summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/l-unicode.lua
diff options
context:
space:
mode:
authorTaco Hoekwater <taco@elvenkind.com>2008-06-12 10:42:53 +0000
committerTaco Hoekwater <taco@elvenkind.com>2008-06-12 10:42:53 +0000
commit0d01365d53c456d246da0ca1f0b3cd9868f02b35 (patch)
tree01a655c8028e17cfb371456b299c1848fe08c05b /Master/texmf-dist/tex/context/base/l-unicode.lua
parent44f3714442da07fdfc36a7f2a8dcd5d4294c5d26 (diff)
ConTeXt release 2008.05.21
git-svn-id: svn://tug.org/texlive/trunk@8691 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/l-unicode.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/l-unicode.lua148
1 files changed, 148 insertions, 0 deletions
diff --git a/Master/texmf-dist/tex/context/base/l-unicode.lua b/Master/texmf-dist/tex/context/base/l-unicode.lua
new file mode 100644
index 00000000000..d0c05bb8629
--- /dev/null
+++ b/Master/texmf-dist/tex/context/base/l-unicode.lua
@@ -0,0 +1,148 @@
+-- filename : l-unicode.lua
+-- comment : split off from luat-inp
+-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL
+-- copyright: PRAGMA ADE / ConTeXt Development Team
+-- license : see context related readme files
+
+if not versions then versions = { } end versions['l-unicode'] = 1.001
+if not unicode then unicode = { } end
+
+if not garbagecollector then
+ garbagecollector = {
+ push = function() collectgarbage("stop") end,
+ pop = function() collectgarbage("restart") end,
+ }
+end
+
+-- 0 EF BB BF UTF-8
+-- 1 FF FE UTF-16-little-endian
+-- 2 FE FF UTF-16-big-endian
+-- 3 FF FE 00 00 UTF-32-little-endian
+-- 4 00 00 FE FF UTF-32-big-endian
+
+unicode.utfname = {
+ [0] = 'utf-8',
+ [1] = 'utf-16-le',
+ [2] = 'utf-16-be',
+ [3] = 'utf-32-le',
+ [4] = 'utf-32-be'
+}
+
+function unicode.utftype(f) -- \000 fails !
+ local str = f:read(4)
+ if not str then
+ f:seek('set')
+ return 0
+ elseif str:find("^%z%z\254\255") then
+ return 4
+ elseif str:find("^\255\254%z%z") then
+ return 3
+ elseif str:find("^\254\255") then
+ f:seek('set',2)
+ return 2
+ elseif str:find("^\255\254") then
+ f:seek('set',2)
+ return 1
+ elseif str:find("^\239\187\191") then
+ f:seek('set',3)
+ return 0
+ else
+ f:seek('set')
+ return 0
+ end
+end
+
+function unicode.utf16_to_utf8(str, endian)
+ garbagecollector.push()
+ local result = { }
+ local tc, uc = table.concat, unicode.utf8.char
+ local tmp, n, m, p = { }, 0, 0, 0
+ -- lf | cr | crlf / (cr:13, lf:10)
+ local function doit()
+ if n == 10 then
+ if p ~= 13 then
+ result[#result+1] = tc(tmp,"")
+ tmp = { }
+ p = 0
+ end
+ elseif n == 13 then
+ result[#result+1] = tc(tmp,"")
+ tmp = { }
+ p = n
+ else
+ tmp[#tmp+1] = uc(n)
+ p = 0
+ end
+ end
+ for l,r in str:bytepairs() do
+ if endian then
+ n = l*256 + r
+ else
+ n = r*256 + l
+ end
+ if m > 0 then
+ n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000
+ m = 0
+ doit()
+ elseif n >= 0xD800 and n <= 0xDBFF then
+ m = n
+ else
+ doit()
+ end
+ end
+ if #tmp > 0 then
+ result[#result+1] = tc(tmp,"")
+ end
+ garbagecollector.pop()
+ return result
+end
+
+function unicode.utf32_to_utf8(str, endian)
+ garbagecollector.push()
+ local result = { }
+ local tc, uc = table.concat, unicode.utf8.char
+ local tmp, n, m, p = { }, 0, -1, 0
+ -- lf | cr | crlf / (cr:13, lf:10)
+ local function doit()
+ if n == 10 then
+ if p ~= 13 then
+ result[#result+1] = tc(tmp,"")
+ tmp = { }
+ p = 0
+ end
+ elseif n == 13 then
+ result[#result+1] = tc(tmp,"")
+ tmp = { }
+ p = n
+ else
+ tmp[#tmp+1] = uc(n)
+ p = 0
+ end
+ end
+ for a,b in str:bytepairs() do
+ if a and b then
+ if m < 0 then
+ if endian then
+ m = a*256*256*256 + b*256*256
+ else
+ m = b*256 + a
+ end
+ else
+ if endian then
+ n = m + a*256 + b
+ else
+ n = m + b*256*256*256 + a*256*256
+ end
+ m = -1
+ doit()
+ end
+ else
+ break
+ end
+ end
+ if #tmp > 0 then
+ result[#result+1] = tc(tmp,"")
+ end
+ garbagecollector.pop()
+ return result
+end