summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/l-unicode.lua
diff options
context:
space:
mode:
authorTaco Hoekwater <taco@elvenkind.com>2009-08-23 11:11:32 +0000
committerTaco Hoekwater <taco@elvenkind.com>2009-08-23 11:11:32 +0000
commit8fc3039c82d48605b5ca8b2eda3f4fdd755681e1 (patch)
tree3cd9bbdd599bc4d1ac0409e167fee2136e4c0ec9 /Master/texmf-dist/tex/context/base/l-unicode.lua
parent850fc99b7cd3ae7a20065531fe866ff7bae642ec (diff)
this is context 2009.08.19 17:10
git-svn-id: svn://tug.org/texlive/trunk@14827 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/l-unicode.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/l-unicode.lua124
1 files changed, 74 insertions, 50 deletions
diff --git a/Master/texmf-dist/tex/context/base/l-unicode.lua b/Master/texmf-dist/tex/context/base/l-unicode.lua
index d0c05bb8629..124a1e24038 100644
--- a/Master/texmf-dist/tex/context/base/l-unicode.lua
+++ b/Master/texmf-dist/tex/context/base/l-unicode.lua
@@ -1,18 +1,17 @@
--- filename : l-unicode.lua
--- comment : split off from luat-inp
--- author : Hans Hagen, PRAGMA-ADE, Hasselt NL
--- copyright: PRAGMA ADE / ConTeXt Development Team
--- license : see context related readme files
+if not modules then modules = { } end modules ['l-unicode'] = {
+ version = 1.001,
+ comment = "companion to luat-lib.tex",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files"
+}
-if not versions then versions = { } end versions['l-unicode'] = 1.001
-if not unicode then unicode = { } end
+utf = utf or unicode.utf8
-if not garbagecollector then
- garbagecollector = {
- push = function() collectgarbage("stop") end,
- pop = function() collectgarbage("restart") end,
- }
-end
+local concat, utfchar, utfgsub = table.concat, utf.char, utf.gsub
+local char, byte, find, bytepairs = string.char, string.byte, string.find, string.bytepairs
+
+unicode = unicode or { }
-- 0 EF BB BF UTF-8
-- 1 FF FE UTF-16-little-endian
@@ -33,17 +32,17 @@ function unicode.utftype(f) -- \000 fails !
if not str then
f:seek('set')
return 0
- elseif str:find("^%z%z\254\255") then
+ elseif find(str,"^%z%z\254\255") then
return 4
- elseif str:find("^\255\254%z%z") then
+ elseif find(str,"^\255\254%z%z") then
return 3
- elseif str:find("^\254\255") then
+ elseif find(str,"^\254\255") then
f:seek('set',2)
return 2
- elseif str:find("^\255\254") then
+ elseif find(str,"^\255\254") then
f:seek('set',2)
return 1
- elseif str:find("^\239\187\191") then
+ elseif find(str,"^\239\187\191") then
f:seek('set',3)
return 0
else
@@ -52,74 +51,70 @@ function unicode.utftype(f) -- \000 fails !
end
end
-function unicode.utf16_to_utf8(str, endian)
- garbagecollector.push()
- local result = { }
- local tc, uc = table.concat, unicode.utf8.char
- local tmp, n, m, p = { }, 0, 0, 0
+function unicode.utf16_to_utf8(str, endian) -- maybe a gsub is faster or an lpeg
+ local result, tmp, n, m, p = { }, { }, 0, 0, 0
-- lf | cr | crlf / (cr:13, lf:10)
local function doit()
if n == 10 then
if p ~= 13 then
- result[#result+1] = tc(tmp,"")
+ result[#result+1] = concat(tmp)
tmp = { }
p = 0
end
elseif n == 13 then
- result[#result+1] = tc(tmp,"")
+ result[#result+1] = concat(tmp)
tmp = { }
p = n
else
- tmp[#tmp+1] = uc(n)
+ tmp[#tmp+1] = utfchar(n)
p = 0
end
end
- for l,r in str:bytepairs() do
- if endian then
- n = l*256 + r
- else
- n = r*256 + l
- end
- if m > 0 then
- n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000
- m = 0
- doit()
- elseif n >= 0xD800 and n <= 0xDBFF then
- m = n
- else
- doit()
+ for l,r in bytepairs(str) do
+ if r then
+ if endian then
+ n = l*256 + r
+ else
+ n = r*256 + l
+ end
+ if m > 0 then
+ n = (m-0xD800)*0x400 + (n-0xDC00) + 0x10000
+ m = 0
+ doit()
+ elseif n >= 0xD800 and n <= 0xDBFF then
+ m = n
+ else
+ doit()
+ end
end
end
if #tmp > 0 then
- result[#result+1] = tc(tmp,"")
+ result[#result+1] = concat(tmp)
end
- garbagecollector.pop()
return result
end
function unicode.utf32_to_utf8(str, endian)
- garbagecollector.push()
local result = { }
- local tc, uc = table.concat, unicode.utf8.char
local tmp, n, m, p = { }, 0, -1, 0
-- lf | cr | crlf / (cr:13, lf:10)
local function doit()
if n == 10 then
if p ~= 13 then
- result[#result+1] = tc(tmp,"")
+ result[#result+1] = concat(tmp)
tmp = { }
p = 0
end
elseif n == 13 then
- result[#result+1] = tc(tmp,"")
+ result[#result+1] = concat(tmp)
tmp = { }
p = n
else
- tmp[#tmp+1] = uc(n)
+ tmp[#tmp+1] = utfchar(n)
p = 0
end
end
- for a,b in str:bytepairs() do
+ for a,b in bytepairs(str) do
if a and b then
if m < 0 then
if endian then
@@ -141,8 +136,37 @@ function unicode.utf32_to_utf8(str, endian)
end
end
if #tmp > 0 then
- result[#result+1] = tc(tmp,"")
+ result[#result+1] = concat(tmp)
end
- garbagecollector.pop()
return result
end
+
+local function little(c)
+ local b = byte(c) -- b = c:byte()
+ if b < 0x10000 then
+ return char(b%256,b/256)
+ else
+ b = b - 0x10000
+ local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00
+ return char(b1%256,b1/256,b2%256,b2/256)
+ end
+end
+
+local function big(c)
+ local b = byte(c)
+ if b < 0x10000 then
+ return char(b/256,b%256)
+ else
+ b = b - 0x10000
+ local b1, b2 = b/1024 + 0xD800, b%1024 + 0xDC00
+ return char(b1/256,b1%256,b2/256,b2%256)
+ end
+end
+
+function unicode.utf8_to_utf16(str,littleendian)
+ if littleendian then
+ return char(255,254) .. utfgsub(str,".",little)
+ else
+ return char(254,255) .. utfgsub(str,".",big)
+ end
+end