summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/l-unicode.lua
diff options
context:
space:
mode:
authorMojca Miklavec <mojca.miklavec@gmail.com>2012-05-14 17:38:55 +0000
committerMojca Miklavec <mojca.miklavec@gmail.com>2012-05-14 17:38:55 +0000
commit15995e10bfc68edf79970c4ea4fbb6678566c46e (patch)
tree2de7ca2a83f2d37ef043ad7429a5cb945bb79ddb /Master/texmf-dist/tex/context/base/l-unicode.lua
parentc9a39f716f1e5ec820ed3aab2c9aef25c5a9d730 (diff)
ConTeXt 2012.05.14 16:00
git-svn-id: svn://tug.org/texlive/trunk@26371 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/l-unicode.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/l-unicode.lua77
1 files changed, 66 insertions, 11 deletions
diff --git a/Master/texmf-dist/tex/context/base/l-unicode.lua b/Master/texmf-dist/tex/context/base/l-unicode.lua
index 2fa50282fe0..246171aecc6 100644
--- a/Master/texmf-dist/tex/context/base/l-unicode.lua
+++ b/Master/texmf-dist/tex/context/base/l-unicode.lua
@@ -218,7 +218,7 @@ local function utf16_to_utf8_be(t)
if right then
local now = 256*left + right
if more > 0 then
- now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000
+ now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong
more = 0
r = r + 1
result[r] = utfchar(now)
@@ -246,7 +246,7 @@ local function utf16_to_utf8_le(t)
if right then
local now = 256*right + left
if more > 0 then
- now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000
+ now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong
more = 0
r = r + 1
result[r] = utfchar(now)
@@ -263,14 +263,14 @@ local function utf16_to_utf8_le(t)
return t
end
-local function utf32_to_utf8_be(str)
+local function utf32_to_utf8_be(t)
if type(t) == "string" then
- t = utfsplitlines(str)
+ t = utfsplitlines(t)
end
local result = { } -- we reuse result
for i=1,#t do
local r, more = 0, -1
- for a,b in bytepairs(str) do
+ for a,b in bytepairs(t[i]) do
if a and b then
if more < 0 then
more = 256*256*256*a + 256*256*b
@@ -285,17 +285,17 @@ local function utf32_to_utf8_be(str)
end
t[i] = concat(result,"",1,r)
end
- return result
+ return t
end
-local function utf32_to_utf8_le(str)
+local function utf32_to_utf8_le(t)
if type(t) == "string" then
- t = utfsplitlines(str)
+ t = utfsplitlines(t)
end
local result = { } -- we reuse result
for i=1,#t do
local r, more = 0, -1
- for a,b in bytepairs(str) do
+ for a,b in bytepairs(t[i]) do
if a and b then
if more < 0 then
more = 256*b + a
@@ -310,7 +310,7 @@ local function utf32_to_utf8_le(str)
end
t[i] = concat(result,"",1,r)
end
- return result
+ return t
end
unicode.utf32_to_utf8_be = utf32_to_utf8_be
@@ -380,12 +380,30 @@ end
--~ print(unicode.utfcodes(str))
local lpegmatch = lpeg.match
-local utftype = lpeg.patterns.utftype
+local patterns = lpeg.patterns
+local utftype = patterns.utftype
function unicode.filetype(data)
return data and lpegmatch(utftype,data) or "unknown"
end
+local toentities = lpeg.Cs (
+ (
+ patterns.utf8one
+ + (
+ patterns.utf8two
+ + patterns.utf8three
+ + patterns.utf8four
+ ) / function(s) local b = utfbyte(s) if b < 127 then return s else return format("&#%X;",b) end end
+ )^0
+)
+
+patterns.toentities = toentities
+
+function utf.toentities(str)
+ return lpegmatch(toentities,str)
+end
+
--~ local utfchr = { } -- 60K -> 2.638 M extra mem but currently not called that often (on latin)
--~
--~ setmetatable(utfchr, { __index = function(t,k) local v = utfchar(k) t[k] = v return v end } )
@@ -408,3 +426,40 @@ end
--~ end
--~ end
--~ print(os.clock()-t,collectgarbage("count")*1024-u)
+
+--~ local byte = string.byte
+--~ local utfchar = utf.char
+--~ local lpegmatch = lpeg.match, lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs
+
+local one = P(1)
+local two = C(1) * C(1)
+local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1)
+
+local pattern = P("\254\255") * Cs( (
+ four / function(a,b,c,d)
+ local ab = 0xFF * byte(a) + byte(b)
+ local cd = 0xFF * byte(c) + byte(d)
+ return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+ end
+ + two / function(a,b)
+ return utfchar(byte(a)*256 + byte(b))
+ end
+ + one
+ )^1 )
+ + P("\255\254") * Cs( (
+ four / function(b,a,d,c)
+ local ab = 0xFF * byte(a) + byte(b)
+ local cd = 0xFF * byte(c) + byte(d)
+ return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000)
+ end
+ + two / function(b,a)
+ return utfchar(byte(a)*256 + byte(b))
+ end
+ + one
+ )^1 )
+
+function string.toutf(s)
+ return lpegmatch(pattern,s) or s -- todo: utf32
+end