diff options
author | Mojca Miklavec <mojca.miklavec@gmail.com> | 2012-05-14 17:38:55 +0000 |
---|---|---|
committer | Mojca Miklavec <mojca.miklavec@gmail.com> | 2012-05-14 17:38:55 +0000 |
commit | 15995e10bfc68edf79970c4ea4fbb6678566c46e (patch) | |
tree | 2de7ca2a83f2d37ef043ad7429a5cb945bb79ddb /Master/texmf-dist/tex/context/base/l-unicode.lua | |
parent | c9a39f716f1e5ec820ed3aab2c9aef25c5a9d730 (diff) |
ConTeXt 2012.05.14 16:00
git-svn-id: svn://tug.org/texlive/trunk@26371 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/l-unicode.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/l-unicode.lua | 77 |
1 files changed, 66 insertions, 11 deletions
diff --git a/Master/texmf-dist/tex/context/base/l-unicode.lua b/Master/texmf-dist/tex/context/base/l-unicode.lua index 2fa50282fe0..246171aecc6 100644 --- a/Master/texmf-dist/tex/context/base/l-unicode.lua +++ b/Master/texmf-dist/tex/context/base/l-unicode.lua @@ -218,7 +218,7 @@ local function utf16_to_utf8_be(t) if right then local now = 256*left + right if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 + now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong more = 0 r = r + 1 result[r] = utfchar(now) @@ -246,7 +246,7 @@ local function utf16_to_utf8_le(t) if right then local now = 256*right + left if more > 0 then - now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 + now = (more-0xD800)*0x400 + (now-0xDC00) + 0x10000 -- the 0x10000 smells wrong more = 0 r = r + 1 result[r] = utfchar(now) @@ -263,14 +263,14 @@ local function utf16_to_utf8_le(t) return t end -local function utf32_to_utf8_be(str) +local function utf32_to_utf8_be(t) if type(t) == "string" then - t = utfsplitlines(str) + t = utfsplitlines(t) end local result = { } -- we reuse result for i=1,#t do local r, more = 0, -1 - for a,b in bytepairs(str) do + for a,b in bytepairs(t[i]) do if a and b then if more < 0 then more = 256*256*256*a + 256*256*b @@ -285,17 +285,17 @@ local function utf32_to_utf8_be(str) end t[i] = concat(result,"",1,r) end - return result + return t end -local function utf32_to_utf8_le(str) +local function utf32_to_utf8_le(t) if type(t) == "string" then - t = utfsplitlines(str) + t = utfsplitlines(t) end local result = { } -- we reuse result for i=1,#t do local r, more = 0, -1 - for a,b in bytepairs(str) do + for a,b in bytepairs(t[i]) do if a and b then if more < 0 then more = 256*b + a @@ -310,7 +310,7 @@ local function utf32_to_utf8_le(str) end t[i] = concat(result,"",1,r) end - return result + return t end unicode.utf32_to_utf8_be = utf32_to_utf8_be @@ -380,12 +380,30 @@ end --~ print(unicode.utfcodes(str)) local lpegmatch = lpeg.match -local utftype = lpeg.patterns.utftype +local patterns = lpeg.patterns +local utftype = patterns.utftype function unicode.filetype(data) return data and lpegmatch(utftype,data) or "unknown" end +local toentities = lpeg.Cs ( + ( + patterns.utf8one + + ( + patterns.utf8two + + patterns.utf8three + + patterns.utf8four + ) / function(s) local b = utfbyte(s) if b < 127 then return s else return format("&#%X;",b) end end + )^0 +) + +patterns.toentities = toentities + +function utf.toentities(str) + return lpegmatch(toentities,str) +end + --~ local utfchr = { } -- 60K -> 2.638 M extra mem but currently not called that often (on latin) --~ --~ setmetatable(utfchr, { __index = function(t,k) local v = utfchar(k) t[k] = v return v end } ) @@ -408,3 +426,40 @@ end --~ end --~ end --~ print(os.clock()-t,collectgarbage("count")*1024-u) + +--~ local byte = string.byte +--~ local utfchar = utf.char +--~ local lpegmatch = lpeg.match, lpeg.P, lpeg.C, lpeg.R, lpeg.Cs + +local P, C, R, Cs = lpeg.P, lpeg.C, lpeg.R, lpeg.Cs + +local one = P(1) +local two = C(1) * C(1) +local four = C(R(utfchar(0xD8),utfchar(0xFF))) * C(1) * C(1) * C(1) + +local pattern = P("\254\255") * Cs( ( + four / function(a,b,c,d) + local ab = 0xFF * byte(a) + byte(b) + local cd = 0xFF * byte(c) + byte(d) + return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) + end + + two / function(a,b) + return utfchar(byte(a)*256 + byte(b)) + end + + one + )^1 ) + + P("\255\254") * Cs( ( + four / function(b,a,d,c) + local ab = 0xFF * byte(a) + byte(b) + local cd = 0xFF * byte(c) + byte(d) + return utfchar((ab-0xD800)*0x400 + (cd-0xDC00) + 0x10000) + end + + two / function(b,a) + return utfchar(byte(a)*256 + byte(b)) + end + + one + )^1 ) + +function string.toutf(s) + return lpegmatch(pattern,s) or s -- todo: utf32 +end |