diff options
author | Norbert Preining <norbert@preining.info> | 2020-05-03 03:02:51 +0000 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2020-05-03 03:02:51 +0000 |
commit | d78de17ddf0443233483fd98f40802cae5aa87c5 (patch) | |
tree | 1b7128cc505808a09c18e158288d6c3cdcf5c2fb /language/chinese/ctex/ctex-zhconv.lua | |
parent | 8626a9c3cb5dc8d02d779d5b5b1e3933d5626f43 (diff) |
CTAN sync 202005030302
Diffstat (limited to 'language/chinese/ctex/ctex-zhconv.lua')
-rw-r--r-- | language/chinese/ctex/ctex-zhconv.lua | 204 |
1 files changed, 204 insertions, 0 deletions
diff --git a/language/chinese/ctex/ctex-zhconv.lua b/language/chinese/ctex/ctex-zhconv.lua new file mode 100644 index 0000000000..d478c2e432 --- /dev/null +++ b/language/chinese/ctex/ctex-zhconv.lua @@ -0,0 +1,204 @@ +-- +-- File ctex-zhconv.lua +-- +-- Copyright (C) 2020 by Qing Lee <sobenlee@gmail.com> +-------------------------------------------------------------------------- +-- +-- This work may be distributed and/or modified under the +-- conditions of the LaTeX Project Public License, either +-- version 1.3c of this license or (at your option) any later +-- version. This version of this license is in +-- http://www.latex-project.org/lppl/lppl-1-3c.txt +-- and the latest version of this license is in +-- http://www.latex-project.org/lppl.txt +-- and version 1.3 or later is part of all distributions of +-- LaTeX version 2005/12/01 or later. +-- +-- This work has the LPPL maintenance status "maintained". +-- +-- The Current Maintainer of this work is Qing Lee. +-- +-- This work consists of the files ctex-zhconv.lua, ctex-zhconv-make.lua +-- and the derived file ctex-zhconv-index.lua. +-- +-------------------------------------------------------------------------- +-- + +if status.kpse_used ~= 1 then + kpse.set_program_name("luatex") +end + +local zhconv = { } +zhconv.module = { + name = "ctex-zhconv", + version = "0.2", + date = "2020/05/02", + description = "GBK/GB18030/Big5 encoder", + author = "Qing Lee", + copyright = "Qing Lee", + license = "LPPL v1.3c" +} + +local utf = require("unicode").utf8 +local ubyte, ugsub = utf.byte, utf.gsub + +local floor = math.floor +local unpack, insert, sort = table.unpack, table.insert, table.sort +local char, format = string.char, string.format + +zhconv.index = require("ctex-zhconv-index") +local index = zhconv.index + +zhconv.mapping = { } +local mapping = zhconv.mapping + +mapping.big5, mapping.gbk, mapping.gb18030 = { }, { }, { } +local gbk, gb18030, big5 = mapping.gbk, mapping.gb18030, mapping.big5 + +zhconv.bytes = { } +local bytes = zhconv.bytes + +-- Let lead be pointer / 190 + 0x81. +-- Let trail be pointer % 190. +-- Let offset be 0x40 if trail is less than 0x3F and 0x41 otherwise. +-- Return two bytes whose values are lead and trail + offset. +function bytes.gbk (pointer) + local lead = floor(pointer / 190) + 0x81 + local trail = pointer % 190 + local offset = trail < 0x3F and 0x40 or 0x41 + return format("%c%c", lead, trail + offset) +end + +-- Let lead be pointer / 157 + 0x81. +-- If lead is less than 0xA1, return error with code point. +-- Let trail be pointer % 157. +-- Let offset be 0x40 if trail is less than 0x3F and 0x62 otherwise. +-- Return two bytes whose values are lead and trail + offset. +function bytes.big5 (pointer) + local lead = floor(pointer / 157) + 0x81 + local trail = pointer % 157 + local offset = trail < 0x3F and 0x40 or 0x62 + return format("%c%c", lead, trail + offset) +end + +-- Let byte1 be pointer / 10 / 126 / 10. +-- Set pointer to pointer - byte1 × 10 × 126 × 10. +-- Let byte2 be pointer / 10 / 126. +-- Set pointer to pointer - byte2 × 10 × 126. +-- Let byte3 be pointer / 10. +-- Let byte4 be pointer - byte3 × 10. +-- Return four bytes whose values are byte1 + 0x81, byte2 + 0x30, byte3 + 0x81, byte4 + 0x30. +function bytes.gb18030 (pointer) + local pointer = pointer + local byte1 = floor(pointer / 12600) + 0x81 + pointer = pointer % 12600 + local byte2 = floor(pointer / 1260) + 0x30 + pointer = pointer % 1260 + local byte3 = floor(pointer / 10) + 0x81 + local byte4 = pointer % 10 + 0x30 + return format("%c%c%c%c", byte1, byte2, byte3, byte4) +end + +-- Let offset be the last code point in index gb18030 ranges that is equal to or +-- less than code point and let pointer offset be its corresponding pointer. +-- Return a pointer whose value is pointer offset + code point - offset. +do + local metatable = { } + local bytes, ranges = bytes.gb18030, index["gb18030_ranges"] + function metatable.__index (t, key) + if type(key) == "number" then + local n = #ranges + if key < 0x10000 then + local s = gbk[key] + if s then return s end + repeat + n = n - 1 + until ranges[n][2] <= key + end + local pointer, offset = unpack(ranges[n]) + return bytes(pointer + key - offset) + end + end + gb18030 = setmetatable(gb18030, metatable) +end + +-- The index Big5 pointer for code point is the return value of these steps: +-- 1. Let index be index Big5 excluding all entries whose pointer +-- is less than (0xA1 - 0x81) × 157. +-- 2. If code point is U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345, +-- return the last pointer corresponding to code point in index. +local big5_last = { + [0x2550] = true , + [0x255E] = true , + [0x2561] = true , + [0x256A] = true , + [0x5341] = true , + [0x5345] = true , +} + +do + +local function spairs (t) + local a = { } + for i, n in pairs(t) do insert(a, {i, n}) end + sort(a, function(b, c) return b[1] < c[1] end) + local i = 0 + local iter = function () + i = i + 1 + if a[i] then + return a[i][1], a[i][2] + end + end + return iter +end + +local chars = bytes.big5 +local m = (0xA1 - 0x81) * 157 +for i, v in spairs(index.big5) do + if i >= m then + if big5[v] then + if big5_last[v] then big5[v] = chars(i) end + else + big5[v] = chars(i) + end + end +end + +local chars = bytes.gbk +for i, v in pairs(index.gb18030) do + gbk[v] = chars(i) +end + +end + +-- If the gbk flag is set and code point is U+20AC, return byte 0x80. +gb18030[0x20AC] = gbk[0x20AC] +gbk[0x20AC] = char(0x80) + +local io_open = io.open +local encode_error = "Encoding %q not available!" +local file_error = "Open file %q failed!" + +function zhconv.conv (encoding, input, output) + local encoding = encoding:lower() + local mapping = assert(mapping[encoding], encode_error:format(encoding)) + local encoder = function (s) + local code_point = ubyte(s) + return code_point > 0x7F and mapping[code_point] + end + if output then + local handle = assert(io_open(input, "rb"), file_error:format(input)) + local stream = handle:read("*all") + handle:close() + handle = assert(io_open(output, "wb"), file_error:format(output)) + stream = stream:gsub("^\xEF\xBB\xBF", "") + stream = ugsub(stream, ".", encoder) + handle:write(stream) + handle:close() + else + local s = ugsub(input, ".", encoder) + return s + end +end + +return zhconv |