summaryrefslogtreecommitdiff
path: root/language/chinese/ctex/zhconv.lua
blob: 8af2161e26f9a73056335f48c8caca91d866c079 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
--
--  File zhconv.lua
--
--     Copyright (C) 2020 by Qing Lee <sobenlee@gmail.com>
--------------------------------------------------------------------------
--
--     This work may be distributed and/or modified under the
--     conditions of the LaTeX Project Public License, either
--     version 1.3c of this license or (at your option) any later
--     version. This version of this license is in
--        http://www.latex-project.org/lppl/lppl-1-3c.txt
--     and the latest version of this license is in
--        http://www.latex-project.org/lppl.txt
--     and version 1.3 or later is part of all distributions of
--     LaTeX version 2005/12/01 or later.
--
--     This work has the LPPL maintenance status "maintained".
--
--     The Current Maintainer of this work is Qing Lee.
--
--     This work consists of the files zhconv.lua, zhconv-make.lua
--               and the derived files zhconv-index.lua.
--
--------------------------------------------------------------------------
--

if status.kpse_used ~= 1 then
  kpse.set_program_name("luatex")
end

zhconv        = zhconv or { }
local zhconv  = zhconv
zhconv.module = {
  name        = "zhconv",
  version     = "0.1",
  date        = "2020/05/01",
  description = "GBK/GB18030/Big5 encoder",
  author      = "Qing Lee",
  copyright   = "Qing Lee",
  license     = "LPPL v1.3c"
}

local utf = require("unicode").utf8
local ubyte, ugsub = utf.byte, utf.gsub

local floor = math.floor
local unpack, insert, sort = table.unpack, table.insert, table.sort
local char, format = string.char, string.format

zhconv.index = require("zhconv-index")
local index = zhconv.index

zhconv.mapping = { }
local mapping = zhconv.mapping

mapping.big5, mapping.gbk, mapping.gb18030 = { }, { }, { }
local gbk, gb18030, big5 = mapping.gbk, mapping.gb18030, mapping.big5

zhconv.bytes = { }
local bytes = zhconv.bytes

-- Let lead be pointer / 190 + 0x81.
-- Let trail be pointer % 190.
-- Let offset be 0x40 if trail is less than 0x3F and 0x41 otherwise.
-- Return two bytes whose values are lead and trail + offset.
function bytes.gbk (pointer)
  local lead = floor(pointer / 190) + 0x81
  local trail = pointer % 190
  local offset = trail < 0x3F and 0x40 or 0x41
  return format("%c%c", lead, trail + offset)
end

-- Let lead be pointer / 157 + 0x81.
-- If lead is less than 0xA1, return error with code point.
-- Let trail be pointer % 157.
-- Let offset be 0x40 if trail is less than 0x3F and 0x62 otherwise.
-- Return two bytes whose values are lead and trail + offset.
function bytes.big5 (pointer)
  local lead = floor(pointer / 157) + 0x81
  local trail = pointer % 157
  local offset = trail < 0x3F and 0x40 or 0x62
  return format("%c%c", lead, trail + offset)
end

-- Let byte1 be pointer / 10 / 126 / 10.
-- Set pointer to pointer - byte1 × 10 × 126 × 10.
-- Let byte2 be pointer / 10 / 126.
-- Set pointer to pointer - byte2 × 10 × 126.
-- Let byte3 be pointer / 10.
-- Let byte4 be pointer - byte3 × 10.
-- Return four bytes whose values are byte1 + 0x81, byte2 + 0x30, byte3 + 0x81, byte4 + 0x30.
function bytes.gb18030 (pointer)
  local pointer = pointer
  local byte1 = floor(pointer / 12600) + 0x81
  pointer = pointer % 12600
  local byte2 = floor(pointer / 1260) + 0x30
  pointer = pointer % 1260
  local byte3 = floor(pointer / 10) + 0x81
  local byte4 = pointer % 10 + 0x30
  return format("%c%c%c%c", byte1, byte2, byte3, byte4)
end

-- Let offset be the last code point in index gb18030 ranges that is equal to or
-- less than code point and let pointer offset be its corresponding pointer.
-- Return a pointer whose value is pointer offset + code point - offset.
do
  local metatable = { }
  local bytes, ranges = bytes.gb18030, index["gb18030_ranges"]
  function metatable.__index (t, key)
    if type(key) == "number" then
      local n = #ranges
      if key < 0x10000 then
        local s = gbk[key]
        if s then return s end
        repeat
          n = n - 1
        until ranges[n][2] <= key
      end
      local pointer, offset = unpack(ranges[n])
      return bytes(pointer + key - offset)
    end
  end
  gb18030 = setmetatable(gb18030, metatable)
end

-- The index Big5 pointer for code point is the return value of these steps:
-- 1. Let index be index Big5 excluding all entries whose pointer
--    is less than (0xA1 - 0x81) × 157.
-- 2. If code point is U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345,
--    return the last pointer corresponding to code point in index.
local big5_last = {
  [0x2550] = true ,
  [0x255E] = true ,
  [0x2561] = true ,
  [0x256A] = true ,
  [0x5341] = true ,
  [0x5345] = true ,
}

do

local function spairs (t)
  local a = { }
  for i, n in pairs(t) do insert(a, {i, n}) end
  sort(a, function(b, c) return b[1] < c[1] end)
  local i = 0
  local iter = function ()
    i = i + 1
    if a[i] then
      return a[i][1], a[i][2]
    end
  end
  return iter
end

local chars = bytes.big5
local m = (0xA1 - 0x81) * 157
for i, v in spairs(index.big5) do
  if i >= m then
    if big5[v] then
      if big5_last[v] then big5[v] = chars(i) end
    else
      big5[v] = chars(i)
    end
  end
end

local chars = bytes.gbk
for i, v in pairs(index.gb18030) do
  gbk[v] = chars(i)
end

end

-- If the gbk flag is set and code point is U+20AC, return byte 0x80.
gb18030[0x20AC] = gbk[0x20AC]
gbk[0x20AC]     = char(0x80)

local io_open = io.open
local encode_error = "Encoding %q not available!"
local file_error   = "Open file %q failed!"

function zhconv.conv (encoding, input, output)
  local encoding = encoding:lower()
  local mapping = assert(mapping[encoding], encode_error:format(encoding))
  local encoder = function (s)
    local code_point = ubyte(s)
    return code_point > 0x7F and mapping[code_point]
  end
  if output then
    local handle = assert(io_open(input, "rb"), file_error:format(input))
    local stream = handle:read("*all")
    handle:close()
    handle = assert(io_open(output, "wb"), file_error:format(output))
    stream = stream:gsub("^\xEF\xBB\xBF", "")
    stream = ugsub(stream, ".", encoder)
    handle:write(stream)
    handle:close()
  else
    local s = ugsub(input, ".", encoder)
    return s
  end
end

return zhconv