1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
|
--
-- File ctex-zhconv.lua
--
-- Copyright (C) 2020 by Qing Lee <sobenlee@gmail.com>
--------------------------------------------------------------------------
--
-- This work may be distributed and/or modified under the
-- conditions of the LaTeX Project Public License, either
-- version 1.3c of this license or (at your option) any later
-- version. This version of this license is in
-- http://www.latex-project.org/lppl/lppl-1-3c.txt
-- and the latest version of this license is in
-- http://www.latex-project.org/lppl.txt
-- and version 1.3 or later is part of all distributions of
-- LaTeX version 2005/12/01 or later.
--
-- This work has the LPPL maintenance status "maintained".
--
-- The Current Maintainer of this work is Qing Lee.
--
-- This work consists of the files ctex-zhconv.lua, ctex-zhconv-make.lua
-- and the derived file ctex-zhconv-index.lua.
--
--------------------------------------------------------------------------
--
if status.kpse_used ~= 1 then
kpse.set_program_name("luatex")
end
local zhconv = { }
zhconv.module = {
name = "ctex-zhconv",
version = "0.2",
date = "2020/05/02",
description = "GBK/GB18030/Big5 encoder",
author = "Qing Lee",
copyright = "Qing Lee",
license = "LPPL v1.3c"
}
local utf = require("unicode").utf8
local ubyte, ugsub = utf.byte, utf.gsub
local floor = math.floor
local unpack, insert, sort = table.unpack, table.insert, table.sort
local char, format = string.char, string.format
zhconv.index = require("ctex-zhconv-index")
local index = zhconv.index
zhconv.mapping = { }
local mapping = zhconv.mapping
mapping.big5, mapping.gbk, mapping.gb18030 = { }, { }, { }
local gbk, gb18030, big5 = mapping.gbk, mapping.gb18030, mapping.big5
zhconv.bytes = { }
local bytes = zhconv.bytes
-- Let lead be pointer / 190 + 0x81.
-- Let trail be pointer % 190.
-- Let offset be 0x40 if trail is less than 0x3F and 0x41 otherwise.
-- Return two bytes whose values are lead and trail + offset.
function bytes.gbk (pointer)
local lead = floor(pointer / 190) + 0x81
local trail = pointer % 190
local offset = trail < 0x3F and 0x40 or 0x41
return format("%c%c", lead, trail + offset)
end
-- Let lead be pointer / 157 + 0x81.
-- If lead is less than 0xA1, return error with code point.
-- Let trail be pointer % 157.
-- Let offset be 0x40 if trail is less than 0x3F and 0x62 otherwise.
-- Return two bytes whose values are lead and trail + offset.
function bytes.big5 (pointer)
local lead = floor(pointer / 157) + 0x81
local trail = pointer % 157
local offset = trail < 0x3F and 0x40 or 0x62
return format("%c%c", lead, trail + offset)
end
-- Let byte1 be pointer / 10 / 126 / 10.
-- Set pointer to pointer - byte1 × 10 × 126 × 10.
-- Let byte2 be pointer / 10 / 126.
-- Set pointer to pointer - byte2 × 10 × 126.
-- Let byte3 be pointer / 10.
-- Let byte4 be pointer - byte3 × 10.
-- Return four bytes whose values are byte1 + 0x81, byte2 + 0x30, byte3 + 0x81, byte4 + 0x30.
function bytes.gb18030 (pointer)
local pointer = pointer
local byte1 = floor(pointer / 12600) + 0x81
pointer = pointer % 12600
local byte2 = floor(pointer / 1260) + 0x30
pointer = pointer % 1260
local byte3 = floor(pointer / 10) + 0x81
local byte4 = pointer % 10 + 0x30
return format("%c%c%c%c", byte1, byte2, byte3, byte4)
end
-- Let offset be the last code point in index gb18030 ranges that is equal to or
-- less than code point and let pointer offset be its corresponding pointer.
-- Return a pointer whose value is pointer offset + code point - offset.
do
local metatable = { }
local bytes, ranges = bytes.gb18030, index["gb18030_ranges"]
function metatable.__index (t, key)
if type(key) == "number" then
local n = #ranges
if key < 0x10000 then
local s = gbk[key]
if s then return s end
repeat
n = n - 1
until ranges[n][2] <= key
end
local pointer, offset = unpack(ranges[n])
return bytes(pointer + key - offset)
end
end
gb18030 = setmetatable(gb18030, metatable)
end
-- The index Big5 pointer for code point is the return value of these steps:
-- 1. Let index be index Big5 excluding all entries whose pointer
-- is less than (0xA1 - 0x81) × 157.
-- 2. If code point is U+2550, U+255E, U+2561, U+256A, U+5341, or U+5345,
-- return the last pointer corresponding to code point in index.
local big5_last = {
[0x2550] = true ,
[0x255E] = true ,
[0x2561] = true ,
[0x256A] = true ,
[0x5341] = true ,
[0x5345] = true ,
}
do
local function spairs (t)
local a = { }
for i, n in pairs(t) do insert(a, {i, n}) end
sort(a, function(b, c) return b[1] < c[1] end)
local i = 0
local iter = function ()
i = i + 1
if a[i] then
return a[i][1], a[i][2]
end
end
return iter
end
local chars = bytes.big5
local m = (0xA1 - 0x81) * 157
for i, v in spairs(index.big5) do
if i >= m then
if big5[v] then
if big5_last[v] then big5[v] = chars(i) end
else
big5[v] = chars(i)
end
end
end
local chars = bytes.gbk
for i, v in pairs(index.gb18030) do
gbk[v] = chars(i)
end
end
-- If the gbk flag is set and code point is U+20AC, return byte 0x80.
gb18030[0x20AC] = gbk[0x20AC]
gbk[0x20AC] = char(0x80)
local io_open = io.open
local encode_error = "Encoding %q not available!"
local file_error = "Open file %q failed!"
function zhconv.conv (encoding, input, output)
local encoding = encoding:lower()
local mapping = assert(mapping[encoding], encode_error:format(encoding))
local encoder = function (s)
local code_point = ubyte(s)
return code_point > 0x7F and mapping[code_point]
end
if output then
local handle = assert(io_open(input, "rb"), file_error:format(input))
local stream = handle:read("*all")
handle:close()
handle = assert(io_open(output, "wb"), file_error:format(output))
stream = stream:gsub("^\xEF\xBB\xBF", "")
stream = ugsub(stream, ".", encoder)
handle:write(stream)
handle:close()
else
local s = ugsub(input, ".", encoder)
return s
end
end
return zhconv
|
