summaryrefslogtreecommitdiff
path: root/macros/luatex/generic/kanaparser/kanaparser.lua
blob: 9034f0196e1efb9618ab99dea3cc7ca343df2bf7 (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
-- Kana Parser lua engine

local vowels = {'a', 'e', 'i', 'o', 'u'} -- latin vowels
local vowelsK = {'ア', 'エ', 'イ', 'オ', 'ウ'} -- katakana vowels
local ambigousToN = {'あ', 'え', 'い', 'お', 'う', 'や', 'よ', 'ゆ'} -- characters ambiguous to preceding "n"
local littleTsuWL = {'s', 't', 'k', 'p', 'c'} -- whitelist for little tsu gemination
local transRaw = { -- latin -> hiragana
	n = 'ん', a = 'あ', e = 'え', i = 'い', o = 'お', u = 'う',
	ba = 'ば', be = 'べ', bi = 'び', bo = 'ぼ', bu = 'ぶ',
	bya = 'びゃ', byo = 'びょ', byu = 'びゅ',
	cha = 'ちゃ', che = 'ちぇ', chi = 'ち', cho = 'ちょ', chu = 'ちゅ',
	da = 'だ', de = 'で', di = 'でぃ', ['do'] = 'ど', du = { 'づ', 'どぅ' },
	dya = 'でゃ', dyo = 'でょ', dyu = 'でゅ',
	fa = 'ふぁ', fe = 'ふぇ', fi = 'ふぃ', fo = 'ふぉ',
	fya = 'ふゃ', fyo = 'ふょ', fyu = 'ふゅ',
	ga = 'が', ge = 'げ', gi = 'ぎ', go = 'ご', gu = 'ぐ',
	gwa = 'ぐぁ', gwe = 'ぐぇ', gwi = 'ぐぃ', gwo = 'ぐぉ', gya = 'ぎゃ', gyo = 'ぎょ', gyu = 'ぎゅ',
	ha = 'は', he = 'へ', hi = 'ひ', ho = 'ほ', hu = 'ふ',
	hya = 'ひゃ', hyo = 'ひょ', hyu = 'ひゅ',
	ja = { 'じゃ', 'ぢゃ' }, je = 'じぇ', ji = { 'じ', 'ぢ' }, jo = { 'じょ', 'ぢょ' }, ju = { 'じゅ', 'ぢゅ' },
	ka = 'か', ke = 'け', ki = 'き', ko = 'こ', ku = 'く',
	kwa = 'くぁ', kwe = 'くぇ', kwi = 'くぃ', kwo = 'くぉ', kya = 'きゃ', kyo = 'きょ',	kyu = 'きゅ',
	ma = 'ま', me = 'め', mi = 'み', mo = 'も', mu = 'む',
	mya = 'みゃ', myo = 'みょ', myu = 'みゅ',
	na = 'な', ne = 'ね', ni = 'に', no = 'の', nu = 'ぬ',
	nya = 'にゃ', nyo = 'にょ', nyu = 'にゅ',
	pa = 'ぱ', pe = 'ぺ', pi = 'ぴ', po = 'ぽ', pu = 'ぷ',
	pya = 'ぴゃ', pyo = 'ぴょ', pyu = 'ぴゅ',
	ra = 'ら', re = 'れ', ri = 'り', ro = 'ろ', ru = 'る',
	rya = 'りゃ', ryo = 'りょ', ryu = 'りゅ',
	sa = 'さ', se = 'せ', si = 'し',	so = 'そ', su = 'す',
	sha = 'しゃ', she = 'しぇ', shi = 'し', sho = 'しょ', shu = 'しゅ',
	ta = 'た', te = 'て', ti = 'てぃ', to = 'と',
	tha = 'てゃ', tho = 'てょ', thu = 'てゅ',
	tsa = 'つぁ', tse = 'つぇ', tsu = 'つ', tsi = 'つぃ', tso = 'つぉ',
	tu = 'つ',
	va = 'ゔぁ', ve = 'ゔぇ', vi = 'ゔぃ', vo = 'ゔぉ', vu = 'ゔぅ',
	vya = 'ゔゃ', vyo = 'ゔょ', vyu = 'ゔゅ',
	wa = 'わ', we = { 'うぇ', 'ゑ' }, wi = 'ゐ', wo = { 'を', 'うぉ' },
	ya = 'や', ye = 'いぇ', yo = 'よ', yu = 'ゆ',
	za = 'ざ', ze = 'ぜ', zo = 'ぞ', zu = 'ず'
}
local transK = { -- hiragana -> katakana
	['ん'] = 'ン', ['あ'] = 'ア', ['え'] = 'エ', ['い'] = 'イ', ['お'] = 'オ', ['う'] = 'ウ',
	['ぁ'] = 'ァ', ['ぃ'] = 'ィ', ['ぅ'] = 'ゥ', ['ぇ'] = 'ェ', ['ぉ'] = 'ォ',
	['ゃ'] = 'ャ', ['ゅ'] = 'ュ', ['ょ'] = 'ョ',
	['は'] = 'ハ', ['へ'] = 'ヘ', ['ひ'] = 'ヒ', ['ほ'] = 'ホ', ['ふ'] = 'フ',
	['ば'] = 'バ', ['べ'] = 'ベ', ['び'] = 'ビ', ['ぼ'] = 'ボ', ['ぶ'] = 'ブ',
	['ぱ'] = 'パ', ['ぺ'] = 'ペ', ['ぴ'] = 'ピ', ['ぽ'] = 'ポ', ['ぷ'] = 'プ',
	['た'] = 'タ', ['て'] = 'テ', ['ち'] = 'チ', ['と'] = 'ト', ['つ'] = 'ツ',
	['だ'] = 'ダ', ['で'] = 'デ', ['ぢ'] = 'ヂ', ['ど'] = 'ド', ['づ'] = 'ヅ',
	['か'] = 'カ', ['け'] = 'ケ', ['き'] = 'キ', ['こ'] = 'コ', ['く'] = 'ク',
	['が'] = 'ガ', ['げ'] = 'ゲ', ['ぎ'] = 'ギ', ['ご'] = 'ゴ', ['ぐ'] = 'グ',
	['ま'] = 'マ', ['め'] = 'マ', ['み'] = 'マ', ['も'] = 'モ', ['む'] = 'マ',
	['な'] = 'ナ', ['ね'] = 'ネ', ['に'] = 'ニ', ['の'] = 'ノ', ['ぬ'] = 'ヌ',
	['ら'] = 'ラ', ['れ'] = 'レ', ['り'] = 'リ', ['ろ'] = 'ロ', ['る'] = 'ル',
	['さ'] = 'サ', ['せ'] = 'セ', ['し'] = 'シ', ['そ'] = 'ソ', ['す'] = 'ス',
	['ざ'] = 'ザ', ['ぜ'] = 'ゼ', ['じ'] = 'ジ', ['ぞ'] = 'ゾ', ['ず'] = 'ズ',
	['わ'] = 'ワ', ['ゑ'] = 'ヱ', ['ゐ'] = 'ヰ', ['を'] = 'ヲ',
	['や'] = 'ヤ', ['よ'] = 'ヨ', ['ゆ'] = 'ユ',
	['ゔ'] = 'ヴ', ['っ'] = 'ッ'
}
local correctionsFromKana = { -- manual transliteration choices
	['し'] = 'shi'
}
local longK = 'ー'
local isolator = '\''
local prolongRules = { -- special rules for prolonging syllables
	o = 'u',
	e = 'i'
}

-- builds a reverse table
local function rev(t)
	local res = {}
	for k, v in pairs(t) do
		if (type(v) == 'table') then
			res[v[1]] = k
			res[v[2]] = k
		else
			res[v] = k
		end
	end
	return res
end

-- builds the default translation tables latin <-> kana from transRaw
local function buildDefaultTransTables()
	local tr, rtr = {}, {}
	
	for k, v in pairs(transRaw) do
		tr[k] = type(v) == 'table' and v[1] or v
	end

	rtr = rev(tr)

	-- apply corrections
	for i, v in pairs(correctionsFromKana) do
		rtr[i] = v
	end

	return tr, rtr, rev(transK)
end

-- decides which wovel should prolong the given vowel
local function prolong(c)
	for i, v in ipairs(vowels) do
		if c == v then
			if prolongRules[c] then return prolongRules[c] else return c end
		end
	end
	return nil
end

-- checks if a katakana token is a vowel and returns its latin representation
local function getWovelK(c)
	for i, v in ipairs(vowelsK) do
		if c == v then return vowels[i] end
	end
	return nil
end

-- checks if a given symbol is ambiguous to preceding n
local function isAmbiguous(c)
	for i, v in ipairs(ambigousToN) do
		if c == v then return true end
	end
	return false
end

-- init translation tables
local trans, revTrans, revTransK = buildDefaultTransTables()

-- init default transliteration choices (everything default to first alternative)
local transChoices = {}

-- checks if two characters are valid candidates for little tsu
local function isValidTsuCandidate(a, b)
	if a ~= b then return false end
	for i, v in ipairs(littleTsuWL) do
		if a == v then return true end
	end
	return false
end

-- checks if two characters are a little tsu used correctly and returns the gemination consonant if true
local function getGeminationConsonant(a, b)
	if a ~= 'っ' then return nil end -- disregard katakana, only hiragana is processed in romanization
	local tr = revTrans[b]
	if not tr then return nil end -- invalid hiragana character
	local fst = string.sub(tr, 1, 1) -- get first character of the transliteration
	for i, v in ipairs(littleTsuWL) do
		if fst == v then return fst end
	end
	return nil -- invalid gemination
end

-- parses an utf8 string into utf8 chars (tokens)
local function tokenize(utf8str)
	assert(type(utf8str) == 'string')
	local res, seq, val = {}, 0, ''
	for i = 1, #utf8str do
		local c = string.byte(utf8str, i)
		if seq == 0 then
			if i ~= 1 then table.insert(res, val) end
			seq = c < 0x80 and 1 or c < 0xE0 and 2 or c < 0xF0 and 3 or
			      c < 0xF8 and 4 or error('invalid UTF-8 character sequence')
			val = string.char(c)
		else
			val = val .. string.char(c)
		end
		seq = seq - 1
	end
	table.insert(res, val)
	return res
end

-- PUBLIC API SECTION

-- toggles used characters for supplied syllables (whitespace-separated)
function toggleChars(input)
	local cur, choices = '', {}
	for s in string.gmatch(input, '%S+') do -- split by whitespaces
		cur = trans[s]
		if cur then -- don't process unknown syllables
			choices = transRaw[s]
			if type(choices) == 'table' then -- only process syllables with alternatives
				trans[s] = cur == choices[1] and choices[2] or choices[1] -- toggle between alternatives
			end
		end
	end
end

-- any kana to latin
function toLatin(input)
	if input == '' then return end
	local tbl = tokenize(input)
	local buffer, res = {}, ''

	-- read tokenized input
	local tjoin, tfst, last, gc = '', '', 0, '' -- last is the last valid transliterated vowel, gc is the last gemination consonant
	for i, v in ipairs(tbl) do
		if revTransK[v] ~= nil then v = revTransK[v] end -- convert all katakana to hiragana
		table.insert(buffer, v)

		if #buffer == 2 then -- kana can be formed with up to two characters, always keep two in buffer
			tjoin, tfst, gc = revTrans[ buffer[1] .. buffer[2] ], revTrans[ buffer[1] ], getGeminationConsonant(buffer[1], buffer[2])
			if tjoin ~= nil then -- double character
				res = res .. tjoin
				buffer, last = {}, string.sub(tjoin, -1)
			elseif gc then -- check for little tsu
				res = res .. gc
				buffer, last = {buffer[2]}, 0
			elseif tfst ~= nil then -- single character
				res = res .. tfst
				if tfst == 'n' and isAmbiguous(buffer[2]) then -- ambiguous character succeeding an "n"
					res = res .. isolator
				end
				buffer, last = {buffer[2]}, string.sub(tfst, -1)
			elseif buffer[1] == longK and prolong(last) ~= nil then -- prolonging dash
				res = res .. prolong(last)
				buffer, last = {buffer[2]}, 0
			else -- cannot transliterate, output as-is
				res = res .. buffer[1]
				buffer, last = {buffer[2]}, 0
			end
		end
	end

	if #buffer == 1 then -- trailing character
		if revTrans[ buffer[1] ] ~= nil then -- single character
			res = res .. revTrans [ buffer[1] ]
		elseif buffer[1] == longK and prolong(last) ~= nil then -- prolonging dash
			res = res .. prolong(last)
		else -- cannot transliterate, output as-is
			res = res .. buffer[1]
		end
	end

	tex.print(res)
end

-- latin or katakana to hiragana, 'raw' parameter is for internal use, leave it blank to get output to TeX
function toHiragana(input, raw)
	if input == '' then return end
	local tbl = tokenize(input)
	local buffer, res = {}, ''
	local t3, t2, t1, last, lastsym, lastcnd = '', '', '', 0, nil, nil

	for i, v in ipairs(tbl) do
		if revTransK[v] then v = revTransK[v] end -- translate katakana to hiragana on the go
		table.insert(buffer, v)

		if #buffer == 3 then
			t3, t2, t1 = trans[ buffer[1] .. buffer[2] .. buffer[3] ], trans[ buffer[1] .. buffer[2] ], trans[ buffer[1] ]
			if t3 ~= nil then -- all three letters yield translation
				if lastcnd then -- add little tsu
					res = res .. 'っ'
					lastcnd = nil
				end
				res = res .. t3
				last = buffer[3]
				buffer = {}
			elseif t2 ~= nil then -- first two letters yield translation
				if lastcnd then -- add little tsu
					res = res .. 'っ'
					lastcnd = nil
				end
				res = res .. t2
				last = buffer[2]
				buffer = {buffer[3]}
			elseif isValidTsuCandidate(buffer[1], buffer[2]) then -- test little tsu candidates
				if lastcnd then res = res .. lastcnd end -- add last consonant in raw form
				lastcnd = buffer[1] -- set last candidate consonant
				last = 0 -- is not vowel
				buffer = {buffer[2], buffer[3]}
			elseif t1 ~= nil then -- first letter yields translation : a, e, i, o, u, n
				res = res .. t1
				last = buffer[1]
				buffer = {buffer[2], buffer[3]}
			elseif buffer[1] == longK and prolong(last) ~= nil then -- valid prolonger sign
				res = res .. trans[prolong(last)]
				buffer, last = {buffer[2], buffer[3]}, 0
			elseif buffer[1] == isolator then -- isolating apostrophe, consume it
				buffer = {buffer[2], buffer[3]}
			else
				if lastcnd then -- add last consonant in raw form
					res = res .. lastcnd
					lastcnd = nil
				end

				-- this code allows for proper conversion of katakana's prolongation dash to hiragana
				t1 = revTrans[ buffer[1] ]
				if t1 then -- symbol is standalone hiragana
					last = string.sub(t1, -1)
					lastsym = buffer[1]
				elseif lastsym then -- attempt to merge symbol with previous symbol
					t1 = revTrans[ lastsym .. buffer[1] ]
					if t1 then -- symbol is a valid non-standalone hiragana compound
						last = string.sub(t1, -1)
					else -- symbol is an invalid non-standalone hiragana compound
						last = nil
					end
					lastsym = nil
				else
					last, lastsym = 0, nil
				end
				
				res = res .. buffer[1]
				buffer = {buffer[2], buffer[3]}
			end
		end
	end

	if #buffer == 2 then
		if trans[ buffer[1] .. buffer[2] ] ~= nil then -- first two symbols yield translation
			if lastcnd then res = res .. 'っ' end -- add little tsu
			res = res .. trans[ buffer[1] .. buffer[2] ]
			last = buffer[2]
			buffer = {}
		elseif trans[ buffer[1] ] ~= nil then -- first symbol yields translation
			res = res .. trans[ buffer[1] ]
			last = buffer[1]
			buffer = {buffer[2]}
		elseif buffer[1] == longK and prolong(last) ~= nil then -- valid prolonger
			res = res .. trans[prolong(last)]
			buffer, last = {buffer[2]}, 0
		elseif buffer[1] == isolator then -- consume isolator
			buffer = {buffer[2]}
		else
			if lastcnd then res = res .. lastcnd end -- add last consonant in raw form

			-- this code allows for proper conversion of katakana's prolongation dash to hiragana
			t1 = revTrans[ buffer[1] ]
			if t1 then -- symbol is standalone hiragana
				last = string.sub(t1, -1)
				lastsym = buffer[1]
			elseif lastsym then -- attempt to merge symbol with previous symbol
				t1 = revTrans[ lastsym .. buffer[1] ]
				if t1 then -- symbol is a valid non-standalone hiragana compound
					last = string.sub(t1, -1)
				else -- symbol is an invalid non-standalone hiragana compound
					last = nil
				end
				lastsym = nil -- erase last valid symbol
			else
				last, lastsym = 0, nil
			end

			res = res .. buffer[1]
			buffer = {buffer[2]}
		end
	end

	if #buffer == 1 then -- remaining symbol
		if trans[ buffer[1] ] ~= nil then
			res = res .. trans[ buffer[1] ]
		elseif buffer[1] == longK and prolong(last) ~= nil then
			res = res .. trans[prolong(last)]
		elseif buffer[1] ~= isolator then
			res = res .. buffer[1]
		end
	end

	if not raw then
		tex.print(res)
	else
		return res -- for internal use
	end
end

-- latin or hiragana to katakana
function toKatakana(input)
	if input == '' then return end
	local hiraganized = tokenize(toHiragana(input, true)) -- convert everything to hiragana

	-- replace hiragana with katakana
	for i, v in ipairs(hiraganized) do
		if transK[v] ~= nil then
			hiraganized[i] = transK[v]
		end
	end

	-- insert prolonging symbols and prepare output
	local prev, nxt, vowel, tprev, tnext, res = hiraganized[1], '', '', '', '', hiraganized[1]
	local merge, toprolong = '', nil
	for i = 2, #hiraganized do
		nxt = hiraganized[i]

		vowel = getWovelK(nxt)

		if not toprolong then -- check prev for ending vowel
			tprev = revTransK[prev]
			if tprev then
				tprev = revTrans[tprev]
				if tprev then
					toprolong = prolong(string.sub(tprev, -1))
				end
			end
		end

		if toprolong then -- check nxt for matching prolonger
			if toprolong == vowel then
				nxt = longK
				toprolong = nil
			elseif vowel then
				toprolong = prolong(vowel)
			else
				toprolong = nil
			end
		end

		-- try merging prev and nxt for a single token
		tprev, tnext = revTransK[prev], revTransK[nxt]
		if tprev and tnext then
			merge = revTrans[tprev .. tnext]
			if merge then
				toprolong = prolong(string.sub(merge, -1))
			end
		end

		res = res .. nxt
		prev = nxt
	end

	tex.print(res)
end