From 96c21343055c250b69580b63f5726ff626328a73 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Tue, 29 Dec 2020 22:01:58 +0000 Subject: uninormalize (29dec20) git-svn-id: svn://tug.org/texlive/trunk@57257 c570f23f-e606-0410-a88d-b1316a301751 --- .../texmf-dist/doc/lualatex/uninormalize/README.md | 64 ++++ .../doc/lualatex/uninormalize/uninormalize-doc.pdf | Bin 0 -> 62530 bytes .../doc/lualatex/uninormalize/uninormalize-doc.tex | 49 +++ .../uninormalize/unicode-normalization.lua | 351 +++++++++++++++++++++ .../uninormalize/unicode-normalize-names.lua | 57 ++++ .../lualatex/uninormalize/unicode-normalize.lua | 226 +++++++++++++ .../tex/lualatex/uninormalize/uninormalize.sty | 35 ++ 7 files changed, 782 insertions(+) create mode 100644 Master/texmf-dist/doc/lualatex/uninormalize/README.md create mode 100644 Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.pdf create mode 100644 Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.tex create mode 100644 Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua create mode 100644 Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize-names.lua create mode 100644 Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize.lua create mode 100644 Master/texmf-dist/tex/lualatex/uninormalize/uninormalize.sty (limited to 'Master/texmf-dist') diff --git a/Master/texmf-dist/doc/lualatex/uninormalize/README.md b/Master/texmf-dist/doc/lualatex/uninormalize/README.md new file mode 100644 index 00000000000..942f128dc48 --- /dev/null +++ b/Master/texmf-dist/doc/lualatex/uninormalize/README.md @@ -0,0 +1,64 @@ +# The `uninormalize` package + +The purpose of this package is to provide Unicode normalization for LuaLaTeX. It is based on Arthur Reutenauer's +[code for GSOC 2008](https://code.google.com/p/google-summer-of-code-2008-tex/downloads/list), which was adapted a little bit to work with +current `Luaotfload`. For more information, see [this question on TeX.sx](http://tex.stackexchange.com/q/229044/7712). + +## What does that mean? + +Citing [Wikipedia](https://en.wikipedia.org/wiki/Unicode_equivalence): + +> Unicode equivalence is the specification by the Unicode character encoding +> standard that some sequences of code points represent essentially the same +> character. This feature was introduced in the standard to allow compatibility +> with preexisting standard character sets, which often included similar or +> identical characters. +> +> Unicode provides two such notions, canonical equivalence and compatibility. +> Code point sequences that are defined as canonically equivalent are assumed to +> have the same appearance and meaning when printed or displayed. For example, +> the code point `U+006E` (the Latin lowercase "n") followed by `U+0303` (the +> combining tilde) is defined by Unicode to be canonically equivalent to the +> single code point `U+00F1` (the lowercase letter "ñ" of the Spanish alphabet). + +## Basic usage + + + \documentclass{article} + \usepackage{fontspec} + \usepackage[czech]{babel} + \setmainfont{Linux Libertine O} + \usepackage{uninormalize} + \begin{document} + + Some tests: + \begin{itemize} + \item combined letter ᾳ %GREEK SMALL LETTER ALPHA (U+03B1) + % + COMBINING GREEK YPOGEGRAMMENI + % (U+0345) + \item normal letter ᾳ % GREEK SMALL LETTER ALPHA WITH + %YPOGEGRAMMENI (U+1FB3) + \end{itemize} + + Some more combined and normal letters: + óóōōöö + + Linux Libertine does support some combined chars: \parbox{4em}{příliš} + + Using the \verb|^^^^| syntax: ^^^^0061^^^^0301 ^^^^0041^^^^0301 + \end{document} + +## Package options + +This package has three options: + + +- **buffer** -- normalize processed document at the moment when it's + source file is read, before processing by \TeX\ starts. This is the default + option, it seems to work better than the next one. +- **nodes** -- normalize LuaTeX nodes. Normalization happens after the full processiny by \TeX. +- **debug** -- print debug messages to the terminal output + +Both **buffer** and **nodes** options are enabled by default, you can disable any of them by using: + + \usepackage[nodes=false,buffer=false]{uninormalize} diff --git a/Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.pdf b/Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.pdf new file mode 100644 index 00000000000..9f45abc1e93 Binary files /dev/null and b/Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.pdf differ diff --git a/Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.tex b/Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.tex new file mode 100644 index 00000000000..d0f6eaf528a --- /dev/null +++ b/Master/texmf-dist/doc/lualatex/uninormalize/uninormalize-doc.tex @@ -0,0 +1,49 @@ +\documentclass{article} +\usepackage{url} +\ifx\HCode\undefined +\usepackage{fontspec} +\setmainfont{Linux Libertine O}[Renderer = Harfbuzz] +\setmonofont{DejaVu Sans Mono}[Scale=MatchLowercase] +\fi +\usepackage{microtype,hyperref} +\usepackage[nodes]{uninormalize} +\usepackage{markdown} +\def\tightlist{} +\begin{document} +\title{The \texttt{uninormalize} package} +\author{Michal Hoftich\footnote{\url{michal.h21@gmail.com}} \and Arthur Reutenauer\footnote{\url{arthur.reutenauer@normalesup.org }}} +\date{Version 0.1\\28/12/2020} +\maketitle + +\markdownInput[hybrid]{README.md} + +\subsection{Example results} + +\begin{itemize} + \item combined letter ᾳ %GREEK SMALL LETTER ALPHA (U+03B1) + COMBINING GREEK YPOGEGRAMMENI (U+0345) + \item normal letter ᾳ% GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI (U+1FB3) +\end{itemize} + +Some more combined and normal letters: +óóōōöö + +Linux Libertine does support some combined chars: \parbox{4em}{příliš} + +Using the \verb|^^^^| syntax: ^^^^0061^^^^0301 ^^^^0041^^^^0301 + +\subsection{License} + +Copyright: 2020 Michal Hoftich + +This work may be distributed and/or modified under the conditions of the +\textit{\LaTeX\ Project Public License}, either version 1.3 of this license or (at your option) +any later version. The latest version of this license is in +\url{http://www.latex-project.org/lppl.txt} and version 1.3 or later is part of all +distributions of \LaTeX\ version 2005/12/01 or later. + +This work has the LPPL maintenance status \textit{maintained}. + +The Current Maintainer of this work is Michal Hoftich. + +\end{document} + diff --git a/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua new file mode 100644 index 00000000000..b8a58b63bb6 --- /dev/null +++ b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalization.lua @@ -0,0 +1,351 @@ +-- char-def now contains all necessary fields, no need for a custom file +if not characters then + require "char-def" +end + +if not unicode then require('unicode') end +unicode.conformance = unicode.conformance or { } + +unicharacters = unicharacters or {} +uni = unicode.utf8 +unidata = characters.data + +function printf(s, ...) print(string.format(s, ...)) end +-- function debug(s, ...) io.write("DEBUG: ", string.format(s, ...), "\n") end +function warn(s, ...) io.write("Warning: ", string.format(s, ...), "\n") end + +function md5sum(any) return md5.hex(md5.sum(any)) end + +-- Rehash the character data +unicharacters.combinee = { } +unicharacters.context = unicharacters.context or { } +local charu = unicode.utf8.char + +function unicharacters.context.rehash2() + for ucode, udata in pairs(unidata) -- *not* ipairs :-) + do + local sp = udata.specials + if sp then + if sp[1] == 'char' then + -- local ucode = udata.unicodeslot + local entry = { combinee = sp[2], combining = sp[3], combined = ucode } + if not unicharacters.combinee[sp[2]] + then unicharacters.combinee[sp[2]] = { } end + local n = #unicharacters.combinee[sp[2]] + unicharacters.combinee[sp[2]][n+1] = entry + end + end + -- copy context's combining field to combclass field + -- this field was in the custom copy of the char-def.lua that we no longer use + udata.combclass = udata.combining + end +end + + +unicharacters.context.rehash2() +combdata = unicharacters.combinee + +--[[ function unicode.conformance.is_hangul(ucode) + return ucode >= 0xAC00 and ucode <= 0xD7A3 +end ]] -- Make it local for the moment +local function is_hangul(char) + return char >= 0xAC00 and char <= 0xD7A3 +end + +local function is_jamo(char) + if char < 0x1100 then return false + elseif char < 0x1160 then return 'choseong' + elseif char < 0x11A8 then return 'jungseong' + elseif char < 0x11FA then return 'jongseong' + else return false + end +end + +local function decompose(ucode, compat) -- if compat then compatibility + local invbuf = { } + local sp = unidata[ucode].specials + if not sp + then return { ucode } else + if compat then compat = (sp[1] == 'compat') else compat = false end + while sp[1] == 'char' or compat do + head, tail = sp[2], sp[3] + if not tail then invbuf[#invbuf + 1] = head break end -- singleton + invbuf[#invbuf + 1] = tail + sp = unidata[head].specials + if not sp then invbuf[#invbuf + 1] = head sp = { } end + -- end -- not unidata[head] + end -- while sp[1] == 'char' or compat + end -- not sp + + local seq = { } + for i = #invbuf, 1, -1 + do seq[#seq + 1] = invbuf[i] + end + return seq +end + +local function canon(seq) -- Canonical reordering + if #seq < 3 then return seq end + local c1, c2, buf + -- I'd never thought I'd implement an actual bubble sort some day ;-) + for k = #seq - 1, 1, -1 do + for i = 2, k do -- was k - 1! Argh! + c1 = unidata[seq[i]].combclass + c2 = unidata[seq[i+1]].combclass + if c1 and c2 then + if c1 > c2 then + buf = seq[i] + seq[i] = seq[i+1] + seq[i+1] = buf + end + end + end + end + return seq +end + +if not math.div then -- from l-math.lua + function math.div(n, m) + return math.floor(n/m) + end +end + +local SBase, LBase, VBase, TBase = 0xAC00, 0x1100, 0x1161, 0x11A7 +local LCount, VCount, TCount = 19, 21, 28 +local NCount = VCount * TCount +local SCount = TCount * NCount + +local function decompose_hangul(ucode) -- assumes input is really a Hangul + local SIndex = ucode - SBase + local L = LBase + math.div(SIndex, NCount) + local V = VBase + math.div((SIndex % NCount), TCount) + local T = TBase + SIndex % TCount + if T == TBase then T = nil end + return { L, V, T } +end + +-- To NFK?D. +function toNF_D_or_KD(unistring, compat) + local nfd, seq = { }, { } + for uchar in uni.gmatch(unistring, '.') do + local ucode = uni.byte(uchar) + if is_hangul(ucode) then + seq = decompose_hangul(ucode) + for _, c in ipairs(seq) + do nfd[#nfd + 1] = c + end + seq = { } + elseif not unidata[ucode] + then nfd[#nfd + 1] = ucode else + local ccc = unidata[ucode].combclass + if not ccc or ccc == 0 then + seq = canon(seq) + for _, c in ipairs(seq) do nfd[#nfd + 1] = c end + seq = decompose(ucode, compat) + else seq[#seq + 1] = ucode + end -- not ccc or ccc == 0 + end -- if is_hangul(ucode) / elseif not unidata[ucode] + end -- for uchar in uni.gmatch(unistring, ".") + + if #seq > 0 then + seq = canon(seq) + for _, c in ipairs(seq) do nfd[#nfd + 1] = c end + end + + local nfdstr = "" + for _, chr in ipairs(nfd) + do nfdstr = string.format("%s%s", nfdstr, uni.char(chr)) end + return nfdstr, nfd +end + +function unicode.conformance.toNFD(unistring) + return toNF_D_or_KD(unistring, false) +end + +function unicode.conformance.toNFKD(unistring) + return toNF_D_or_KD(unistring, true) +end + +local function compose(seq) + local base = seq[1] + if not combdata[base] then return seq else + local i = 2 + while i <= #seq do -- can I play with 'i' in a for loop? + local cbng = seq[i] + local cccprev + if unidata[seq[i-1]] then cccprev = unidata[seq[i-1]].combclass end + if not cccprev then cccprev = -1 end + if unidata[cbng].combclass > cccprev then + if not combdata[base] then return seq else + for _, cbdata in ipairs(combdata[base]) do + if cbdata.combining == cbng then + seq[1] = cbdata.combined + base = seq[1] + for k = i, #seq - 1 + do seq[k] = seq[k+1] + end -- for k = i, #seq - 1 + seq[#seq] = nil + i = i - 1 + end -- if cbdata.combining == cbng + end -- for _, cbdata in ipairs(combdata[base]) + end -- if unidata[cbng.combclass > cccprev + end -- if not combdata[base] + i = i + 1 + end -- while i <= #seq + end -- if not combdata[base] + return seq +end + +-- To NFC from NFD. +-- Does not yet take all the composition exclusions in account +-- (missing types 1 and 2 as defined by UAX #15 X6) +function unicode.conformance.toNFC_fromNFD(nfd) + local nfc = { } + local seq = { } + for uchar in uni.gmatch(nfd, '.') do + local ucode = uni.byte(uchar) + if not unidata[ucode] + then nfc[#nfc + 1] = ucode else + local cb = unidata[ucode].combclass + if not cb or (cb == 0) then + -- if seq ~= { } then -- Dubious ... + if #seq > 0 then + seq = compose(seq) -- There was a check for #seq == 1 here + for i = 1, #seq do nfc[#nfc + 1] = seq[i] end + end -- #seq > 0 + seq = { ucode } + else seq[#seq + 1] = ucode --[[ Maybe check if seq is not empty ... ]] + end -- not cb or cb == 0 + end + end + + seq = compose(seq) + for i = 1, #seq do nfc[#nfc + 1] = seq[i] end + + local nfcstr = "" + for _, chr in ipairs(nfc) + do nfcstr = string.format("%s%s", nfcstr, uni.char(chr)) end + return nfcstr, nfc +end + +local function cancompose(seq, compat) + local dec = { } -- new table to hold the decomposed sequence + + local shift + if #seq >= 2 then -- let's do it the brutal way :-) + if is_jamo(seq[1]) == 'choseong' and + is_jamo(seq[2]) == 'jungseong' then + LIndex = seq[1] - LBase + VIndex = seq[2] - VBase + if #seq == 2 or is_jamo(seq[3]) ~= 'jongseong' then + TIndex = 0 + shift = 1 + else + TIndex = seq[3] - TBase + shift = 2 + end + seq[1] = (LIndex * VCount + VIndex) * TCount + TIndex + SBase + for i = 2, #seq -- this shifts and shrinks the table at the same time + do seq[i] = seq[i + shift] + end + end + end + + dec[1] = seq[1] + for i = 2, #seq do + local u = seq[i] + local sp = unidata[u].specials + if sp then + if compat then compat = (sp[1] == 'compat') else compat = false end + if (sp[1] == 'char') or compat then + for i = 2, #sp + do dec[#dec + 1] = sp[i] + end + end + else dec[#dec + 1] = u + end + end -- we have the fully decomposed sequence; now sort it + + for i = #dec - 1, 2, -1 do -- bubble sort! + for j = 2, #dec - 1 do + local u = dec[j] + local ccc1 = unidata[u].combclass + local v = dec[j+1] + local ccc2 = unidata[v].combclass + if ccc1 > ccc2 then -- swap + dec[j+1] = u + dec[j] = v + end + end + end -- dec sorted; now recursively compose + + local base, i, n = dec[1], 2, #dec + local cbd = combdata[base] + local incr_i = true + while i <= n do + local cbg = dec[i] + if cbd then + for _, cb in ipairs(cbd) do + if cb.combining == cbg then + -- NO :-) -- if cbd[cbg] then -- base and cbg combine; compose + dec[1] = cb.combined + base = dec[1] + cbd = combdata[base] + for j = i, n-1 -- shift table elements right of i + do dec[j] = dec[j+1] end + dec[n] = nil + n = n-1 -- table has shrunk by 1, and i doesn't grow + incr_i = false + end + end + end + if incr_i then i = i + 1 + else incr_i = true end + end -- we're finally through! return + return dec +end + +function toNF_C_or_KC(unistring, compat) + if unistring == "" then return "" end + local nfc, seq = "", { } + local start, space = true, "" + for uchar in uni.gmatch(unistring, '.') do + local ucode = uni.byte(uchar) + if start then space = ", " start = true end + if not unidata[ucode] then -- unknown to the UCD, will not compose + nfc = string.format("%s%s", nfc, uchar) + else + local ccc = unidata[ucode].combclass + if not (ccc or is_jamo(ucode) == 'jongseong' + or is_jamo(ucode) == 'jungseong') + or ccc == 0 or (is_jamo(ucode) == 'choseong') then + -- and is actually good :-) -- Well, yes and no ;-) + if #seq == 0 then -- add ucode and go to next item of the loop + seq = { ucode } + else -- seq contains unicharacters, try and compose them + if #seq == 1 then nfc = string.format("%s%s", nfc, uni.char(seq[1])) + else dec = cancompose(seq, compat) + for _, c in ipairs(dec) -- add the whole sequence to nfc + do nfc = string.format("%s%s", nfc, uni.char(c)) end + end + seq = { ucode } -- don't forget to reinitialize seq with current char + end + else -- not ccc or ccc == 0 and is_choseong: + -- character is combining, add it to seq + seq[#seq + 1] = ucode + end + end + end + if #seq > 0 then dec = cancompose(seq, compat) end + for _, c in ipairs(dec) + do nfc = string.format("%s%s", nfc, uni.char(c)) end + return nfc +end + +function unicode.conformance.toNFC(unistring) + return toNF_C_or_KC(unistring, false) +end + +function unicode.conformance.toNFKC(unistring) + return toNF_C_or_KC(unistring, true) +end diff --git a/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize-names.lua b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize-names.lua new file mode 100644 index 00000000000..a15d6b09785 --- /dev/null +++ b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize-names.lua @@ -0,0 +1,57 @@ +-- Unicode names + +if not characters then + require "char-def" +end + +unicode = unicode or { } +unicode.conformance = unicode.conformance or { } + +unidata = characters.data + +if not math.div then -- from l-math.lua + function math.div(n, m) + return math.floor(n/m) + end +end + +local function is_hangul(char) + return char >= 0xAC00 and char <= 0xD7A3 +end + +local function is_han_character(char) -- from font-otf.lua (check) + return + (char>=0x04E00 and char<=0x09FFF) or + (char>=0x03400 and char<=0x04DFF) or + (char>=0x20000 and char<=0x2A6DF) or + (char>=0x0F900 and char<=0x0FAFF) or + (char>=0x2F800 and char<=0x2FA1F) +end + +local SBase, LBase, VBase, TBase = 0xAC00, 0x1100, 0x1161, 0x11A7 +local LCount, VCount, TCount = 19, 21, 28 +local NCount = VCount * TCount +local SCount = LCount * NCount + +local JAMO_L_TABLE = { [0] = "G", "GG", "N", "D", "DD", "R", "M", "B", "BB", + "S", "SS", "", "J", "JJ", "C", "K", "T", "P", "H" } +local JAMO_V_TABLE = { [0] = "A", "AE", "YA", "YAE", "EO", "E", "YEO", "YE", + "O", "WA", "WAE", "OE", "YO", "U", "WEO", "WE", "WI", "YU", "EU", "YI", "I" } +local JAMO_T_TABLE = { [0] = "", "G", "GG", "GS", "N", "NJ", "NH", "D", "L", + "LG", "LM", "LB", "LS", "LT", "LP", "LH", "M", "B", "BS", "S", "SS", "NG", + "J", "C", "K", "T", "P", "H" } + +function unicode.conformance.name(char) + if is_hangul(char) then + local SIndex = char - SBase + local LIndex = math.div(SIndex, NCount) + local VIndex = math.div(SIndex % NCount, TCount) + local TIndex = SIndex % TCount + return string.format("HANGUL SYLLABLE %s%s%s", JAMO_L_TABLE[LIndex], + JAMO_V_TABLE[VIndex], JAMO_T_TABLE[TIndex]) + elseif is_han_character(char) + then return string.format("CJK UNIFIED IDEOGRAPH-%04X", char) + elseif unidata[char] -- if unidata[char] exists, the name exists + then return unidata[char].description + end +end diff --git a/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize.lua b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize.lua new file mode 100644 index 00000000000..0bd797d4b7f --- /dev/null +++ b/Master/texmf-dist/tex/lualatex/uninormalize/unicode-normalize.lua @@ -0,0 +1,226 @@ +local M = {} +require("unicode-normalize-names") +require('unicode-normalization') +local NFC = unicode.conformance.toNFC +local char = unicode.utf8.char +local gmatch = unicode.utf8.gmatch +local name = unicode.conformance.name +local byte = unicode.utf8.byte +-- local unidata = unicharacters.data +local length = unicode.utf8.len + +local glyph_id = node.id "glyph" + +M.debug = false + +-- for some reason variable number of arguments doesn't work +local function debug_msg(a,b,c,d,e,f,g,h,i) + if M.debug then + local t = {a,b,c,d,e,f,g,h,i} + print("[uninormalize]", table.unpack(t)) + end +end + +local function make_hash (t) + local y = {} + for _,v in ipairs(t) do + y[v] = true + end + return y +end + +local letter_categories = make_hash {"lu","ll","lt","lo","lm"} + +local mark_categories = make_hash {"mn","mc","me"} + +local function printchars(s) + local t = {} + for x in gmatch(s,".") do + t[#t+1] = name(byte(x)) + end + debug_msg("characters",table.concat(t,":")) +end + +local categories = {} + + +local function get_category(charcode) + local charcode = charcode or "" + if categories[charcode] then + return categories[charcode] + else + local unidatacode = unidata[charcode] or {} + local category = unidatacode.category + categories[charcode] = category + return category + end +end + +-- get glyph char and category +local function glyph_info(n) + local char = n.char + return char, get_category(char) +end + +local function get_mark(n) + if n.id == glyph_id then + local character, cat = glyph_info(n) + if mark_categories[cat] then + return char(character) + end + end + return false +end + +local function make_glyphs(head, nextn,s, lang, font, subtype) + local g = function(a) + local new_n = node.new(glyph_id, subtype) + new_n.lang = lang + new_n.font = font + new_n.char = byte(a) + return new_n + end + if length(s) == 1 then + return node.insert_before(head, nextn,g(s)) + else + local t = {} + local first = true + for x in gmatch(s,".") do + debug_msg("multi letter",x) + head, newn = node.insert_before(head, nextn, g(x)) + end + return head + end +end + +local function normalize_marks(head, n) + local lang, font, subtype = n.lang, n.font, n.subtype + local text = {} + text[#text+1] = char(n.char) + local head, nextn = node.remove(head, n) + --local nextn = n.next + local info = get_mark(nextn) + while(info) do + text[#text+1] = info + head, nextn = node.remove(head,nextn) + info = get_mark(nextn) + end + local s = NFC(table.concat(text)) + debug_msg("We've got mark: " .. s) + local new_n = node.new(glyph_id, subtype) + new_n.lang = lang + new_n.font = font + new_n.char = byte(s) + --head, new_n = node.insert_before(head, nextn, new_n) + -- head, new_n = node.insert_before(head, nextn, make_glyphs(s, lang, font, subtype)) + head, new_n = make_glyphs(head, nextn, s, lang, font, subtype) + local t = {} + for x in node.traverse_id(glyph_id,head) do + t[#t+1] = char(x.char) + end + debug_msg("Variables ", table.concat(t,":"), table.concat(text,";"), char(byte(s)),length(s)) + return head, nextn +end + +local function normalize_glyphs(head, n) + --local charcode = n.char + --local category = get_category(charcode) + local charcode, category = glyph_info(n) + if letter_categories[category] then + local nextn = n.next + if nextn and nextn.id == glyph_id then + --local nextchar = nextn.char + --local nextcat = get_category(nextchar) + local nextchar, nextcat = glyph_info(nextn) + if mark_categories[nextcat] then + return normalize_marks(head,n) + end + end + end + return head, n.next +end + + +function M.nodes(head) + local t = {} + local text = false + local n = head + -- for n in node.traverse(head) do + while n do + if n.id == glyph_id then + local charcode = n.char + debug_msg("unicode name",name(charcode)) + debug_msg("character category",get_category(charcode)) + t[#t+1]= char(charcode) + text = true + head, n = normalize_glyphs(head, n) + else + if text then + local s = table.concat(t) + debug_msg("text chunk",s) + --printchars(NFC(s)) + debug_msg("----------") + end + text = false + t = {} + n = n.next + end + end + return head +end + +local unibytes = {} + +local function get_charcategory(s) + local s = s or "" + local b = unibytes[s] or byte(s) or 0 + unibytes[s] = b + return get_category(b) +end + +local function normalize_charmarks(t,i) + local c = {t[i]} + local i = i + 1 + local s = get_charcategory(t[i]) + while mark_categories[s] do + c[#c+1] = t[i] + i = i + 1 + s = get_charcategory(t[i]) + end + return NFC(table.concat(c)), i +end + +local function normalize_char(t,i) + local ch = t[i] + local c = get_charcategory(ch) + if letter_categories[c] then + local nextc = get_charcategory(t[i+1]) + if mark_categories[nextc] then + return normalize_charmarks(t,i) + end + end + return ch, i+1 +end + +function M.buffer(line) + local t = {} + local new_t = {} + -- we need to make table witl all uni chars on the line + for x in gmatch(line,".") do + t[#t+1] = x + end + local i = 1 + -- normalize next char + local c, i = normalize_char(t, i) + new_t[#new_t+1] = c + while t[i] do + c, i = normalize_char(t,i) + -- local c = t[i] + -- i = i + 1 + new_t[#new_t+1] = c + end + return table.concat(new_t) +end + + +return M diff --git a/Master/texmf-dist/tex/lualatex/uninormalize/uninormalize.sty b/Master/texmf-dist/tex/lualatex/uninormalize/uninormalize.sty new file mode 100644 index 00000000000..4e1e4b15716 --- /dev/null +++ b/Master/texmf-dist/tex/lualatex/uninormalize/uninormalize.sty @@ -0,0 +1,35 @@ +\ProvidesPackage{uninormalize} +\RequirePackage{luatexbase} +\RequirePackage{luacode} +\RequirePackage{kvoptions} +\DeclareBoolOption[true]{nodes} +\DeclareBoolOption[true]{buffer} +\DeclareBoolOption{debug} +\ProcessKeyvalOptions* +\ifuninormalize@nodes + \luaexec{processnodes=true} +\fi +\ifuninormalize@buffer + \luaexec{processbuffer=true} +\fi +\ifuninormalize@debug + \luaexec{uninormalize_debug = true} +\fi +\begin{luacode*} +local normalize = require "unicode-normalize" +if processnodes==true then + print "[uninormalize] process nodes on" + luatexbase.add_to_callback("pre_linebreak_filter",normalize.nodes, "normalize unicode") + luatexbase.add_to_callback("hpack_filter",normalize.nodes, "normalize unicode") +end +if processbuffer== true then + print "[uninormalize] process buffer on" + luatexbase.add_to_callback("process_input_buffer", normalize.buffer," normalize unicode") +end +if uninormalize_debug then + normalize.debug = true +end +\end{luacode*} + + +\endinput -- cgit v1.2.3