summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/mkxl/char-tex.lmt
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/mkxl/char-tex.lmt')
-rw-r--r--Master/texmf-dist/tex/context/base/mkxl/char-tex.lmt994
1 files changed, 994 insertions, 0 deletions
diff --git a/Master/texmf-dist/tex/context/base/mkxl/char-tex.lmt b/Master/texmf-dist/tex/context/base/mkxl/char-tex.lmt
new file mode 100644
index 00000000000..3d8f6a2598e
--- /dev/null
+++ b/Master/texmf-dist/tex/context/base/mkxl/char-tex.lmt
@@ -0,0 +1,994 @@
+if not modules then modules = { } end modules ['char-tex'] = {
+ version = 1.001,
+ comment = "companion to char-ini.mkiv",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files"
+}
+
+local lpeg = lpeg
+local tonumber, next, type = tonumber, next, type
+local format, find, gmatch, match, gsub = string.format, string.find, string.gmatch, string.match, string.gsub
+local utfchar, utfbyte = utf.char, utf.byte
+local concat, tohash = table.concat, table.tohash
+local P, C, R, S, V, Cs, Cc = lpeg.P, lpeg.C, lpeg.R, lpeg.S, lpeg.V, lpeg.Cs, lpeg.Cc
+
+local lpegpatterns = lpeg.patterns
+local lpegmatch = lpeg.match
+local utfchartabletopattern = lpeg.utfchartabletopattern
+
+local allocate = utilities.storage.allocate
+local mark = utilities.storage.mark
+
+local context = context
+local commands = commands
+
+local characters = characters
+local texcharacters = { }
+characters.tex = texcharacters
+local utffilters = characters.filters.utf
+
+local allocate = utilities.storage.allocate or function() return { } end
+local mark = utilities.storage.mark or allocate
+
+local is_character = characters.is_character
+local is_letter = characters.is_letter
+local is_command = characters.is_command
+local is_spacing = characters.is_spacing
+local is_mark = characters.is_mark
+local is_punctuation = characters.is_punctuation
+
+local data = characters.data if not data then return end
+local blocks = characters.blocks
+
+local trace_defining = false trackers.register("characters.defining", function(v) characters_defining = v end)
+
+local report_defining = logs.reporter("characters")
+
+--[[ldx--
+<p>In order to deal with 8-bit output, we need to find a way to go from <l n='utf'/> to
+8-bit. This is handled in the <l n='luatex'/> engine itself.</p>
+
+<p>This leaves us problems with characters that are specific to <l n='tex'/> like
+<type>{}</type>, <type>$</type> and alike. We can remap some chars that tex input files
+are sensitive for to a private area (while writing to a utility file) and revert then
+to their original slot when we read in such a file. Instead of reverting, we can (when
+we resolve characters to glyphs) map them to their right glyph there. For this purpose
+we can use the private planes 0x0F0000 and 0x100000.</p>
+--ldx]]--
+
+local low = allocate()
+local high = allocate()
+local escapes = allocate()
+local special = "~#$%^&_{}\\|" -- "~#$%{}\\|"
+
+local private = {
+ low = low,
+ high = high,
+ escapes = escapes,
+}
+
+utffilters.private = private
+
+for ch in gmatch(special,".") do
+ local cb
+ if type(ch) == "number" then
+ cb, ch = ch, utfchar(ch)
+ else
+ cb = utfbyte(ch)
+ end
+ if cb < 256 then
+ escapes[ch] = "\\" .. ch
+ low[ch] = utfchar(0x0F0000 + cb)
+ if ch == "%" then
+ ch = "%%" -- nasty, but we need this as in replacements (also in lpeg) % is interpreted
+ end
+ high[utfchar(0x0F0000 + cb)] = ch
+ end
+end
+
+local tohigh = lpeg.replacer(low) -- frozen, only for basic tex
+local tolow = lpeg.replacer(high) -- frozen, only for basic tex
+
+lpegpatterns.utftohigh = tohigh
+lpegpatterns.utftolow = tolow
+
+function utffilters.harden(str)
+ return lpegmatch(tohigh,str)
+end
+
+function utffilters.soften(str)
+ return lpegmatch(tolow,str)
+end
+
+private.escape = utf.remapper(escapes) -- maybe: ,"dynamic"
+private.replace = utf.remapper(low) -- maybe: ,"dynamic"
+private.revert = utf.remapper(high) -- maybe: ,"dynamic"
+
+--[[ldx--
+<p>We get a more efficient variant of this when we integrate
+replacements in collapser. This more or less renders the previous
+private code redundant. The following code is equivalent but the
+first snippet uses the relocated dollars.</p>
+
+<typing>
+[󰀤x󰀤] [$x$]
+</typing>
+--ldx]]--
+
+-- using the tree-lpeg-mapper would be nice but we also need to deal with end-of-string
+-- cases: "\"\i" and don't want "\relax" to be seen as \r e lax" (for which we need to mess
+-- with spaces
+
+local accentmapping = allocate {
+ ['"'] = { [""] = "¨",
+ A = "Ä", a = "ä",
+ E = "Ë", e = "ë",
+ I = "Ï", i = "ï", ["ı"] = "ï", ["\\i"] = "ï",
+ O = "Ö", o = "ö",
+ U = "Ü", u = "ü",
+ Y = "Ÿ", y = "ÿ",
+ },
+ ["'"] = { [""] = "´",
+ A = "Á", a = "á",
+ C = "Ć", c = "ć",
+ E = "É", e = "é",
+ I = "Í", i = "í", ["ı"] = "í", ["\\i"] = "í",
+ L = "Ĺ", l = "ĺ",
+ N = "Ń", n = "ń",
+ O = "Ó", o = "ó",
+ R = "Ŕ", r = "ŕ",
+ S = "Ś", s = "ś",
+ U = "Ú", u = "ú",
+ Y = "Ý", y = "ý",
+ Z = "Ź", z = "ź",
+ },
+ ["."] = { [""] = "˙",
+ C = "Ċ", c = "ċ",
+ E = "Ė", e = "ė",
+ G = "Ġ", g = "ġ",
+ I = "İ", i = "i", ["ı"] = "i", ["\\i"] = "i",
+ Z = "Ż", z = "ż",
+ },
+ ["="] = { [""] = "¯",
+ A = "Ā", a = "ā",
+ E = "Ē", e = "ē",
+ I = "Ī", i = "ī", ["ı"] = "ī", ["\\i"] = "ī",
+ O = "Ō", o = "ō",
+ U = "Ū", u = "ū",
+ },
+ ["H"] = { [""] = "˝",
+ O = "Ő", o = "ő",
+ U = "Ű", u = "ű",
+ },
+ ["^"] = { [""] = "ˆ",
+ A = "Â", a = "â",
+ C = "Ĉ", c = "ĉ",
+ E = "Ê", e = "ê",
+ G = "Ĝ", g = "ĝ",
+ H = "Ĥ", h = "ĥ",
+ I = "Î", i = "î", ["ı"] = "î", ["\\i"] = "î",
+ J = "Ĵ", j = "ĵ",
+ O = "Ô", o = "ô",
+ S = "Ŝ", s = "ŝ",
+ U = "Û", u = "û",
+ W = "Ŵ", w = "ŵ",
+ Y = "Ŷ", y = "ŷ",
+ },
+ ["`"] = { [""] = "`",
+ A = "À", a = "à",
+ E = "È", e = "è",
+ I = "Ì", i = "ì", ["ı"] = "ì", ["\\i"] = "ì",
+ O = "Ò", o = "ò",
+ U = "Ù", u = "ù",
+ Y = "Ỳ", y = "ỳ",
+ },
+ ["c"] = { [""] = "¸",
+ C = "Ç", c = "ç",
+ K = "Ķ", k = "ķ",
+ L = "Ļ", l = "ļ",
+ N = "Ņ", n = "ņ",
+ R = "Ŗ", r = "ŗ",
+ S = "Ş", s = "ş",
+ T = "Ţ", t = "ţ",
+ },
+ ["k"] = { [""] = "˛",
+ A = "Ą", a = "ą",
+ E = "Ę", e = "ę",
+ I = "Į", i = "į",
+ U = "Ų", u = "ų",
+ },
+ ["r"] = { [""] = "˚",
+ A = "Å", a = "å",
+ U = "Ů", u = "ů",
+ },
+ ["u"] = { [""] = "˘",
+ A = "Ă", a = "ă",
+ E = "Ĕ", e = "ĕ",
+ G = "Ğ", g = "ğ",
+ I = "Ĭ", i = "ĭ", ["ı"] = "ĭ", ["\\i"] = "ĭ",
+ O = "Ŏ", o = "ŏ",
+ U = "Ŭ", u = "ŭ",
+ },
+ ["v"] = { [""] = "ˇ",
+ C = "Č", c = "č",
+ D = "Ď", d = "ď",
+ E = "Ě", e = "ě",
+ L = "Ľ", l = "ľ",
+ N = "Ň", n = "ň",
+ R = "Ř", r = "ř",
+ S = "Š", s = "š",
+ T = "Ť", t = "ť",
+ Z = "Ž", z = "ž",
+ },
+ ["~"] = { [""] = "˜",
+ A = "Ã", a = "ã",
+ I = "Ĩ", i = "ĩ", ["ı"] = "ĩ", ["\\i"] = "ĩ",
+ N = "Ñ", n = "ñ",
+ O = "Õ", o = "õ",
+ U = "Ũ", u = "ũ",
+ },
+}
+
+texcharacters.accentmapping = accentmapping
+
+local accent_map = allocate { -- incomplete
+ ['~'] = "̃" , -- ̃ Ẽ
+ ['"'] = "̈" , -- ̈ Ë
+ ["`"] = "̀" , -- ̀ È
+ ["'"] = "́" , -- ́ É
+ ["^"] = "̂" , -- ̂ Ê
+ -- ̄ Ē
+ -- ̆ Ĕ
+ -- ̇ Ė
+ -- ̉ Ẻ
+ -- ̌ Ě
+ -- ̏ Ȅ
+ -- ̑ Ȇ
+ -- ̣ Ẹ
+ -- ̧ Ȩ
+ -- ̨ Ę
+ -- ̭ Ḙ
+ -- ̰ Ḛ
+}
+
+-- local accents = concat(table.keys(accentmapping)) -- was _map
+
+local function remap_accent(a,c,braced)
+ local m = accentmapping[a]
+ if m then
+ local n = m[c]
+ if n then
+ return n
+ end
+ end
+-- local m = accent_map[a]
+-- if m then
+-- return c .. m
+-- elseif braced then -- or #c > 0
+ if braced then -- or #c > 0
+ return "\\" .. a .. "{" .. c .. "}"
+ else
+ return "\\" .. a .. " " .. c
+ end
+end
+
+local commandmapping = allocate {
+ ["aa"] = "å", ["AA"] = "Å",
+ ["ae"] = "æ", ["AE"] = "Æ",
+ ["cc"] = "ç", ["CC"] = "Ç",
+ ["i"] = "ı", ["j"] = "ȷ",
+ ["ij"] = "ij", ["IJ"] = "IJ",
+ ["l"] = "ł", ["L"] = "Ł",
+ ["o"] = "ø", ["O"] = "Ø",
+ ["oe"] = "œ", ["OE"] = "Œ",
+ ["sz"] = "ß", ["SZ"] = "SZ", ["ss"] = "ß", ["SS"] = "ß", -- uppercase: ẞ
+}
+
+texcharacters.commandmapping = commandmapping
+
+local ligaturemapping = allocate {
+ ["''"] = "”",
+ ["``"] = "“",
+ ["--"] = "–",
+ ["---"] = "—",
+}
+
+-- Older accent handling code can be found in char-def.lua but in the meantime
+-- we moved on. First the one with commands:
+
+local untex, pattern
+
+local function toutfpattern()
+ if not untex then
+ local hash = { }
+ for k, v in next, accentmapping do
+ for kk, vv in next, v do
+ if (k >= "a" and k <= "z") or (k >= "A" and k <= "Z") then
+ hash[ "\\"..k.." "..kk ] = vv
+ hash["{\\"..k.." "..kk.."}"] = vv
+ else
+ hash["\\" ..k ..kk ] = vv
+ hash["{\\"..k ..kk.."}"] = vv
+ end
+ hash["\\" ..k.."{"..kk.."}" ] = vv
+ hash["{\\"..k.."{"..kk.."}}"] = vv
+ end
+ end
+ for k, v in next, commandmapping do
+ hash["\\"..k.." "] = v
+ hash["{\\"..k.."}"] = v
+ hash["{\\"..k.." }"] = v
+ end
+ for k, v in next, ligaturemapping do
+ hash[k] = v
+ end
+ untex = utfchartabletopattern(hash) / hash
+ end
+ return untex
+end
+
+local function prepare()
+ pattern = Cs((toutfpattern() + P(1))^0)
+ return pattern
+end
+
+local function textoutf(str,strip)
+ if str == "" then
+ return str
+ elseif not find(str,"\\",1,true) then
+ return str
+ -- elseif strip then
+ else
+ return lpegmatch(pattern or prepare(),str)
+ end
+end
+
+texcharacters.toutfpattern = toutfpattern
+texcharacters.toutf = textoutf
+
+-- print(texcharacters.toutf([[\~{Z}]],true))
+-- print(texcharacters.toutf([[\'\i]],true))
+-- print(texcharacters.toutf([[\'{\i}]],true))
+-- print(texcharacters.toutf([[\"{e}]],true))
+-- print(texcharacters.toutf([[\" {e}]],true))
+-- print(texcharacters.toutf([[{\"{e}}]],true))
+-- print(texcharacters.toutf([[{\" {e}}]],true))
+-- print(texcharacters.toutf([[{\l}]],true))
+-- print(texcharacters.toutf([[{\l }]],true))
+-- print(texcharacters.toutf([[\v{r}]],true))
+-- print(texcharacters.toutf([[fo{\"o}{\ss}ar]],true))
+-- print(texcharacters.toutf([[H{\'a}n Th\^e\llap{\raise 0.5ex\hbox{\'{\relax}}} Th{\'a}nh]],true))
+
+-- Next the ones without backslash
+
+local untex, pattern
+
+local function toutfpattern()
+ if not untex then
+ local hash = { }
+ for k, v in next, accentmapping do
+ for kk, vv in next, v do
+ hash[k..kk] = vv
+ end
+ end
+ for k, v in next, commandmapping do
+ hash[k] = v
+ end
+ for k, v in next, ligaturemapping do
+ hash[k] = v
+ end
+ untex = utfchartabletopattern(hash) / hash
+ end
+ return untex
+end
+
+local function prepare()
+ pattern = Cs((toutfpattern() + P(1))^0)
+ return pattern
+end
+
+local function textoutf(str)
+ return lpegmatch(pattern or prepare(),str)
+end
+
+texcharacters.strtoutfpattern = toutfpattern
+texcharacters.strtextoutf = textoutf
+
+local collapse = utffilters.collapse
+local combine = utffilters.combine
+
+if not interfaces then return end
+
+local implement = interfaces.implement
+
+local pattern
+
+local verbosemarks = characters.verbosemarks
+
+if verbosemarks then
+
+ mark(verbosemarks)
+
+else
+
+ verbosemarks = allocate {
+ ["stroke"] = utfchar(0x02F), ["slash"] = utfchar(0x02F),
+ ["middle dot"] = utfchar(0x0B7),
+
+ ["grave"] = utfchar(0x300),
+ ["acute"] = utfchar(0x301),
+ ["circumflex"] = utfchar(0x302),
+ ["tilde"] = utfchar(0x303),
+ ["macron"] = utfchar(0x304), ["line"] = utfchar(0x304),
+ ["overline"] = utfchar(0x305),
+ ["breve"] = utfchar(0x306),
+ ["dot"] = utfchar(0x307),
+ ["dieresis"] = utfchar(0x308), ["diaeresis"] = utfchar(0x308),
+ ["hook"] = utfchar(0x309),
+ ["ring"] = utfchar(0x30A),
+ ["double acute"] = utfchar(0x30B), ["hungarumlaut"] = utfchar(0x30B), -- tex speak
+ ["caron"] = utfchar(0x30C),
+ ["vertical line"] = utfchar(0x30D),
+ ["double vertical line"] = utfchar(0x30E),
+ ["double grave"] = utfchar(0x30F),
+ ["inverted breve"] = utfchar(0x311),
+ ["dot below"] = utfchar(0x323),
+ ["ring below"] = utfchar(0x325),
+ ["cedilla"] = utfchar(0x327), ["comma below"] = utfchar(0x327),
+ ["ogonek"] = utfchar(0x328),
+ ["caron below"] = utfchar(0x32C),
+ ["circumflex below"] = utfchar(0x32D),
+ ["tilde below"] = utfchar(0x330),
+ ["macron below"] = utfchar(0x331), ["line below"] = utfchar(0x331),
+
+ ["hook below"] = utfchar(0x1FA9D),
+ }
+
+ characters.verbosemarks = verbosemarks
+
+ if storage then
+ storage.register("characters/verbosemarks", verbosemarks, "characters.verbosemarks")
+ end
+
+end
+
+local function prepare()
+ pattern = Cs((utfchartabletopattern(verbosemarks) / verbosemarks + lpegpatterns.space/"" + lpegpatterns.utf8character)^0)
+ return pattern
+end
+
+local hash = table.setmetatableindex(function(t,k)
+ local f = ""
+ k = lpegmatch(pattern or prepare(),k) or k
+ local v = collapse(k) or k -- char specials
+-- print("collapse",k,v)
+ if k ~= v then
+ goto DONE
+ end
+ v = combine(k) or k -- with specials
+-- print("combine",k,v)
+ if k ~= v then
+ goto DONE
+ end
+ v = commandmapping[k] or k
+-- print("command",k,v)
+ if k ~= v then
+ f = "\\"
+ goto DONE
+ end
+ v = textoutf(k) or k
+-- print("utf",k,v)
+ if k ~= v then
+ f = "\\"
+ goto DONE
+ end
+ ::DONE::
+ report_defining("instead of old school '%s%s' you can input the utf sequence %s",f,k,v)
+ t[k] = v
+ return v
+end)
+
+implement {
+ name = "chr",
+ arguments = "argument",
+ public = true,
+ actions = function(str)
+ context(hash[str]) -- expandable
+ end
+}
+
+function texcharacters.safechar(n) -- was characters.safechar
+ local c = data[n]
+ if c and c.contextname then
+ return "\\" .. c.contextname
+ else
+ return utfchar(n)
+ end
+end
+
+if not context or not commands then
+ -- used in e.g. mtx-bibtex
+ return
+end
+
+-- all kind of initializations
+
+local tex = tex
+local texsetlccode = tex.setlccode
+local texsetsfcode = tex.setsfcode
+local texsetcatcode = tex.setcatcode
+
+local contextsprint = context.sprint
+local ctxcatcodes = catcodes.numbers.ctxcatcodes
+
+local texsetmacro = tokens.setters.macro
+local texsetchar = tokens.setters.char
+
+-- function texcharacters.defineaccents()
+-- local ctx_dodefineaccentcommand = context.dodefineaccent
+-- local ctx_dodefineaccent = context.dodefineaccent
+-- local ctx_dodefinecommand = context.dodefinecommand
+-- for accent, group in next, accentmapping do
+-- ctx_dodefineaccentcommand(accent)
+-- for character, mapping in next, group do
+-- ctx_dodefineaccent(accent,character,mapping)
+-- end
+-- end
+-- for command, mapping in next, commandmapping do
+-- ctx_dodefinecommand(command,mapping)
+-- end
+-- os.exit()
+-- end
+
+function texcharacters.defineaccents()
+ local ctx_dodefinecombine = context.dodefinecombine
+ local ctx_dodefinecommand = context.dodefinecommand
+ for verbose, mark in next, verbosemarks do
+ ctx_dodefinecombine((gsub(verbose," ","")),mark)
+ end
+ for command, mapping in next, commandmapping do
+ ctx_dodefinecommand(command,mapping)
+ end
+end
+
+implement { -- a waste of scanner but consistent
+ name = "defineaccents",
+ actions = texcharacters.defineaccents
+}
+
+--[[ldx--
+<p>Instead of using a <l n='tex'/> file to define the named glyphs, we
+use the table. After all, we have this information available anyway.</p>
+--ldx]]--
+
+local function to_number(s)
+ local n = tonumber(s)
+ if n then
+ return n
+ end
+ return tonumber(match(s,'^"(.*)$'),16) or 0
+end
+
+implement {
+ name = "utfchar",
+ actions = { to_number, utfchar, contextsprint },
+ arguments = "string"
+}
+
+implement {
+ name = "safechar",
+ actions = { to_number, texcharacters.safechar, contextsprint },
+ arguments = "string"
+}
+
+implement {
+ name = "uchar",
+ arguments = { "integer", "integer" },
+ actions = function(h,l)
+ context(utfchar(h*256+l))
+ end
+}
+
+tex.uprint = commands.utfchar
+
+-- in contect we don't use lc and uc codes (in fact in luatex we should have a hf code)
+-- so at some point we might drop this
+
+-- The following get set at the TeX end:
+
+local forbidden = tohash {
+ 0x000A0, -- zs nobreakspace <self>
+ 0x000AD, -- cf softhyphen <self>
+ -- 0x00600, -- cf arabicnumber <self>
+ -- 0x00601, -- cf arabicsanah <self>
+ -- 0x00602, -- cf arabicfootnotemarker <self>
+ -- 0x00603, -- cf arabicsafha <self>
+ -- 0x00604, -- cf arabicsamvat <self>
+ -- 0x00605, -- cf arabicnumberabove <self>
+ -- 0x0061C, -- cf arabiclettermark <self>
+ -- 0x006DD, -- cf arabicendofayah <self>
+ -- 0x008E2, -- cf arabicdisputedendofayah <self>
+ 0x02000, -- zs enquad <self>
+ 0x02001, -- zs emquad <self>
+ 0x02002, -- zs enspace \kern .5\emwidth
+ 0x02003, -- zs emspace \hskip \emwidth
+ 0x02004, -- zs threeperemspace <self>
+ 0x02005, -- zs fourperemspace <self>
+ 0x02006, -- zs sixperemspace <self>
+ 0x02007, -- zs figurespace <self>
+ 0x02008, -- zs punctuationspace <self>
+ 0x02009, -- zs breakablethinspace <self>
+ 0x0200A, -- zs hairspace <self>
+ 0x0200B, -- cf zerowidthspace <self>
+ 0x0200C, -- cf zwnj <self>
+ 0x0200D, -- cf zwj <self>
+ 0x0202F, -- zs narrownobreakspace <self>
+ 0x0205F, -- zs medspace \textormathspace +\medmuskip 2
+ -- 0x03000, -- zs ideographicspace <self>
+ -- 0x0FEFF, -- cf zerowidthnobreakspace \penalty \plustenthousand \kern \zeropoint
+}
+
+local csletters = characters.csletters -- also a signal that we have initialized
+local activated = { }
+local sfmode = "unset" -- unset, traditional, normal
+local block_too = false
+
+directives.register("characters.blockstoo",function(v) block_too = v end)
+
+-- If this is something that is not documentwide and used a lot, then we
+-- need a more clever approach (trivial but not now).
+
+local function setuppersfcodes(v,n)
+ if sfstate ~= "unset" then
+ report_defining("setting uppercase sf codes to %a",n)
+ for u, chr in next, data do
+ if chr.category == "lu" then
+ texsetsfcode(u,n)
+ end
+ end
+ end
+ sfstate = v
+end
+
+directives.register("characters.spaceafteruppercase",function(v)
+ if v == "traditional" then
+ setuppersfcodes(v,999)
+ elseif v == "normal" then
+ setuppersfcodes(v,1000)
+ end
+end)
+
+if not csletters then
+
+ csletters = allocate()
+ characters.csletters = csletters
+
+ report_defining("setting up character related codes and commands")
+
+ if sfstate == "unset" then
+ sfstate = "traditional"
+ end
+
+ local traditional = sfstate == "traditional"
+
+ for u, chr in next, data do -- will move up
+ local contextname = chr.contextname
+ local category = chr.category
+ local isletter = is_letter[category]
+ if contextname then
+ if is_character[category] then
+ if chr.unicodeslot < 128 then
+ if isletter then
+ local c = utfchar(u)
+ csletters[c] = u
+ end
+ else
+ local c = utfchar(u)
+ if isletter and u >= 32 and u <= 65536 then
+ csletters[c] = u
+ end
+ end
+ if isletter then
+ local lc = chr.lccode
+ local uc = chr.uccode
+ if not lc then
+ chr.lccode = u
+ lc = u
+ elseif type(lc) == "table" then
+ lc = u
+ end
+ if not uc then
+ chr.uccode = u
+ uc = u
+ elseif type(uc) == "table" then
+ uc = u
+ end
+ texsetlccode(u,lc,uc)
+ if traditional and category == "lu" then
+ texsetsfcode(code,999)
+ end
+ end
+ elseif is_command[category] and not forbidden[u] then
+ -- skip
+ elseif is_mark[category] then
+ texsetlccode(u,u,u) -- for hyphenation
+ end
+ elseif isletter then
+ csletters[utfchar(u)] = u
+ local lc, uc = chr.lccode, chr.uccode
+ if not lc then
+ chr.lccode = u
+ lc = u
+ elseif type(lc) == "table" then
+ lc = u
+ end
+ if not uc then
+ chr.uccode = u
+ uc = u
+ elseif type(uc) == "table" then
+ uc = u
+ end
+ texsetlccode(u,lc,uc)
+ if traditional and category == "lu" then
+ texsetsfcode(code,999)
+ end
+ elseif is_mark[category] then
+ texsetlccode(u,u,u) -- for hyphenation
+ end
+ end
+
+ if blocks_too then
+ -- this slows down format generation by over 10 percent
+ for k, v in next, blocks do
+ if v.catcode == "letter" then
+ local first = v.first
+ local last = v.last
+ local gaps = v.gaps
+ if first and last then
+ for u=first,last do
+ csletters[utfchar(u)] = u
+ --
+ -- texsetlccode(u,u,u) -- self self
+ --
+ end
+ end
+ if gaps then
+ for i=1,#gaps do
+ local u = gaps[i]
+ csletters[utfchar(u)] = u
+ --
+ -- texsetlccode(u,u,u) -- self self
+ --
+ end
+ end
+ end
+ end
+ end
+
+ if storage then
+ storage.register("characters/csletters", csletters, "characters.csletters")
+ end
+
+ function characters.setcharacternames(ctt)
+ for u, chr in next, data do -- will move up
+ local contextname = chr.contextname
+ local category = chr.category
+ local isletter = is_letter[category]
+ if contextname then
+ if is_character[category] then
+ if chr.unicodeslot < 128 then
+ if isletter then
+ texsetmacro(contextname,utfchar(u),"immutable")
+ else
+ texsetchar(contextname,u,"immutable")
+ end
+ else
+ texsetmacro(contextname,utfchar(u),"immutable")
+ end
+ elseif is_command[category] and not forbidden[u] then
+ texsetmacro(contextname,utfchar(u),"immutable")
+ end
+ end
+ end
+ end
+
+else
+ mark(csletters)
+end
+
+lpegpatterns.csletter = utfchartabletopattern(csletters)
+
+-- The engine presets the letters to 11 (always).
+
+function characters.setlettercatcodes(cct)
+ if trace_defining then
+ report_defining("assigning letter catcodes to catcode table %a",cct)
+ end
+ local saved = tex.catcodetable
+ tex.catcodetable = cct
+ texsetcatcode(0x200C,11) -- non-joiner
+ texsetcatcode(0x200D,11) -- joiner
+ for c, u in next, csletters do
+ texsetcatcode(u,11)
+ end
+ tex.catcodetable = saved
+end
+
+function characters.setothercatcodes(cct)
+ if trace_defining then
+ report_defining("assigning other catcodes to catcode table %a",cct)
+ end
+ local saved = tex.catcodetable
+ tex.catcodetable = cct
+ for u=65,90 do
+ texsetcatcode(u,12)
+ end
+ for u=97,122 do
+ texsetcatcode(u,12)
+ end
+ tex.catcodetable = saved
+end
+
+function characters.setactivecatcodes(cct)
+ local saved = tex.catcodetable
+ tex.catcodetable = cct
+ for i=1,#activated do
+ local u = activated[i]
+ texsetcatcode(u,13)
+ if trace_defining then
+ report_defining("character %U (%s) is active in set %a",u,data[u].description,cct)
+ end
+ end
+ tex.catcodetable = saved
+end
+
+--[[ldx--
+<p>Setting the lccodes is also done in a loop over the data table.</p>
+--ldx]]--
+
+implement {
+ name = "chardescription",
+ arguments = "integer",
+ actions = function(slot)
+ local d = data[slot]
+ if d then
+ context(d.description)
+ end
+ end,
+}
+
+if characters.setcharacternames then -- only in ini mode
+
+ implement { name = "setlettercatcodes", scope = "private", actions = characters.setlettercatcodes, arguments = "integer" }
+ implement { name = "setothercatcodes", scope = "private", actions = characters.setothercatcodes, arguments = "integer" }
+ implement { name = "setactivecatcodes", scope = "private", actions = characters.setactivecatcodes, arguments = "integer" }
+ implement { name = "setcharacternames", scope = "private", actions = characters.setcharacternames, arguments = "integer" }
+
+end
+
+-- experiment (some can move to char-ini.lua)
+
+local function overload(c,u,code,codes)
+ local c = tonumber(c)
+ if not c then
+ return
+ end
+ local u = utilities.parsers.settings_to_array(u)
+ local n = #u
+ if n == 0 then
+ return
+ end
+ local t = nil
+ if n == 1 then
+ t = tonumber(u[1])
+ else
+ t = { }
+ for i=1,n do
+ t[#t+1] = tonumber(u[i])
+ end
+ end
+ if t then
+ data[c][code] = t
+ characters[codes][c] = nil
+ end
+end
+
+implement {
+ name = "overloaduppercase",
+ arguments = "2 strings",
+ actions = function(c,u)
+ overload(c,u,"uccode","uccodes")
+ end
+}
+
+implement {
+ name = "overloadlowercase",
+ arguments = "2 strings",
+ actions = function(c,u)
+ overload(c,u,"lccode","lccodes")
+ end
+}
+
+-- Just for fun we support keywords:
+--
+-- \startTEXpage[offset=10pt]
+-- abg"
+-- \sl \showboxes
+-- \accent `" h%
+-- \accent `" x%
+-- \accent yoffset .2ex `" x
+-- \accent yoffset 1.1ex `x x%
+-- \stopTEXpage
+--
+-- We could do this:
+--
+-- \startTEXpage[offset=10pt]
+-- abg"
+-- \sl \showboxes
+-- \withaccent `" h%
+-- \withaccent `" x%
+-- \withaccent yoffset .2ex `" x
+-- \withaccent yoffset 1.1ex accent `x base `x%
+-- \stopTEXpage
+--
+-- But only when users demand it:
+--
+-- do
+--
+-- local new_glyph = nodes.pool.glyph
+--
+-- local scankeyword = tokens.scanners.keyword
+-- local scaninteger = tokens.scanners.integer
+-- local scandimension = tokens.scanners.dimension
+-- local scantoken = tokens.scanners.token
+--
+-- implement {
+-- name = "withaccent",
+-- public = true,
+-- protected = true,
+-- actions = function()
+-- local xoffset = 0
+-- local yoffset = 0
+-- local accent = false
+-- local base = false
+-- local zwj = 0x200D
+-- while true do
+-- if scankeyword("xoffset") then
+-- xoffset = scandimension()
+-- elseif scankeyword("yoffset") then
+-- yoffset = scandimension()
+-- elseif scankeyword("accent") then
+-- accent = scaninteger()
+-- elseif scankeyword("base") then
+-- base = scaninteger()
+-- else
+-- break
+-- end
+-- end
+-- if not accent then
+-- accent = scaninteger()
+-- end
+-- if not base then
+-- local nxttok = scantoken()
+-- base = nxttok.cmdname == "char_number" and scaninteger() or nxttok.index
+-- end
+-- if base and accent and base > 0 and accent > 0 then
+-- base = new_glyph(true,base)
+-- zwj = new_glyph(true,zwj)
+-- accent = new_glyph(true,accent)
+-- local slant = fonts.hashes.parameters[true].slant / 65536 -- a la tex
+-- local xheight = fonts.hashes.parameters[true].xheight -- hm, compensated for glyphscale?
+-- accent.xoffset = xoffset - .5*(base.width -accent.width) + .5*(base.height-accent.height) * slant
+-- accent.yoffset = yoffset - (xheight - accent.height)
+-- accent.left = accent.width
+-- accent.options = accent.options | 0x40 | 0x80
+-- context.dontleavehmode()
+-- context(base)
+-- context(zwj)
+-- context(accent)
+-- end
+-- end,
+-- }
+--
+-- end