summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/char-ini.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/char-ini.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/char-ini.lua1074
1 files changed, 682 insertions, 392 deletions
diff --git a/Master/texmf-dist/tex/context/base/char-ini.lua b/Master/texmf-dist/tex/context/base/char-ini.lua
index 5c4a40bad51..89f8336f826 100644
--- a/Master/texmf-dist/tex/context/base/char-ini.lua
+++ b/Master/texmf-dist/tex/context/base/char-ini.lua
@@ -6,19 +6,32 @@ if not modules then modules = { } end modules ['char-ini'] = {
license = "see context related readme files"
}
-tex = tex or { }
-xml = xml or { }
+-- todo: make two files, one for format generation, one for format use
+local tex = tex
local utf = unicode.utf8
local utfchar, utfbyte, utfvalues = utf.char, utf.byte, string.utfvalues
-local concat = table.concat
-local next, tonumber = next, tonumber
+local ustring = unicode.ustring
+local concat, unpack = table.concat, table.unpack
+local next, tonumber, type, rawget, rawset = next, tonumber, type, rawget, rawset
local texsprint, texprint = tex.sprint, tex.print
local format, lower, gsub, match, gmatch = string.format, string.lower, string.gsub, string.match, string.match, string.gmatch
+local P, R, lpegmatch = lpeg.P, lpeg.R, lpeg.match
-local ctxcatcodes = tex.ctxcatcodes
-local texcatcodes = tex.texcatcodes
+local allocate = utilities.storage.allocate
+local mark = utilities.storage.mark
+local texsetlccode = tex.setlccode
+local texsetuccode = tex.setuccode
+local texsetsfcode = tex.setsfcode
+local texsetcatcode = tex.setcatcode
+local ctxcatcodes = tex.ctxcatcodes
+local texcatcodes = tex.texcatcodes
+local setmetatableindex = table.setmetatableindex
+
+local trace_defining = false trackers.register("characters.defining", function(v) characters_defining = v end)
+
+local report_defining = logs.reporter("characters")
--[[ldx--
<p>This module implements some methods and creates additional datastructured
@@ -29,215 +42,279 @@ from the big character table that we use for all kind of purposes:
loaded!</p>
--ldx]]--
-characters = characters or { }
-characters.data = characters.data or { }
+characters = characters or { }
+local characters = characters
-local data = characters.data
+local data = characters.data
-if not characters.ranges then
- characters.ranges = { }
- for k, v in next, data do
- characters.ranges[#characters.ranges+1] = k
+if data then
+ mark(data) -- why does this fail
+else
+ report_defining("fatal error: 'char-def.lua' is not loaded")
+ os.exit()
+end
+
+--[[ldx--
+<p>This converts a string (if given) into a number.</p>
+--ldx]]--
+
+local pattern = (P("0x") + P("U+")) * ((R("09","AF")^1 * P(-1)) / function(s) return tonumber(s,16) end)
+
+lpeg.patterns.chartonumber = pattern
+
+local function chartonumber(k)
+ if type(k) == "string" then
+ local u = lpegmatch(pattern,k)
+ if u then
+ return utfbyte(u)
+ else
+ return utfbyte(k) or 0
+ end
+ else
+ return k or 0
end
end
-storage.register("characters/ranges",characters.ranges,"characters.ranges")
+local function charfromnumber(k)
+ if type(k) == "number" then
+ return utfchar(k) or ""
+ else
+ local u = lpegmatch(pattern,k)
+ if u then
+ return utfchar(u)
+ else
+ return k
+ end
+ end
+end
-local ranges = characters.ranges
+--~ print(chartonumber(97), chartonumber("a"), chartonumber("0x61"), chartonumber("U+61"))
-setmetatable(data, {
- __index = function(t,k)
+characters.tonumber = chartonumber
+characters.fromnumber = charfromnumber
+
+local private = {
+ description = "PRIVATE SLOT",
+}
+
+local ranges = allocate()
+characters.ranges = ranges
+
+setmetatableindex(data, function(t,k)
+ local tk = type(k)
+ if tk == "string" then
+ k = lpegmatch(pattern,k) or utfbyte(k)
+ if k then
+ local tk = rawget(t,k)
+ if tk then
+ return tk
+ else
+ -- goes to ranges
+ end
+ else
+ return private
+ end
+ end
+ if tk == "number" and k < 0xF0000 then
for r=1,#ranges do
- local rr = ranges[r] -- first in range
- if k > rr and k <= data[rr].range then
- t[k] = t[rr]
- return t[k]
+ local rr = ranges[r]
+ if k >= rr.first and k <= rr.last then
+ local extender = rr.extender
+ if extender then
+ v = extender(k,v)
+ end
+ t[k] = v
+ return v
end
end
- return nil
end
-})
-
-characters.blocks = {
- ["aegeannumbers"] = { 0x10100, 0x1013F, "Aegean Numbers" },
- ["alphabeticpresentationforms"] = { 0x0FB00, 0x0FB4F, "Alphabetic Presentation Forms" },
- ["ancientgreekmusicalnotation"] = { 0x1D200, 0x1D24F, "Ancient Greek Musical Notation" },
- ["ancientgreeknumbers"] = { 0x10140, 0x1018F, "Ancient Greek Numbers" },
- ["ancientsymbols"] = { 0x10190, 0x101CF, "Ancient Symbols" },
- ["arabic"] = { 0x00600, 0x006FF, "Arabic" },
- ["arabicpresentationformsa"] = { 0x0FB50, 0x0FDFF, "Arabic Presentation Forms-A" },
- ["arabicpresentationformsb"] = { 0x0FE70, 0x0FEFF, "Arabic Presentation Forms-B" },
- ["arabicsupplement"] = { 0x00750, 0x0077F, "Arabic Supplement" },
- ["armenian"] = { 0x00530, 0x0058F, "Armenian" },
- ["arrows"] = { 0x02190, 0x021FF, "Arrows" },
- ["balinese"] = { 0x01B00, 0x01B7F, "Balinese" },
- ["basiclatin"] = { 0x00000, 0x0007F, "Basic Latin" },
- ["bengali"] = { 0x00980, 0x009FF, "Bengali" },
- ["blockelements"] = { 0x02580, 0x0259F, "Block Elements" },
- ["bopomofo"] = { 0x03100, 0x0312F, "Bopomofo" },
- ["bopomofoextended"] = { 0x031A0, 0x031BF, "Bopomofo Extended" },
- ["boxdrawing"] = { 0x02500, 0x0257F, "Box Drawing" },
- ["braillepatterns"] = { 0x02800, 0x028FF, "Braille Patterns" },
- ["buginese"] = { 0x01A00, 0x01A1F, "Buginese" },
- ["buhid"] = { 0x01740, 0x0175F, "Buhid" },
- ["byzantinemusicalsymbols"] = { 0x1D000, 0x1D0FF, "Byzantine Musical Symbols" },
- ["carian"] = { 0x102A0, 0x102DF, "Carian" },
- ["cham"] = { 0x0AA00, 0x0AA5F, "Cham" },
- ["cherokee"] = { 0x013A0, 0x013FF, "Cherokee" },
- ["cjkcompatibility"] = { 0x03300, 0x033FF, "CJK Compatibility" },
- ["cjkcompatibilityforms"] = { 0x0FE30, 0x0FE4F, "CJK Compatibility Forms" },
- ["cjkcompatibilityideographs"] = { 0x0F900, 0x0FAFF, "CJK Compatibility Ideographs" },
- ["cjkcompatibilityideographssupplement"] = { 0x2F800, 0x2FA1F, "CJK Compatibility Ideographs Supplement" },
- ["cjkradicalssupplement"] = { 0x02E80, 0x02EFF, "CJK Radicals Supplement" },
- ["cjkstrokes"] = { 0x031C0, 0x031EF, "CJK Strokes" },
- ["cjksymbolsandpunctuation"] = { 0x03000, 0x0303F, "CJK Symbols and Punctuation" },
- ["cjkunifiedideographs"] = { 0x04E00, 0x09FFF, "CJK Unified Ideographs" },
- ["cjkunifiedideographsextensiona"] = { 0x03400, 0x04DBF, "CJK Unified Ideographs Extension A" },
- ["cjkunifiedideographsextensionb"] = { 0x20000, 0x2A6DF, "CJK Unified Ideographs Extension B" },
- ["combiningdiacriticalmarks"] = { 0x00300, 0x0036F, "Combining Diacritical Marks" },
- ["combiningdiacriticalmarksforsymbols"] = { 0x020D0, 0x020FF, "Combining Diacritical Marks for Symbols" },
- ["combiningdiacriticalmarkssupplement"] = { 0x01DC0, 0x01DFF, "Combining Diacritical Marks Supplement" },
- ["combininghalfmarks"] = { 0x0FE20, 0x0FE2F, "Combining Half Marks" },
- ["controlpictures"] = { 0x02400, 0x0243F, "Control Pictures" },
- ["coptic"] = { 0x02C80, 0x02CFF, "Coptic" },
- ["countingrodnumerals"] = { 0x1D360, 0x1D37F, "Counting Rod Numerals" },
- ["cuneiform"] = { 0x12000, 0x123FF, "Cuneiform" },
- ["cuneiformnumbersandpunctuation"] = { 0x12400, 0x1247F, "Cuneiform Numbers and Punctuation" },
- ["currencysymbols"] = { 0x020A0, 0x020CF, "Currency Symbols" },
- ["cypriotsyllabary"] = { 0x10800, 0x1083F, "Cypriot Syllabary" },
- ["cyrillic"] = { 0x00400, 0x004FF, "Cyrillic" },
- ["cyrillicextendeda"] = { 0x02DE0, 0x02DFF, "Cyrillic Extended-A" },
- ["cyrillicextendedb"] = { 0x0A640, 0x0A69F, "Cyrillic Extended-B" },
- ["cyrillicsupplement"] = { 0x00500, 0x0052F, "Cyrillic Supplement" },
- ["deseret"] = { 0x10400, 0x1044F, "Deseret" },
- ["devanagari"] = { 0x00900, 0x0097F, "Devanagari" },
- ["dingbats"] = { 0x02700, 0x027BF, "Dingbats" },
- ["dominotiles"] = { 0x1F030, 0x1F09F, "Domino Tiles" },
- ["enclosedalphanumerics"] = { 0x02460, 0x024FF, "Enclosed Alphanumerics" },
- ["enclosedcjklettersandmonths"] = { 0x03200, 0x032FF, "Enclosed CJK Letters and Months" },
- ["ethiopic"] = { 0x01200, 0x0137F, "Ethiopic" },
- ["ethiopicextended"] = { 0x02D80, 0x02DDF, "Ethiopic Extended" },
- ["ethiopicsupplement"] = { 0x01380, 0x0139F, "Ethiopic Supplement" },
- ["generalpunctuation"] = { 0x02000, 0x0206F, "General Punctuation" },
- ["geometricshapes"] = { 0x025A0, 0x025FF, "Geometric Shapes" },
- ["georgian"] = { 0x010A0, 0x010FF, "Georgian" },
- ["georgiansupplement"] = { 0x02D00, 0x02D2F, "Georgian Supplement" },
- ["glagolitic"] = { 0x02C00, 0x02C5F, "Glagolitic" },
- ["gothic"] = { 0x10330, 0x1034F, "Gothic" },
- ["greekandcoptic"] = { 0x00370, 0x003FF, "Greek and Coptic" },
- ["greekextended"] = { 0x01F00, 0x01FFF, "Greek Extended" },
- ["gujarati"] = { 0x00A80, 0x00AFF, "Gujarati" },
- ["gurmukhi"] = { 0x00A00, 0x00A7F, "Gurmukhi" },
- ["halfwidthandfullwidthforms"] = { 0x0FF00, 0x0FFEF, "Halfwidth and Fullwidth Forms" },
- ["hangulcompatibilityjamo"] = { 0x03130, 0x0318F, "Hangul Compatibility Jamo" },
- ["hanguljamo"] = { 0x01100, 0x011FF, "Hangul Jamo" },
- ["hangulsyllables"] = { 0x0AC00, 0x0D7AF, "Hangul Syllables" },
- ["hanunoo"] = { 0x01720, 0x0173F, "Hanunoo" },
- ["hebrew"] = { 0x00590, 0x005FF, "Hebrew" },
- ["highprivateusesurrogates"] = { 0x0DB80, 0x0DBFF, "High Private Use Surrogates" },
- ["highsurrogates"] = { 0x0D800, 0x0DB7F, "High Surrogates" },
- ["hiragana"] = { 0x03040, 0x0309F, "Hiragana" },
- ["ideographicdescriptioncharacters"] = { 0x02FF0, 0x02FFF, "Ideographic Description Characters" },
- ["ipaextensions"] = { 0x00250, 0x02AF, "IPA Extensions" },
- ["kanbun"] = { 0x03190, 0x0319F, "Kanbun" },
- ["kangxiradicals"] = { 0x02F00, 0x02FDF, "Kangxi Radicals" },
- ["kannada"] = { 0x00C80, 0x00CFF, "Kannada" },
- ["katakana"] = { 0x030A0, 0x030FF, "Katakana" },
- ["katakanaphoneticextensions"] = { 0x031F0, 0x031FF, "Katakana Phonetic Extensions" },
- ["kayahli"] = { 0x0A900, 0x0A92F, "Kayah Li" },
- ["kharoshthi"] = { 0x10A00, 0x10A5F, "Kharoshthi" },
- ["khmer"] = { 0x01780, 0x017FF, "Khmer" },
- ["khmersymbols"] = { 0x019E0, 0x019FF, "Khmer Symbols" },
- ["lao"] = { 0x00E80, 0x00EFF, "Lao" },
- ["latinextendeda"] = { 0x00100, 0x0017F, "Latin Extended-A" },
- ["latinextendedadditional"] = { 0x01E00, 0x01EFF, "Latin Extended Additional" },
- ["latinextendedb"] = { 0x00180, 0x0024F, "Latin Extended-B" },
- ["latinextendedc"] = { 0x02C60, 0x02C7F, "Latin Extended-C" },
- ["latinextendedd"] = { 0x0A720, 0x0A7FF, "Latin Extended-D" },
- ["latinsupplement"] = { 0x00080, 0x000FF, "Latin-1 Supplement" },
- ["lepcha"] = { 0x01C00, 0x01C4F, "Lepcha" },
- ["letterlikesymbols"] = { 0x02100, 0x0214F, "Letterlike Symbols" },
- ["limbu"] = { 0x01900, 0x0194F, "Limbu" },
- ["linearbideograms"] = { 0x10080, 0x100FF, "Linear B Ideograms" },
- ["linearbsyllabary"] = { 0x10000, 0x1007F, "Linear B Syllabary" },
- ["lowsurrogates"] = { 0x0DC00, 0x0DFFF, "Low Surrogates" },
- ["lycian"] = { 0x10280, 0x1029F, "Lycian" },
- ["lydian"] = { 0x10920, 0x1093F, "Lydian" },
- ["mahjongtiles"] = { 0x1F000, 0x1F02F, "Mahjong Tiles" },
- ["malayalam"] = { 0x00D00, 0x00D7F, "Malayalam" },
- ["mathematicalalphanumericsymbols"] = { 0x1D400, 0x1D7FF, "Mathematical Alphanumeric Symbols" },
- ["mathematicaloperators"] = { 0x02200, 0x022FF, "Mathematical Operators" },
- ["miscellaneousmathematicalsymbolsa"] = { 0x027C0, 0x027EF, "Miscellaneous Mathematical Symbols-A" },
- ["miscellaneousmathematicalsymbolsb"] = { 0x02980, 0x029FF, "Miscellaneous Mathematical Symbols-B" },
- ["miscellaneoussymbols"] = { 0x02600, 0x026FF, "Miscellaneous Symbols" },
- ["miscellaneoussymbolsandarrows"] = { 0x02B00, 0x02BFF, "Miscellaneous Symbols and Arrows" },
- ["miscellaneoustechnical"] = { 0x02300, 0x023FF, "Miscellaneous Technical" },
- ["modifiertoneletters"] = { 0x0A700, 0x0A71F, "Modifier Tone Letters" },
- ["mongolian"] = { 0x01800, 0x018AF, "Mongolian" },
- ["musicalsymbols"] = { 0x1D100, 0x1D1FF, "Musical Symbols" },
- ["myanmar"] = { 0x01000, 0x0109F, "Myanmar" },
- ["newtailue"] = { 0x01980, 0x019DF, "New Tai Lue" },
- ["nko"] = { 0x007C0, 0x007FF, "NKo" },
- ["numberforms"] = { 0x02150, 0x0218F, "Number Forms" },
- ["ogham"] = { 0x01680, 0x0169F, "Ogham" },
- ["olchiki"] = { 0x01C50, 0x01C7F, "Ol Chiki" },
- ["olditalic"] = { 0x10300, 0x1032F, "Old Italic" },
- ["oldpersian"] = { 0x103A0, 0x103DF, "Old Persian" },
- ["opticalcharacterrecognition"] = { 0x02440, 0x0245F, "Optical Character Recognition" },
- ["oriya"] = { 0x00B00, 0x00B7F, "Oriya" },
- ["osmanya"] = { 0x10480, 0x104AF, "Osmanya" },
- ["phagspa"] = { 0x0A840, 0x0A87F, "Phags-pa" },
- ["phaistosdisc"] = { 0x101D0, 0x101FF, "Phaistos Disc" },
- ["phoenician"] = { 0x10900, 0x1091F, "Phoenician" },
- ["phoneticextensions"] = { 0x01D00, 0x01D7F, "Phonetic Extensions" },
- ["phoneticextensionssupplement"] = { 0x01D80, 0x01DBF, "Phonetic Extensions Supplement" },
- ["privateusearea"] = { 0x0E000, 0x0F8FF, "Private Use Area" },
- ["rejang"] = { 0x0A930, 0x0A95F, "Rejang" },
- ["runic"] = { 0x016A0, 0x016FF, "Runic" },
- ["saurashtra"] = { 0x0A880, 0x0A8DF, "Saurashtra" },
- ["shavian"] = { 0x10450, 0x1047F, "Shavian" },
- ["sinhala"] = { 0x00D80, 0x00DFF, "Sinhala" },
- ["smallformvariants"] = { 0x0FE50, 0x0FE6F, "Small Form Variants" },
- ["spacingmodifierletters"] = { 0x002B0, 0x002FF, "Spacing Modifier Letters" },
- ["specials"] = { 0x0FFF0, 0x0FFFF, "Specials" },
- ["sundanese"] = { 0x01B80, 0x01BBF, "Sundanese" },
- ["superscriptsandsubscripts"] = { 0x02070, 0x0209F, "Superscripts and Subscripts" },
- ["supplementalarrowsa"] = { 0x027F0, 0x027FF, "Supplemental Arrows-A" },
- ["supplementalarrowsb"] = { 0x02900, 0x0297F, "Supplemental Arrows-B" },
- ["supplementalmathematicaloperators"] = { 0x02A00, 0x02AFF, "Supplemental Mathematical Operators" },
- ["supplementalpunctuation"] = { 0x02E00, 0x02E7F, "Supplemental Punctuation" },
- ["supplementaryprivateuseareaa"] = { 0xF0000, 0xFFFFF, "Supplementary Private Use Area-A" },
- ["supplementaryprivateuseareab"] = { 0x100000,0x10FFFF,"Supplementary Private Use Area-B" },
- ["sylotinagri"] = { 0x0A800, 0x0A82F, "Syloti Nagri" },
- ["syriac"] = { 0x00700, 0x0074F, "Syriac" },
- ["tagalog"] = { 0x01700, 0x0171F, "Tagalog" },
- ["tagbanwa"] = { 0x01760, 0x0177F, "Tagbanwa" },
- ["tags"] = { 0xE0000, 0xE007F, "Tags" },
- ["taile"] = { 0x01950, 0x0197F, "Tai Le" },
- ["taixuanjingsymbols"] = { 0x1D300, 0x1D35F, "Tai Xuan Jing Symbols" },
- ["tamil"] = { 0x00B80, 0x00BFF, "Tamil" },
- ["telugu"] = { 0x00C00, 0x00C7F, "Telugu" },
- ["thaana"] = { 0x00780, 0x007BF, "Thaana" },
- ["thai"] = { 0x00E00, 0x00E7F, "Thai" },
- ["tibetan"] = { 0x00F00, 0x00FFF, "Tibetan" },
- ["tifinagh"] = { 0x02D30, 0x02D7F, "Tifinagh" },
- ["ugaritic"] = { 0x10380, 0x1039F, "Ugaritic" },
- ["unifiedcanadianaboriginalsyllabics"] = { 0x01400, 0x0167F, "Unified Canadian Aboriginal Syllabics" },
- ["vai"] = { 0x0A500, 0x0A63F, "Vai" },
- ["variationselectors"] = { 0x0FE00, 0x0FE0F, "Variation Selectors" },
- ["variationselectorssupplement"] = { 0xE0100, 0xE01EF, "Variation Selectors Supplement" },
- ["verticalforms"] = { 0x0FE10, 0x0FE1F, "Vertical Forms" },
- ["yijinghexagramsymbols"] = { 0x04DC0, 0x04DFF, "Yijing Hexagram Symbols" },
- ["yiradicals"] = { 0x0A490, 0x0A4CF, "Yi Radicals" },
- ["yisyllables"] = { 0x0A000, 0x0A48F, "Yi Syllables" },
+ return private -- handy for when we loop over characters in fonts and check for a property
+end)
+
+local blocks = allocate {
+ ["aegeannumbers"] = { first = 0x10100, last = 0x1013F, description = "Aegean Numbers" },
+ ["alphabeticpresentationforms"] = { first = 0x0FB00, last = 0x0FB4F, description = "Alphabetic Presentation Forms" },
+ ["ancientgreekmusicalnotation"] = { first = 0x1D200, last = 0x1D24F, description = "Ancient Greek Musical Notation" },
+ ["ancientgreeknumbers"] = { first = 0x10140, last = 0x1018F, description = "Ancient Greek Numbers" },
+ ["ancientsymbols"] = { first = 0x10190, last = 0x101CF, description = "Ancient Symbols" },
+ ["arabic"] = { first = 0x00600, last = 0x006FF, description = "Arabic" },
+ ["arabicpresentationformsa"] = { first = 0x0FB50, last = 0x0FDFF, description = "Arabic Presentation Forms-A" },
+ ["arabicpresentationformsb"] = { first = 0x0FE70, last = 0x0FEFF, description = "Arabic Presentation Forms-B" },
+ ["arabicsupplement"] = { first = 0x00750, last = 0x0077F, description = "Arabic Supplement" },
+ ["armenian"] = { first = 0x00530, last = 0x0058F, description = "Armenian" },
+ ["arrows"] = { first = 0x02190, last = 0x021FF, description = "Arrows" },
+ ["balinese"] = { first = 0x01B00, last = 0x01B7F, description = "Balinese" },
+ ["basiclatin"] = { first = 0x00000, last = 0x0007F, description = "Basic Latin" },
+ ["bengali"] = { first = 0x00980, last = 0x009FF, description = "Bengali" },
+ ["blockelements"] = { first = 0x02580, last = 0x0259F, description = "Block Elements" },
+ ["bopomofo"] = { first = 0x03100, last = 0x0312F, description = "Bopomofo" },
+ ["bopomofoextended"] = { first = 0x031A0, last = 0x031BF, description = "Bopomofo Extended" },
+ ["boxdrawing"] = { first = 0x02500, last = 0x0257F, description = "Box Drawing" },
+ ["braillepatterns"] = { first = 0x02800, last = 0x028FF, description = "Braille Patterns" },
+ ["buginese"] = { first = 0x01A00, last = 0x01A1F, description = "Buginese" },
+ ["buhid"] = { first = 0x01740, last = 0x0175F, description = "Buhid" },
+ ["byzantinemusicalsymbols"] = { first = 0x1D000, last = 0x1D0FF, description = "Byzantine Musical Symbols" },
+ ["carian"] = { first = 0x102A0, last = 0x102DF, description = "Carian" },
+ ["cham"] = { first = 0x0AA00, last = 0x0AA5F, description = "Cham" },
+ ["cherokee"] = { first = 0x013A0, last = 0x013FF, description = "Cherokee" },
+ ["cjkcompatibility"] = { first = 0x03300, last = 0x033FF, description = "CJK Compatibility" },
+ ["cjkcompatibilityforms"] = { first = 0x0FE30, last = 0x0FE4F, description = "CJK Compatibility Forms" },
+ ["cjkcompatibilityideographs"] = { first = 0x0F900, last = 0x0FAFF, description = "CJK Compatibility Ideographs" },
+ ["cjkcompatibilityideographssupplement"] = { first = 0x2F800, last = 0x2FA1F, description = "CJK Compatibility Ideographs Supplement" },
+ ["cjkradicalssupplement"] = { first = 0x02E80, last = 0x02EFF, description = "CJK Radicals Supplement" },
+ ["cjkstrokes"] = { first = 0x031C0, last = 0x031EF, description = "CJK Strokes" },
+ ["cjksymbolsandpunctuation"] = { first = 0x03000, last = 0x0303F, description = "CJK Symbols and Punctuation" },
+ ["cjkunifiedideographs"] = { first = 0x04E00, last = 0x09FFF, description = "CJK Unified Ideographs" },
+ ["cjkunifiedideographsextensiona"] = { first = 0x03400, last = 0x04DBF, description = "CJK Unified Ideographs Extension A" },
+ ["cjkunifiedideographsextensionb"] = { first = 0x20000, last = 0x2A6DF, description = "CJK Unified Ideographs Extension B" },
+ ["combiningdiacriticalmarks"] = { first = 0x00300, last = 0x0036F, description = "Combining Diacritical Marks" },
+ ["combiningdiacriticalmarksforsymbols"] = { first = 0x020D0, last = 0x020FF, description = "Combining Diacritical Marks for Symbols" },
+ ["combiningdiacriticalmarkssupplement"] = { first = 0x01DC0, last = 0x01DFF, description = "Combining Diacritical Marks Supplement" },
+ ["combininghalfmarks"] = { first = 0x0FE20, last = 0x0FE2F, description = "Combining Half Marks" },
+ ["controlpictures"] = { first = 0x02400, last = 0x0243F, description = "Control Pictures" },
+ ["coptic"] = { first = 0x02C80, last = 0x02CFF, description = "Coptic" },
+ ["countingrodnumerals"] = { first = 0x1D360, last = 0x1D37F, description = "Counting Rod Numerals" },
+ ["cuneiform"] = { first = 0x12000, last = 0x123FF, description = "Cuneiform" },
+ ["cuneiformnumbersandpunctuation"] = { first = 0x12400, last = 0x1247F, description = "Cuneiform Numbers and Punctuation" },
+ ["currencysymbols"] = { first = 0x020A0, last = 0x020CF, description = "Currency Symbols" },
+ ["cypriotsyllabary"] = { first = 0x10800, last = 0x1083F, description = "Cypriot Syllabary" },
+ ["cyrillic"] = { first = 0x00400, last = 0x004FF, description = "Cyrillic" },
+ ["cyrillicextendeda"] = { first = 0x02DE0, last = 0x02DFF, description = "Cyrillic Extended-A" },
+ ["cyrillicextendedb"] = { first = 0x0A640, last = 0x0A69F, description = "Cyrillic Extended-B" },
+ ["cyrillicsupplement"] = { first = 0x00500, last = 0x0052F, description = "Cyrillic Supplement" },
+ ["deseret"] = { first = 0x10400, last = 0x1044F, description = "Deseret" },
+ ["devanagari"] = { first = 0x00900, last = 0x0097F, description = "Devanagari" },
+ ["dingbats"] = { first = 0x02700, last = 0x027BF, description = "Dingbats" },
+ ["dominotiles"] = { first = 0x1F030, last = 0x1F09F, description = "Domino Tiles" },
+ ["enclosedalphanumerics"] = { first = 0x02460, last = 0x024FF, description = "Enclosed Alphanumerics" },
+ ["enclosedcjklettersandmonths"] = { first = 0x03200, last = 0x032FF, description = "Enclosed CJK Letters and Months" },
+ ["ethiopic"] = { first = 0x01200, last = 0x0137F, description = "Ethiopic" },
+ ["ethiopicextended"] = { first = 0x02D80, last = 0x02DDF, description = "Ethiopic Extended" },
+ ["ethiopicsupplement"] = { first = 0x01380, last = 0x0139F, description = "Ethiopic Supplement" },
+ ["generalpunctuation"] = { first = 0x02000, last = 0x0206F, description = "General Punctuation" },
+ ["geometricshapes"] = { first = 0x025A0, last = 0x025FF, description = "Geometric Shapes" },
+ ["georgian"] = { first = 0x010A0, last = 0x010FF, description = "Georgian" },
+ ["georgiansupplement"] = { first = 0x02D00, last = 0x02D2F, description = "Georgian Supplement" },
+ ["glagolitic"] = { first = 0x02C00, last = 0x02C5F, description = "Glagolitic" },
+ ["gothic"] = { first = 0x10330, last = 0x1034F, description = "Gothic" },
+ ["greekandcoptic"] = { first = 0x00370, last = 0x003FF, description = "Greek and Coptic" },
+ ["greekextended"] = { first = 0x01F00, last = 0x01FFF, description = "Greek Extended" },
+ ["gujarati"] = { first = 0x00A80, last = 0x00AFF, description = "Gujarati" },
+ ["gurmukhi"] = { first = 0x00A00, last = 0x00A7F, description = "Gurmukhi" },
+ ["halfwidthandfullwidthforms"] = { first = 0x0FF00, last = 0x0FFEF, description = "Halfwidth and Fullwidth Forms" },
+ ["hangulcompatibilityjamo"] = { first = 0x03130, last = 0x0318F, description = "Hangul Compatibility Jamo" },
+ ["hanguljamo"] = { first = 0x01100, last = 0x011FF, description = "Hangul Jamo" },
+ ["hangulsyllables"] = { first = 0x0AC00, last = 0x0D7AF, description = "Hangul Syllables" },
+ ["hanunoo"] = { first = 0x01720, last = 0x0173F, description = "Hanunoo" },
+ ["hebrew"] = { first = 0x00590, last = 0x005FF, description = "Hebrew" },
+ ["highprivateusesurrogates"] = { first = 0x0DB80, last = 0x0DBFF, description = "High Private Use Surrogates" },
+ ["highsurrogates"] = { first = 0x0D800, last = 0x0DB7F, description = "High Surrogates" },
+ ["hiragana"] = { first = 0x03040, last = 0x0309F, description = "Hiragana" },
+ ["ideographicdescriptioncharacters"] = { first = 0x02FF0, last = 0x02FFF, description = "Ideographic Description Characters" },
+ ["ipaextensions"] = { first = 0x00250, last = 0x002AF, description = "IPA Extensions" },
+ ["kanbun"] = { first = 0x03190, last = 0x0319F, description = "Kanbun" },
+ ["kangxiradicals"] = { first = 0x02F00, last = 0x02FDF, description = "Kangxi Radicals" },
+ ["kannada"] = { first = 0x00C80, last = 0x00CFF, description = "Kannada" },
+ ["katakana"] = { first = 0x030A0, last = 0x030FF, description = "Katakana" },
+ ["katakanaphoneticextensions"] = { first = 0x031F0, last = 0x031FF, description = "Katakana Phonetic Extensions" },
+ ["kayahli"] = { first = 0x0A900, last = 0x0A92F, description = "Kayah Li" },
+ ["kharoshthi"] = { first = 0x10A00, last = 0x10A5F, description = "Kharoshthi" },
+ ["khmer"] = { first = 0x01780, last = 0x017FF, description = "Khmer" },
+ ["khmersymbols"] = { first = 0x019E0, last = 0x019FF, description = "Khmer Symbols" },
+ ["lao"] = { first = 0x00E80, last = 0x00EFF, description = "Lao" },
+ ["latinextendeda"] = { first = 0x00100, last = 0x0017F, description = "Latin Extended-A" },
+ ["latinextendedadditional"] = { first = 0x01E00, last = 0x01EFF, description = "Latin Extended Additional" },
+ ["latinextendedb"] = { first = 0x00180, last = 0x0024F, description = "Latin Extended-B" },
+ ["latinextendedc"] = { first = 0x02C60, last = 0x02C7F, description = "Latin Extended-C" },
+ ["latinextendedd"] = { first = 0x0A720, last = 0x0A7FF, description = "Latin Extended-D" },
+ ["latinsupplement"] = { first = 0x00080, last = 0x000FF, description = "Latin-1 Supplement" },
+ ["lepcha"] = { first = 0x01C00, last = 0x01C4F, description = "Lepcha" },
+ ["letterlikesymbols"] = { first = 0x02100, last = 0x0214F, description = "Letterlike Symbols" },
+ ["limbu"] = { first = 0x01900, last = 0x0194F, description = "Limbu" },
+ ["linearbideograms"] = { first = 0x10080, last = 0x100FF, description = "Linear B Ideograms" },
+ ["linearbsyllabary"] = { first = 0x10000, last = 0x1007F, description = "Linear B Syllabary" },
+ ["lowsurrogates"] = { first = 0x0DC00, last = 0x0DFFF, description = "Low Surrogates" },
+ ["lycian"] = { first = 0x10280, last = 0x1029F, description = "Lycian" },
+ ["lydian"] = { first = 0x10920, last = 0x1093F, description = "Lydian" },
+ ["mahjongtiles"] = { first = 0x1F000, last = 0x1F02F, description = "Mahjong Tiles" },
+ ["malayalam"] = { first = 0x00D00, last = 0x00D7F, description = "Malayalam" },
+ ["mathematicalalphanumericsymbols"] = { first = 0x1D400, last = 0x1D7FF, description = "Mathematical Alphanumeric Symbols" },
+ ["mathematicaloperators"] = { first = 0x02200, last = 0x022FF, description = "Mathematical Operators" },
+ ["miscellaneousmathematicalsymbolsa"] = { first = 0x027C0, last = 0x027EF, description = "Miscellaneous Mathematical Symbols-A" },
+ ["miscellaneousmathematicalsymbolsb"] = { first = 0x02980, last = 0x029FF, description = "Miscellaneous Mathematical Symbols-B" },
+ ["miscellaneoussymbols"] = { first = 0x02600, last = 0x026FF, description = "Miscellaneous Symbols" },
+ ["miscellaneoussymbolsandarrows"] = { first = 0x02B00, last = 0x02BFF, description = "Miscellaneous Symbols and Arrows" },
+ ["miscellaneoustechnical"] = { first = 0x02300, last = 0x023FF, description = "Miscellaneous Technical" },
+ ["modifiertoneletters"] = { first = 0x0A700, last = 0x0A71F, description = "Modifier Tone Letters" },
+ ["mongolian"] = { first = 0x01800, last = 0x018AF, description = "Mongolian" },
+ ["musicalsymbols"] = { first = 0x1D100, last = 0x1D1FF, description = "Musical Symbols" },
+ ["myanmar"] = { first = 0x01000, last = 0x0109F, description = "Myanmar" },
+ ["newtailue"] = { first = 0x01980, last = 0x019DF, description = "New Tai Lue" },
+ ["nko"] = { first = 0x007C0, last = 0x007FF, description = "NKo" },
+ ["numberforms"] = { first = 0x02150, last = 0x0218F, description = "Number Forms" },
+ ["ogham"] = { first = 0x01680, last = 0x0169F, description = "Ogham" },
+ ["olchiki"] = { first = 0x01C50, last = 0x01C7F, description = "Ol Chiki" },
+ ["olditalic"] = { first = 0x10300, last = 0x1032F, description = "Old Italic" },
+ ["oldpersian"] = { first = 0x103A0, last = 0x103DF, description = "Old Persian" },
+ ["opticalcharacterrecognition"] = { first = 0x02440, last = 0x0245F, description = "Optical Character Recognition" },
+ ["oriya"] = { first = 0x00B00, last = 0x00B7F, description = "Oriya" },
+ ["osmanya"] = { first = 0x10480, last = 0x104AF, description = "Osmanya" },
+ ["phagspa"] = { first = 0x0A840, last = 0x0A87F, description = "Phags-pa" },
+ ["phaistosdisc"] = { first = 0x101D0, last = 0x101FF, description = "Phaistos Disc" },
+ ["phoenician"] = { first = 0x10900, last = 0x1091F, description = "Phoenician" },
+ ["phoneticextensions"] = { first = 0x01D00, last = 0x01D7F, description = "Phonetic Extensions" },
+ ["phoneticextensionssupplement"] = { first = 0x01D80, last = 0x01DBF, description = "Phonetic Extensions Supplement" },
+ ["privateusearea"] = { first = 0x0E000, last = 0x0F8FF, description = "Private Use Area" },
+ ["rejang"] = { first = 0x0A930, last = 0x0A95F, description = "Rejang" },
+ ["runic"] = { first = 0x016A0, last = 0x016FF, description = "Runic" },
+ ["saurashtra"] = { first = 0x0A880, last = 0x0A8DF, description = "Saurashtra" },
+ ["shavian"] = { first = 0x10450, last = 0x1047F, description = "Shavian" },
+ ["sinhala"] = { first = 0x00D80, last = 0x00DFF, description = "Sinhala" },
+ ["smallformvariants"] = { first = 0x0FE50, last = 0x0FE6F, description = "Small Form Variants" },
+ ["spacingmodifierletters"] = { first = 0x002B0, last = 0x002FF, description = "Spacing Modifier Letters" },
+ ["specials"] = { first = 0x0FFF0, last = 0x0FFFF, description = "Specials" },
+ ["sundanese"] = { first = 0x01B80, last = 0x01BBF, description = "Sundanese" },
+ ["superscriptsandsubscripts"] = { first = 0x02070, last = 0x0209F, description = "Superscripts and Subscripts" },
+ ["supplementalarrowsa"] = { first = 0x027F0, last = 0x027FF, description = "Supplemental Arrows-A" },
+ ["supplementalarrowsb"] = { first = 0x02900, last = 0x0297F, description = "Supplemental Arrows-B" },
+ ["supplementalmathematicaloperators"] = { first = 0x02A00, last = 0x02AFF, description = "Supplemental Mathematical Operators" },
+ ["supplementalpunctuation"] = { first = 0x02E00, last = 0x02E7F, description = "Supplemental Punctuation" },
+ ["supplementaryprivateuseareaa"] = { first = 0xF0000, last = 0xFFFFF, description = "Supplementary Private Use Area-A" },
+ ["supplementaryprivateuseareab"] = { first = 0x100000,last = 0x10FFFF,description = "Supplementary Private Use Area-B" },
+ ["sylotinagri"] = { first = 0x0A800, last = 0x0A82F, description = "Syloti Nagri" },
+ ["syriac"] = { first = 0x00700, last = 0x0074F, description = "Syriac" },
+ ["tagalog"] = { first = 0x01700, last = 0x0171F, description = "Tagalog" },
+ ["tagbanwa"] = { first = 0x01760, last = 0x0177F, description = "Tagbanwa" },
+ ["tags"] = { first = 0xE0000, last = 0xE007F, description = "Tags" },
+ ["taile"] = { first = 0x01950, last = 0x0197F, description = "Tai Le" },
+ ["taixuanjingsymbols"] = { first = 0x1D300, last = 0x1D35F, description = "Tai Xuan Jing Symbols" },
+ ["tamil"] = { first = 0x00B80, last = 0x00BFF, description = "Tamil" },
+ ["telugu"] = { first = 0x00C00, last = 0x00C7F, description = "Telugu" },
+ ["thaana"] = { first = 0x00780, last = 0x007BF, description = "Thaana" },
+ ["thai"] = { first = 0x00E00, last = 0x00E7F, description = "Thai" },
+ ["tibetan"] = { first = 0x00F00, last = 0x00FFF, description = "Tibetan" },
+ ["tifinagh"] = { first = 0x02D30, last = 0x02D7F, description = "Tifinagh" },
+ ["ugaritic"] = { first = 0x10380, last = 0x1039F, description = "Ugaritic" },
+ ["unifiedcanadianaboriginalsyllabics"] = { first = 0x01400, last = 0x0167F, description = "Unified Canadian Aboriginal Syllabics" },
+ ["vai"] = { first = 0x0A500, last = 0x0A63F, description = "Vai" },
+ ["variationselectors"] = { first = 0x0FE00, last = 0x0FE0F, description = "Variation Selectors" },
+ ["variationselectorssupplement"] = { first = 0xE0100, last = 0xE01EF, description = "Variation Selectors Supplement" },
+ ["verticalforms"] = { first = 0x0FE10, last = 0x0FE1F, description = "Vertical Forms" },
+ ["yijinghexagramsymbols"] = { first = 0x04DC0, last = 0x04DFF, description = "Yijing Hexagram Symbols" },
+ ["yiradicals"] = { first = 0x0A490, last = 0x0A4CF, description = "Yi Radicals" },
+ ["yisyllables"] = { first = 0x0A000, last = 0x0A48F, description = "Yi Syllables" },
}
-function characters.getrange(name)
- local tag = lower(name)
- tag = gsub(name,"[^a-z]", "")
- local range = characters.blocks[tag]
+characters.blocks = blocks
+
+setmetatableindex(blocks, function(t,k)
+ return k and rawget(t,lower(gsub(k,"[^a-zA-Z]","")))
+end)
+
+function characters.getrange(name) -- used in font fallback definitions (name or range)
+ local range = blocks[name]
if range then
- return range[1], range[2], range[3]
+ return range.first, range.last, range.description
end
name = gsub(name,'"',"0x") -- goodie: tex hex notation
local start, stop = match(name,"^(.-)[%-%:](.-)$")
@@ -251,7 +328,7 @@ function characters.getrange(name)
return slot, slot, nil
end
-characters.categories = {
+local categorytags = allocate {
lu = "Letter Uppercase",
ll = "Letter Lowercase",
lt = "Letter Titlecase",
@@ -284,26 +361,58 @@ characters.categories = {
cn = "Other Not Assigned",
}
+characters.categorytags = categorytags
+
--~ special : cf (softhyphen) zs (emspace)
--~ characters: ll lm lo lt lu mn nl no pc pd pe pf pi po ps sc sk sm so
-characters.is_character = table.tohash {
+local is_character = allocate ( table.tohash {
"lu","ll","lt","lm","lo",
"nd","nl","no",
"mn",
"nl","no",
"pc","pd","ps","pe","pi","pf","po",
"sm","sc","sk","so"
-}
+} )
-characters.is_letter = table.tohash {
+local is_letter = allocate ( table.tohash {
"ll","lm","lo","lt","lu"
-}
+} )
-characters.is_command = table.tohash {
+local is_command = allocate ( table.tohash {
"cf","zs"
+} )
+
+local is_spacing = allocate ( table.tohash {
+ "zs", "zl","zp",
+} )
+
+local is_mark = allocate ( table.tohash {
+ "mn", "ms",
+} )
+
+characters.is_character = is_character
+characters.is_letter = is_letter
+characters.is_command = is_command
+characters.is_spacing = is_spacing
+characters.is_mark = is_mark
+
+local mt = { -- yes or no ?
+ __index = function(t,k)
+ if type(k) == "number" then
+ local c = characters.data[k].category
+ return c and rawget(t,c)
+ else
+ -- avoid auto conversion in data.characters lookups
+ end
+ end
}
+setmetatableindex(characters.is_character, mt)
+setmetatableindex(characters.is_letter, mt)
+setmetatableindex(characters.is_command, mt)
+setmetatableindex(characters.is_spacing, mt)
+
-- linebreak: todo: hash
--
-- normative : BK CR LF CM SG GL CB SP ZW NL WJ JL JV JT H2 H3
@@ -313,7 +422,7 @@ characters.is_command = table.tohash {
--
-- N A H W F Na
-characters.bidi = {
+characters.bidi = allocate {
l = "Left-to-Right",
lre = "Left-to-Right Embedding",
lro = "Left-to-Right Override",
@@ -335,14 +444,6 @@ characters.bidi = {
on = "Other Neutrals",
}
-local _empty_table_ = { __index = function(t,k) return "" end }
-
-function table.set_empty_metatable(t)
- setmetatable(t,_empty_table_)
-end
-
-table.set_empty_metatable(data)
-
--[[ldx--
<p>At this point we assume that the big data table is loaded. From this
table we derive a few more.</p>
@@ -350,6 +451,8 @@ table we derive a few more.</p>
if not characters.fallbacks then
+ -- we could the definition by using a metatable
+
characters.fallbacks = { }
characters.directions = { }
@@ -368,8 +471,8 @@ if not characters.fallbacks then
end
-storage.register("characters.fallbacks", characters.fallbacks, "characters.fallbacks")
-storage.register("characters.directions", characters.directions, "characters.directions")
+storage.register("characters/fallbacks", characters.fallbacks, "characters.fallbacks") -- accents and such
+storage.register("characters/directions", characters.directions, "characters.directions")
--[[ldx--
<p>The <type>context</type> namespace is used to store methods and data
@@ -383,108 +486,149 @@ use the table. After all, we have this information available anyway.</p>
function characters.makeactive(n,name) -- let ?
texsprint(ctxcatcodes,format("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name))
+ -- context("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name)
end
-function tex.uprint(n)
- texsprint(ctxcatcodes,utfchar(n))
+function tex.uprint(c,n)
+ if n then
+ -- texsprint(c,charfromnumber(n))
+ texsprint(c,utfchar(n))
+ else
+ -- texsprint(charfromnumber(c))
+ texsprint(utfchar(c))
+ end
end
-local template_a = "\\startextendcatcodetable{%s}\\chardef\\l=11\\chardef\\a=13\\let\\c\\catcode%s\\let\\a\\undefined\\let\\l\\undefined\\let\\c\\undefined\\stopextendcatcodetable"
-local template_b = "\\chardef\\l=11\\chardef\\a=13\\let\\c\\catcode%s\\let\\a\\undefined\\let\\l\\undefined\\let\\c\\undefined"
+function characters.define(tobelettered, tobeactivated) -- catcodetables
+
+ if trace_defining then
+ report_defining("defining active character commands")
+ end
--- we need a function for setting the codes ....
+ local activated, a = { }, 0
-function characters.define(tobelettered, tobeactivated) -- catcodetables
- local is_character, is_command, is_letter = characters.is_character, characters.is_command, characters.is_letter
- local lettered, activated = { }, { }
- for u, chr in next, data do
- -- we can use a macro instead of direct settings
+ for u, chr in next, data do -- these will be commands
local fallback = chr.fallback
if fallback then
- -- texprint(format("{\\catcode %s=13\\unexpanded\\gdef %s{\\checkedchar{%s}{%s}}}",u,utfchar(u),u,fallback))
texsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\checkedchar{",u,"}{",fallback,"}}}") -- no texprint
- activated[#activated+1] = "\\c"..u.."\\a"
+ a = a + 1
+ activated[a] = u
else
local contextname = chr.contextname
- local category = chr.category
if contextname then
+ local category = chr.category
if is_character[category] then
- -- by this time, we're still in normal catcode mode
- -- subtle: not "\\",contextname but "\\"..contextname
if chr.unicodeslot < 128 then
- -- texprint(ctxcatcodes, "\\chardef\\"..contextname,"=",u)
- texprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u))
- else
- -- texprint(ctxcatcodes, "\\let\\"..contextname,"=",utfchar(u))
- texprint(ctxcatcodes,format("\\let\\%s=%s",contextname,utfchar(u)))
if is_letter[category] then
- lettered[#lettered+1] = "\\c"..u.."\\l"
+ texprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u)))
+ else
+ texprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u))
end
+ else
+ texprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u)))
end
elseif is_command[category] then
- -- this might change: contextcommand ipv contextname
- -- texprint(format("{\\catcode %s=13\\unexpanded\\gdef %s{\\%s}}",u,utfchar(u),contextname))
texsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\"..contextname,"}}") -- no texprint
- activated[#activated+1] = "\\c"..u.."\\a"
- end
- elseif is_letter[category] then
- if u >= 128 and u <= 65536 then -- catch private mess
- lettered[#lettered+1] = "\\c"..u.."\\l"
+ a = a + 1
+ activated[a] = u
end
end
end
- if chr.range then
- lettered[#lettered+1] = format('\\dofastrecurse{"%05X}{"%05X}{1}{\\c\\fastrecursecounter\\l}',u,chr.range)
- end
end
- -- if false then
- lettered[#lettered+1] = "\\c"..0x200C.."\\l" -- non-joiner
- lettered[#lettered+1] = "\\c"..0x200D.."\\l" -- joiner
- -- fi
- if tobelettered then
- lettered = concat(lettered)
- if true then
- texsprint(ctxcatcodes,format(template_b,lettered))
- else
- for l=1,#tobelettered do
- texsprint(ctxcatcodes,format(template_a,tobelettered[l],lettered))
+
+ if tobelettered then -- shared
+ local saved = tex.catcodetable
+ for i=1,#tobelettered do
+ tex.catcodetable = tobelettered[i]
+ if trace_defining then
+ report_defining("defining letters (global, shared)")
+ end
+ for u, chr in next, data do
+ if not chr.fallback and is_letter[chr.category] and u >= 128 and u <= 65536 then
+ texsetcatcode(u,11)
+ end
+ local range = chr.range
+ if range then
+ for i=1,range.first,range.last do
+ texsetcatcode(i,11)
+ end
+ end
end
+ texsetcatcode(0x200C,11) -- non-joiner
+ texsetcatcode(0x200D,11) -- joiner
end
+ tex.catcodetable = saved
end
- if tobeactivated then
- activated = concat(activated)
- for a=1,#tobeactivated do
- texsprint(ctxcatcodes,format(template_a,tobeactivated[a],activated))
+
+ local nofactivated = #tobeactivated
+ if tobeactivated and nofactivated > 0 then
+ for i=1,nofactivated do
+ local u = activated[i]
+ report_defining("character 0x%05X is active in sets %s (%s)",u,concat(tobeactivated,","),data[u].description)
+ end
+ local saved = tex.catcodetable
+ for i=1,#tobeactivated do
+ local vector = tobeactivated[i]
+ if trace_defining then
+ report_defining("defining %s active characters in vector %s",nofactivated,vector)
+ end
+ tex.catcodetable = vector
+ for i=1,nofactivated do
+ texsetcatcode(activated[i],13)
+ end
end
+ tex.catcodetable = saved
end
-end
-function characters.charcode(box)
- local b = tex.box[box]
- local l = b.list
- texsprint((l and l.id == node.id('glyph') and l.char) or 0)
end
--[[ldx--
<p>Setting the lccodes is also done in a loop over the data table.</p>
--ldx]]--
+--~ function tex.setsfcode (index,sf) ... end
+--~ function tex.setlccode (index,lc,[uc]) ... end -- optional third value, safes call
+--~ function tex.setuccode (index,uc,[lc]) ... end
+--~ function tex.setcatcode(index,cc) ... end
+
-- we need a function ...
+--~ tex.lccode
+--~ tex.uccode
+--~ tex.sfcode
+--~ tex.catcode
+
function characters.setcodes()
+ if trace_defining then
+ report_defining("defining lc and uc codes")
+ end
for code, chr in next, data do
local cc = chr.category
- if cc == 'll' or cc == 'lu' or cc == 'lt' then
- local lc, uc = chr.lccode, chr.uccode
- if not lc then chr.lccode, lc = code, code end
- if not uc then chr.uccode, uc = code, code end
- texsprint(ctxcatcodes,format("\\setcclcuc{%i}{%i}{%i}",code,lc,uc))
- end
- if cc == "lu" then
- texprint(ctxcatcodes,"\\sfcode ",code,"999 ")
- end
- if cc == "lo" and chr.range then
- texsprint(ctxcatcodes,format('\\dofastrecurse{"%05X}{"%05X}{1}{\\setcclcucself\\fastrecursecounter}',code,chr.range))
+ if is_letter[cc] then
+ local range = chr.range
+ if range then
+ for i=range.first,range.last do
+ texsetcatcode(i,11) -- letter
+ texsetlccode(i,i,i) -- self self
+ end
+ else
+ local lc, uc = chr.lccode, chr.uccode
+ if not lc then chr.lccode, lc = code, code end
+ if not uc then chr.uccode, uc = code, code end
+ texsetcatcode(code,11) -- letter
+ if type(lc) == "table" then
+ lc = code
+ end
+ if type(uc) == "table" then
+ uc = code
+ end
+ texsetlccode(code,lc,uc)
+ if cc == "lu" then
+ texsetsfcode(code,999)
+ end
+ end
+ elseif is_mark[cc] then
+ texsetlccode(code,code,code) -- for hyphenation
end
end
end
@@ -495,88 +639,236 @@ of the official <l n='api'/>.</p>
--ldx]]--
--[[ldx--
-<p>This converts a string (if given) into a number.</p>
+<p>A couple of convenience methods. Beware, these are slower than directly
+accessing the data table.</p>
--ldx]]--
-function characters.number(n)
- if type(n) == "string" then return tonumber(n,16) else return n end
-end
+-- we could make them virtual: characters.contextnames[n]
---[[ldx--
-<p>Checking for valid characters.</p>
---ldx]]--
+function characters.contextname(n) return data[n].contextname or "" end
+function characters.adobename (n) return data[n].adobename or "" end
+function characters.description(n) return data[n].description or "" end
+-------- characters.category (n) return data[n].category or "" end
-function characters.is_valid(s)
- return s or ""
+function characters.category(n,verbose)
+ local c = data[n].category
+ if not c then
+ return ""
+ elseif verbose then
+ return categorytags[c]
+ else
+ return c
+ end
end
-function characters.checked(s, default)
- return s or default
+-- xml support (moved)
+
+function characters.remapentity(chr,slot)
+ texsprint(format("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr))
end
-characters.valid = characters.is_valid
+characters.activeoffset = 0x10000 -- there will be remapped in that byte range
---[[ldx--
-<p></p>
---ldx]]--
--- set a table entry; index is number (can be different from unicodeslot)
+-- xml.entities = xml.entities or { }
+--
+-- storage.register("xml/entities",xml.entities,"xml.entities") -- this will move to lxml
+--
+-- function characters.setmkiventities()
+-- local entities = xml.entities
+-- entities.lt = "<"
+-- entities.amp = "&"
+-- entities.gt = ">"
+-- end
+--
+-- function characters.setmkiientities()
+-- local entities = xml.entities
+-- entities.lt = utfchar(characters.activeoffset + utfbyte("<"))
+-- entities.amp = utfchar(characters.activeoffset + utfbyte("&"))
+-- entities.gt = utfchar(characters.activeoffset + utfbyte(">"))
+-- end
+
+-- some day we will make a table
-function characters.set(n, c)
- data[characters.number(n)] = c
+local function utfstring(s)
+ if type(s) == "table" then
+ return utfchar(unpack(s)) -- concat { utfchar( unpack(s) ) }
+ else
+ return utfchar(s)
+ end
end
---[[ldx--
-<p>Get a table entry happens by number. Keep in mind that the unicodeslot
-can be different (not likely).</p>
---ldx]]--
+utf.string = utf.string or utfstring
-function characters.get(n)
- return data[characters.number(n)]
-end
+local categories = allocate() characters.categories = categories -- lazy table
---[[ldx--
-<p>A couple of convenience methods. Beware, these are not that fast due
-to the checking.</p>
---ldx]]--
+setmetatableindex(categories, function(t,u) if u then local c = data[u] c = c and c.category or u t[u] = c return c end end)
-function characters.hexindex(n)
- return format("%04X", characters.valid(data[characters.number(n)].unicodeslot))
-end
+local lccodes = allocate() characters.lccodes = lccodes -- lazy table
+local uccodes = allocate() characters.uccodes = uccodes -- lazy table
+local shcodes = allocate() characters.shcodes = shcodes -- lazy table
+local fscodes = allocate() characters.fscodes = fscodes -- lazy table
+
+setmetatableindex(lccodes, function(t,u) if u then local c = data[u] c = c and c.lccode or (type(u) == "string" and utfbyte(u)) or u t[u] = c return c end end)
+setmetatableindex(uccodes, function(t,u) if u then local c = data[u] c = c and c.uccode or (type(u) == "string" and utfbyte(u)) or u t[u] = c return c end end)
+setmetatableindex(shcodes, function(t,u) if u then local c = data[u] c = c and c.shcode or (type(u) == "string" and utfbyte(u)) or u t[u] = c return c end end)
+setmetatableindex(fscodes, function(t,u) if u then local c = data[u] c = c and c.fscode or (type(u) == "string" and utfbyte(u)) or u t[u] = c return c end end)
+
+local lcchars = allocate() characters.lcchars = lcchars -- lazy table
+local ucchars = allocate() characters.ucchars = ucchars -- lazy table
+local shchars = allocate() characters.shchars = shchars -- lazy table
+local fschars = allocate() characters.fschars = fschars -- lazy table
+
+setmetatableindex(lcchars, function(t,u) if u then local c = data[u] c = c and c.lccode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end)
+setmetatableindex(ucchars, function(t,u) if u then local c = data[u] c = c and c.uccode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end)
+setmetatableindex(shchars, function(t,u) if u then local c = data[u] c = c and c.shcode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end)
+setmetatableindex(fschars, function(t,u) if u then local c = data[u] c = c and c.fscode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end)
-function characters.contextname(n)
- return characters.valid(data[characters.number(n)].contextname)
+local decomposed = allocate() characters.decomposed = decomposed -- lazy table
+
+setmetatableindex(decomposed, function(t,u) -- either a table or false
+ if u then
+ local c = data[u]
+ local s = c and c.decomposed or false
+ t[u] = s
+ return s
+ end
+end)
+
+local specialchars = allocate() characters.specialchars = specialchars -- lazy table
+local descriptions = allocate() characters.descriptions = descriptions -- lazy table
+
+setmetatableindex(specialchars, function(t,u)
+ if u then
+ local c = data[u]
+ local s = c and c.specials
+ if s then
+ local tt, ttn = { }, 0
+ for i=2,#s do
+ local si = s[i]
+ local c = data[si]
+ if is_letter[c.category] then
+ ttn = ttn + 1
+ tt[ttn] = utfchar(si)
+ end
+ end
+ c = concat(tt)
+ t[u] = c
+ return c
+ else
+ if type(u) == "number" then
+ u = utfchar(u)
+ end
+ t[u] = u
+ return u
+ end
+ end
+end)
+
+setmetatableindex(descriptions, function(t,k)
+ -- 0.05 - 0.10 sec
+ for u, c in next, data do
+ local d = c.description
+ if d then
+ d = gsub(d," ","")
+ d = lower(d)
+ t[d] = u
+ end
+ end
+ local d = rawget(t,k)
+ if not d then
+ t[k] = k
+ end
+ return d
+end)
+
+function characters.unicodechar(asked)
+ local n = tonumber(asked)
+ if n then
+ return n
+ elseif type(asked) == "string" then
+ asked = gsub(asked," ","")
+ return descriptions[asked]
+ end
end
-function characters.adobename(n)
- return characters.valid(data[characters.number(n)].adobename)
+function characters.lower(str)
+ local new, n = { }, 0
+ for u in utfvalues(str) do
+ n = n + 1
+ new[n] = lcchars[u]
+ end
+ return concat(new)
end
-function characters.description(n)
- return characters.valid(data[characters.number(n)].description)
+function characters.upper(str)
+ local new, n = { }, 0
+ for u in utfvalues(str) do
+ n = n + 1
+ new[n] = ucchars[u]
+ end
+ return concat(new)
end
-function characters.category(n)
- return characters.valid(data[characters.number(n)].category)
+function characters.shaped(str)
+ local new, n = { }, 0
+ for u in utfvalues(str) do
+ n = n + 1
+ new[n] = shchars[u]
+ end
+ return concat(new)
end
+function characters.lettered(str,spacing)
+ local new, n = { }, 0
+ if spacing then
+ local done = false
+ for u in utfvalues(str) do
+ local c = data[u].category
+ if is_letter[c] then
+ if done and n > 1 then
+ n = n + 1
+ new[n] = " "
+ done = false
+ end
+ n = n + 1
+ new[n] = utfchar(u)
+ elseif spacing and is_spacing[c] then
+ done = true
+ end
+ end
+ else
+ for u in utfvalues(str) do
+ if is_letter[data[u].category] then
+ n = n + 1
+ new[n] = utfchar(u)
+ end
+ end
+ end
+ return concat(new)
+end
--[[ldx--
<p>Requesting lower and uppercase codes:</p>
--ldx]]--
-function characters.uccode(n) return data[n].uccode or n end
-function characters.lccode(n) return data[n].lccode or n end
+function characters.uccode(n) return uccodes[n] end -- obsolete
+function characters.lccode(n) return lccodes[n] end -- obsolete
-function characters.flush(n)
+function characters.flush(n,direct)
local c = data[n]
if c and c.contextname then
- texsprint(texcatcodes, "\\"..c.contextname)
+ c = "\\" .. c.contextname
else
- texsprint(utfchar(n))
+ c = utfchar(n)
+ end
+ if direct then
+ return c
+ else
+ texsprint(c)
end
end
function characters.shape(n)
- local shcode = data[n].shcode
+ local shcode = shcodes[n]
if not shcode then
return n, nil
elseif type(shcode) == "table" then
@@ -586,72 +878,6 @@ function characters.shape(n)
end
end
---[[ldx--
-<p>Categories play an important role, so here are some checkers.</p>
---ldx]]--
-
-function characters.is_of_category(token,category)
- if type(token) == "string" then
- return data[utfbyte(token)].category == category
- else
- return data[token].category == category
- end
-end
-
-function characters.i_is_of_category(i,category) -- by index (number)
- local cd = data[i]
- return cd and cd.category == category
-end
-
-function characters.n_is_of_category(n,category) -- by name (string)
- local cd = data[utfbyte(n)]
- return cd and cd.category == category
-end
-
--- xml support (moved)
-
-function characters.remapentity(chr,slot)
- texsprint(format("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr))
-end
-
-characters.active_offset = 0x10000 -- there will be remapped in that byte range
-
--- xml.entities = xml.entities or { }
---
--- storage.register("xml/entities",xml.entities,"xml.entities") -- this will move to lxml
---
--- function characters.setmkiventities()
--- local entities = xml.entities
--- entities.lt = "<"
--- entities.amp = "&"
--- entities.gt = ">"
--- end
---
--- function characters.setmkiientities()
--- local entities = xml.entities
--- entities.lt = utfchar(characters.active_offset + utfbyte("<"))
--- entities.amp = utfchar(characters.active_offset + utfbyte("&"))
--- entities.gt = utfchar(characters.active_offset + utfbyte(">"))
--- end
-
--- some day we will make a table
-
-function characters.lower(str)
- local new = { }
- for u in utfvalues(str) do
- new[#new+1] = utfchar(data[u].lccode or u)
- end
- return concat(new)
-end
-
-function characters.upper(str)
- local new = { }
- for u in utfvalues(str) do
- new[#new+1] = utfchar(data[u].uccode or u)
- end
- return concat(new)
-end
-
-- -- some day we might go this route, but it does not really save that much
-- -- so not now (we can generate a lot using mtx-unicode that operates on the
-- -- database)
@@ -696,3 +922,67 @@ end
-- end
--~ characters.data, characters.groups = chardata, groupdata
+
+--~ [0xF0000]={
+--~ category="co",
+--~ cjkwd="a",
+--~ description="<Plane 0x000F Private Use, First>",
+--~ direction="l",
+--~ unicodeslot=0xF0000,
+--~ },
+--~ [0xFFFFD]={
+--~ category="co",
+--~ cjkwd="a",
+--~ description="<Plane 0x000F Private Use, Last>",
+--~ direction="l",
+--~ unicodeslot=0xFFFFD,
+--~ },
+--~ [0x100000]={
+--~ category="co",
+--~ cjkwd="a",
+--~ description="<Plane 0x0010 Private Use, First>",
+--~ direction="l",
+--~ unicodeslot=0x100000,
+--~ },
+--~ [0x10FFFD]={
+--~ category="co",
+--~ cjkwd="a",
+--~ description="<Plane 0x0010 Private Use, Last>",
+--~ direction="l",
+--~ unicodeslot=0x10FFFD,
+--~ },
+
+if not characters.superscripts then
+
+ local superscripts = allocate() characters.superscripts = superscripts
+ local subscripts = allocate() characters.subscripts = subscripts
+
+ -- skipping U+02120 (service mark) U+02122 (trademark)
+
+ for k, v in next, data do
+ local specials = v.specials
+ if specials then
+ local what = specials[1]
+ if what == "super" then
+ if #specials == 2 then
+ superscripts[k] = specials[2]
+ else
+ report_defining("ignoring superscript %s %s: %s",ustring(k),utfchar(k),v.description)
+ end
+ elseif what == "sub" then
+ if #specials == 2 then
+ subscripts[k] = specials[2]
+ else
+ report_defining("ignoring subscript %s %s: %s",ustring(k),utfchar(k),v.description)
+ end
+ end
+ end
+ end
+
+ -- print(table.serialize(superscripts, "superscripts", { hexify = true }))
+ -- print(table.serialize(subscripts, "subscripts", { hexify = true }))
+
+ storage.register("characters/superscripts", superscripts, "characters.superscripts")
+ storage.register("characters/subscripts", subscripts, "characters.subscripts")
+
+end