diff options
author | Karl Berry <karl@freefriends.org> | 2013-04-08 00:43:40 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2013-04-08 00:43:40 +0000 |
commit | 824f7b0903de8ad7f6ee3d7656005e4c59155e06 (patch) | |
tree | abfc8673ef9916f3ab7074e811207384c301492b /Master/texmf-dist/tex/context/base/char-ini.lua | |
parent | 689aefb0727676ed3cddf331337b4be226495e72 (diff) |
context import for TL13, from www.pragma-ade.com/context/beta/cont-tmf.zip
git-svn-id: svn://tug.org/texlive/trunk@29731 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/tex/context/base/char-ini.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/char-ini.lua | 1033 |
1 files changed, 585 insertions, 448 deletions
diff --git a/Master/texmf-dist/tex/context/base/char-ini.lua b/Master/texmf-dist/tex/context/base/char-ini.lua index 244b8d2a70e..b75f5eda758 100644 --- a/Master/texmf-dist/tex/context/base/char-ini.lua +++ b/Master/texmf-dist/tex/context/base/char-ini.lua @@ -10,26 +10,17 @@ if not modules then modules = { } end modules ['char-ini'] = { -- we can remove the tag range starting at 0xE0000 (special applications) -local tex = tex -local utf = unicode.utf8 - -local utfchar, utfbyte, utfvalues = utf.char, utf.byte, string.utfvalues -local ustring = unicode.ustring +local utfchar, utfbyte, utfvalues, ustring = utf.char, utf.byte, utf.values, utf.ustring local concat, unpack, tohash = table.concat, table.unpack, table.tohash local next, tonumber, type, rawget, rawset = next, tonumber, type, rawget, rawset local format, lower, gsub, match, gmatch = string.format, string.lower, string.gsub, string.match, string.match, string.gmatch -local P, R, lpegmatch = lpeg.P, lpeg.R, lpeg.match +local P, R, Cs, lpegmatch, patterns = lpeg.P, lpeg.R, lpeg.Cs, lpeg.match, lpeg.patterns + +local utf8byte = patterns.utf8byte +local utf8char = patterns.utf8char local allocate = utilities.storage.allocate local mark = utilities.storage.mark -local texsetlccode = tex.setlccode -local texsetuccode = tex.setuccode -local texsetsfcode = tex.setsfcode -local texsetcatcode = tex.setcatcode - -local contextsprint = context.sprint -local ctxcatcodes = tex.ctxcatcodes -local texcatcodes = tex.texcatcodes local setmetatableindex = table.setmetatableindex @@ -48,7 +39,6 @@ loaded!</p> characters = characters or { } local characters = characters - local data = characters.data if data then @@ -64,7 +54,7 @@ end local pattern = (P("0x") + P("U+")) * ((R("09","AF")^1 * P(-1)) / function(s) return tonumber(s,16) end) -lpeg.patterns.chartonumber = pattern +patterns.chartonumber = pattern local function chartonumber(k) if type(k) == "string" then @@ -136,185 +126,259 @@ setmetatableindex(data, function(t,k) end) local blocks = allocate { - ["aegeannumbers"] = { first = 0x10100, last = 0x1013F, description = "Aegean Numbers" }, - ["alphabeticpresentationforms"] = { first = 0x0FB00, last = 0x0FB4F, description = "Alphabetic Presentation Forms" }, - ["ancientgreekmusicalnotation"] = { first = 0x1D200, last = 0x1D24F, description = "Ancient Greek Musical Notation" }, - ["ancientgreeknumbers"] = { first = 0x10140, last = 0x1018F, description = "Ancient Greek Numbers" }, - ["ancientsymbols"] = { first = 0x10190, last = 0x101CF, description = "Ancient Symbols" }, - ["arabic"] = { first = 0x00600, last = 0x006FF, description = "Arabic" }, - ["arabicpresentationformsa"] = { first = 0x0FB50, last = 0x0FDFF, description = "Arabic Presentation Forms-A" }, - ["arabicpresentationformsb"] = { first = 0x0FE70, last = 0x0FEFF, description = "Arabic Presentation Forms-B" }, - ["arabicsupplement"] = { first = 0x00750, last = 0x0077F, description = "Arabic Supplement" }, - ["armenian"] = { first = 0x00530, last = 0x0058F, description = "Armenian" }, - ["arrows"] = { first = 0x02190, last = 0x021FF, description = "Arrows" }, - ["balinese"] = { first = 0x01B00, last = 0x01B7F, description = "Balinese" }, - ["basiclatin"] = { first = 0x00000, last = 0x0007F, description = "Basic Latin" }, - ["bengali"] = { first = 0x00980, last = 0x009FF, description = "Bengali" }, - ["blockelements"] = { first = 0x02580, last = 0x0259F, description = "Block Elements" }, - ["bopomofo"] = { first = 0x03100, last = 0x0312F, description = "Bopomofo" }, - ["bopomofoextended"] = { first = 0x031A0, last = 0x031BF, description = "Bopomofo Extended" }, - ["boxdrawing"] = { first = 0x02500, last = 0x0257F, description = "Box Drawing" }, - ["braillepatterns"] = { first = 0x02800, last = 0x028FF, description = "Braille Patterns" }, - ["buginese"] = { first = 0x01A00, last = 0x01A1F, description = "Buginese" }, - ["buhid"] = { first = 0x01740, last = 0x0175F, description = "Buhid" }, - ["byzantinemusicalsymbols"] = { first = 0x1D000, last = 0x1D0FF, description = "Byzantine Musical Symbols" }, - ["carian"] = { first = 0x102A0, last = 0x102DF, description = "Carian" }, - ["cham"] = { first = 0x0AA00, last = 0x0AA5F, description = "Cham" }, - ["cherokee"] = { first = 0x013A0, last = 0x013FF, description = "Cherokee" }, - ["cjkcompatibility"] = { first = 0x03300, last = 0x033FF, description = "CJK Compatibility" }, - ["cjkcompatibilityforms"] = { first = 0x0FE30, last = 0x0FE4F, description = "CJK Compatibility Forms" }, - ["cjkcompatibilityideographs"] = { first = 0x0F900, last = 0x0FAFF, description = "CJK Compatibility Ideographs" }, - ["cjkcompatibilityideographssupplement"] = { first = 0x2F800, last = 0x2FA1F, description = "CJK Compatibility Ideographs Supplement" }, - ["cjkradicalssupplement"] = { first = 0x02E80, last = 0x02EFF, description = "CJK Radicals Supplement" }, - ["cjkstrokes"] = { first = 0x031C0, last = 0x031EF, description = "CJK Strokes" }, - ["cjksymbolsandpunctuation"] = { first = 0x03000, last = 0x0303F, description = "CJK Symbols and Punctuation" }, - ["cjkunifiedideographs"] = { first = 0x04E00, last = 0x09FFF, description = "CJK Unified Ideographs" }, - ["cjkunifiedideographsextensiona"] = { first = 0x03400, last = 0x04DBF, description = "CJK Unified Ideographs Extension A" }, - ["cjkunifiedideographsextensionb"] = { first = 0x20000, last = 0x2A6DF, description = "CJK Unified Ideographs Extension B" }, - ["combiningdiacriticalmarks"] = { first = 0x00300, last = 0x0036F, description = "Combining Diacritical Marks" }, - ["combiningdiacriticalmarksforsymbols"] = { first = 0x020D0, last = 0x020FF, description = "Combining Diacritical Marks for Symbols" }, - ["combiningdiacriticalmarkssupplement"] = { first = 0x01DC0, last = 0x01DFF, description = "Combining Diacritical Marks Supplement" }, - ["combininghalfmarks"] = { first = 0x0FE20, last = 0x0FE2F, description = "Combining Half Marks" }, - ["controlpictures"] = { first = 0x02400, last = 0x0243F, description = "Control Pictures" }, - ["coptic"] = { first = 0x02C80, last = 0x02CFF, description = "Coptic" }, - ["countingrodnumerals"] = { first = 0x1D360, last = 0x1D37F, description = "Counting Rod Numerals" }, - ["cuneiform"] = { first = 0x12000, last = 0x123FF, description = "Cuneiform" }, - ["cuneiformnumbersandpunctuation"] = { first = 0x12400, last = 0x1247F, description = "Cuneiform Numbers and Punctuation" }, - ["currencysymbols"] = { first = 0x020A0, last = 0x020CF, description = "Currency Symbols" }, - ["cypriotsyllabary"] = { first = 0x10800, last = 0x1083F, description = "Cypriot Syllabary" }, - ["cyrillic"] = { first = 0x00400, last = 0x004FF, description = "Cyrillic" }, - ["cyrillicextendeda"] = { first = 0x02DE0, last = 0x02DFF, description = "Cyrillic Extended-A" }, - ["cyrillicextendedb"] = { first = 0x0A640, last = 0x0A69F, description = "Cyrillic Extended-B" }, - ["cyrillicsupplement"] = { first = 0x00500, last = 0x0052F, description = "Cyrillic Supplement" }, - ["deseret"] = { first = 0x10400, last = 0x1044F, description = "Deseret" }, - ["devanagari"] = { first = 0x00900, last = 0x0097F, description = "Devanagari" }, - ["dingbats"] = { first = 0x02700, last = 0x027BF, description = "Dingbats" }, - ["dominotiles"] = { first = 0x1F030, last = 0x1F09F, description = "Domino Tiles" }, - ["enclosedalphanumerics"] = { first = 0x02460, last = 0x024FF, description = "Enclosed Alphanumerics" }, - ["enclosedcjklettersandmonths"] = { first = 0x03200, last = 0x032FF, description = "Enclosed CJK Letters and Months" }, - ["ethiopic"] = { first = 0x01200, last = 0x0137F, description = "Ethiopic" }, - ["ethiopicextended"] = { first = 0x02D80, last = 0x02DDF, description = "Ethiopic Extended" }, - ["ethiopicsupplement"] = { first = 0x01380, last = 0x0139F, description = "Ethiopic Supplement" }, - ["generalpunctuation"] = { first = 0x02000, last = 0x0206F, description = "General Punctuation" }, - ["geometricshapes"] = { first = 0x025A0, last = 0x025FF, description = "Geometric Shapes" }, - ["georgian"] = { first = 0x010A0, last = 0x010FF, description = "Georgian" }, - ["georgiansupplement"] = { first = 0x02D00, last = 0x02D2F, description = "Georgian Supplement" }, - ["glagolitic"] = { first = 0x02C00, last = 0x02C5F, description = "Glagolitic" }, - ["gothic"] = { first = 0x10330, last = 0x1034F, description = "Gothic" }, - ["greekandcoptic"] = { first = 0x00370, last = 0x003FF, description = "Greek and Coptic" }, - ["greekextended"] = { first = 0x01F00, last = 0x01FFF, description = "Greek Extended" }, - ["gujarati"] = { first = 0x00A80, last = 0x00AFF, description = "Gujarati" }, - ["gurmukhi"] = { first = 0x00A00, last = 0x00A7F, description = "Gurmukhi" }, - ["halfwidthandfullwidthforms"] = { first = 0x0FF00, last = 0x0FFEF, description = "Halfwidth and Fullwidth Forms" }, - ["hangulcompatibilityjamo"] = { first = 0x03130, last = 0x0318F, description = "Hangul Compatibility Jamo" }, - ["hanguljamo"] = { first = 0x01100, last = 0x011FF, description = "Hangul Jamo" }, - ["hangulsyllables"] = { first = 0x0AC00, last = 0x0D7AF, description = "Hangul Syllables" }, - ["hanunoo"] = { first = 0x01720, last = 0x0173F, description = "Hanunoo" }, - ["hebrew"] = { first = 0x00590, last = 0x005FF, description = "Hebrew" }, - ["highprivateusesurrogates"] = { first = 0x0DB80, last = 0x0DBFF, description = "High Private Use Surrogates" }, - ["highsurrogates"] = { first = 0x0D800, last = 0x0DB7F, description = "High Surrogates" }, - ["hiragana"] = { first = 0x03040, last = 0x0309F, description = "Hiragana" }, - ["ideographicdescriptioncharacters"] = { first = 0x02FF0, last = 0x02FFF, description = "Ideographic Description Characters" }, - ["ipaextensions"] = { first = 0x00250, last = 0x002AF, description = "IPA Extensions" }, - ["kanbun"] = { first = 0x03190, last = 0x0319F, description = "Kanbun" }, - ["kangxiradicals"] = { first = 0x02F00, last = 0x02FDF, description = "Kangxi Radicals" }, - ["kannada"] = { first = 0x00C80, last = 0x00CFF, description = "Kannada" }, - ["katakana"] = { first = 0x030A0, last = 0x030FF, description = "Katakana" }, - ["katakanaphoneticextensions"] = { first = 0x031F0, last = 0x031FF, description = "Katakana Phonetic Extensions" }, - ["kayahli"] = { first = 0x0A900, last = 0x0A92F, description = "Kayah Li" }, - ["kharoshthi"] = { first = 0x10A00, last = 0x10A5F, description = "Kharoshthi" }, - ["khmer"] = { first = 0x01780, last = 0x017FF, description = "Khmer" }, - ["khmersymbols"] = { first = 0x019E0, last = 0x019FF, description = "Khmer Symbols" }, - ["lao"] = { first = 0x00E80, last = 0x00EFF, description = "Lao" }, - ["latinextendeda"] = { first = 0x00100, last = 0x0017F, description = "Latin Extended-A" }, - ["latinextendedadditional"] = { first = 0x01E00, last = 0x01EFF, description = "Latin Extended Additional" }, - ["latinextendedb"] = { first = 0x00180, last = 0x0024F, description = "Latin Extended-B" }, - ["latinextendedc"] = { first = 0x02C60, last = 0x02C7F, description = "Latin Extended-C" }, - ["latinextendedd"] = { first = 0x0A720, last = 0x0A7FF, description = "Latin Extended-D" }, - ["latinsupplement"] = { first = 0x00080, last = 0x000FF, description = "Latin-1 Supplement" }, - ["lepcha"] = { first = 0x01C00, last = 0x01C4F, description = "Lepcha" }, - ["letterlikesymbols"] = { first = 0x02100, last = 0x0214F, description = "Letterlike Symbols" }, - ["limbu"] = { first = 0x01900, last = 0x0194F, description = "Limbu" }, - ["linearbideograms"] = { first = 0x10080, last = 0x100FF, description = "Linear B Ideograms" }, - ["linearbsyllabary"] = { first = 0x10000, last = 0x1007F, description = "Linear B Syllabary" }, - ["lowsurrogates"] = { first = 0x0DC00, last = 0x0DFFF, description = "Low Surrogates" }, - ["lycian"] = { first = 0x10280, last = 0x1029F, description = "Lycian" }, - ["lydian"] = { first = 0x10920, last = 0x1093F, description = "Lydian" }, - ["mahjongtiles"] = { first = 0x1F000, last = 0x1F02F, description = "Mahjong Tiles" }, - ["malayalam"] = { first = 0x00D00, last = 0x00D7F, description = "Malayalam" }, - ["mathematicalalphanumericsymbols"] = { first = 0x1D400, last = 0x1D7FF, description = "Mathematical Alphanumeric Symbols" }, - ["mathematicaloperators"] = { first = 0x02200, last = 0x022FF, description = "Mathematical Operators" }, - ["miscellaneousmathematicalsymbolsa"] = { first = 0x027C0, last = 0x027EF, description = "Miscellaneous Mathematical Symbols-A" }, - ["miscellaneousmathematicalsymbolsb"] = { first = 0x02980, last = 0x029FF, description = "Miscellaneous Mathematical Symbols-B" }, - ["miscellaneoussymbols"] = { first = 0x02600, last = 0x026FF, description = "Miscellaneous Symbols" }, - ["miscellaneoussymbolsandarrows"] = { first = 0x02B00, last = 0x02BFF, description = "Miscellaneous Symbols and Arrows" }, - ["miscellaneoustechnical"] = { first = 0x02300, last = 0x023FF, description = "Miscellaneous Technical" }, - ["modifiertoneletters"] = { first = 0x0A700, last = 0x0A71F, description = "Modifier Tone Letters" }, - ["mongolian"] = { first = 0x01800, last = 0x018AF, description = "Mongolian" }, - ["musicalsymbols"] = { first = 0x1D100, last = 0x1D1FF, description = "Musical Symbols" }, - ["myanmar"] = { first = 0x01000, last = 0x0109F, description = "Myanmar" }, - ["newtailue"] = { first = 0x01980, last = 0x019DF, description = "New Tai Lue" }, - ["nko"] = { first = 0x007C0, last = 0x007FF, description = "NKo" }, - ["numberforms"] = { first = 0x02150, last = 0x0218F, description = "Number Forms" }, - ["ogham"] = { first = 0x01680, last = 0x0169F, description = "Ogham" }, - ["olchiki"] = { first = 0x01C50, last = 0x01C7F, description = "Ol Chiki" }, - ["olditalic"] = { first = 0x10300, last = 0x1032F, description = "Old Italic" }, - ["oldpersian"] = { first = 0x103A0, last = 0x103DF, description = "Old Persian" }, - ["opticalcharacterrecognition"] = { first = 0x02440, last = 0x0245F, description = "Optical Character Recognition" }, - ["oriya"] = { first = 0x00B00, last = 0x00B7F, description = "Oriya" }, - ["osmanya"] = { first = 0x10480, last = 0x104AF, description = "Osmanya" }, - ["phagspa"] = { first = 0x0A840, last = 0x0A87F, description = "Phags-pa" }, - ["phaistosdisc"] = { first = 0x101D0, last = 0x101FF, description = "Phaistos Disc" }, - ["phoenician"] = { first = 0x10900, last = 0x1091F, description = "Phoenician" }, - ["phoneticextensions"] = { first = 0x01D00, last = 0x01D7F, description = "Phonetic Extensions" }, - ["phoneticextensionssupplement"] = { first = 0x01D80, last = 0x01DBF, description = "Phonetic Extensions Supplement" }, - ["privateusearea"] = { first = 0x0E000, last = 0x0F8FF, description = "Private Use Area" }, - ["rejang"] = { first = 0x0A930, last = 0x0A95F, description = "Rejang" }, - ["runic"] = { first = 0x016A0, last = 0x016FF, description = "Runic" }, - ["saurashtra"] = { first = 0x0A880, last = 0x0A8DF, description = "Saurashtra" }, - ["shavian"] = { first = 0x10450, last = 0x1047F, description = "Shavian" }, - ["sinhala"] = { first = 0x00D80, last = 0x00DFF, description = "Sinhala" }, - ["smallformvariants"] = { first = 0x0FE50, last = 0x0FE6F, description = "Small Form Variants" }, - ["spacingmodifierletters"] = { first = 0x002B0, last = 0x002FF, description = "Spacing Modifier Letters" }, - ["specials"] = { first = 0x0FFF0, last = 0x0FFFF, description = "Specials" }, - ["sundanese"] = { first = 0x01B80, last = 0x01BBF, description = "Sundanese" }, - ["superscriptsandsubscripts"] = { first = 0x02070, last = 0x0209F, description = "Superscripts and Subscripts" }, - ["supplementalarrowsa"] = { first = 0x027F0, last = 0x027FF, description = "Supplemental Arrows-A" }, - ["supplementalarrowsb"] = { first = 0x02900, last = 0x0297F, description = "Supplemental Arrows-B" }, - ["supplementalmathematicaloperators"] = { first = 0x02A00, last = 0x02AFF, description = "Supplemental Mathematical Operators" }, - ["supplementalpunctuation"] = { first = 0x02E00, last = 0x02E7F, description = "Supplemental Punctuation" }, - ["supplementaryprivateuseareaa"] = { first = 0xF0000, last = 0xFFFFF, description = "Supplementary Private Use Area-A" }, - ["supplementaryprivateuseareab"] = { first = 0x100000,last = 0x10FFFF,description = "Supplementary Private Use Area-B" }, - ["sylotinagri"] = { first = 0x0A800, last = 0x0A82F, description = "Syloti Nagri" }, - ["syriac"] = { first = 0x00700, last = 0x0074F, description = "Syriac" }, - ["tagalog"] = { first = 0x01700, last = 0x0171F, description = "Tagalog" }, - ["tagbanwa"] = { first = 0x01760, last = 0x0177F, description = "Tagbanwa" }, - ["tags"] = { first = 0xE0000, last = 0xE007F, description = "Tags" }, - ["taile"] = { first = 0x01950, last = 0x0197F, description = "Tai Le" }, - ["taixuanjingsymbols"] = { first = 0x1D300, last = 0x1D35F, description = "Tai Xuan Jing Symbols" }, - ["tamil"] = { first = 0x00B80, last = 0x00BFF, description = "Tamil" }, - ["telugu"] = { first = 0x00C00, last = 0x00C7F, description = "Telugu" }, - ["thaana"] = { first = 0x00780, last = 0x007BF, description = "Thaana" }, - ["thai"] = { first = 0x00E00, last = 0x00E7F, description = "Thai" }, - ["tibetan"] = { first = 0x00F00, last = 0x00FFF, description = "Tibetan" }, - ["tifinagh"] = { first = 0x02D30, last = 0x02D7F, description = "Tifinagh" }, - ["ugaritic"] = { first = 0x10380, last = 0x1039F, description = "Ugaritic" }, - ["unifiedcanadianaboriginalsyllabics"] = { first = 0x01400, last = 0x0167F, description = "Unified Canadian Aboriginal Syllabics" }, - ["vai"] = { first = 0x0A500, last = 0x0A63F, description = "Vai" }, - ["variationselectors"] = { first = 0x0FE00, last = 0x0FE0F, description = "Variation Selectors" }, - ["variationselectorssupplement"] = { first = 0xE0100, last = 0xE01EF, description = "Variation Selectors Supplement" }, - ["verticalforms"] = { first = 0x0FE10, last = 0x0FE1F, description = "Vertical Forms" }, - ["yijinghexagramsymbols"] = { first = 0x04DC0, last = 0x04DFF, description = "Yijing Hexagram Symbols" }, - ["yiradicals"] = { first = 0x0A490, last = 0x0A4CF, description = "Yi Radicals" }, - ["yisyllables"] = { first = 0x0A000, last = 0x0A48F, description = "Yi Syllables" }, + ["aegeannumbers"] = { first = 0x10100, last = 0x1013F, description = "Aegean Numbers" }, + ["alchemicalsymbols"] = { first = 0x1F700, last = 0x1F77F, description = "Alchemical Symbols" }, + ["alphabeticpresentationforms"] = { first = 0x0FB00, last = 0x0FB4F, otf="latn", description = "Alphabetic Presentation Forms" }, + ["ancientgreekmusicalnotation"] = { first = 0x1D200, last = 0x1D24F, otf="grek", description = "Ancient Greek Musical Notation" }, + ["ancientgreeknumbers"] = { first = 0x10140, last = 0x1018F, otf="grek", description = "Ancient Greek Numbers" }, + ["ancientsymbols"] = { first = 0x10190, last = 0x101CF, otf="grek", description = "Ancient Symbols" }, + ["arabic"] = { first = 0x00600, last = 0x006FF, otf="arab", description = "Arabic" }, + ["arabicextendeda"] = { first = 0x008A0, last = 0x008FF, description = "Arabic Extended-A" }, + ["arabicmathematicalalphabeticsymbols"] = { first = 0x1EE00, last = 0x1EEFF, description = "Arabic Mathematical Alphabetic Symbols" }, + ["arabicpresentationformsa"] = { first = 0x0FB50, last = 0x0FDFF, otf="arab", description = "Arabic Presentation Forms-A" }, + ["arabicpresentationformsb"] = { first = 0x0FE70, last = 0x0FEFF, otf="arab", description = "Arabic Presentation Forms-B" }, + ["arabicsupplement"] = { first = 0x00750, last = 0x0077F, otf="arab", description = "Arabic Supplement" }, + ["armenian"] = { first = 0x00530, last = 0x0058F, otf="armn", description = "Armenian" }, + ["arrows"] = { first = 0x02190, last = 0x021FF, description = "Arrows" }, + ["avestan"] = { first = 0x10B00, last = 0x10B3F, description = "Avestan" }, + ["balinese"] = { first = 0x01B00, last = 0x01B7F, otf="bali", description = "Balinese" }, + ["bamum"] = { first = 0x0A6A0, last = 0x0A6FF, description = "Bamum" }, + ["bamumsupplement"] = { first = 0x16800, last = 0x16A3F, description = "Bamum Supplement" }, + ["basiclatin"] = { first = 0x00000, last = 0x0007F, otf="latn", description = "Basic Latin" }, + ["batak"] = { first = 0x01BC0, last = 0x01BFF, description = "Batak" }, + ["bengali"] = { first = 0x00980, last = 0x009FF, otf="beng", description = "Bengali" }, + ["blockelements"] = { first = 0x02580, last = 0x0259F, otf="bopo", description = "Block Elements" }, + ["bopomofo"] = { first = 0x03100, last = 0x0312F, otf="bopo", description = "Bopomofo" }, + ["bopomofoextended"] = { first = 0x031A0, last = 0x031BF, otf="bopo", description = "Bopomofo Extended" }, + ["boxdrawing"] = { first = 0x02500, last = 0x0257F, description = "Box Drawing" }, + ["brahmi"] = { first = 0x11000, last = 0x1107F, description = "Brahmi" }, + ["braillepatterns"] = { first = 0x02800, last = 0x028FF, otf="brai", description = "Braille Patterns" }, + ["buginese"] = { first = 0x01A00, last = 0x01A1F, otf="bugi", description = "Buginese" }, + ["buhid"] = { first = 0x01740, last = 0x0175F, otf="buhd", description = "Buhid" }, + ["byzantinemusicalsymbols"] = { first = 0x1D000, last = 0x1D0FF, otf="byzm", description = "Byzantine Musical Symbols" }, + ["commonindicnumberforms"] = { first = 0x0A830, last = 0x0A83F, description = "Common Indic Number Forms" }, + ["carian"] = { first = 0x102A0, last = 0x102DF, description = "Carian" }, + ["cham"] = { first = 0x0AA00, last = 0x0AA5F, description = "Cham" }, + ["cherokee"] = { first = 0x013A0, last = 0x013FF, otf="cher", description = "Cherokee" }, + ["cjkcompatibility"] = { first = 0x03300, last = 0x033FF, otf="hang", description = "CJK Compatibility" }, + ["cjkcompatibilityforms"] = { first = 0x0FE30, last = 0x0FE4F, otf="hang", description = "CJK Compatibility Forms" }, + ["cjkcompatibilityideographs"] = { first = 0x0F900, last = 0x0FAFF, otf="hang", description = "CJK Compatibility Ideographs" }, + ["cjkcompatibilityideographssupplement"] = { first = 0x2F800, last = 0x2FA1F, otf="hang", description = "CJK Compatibility Ideographs Supplement" }, + ["cjkradicalssupplement"] = { first = 0x02E80, last = 0x02EFF, otf="hang", description = "CJK Radicals Supplement" }, + ["cjkstrokes"] = { first = 0x031C0, last = 0x031EF, otf="hang", description = "CJK Strokes" }, + ["cjksymbolsandpunctuation"] = { first = 0x03000, last = 0x0303F, otf="hang", description = "CJK Symbols and Punctuation" }, + ["cjkunifiedideographs"] = { first = 0x04E00, last = 0x09FFF, otf="hang", description = "CJK Unified Ideographs" }, + ["cjkunifiedideographsextensiona"] = { first = 0x03400, last = 0x04DBF, otf="hang", description = "CJK Unified Ideographs Extension A" }, + ["cjkunifiedideographsextensionb"] = { first = 0x20000, last = 0x2A6DF, otf="hang", description = "CJK Unified Ideographs Extension B" }, + ["combiningdiacriticalmarks"] = { first = 0x00300, last = 0x0036F, description = "Combining Diacritical Marks" }, + ["combiningdiacriticalmarksforsymbols"] = { first = 0x020D0, last = 0x020FF, description = "Combining Diacritical Marks for Symbols" }, + ["combiningdiacriticalmarkssupplement"] = { first = 0x01DC0, last = 0x01DFF, description = "Combining Diacritical Marks Supplement" }, + ["combininghalfmarks"] = { first = 0x0FE20, last = 0x0FE2F, description = "Combining Half Marks" }, + ["controlpictures"] = { first = 0x02400, last = 0x0243F, description = "Control Pictures" }, + ["coptic"] = { first = 0x02C80, last = 0x02CFF, otf="copt", description = "Coptic" }, + ["countingrodnumerals"] = { first = 0x1D360, last = 0x1D37F, description = "Counting Rod Numerals" }, + ["cuneiform"] = { first = 0x12000, last = 0x123FF, otf="xsux", description = "Cuneiform" }, + ["cuneiformnumbersandpunctuation"] = { first = 0x12400, last = 0x1247F, otf="xsux", description = "Cuneiform Numbers and Punctuation" }, + ["currencysymbols"] = { first = 0x020A0, last = 0x020CF, description = "Currency Symbols" }, + ["cypriotsyllabary"] = { first = 0x10800, last = 0x1083F, otf="cprt", description = "Cypriot Syllabary" }, + ["cyrillic"] = { first = 0x00400, last = 0x004FF, otf="cyrl", description = "Cyrillic" }, + ["cyrillicextendeda"] = { first = 0x02DE0, last = 0x02DFF, otf="cyrl", description = "Cyrillic Extended-A" }, + ["cyrillicextendedb"] = { first = 0x0A640, last = 0x0A69F, otf="cyrl", description = "Cyrillic Extended-B" }, + ["cyrillicsupplement"] = { first = 0x00500, last = 0x0052F, otf="cyrl", description = "Cyrillic Supplement" }, + ["deseret"] = { first = 0x10400, last = 0x1044F, otf="dsrt", description = "Deseret" }, + ["devanagari"] = { first = 0x00900, last = 0x0097F, otf="deva", description = "Devanagari" }, + ["devanagariextended"] = { first = 0x0A8E0, last = 0x0A8FF, description = "Devanagari Extended" }, + ["dingbats"] = { first = 0x02700, last = 0x027BF, description = "Dingbats" }, + ["dominotiles"] = { first = 0x1F030, last = 0x1F09F, description = "Domino Tiles" }, + ["egyptianhieroglyphs"] = { first = 0x13000, last = 0x1342F, description = "Egyptian Hieroglyphs" }, + ["emoticons"] = { first = 0x1F600, last = 0x1F64F, description = "Emoticons" }, + ["enclosedalphanumericsupplement"] = { first = 0x1F100, last = 0x1F1FF, description = "Enclosed Alphanumeric Supplement" }, + ["enclosedalphanumerics"] = { first = 0x02460, last = 0x024FF, description = "Enclosed Alphanumerics" }, + ["enclosedcjklettersandmonths"] = { first = 0x03200, last = 0x032FF, description = "Enclosed CJK Letters and Months" }, + ["enclosedideographicsupplement"] = { first = 0x1F200, last = 0x1F2FF, description = "Enclosed Ideographic Supplement" }, + ["ethiopic"] = { first = 0x01200, last = 0x0137F, otf="ethi", description = "Ethiopic" }, + ["ethiopicextended"] = { first = 0x02D80, last = 0x02DDF, otf="ethi", description = "Ethiopic Extended" }, + ["ethiopicextendeda"] = { first = 0x0AB00, last = 0x0AB2F, description = "Ethiopic Extended-A" }, + ["ethiopicsupplement"] = { first = 0x01380, last = 0x0139F, otf="ethi", description = "Ethiopic Supplement" }, + ["generalpunctuation"] = { first = 0x02000, last = 0x0206F, description = "General Punctuation" }, + ["geometricshapes"] = { first = 0x025A0, last = 0x025FF, description = "Geometric Shapes" }, + ["georgian"] = { first = 0x010A0, last = 0x010FF, otf="geor", description = "Georgian" }, + ["georgiansupplement"] = { first = 0x02D00, last = 0x02D2F, otf="geor", description = "Georgian Supplement" }, + ["glagolitic"] = { first = 0x02C00, last = 0x02C5F, otf="glag", description = "Glagolitic" }, + ["gothic"] = { first = 0x10330, last = 0x1034F, otf="goth", description = "Gothic" }, + ["greekandcoptic"] = { first = 0x00370, last = 0x003FF, otf="grek", description = "Greek and Coptic" }, + ["greekextended"] = { first = 0x01F00, last = 0x01FFF, otf="grek", description = "Greek Extended" }, + ["gujarati"] = { first = 0x00A80, last = 0x00AFF, otf="gujr", description = "Gujarati" }, + ["gurmukhi"] = { first = 0x00A00, last = 0x00A7F, otf="guru", description = "Gurmukhi" }, + ["halfwidthandfullwidthforms"] = { first = 0x0FF00, last = 0x0FFEF, description = "Halfwidth and Fullwidth Forms" }, + ["hangulcompatibilityjamo"] = { first = 0x03130, last = 0x0318F, otf="jamo", description = "Hangul Compatibility Jamo" }, + ["hanguljamo"] = { first = 0x01100, last = 0x011FF, otf="jamo", description = "Hangul Jamo" }, + ["hanguljamoextendeda"] = { first = 0x0A960, last = 0x0A97F, description = "Hangul Jamo Extended-A" }, + ["hanguljamoextendedb"] = { first = 0x0D7B0, last = 0x0D7FF, description = "Hangul Jamo Extended-B" }, + ["hangulsyllables"] = { first = 0x0AC00, last = 0x0D7AF, otf="hang", description = "Hangul Syllables" }, + ["hanunoo"] = { first = 0x01720, last = 0x0173F, otf="hano", description = "Hanunoo" }, + ["hebrew"] = { first = 0x00590, last = 0x005FF, otf="hebr", description = "Hebrew" }, + ["highprivateusesurrogates"] = { first = 0x0DB80, last = 0x0DBFF, description = "High Private Use Surrogates" }, + ["highsurrogates"] = { first = 0x0D800, last = 0x0DB7F, description = "High Surrogates" }, + ["hiragana"] = { first = 0x03040, last = 0x0309F, otf="kana", description = "Hiragana" }, + ["ideographicdescriptioncharacters"] = { first = 0x02FF0, last = 0x02FFF, description = "Ideographic Description Characters" }, + ["imperialaramaic"] = { first = 0x10840, last = 0x1085F, description = "Imperial Aramaic" }, + ["inscriptionalpahlavi"] = { first = 0x10B60, last = 0x10B7F, description = "Inscriptional Pahlavi" }, + ["inscriptionalparthian"] = { first = 0x10B40, last = 0x10B5F, description = "Inscriptional Parthian" }, + ["ipaextensions"] = { first = 0x00250, last = 0x002AF, description = "IPA Extensions" }, + ["javanese"] = { first = 0x0A980, last = 0x0A9DF, description = "Javanese" }, + ["kaithi"] = { first = 0x11080, last = 0x110CF, description = "Kaithi" }, + ["kanasupplement"] = { first = 0x1B000, last = 0x1B0FF, description = "Kana Supplement" }, + ["kanbun"] = { first = 0x03190, last = 0x0319F, description = "Kanbun" }, + ["kangxiradicals"] = { first = 0x02F00, last = 0x02FDF, description = "Kangxi Radicals" }, + ["kannada"] = { first = 0x00C80, last = 0x00CFF, otf="knda", description = "Kannada" }, + ["katakana"] = { first = 0x030A0, last = 0x030FF, otf="kana", description = "Katakana" }, + ["katakanaphoneticextensions"] = { first = 0x031F0, last = 0x031FF, otf="kana", description = "Katakana Phonetic Extensions" }, + ["kayahli"] = { first = 0x0A900, last = 0x0A92F, description = "Kayah Li" }, + ["kharoshthi"] = { first = 0x10A00, last = 0x10A5F, otf="khar", description = "Kharoshthi" }, + ["khmer"] = { first = 0x01780, last = 0x017FF, otf="khmr", description = "Khmer" }, + ["khmersymbols"] = { first = 0x019E0, last = 0x019FF, otf="khmr", description = "Khmer Symbols" }, + ["lao"] = { first = 0x00E80, last = 0x00EFF, otf="lao", description = "Lao" }, + ["latinextendeda"] = { first = 0x00100, last = 0x0017F, otf="latn", description = "Latin Extended-A" }, + ["latinextendedadditional"] = { first = 0x01E00, last = 0x01EFF, otf="latn", description = "Latin Extended Additional" }, + ["latinextendedb"] = { first = 0x00180, last = 0x0024F, otf="latn", description = "Latin Extended-B" }, + ["latinextendedc"] = { first = 0x02C60, last = 0x02C7F, otf="latn", description = "Latin Extended-C" }, + ["latinextendedd"] = { first = 0x0A720, last = 0x0A7FF, otf="latn", description = "Latin Extended-D" }, + ["latinsupplement"] = { first = 0x00080, last = 0x000FF, otf="latn", description = "Latin-1 Supplement" }, + ["lepcha"] = { first = 0x01C00, last = 0x01C4F, description = "Lepcha" }, + ["letterlikesymbols"] = { first = 0x02100, last = 0x0214F, description = "Letterlike Symbols" }, + ["limbu"] = { first = 0x01900, last = 0x0194F, otf="limb", description = "Limbu" }, + ["linearbideograms"] = { first = 0x10080, last = 0x100FF, otf="linb", description = "Linear B Ideograms" }, + ["linearbsyllabary"] = { first = 0x10000, last = 0x1007F, otf="linb", description = "Linear B Syllabary" }, + ["lisu"] = { first = 0x0A4D0, last = 0x0A4FF, description = "Lisu" }, + ["lowsurrogates"] = { first = 0x0DC00, last = 0x0DFFF, description = "Low Surrogates" }, + ["lycian"] = { first = 0x10280, last = 0x1029F, description = "Lycian" }, + ["lydian"] = { first = 0x10920, last = 0x1093F, description = "Lydian" }, + ["mahjongtiles"] = { first = 0x1F000, last = 0x1F02F, description = "Mahjong Tiles" }, + ["malayalam"] = { first = 0x00D00, last = 0x00D7F, otf="mlym", description = "Malayalam" }, + ["mandiac"] = { first = 0x00840, last = 0x0085F, otf="mand", description = "Mandaic" }, + ["mathematicalalphanumericsymbols"] = { first = 0x1D400, last = 0x1D7FF, description = "Mathematical Alphanumeric Symbols" }, + ["mathematicaloperators"] = { first = 0x02200, last = 0x022FF, description = "Mathematical Operators" }, + ["meeteimayek"] = { first = 0x0ABC0, last = 0x0ABFF, description = "Meetei Mayek" }, + ["meeteimayekextensions"] = { first = 0x0AAE0, last = 0x0AAFF, description = "Meetei Mayek Extensions" }, + ["meroiticcursive"] = { first = 0x109A0, last = 0x109FF, description = "Meroitic Cursive" }, + ["meroitichieroglyphs"] = { first = 0x10980, last = 0x1099F, description = "Meroitic Hieroglyphs" }, + ["miao"] = { first = 0x16F00, last = 0x16F9F, description = "Miao" }, + ["miscellaneousmathematicalsymbolsa"] = { first = 0x027C0, last = 0x027EF, description = "Miscellaneous Mathematical Symbols-A" }, + ["miscellaneousmathematicalsymbolsb"] = { first = 0x02980, last = 0x029FF, description = "Miscellaneous Mathematical Symbols-B" }, + ["miscellaneoussymbols"] = { first = 0x02600, last = 0x026FF, description = "Miscellaneous Symbols" }, + ["miscellaneoussymbolsandarrows"] = { first = 0x02B00, last = 0x02BFF, description = "Miscellaneous Symbols and Arrows" }, + ["miscellaneoussymbolsandpictographs"] = { first = 0x1F300, last = 0x1F5FF, description = "Miscellaneous Symbols And Pictographs" }, + ["miscellaneoustechnical"] = { first = 0x02300, last = 0x023FF, description = "Miscellaneous Technical" }, + ["modifiertoneletters"] = { first = 0x0A700, last = 0x0A71F, description = "Modifier Tone Letters" }, + ["mongolian"] = { first = 0x01800, last = 0x018AF, otf="mong", description = "Mongolian" }, + ["musicalsymbols"] = { first = 0x1D100, last = 0x1D1FF, otf="musc", description = "Musical Symbols" }, + ["myanmar"] = { first = 0x01000, last = 0x0109F, otf="mymr", description = "Myanmar" }, + ["myanmarextendeda"] = { first = 0x0AA60, last = 0x0AA7F, description = "Myanmar Extended-A" }, + ["newtailue"] = { first = 0x01980, last = 0x019DF, description = "New Tai Lue" }, + ["nko"] = { first = 0x007C0, last = 0x007FF, otf="nko", description = "NKo" }, + ["numberforms"] = { first = 0x02150, last = 0x0218F, description = "Number Forms" }, + ["ogham"] = { first = 0x01680, last = 0x0169F, otf="ogam", description = "Ogham" }, + ["olchiki"] = { first = 0x01C50, last = 0x01C7F, description = "Ol Chiki" }, + ["olditalic"] = { first = 0x10300, last = 0x1032F, otf="ital", description = "Old Italic" }, + ["oldpersian"] = { first = 0x103A0, last = 0x103DF, otf="xpeo", description = "Old Persian" }, + ["oldsoutharabian"] = { first = 0x10A60, last = 0x10A7F, description = "Old South Arabian" }, + ["odlturkic"] = { first = 0x10C00, last = 0x10C4F, description = "Old Turkic" }, + ["opticalcharacterrecognition"] = { first = 0x02440, last = 0x0245F, description = "Optical Character Recognition" }, + ["oriya"] = { first = 0x00B00, last = 0x00B7F, otf="orya", description = "Oriya" }, + ["osmanya"] = { first = 0x10480, last = 0x104AF, otf="osma", description = "Osmanya" }, + ["phagspa"] = { first = 0x0A840, last = 0x0A87F, otf="phag", description = "Phags-pa" }, + ["phaistosdisc"] = { first = 0x101D0, last = 0x101FF, description = "Phaistos Disc" }, + ["phoenician"] = { first = 0x10900, last = 0x1091F, otf="phnx", description = "Phoenician" }, + ["phoneticextensions"] = { first = 0x01D00, last = 0x01D7F, description = "Phonetic Extensions" }, + ["phoneticextensionssupplement"] = { first = 0x01D80, last = 0x01DBF, description = "Phonetic Extensions Supplement" }, + ["playingcards"] = { first = 0x1F0A0, last = 0x1F0FF, description = "Playing Cards" }, + ["privateusearea"] = { first = 0x0E000, last = 0x0F8FF, description = "Private Use Area" }, + ["rejang"] = { first = 0x0A930, last = 0x0A95F, description = "Rejang" }, + ["ruminumeralsymbols"] = { first = 0x10E60, last = 0x10E7F, description = "Rumi Numeral Symbols" }, + ["runic"] = { first = 0x016A0, last = 0x016FF, otf="runr", description = "Runic" }, + ["samaritan"] = { first = 0x00800, last = 0x0083F, description = "Samaritan" }, + ["saurashtra"] = { first = 0x0A880, last = 0x0A8DF, description = "Saurashtra" }, + ["sharada"] = { first = 0x11180, last = 0x111DF, description = "Sharada" }, + ["shavian"] = { first = 0x10450, last = 0x1047F, otf="shaw", description = "Shavian" }, + ["sinhala"] = { first = 0x00D80, last = 0x00DFF, otf="sinh", description = "Sinhala" }, + ["smallformvariants"] = { first = 0x0FE50, last = 0x0FE6F, description = "Small Form Variants" }, + ["sorasompeng"] = { first = 0x110D0, last = 0x110FF, description = "Sora Sompeng" }, + ["spacingmodifierletters"] = { first = 0x002B0, last = 0x002FF, description = "Spacing Modifier Letters" }, + ["specials"] = { first = 0x0FFF0, last = 0x0FFFF, description = "Specials" }, + ["sundanese"] = { first = 0x01B80, last = 0x01BBF, description = "Sundanese" }, + ["sundanesesupplement"] = { first = 0x01CC0, last = 0x01CCF, description = "Sundanese Supplement" }, + ["superscriptsandsubscripts"] = { first = 0x02070, last = 0x0209F, description = "Superscripts and Subscripts" }, + ["supplementalarrowsa"] = { first = 0x027F0, last = 0x027FF, description = "Supplemental Arrows-A" }, + ["supplementalarrowsb"] = { first = 0x02900, last = 0x0297F, description = "Supplemental Arrows-B" }, + ["supplementalmathematicaloperators"] = { first = 0x02A00, last = 0x02AFF, description = "Supplemental Mathematical Operators" }, + ["supplementalpunctuation"] = { first = 0x02E00, last = 0x02E7F, description = "Supplemental Punctuation" }, + ["supplementaryprivateuseareaa"] = { first = 0xF0000, last = 0xFFFFF, description = "Supplementary Private Use Area-A" }, + ["supplementaryprivateuseareab"] = { first = 0x100000,last = 0x10FFFF, description = "Supplementary Private Use Area-B" }, + ["sylotinagri"] = { first = 0x0A800, last = 0x0A82F, otf="sylo", description = "Syloti Nagri" }, + ["syriac"] = { first = 0x00700, last = 0x0074F, otf="syrc", description = "Syriac" }, + ["tagalog"] = { first = 0x01700, last = 0x0171F, otf="tglg", description = "Tagalog" }, + ["tagbanwa"] = { first = 0x01760, last = 0x0177F, otf="tagb", description = "Tagbanwa" }, + ["tags"] = { first = 0xE0000, last = 0xE007F, description = "Tags" }, + ["taile"] = { first = 0x01950, last = 0x0197F, otf="tale", description = "Tai Le" }, + ["taitham"] = { first = 0x01A20, last = 0x01AAF, description = "Tai Tham" }, + ["taiviet"] = { first = 0x0AA80, last = 0x0AADF, description = "Tai Viet" }, + ["taixuanjingsymbols"] = { first = 0x1D300, last = 0x1D35F, description = "Tai Xuan Jing Symbols" }, + ["takri"] = { first = 0x11680, last = 0x116CF, description = "Takri" }, + ["tamil"] = { first = 0x00B80, last = 0x00BFF, otf="taml", description = "Tamil" }, + ["telugu"] = { first = 0x00C00, last = 0x00C7F, otf="telu", description = "Telugu" }, + ["thaana"] = { first = 0x00780, last = 0x007BF, otf="thaa", description = "Thaana" }, + ["thai"] = { first = 0x00E00, last = 0x00E7F, otf="thai", description = "Thai" }, + ["tibetan"] = { first = 0x00F00, last = 0x00FFF, otf="tibt", description = "Tibetan" }, + ["tifinagh"] = { first = 0x02D30, last = 0x02D7F, otf="tfng", description = "Tifinagh" }, + ["transportandmapsymbols"] = { first = 0x1F680, last = 0x1F6FF, description = "Transport And Map Symbols" }, + ["ugaritic"] = { first = 0x10380, last = 0x1039F, otf="ugar", description = "Ugaritic" }, + ["unifiedcanadianaboriginalsyllabics"] = { first = 0x01400, last = 0x0167F, otf="cans", description = "Unified Canadian Aboriginal Syllabics" }, + ["unifiedcanadianaboriginalsyllabicsextended"] = { first = 0x018B0, last = 0x018FF, description = "Unified Canadian Aboriginal Syllabics Extended" }, + ["vai"] = { first = 0x0A500, last = 0x0A63F, description = "Vai" }, + ["variationselectors"] = { first = 0x0FE00, last = 0x0FE0F, description = "Variation Selectors" }, + ["variationselectorssupplement"] = { first = 0xE0100, last = 0xE01EF, description = "Variation Selectors Supplement" }, + ["vedicextensions"] = { first = 0x01CD0, last = 0x01CFF, description = "Vedic Extensions" }, + ["verticalforms"] = { first = 0x0FE10, last = 0x0FE1F, description = "Vertical Forms" }, + ["yijinghexagramsymbols"] = { first = 0x04DC0, last = 0x04DFF, otf="yi", description = "Yijing Hexagram Symbols" }, + ["yiradicals"] = { first = 0x0A490, last = 0x0A4CF, otf="yi", description = "Yi Radicals" }, + ["yisyllables"] = { first = 0x0A000, last = 0x0A48F, otf="yi", description = "Yi Syllables" }, } characters.blocks = blocks -setmetatableindex(blocks, function(t,k) +function characters.blockrange(name) + local b = blocks[name] + if b then + return b.first, b.last + else + return 0, 0 + end +end + +setmetatableindex(blocks, function(t,k) -- we could use an intermediate table if called often return k and rawget(t,lower(gsub(k,"[^a-zA-Z]",""))) end) +local otfscripts = utilities.storage.allocate() +characters.otfscripts = otfscripts + +setmetatableindex(otfscripts,function(t,unicode) + for k, v in next, blocks do + local first, last = v.first, v.last + if unicode >= first and unicode <= last then + local script = v.otf or "dflt" + for u=first,last do + t[u] = script + end + return script + end + end + -- pretty slow when we're here + t[unicode] = "dflt" + return "dflt" +end) + function characters.getrange(name) -- used in font fallback definitions (name or range) local range = blocks[name] if range then @@ -395,6 +459,8 @@ local is_mark = allocate ( tohash { "mn", "ms", } ) +-- to be redone: store checked characters + characters.is_character = is_character characters.is_letter = is_letter characters.is_command = is_command @@ -404,7 +470,7 @@ characters.is_mark = is_mark local mt = { -- yes or no ? __index = function(t,k) if type(k) == "number" then - local c = characters.data[k].category + local c = data[k].category return c and rawget(t,c) else -- avoid auto conversion in data.characters lookups @@ -455,213 +521,47 @@ table we derive a few more.</p> if not characters.fallbacks then - -- we could the definition by using a metatable + characters.fallbacks = { } -- not than many - characters.fallbacks = { } - characters.directions = { } + local fallbacks = characters.fallbacks - local fallbacks = characters.fallbacks - local directions = characters.directions - - for k,v in next, data do - local specials = v.specials - if specials and specials[1] == "compat" and specials[2] == 0x0020 and specials[3] then + for k, d in next, data do + local specials = d.specials + if specials and specials[1] == "compat" and specials[2] == 0x0020 then local s = specials[3] - fallbacks[k] = s - fallbacks[s] = k - end - directions[k] = v.direction - end - -end - -storage.register("characters/fallbacks", characters.fallbacks, "characters.fallbacks") -- accents and such -storage.register("characters/directions", characters.directions, "characters.directions") - ---[[ldx-- -<p>The <type>context</type> namespace is used to store methods and data -which is rather specific to <l n='context'/>.</p> ---ldx]]-- - ---[[ldx-- -<p>Instead of using a <l n='tex'/> file to define the named glyphs, we -use the table. After all, we have this information available anyway.</p> ---ldx]]-- - -function characters.makeactive(n,name) -- let ? - contextsprint(ctxcatcodes,format("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name)) - -- context("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name) -end - -function tex.uprint(c,n) - if n then - -- contextsprint(c,charfromnumber(n)) - contextsprint(c,utfchar(n)) - else - -- contextsprint(charfromnumber(c)) - contextsprint(utfchar(c)) - end -end - -local temphack = tohash { - 0x00A0, - 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x200B, 0x200C, 0x200D, - 0x202F, - 0x205F, - -- 0xFEFF, -} - -function characters.define(tobelettered, tobeactivated) -- catcodetables - - if trace_defining then - report_defining("defining active character commands") - end - - local activated, a = { }, 0 - - for u, chr in next, data do -- these will be commands - local fallback = chr.fallback - if fallback then - contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\checkedchar{",u,"}{",fallback,"}}}") - a = a + 1 - activated[a] = u - else - local contextname = chr.contextname - if contextname then - local category = chr.category - if is_character[category] then - if chr.unicodeslot < 128 then - if is_letter[category] then - contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s - else - contextsprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u)) -- has no s - end - else - contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s - end - elseif is_command[category] then -if not temphack[u] then - contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\"..contextname,"}}") - a = a + 1 - activated[a] = u -end - end + if s then + fallbacks[k] = s + fallbacks[s] = k end end end - if tobelettered then -- shared - local saved = tex.catcodetable - for i=1,#tobelettered do - tex.catcodetable = tobelettered[i] - if trace_defining then - report_defining("defining letters (global, shared)") - end - for u, chr in next, data do - if not chr.fallback and is_letter[chr.category] and u >= 128 and u <= 65536 then - texsetcatcode(u,11) - end - local range = chr.range - if range then - for i=1,range.first,range.last do - texsetcatcode(i,11) - end - end - end - texsetcatcode(0x200C,11) -- non-joiner - texsetcatcode(0x200D,11) -- joiner - end - tex.catcodetable = saved - end - - local nofactivated = #tobeactivated - if tobeactivated and nofactivated > 0 then - for i=1,nofactivated do - local u = activated[i] - if u then - report_defining("character 0x%05X is active in sets %s (%s)",u,concat(tobeactivated,","),data[u].description) - end - end - local saved = tex.catcodetable - for i=1,#tobeactivated do - local vector = tobeactivated[i] - if trace_defining then - report_defining("defining %s active characters in vector %s",nofactivated,vector) - end - tex.catcodetable = vector - for i=1,nofactivated do - local u = activated[i] - if u then - texsetcatcode(u,13) - end - end - end - tex.catcodetable = saved - end - end ---[[ldx-- -<p>Setting the lccodes is also done in a loop over the data table.</p> ---ldx]]-- - ---~ function tex.setsfcode (index,sf) ... end ---~ function tex.setlccode (index,lc,[uc]) ... end -- optional third value, safes call ---~ function tex.setuccode (index,uc,[lc]) ... end ---~ function tex.setcatcode(index,cc) ... end - --- we need a function ... +if storage then + storage.register("characters/fallbacks", characters.fallbacks, "characters.fallbacks") -- accents and such +end ---~ tex.lccode ---~ tex.uccode ---~ tex.sfcode ---~ tex.catcode +characters.directions = { } -function characters.setcodes() - if trace_defining then - report_defining("defining lc and uc codes") - end - for code, chr in next, data do - local cc = chr.category - if is_letter[cc] then - local range = chr.range - if range then - for i=range.first,range.last do - texsetcatcode(i,11) -- letter - texsetlccode(i,i,i) -- self self - end - else - local lc, uc = chr.lccode, chr.uccode - if not lc then chr.lccode, lc = code, code end - if not uc then chr.uccode, uc = code, code end - texsetcatcode(code,11) -- letter - if type(lc) == "table" then - lc = code - end - if type(uc) == "table" then - uc = code - end - texsetlccode(code,lc,uc) - if cc == "lu" then - texsetsfcode(code,999) - end - end - elseif is_mark[cc] then - texsetlccode(code,code,code) -- for hyphenation +setmetatableindex(characters.directions,function(t,k) + local d = data[k] + if d then + local v = d.direction + if v then + t[k] = v + return v end end -end + t[k] = false -- maybe 'l' + return v +end) --[[ldx-- <p>Next comes a whole series of helper methods. These are (will be) part of the official <l n='api'/>.</p> --ldx]]-- ---[[ldx-- -<p>A couple of convenience methods. Beware, these are slower than directly -accessing the data table.</p> ---ldx]]-- - -- we could make them virtual: characters.contextnames[n] function characters.contextname(n) return data[n].contextname or "" end @@ -680,35 +580,17 @@ function characters.category(n,verbose) end end --- xml support (moved) - -function characters.remapentity(chr,slot) - contextsprint(format("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr)) -end - -characters.activeoffset = 0x10000 -- there will be remapped in that byte range - --- xml.entities = xml.entities or { } +-- -- some day we will make a table .. not that many calls to utfchar -- --- storage.register("xml/entities",xml.entities,"xml.entities") -- this will move to lxml +-- local utfchar = utf.char +-- local utfbyte = utf.byte +-- local utfbytes = { } +-- local utfchars = { } -- --- function characters.setmkiventities() --- local entities = xml.entities --- entities.lt = "<" --- entities.amp = "&" --- entities.gt = ">" --- end --- --- function characters.setmkiientities() --- local entities = xml.entities --- entities.lt = utfchar(characters.activeoffset + utfbyte("<")) --- entities.amp = utfchar(characters.activeoffset + utfbyte("&")) --- entities.gt = utfchar(characters.activeoffset + utfbyte(">")) --- end - --- some day we will make a table +-- table.setmetatableindex(utfbytes,function(t,k) local v = utfchar(k) t[k] = v return v end) +-- table.setmetatableindex(utfchars,function(t,k) local v = utfbyte(k) t[k] = v return v end) -local function utfstring(s) +local function toutfstring(s) if type(s) == "table" then return utfchar(unpack(s)) -- concat { utfchar( unpack(s) ) } else @@ -716,7 +598,7 @@ local function utfstring(s) end end -utf.string = utf.string or utfstring +utf.tostring = toutfstring local categories = allocate() characters.categories = categories -- lazy table @@ -737,10 +619,10 @@ local ucchars = allocate() characters.ucchars = ucchars -- lazy table local shchars = allocate() characters.shchars = shchars -- lazy table local fschars = allocate() characters.fschars = fschars -- lazy table -setmetatableindex(lcchars, function(t,u) if u then local c = data[u] c = c and c.lccode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) -setmetatableindex(ucchars, function(t,u) if u then local c = data[u] c = c and c.uccode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) -setmetatableindex(shchars, function(t,u) if u then local c = data[u] c = c and c.shcode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) -setmetatableindex(fschars, function(t,u) if u then local c = data[u] c = c and c.fscode c = c and utfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) +setmetatableindex(lcchars, function(t,u) if u then local c = data[u] c = c and c.lccode c = c and toutfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) +setmetatableindex(ucchars, function(t,u) if u then local c = data[u] c = c and c.uccode c = c and toutfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) +setmetatableindex(shchars, function(t,u) if u then local c = data[u] c = c and c.shcode c = c and toutfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) +setmetatableindex(fschars, function(t,u) if u then local c = data[u] c = c and c.fscode c = c and toutfstring(c) or (type(u) == "number" and utfchar(u)) or u t[u] = c return c end end) local decomposed = allocate() characters.decomposed = decomposed -- lazy table local specials = allocate() characters.specials = specials -- lazy table @@ -815,37 +697,52 @@ function characters.unicodechar(asked) if n then return n elseif type(asked) == "string" then - asked = gsub(asked," ","") - return descriptions[asked] + return descriptions[asked] or descriptions[gsub(asked," ","")] end end -function characters.lower(str) - local new, n = { }, 0 - for u in utfvalues(str) do - n = n + 1 - new[n] = lcchars[u] - end - return concat(new) -end +-- function characters.lower(str) +-- local new, n = { }, 0 +-- for u in utfvalues(str) do +-- n = n + 1 +-- new[n] = lcchars[u] +-- end +-- return concat(new) +-- end +-- +-- function characters.upper(str) +-- local new, n = { }, 0 +-- for u in utfvalues(str) do +-- n = n + 1 +-- new[n] = ucchars[u] +-- end +-- return concat(new) +-- end +-- +-- function characters.shaped(str) +-- local new, n = { }, 0 +-- for u in utfvalues(str) do +-- n = n + 1 +-- new[n] = shchars[u] +-- end +-- return concat(new) +-- end -function characters.upper(str) - local new, n = { }, 0 - for u in utfvalues(str) do - n = n + 1 - new[n] = ucchars[u] - end - return concat(new) -end +----- tolower = Cs((utf8byte/lcchars)^0) +----- toupper = Cs((utf8byte/ucchars)^0) +----- toshape = Cs((utf8byte/shchars)^0) -function characters.shaped(str) - local new, n = { }, 0 - for u in utfvalues(str) do - n = n + 1 - new[n] = shchars[u] - end - return concat(new) -end +local tolower = Cs((utf8char/lcchars)^0) +local toupper = Cs((utf8char/ucchars)^0) +local toshape = Cs((utf8char/shchars)^0) + +patterns.tolower = tolower +patterns.toupper = toupper +patterns.toshape = toshape + +function characters.lower (str) return lpegmatch(tolower,str) end +function characters.upper (str) return lpegmatch(toupper,str) end +function characters.shaped(str) return lpegmatch(toshape,str) end function characters.lettered(str,spacing) local new, n = { }, 0 @@ -875,6 +772,7 @@ function characters.lettered(str,spacing) end return concat(new) end + --[[ldx-- <p>Requesting lower and uppercase codes:</p> --ldx]]-- @@ -882,17 +780,12 @@ end function characters.uccode(n) return uccodes[n] end -- obsolete function characters.lccode(n) return lccodes[n] end -- obsolete -function characters.flush(n,direct) +function characters.safechar(n) local c = data[n] if c and c.contextname then - c = "\\" .. c.contextname - else - c = utfchar(n) - end - if direct then - return c + return "\\" .. c.contextname else - contextsprint(c) + return utfchar(n) end end @@ -996,13 +889,13 @@ if not characters.superscripts then if #specials == 2 then superscripts[k] = specials[2] else - report_defining("ignoring superscript %s %s: %s",ustring(k),utfchar(k),v.description) + report_defining("ignoring %s %a, char %c, description %a","superscript",ustring(k),k,v.description) end elseif what == "sub" then if #specials == 2 then subscripts[k] = specials[2] else - report_defining("ignoring subscript %s %s: %s",ustring(k),utfchar(k),v.description) + report_defining("ignoring %s %a, char %c, description %a","subscript",ustring(k),k,v.description) end end end @@ -1011,11 +904,255 @@ if not characters.superscripts then -- print(table.serialize(superscripts, "superscripts", { hexify = true })) -- print(table.serialize(subscripts, "subscripts", { hexify = true })) - storage.register("characters/superscripts", superscripts, "characters.superscripts") - storage.register("characters/subscripts", subscripts, "characters.subscripts") + if storage then + storage.register("characters/superscripts", superscripts, "characters.superscripts") + storage.register("characters/subscripts", subscripts, "characters.subscripts") + end + +end + +-- for the moment only a few + +local tracedchars = utilities.strings.tracers + +tracedchars[0x00] = "[signal]" +tracedchars[0x20] = "[space]" + +-- the following code will move to char-tex.lua + +-- tex + +if not tex or not context or not commands then return characters end + +local tex = tex +local texsetlccode = tex.setlccode +local texsetuccode = tex.setuccode +local texsetsfcode = tex.setsfcode +local texsetcatcode = tex.setcatcode + +local contextsprint = context.sprint +local ctxcatcodes = catcodes.numbers.ctxcatcodes + +--[[ldx-- +<p>Instead of using a <l n='tex'/> file to define the named glyphs, we +use the table. After all, we have this information available anyway.</p> +--ldx]]-- + +function commands.makeactive(n,name) -- + contextsprint(ctxcatcodes,format("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name)) + -- context("\\catcode%s=13\\unexpanded\\def %s{\\%s}",n,utfchar(n),name) +end + +function commands.utfchar(c,n) + if n then + -- contextsprint(c,charfromnumber(n)) + contextsprint(c,utfchar(n)) + else + -- contextsprint(charfromnumber(c)) + contextsprint(utfchar(c)) + end +end +function commands.safechar(n) + local c = data[n] + if c and c.contextname then + contextsprint("\\" .. c.contextname) -- context[c.contextname]() + else + contextsprint(utfchar(n)) + end end --- interface +tex.uprint = commands.utfchar + +local forbidden = tohash { -- at least now + 0x00A0, + 0x2000, 0x2001, 0x2002, 0x2003, 0x2004, 0x2005, 0x2006, 0x2007, 0x2008, 0x2009, 0x200A, 0x200B, 0x200C, 0x200D, + 0x202F, + 0x205F, + -- 0xFEFF, +} + +function characters.define(tobelettered, tobeactivated) -- catcodetables + + if trace_defining then + report_defining("defining active character commands") + end + + local activated, a = { }, 0 + + for u, chr in next, data do -- these will be commands + local fallback = chr.fallback + if fallback then + contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\checkedchar{",u,"}{",fallback,"}}}") + a = a + 1 + activated[a] = u + else + local contextname = chr.contextname + if contextname then + local category = chr.category + if is_character[category] then + if chr.unicodeslot < 128 then + if is_letter[category] then + contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s + else + contextsprint(ctxcatcodes,format("\\chardef\\%s=%s",contextname,u)) -- has no s + end + else + contextsprint(ctxcatcodes,format("\\def\\%s{%s}",contextname,utfchar(u))) -- has no s + end + elseif is_command[category] and not forbidden[u] then + contextsprint("{\\catcode",u,"=13\\unexpanded\\gdef ",utfchar(u),"{\\"..contextname,"}}") + a = a + 1 + activated[a] = u + end + end + end + end + + if tobelettered then -- shared + local saved = tex.catcodetable + for i=1,#tobelettered do + tex.catcodetable = tobelettered[i] + if trace_defining then + report_defining("defining letters (global, shared)") + end + for u, chr in next, data do + if not chr.fallback and is_letter[chr.category] and u >= 128 and u <= 65536 then + texsetcatcode(u,11) + end + local range = chr.range + if range then + for i=1,range.first,range.last do + texsetcatcode(i,11) + end + end + end + texsetcatcode(0x200C,11) -- non-joiner + texsetcatcode(0x200D,11) -- joiner + end + tex.catcodetable = saved + end + + local nofactivated = #tobeactivated + if tobeactivated and nofactivated > 0 then + for i=1,nofactivated do + local u = activated[i] + if u then + report_defining("character %U is active in set %a, containing %a",u,data[u].description,tobeactivated) + end + end + local saved = tex.catcodetable + for i=1,#tobeactivated do + local vector = tobeactivated[i] + if trace_defining then + report_defining("defining %a active characters in vector %a",nofactivated,vector) + end + tex.catcodetable = vector + for i=1,nofactivated do + local u = activated[i] + if u then + texsetcatcode(u,13) + end + end + end + tex.catcodetable = saved + end + +end + +--[[ldx-- +<p>Setting the lccodes is also done in a loop over the data table.</p> +--ldx]]-- + +local sfmode = "unset" -- unset, traditional, normal + +function characters.setcodes() + if trace_defining then + report_defining("defining lc and uc codes") + end + local traditional = sfstate == "traditional" or sfstate == "unset" + for code, chr in next, data do + local cc = chr.category + if is_letter[cc] then + local range = chr.range + if range then + for i=range.first,range.last do + texsetcatcode(i,11) -- letter + texsetlccode(i,i,i) -- self self + end + else + local lc, uc = chr.lccode, chr.uccode + if not lc then + chr.lccode, lc = code, code + elseif type(lc) == "table" then + lc = code + end + if not uc then + chr.uccode, uc = code, code + elseif type(uc) == "table" then + uc = code + end + texsetcatcode(code,11) -- letter + texsetlccode(code,lc,uc) + if traditional and cc == "lu" then + texsetsfcode(code,999) + end + end + elseif is_mark[cc] then + texsetlccode(code,code,code) -- for hyphenation + end + end + if traditional then + sfstate = "traditional" + end +end + +-- If this is something that is not documentwide and used a lot, then we +-- need a more clever approach (trivial but not now). + +local function setuppersfcodes(v,n) + if sfstate ~= "unset" then + report_defining("setting uppercase sf codes to %a",n) + for code, chr in next, data do + if chr.category == "lu" then + texsetsfcode(code,n) + end + end + end + sfstate = v +end + +directives.register("characters.spaceafteruppercase",function(v) + if v == "traditional" then + setuppersfcodes(v,999) + elseif v == "normal" then + setuppersfcodes(v,1000) + end +end) + +-- xml + +characters.activeoffset = 0x10000 -- there will be remapped in that byte range + +function commands.remapentity(chr,slot) + contextsprint(format("{\\catcode%s=13\\xdef%s{\\string%s}}",slot,utfchar(slot),chr)) +end + +-- xml.entities = xml.entities or { } +-- +-- storage.register("xml/entities",xml.entities,"xml.entities") -- this will move to lxml +-- +-- function characters.setmkiventities() +-- local entities = xml.entities +-- entities.lt = "<" +-- entities.amp = "&" +-- entities.gt = ">" +-- end +-- +-- function characters.setmkiientities() +-- local entities = xml.entities +-- entities.lt = utfchar(characters.activeoffset + utfbyte("<")) +-- entities.amp = utfchar(characters.activeoffset + utfbyte("&")) +-- entities.gt = utfchar(characters.activeoffset + utfbyte(">")) +-- end -commands.utfchar = tex.uprint |