diff options
author | Akira Kakuto <kakuto@fuk.kindai.ac.jp> | 2020-12-24 07:21:30 +0000 |
---|---|---|
committer | Akira Kakuto <kakuto@fuk.kindai.ac.jp> | 2020-12-24 07:21:30 +0000 |
commit | f88f99b079f2902f7705a2aff36c0ff8c13df8a8 (patch) | |
tree | 9a4408ede2a693471e9316ce59708ca0eca1b68c /Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py | |
parent | d9a6a275c51f6dcebc04e63329571f020c5de19a (diff) |
harfbuzz 2.7.3
git-svn-id: svn://tug.org/texlive/trunk@57216 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py')
-rwxr-xr-x | Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py | 248 |
1 files changed, 144 insertions, 104 deletions
diff --git a/Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py b/Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py index e4b48353f33..a1865e96770 100755 --- a/Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py +++ b/Build/source/libs/harfbuzz/harfbuzz-src/src/gen-use-table.py @@ -1,32 +1,53 @@ #!/usr/bin/env python3 # flake8: noqa: F821 -"""usage: ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt +"""usage: ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt ArabicShaping.txt Blocks.txt IndicSyllabicCategory-Additional.txt IndicPositionalCategory-Additional.txt -Input file: +Input files: * https://unicode.org/Public/UCD/latest/ucd/IndicSyllabicCategory.txt * https://unicode.org/Public/UCD/latest/ucd/IndicPositionalCategory.txt * https://unicode.org/Public/UCD/latest/ucd/UnicodeData.txt +* https://unicode.org/Public/UCD/latest/ucd/ArabicShaping.txt * https://unicode.org/Public/UCD/latest/ucd/Blocks.txt +* ms-use/IndicSyllabicCategory-Additional.txt +* ms-use/IndicPositionalCategory-Additional.txt """ import sys -if len (sys.argv) != 5: +if len (sys.argv) != 8: sys.exit (__doc__) -BLACKLISTED_BLOCKS = ["Thai", "Lao"] +BLACKLISTED_BLOCKS = [ + 'Samaritan', + 'Thai', + 'Lao', +] files = [open (x, encoding='utf-8') for x in sys.argv[1:]] headers = [[f.readline () for i in range (2)] for j,f in enumerate(files) if j != 2] +for j in range(5, 7): + for line in files[j]: + line = line.rstrip() + if not line: + break + headers[j - 1].append(line) headers.append (["UnicodeData.txt does not have a header."]) data = [{} for _ in files] values = [{} for _ in files] for i, f in enumerate (files): + extended = False + for line in f: + # TODO: https://github.com/MicrosoftDocs/typography-issues/issues/522 + if extended and line.startswith ('# ') and line.find (';'): + line = line[2:] + elif 'USE_Syllabic_Category' in line: + extended = True + j = line.find ('#') if j >= 0: line = line[:j] @@ -42,16 +63,25 @@ for i, f in enumerate (files): else: end = int (uu[1], 16) - t = fields[1 if i != 2 else 2] + t = fields[1 if i not in [2, 3] else 2] + + if i == 3: + t = 'jt_' + t + elif i == 5 and t == 'Consonant_Final_Modifier': + # TODO: https://github.com/MicrosoftDocs/typography-issues/issues/336 + t = 'Syllable_Modifier' + elif i == 6 and t == 'NA': + t = 'Not_Applicable' + i0 = i if i < 5 else i - 5 for u in range (start, end + 1): - data[i][u] = t - values[i][t] = values[i].get (t, 0) + end - start + 1 + data[i0][u] = t + values[i0][t] = values[i0].get (t, 0) + end - start + 1 -defaults = ('Other', 'Not_Applicable', 'Cn', 'No_Block') +defaults = ('Other', 'Not_Applicable', 'Cn', 'jt_X', 'No_Block') # TODO Characters that are not in Unicode Indic files, but used in USE -data[0][0x034F] = defaults[0] +data[0][0x0640] = defaults[0] data[0][0x1B61] = defaults[0] data[0][0x1B63] = defaults[0] data[0][0x1B64] = defaults[0] @@ -61,6 +91,29 @@ data[0][0x1B67] = defaults[0] data[0][0x1B69] = defaults[0] data[0][0x1B6A] = defaults[0] data[0][0x2060] = defaults[0] +for u in range (0x07CA, 0x07EA + 1): + data[0][u] = defaults[0] +data[0][0x07FA] = defaults[0] +for u in range (0x0840, 0x0858 + 1): + data[0][u] = defaults[0] +for u in range (0x1887, 0x18A8 + 1): + data[0][u] = defaults[0] +data[0][0x18AA] = defaults[0] +for u in range (0xA840, 0xA872 + 1): + data[0][u] = defaults[0] +for u in range (0x10B80, 0x10B91 + 1): + data[0][u] = defaults[0] +for u in range (0x10BA9, 0x10BAE + 1): + data[0][u] = defaults[0] +data[0][0x10FB0] = defaults[0] +for u in range (0x10FB2, 0x10FB6 + 1): + data[0][u] = defaults[0] +for u in range (0x10FB8, 0x10FBF + 1): + data[0][u] = defaults[0] +for u in range (0x10FC1, 0x10FC4 + 1): + data[0][u] = defaults[0] +for u in range (0x10FC9, 0x10FCB + 1): + data[0][u] = defaults[0] # TODO https://github.com/harfbuzz/harfbuzz/pull/1685 data[0][0x1B5B] = 'Consonant_Placeholder' data[0][0x1B5C] = 'Consonant_Placeholder' @@ -72,8 +125,6 @@ data[0][0x11C44] = 'Consonant_Placeholder' data[0][0x11C45] = 'Consonant_Placeholder' # TODO https://github.com/harfbuzz/harfbuzz/pull/1399 data[0][0x111C8] = 'Consonant_Placeholder' -for u in range (0xFE00, 0xFE0F + 1): - data[0][u] = defaults[0] # Merge data into one dict: for i,v in enumerate (defaults): @@ -86,7 +137,7 @@ for i,d in enumerate (data): if not u in combined: combined[u] = list (defaults) combined[u][i] = v -combined = {k:v for k,v in combined.items() if v[3] not in BLACKLISTED_BLOCKS} +combined = {k:v for k,v in combined.items() if v[4] not in BLACKLISTED_BLOCKS} data = combined del combined @@ -133,6 +184,10 @@ property_names = [ 'Number_Joiner', 'Number', 'Brahmi_Joining_Number', + 'Hieroglyph', + 'Hieroglyph_Joiner', + 'Hieroglyph_Segment_Begin', + 'Hieroglyph_Segment_End', # Indic_Positional_Category 'Not_Applicable', 'Right', @@ -150,6 +205,14 @@ property_names = [ 'Bottom_And_Right', 'Top_And_Bottom_And_Right', 'Overstruck', + # Joining_Type + 'jt_C', + 'jt_D', + 'jt_L', + 'jt_R', + 'jt_T', + 'jt_U', + 'jt_X', ] class PropertyValue(object): @@ -174,99 +237,86 @@ for name in property_names: globals().update(property_values) -def is_BASE(U, UISC, UGC): +def is_BASE(U, UISC, UGC, AJT): return (UISC in [Number, Consonant, Consonant_Head_Letter, - #SPEC-DRAFT Consonant_Placeholder, Tone_Letter, - Vowel_Independent #SPEC-DRAFT + Vowel_Independent, ] or + # TODO: https://github.com/MicrosoftDocs/typography-issues/issues/484 + AJT in [jt_C, jt_D, jt_L, jt_R] and UISC != Joiner or (UGC == Lo and UISC in [Avagraha, Bindu, Consonant_Final, Consonant_Medial, Consonant_Subjoined, Vowel, Vowel_Dependent])) -def is_BASE_IND(U, UISC, UGC): - #SPEC-DRAFT return (UISC in [Consonant_Dead, Modifying_Letter] or UGC == Po) - return (UISC in [Consonant_Dead, Modifying_Letter] or - (UGC == Po and not U in [0x104B, 0x104E, 0x1B5B, 0x1B5C, 0x1B5F, 0x2022, 0x111C8, 0x11A3F, 0x11A45, 0x11C44, 0x11C45]) or - False # SPEC-DRAFT-OUTDATED! U == 0x002D - ) -def is_BASE_NUM(U, UISC, UGC): +def is_BASE_NUM(U, UISC, UGC, AJT): return UISC == Brahmi_Joining_Number -def is_BASE_OTHER(U, UISC, UGC): - if UISC == Consonant_Placeholder: return True #SPEC-DRAFT - #SPEC-DRAFT return U in [0x00A0, 0x00D7, 0x2015, 0x2022, 0x25CC, 0x25FB, 0x25FC, 0x25FD, 0x25FE] +def is_BASE_OTHER(U, UISC, UGC, AJT): + if UISC == Consonant_Placeholder: return True return U in [0x2015, 0x2022, 0x25FB, 0x25FC, 0x25FD, 0x25FE] -def is_CGJ(U, UISC, UGC): - return U == 0x034F -def is_CONS_FINAL(U, UISC, UGC): +def is_CONS_FINAL(U, UISC, UGC, AJT): return ((UISC == Consonant_Final and UGC != Lo) or UISC == Consonant_Succeeding_Repha) -def is_CONS_FINAL_MOD(U, UISC, UGC): - #SPEC-DRAFT return UISC in [Consonant_Final_Modifier, Syllable_Modifier] - return UISC == Syllable_Modifier -def is_CONS_MED(U, UISC, UGC): +def is_CONS_FINAL_MOD(U, UISC, UGC, AJT): + return UISC == Syllable_Modifier +def is_CONS_MED(U, UISC, UGC, AJT): # Consonant_Initial_Postfixed is new in Unicode 11; not in the spec. return (UISC == Consonant_Medial and UGC != Lo or UISC == Consonant_Initial_Postfixed) -def is_CONS_MOD(U, UISC, UGC): - return UISC in [Nukta, Gemination_Mark, Consonant_Killer] -def is_CONS_SUB(U, UISC, UGC): - #SPEC-DRAFT return UISC == Consonant_Subjoined +def is_CONS_MOD(U, UISC, UGC, AJT): + return (UISC in [Nukta, Gemination_Mark, Consonant_Killer] and + not is_SYM_MOD(U, UISC, UGC, AJT)) +def is_CONS_SUB(U, UISC, UGC, AJT): return UISC == Consonant_Subjoined and UGC != Lo -def is_CONS_WITH_STACKER(U, UISC, UGC): +def is_CONS_WITH_STACKER(U, UISC, UGC, AJT): return UISC == Consonant_With_Stacker -def is_HALANT(U, UISC, UGC): +def is_HALANT(U, UISC, UGC, AJT): return (UISC in [Virama, Invisible_Stacker] - and not is_HALANT_OR_VOWEL_MODIFIER(U, UISC, UGC) - and not is_SAKOT(U, UISC, UGC)) -def is_HALANT_OR_VOWEL_MODIFIER(U, UISC, UGC): + and not is_HALANT_OR_VOWEL_MODIFIER(U, UISC, UGC, AJT) + and not is_SAKOT(U, UISC, UGC, AJT)) +def is_HALANT_OR_VOWEL_MODIFIER(U, UISC, UGC, AJT): # https://github.com/harfbuzz/harfbuzz/issues/1102 # https://github.com/harfbuzz/harfbuzz/issues/1379 return U in [0x11046, 0x1134D] -def is_HALANT_NUM(U, UISC, UGC): +def is_HALANT_NUM(U, UISC, UGC, AJT): return UISC == Number_Joiner -def is_ZWNJ(U, UISC, UGC): +def is_HIEROGLYPH(U, UISC, UGC, AJT): + return UISC == Hieroglyph +def is_HIEROGLYPH_JOINER(U, UISC, UGC, AJT): + return UISC == Hieroglyph_Joiner +def is_HIEROGLYPH_SEGMENT_BEGIN(U, UISC, UGC, AJT): + return UISC == Hieroglyph_Segment_Begin +def is_HIEROGLYPH_SEGMENT_END(U, UISC, UGC, AJT): + return UISC == Hieroglyph_Segment_End +def is_ZWNJ(U, UISC, UGC, AJT): return UISC == Non_Joiner -def is_ZWJ(U, UISC, UGC): - return UISC == Joiner -def is_Word_Joiner(U, UISC, UGC): - return U == 0x2060 -def is_OTHER(U, UISC, UGC): - #SPEC-OUTDATED return UGC == Zs # or any other SCRIPT_COMMON characters - return (UISC == Other - and not is_SYM(U, UISC, UGC) - and not is_SYM_MOD(U, UISC, UGC) - and not is_CGJ(U, UISC, UGC) - and not is_Word_Joiner(U, UISC, UGC) - and not is_VARIATION_SELECTOR(U, UISC, UGC) +def is_OTHER(U, UISC, UGC, AJT): + return ((UGC in [Cn, Po] or UISC in [Consonant_Dead, Joiner, Modifying_Letter, Other]) + and not is_BASE(U, UISC, UGC, AJT) + and not is_BASE_OTHER(U, UISC, UGC, AJT) + and not is_SYM(U, UISC, UGC, AJT) + and not is_SYM_MOD(U, UISC, UGC, AJT) ) -def is_Reserved(U, UISC, UGC): - return UGC == 'Cn' -def is_REPHA(U, UISC, UGC): +def is_REPHA(U, UISC, UGC, AJT): return UISC in [Consonant_Preceding_Repha, Consonant_Prefixed] -def is_SAKOT(U, UISC, UGC): +def is_SAKOT(U, UISC, UGC, AJT): return U == 0x1A60 -def is_SYM(U, UISC, UGC): - if U == 0x25CC: return False #SPEC-DRAFT - #SPEC-DRAFT return UGC in [So, Sc] or UISC == Symbol_Letter - return UGC in [So, Sc] and U not in [0x1B62, 0x1B68] -def is_SYM_MOD(U, UISC, UGC): +def is_SYM(U, UISC, UGC, AJT): + if U in [0x25CC, 0x1E14F]: return False + return UGC in [So, Sc] and U not in [0x0F01, 0x1B62, 0x1B68] +def is_SYM_MOD(U, UISC, UGC, AJT): return U in [0x1B6B, 0x1B6C, 0x1B6D, 0x1B6E, 0x1B6F, 0x1B70, 0x1B71, 0x1B72, 0x1B73] -def is_VARIATION_SELECTOR(U, UISC, UGC): - return 0xFE00 <= U <= 0xFE0F -def is_VOWEL(U, UISC, UGC): +def is_VOWEL(U, UISC, UGC, AJT): # https://github.com/harfbuzz/harfbuzz/issues/376 return (UISC == Pure_Killer or (UGC != Lo and UISC in [Vowel, Vowel_Dependent] and U not in [0xAA29])) -def is_VOWEL_MOD(U, UISC, UGC): +def is_VOWEL_MOD(U, UISC, UGC, AJT): # https://github.com/harfbuzz/harfbuzz/issues/376 return (UISC in [Tone_Mark, Cantillation_Mark, Register_Shifter, Visarga] or (UGC != Lo and (UISC == Bindu or U in [0xAA29]))) +# CGJ, VS, WJ, and ZWJ are handled in find_syllables use_mapping = { 'B': is_BASE, - 'IND': is_BASE_IND, 'N': is_BASE_NUM, 'GB': is_BASE_OTHER, - 'CGJ': is_CGJ, 'F': is_CONS_FINAL, 'FM': is_CONS_FINAL_MOD, 'M': is_CONS_MED, @@ -276,16 +326,16 @@ use_mapping = { 'H': is_HALANT, 'HVM': is_HALANT_OR_VOWEL_MODIFIER, 'HN': is_HALANT_NUM, + 'G': is_HIEROGLYPH, + 'J': is_HIEROGLYPH_JOINER, + 'SB': is_HIEROGLYPH_SEGMENT_BEGIN, + 'SE': is_HIEROGLYPH_SEGMENT_END, 'ZWNJ': is_ZWNJ, - 'ZWJ': is_ZWJ, - 'WJ': is_Word_Joiner, 'O': is_OTHER, - 'Rsv': is_Reserved, 'R': is_REPHA, 'S': is_SYM, 'Sk': is_SAKOT, 'SM': is_SYM_MOD, - 'VS': is_VARIATION_SELECTOR, 'V': is_VOWEL, 'VM': is_VOWEL_MOD, } @@ -304,13 +354,13 @@ use_positions = { }, 'CM': { 'Abv': [Top], - 'Blw': [Bottom], + 'Blw': [Bottom, Overstruck], }, 'V': { 'Abv': [Top, Top_And_Bottom, Top_And_Bottom_And_Right, Top_And_Right], 'Blw': [Bottom, Overstruck, Bottom_And_Right], - 'Pst': [Right, Top_And_Left, Top_And_Left_And_Right, Left_And_Right], - 'Pre': [Left], + 'Pst': [Right], + 'Pre': [Left, Top_And_Left, Top_And_Left_And_Right, Left_And_Right], }, 'VM': { 'Abv': [Top], @@ -330,13 +380,14 @@ use_positions = { 'Blw': [Bottom], 'Pst': [Not_Applicable], }, + 'R': None, 'SUB': None, } def map_to_use(data): out = {} items = use_mapping.items() - for U,(UISC,UIPC,UGC,UBlock) in data.items(): + for U,(UISC,UIPC,UGC,AJT,UBlock) in data.items(): # Resolve Indic_Syllabic_Category @@ -346,20 +397,6 @@ def map_to_use(data): # Tibetan: # TODO: These don't have UISC assigned in Unicode 13.0.0, but have UIPC if 0x0F18 <= U <= 0x0F19 or 0x0F3E <= U <= 0x0F3F: UISC = Vowel_Dependent - if 0x0F86 <= U <= 0x0F87: UISC = Tone_Mark - # Overrides to allow NFC order matching syllable - # https://github.com/harfbuzz/harfbuzz/issues/1012 - if UBlock == 'Tibetan' and is_VOWEL (U, UISC, UGC): - if UIPC == Top: - UIPC = Bottom - - # TODO: https://github.com/harfbuzz/harfbuzz/pull/982 - # also https://github.com/harfbuzz/harfbuzz/issues/1012 - if UBlock == 'Chakma' and is_VOWEL (U, UISC, UGC): - if UIPC == Top: - UIPC = Bottom - elif UIPC == Bottom: - UIPC = Top # TODO: https://github.com/harfbuzz/harfbuzz/pull/627 if 0x1BF2 <= U <= 0x1BF3: UISC = Nukta; UIPC = Bottom @@ -368,11 +405,11 @@ def map_to_use(data): # the nasalization marks, maybe only for U+1CE9..U+1CF1. if U == 0x1CED: UISC = Tone_Mark - # TODO: https://github.com/harfbuzz/harfbuzz/issues/1105 - if U == 0x11134: UISC = Gemination_Mark + # TODO: https://github.com/microsoft/font-tools/issues/1 + if U == 0xA982: UISC = Consonant_Succeeding_Repha - values = [k for k,v in items if v(U,UISC,UGC)] - assert len(values) == 1, "%s %s %s %s" % (hex(U), UISC, UGC, values) + values = [k for k,v in items if v(U,UISC,UGC,AJT)] + assert len(values) == 1, "%s %s %s %s %s" % (hex(U), UISC, UGC, AJT, values) USE = values[0] # Resolve Indic_Positional_Category @@ -380,9 +417,6 @@ def map_to_use(data): # TODO: These should die, but have UIPC in Unicode 13.0.0 if U in [0x953, 0x954]: UIPC = Not_Applicable - # TODO: https://github.com/harfbuzz/harfbuzz/pull/2012 - if U == 0x1C29: UIPC = Left - # TODO: These are not in USE's override list that we have, nor are they in Unicode 13.0.0 if 0xA926 <= U <= 0xA92A: UIPC = Top # TODO: https://github.com/harfbuzz/harfbuzz/pull/1037 @@ -390,14 +424,18 @@ def map_to_use(data): if U in [0x11302, 0x11303, 0x114C1]: UIPC = Top if 0x1CF8 <= U <= 0x1CF9: UIPC = Top - assert (UIPC in [Not_Applicable, Visual_Order_Left] or - USE == 'R' or - USE in use_positions), "%s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC) + # TODO: https://github.com/harfbuzz/harfbuzz/pull/982 + # also https://github.com/harfbuzz/harfbuzz/issues/1012 + if 0x1112A <= U <= 0x1112B: UIPC = Top + if 0x11131 <= U <= 0x11132: UIPC = Top + + assert (UIPC in [Not_Applicable, Visual_Order_Left] or U == 0x0F7F or + USE in use_positions), "%s %s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC, AJT) pos_mapping = use_positions.get(USE, None) if pos_mapping: values = [k for k,v in pos_mapping.items() if v and UIPC in v] - assert len(values) == 1, "%s %s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC, values) + assert len(values) == 1, "%s %s %s %s %s %s %s" % (hex(U), UIPC, USE, UISC, UGC, AJT, values) USE = USE + values[0] out[U] = (USE, UBlock) @@ -410,7 +448,7 @@ print ("/* == Start of generated table == */") print ("/*") print (" * The following table is generated by running:") print (" *") -print (" * ./gen-use-table.py IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt Blocks.txt") +print (" * {} IndicSyllabicCategory.txt IndicPositionalCategory.txt UnicodeData.txt ArabicShaping.txt Blocks.txt IndicSyllabicCategory-Additional.txt IndicPositionalCategory-Additional.txt".format (sys.argv[0])) print (" *") print (" * on files with these headers:") print (" *") @@ -477,6 +515,8 @@ print ("static const USE_TABLE_ELEMENT_TYPE use_table[] = {") for u in uu: if u <= last: continue + if data[u][0] == 'O': + continue block = data[u][1] start = u//8*8 |