summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source
diff options
context:
space:
mode:
authorMojca Miklavec <mojca.miklavec@gmail.com>2013-05-07 23:38:30 +0000
committerMojca Miklavec <mojca.miklavec@gmail.com>2013-05-07 23:38:30 +0000
commit1a8e9572db5e1902de28cf84b45f8ed8b45de5c0 (patch)
treecb220baa38f0f8b44f52e0f5340ed4c4c011ca13 /Master/texmf-dist/source
parenta51a5c6021b880f32efe163726f67cc673551c68 (diff)
hyph-utf8 (2012.05.08)
- add Georgian hyphenation patterns (by Levan Shoshiashvili) - add Thai hyphenation patterns (by Theppitak Karoonboonyanan) [not yet added as a package] - cleanup luatex-hyphen module - fix adding a dialect of a synonym in LuaTeX git-svn-id: svn://tug.org/texlive/trunk@30326 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source')
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/data/encodings/lth.dat87
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/data/encodings/t8m.dat256
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb196
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb2
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb34
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb8
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/languages.rb46
-rw-r--r--Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx39
8 files changed, 573 insertions, 95 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/lth.dat b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/lth.dat
new file mode 100644
index 00000000000..4d0839cbb26
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/lth.dat
@@ -0,0 +1,87 @@
+0xA1 U+0E01 1
+0xA2 U+0E02 1
+0xA3 U+0E03 1
+0xA4 U+0E04 1
+0xA5 U+0E05 1
+0xA6 U+0E06 1
+0xA7 U+0E07 1
+0xA8 U+0E08 1
+0xA9 U+0E09 1
+0xAA U+0E0A 1
+0xAB U+0E0B 1
+0xAC U+0E0C 1
+0xAD U+0E0D 1
+0xAE U+0E0E 1
+0xAF U+0E0F 1
+0xB0 U+0E10 1
+0xB1 U+0E11 1
+0xB2 U+0E12 1
+0xB3 U+0E13 1
+0xB4 U+0E14 1
+0xB5 U+0E15 1
+0xB6 U+0E16 1
+0xB7 U+0E17 1
+0xB8 U+0E18 1
+0xB9 U+0E19 1
+0xBA U+0E1A 1
+0xBB U+0E1B 1
+0xBC U+0E1C 1
+0xBD U+0E1D 1
+0xBE U+0E1E 1
+0xBF U+0E1F 1
+0xC0 U+0E20 1
+0xC1 U+0E21 1
+0xC2 U+0E22 1
+0xC3 U+0E23 1
+0xC4 U+0E24 1
+0xC5 U+0E25 1
+0xC6 U+0E26 1
+0xC7 U+0E27 1
+0xC8 U+0E28 1
+0xC9 U+0E29 1
+0xCA U+0E2A 1
+0xCB U+0E2B 1
+0xCC U+0E2C 1
+0xCD U+0E2D 1
+0xCE U+0E2E 1
+0xCF U+0E2F 1
+0xD0 U+0E30 1
+0xD1 U+0E31 1
+0xD2 U+0E32 1
+0xD3 U+0E33 1
+0xD4 U+0E34 1
+0xD5 U+0E35 1
+0xD6 U+0E36 1
+0xD7 U+0E37 1
+0xD8 U+0E38 1
+0xD9 U+0E39 1
+0xDA U+0E3A 1
+0xDF U+0E3F
+0xE0 U+0E40 1
+0xE1 U+0E41 1
+0xE2 U+0E42 1
+0xE3 U+0E43 1
+0xE4 U+0E44 1
+0xE5 U+0E45 1
+0xE6 U+0E46
+0xE7 U+0E47 1
+0xE8 U+0E48 1
+0xE9 U+0E49 1
+0xEA U+0E4A 1
+0xEB U+0E4B 1
+0xEC U+0E4C 1
+0xED U+0E4D 1
+0xEE U+0E4E 1
+0xEF U+0E4F
+0xF0 U+0E50
+0xF1 U+0E51
+0xF2 U+0E52
+0xF3 U+0E53
+0xF4 U+0E54
+0xF5 U+0E55
+0xF6 U+0E56
+0xF7 U+0E57
+0xF8 U+0E58
+0xF9 U+0E59
+0xFA U+0E5A
+0xFB U+0E5B
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/t8m.dat b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/t8m.dat
new file mode 100644
index 00000000000..78e09abc945
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/t8m.dat
@@ -0,0 +1,256 @@
+0x00 U+.... .notdef
+0x01 U+.... .notdef
+0x02 U+.... .notdef
+0x03 U+.... .notdef
+0x04 U+.... .notdef
+0x05 U+.... .notdef
+0x06 U+.... .notdef
+0x07 U+.... .notdef
+0x08 U+.... .notdef
+0x09 U+00A9 copyright
+0x0A U+00AB guillemotleft
+0x0B U+00BB guillemotright
+0x0C U+00AE registered
+0x0D U+00A3 sterling
+0x0E U+2020 dagger
+0x0F U+2116 uni2116
+0x10 U+201E quotedblbase
+0x11 U+201C quotedblleft
+0x12 U+.... .notdef
+0x13 U+.... .notdef
+0x14 U+.... .notdef
+0x15 U+.... .notdef
+0x16 U+.... .notdef
+0x17 U+.... .notdef
+0x18 U+.... .notdef
+0x19 U+.... .notdef
+0x1A U+.... .notdef
+0x1B U+.... .notdef
+0x1C U+.... .notdef
+0x1D U+.... .notdef
+0x1E U+.... .notdef
+0x1F U+.... .notdef
+0x20 U+0020 space
+0x21 U+0021 exclam
+0x22 U+0022 quotedbl
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+0027 quotesingle
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+003C less
+0x3D U+003D equal
+0x3E U+003E greater
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0049 I
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+005C backslash
+0x5D U+005D bracketright
+0x5E U+005E asciicircum
+0x5F U+005F underscore
+0x60 U+0060 grave
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0069 i
+0x6A U+006A j
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+007B braceleft
+0x7C U+007C bar
+0x7D U+007D braceright
+0x7E U+007E asciitilde
+0x7F U+.... .notdef
+0x80 U+10FB uni10FB
+0x81 U+10A0 uni10A0
+0x82 U+10A1 uni10A1
+0x83 U+10A2 uni10A2
+0x84 U+10A3 uni10A3
+0x85 U+10A4 uni10A4
+0x86 U+10A5 uni10A5
+0x87 U+10A6 uni10A6
+0x88 U+10A7 uni10A7
+0x89 U+10A8 uni10A8
+0x8A U+10A9 uni10A9
+0x8B U+10AA uni10AA
+0x8C U+10AB uni10AB
+0x8D U+10AC uni10AC
+0x8E U+10AD uni10AD
+0x8F U+10AE uni10AE
+0x90 U+10AF uni10AF
+0x91 U+10B0 uni10B0
+0x92 U+10B1 uni10B1
+0x93 U+10B2 uni10B2
+0x94 U+10B3 uni10B3
+0x95 U+10B4 uni10B4
+0x96 U+10B5 uni10B5
+0x97 U+10B6 uni10B6
+0x98 U+10B7 uni10B7
+0x99 U+10B8 uni10B8
+0x9A U+10B9 uni10B9
+0x9B U+10BA uni10BA
+0x9C U+10BB uni10BB
+0x9D U+2013 endash
+0x9E U+2014 emdash
+0x9F U+00A7 section
+0xA0 U+00A0 nonbreakingspace
+0xA1 U+10D0 1 uni10D0
+0xA2 U+10D1 1 uni10D1
+0xA3 U+10D2 1 uni10D2
+0xA4 U+10D3 1 uni10D3
+0xA5 U+10D4 1 uni10D4
+0xA6 U+10D5 1 uni10D5
+0xA7 U+10D6 1 uni10D6
+0xA8 U+10D7 1 uni10D7
+0xA9 U+10D8 1 uni10D8
+0xAA U+10D9 1 uni10D9
+0xAB U+10DA 1 uni10DA
+0xAC U+10DB 1 uni10DB
+0xAD U+10DC 1 uni10DC
+0xAE U+10DD 1 uni10DD
+0xAF U+10DE 1 uni10DE
+0xB0 U+10DF 1 uni10DF
+0xB1 U+10E0 1 uni10E0
+0xB2 U+10E1 1 uni10E1
+0xB3 U+10E2 1 uni10E2
+0xB4 U+10E3 1 uni10E3
+0xB5 U+10E4 1 uni10E4
+0xB6 U+10E5 1 uni10E5
+0xB7 U+10E6 1 uni10E6
+0xB8 U+10E7 1 uni10E7
+0xB9 U+10E8 1 uni10E8
+0xBA U+10E9 1 uni10E9
+0xBB U+10EA 1 uni10EA
+0xBC U+10EB 1 uni10EB
+0xBD U+.... .notdef
+0xBE U+.... .notdef
+0xBF U+.... .notdef
+0xC0 U+10BC uni10BC
+0xC1 U+10BD uni10BD
+0xC2 U+10BE uni10BE
+0xC3 U+10BF uni10BF
+0xC4 U+10C0 uni10C0
+0xC5 U+10C1 uni10C1
+0xC6 U+10C2 uni10C2
+0xC7 U+10C3 uni10C3
+0xC8 U+10C4 uni10C4
+0xC9 U+10C5 uni10C5
+0xCA U+.... .notdef
+0xCB U+.... .notdef
+0xCC U+.... .notdef
+0xCD U+.... .notdef
+0xCE U+.... .notdef
+0xCF U+.... .notdef
+0xD0 U+.... .notdef
+0xD1 U+.... .notdef
+0xD2 U+.... .notdef
+0xD3 U+.... .notdef
+0xD4 U+.... .notdef
+0xD5 U+.... .notdef
+0xD6 U+.... .notdef
+0xD7 U+.... .notdef
+0xD8 U+.... .notdef
+0xD9 U+.... .notdef
+0xDA U+.... .notdef
+0xDB U+.... .notdef
+0xDC U+.... .notdef
+0xDD U+.... .notdef
+0xDE U+.... .notdef
+0xDF U+.... .notdef
+0xE0 U+10EC 1 uni10EC
+0xE1 U+10ED 1 uni10ED
+0xE2 U+10EE 1 uni10EE
+0xE3 U+10EF 1 uni10EF
+0xE4 U+10F0 1 uni10F0
+0xE5 U+10F1 1 uni10F1
+0xE6 U+10F2 1 uni10F2
+0xE7 U+10F3 1 uni10F3
+0xE8 U+10F4 1 uni10F4
+0xE9 U+10F5 1 uni10F5
+0xEA U+10F6 1 uni10F6
+0xEB U+10F7 1 uni10F7
+0xEC U+10F8 1 uni10F8
+0xED U+10F9 1 uni10F9
+0xEE U+10FA 1 uni10FA
+0xEF U+10FC uni10FC
+0xF0 U+.... .notdef
+0xF1 U+.... .notdef
+0xF2 U+.... .notdef
+0xF3 U+.... .notdef
+0xF4 U+.... .notdef
+0xF5 U+.... .notdef
+0xF6 U+.... .notdef
+0xF7 U+.... .notdef
+0xF8 U+.... .notdef
+0xF9 U+.... .notdef
+0xFA U+.... .notdef
+0xFB U+.... .notdef
+0xFC U+.... .notdef
+0xFD U+.... .notdef
+0xFE U+.... .notdef
+0xFF U+.... .notdef
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb
index db699173334..21968cbd079 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb
@@ -3,76 +3,164 @@
require 'hyph-utf8'
$encoding_data_dir = "data/encodings"
-$encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"]
+# $encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"]
+$encodings = ["t8m", "lth"]
$path_root=File.expand_path("../../..")
$output_data_dir = "#{$path_root}/tex/generic/hyph-utf8/conversions"
-# 0x19; U+0131; 1; dotlessi
$encodings.each do |encoding|
+ # load encoding
+ e = Encoding.new(encoding)
+
+ # open file
+ file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w")
+
+ # copyright notice
+ file_out.puts "% conv-utf8-#{encoding}.tex"
+ file_out.puts "%"
+ file_out.puts "% Conversion from UTF-8 to #{encoding.upcase},"
+ file_out.puts "% used before loading hyphenation patterns for 8-bit TeX engines."
+ file_out.puts "%"
+ file_out.puts "% This file is part of hyph-utf8 package and autogenerated."
+ file_out.puts "% See http://tug.org/tex-hyphen"
+ file_out.puts "%"
+ file_out.puts "% Copyright 2008-2013 TeX Users Group."
+ file_out.puts "% You may freely use, modify and/or distribute this file."
+ file_out.puts "% (But consider adapting the scripts if you need modifications.)"
+ file_out.puts
+
+ # macro to get mapping unicode -> font encoding & error message if screwed up
+ file_out.puts '% macros adapted from ConTeXt MKII; see unic-ini.mkii'
+ file_out.puts '\def\unicodechar#1{%'
+ file_out.puts ' \ifcsname unichar@\number#1\endcsname'
+ file_out.puts ' \csname unichar@\number#1\endcsname'
+ file_out.puts ' \else'
+ file_out.puts ' \errmessage{Unicode character [#1] not in encoding.}%'
+ file_out.puts ' \fi}'
+
+ # minimal and maximal lenght of characters in the encoding (until now just 2 & 3)
+ unicode_characters_array = e.unicode_characters.sort
+ length_min = unicode_characters_array.first[1].bytes.size
+ length_max = unicode_characters_array.last[1].bytes.size
+
+ # only output the necessary macros for transforming UTF-8 -> Unicode number
+ if length_min <= 2 and length_max >= 2 then
+ file_out.puts '\def\utftwouniglyph#1#2%'
+ file_out.puts ' {\expandafter\unicodechar\expandafter'
+ file_out.puts ' {\the\numexpr64*(#1-192)+`#2-128\relax}}'
+ end
+ if length_min <= 3 and length_max >= 3 then
+ file_out.puts '\def\utfthreeuniglyph#1#2#3%'
+ file_out.puts ' {\expandafter\unicodechar\expandafter'
+ file_out.puts ' {\the\numexpr4096*(#1-224)+64*(`#2-128)+`#3-128\relax}}'
+ end
+ if length_min <= 4 and length_max >= 4 then
+ file_out.puts '\def\utffouruniglyph#1#2#3#4%'
+ file_out.puts ' {\expandafter\unicodechar\expandafter'
+ file_out.puts ' {\the\numexpr262144*(#1-240)+4096*(`#2-128)+64*(`#3-128)+`#4-128\relax}}'
+ end
+
+ # macro to store mapping unicode -> font encoding
+ file_out.puts
+ file_out.puts '\def\addunichar #1 #2 {\expandafter\def\csname unichar@\number#1\endcsname{#2}}'
+ file_out.puts
+ file_out.puts '% \addunichar "unicode_code - ^^font_encoding_code'
+
+ # all unicode characters in the encoding
+ e.unicode_characters.sort.each do |code,c|
+ file_out.puts sprintf("\\addunichar \"%04X ^^%02x \\lccode\"%02X=\"%02X %% %s - %s",
+ c.code_uni, c.code_enc, c.code_enc, c.code_enc, [c.code_uni].pack('U'), c.name)
+ end
+ file_out.puts
+
+ # make all the possible first characters active
+ # output the definition into file
+ e.unicode_characters_first_byte.sort.each do |first_byte_code,chars|
+ byte = first_byte_code.hex
+ size = chars[0].bytes.size
+ # 2-byte: 0b11000000 <= byte < 0b11100000
+ if size == 2 then
+ str = "two"
+ # 3-byte: 0b11100000 <= byte < 0b11110000
+ elsif size == 3 then
+ str = "three"
+ # 4-byte: 0b11110000 <= byte < 0b11111000
+ elsif size == 4 then
+ str = "four"
+ end
+ file_out.puts sprintf("\\catcode\"%02X=\\active \\def^^%02x{\\utf%suniglyph{\"%02X}}", byte, byte, str, byte)
+ end
+
+ file_out.close
+end
+
+$encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"]
+
+$encodings.each do |encoding|
+ # load encoding
e = Encoding.new(encoding)
- $file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w")
- $file_out.puts "% conv-utf8-#{encoding}.tex"
- $file_out.puts "%"
- $file_out.puts "% Conversion from UTF-8 to #{encoding.upcase},"
- $file_out.puts "% used before loading hyphenation patterns for 8-bit TeX engines."
- $file_out.puts "%"
- $file_out.puts "% This file is part of hyph-utf8 package and autogenerated."
- $file_out.puts "% See http://tug.org/tex-hyphen"
- $file_out.puts "%"
- $file_out.puts "% Copyright 2008 TeX Users Group."
- $file_out.puts "% You may freely use, modify and/or distribute this file."
- $file_out.puts "% (But consider adapting the scripts if you need modifications.)"
- $file_out.puts "%"
-
- e.unicode_characters_first_byte.sort.each do |first_byte|
+ # open file
+ file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w")
+
+ # copyright notice
+ file_out.puts "% conv-utf8-#{encoding}.tex"
+ file_out.puts "%"
+ file_out.puts "% Conversion from UTF-8 to #{encoding.upcase},"
+ file_out.puts "% used before loading hyphenation patterns for 8-bit TeX engines."
+ file_out.puts "%"
+ file_out.puts "% This file is part of hyph-utf8 package and autogenerated."
+ file_out.puts "% See http://tug.org/tex-hyphen"
+ file_out.puts "%"
+ file_out.puts "% Copyright 2008-2013 TeX Users Group."
+ file_out.puts "% You may freely use, modify and/or distribute this file."
+ file_out.puts "% (But consider adapting the scripts if you need modifications.)"
+ file_out.puts "%"
+
+ e.unicode_characters_first_byte.sort.each do |first_byte_code,chars|
# sorting all the second characters alphabetically
- first_byte[1].sort!{|x,y| x.code_uni <=> y.code_uni }
+ chars.sort!{|x,y| x.code_uni <=> y.code_uni }
# make all the possible first characters active
# output the definition into file
- $file_out.puts "\\catcode\"#{first_byte[0].upcase}=\\active"
+ file_out.puts sprintf("\\catcode\"%02X=\\active", first_byte_code.hex)
end
- $file_out.puts "%"
- e.unicode_characters_first_byte.sort.each do |first_byte|
- $file_out.puts "\\def^^#{first_byte[0]}#1{%"
- string_fi = ""
- for i in 1..(first_byte[1].size)
- uni_character = first_byte[1][i-1]
-
- second_byte = uni_character.bytes[1]
- enc_byte = uni_character.code_enc
- enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2')
- ux_code = sprintf("U+%04X", uni_character.code_uni)
- $file_out.puts "\t\\ifx#1^^#{second_byte}^^#{enc_byte}\\else % #{[uni_character.code_uni].pack('U')} - #{ux_code} - #{uni_character.name}"
- string_fi = string_fi + "\\fi"
+ file_out.puts "%"
+ e.unicode_characters_first_byte.sort.each do |first_byte_code,chars|
+ first_byte_code = first_byte_code.hex
+ size = chars[0].bytes.size
+ if size != 2 then
+ throw "The encoding #{encoding} uses more than two bytes to encode characters"
+ else
+
+ file_out.puts sprintf("\\def^^%02x#1{%", first_byte_code)
+ string_fi = ""
+ for i in 1..(chars.size)
+ uni_character = chars[i-1]
+ enc_byte = uni_character.code_enc
+ enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2')
+ ux_code = sprintf("U+%04X", uni_character.code_uni)
+ file_out.puts sprintf("\t\\ifx#1^^%02x^^%02x\\else %% %s - U+%04X - %s", uni_character.bytes[1], uni_character.code_enc, [uni_character.code_uni].pack('U'), uni_character.code_uni, uni_character.name)
+ string_fi = string_fi + "\\fi"
+ end
+
+ # at least three bytes
end
- $file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}"
- $file_out.puts string_fi+"}"
+ file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}"
+ file_out.puts string_fi + "}"
end
- $file_out.puts "%"
- $file_out.puts "% ensure all the chars above have valid \lccode values"
- $file_out.puts "%"
- e.lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc }.each do |character|
+ file_out.puts '%'
+ file_out.puts '% ensure all the chars above have valid \lccode values'
+ file_out.puts '%'
+ e.lowercase_characters.each do |character|
code = [ character.code_enc ].pack("c").unpack("H2").first.upcase
# \lccode"FF="FF
ux_code = sprintf("U+%04X", character.code_uni)
- $file_out.puts "\\lccode\"#{code}=\"#{code} % #{[character.code_uni].pack('U')} - #{ux_code} - #{character.name}"
+ file_out.puts "\\lccode\"#{code}=\"#{code} % #{[character.code_uni].pack('U')} - #{ux_code} - #{character.name}"
end
- $file_out.puts
-
- # if encoding == 'ec'
- # $file_out.puts '% TODO: test if needed and the exact syntax'
- # $file_out.puts '% some patterns use apostrophe and hyphen'
- # # \lccode`\'=`\'
- # $file_out.puts "\\lccode`\\'=`\\'"
- # $file_out.puts "\\catcode`-=11"
- # $file_out.puts
- # end
- # if encoding == 't2a'
- # $file_out.puts "\\lccode`\\'=`\\'"
- # end
-
- $file_out.close
+ file_out.puts
+
+ file_out.close
end
+
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
index a1f6b0d492e..0f8bd7bf915 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
@@ -34,7 +34,7 @@ text_header =
% source/generic/hyph-utf8/generate-pattern-loaders.rb
% See also http://tug.org/tex-hyphen
%
-% Copyright 2008-2011 TeX Users Group.
+% Copyright 2008-2013 TeX Users Group.
% You may freely use, modify and/or distribute this file.
% (But consider adapting the scripts if you need modifications.)
%
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
index 48befd6f011..b4f2a3c82da 100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
@@ -15,7 +15,7 @@ $path_root=File.expand_path("../../..")
$path_ptex="#{$path_root}/tex/generic/hyph-utf8/patterns/ptex"
# load encodings
-encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"]
+encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"]
encodings = Hash.new
encodings_list.each do |encoding_name|
encodings[encoding_name] = Encoding.new(encoding_name)
@@ -30,14 +30,6 @@ language_codes = Hash.new
languages.each do |language|
language_codes[language.code] = language.code
end
-# language_codes['mn-cyrl-x-lmc'] = nil
-# language_codes['sh-latn'] = 'sr-latn'
-# language_codes['sh-cyrl'] = nil
-
-# e = Encoding.new("ec")
-# puts e.convert_string_to_escaped_characters("moja čaša")
-#
-# return
languages.sort{|x,y| x.code <=> y.code }.each do |language|
encoding = nil
@@ -70,6 +62,8 @@ languages.sort{|x,y| x.code <=> y.code }.each do |language|
patterns = $l['no'].get_patterns
end
+ characters = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq
+
if language.encoding != 'ascii' then
patterns = encoding.convert_to_escaped_characters(patterns)
exceptions = encoding.convert_to_escaped_characters(exceptions)
@@ -85,24 +79,24 @@ languages.sort{|x,y| x.code <=> y.code }.each do |language|
file_ptex.puts("% See the original file for details about author, licence etc.")
file_ptex.puts("%")
+ file_ptex.puts("\\bgroup")
+ # setting lccodes for letters
+ characters.each do |c|
+ if c >= 128 then
+ code = encoding.unicode_characters[c].code_enc
+ file_ptex.puts sprintf("\\lccode\"%02X=\"%02X", code, code)
+ end
+ end
+ # patterns
if patterns.length > 0 then
- # file_ptex.puts("\\patterns{\n#{encoding.convert_to_escaped_characters(patterns.join("\n"))}\n}")
file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}")
end
+ # exceptions
if exceptions.length > 0 then
- # file_ptex.puts("\\hyphenation{\n#{encoding.convert_to_escaped_characters(exceptions.join("\n"))}\n}")
file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}")
end
+ file_ptex.puts("\\egroup")
-
- # # patterns
- # patterns.each do |pattern|
- # $file_pat.puts pattern.gsub(/'/,"’")
- # end
- # # exceptions
- # if exceptions != ""
- # $file_hyp.puts exceptions
- # end
file_ptex.close
end
end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb b/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb
index e0004a76630..6d9606f4bc1 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb
@@ -6,8 +6,7 @@ class UnicodeCharacter
def initialize(code_uni, code_enc, name)
@code_uni = code_uni
@code_enc = code_enc
- # TODO: might be longer or shorter
- @bytes = [code_uni].pack('U').unpack('H2H2')
+ @bytes = [code_uni].pack('U').unpack('C*')
@name = name
end
@@ -67,7 +66,7 @@ class Encoding
elsif c == 8217 # ’
new_string[i] = "'"
else
- puts "There must be an error: character #{c} in string #{str} is not ASCII."
+ puts sprintf("There must be an error: character U+%04X in string '%s' is not ASCII or %s.", c, str, @encoding_name.upcase)
end
# an unicode character
else
@@ -84,7 +83,7 @@ private
filename = "#{encoding_data_dir}/#{@encoding_name}.dat"
if File.exists?(filename) then
- File.open(filename).grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z\.]*)$/) do |line|
+ File.open(filename).grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z0-9\.]*)$/) do |line|
# puts line
code_enc = $1.hex
code_uni = $2.hex
@@ -99,6 +98,7 @@ private
@unicode_characters.add_new_character(code_uni, code_enc, name)
@lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name))
end
+ @lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc}
end
else
# TODO: throw an error
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
index d6bb1d5e6c9..97f32a7eaad 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
@@ -178,6 +178,8 @@ authors = {
"rene_bastian" => ["René", "Bastian", nil, false, false], # TODO make contact
"daniel_flipo" => ["Daniel", "Flipo", nil, false, false], # TODO make contact
"bernard_gaulle" => ["Bernard", "Gaulle", nil, false, false], # Deceased...
+ "theppitak_karoonboonyanan" => ["Theppitak", "Karoonboonyanan" "thep{at}linux{dot}thai{dot}net", true, true],
+ "levan_shoshiashvili" => ["Levan", "Shoshiashvili", "shoshia{at}hotmail{dot}com", true, true],
}
@@ -1896,6 +1898,28 @@ class Languages < Hash
"exceptions" => false,
"message" => "Telugu hyphenation patterns",
},
+# thai
+{
+ "code" => "th",
+ "name" => "thai",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [2,3],
+ "encoding" => "lth",
+ "exceptions" => false,
+ "message" => "Thai hyphenation patterns",
+
+ "version" => nil,
+ "last_modified" => "2013-01-28",
+ "type" => "dictionary",
+ "authors" => [ "theppitak_karoonboonyanan" ],
+ "licence" => "LPPL",
+ "description_s" => "Thai hyphenation patterns",
+ "description_l" => [
+ #......................................................................#
+ "Hyphenation patterns for Thai in LTH and UTF-8 encodings.",
+ ],
+},
# lao
#{
# "code" => "lo",
@@ -1942,6 +1966,28 @@ class Languages < Hash
"by files tailored to individual languages.",
],
},
+# georgian
+{
+ "code" => "ka",
+ "name" => "georgian",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [2,3], # TODO
+ "encoding" => "t8m",
+ "exceptions" => false,
+ "message" => "Georgian hyphenation patterns",
+
+ "version" => nil,
+ "last_modified" => "2013-02-25", # TODO
+ "type" => "dictionary",
+ "authors" => [ "levan_shoshiashvili" ],
+ "licence" => "other-free",
+ "description_s" => "Georgian hyphenation patterns",
+ "description_l" => [
+ #......................................................................#
+ "Hyphenation patterns for Georgian in T8M, T8K and UTF-8 encodings.",
+ ],
+},
# dumylang -> dumyhyph.tex
# nohyphenation -> zerohyph.tex
# arabic -> zerohyph.tex
diff --git a/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx b/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx
index 6d7f4cc5d35..74b6c88f399 100644
--- a/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx
+++ b/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx
@@ -253,15 +253,7 @@ This work is under the CC0 license.
%<*lua>
% \end{macrocode}
%
-% Start a Lua module, importing only the necessary functions as locals.
-%
-% \begin{macrocode}
-local error, dofile, pairs, ipairs = error, dofile, pairs, ipairs
-local io, texio, lang, kpse = io, texio, lang, kpse
-module('luatexhyphen')
-% \end{macrocode}
-%
-% Two functions for error and information reporting.
+% Start a Lua module, two functions for error and information reporting.
%
% \begin{macrocode}
local function wlog(msg, ...)
@@ -291,7 +283,7 @@ end
% information, as well as the canonical name of the language.
%
% \begin{macrocode}
-function lookupname(name)
+local function lookupname(name)
if language_dat[name] then
return language_dat[name], name
else
@@ -311,7 +303,7 @@ end
% anything, but will call \verb+error()+ if things go wrong.
%
% \begin{macrocode}
-function loadlanguage(lname, id)
+local function loadlanguage(lname, id)
if id == 0 then
return
end
@@ -363,11 +355,16 @@ function loadlanguage(lname, id)
end
end
end
-function adddialect(dialect, language)
+% \end{macrocode}
+%
+% Add Babe's ``dialects'' as synonyms.
+%
+% \begin{macrocode}
+local function adddialect(dialect, language)
if dialect ~= '0' then
dialect = dialect:gsub('l@', '')
language = language:gsub('l@', '')
- data = language_dat[language]
+ data = lookupname(language)
if data then
data.synonyms[#data.synonyms+1] = dialect
end
@@ -375,6 +372,16 @@ function adddialect(dialect, language)
end
% \end{macrocode}
%
+% Return module functions to the loader.
+%
+% \begin{macrocode}
+return {
+ lookupname = lookupname,
+ loadlanguage = loadlanguage,
+ adddialect = adddialect,
+}
+% \end{macrocode}
+%
% \begin{macrocode}
%</lua>
% \end{macrocode}
@@ -481,7 +488,7 @@ end
\ifx\directlua\relax\else
\directlua{
if not luatexhyphen then
- dofile(assert(kpse.find_file("luatex-hyphen.lua")))
+ luatexhyphen = require("luatex-hyphen")
end
luatexhyphen.adddialect("\string#1", "\string#2")
}%
@@ -623,7 +630,7 @@ end
\global\@namedef{lu@texhyphen@loaded@\the\language}{}%
\directlua{
if not luatexhyphen then
- dofile(assert(kpse.find_file("luatex-hyphen.lua")))
+ luatexhyphen = require("luatex-hyphen")
end
luatexhyphen.loadlanguage("\luatexluaescapestring{#1}",
\the\language)}%
@@ -736,7 +743,7 @@ end
\else
\directlua{
if not luatexhyphen then
- dofile(assert(kpse.find_file("luatex-hyphen.lua")))
+ luatexhyphen = require("luatex-hyphen")
end
processnow = (tex.language == 0) or
(luatexhyphen.lookupname("\luatexluaescapestring{#1}") == nil)}%