summaryrefslogtreecommitdiff
path: root/language/hyph-utf8/source
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /language/hyph-utf8/source
Initial commit
Diffstat (limited to 'language/hyph-utf8/source')
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/README57
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl50
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat257
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat128
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat6
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat256
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat70
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat87
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat256
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat256
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat256
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat256
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb177
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb122
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb68
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb82
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb69
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb116
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/language-data.rb327
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages.rb60
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/es/README83
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua136
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src948
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb137
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README25
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex25
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex7
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex19
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex19
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex19
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex25
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex459
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb69
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua154
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb135
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README61
-rwxr-xr-xlanguage/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb120
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb1
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml452
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb364
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml94
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb25
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb22
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb193
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb126
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb117
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb7
-rw-r--r--language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb985
-rw-r--r--language/hyph-utf8/source/luatex/hyph-utf8/Makefile35
-rw-r--r--language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx337
50 files changed, 8155 insertions, 0 deletions
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/README b/language/hyph-utf8/source/generic/hyph-utf8/README
new file mode 100644
index 0000000000..97f8737876
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/README
@@ -0,0 +1,57 @@
+generate-converters.rb
+======================
+INPUT:
+- source/generic/hyph-utf8/data/encodings/*.dat
+OUTPUT:
+- tex/generic/hyph-utf8/conversions/conv-utf8-*.dat
+
+Auto-generates conversions from UTF-8 to some particular encoding.
+
+Needs to be run only in case that a new encoding is added or an old encoding is fixed.
+
+generate-pattern-loaders.rb
+===========================
+INPUT:
+- language.rb with data
+OUTPUT:
+- tex/generic/hyph-utf8/loadhyph/loadhyph-*.tex
+
+Auto-generates pattern loaders for languages that should be used in TeX Live or other distros.
+
+Needs to be run when a new language is added, a bug needs to be fixed or a strategy is changed.
+
+generate-tl-files.rb
+====================
+Has been used once for the very first TL script generation.
+Out-of-date and not maintaned; will probably be removed.
+
+generate-webpage.rb
+===================
+Used to generated webpage with language overview.
+
+languages.rb
+============
+Database with language information used for almost every script.
+Needs some clean-up and some updates.
+
+
+languages/eu/generate_patterns_eu.rb
+====================================
+Generator for patterns for Basque.
+Author: Juan M. Aguirregabiria, adapted by Mojca Miklavec
+
+languages/gl
+============
+Generator for patterns for Galician.
+Author: Javier A. Múgica
+
+languages/tr/generate_patterns_tr.rb
+====================================
+Generator for patterns for Turkish.
+Author: See file, modified by Mojca Miklavec
+
+
+contributed/make-exhyph.pl
+==========================
+Imported script for hack related to breaking compound words.
+
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl b/language/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl
new file mode 100755
index 0000000000..3fe44239d2
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl
@@ -0,0 +1,50 @@
+#!/usr/bin/perl
+# -*- coding: utf-8; -*-
+# Copyright (C) 2008 Vladimir Volovich.
+# You may freely use, modify and/or distribute this file.
+#
+# Generate additional patterns (for some language), to be used in case
+# there is an alternative hyphen character available.
+#
+# It enables the hyphenation of words containing explicit hyphens
+# when using fonts with \hyphenchar\font <> `\- (e.g. T1 or T2A encoding).
+#
+# sample usage: perl make-exhyph.pl ru > exhyph-ru.tex
+
+use strict;
+use utf8;
+binmode(STDOUT, ":utf8");
+
+my %alphabet = (
+ 'ru' => [ qw( а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я ) ],
+ 'uk' => [ qw( а б в г ґ д е є ж з и і ї й к л м н о п р с т у ф х ц ч ш щ ь ю я ' ) ]
+);
+
+my $lang = $ARGV[0];
+die "unspecified lang" if ! defined $lang;
+die "invalid lang" if ! exists $alphabet{$lang};
+
+my @alphabet = @{$alphabet{$lang}};
+
+print <<\EOF;
+\begingroup
+\lccode45=45 % Make hyphen "-" a word constituent
+\patterns{
+8-7
+--8
+EOF
+
+for my $l (@alphabet) {
+ print ".${l}-8\n";
+}
+
+for my $l1 (@alphabet) {
+ for my $l2 (@alphabet) {
+ print "-${l1}8${l2}8\n8${l1}8${l2}-\n";
+ }
+}
+
+print <<\EOF;
+}
+\endgroup
+EOF
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat
new file mode 100644
index 0000000000..905d5b1d04
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat
@@ -0,0 +1,257 @@
+0x00 U+0060 grave
+0x01 U+00B4 acute
+0x02 U+02C6 circumflex
+0x03 U+02DC tilde
+0x04 U+00A8 dieresis
+0x05 U+02DD hungarumlaut
+0x06 U+02DA ring
+0x07 U+02C7 caron
+0x08 U+02D8 breve
+0x09 U+00AF macron
+0x0A U+02D9 dotaccent
+0x0B U+00B8 cedilla
+0x0C U+02DB ogonek
+0x0D U+201A quotesinglbase
+0x0E U+2039 guilsinglleft
+0x0F U+203A guilsinglright
+0x10 U+201C quotedblleft
+0x11 U+201D quotedblright
+0x12 U+201E quotedblbase
+0x13 U+00AB guillemotleft
+0x14 U+00BB guillemotright
+0x15 U+2013 endash
+0x16 U+2014 emdash
+0x17 U+200B cwm
+0x18 U+.... perthousandzero
+0x19 U+0131 1 dotlessi
+0x1A U+0237 1 dotlessj
+0x1B U+FB00 ff
+0x1C U+FB01 fi
+0x1D U+FB02 fl
+0x1E U+FB03 ffi
+0x1F U+FB04 ffl
+0x20 U+2423 visiblespace
+0x21 U+0021 exclam
+0x22 U+0022 quotedbl
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+2019 quoteright
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+003C less
+0x3D U+003D equal
+0x3E U+003E greater
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0049 I
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+005C backslash
+0x5D U+005D bracketright
+0x5E U+005E asciicircum
+0x5F U+005F underscore
+0x60 U+2018 quoteleft
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0069 i
+0x6A U+006A j
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+007B braceleft
+0x7C U+007C bar
+0x7D U+007D braceright
+0x7E U+007E asciitilde
+0x7F U+002D hyphenchar
+0x80 U+0102 Abreve
+0x81 U+0104 Aogonek
+0x82 U+0106 Cacute
+0x83 U+010C Ccaron
+0x84 U+010E Dcaron
+0x85 U+011A Ecaron
+0x86 U+0118 Eogonek
+0x87 U+011E Gbreve
+0x88 U+0139 Lacute
+0x89 U+013D Lcaron
+0x8A U+0141 Lslash
+0x8B U+0143 Nacute
+0x8C U+0147 Ncaron
+0x8D U+014A Eng
+0x8E U+0150 Ohungarumlaut
+0x8F U+0154 Racute
+0x90 U+0158 Rcaron
+0x91 U+015A Sacute
+0x92 U+0160 Scaron
+0x93 U+015E Scedilla
+0x94 U+0164 Tcaron
+0x95 U+0162 Tcedilla
+0x96 U+0170 Uhungarumlaut
+0x97 U+016E Uring
+0x98 U+0178 Ydieresis
+0x99 U+0179 Zacute
+0x9A U+017D Zcaron
+0x9B U+017B Zdotaccent
+0x9C U+0132 IJ
+0x9D U+0130 Idotaccent
+0x9E U+0111 1 dcroat
+0x9F U+00A7 section
+0xA0 U+0103 1 abreve
+0xA1 U+0105 1 aogonek
+0xA2 U+0107 1 cacute
+0xA3 U+010D 1 ccaron
+0xA4 U+010F 1 dcaron
+0xA5 U+011B 1 ecaron
+0xA6 U+0119 1 eogonek
+0xA7 U+011F 1 gbreve
+0xA8 U+013A 1 lacute
+0xA9 U+013E 1 lcaron
+0xAA U+0142 1 lslash
+0xAB U+0144 1 nacute
+0xAC U+0148 1 ncaron
+0xAD U+014B 1 eng
+0xAE U+0151 1 ohungarumlaut
+0xAF U+0155 1 racute
+0xB0 U+0159 1 rcaron
+0xB1 U+015B 1 sacute
+0xB2 U+0161 1 scaron
+0xB3 U+015F 1 scedilla
+0xB3 U+0219 1 scommaaccent
+0xB4 U+0165 1 tcaron
+0xB5 U+021B 1 tcommaaccent
+0xB6 U+0171 1 uhungarumlaut
+0xB7 U+016F 1 uring
+0xB8 U+00FF 1 ydieresis
+0xB9 U+017A 1 zacute
+0xBA U+017E 1 zcaron
+0xBB U+017C 1 zdotaccent
+0xBC U+0133 1 ij
+0xBD U+00A1 exclamdown
+0xBE U+00BF questiondown
+0xBF U+00A3 sterling
+0xC0 U+00C0 Agrave
+0xC1 U+00C1 Aacute
+0xC2 U+00C2 Acircumflex
+0xC3 U+00C3 Atilde
+0xC4 U+00C4 Adieresis
+0xC5 U+00C5 Aring
+0xC6 U+00C6 AE
+0xC7 U+00C7 Ccedilla
+0xC8 U+00C8 Egrave
+0xC9 U+00C9 Eacute
+0xCA U+00CA Ecircumflex
+0xCB U+00CB Edieresis
+0xCC U+00CC Igrave
+0xCD U+00CD Iacute
+0xCE U+00CE Icircumflex
+0xCF U+00CF Idieresis
+0xD0 U+00D0 Eth
+0xD1 U+00D1 Ntilde
+0xD2 U+00D2 Ograve
+0xD3 U+00D3 Oacute
+0xD4 U+00D4 Ocircumflex
+0xD5 U+00D5 Otilde
+0xD6 U+00D6 Odieresis
+0xD7 U+0152 OE
+0xD8 U+00D8 Oslash
+0xD9 U+00D9 Ugrave
+0xDA U+00DA Uacute
+0xDB U+00DB Ucircumflex
+0xDC U+00DC Udieresis
+0xDD U+00DD Yacute
+0xDE U+00DE Thorn
+0xDF U+1E9E Germandbls
+0xE0 U+00E0 1 agrave
+0xE1 U+00E1 1 aacute
+0xE2 U+00E2 1 acircumflex
+0xE3 U+00E3 1 atilde
+0xE4 U+00E4 1 adieresis
+0xE5 U+00E5 1 aring
+0xE6 U+00E6 1 ae
+0xE7 U+00E7 1 ccedilla
+0xE8 U+00E8 1 egrave
+0xE9 U+00E9 1 eacute
+0xEA U+00EA 1 ecircumflex
+0xEB U+00EB 1 edieresis
+0xEC U+00EC 1 igrave
+0xED U+00ED 1 iacute
+0xEE U+00EE 1 icircumflex
+0xEF U+00EF 1 idieresis
+0xF0 U+00F0 1 eth
+0xF1 U+00F1 1 ntilde
+0xF2 U+00F2 1 ograve
+0xF3 U+00F3 1 oacute
+0xF4 U+00F4 1 ocircumflex
+0xF5 U+00F5 1 otilde
+0xF6 U+00F6 1 odieresis
+0xF7 U+0153 1 oe
+0xF8 U+00F8 1 oslash
+0xF9 U+00F9 1 ugrave
+0xFA U+00FA 1 uacute
+0xFB U+00FB 1 ucircumflex
+0xFC U+00FC 1 udieresis
+0xFD U+00FD 1 yacute
+0xFE U+00FE 1 thorn
+0xFF U+00DF 1 germandbls
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat
new file mode 100644
index 0000000000..421230ecc3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat
@@ -0,0 +1,128 @@
+0x80 U+0080
+0x81 U+0081
+0x82 U+0082
+0x83 U+0083
+0x84 U+0084
+0x85 U+0085
+0x86 U+0086
+0x87 U+0087
+0x88 U+0088
+0x89 U+0089
+0x8A U+008A
+0x8B U+008B
+0x8C U+008C
+0x8D U+008D
+0x8E U+008E
+0x8F U+008F
+0x90 U+0090
+0x91 U+0091
+0x92 U+0092
+0x93 U+0093
+0x94 U+0094
+0x95 U+0095
+0x96 U+0096
+0x97 U+0097
+0x98 U+0098
+0x99 U+0099
+0x9A U+009A
+0x9B U+009B
+0x9C U+009C
+0x9D U+009D
+0x9E U+009E
+0x9F U+009F
+0xA0 U+00A0
+0xA1 U+0104 Aogonek
+0xA2 U+02D8 breve
+0xA3 U+0141 Lslash
+0xA4 U+00A4 currency
+0xA5 U+013D Lcaron
+0xA6 U+015A Sacute
+0xA7 U+00A7 section
+0xA8 U+00A8 dieresis
+0xA9 U+0160 Scaron
+0xAA U+015E Scedilla
+0xAB U+0164 Tcaron
+0xAC U+0179 Zacute
+0xAD U+00AD
+0xAE U+017D Zcaron
+0xAF U+017B Zdotaccent
+0xB0 U+00B0 degree
+0xB1 U+0105 1 aogonek
+0xB2 U+02DB ogonek
+0xB3 U+0142 1 lslash
+0xB4 U+00B4 acute
+0xB5 U+013E 1 lcaron
+0xB6 U+015B 1 sacute
+0xB7 U+02C7 caron
+0xB8 U+00B8 cedilla
+0xB9 U+0161 1 scaron
+0xBA U+015F 1 scedilla
+0xBB U+0165 1 tcaron
+0xBC U+017A 1 zacute
+0xBD U+02DD hungarumlaut
+0xBE U+017E 1 zcaron
+0xBF U+017C 1 zdotaccent
+0xC0 U+0154 Racute
+0xC1 U+00C1 Aacute
+0xC2 U+00C2 Acircumflex
+0xC3 U+0102 Abreve
+0xC4 U+00C4 Adieresis
+0xC5 U+0139 Lacute
+0xC6 U+0106 Cacute
+0xC7 U+00C7 Ccedilla
+0xC8 U+010C Ccaron
+0xC9 U+00C9 Eacute
+0xCA U+0118 Eogonek
+0xCB U+00CB Edieresis
+0xCC U+011A Ecaron
+0xCD U+00CD Iacute
+0xCE U+00CE Icircumflex
+0xCF U+010E Dcaron
+0xD0 U+0110 Dcroat
+0xD1 U+0143 Nacute
+0xD2 U+0147 Ncaron
+0xD3 U+00D3 Oacute
+0xD4 U+00D4 Ocircumflex
+0xD5 U+0150 Ohungarumlaut
+0xD6 U+00D6 Odieresis
+0xD7 U+00D7 multiply
+0xD8 U+0158 Rcaron
+0xD9 U+016E Uring
+0xDA U+00DA Uacute
+0xDB U+0170 Uhungarumlaut
+0xDC U+00DC Udieresis
+0xDD U+00DD Yacute
+0xDE U+0162 Tcommaaccent
+0xDF U+00DF 1 germandbls
+0xE0 U+0155 1 racute
+0xE1 U+00E1 1 aacute
+0xE2 U+00E2 1 acircumflex
+0xE3 U+0103 1 abreve
+0xE4 U+00E4 1 adieresis
+0xE5 U+013A 1 lacute
+0xE6 U+0107 1 cacute
+0xE7 U+00E7 1 ccedilla
+0xE8 U+010D 1 ccaron
+0xE9 U+00E9 1 eacute
+0xEA U+0119 1 eogonek
+0xEB U+00EB 1 edieresis
+0xEC U+011B 1 ecaron
+0xED U+00ED 1 iacute
+0xEE U+00EE 1 icircumflex
+0xEF U+010F 1 dcaron
+0xF0 U+0111 1 dcroat
+0xF1 U+0144 1 nacute
+0xF2 U+0148 1 ncaron
+0xF3 U+00F3 1 oacute
+0xF4 U+00F4 1 ocircumflex
+0xF5 U+0151 1 ohungarumlaut
+0xF6 U+00F6 1 odieresis
+0xF7 U+00F7 divide
+0xF8 U+0159 1 rcaron
+0xF9 U+016F 1 uring
+0xFA U+00FA 1 uacute
+0xFB U+0171 1 uhungarumlaut
+0xFC U+00FC 1 udieresis
+0xFD U+00FD 1 yacute
+0xFE U+0163 1 tcommaaccent
+0xFF U+02D9 dotaccent
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat
new file mode 100644
index 0000000000..7e7d856a19
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat
@@ -0,0 +1,6 @@
+0xB6 U+0125 1 hcircumflex
+0xBC U+0135 1 jcircumflex
+0xE6 U+0109 1 ccircumflex
+0xF8 U+011D 1 gcircumflex
+0xFD U+016D 1 ubreve
+0xFE U+015D 1 scircumflex
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat
new file mode 100644
index 0000000000..c3e9fa0890
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat
@@ -0,0 +1,256 @@
+0x00 U+0060 grave
+0x01 U+00B4 acute
+0x02 U+02C6 circumflex
+0x03 U+02DC tilde
+0x04 U+00A8 dieresis
+0x05 U+02DD hungarumlaut
+0x06 U+02DA ring
+0x07 U+02C7 caron
+0x08 U+02D8 breve
+0x09 U+00AF macron
+0x0A U+02D9 dotaccent
+0x0B U+00B8 cedilla
+0x0C U+02DB ogonek
+0x0D U+201A quotesinglbase
+0x0E U+2039 guilsinglleft
+0x0F U+203A guilsinglright
+0x10 U+201C quotedblleft
+0x11 U+201D quotedblright
+0x12 U+201E quotedblbase
+0x13 U+00AB guillemotleft
+0x14 U+00BB guillemotright
+0x15 U+2013 endash
+0x16 U+2014 emdash
+0x17 U+200B cwm
+0x18 U+.... perthousandzero
+0x19 U+0131 1 dotlessi
+0x1A U+0237 1 dotlessj
+0x1B U+FB00 ff
+0x1C U+FB01 fi
+0x1D U+FB02 fl
+0x1E U+FB03 ffi
+0x1F U+FB04 ffl
+0x20 U+0020 space
+0x21 U+0021 exclam
+0x22 U+0022 quotedbl
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+2019 quoteright
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+003C less
+0x3D U+003D equal
+0x3E U+003E greater
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0049 I
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+005C backslash
+0x5D U+005D bracketright
+0x5E U+005E asciicircum
+0x5F U+005F underscore
+0x60 U+2018 quoteleft
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0069 i
+0x6A U+006A j
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+007B braceleft
+0x7C U+007C bar
+0x7D U+007D braceright
+0x7E U+007E asciitilde
+0x7F U+.... .notdef
+0x80 U+20AC Euro
+0x81 U+.... .notdef
+0x82 U+201A quotesinglbase
+0x83 U+0192 florin
+0x84 U+201E quotedblbase
+0x85 U+2026 ellipsis
+0x86 U+2020 dagger
+0x87 U+2021 daggerdbl
+0x88 U+.... .notdef
+0x89 U+2030 perthousand
+0x8A U+.... .notdef
+0x8B U+2039 guilsinglleft
+0x8C U+0152 OE
+0x8D U+00A8 dieresis
+0x8E U+02C7 caron
+0x8F U+00B8 cedilla
+0x90 U+.... .notdef
+0x91 U+2018 quoteleft
+0x92 U+2019 quoteright
+0x93 U+201C quotedblleft
+0x94 U+201D quotedblright
+0x95 U+2022 bullet
+0x96 U+2013 endash
+0x97 U+2014 emdash
+0x98 U+.... .notdef
+0x99 U+2122 trademark
+0x9A U+.... .notdef
+0x9B U+203A guilsinglright
+0x9C U+0153 1 oe
+0x9D U+00AF macron
+0x9E U+02DB ogonek
+0x9F U+.... .notdef
+0xA0 U+0020 space
+0xA1 U+201D quotedblright
+0xA2 U+00A2 cent
+0xA3 U+00A3 sterling
+0xA4 U+00A4 currency
+0xA5 U+201E quotedblbase
+0xA6 U+00A6 brokenbar
+0xA7 U+00A7 section
+0xA8 U+00D8 Oslash
+0xA9 U+00A9 copyright
+0xAA U+0156 Rcommaaccent
+0xAB U+00AB guillemotleft
+0xAC U+00AC logicalnot
+0xAD U+002D hyphen
+0xAE U+00AE registered
+0xAF U+00C6 AE
+0xB0 U+00B0 degree
+0xB1 U+00B1 plusminus
+0xB2 U+00B2 twosuperior
+0xB3 U+00B3 threesuperior
+0xB4 U+201C quotedblleft
+0xB5 U+00B5 mu
+0xB6 U+00B6 paragraph
+0xB7 U+00B7 periodcentered
+0xB8 U+00F8 1 oslash
+0xB9 U+00B9 onesuperior
+0xBA U+0157 1 rcommaaccent
+0xBB U+00BB guillemotright
+0xBC U+00BC onequarter
+0xBD U+00BD onehalf
+0xBE U+00BE threequarters
+0xBF U+00E6 1 ae
+0xC0 U+0104 Aogonek
+0xC1 U+012E Iogonek
+0xC2 U+0100 Amacron
+0xC3 U+0106 Cacute
+0xC4 U+00C4 Adieresis
+0xC5 U+00C5 Aring
+0xC6 U+0118 Eogonek
+0xC7 U+0112 Emacron
+0xC8 U+010C Ccaron
+0xC9 U+00C9 Eacute
+0xCA U+0179 Zacute
+0xCB U+0116 Edotaccent
+0xCC U+0122 Gcommaaccent
+0xCD U+0136 Kcommaaccent
+0xCE U+012A Imacron
+0xCF U+013B Lcommaaccent
+0xD0 U+0160 Scaron
+0xD1 U+0143 Nacute
+0xD2 U+0145 Ncommaaccent
+0xD3 U+00D3 Oacute
+0xD4 U+014C Omacron
+0xD5 U+00D5 Otilde
+0xD6 U+00D6 Odieresis
+0xD7 U+00D7 multiply
+0xD8 U+0172 Uogonek
+0xD9 U+0141 Lslash
+0xDA U+015A Sacute
+0xDB U+016A Umacron
+0xDC U+00DC Udieresis
+0xDD U+017B Zdotaccent
+0xDE U+017D Zcaron
+0xDF U+00DF 1 germandbls
+0xE0 U+0105 1 aogonek
+0xE1 U+012F 1 iogonek
+0xE2 U+0101 1 amacron
+0xE3 U+0107 1 cacute
+0xE4 U+00E4 1 adieresis
+0xE5 U+00E5 1 aring
+0xE6 U+0119 1 eogonek
+0xE7 U+0113 1 emacron
+0xE8 U+010D 1 ccaron
+0xE9 U+00E9 1 eacute
+0xEA U+017A 1 zacute
+0xEB U+0117 1 edotaccent
+0xEC U+0123 1 gcommaaccent
+0xED U+0137 1 kcommaaccent
+0xEE U+012B 1 imacron
+0xEF U+013C 1 lcommaaccent
+0xF0 U+0161 1 scaron
+0xF1 U+0144 1 nacute
+0xF2 U+0146 1 ncommaaccent
+0xF3 U+00F3 1 oacute
+0xF4 U+014D 1 omacron
+0xF5 U+00F5 1 otilde
+0xF6 U+00F6 1 odieresis
+0xF7 U+00F7 divide
+0xF8 U+0173 1 uogonek
+0xF9 U+0142 1 lslash
+0xFA U+015B 1 sacute
+0xFB U+016B 1 umacron
+0xFC U+00FC 1 udieresis
+0xFD U+017C 1 zdotaccent
+0xFE U+017E 1 zcaron
+0xFF U+2019 quoteright
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat
new file mode 100644
index 0000000000..2fefcb77c3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat
@@ -0,0 +1,70 @@
+0x41 U+0410 CYRA
+0x42 U+0411 CYRB
+0x43 U+0426 CYRC
+0x44 U+0414 CYRD
+0x45 U+0415 CYRE
+0x46 U+0424 CYRF
+0x47 U+0413 CYRG
+0x49 U+0418 CYRI
+0x4A U+0416 CYRZH
+0x4B U+041A CYRK
+0x4C U+041B CYRL
+0x4D U+041C CYRM
+0x4E U+041D CYRN
+0x4F U+041E CYRO
+0x50 U+041F CYRP
+0x51 U+0427 CYRCH
+0x52 U+0420 CYRR
+0x53 U+0421 CYRS
+0x54 U+0422 CYRT
+0x55 U+0423 CYRU
+0x57 U+0412 CYRV
+0x58 U+0425 CYRH
+0x59 U+042B CYRERY
+0x5A U+0417 CYRZ
+0x61 U+0430 1 cyra
+0x62 U+0431 1 cyrb
+0x63 U+0446 1 cyrc
+0x64 U+0434 1 cyrd
+0x65 U+0435 1 cyre
+0x66 U+0444 1 cyrf
+0x67 U+0433 1 cyrg
+0x69 U+0438 1 cyri
+0x6A U+0436 1 cyrzh
+0x6B U+043A 1 cyrk
+0x6C U+043B 1 cyrl
+0x6D U+043C 1 cyrm
+0x6E U+043D 1 cyrn
+0x6F U+043E 1 cyro
+0x70 U+043F 1 cyrp
+0x71 U+0447 1 cyrch
+0x72 U+0440 1 cyrr
+0x73 U+0441 1 cyrs
+0x74 U+0442 1 cyrt
+0x75 U+0443 1 cyru
+0x77 U+0432 1 cyrv
+0x78 U+0445 1 cyrh
+0x79 U+044B 1 cyrery
+0x7A U+0437 1 cyrz
+0x90 U+0429 CYRSHCH
+0x92 U+0428 CYRSH
+0x93 U+042A CYRHRDSN
+0x94 U+042C CYRSFTSN
+0x95 U+042E CYRYU
+0x96 U+042F CYRYA
+0xB0 U+0449 1 cyrshch
+0xB2 U+0448 1 cyrsh
+0xB3 U+044A 1 cyrhrdsn
+0xB4 U+0131 1 i
+0xB5 U+044E 1 cyryu
+0xB6 U+044F 1 cyrya
+0xC4 U+042D CYREREV
+0xCB U+0401 CYRYO
+0xCF U+0419 CYRISHRT
+0xD6 U+04E8 CYROTLD
+0xDC U+04AE CYRY
+0xE4 U+044D 1 cyrerev
+0xEB U+0451 1 cyryo
+0xEF U+0439 1 cyrishrt
+0xF6 U+04E9 1 cyrotld
+0xFC U+04AF 1 cyry
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat
new file mode 100644
index 0000000000..4d0839cbb2
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat
@@ -0,0 +1,87 @@
+0xA1 U+0E01 1
+0xA2 U+0E02 1
+0xA3 U+0E03 1
+0xA4 U+0E04 1
+0xA5 U+0E05 1
+0xA6 U+0E06 1
+0xA7 U+0E07 1
+0xA8 U+0E08 1
+0xA9 U+0E09 1
+0xAA U+0E0A 1
+0xAB U+0E0B 1
+0xAC U+0E0C 1
+0xAD U+0E0D 1
+0xAE U+0E0E 1
+0xAF U+0E0F 1
+0xB0 U+0E10 1
+0xB1 U+0E11 1
+0xB2 U+0E12 1
+0xB3 U+0E13 1
+0xB4 U+0E14 1
+0xB5 U+0E15 1
+0xB6 U+0E16 1
+0xB7 U+0E17 1
+0xB8 U+0E18 1
+0xB9 U+0E19 1
+0xBA U+0E1A 1
+0xBB U+0E1B 1
+0xBC U+0E1C 1
+0xBD U+0E1D 1
+0xBE U+0E1E 1
+0xBF U+0E1F 1
+0xC0 U+0E20 1
+0xC1 U+0E21 1
+0xC2 U+0E22 1
+0xC3 U+0E23 1
+0xC4 U+0E24 1
+0xC5 U+0E25 1
+0xC6 U+0E26 1
+0xC7 U+0E27 1
+0xC8 U+0E28 1
+0xC9 U+0E29 1
+0xCA U+0E2A 1
+0xCB U+0E2B 1
+0xCC U+0E2C 1
+0xCD U+0E2D 1
+0xCE U+0E2E 1
+0xCF U+0E2F 1
+0xD0 U+0E30 1
+0xD1 U+0E31 1
+0xD2 U+0E32 1
+0xD3 U+0E33 1
+0xD4 U+0E34 1
+0xD5 U+0E35 1
+0xD6 U+0E36 1
+0xD7 U+0E37 1
+0xD8 U+0E38 1
+0xD9 U+0E39 1
+0xDA U+0E3A 1
+0xDF U+0E3F
+0xE0 U+0E40 1
+0xE1 U+0E41 1
+0xE2 U+0E42 1
+0xE3 U+0E43 1
+0xE4 U+0E44 1
+0xE5 U+0E45 1
+0xE6 U+0E46
+0xE7 U+0E47 1
+0xE8 U+0E48 1
+0xE9 U+0E49 1
+0xEA U+0E4A 1
+0xEB U+0E4B 1
+0xEC U+0E4C 1
+0xED U+0E4D 1
+0xEE U+0E4E 1
+0xEF U+0E4F
+0xF0 U+0E50
+0xF1 U+0E51
+0xF2 U+0E52
+0xF3 U+0E53
+0xF4 U+0E54
+0xF5 U+0E55
+0xF6 U+0E56
+0xF7 U+0E57
+0xF8 U+0E58
+0xF9 U+0E59
+0xFA U+0E5A
+0xFB U+0E5B
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat
new file mode 100644
index 0000000000..57abcac177
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat
@@ -0,0 +1,256 @@
+0x00 U+.... .notdef
+0x01 U+2206 Delta
+0x02 U+.... .notdef
+0x03 U+.... .notdef
+0x04 U+.... .notdef
+0x05 U+03A0 Pi
+0x06 U+03A3 Sigma
+0x07 U+00B5 mu
+0x08 U+2026 ellipsis
+0x09 U+.... f_k
+0x0A U+2126 Omega
+0x0B U+FB00 ff
+0x0C U+FB01 fi
+0x0D U+FB02 fl
+0x0E U+FB03 ffi
+0x0F U+FB04 ffl
+0x10 U+0131 1 dotlessi
+0x11 U+0237 1 dotlessj
+0x12 U+0060 grave
+0x13 U+00B4 acute
+0x14 U+02C7 caron
+0x15 U+02D8 breve
+0x16 U+00AF macron
+0x17 U+02DA ring
+0x18 U+00B8 cedilla
+0x19 U+00DF 1 germandbls
+0x1A U+00E6 1 ae
+0x1B U+0153 1 oe
+0x1C U+00F8 1 oslash
+0x1D U+00C6 AE
+0x1E U+0152 OE
+0x1F U+00D8 Oslash
+0x20 U+0020 space
+0x21 U+0021 exclam
+0x22 U+201D quotedblright
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+2019 quoteright
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+00A1 exclamdown
+0x3D U+003D equal
+0x3E U+00BF questiondown
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0049 I
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+201C quotedblleft
+0x5D U+005D bracketright
+0x5E U+02C6 circumflex
+0x5F U+02D9 dotaccent
+0x60 U+2018 quoteleft
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0069 i
+0x6A U+006A j
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+2013 endash
+0x7C U+2014 emdash
+0x7D U+02DD hungarumlaut
+0x7E U+02DC tilde
+0x7F U+00A8 dieresis
+0x80 U+20AC Euro
+0x81 U+0104 Aogonek
+0x82 U+0106 Cacute
+0x83 U+003E greater
+0x84 U+.... .notdef
+0x85 U+.... .notdef
+0x86 U+0118 Eogonek
+0x87 U+012E Iogonek
+0x88 U+003C less
+0x89 U+.... .notdef
+0x8A U+0141 Lslash
+0x8B U+0143 Nacute
+0x8C U+007E asciitilde
+0x8D U+005E asciicircum
+0x8E U+.... .notdef
+0x8F U+2020 dagger
+0x90 U+2021 daggerdbl
+0x91 U+015A Sacute
+0x92 U+0160 Scaron
+0x93 U+0218 Scommaaccent
+0x94 U+00B0 degree
+0x95 U+0162 Tcommaaccent
+0x96 U+02DB ogonek
+0x97 U+0172 Uogonek
+0x98 U+0178 Ydieresis
+0x99 U+0179 Zacute
+0x9A U+017D Zcaron
+0x9B U+017B Zdotaccent
+0x9C U+0132 IJ
+0x9D U+007B braceleft
+0x9E U+007D braceright
+0x9F U+00A7 section
+0xA0 U+.... .notdef
+0xA1 U+0105 1 aogonek
+0xA2 U+0107 1 cacute
+0xA3 U+00AE registered
+0xA4 U+00A9 copyright
+0xA5 U+00F7 divide
+0xA6 U+0119 1 eogonek
+0xA7 U+012F 1 iogonek
+0xA8 U+2212 minus
+0xA9 U+00D7 multiply
+0xAA U+0142 1 lslash
+0xAB U+0144 1 nacute
+0xAC U+00B1 plusminus
+0xAD U+221E infinity
+0xAE U+00AB guillemotleft
+0xAF U+00BB guillemotright
+0xB0 U+00B6 paragraph
+0xB1 U+015B 1 sacute
+0xB2 U+0161 1 scaron
+0xB3 U+0219 1 scommaaccent
+0xB4 U+2022 bullet
+0xB5 U+0163 1 tcommaaccent
+0xB6 U+.... threequartersemdash
+0xB7 U+0173 1 uogonek
+0xB8 U+00FF 1 ydieresis
+0xB9 U+017A 1 zacute
+0xBA U+017E 1 zcaron
+0xBB U+017C 1 zdotaccent
+0xBC U+0133 1 ij
+0xBD U+00B7 periodcentered
+0xBE U+0022 quotedbl
+0xBF U+0027 quotesingle
+0xC0 U+00C0 Agrave
+0xC1 U+00C1 Aacute
+0xC2 U+00C2 Acircumflex
+0xC3 U+00C3 Atilde
+0xC4 U+00C4 Adieresis
+0xC5 U+00C5 Aring
+0xC6 U+005C backslash
+0xC7 U+00C7 Ccedilla
+0xC8 U+00C8 Egrave
+0xC9 U+00C9 Eacute
+0xCA U+00CA Ecircumflex
+0xCB U+00CB Edieresis
+0xCC U+00CC Igrave
+0xCD U+00CD Iacute
+0xCE U+00CE Icircumflex
+0xCF U+00CF Idieresis
+0xD0 U+00D0 Eth
+0xD1 U+00D1 Ntilde
+0xD2 U+00D2 Ograve
+0xD3 U+00D3 Oacute
+0xD4 U+00D4 Ocircumflex
+0xD5 U+00D5 Otilde
+0xD6 U+00D6 Odieresis
+0xD7 U+00A4 currency
+0xD8 U+2030 perthousand
+0xD9 U+00D9 Ugrave
+0xDA U+00DA Uacute
+0xDB U+00DB Ucircumflex
+0xDC U+00DC Udieresis
+0xDD U+00DD Yacute
+0xDE U+00DE Thorn
+0xDF U+007C bar
+0xE0 U+00E0 1 agrave
+0xE1 U+00E1 1 aacute
+0xE2 U+00E2 1 acircumflex
+0xE3 U+00E3 1 atilde
+0xE4 U+00E4 1 adieresis
+0xE5 U+00E5 1 aring
+0xE6 U+005F underscore
+0xE7 U+00E7 1 ccedilla
+0xE8 U+00E8 1 egrave
+0xE9 U+00E9 1 eacute
+0xEA U+00EA 1 ecircumflex
+0xEB U+00EB 1 edieresis
+0xEC U+00EC 1 igrave
+0xED U+00ED 1 iacute
+0xEE U+00EE 1 icircumflex
+0xEF U+00EF 1 idieresis
+0xF0 U+00F0 1 eth
+0xF1 U+00F1 1 ntilde
+0xF2 U+00F2 1 ograve
+0xF3 U+00F3 1 oacute
+0xF4 U+00F4 1 ocircumflex
+0xF5 U+00F5 1 otilde
+0xF6 U+00F6 1 odieresis
+0xF7 U+2222 anglearc
+0xF8 U+2300 diameter
+0xF9 U+00F9 1 ugrave
+0xFA U+00FA 1 uacute
+0xFB U+00FB 1 ucircumflex
+0xFC U+00FC 1 udieresis
+0xFD U+00FD 1 yacute
+0xFE U+00FE 1 thorn
+0xFF U+201E quotedblbase
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat
new file mode 100644
index 0000000000..23b8d8fa4a
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat
@@ -0,0 +1,256 @@
+0x00 U+0060
+0x01 U+00B4
+0x02 U+02C6
+0x03 U+02DC
+0x04 U+00A8
+0x05 U+02DD
+0x06 U+02DA
+0x07 U+02C7
+0x08 U+02D8
+0x09 U+00AF
+0x0A U+02D9
+0x0B U+00B8
+0x0C U+02DB
+0x0D U+04C0 CYRpalochka
+0x0E U+2329 textlangle
+0x0F U+232A textrangle
+0x10 U+201C textquotedblleft
+0x11 U+201D textquotedblright
+0x12 U+F6D5
+0x13 U+F6D6
+0x14 U+F6D4
+0x15 U+2013 textendash
+0x16 U+2014 textemdash
+# 0x17 U+0x17 Nothing?
+# 0x18 U+0x18 ?
+0x19 U+0131 1 i
+0x1A U+F6BE
+0x1B U+FB00 # Hand-edited!
+0x1C U+FB01 #
+0x1D U+FB02 #
+0x1E U+FB03 #
+0x1F U+FB04 #
+0x20 U+0020
+0x21 U+0021 exclam
+0x22 U+0022 quotedbl
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+2019 quoteright
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+003C less
+0x3D U+003D equal
+0x3E U+003E greater
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0406 CYRII
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+005C backslash
+0x5D U+005D bracketright
+0x5E U+02C6 circumflex
+0x5F U+005F underscore
+0x60 U+2018 quoteleft
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0456 1 cyrii
+# Hum ... (fix for Serbian)
+0x6A U+0458 1 cyrj
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+007B braceleft
+0x7C U+007C bar
+0x7D U+007D braceright
+0x7E U+02DC tilde
+0x80 U+0490 CYRGUP
+0x81 U+0492 CYRGHCRS
+0x82 U+0402 CYRDJE
+0x83 U+040B CYRTSHE
+0x84 U+04BA CYRSHHA
+0x85 U+0496 CYRZHDSC
+0x86 U+0498 CYRZDSC
+0x87 U+0409 CYRLJE
+0x88 U+0407 CYRYI
+0x89 U+049A CYRKDSC
+0x8A U+04A0
+0x8B U+049C CYRKVCRS
+0x8C U+04D4 CYRAE
+0x8D U+04A2 CYRNDSC
+0x8E U+04A4 CYRNG
+0x8F U+0405 CYRDZE
+0x90 U+04E8 CYROTLD
+0x91 U+04AA CYRSDSC
+0x92 U+040E CYRUSHRT
+0x93 U+04AE CYRY
+0x94 U+04B0 CYRYHCRS
+0x95 U+04B2 CYRHDSC
+0x96 U+040F CYRDZHE
+0x97 U+04B8 CYRCHVCRS
+0x98 U+04B6 CYRCHRDSC
+0x99 U+0404 CYRIE
+0x9A U+04D8 CYRSCHWA
+0x9B U+040A CYRNJE
+0x9C U+0401 CYRYO
+0x9D U+2116 textnumero
+0x9E U+00A4 textcurrency
+0x9F U+00A7 textsection
+0xA0 U+0491 1 cyrgup
+0xA1 U+0493 1 cyrghcrs
+0xA2 U+0452 1 cyrdje
+0xA3 U+045B 1 cyrtshe
+0xA4 U+04BB 1 cyrshha
+0xA5 U+0497 1 cyrzhdsc
+0xA6 U+0499 1 cyrzdsc
+0xA7 U+0459 1 cyrlje
+0xA8 U+0457 1 cyryi
+0xA9 U+049B 1 cyrkdsc
+0xAA U+04A1 1
+0xAB U+049D 1 cyrkvcrs
+0xAC U+04D5 1 cyrae
+0xAD U+04A3 1 cyrndsc
+0xAE U+04A5 1 cyrng
+0xAF U+0455 1 cyrdze
+0xB0 U+04E9 1 cyrotld
+0xB1 U+04AB 1 cyrsdsc
+0xB2 U+045E 1 cyrushrt
+0xB3 U+04AF 1 cyry
+0xB4 U+04B1 1 cyryhcrs
+0xB5 U+04B3 1 cyrhdsc
+0xB6 U+045F 1 cyrdzhe
+0xB7 U+04B9 1 cyrchvcrs
+0xB8 U+04B7 1 cyrchrdsc
+0xB9 U+0454 1 cyrie
+0xBA U+04D9 1 cyrschwa
+0xBB U+045A 1 cyrnje
+0xBC U+0451 1 cyryo
+0xBD U+201E quotedblbase
+0xBE U+00AB guillemotleft
+0xBF U+00BB guillemotright
+0xC0 U+0410 CYRA
+0xC1 U+0411 CYRB
+0xC2 U+0412 CYRV
+0xC3 U+0413 CYRG
+0xC4 U+0414 CYRD
+0xC5 U+0415 CYRE
+0xC6 U+0416 CYRZH
+0xC7 U+0417 CYRZ
+0xC8 U+0418 CYRI
+0xC9 U+0419 CYRISHRT
+0xCA U+041A CYRK
+0xCB U+041B CYRL
+0xCC U+041C CYRM
+0xCD U+041D CYRN
+0xCE U+041E CYRO
+0xCF U+041F CYRP
+0xD0 U+0420 CYRR
+0xD1 U+0421 CYRS
+0xD2 U+0422 CYRT
+0xD3 U+0423 CYRU
+0xD4 U+0424 CYRF
+0xD5 U+0425 CYRH
+0xD6 U+0426 CYRC
+0xD7 U+0427 CYRCH
+0xD8 U+0428 CYRSH
+0xD9 U+0429 CYRSHCH
+0xDA U+042A CYRHRDSN
+0xDB U+042B CYRERY
+0xDC U+042C CYRSFTSN
+0xDD U+042D CYREREV
+0xDE U+042E CYRYU
+0xDF U+042F CYRYA
+0xE0 U+0430 1 cyra
+0xE1 U+0431 1 cyrb
+0xE2 U+0432 1 cyrv
+0xE3 U+0433 1 cyrg
+0xE4 U+0434 1 cyrd
+0xE5 U+0435 1 cyre
+0xE6 U+0436 1 cyrzh
+0xE7 U+0437 1 cyrz
+0xE8 U+0438 1 cyri
+0xE9 U+0439 1 cyrishrt
+0xEA U+043A 1 cyrk
+0xEB U+043B 1 cyrl
+0xEC U+043C 1 cyrm
+0xED U+043D 1 cyrn
+0xEE U+043E 1 cyro
+0xEF U+043F 1 cyrp
+0xF0 U+0440 1 cyrr
+0xF1 U+0441 1 cyrs
+0xF2 U+0442 1 cyrt
+0xF3 U+0443 1 cyru
+0xF4 U+0444 1 cyrf
+0xF5 U+0445 1 cyrh
+0xF6 U+0446 1 cyrc
+0xF7 U+0447 1 cyrch
+0xF8 U+0448 1 cyrsh
+0xF9 U+0449 1 cyrshch
+0xFA U+044A 1 cyrhrdsn
+0xFB U+044B 1 cyrery
+0xFC U+044C 1 cyrsftsn
+0xFD U+044D 1 cyrerev
+0xFE U+044E 1 cyryu
+0xFF U+044F 1 cyrya
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat
new file mode 100644
index 0000000000..78e09abc94
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat
@@ -0,0 +1,256 @@
+0x00 U+.... .notdef
+0x01 U+.... .notdef
+0x02 U+.... .notdef
+0x03 U+.... .notdef
+0x04 U+.... .notdef
+0x05 U+.... .notdef
+0x06 U+.... .notdef
+0x07 U+.... .notdef
+0x08 U+.... .notdef
+0x09 U+00A9 copyright
+0x0A U+00AB guillemotleft
+0x0B U+00BB guillemotright
+0x0C U+00AE registered
+0x0D U+00A3 sterling
+0x0E U+2020 dagger
+0x0F U+2116 uni2116
+0x10 U+201E quotedblbase
+0x11 U+201C quotedblleft
+0x12 U+.... .notdef
+0x13 U+.... .notdef
+0x14 U+.... .notdef
+0x15 U+.... .notdef
+0x16 U+.... .notdef
+0x17 U+.... .notdef
+0x18 U+.... .notdef
+0x19 U+.... .notdef
+0x1A U+.... .notdef
+0x1B U+.... .notdef
+0x1C U+.... .notdef
+0x1D U+.... .notdef
+0x1E U+.... .notdef
+0x1F U+.... .notdef
+0x20 U+0020 space
+0x21 U+0021 exclam
+0x22 U+0022 quotedbl
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+0027 quotesingle
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+003C less
+0x3D U+003D equal
+0x3E U+003E greater
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0049 I
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+005C backslash
+0x5D U+005D bracketright
+0x5E U+005E asciicircum
+0x5F U+005F underscore
+0x60 U+0060 grave
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0069 i
+0x6A U+006A j
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+007B braceleft
+0x7C U+007C bar
+0x7D U+007D braceright
+0x7E U+007E asciitilde
+0x7F U+.... .notdef
+0x80 U+10FB uni10FB
+0x81 U+10A0 uni10A0
+0x82 U+10A1 uni10A1
+0x83 U+10A2 uni10A2
+0x84 U+10A3 uni10A3
+0x85 U+10A4 uni10A4
+0x86 U+10A5 uni10A5
+0x87 U+10A6 uni10A6
+0x88 U+10A7 uni10A7
+0x89 U+10A8 uni10A8
+0x8A U+10A9 uni10A9
+0x8B U+10AA uni10AA
+0x8C U+10AB uni10AB
+0x8D U+10AC uni10AC
+0x8E U+10AD uni10AD
+0x8F U+10AE uni10AE
+0x90 U+10AF uni10AF
+0x91 U+10B0 uni10B0
+0x92 U+10B1 uni10B1
+0x93 U+10B2 uni10B2
+0x94 U+10B3 uni10B3
+0x95 U+10B4 uni10B4
+0x96 U+10B5 uni10B5
+0x97 U+10B6 uni10B6
+0x98 U+10B7 uni10B7
+0x99 U+10B8 uni10B8
+0x9A U+10B9 uni10B9
+0x9B U+10BA uni10BA
+0x9C U+10BB uni10BB
+0x9D U+2013 endash
+0x9E U+2014 emdash
+0x9F U+00A7 section
+0xA0 U+00A0 nonbreakingspace
+0xA1 U+10D0 1 uni10D0
+0xA2 U+10D1 1 uni10D1
+0xA3 U+10D2 1 uni10D2
+0xA4 U+10D3 1 uni10D3
+0xA5 U+10D4 1 uni10D4
+0xA6 U+10D5 1 uni10D5
+0xA7 U+10D6 1 uni10D6
+0xA8 U+10D7 1 uni10D7
+0xA9 U+10D8 1 uni10D8
+0xAA U+10D9 1 uni10D9
+0xAB U+10DA 1 uni10DA
+0xAC U+10DB 1 uni10DB
+0xAD U+10DC 1 uni10DC
+0xAE U+10DD 1 uni10DD
+0xAF U+10DE 1 uni10DE
+0xB0 U+10DF 1 uni10DF
+0xB1 U+10E0 1 uni10E0
+0xB2 U+10E1 1 uni10E1
+0xB3 U+10E2 1 uni10E2
+0xB4 U+10E3 1 uni10E3
+0xB5 U+10E4 1 uni10E4
+0xB6 U+10E5 1 uni10E5
+0xB7 U+10E6 1 uni10E6
+0xB8 U+10E7 1 uni10E7
+0xB9 U+10E8 1 uni10E8
+0xBA U+10E9 1 uni10E9
+0xBB U+10EA 1 uni10EA
+0xBC U+10EB 1 uni10EB
+0xBD U+.... .notdef
+0xBE U+.... .notdef
+0xBF U+.... .notdef
+0xC0 U+10BC uni10BC
+0xC1 U+10BD uni10BD
+0xC2 U+10BE uni10BE
+0xC3 U+10BF uni10BF
+0xC4 U+10C0 uni10C0
+0xC5 U+10C1 uni10C1
+0xC6 U+10C2 uni10C2
+0xC7 U+10C3 uni10C3
+0xC8 U+10C4 uni10C4
+0xC9 U+10C5 uni10C5
+0xCA U+.... .notdef
+0xCB U+.... .notdef
+0xCC U+.... .notdef
+0xCD U+.... .notdef
+0xCE U+.... .notdef
+0xCF U+.... .notdef
+0xD0 U+.... .notdef
+0xD1 U+.... .notdef
+0xD2 U+.... .notdef
+0xD3 U+.... .notdef
+0xD4 U+.... .notdef
+0xD5 U+.... .notdef
+0xD6 U+.... .notdef
+0xD7 U+.... .notdef
+0xD8 U+.... .notdef
+0xD9 U+.... .notdef
+0xDA U+.... .notdef
+0xDB U+.... .notdef
+0xDC U+.... .notdef
+0xDD U+.... .notdef
+0xDE U+.... .notdef
+0xDF U+.... .notdef
+0xE0 U+10EC 1 uni10EC
+0xE1 U+10ED 1 uni10ED
+0xE2 U+10EE 1 uni10EE
+0xE3 U+10EF 1 uni10EF
+0xE4 U+10F0 1 uni10F0
+0xE5 U+10F1 1 uni10F1
+0xE6 U+10F2 1 uni10F2
+0xE7 U+10F3 1 uni10F3
+0xE8 U+10F4 1 uni10F4
+0xE9 U+10F5 1 uni10F5
+0xEA U+10F6 1 uni10F6
+0xEB U+10F7 1 uni10F7
+0xEC U+10F8 1 uni10F8
+0xED U+10F9 1 uni10F9
+0xEE U+10FA 1 uni10FA
+0xEF U+10FC uni10FC
+0xF0 U+.... .notdef
+0xF1 U+.... .notdef
+0xF2 U+.... .notdef
+0xF3 U+.... .notdef
+0xF4 U+.... .notdef
+0xF5 U+.... .notdef
+0xF6 U+.... .notdef
+0xF7 U+.... .notdef
+0xF8 U+.... .notdef
+0xF9 U+.... .notdef
+0xFA U+.... .notdef
+0xFB U+.... .notdef
+0xFC U+.... .notdef
+0xFD U+.... .notdef
+0xFE U+.... .notdef
+0xFF U+.... .notdef
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat
new file mode 100644
index 0000000000..96aaea03a0
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat
@@ -0,0 +1,256 @@
+0x00 U+.... .notdef
+0x01 U+20AC Euro
+0x02 U+.... .notdef
+0x03 U+.... .notdef
+0x04 U+2044 fraction
+0x05 U+02D9 dotaccent
+0x06 U+02DD hungarumlaut
+0x07 U+02DB ogonek
+0x08 U+FB02 fl
+0x09 U+.... .notdef
+0x0A U+200B cwm
+0x0B U+FB00 ff
+0x0C U+FB01 fi
+0x0D U+.... .notdef
+0x0E U+FB03 ffi
+0x0F U+FB04 ffl
+0x10 U+0131 1 dotlessi
+0x11 U+0237 1 dotlessj
+0x12 U+0060 grave
+0x13 U+00B4 acute
+0x14 U+02C7 caron
+0x15 U+02D8 breve
+0x16 U+00AF macron
+0x17 U+02DA ring
+0x18 U+00B8 cedilla
+0x19 U+00DF 1 germandbls
+0x1A U+00E6 1 ae
+0x1B U+0153 1 oe
+0x1C U+00F8 1 oslash
+0x1D U+00C6 AE
+0x1E U+0152 OE
+0x1F U+00D8 Oslash
+0x20 U+0020 space
+0x21 U+0021 exclam
+0x22 U+0022 quotedbl
+0x23 U+0023 numbersign
+0x24 U+0024 dollar
+0x25 U+0025 percent
+0x26 U+0026 ampersand
+0x27 U+2019 quoteright
+0x28 U+0028 parenleft
+0x29 U+0029 parenright
+0x2A U+002A asterisk
+0x2B U+002B plus
+0x2C U+002C comma
+0x2D U+002D hyphen
+0x2E U+002E period
+0x2F U+002F slash
+0x30 U+0030 zero
+0x31 U+0031 one
+0x32 U+0032 two
+0x33 U+0033 three
+0x34 U+0034 four
+0x35 U+0035 five
+0x36 U+0036 six
+0x37 U+0037 seven
+0x38 U+0038 eight
+0x39 U+0039 nine
+0x3A U+003A colon
+0x3B U+003B semicolon
+0x3C U+003C less
+0x3D U+003D equal
+0x3E U+003E greater
+0x3F U+003F question
+0x40 U+0040 at
+0x41 U+0041 A
+0x42 U+0042 B
+0x43 U+0043 C
+0x44 U+0044 D
+0x45 U+0045 E
+0x46 U+0046 F
+0x47 U+0047 G
+0x48 U+0048 H
+0x49 U+0049 I
+0x4A U+004A J
+0x4B U+004B K
+0x4C U+004C L
+0x4D U+004D M
+0x4E U+004E N
+0x4F U+004F O
+0x50 U+0050 P
+0x51 U+0051 Q
+0x52 U+0052 R
+0x53 U+0053 S
+0x54 U+0054 T
+0x55 U+0055 U
+0x56 U+0056 V
+0x57 U+0057 W
+0x58 U+0058 X
+0x59 U+0059 Y
+0x5A U+005A Z
+0x5B U+005B bracketleft
+0x5C U+005C backslash
+0x5D U+005D bracketright
+0x5E U+02C6 circumflex
+0x5F U+005F underscore
+0x60 U+2018 quoteleft
+0x61 U+0061 a
+0x62 U+0062 b
+0x63 U+0063 c
+0x64 U+0064 d
+0x65 U+0065 e
+0x66 U+0066 f
+0x67 U+0067 g
+0x68 U+0068 h
+0x69 U+0069 i
+0x6A U+006A j
+0x6B U+006B k
+0x6C U+006C l
+0x6D U+006D m
+0x6E U+006E n
+0x6F U+006F o
+0x70 U+0070 p
+0x71 U+0071 q
+0x72 U+0072 r
+0x73 U+0073 s
+0x74 U+0074 t
+0x75 U+0075 u
+0x76 U+0076 v
+0x77 U+0077 w
+0x78 U+0078 x
+0x79 U+0079 y
+0x7A U+007A z
+0x7B U+007B braceleft
+0x7C U+007C bar
+0x7D U+007D braceright
+0x7E U+02DC tilde
+0x7F U+00A8 dieresis
+0x80 U+0141 Lslash
+0x81 U+0027 quotesingle
+0x82 U+201A quotesinglbase
+0x83 U+0192 1 florin
+0x84 U+201E quotedblbase
+0x85 U+2026 ellipsis
+0x86 U+2020 dagger
+0x87 U+2021 daggerdbl
+0x88 U+02C6 circumflex
+0x89 U+2030 perthousand
+0x8A U+0160 Scaron
+0x8B U+2039 guilsinglleft
+0x8C U+0152 OE
+0x8D U+017D Zcaron
+0x8E U+005E asciicircum
+0x8F U+2212 minus
+0x90 U+0142 1 lslash
+0x91 U+2018 quoteleft
+0x92 U+2019 quoteright
+0x93 U+201C quotedblleft
+0x94 U+201D quotedblright
+0x95 U+2022 bullet
+0x96 U+2013 endash
+0x97 U+2014 emdash
+0x98 U+02DC tilde
+0x99 U+2122 trademark
+0x9A U+0161 1 scaron
+0x9B U+203A guilsinglright
+0x9C U+0153 1 oe
+0x9D U+017E 1 zcaron
+0x9E U+007E asciitilde
+0x9F U+0178 Ydieresis
+0xA0 U+00A0 nbspace
+0xA1 U+00A1 exclamdown
+0xA2 U+00A2 cent
+0xA3 U+00A3 sterling
+0xA4 U+00A4 currency
+0xA5 U+00A5 yen
+0xA6 U+00A6 brokenbar
+0xA7 U+00A7 section
+0xA8 U+00A8 dieresis
+0xA9 U+00A9 copyright
+0xAA U+00AA 1 ordfeminine
+0xAB U+00AB guillemotleft
+0xAC U+00AC logicalnot
+0xAD U+00AD sfthyphen
+0xAE U+00AE registered
+0xAF U+00AF macron
+0xB0 U+00B0 degree
+0xB1 U+00B1 plusminus
+0xB2 U+00B2 twosuperior
+0xB3 U+00B3 threesuperior
+0xB4 U+00B4 acute
+0xB5 U+00B5 mu
+0xB6 U+00B6 paragraph
+0xB7 U+00B7 periodcentered
+0xB8 U+00B8 cedilla
+0xB9 U+00B9 onesuperior
+0xBA U+00BA 1 ordmasculine
+0xBB U+00BB guillemotright
+0xBC U+00BC onequarter
+0xBD U+00BD onehalf
+0xBE U+00BE threequarters
+0xBF U+00BF questiondown
+0xC0 U+00C0 Agrave
+0xC1 U+00C1 Aacute
+0xC2 U+00C2 Acircumflex
+0xC3 U+00C3 Atilde
+0xC4 U+00C4 Adieresis
+0xC5 U+00C5 Aring
+0xC6 U+00C6 AE
+0xC7 U+00C7 Ccedilla
+0xC8 U+00C8 Egrave
+0xC9 U+00C9 Eacute
+0xCA U+00CA Ecircumflex
+0xCB U+00CB Edieresis
+0xCC U+00CC Igrave
+0xCD U+00CD Iacute
+0xCE U+00CE Icircumflex
+0xCF U+00CF Idieresis
+0xD0 U+00D0 Eth
+0xD1 U+00D1 Ntilde
+0xD2 U+00D2 Ograve
+0xD3 U+00D3 Oacute
+0xD4 U+00D4 Ocircumflex
+0xD5 U+00D5 Otilde
+0xD6 U+00D6 Odieresis
+0xD7 U+00D7 multiply
+0xD8 U+00D8 Oslash
+0xD9 U+00D9 Ugrave
+0xDA U+00DA Uacute
+0xDB U+00DB Ucircumflex
+0xDC U+00DC Udieresis
+0xDD U+00DD Yacute
+0xDE U+00DE Thorn
+0xDF U+00DF 1 germandbls
+0xE0 U+00E0 1 agrave
+0xE1 U+00E1 1 aacute
+0xE2 U+00E2 1 acircumflex
+0xE3 U+00E3 1 atilde
+0xE4 U+00E4 1 adieresis
+0xE5 U+00E5 1 aring
+0xE6 U+00E6 1 ae
+0xE7 U+00E7 1 ccedilla
+0xE8 U+00E8 1 egrave
+0xE9 U+00E9 1 eacute
+0xEA U+00EA 1 ecircumflex
+0xEB U+00EB 1 edieresis
+0xEC U+00EC 1 igrave
+0xED U+00ED 1 iacute
+0xEE U+00EE 1 icircumflex
+0xEF U+00EF 1 idieresis
+0xF0 U+00F0 1 eth
+0xF1 U+00F1 1 ntilde
+0xF2 U+00F2 1 ograve
+0xF3 U+00F3 1 oacute
+0xF4 U+00F4 1 ocircumflex
+0xF5 U+00F5 1 otilde
+0xF6 U+00F6 1 odieresis
+0xF7 U+00F7 divide
+0xF8 U+00F8 1 oslash
+0xF9 U+00F9 1 ugrave
+0xFA U+00FA 1 uacute
+0xFB U+00FB 1 ucircumflex
+0xFC U+00FC 1 udieresis
+0xFD U+00FD 1 yacute
+0xFE U+00FE 1 thorn
+0xFF U+00FF 1 ydieresis
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb
new file mode 100755
index 0000000000..d76aa29657
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb
@@ -0,0 +1,177 @@
+#!/usr/bin/env ruby
+
+require_relative 'hyph-utf8'
+
+$path_root=File.expand_path("../../../..", __FILE__)
+$encoding_data_dir = File.expand_path("../data/encodings", __FILE__)
+$output_data_dir = "#{$path_root}/tex/generic/hyph-utf8/conversions"
+
+def output_file_name(encoding)
+ File.join($output_data_dir, sprintf('conv-utf8-%s.tex', encoding))
+end
+
+$header = <<__EOHEADER__
+%% conv-utf8-%s.tex
+%%
+%% Conversion from UTF-8 to %s,
+%% used before loading hyphenation patterns for 8-bit TeX engines.
+%%
+%% This file is part of hyph-utf8 package and autogenerated.
+%% See http://tug.org/tex-hyphen
+%%
+%% Copyright 2008-%d TeX Users Group.
+%% You may freely use, modify and/or distribute this file.
+%% (But consider adapting the scripts if you need modifications.)
+__EOHEADER__
+
+def output_copyright_notice(outfile, encoding)
+ outfile.printf $header, encoding, encoding.upcase, Time.new.year
+end
+
+$uniconvmacro1 = <<__EOUNIMAC1__
+% macros adapted from ConTeXt MKII; see unic-ini.mkii
+\\def\\unicodechar#1{%
+ \\ifcsname unichar@\\number#1\\endcsname
+ \\csname unichar@\\number#1\\endcsname
+ \\else
+ \\errmessage{Unicode character [#1] not in encoding.}%
+ \\fi}
+__EOUNIMAC1__
+
+$uniconvmacros = [nil, nil]
+
+$uniconvmacros << <<__EOTWOBYTES__
+\\def\\utftwouniglyph#1#2%
+ {\\expandafter\\unicodechar\\expandafter
+ {\\the\\numexpr64*(#1-192)+`#2-128\\relax}}
+__EOTWOBYTES__
+
+$uniconvmacros << <<__EOTHREEBYTES__
+\\def\\utfthreeuniglyph#1#2#3%
+ {\\expandafter\\unicodechar\\expandafter
+ {\\the\\numexpr4096*(#1-224)+64*(`#2-128)+`#3-128\\relax}}
+__EOTHREEBYTES__
+
+$uniconvmacros << <<__EOFOURBYTES__
+\\def\\utffouruniglyph#1#2#3#4%
+ {\\expandafter\\unicodechar\\expandafter
+ {\\the\\numexpr262144*(#1-240)+4096*(`#2-128)+64*(`#3-128)+`#4-128\\relax}}
+__EOFOURBYTES__
+
+$uniconvmacro2 = <<__EOUNIMAC2__
+
+\\def\\addunichar #1 #2 {\\expandafter\\def\\csname unichar@\\number#1\\endcsname{#2}}
+
+% \\addunichar "unicode_code - ^^font_encoding_code
+__EOUNIMAC2__
+
+["t8m", "lth"].each do |encoding|
+ # load encoding
+ e = HyphEncoding.new(encoding)
+
+ # open file
+ File.open(output_file_name(encoding), "w") do |file_out|
+
+ # copyright notice
+ output_copyright_notice(file_out, encoding)
+ file_out.puts
+
+ # macro to get mapping unicode -> font encoding & error message if screwed up
+ file_out.puts $uniconvmacro1
+
+ # minimal and maximal length of characters in the encoding (until now just 2 & 3)
+ unicode_characters_array = e.unicode_characters.sort
+ length_min = unicode_characters_array.first[1].bytes.size
+ length_max = unicode_characters_array.last[1].bytes.size
+
+ # only output the necessary macros for transforming UTF-8 -> Unicode number
+ (length_min..length_max).each do |nbytes|
+ file_out.puts $uniconvmacros[nbytes]
+ end
+
+ # macro to store mapping unicode -> font encoding
+ file_out.puts $uniconvmacro2
+
+ # all unicode characters in the encoding
+ e.unicode_characters.sort.each do |code,c|
+ file_out.printf("\\addunichar \"%04X ^^%02x \\lccode\"%02X=\"%02X %% %s - %s\n",
+ c.code_uni, c.code_enc, c.code_enc, c.code_enc, [c.code_uni].pack('U'), c.name)
+ end
+ file_out.puts
+
+ # make all the possible first characters active
+ # output the definition into file
+ e.unicode_characters_first_byte.sort.each do |first_byte_code,chars|
+ byte = first_byte_code.hex
+ size = chars[0].bytes.size
+ # 2-byte: 0b11000000 <= byte < 0b11100000
+ str = case size
+ when 2 then
+ "two"
+ # 3-byte: 0b11100000 <= byte < 0b11110000
+ when 3 then
+ "three"
+ # 4-byte: 0b11110000 <= byte < 0b11111000
+ when 4 then
+ "four"
+ end
+ file_out.printf("\\catcode\"%02X=\\active \\def^^%02x{\\utf%suniglyph{\"%02X}}\n", byte, byte, str, byte)
+ end
+ end
+end
+
+["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"].each do |encoding|
+ # load encoding
+ e = HyphEncoding.new(encoding)
+
+ # open file
+ File.open(output_file_name(encoding), "w") do |file_out|
+
+ # copyright notice
+ output_copyright_notice(file_out, encoding)
+ file_out.puts '%'
+
+ e.unicode_characters_first_byte.sort.each do |first_byte_code,chars|
+ # sorting all the second characters alphabetically
+ chars.sort!{|x,y| x.code_uni <=> y.code_uni }
+ # make all the possible first characters active
+ # output the definition into file
+ file_out.printf("\\catcode\"%02X=\\active\n", first_byte_code.hex)
+ end
+ file_out.puts "%"
+ e.unicode_characters_first_byte.sort.each do |first_byte_code,chars|
+ first_byte_code = first_byte_code.hex
+ size = chars[0].bytes.size
+ if size != 2 then
+ throw "The encoding #{encoding} uses more than two bytes to encode characters"
+ else
+
+ file_out.printf("\\def^^%02x#1{%%\n", first_byte_code)
+ string_fi = ""
+ for i in 1..(chars.size)
+ uni_character = chars[i-1]
+ enc_byte = uni_character.code_enc
+ enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2')
+ file_out.printf("\t\\ifx#1^^%02x^^%02x\\else %% %s - U+%04X - %s\n", uni_character.bytes[1], uni_character.code_enc, [uni_character.code_uni].pack('U'), uni_character.code_uni, uni_character.name)
+ string_fi = string_fi + "\\fi"
+ end
+
+ # at least three bytes
+ end
+ file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}"
+ file_out.puts string_fi + "}"
+ end
+ file_out.puts '%'
+ file_out.puts '% ensure all the chars above have valid \lccode values'
+ file_out.puts '%'
+ e.lowercase_characters.each do |character|
+ code = [ character.code_enc ].pack("c").unpack("H2").first.upcase
+ # \lccode"FF="FF
+ file_out.printf "\\lccode\"%s=\"%s %% %s - U+%04X - %s\n", code, code, [character.code_uni].pack('U'), character.code_uni, character.name
+ end
+
+ file_out.puts
+ end
+end
+
+
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb
new file mode 100755
index 0000000000..c0781932a3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb
@@ -0,0 +1,122 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# this file auto-generates loaders for hyphenation patterns - to be improved # No shit, Sherlock -- AR 2018-11-27
+
+require_relative 'lib/tex/hyphen'
+require_relative 'lib/tex/hyphen/texlive'
+include TeX::Hyphen
+include TeXLive
+
+#text_if_native_utf = "\input pattern-loader.tex\n\\ifNativeUtfEightPatterns"
+
+def output(file, string, indent = 0)
+ if string.is_a? Enumerable
+ string.each { |line| output(file, line, indent) }
+ else
+ file.print ' ' * indent
+ file.puts(string)
+ end
+end
+
+print 'Generating loaders for '
+Language.all.each do |language|
+
+# puts language.bcp47
+
+################
+# Header texts #
+################
+
+# a message about auto-generation
+# TODO: write a more comprehensive one
+ text_header = <<-EOHEADER
+% filename: loadhyph-#{language.bcp47}.tex
+% language: #{language.babelname}
+%
+% Loader for hyphenation patterns, generated by
+% source/generic/hyph-utf8/generate-pattern-loaders.rb
+% See also http://tug.org/tex-hyphen
+%
+% Copyright 2008-#{Time.now.year} TeX Users Group.
+% You may freely use, modify and/or distribute this file.
+% (But consider adapting the scripts if you need modifications.)
+%
+% Once it turns out that more than a simple definition is needed,
+% these lines may be moved to a separate file.
+%
+ EOHEADER
+
+###########
+# lccodes #
+###########
+
+ lccodes_common = []
+ if language.has_apostrophes? then
+ lccodes_common.push("\\lccode`\\'=`\\'")
+ end
+ if language.has_hyphens? then
+ lccodes_common.push("\\lccode`\\-=`\\-")
+ end
+
+ next if language.use_old_loader
+ print language.bcp47, ' '
+
+ filename = File.join(PATH::LOADER, language.loadhyph)
+ File.open(filename, "w") do |file|
+ # puts language.bcp47
+ file.puts text_header
+ file.puts('\begingroup')
+
+ if lccodes_common.length > 0 then
+ file.puts lccodes_common.join("\n")
+ end
+
+ # for ASCII encoding, we don't load any special support files, but simply load everything
+ if language.encoding == 'ascii' && !language.italic?
+ file.puts "% ASCII patterns - no additional support is needed"
+ file.puts "\\message{ASCII #{language.message}}"
+ file.puts "\\input hyph-#{language.bcp47}.tex"
+ else
+ file.puts '% Test for pTeX
+\\ifx\\kanjiskip\\undefined
+% Test for native UTF-8 (which gets only a single argument)
+% That\'s Tau (as in Taco or ΤΕΧ, Tau-Epsilon-Chi), a 2-byte UTF-8 character
+\\def\\testengine#1#2!{\\def\\secondarg{#2}}\\testengine Τ!\\relax
+\\ifx\\secondarg\\empty'
+ output(file, language.format_inputs(language.utf8_chunk), 2)
+ file.puts("\\else\n")
+ output(file, language.format_inputs(language.nonutf8_chunk('8-bit')), 2)
+ file.puts("\\fi\\else\n")
+ output(file, language.format_inputs(language.nonutf8_chunk('pTeX')), 2)
+ file.puts("\\fi\n")
+ end
+
+########################################
+# GROUP nr. 1 - ONLY USABLE WITH UTF-8 #
+########################################
+ # some special cases first
+ #
+ # some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines
+ # TODO - maybe consider doing something similar for ibycus
+
+#######################
+# GROUP nr. 2 - ASCII #
+#######################
+
+####################################
+# GROUP nr. 3 - different patterns #
+####################################
+ # when lanugage uses old patterns for 8-bit engines, load two different patterns rather than using the converter
+ # greek, coptic
+#########################
+# GROUP nr. 4 - regular #
+#########################
+#######
+# end #
+#######
+ file.puts('\endgroup')
+ end
+end
+
+puts
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb
new file mode 100755
index 0000000000..eaa9cbf608
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb
@@ -0,0 +1,68 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# this file generates plain patterns (one-per-line) out of TeX source
+
+# use 'gem install unicode' if unicode is missing on your computer
+require 'unicode'
+
+require_relative 'lib/tex/hyphen/language.rb'
+# include OldLanguage::TeXLive
+include TeX::Hyphen
+
+# FIXME Close files!
+# FIXME sr-cyrl?
+print 'Generating plain files for (parenthesised tags are skipped) '
+Language.all.sort.each do |language|
+ bcp47 = language.bcp47
+
+ if language.use_old_loader || bcp47 == 'mn-cyrl-x-lmc'
+ print '(', language.bcp47, ') '
+ next
+ else
+ print bcp47, ' '
+ end
+
+ outfile = Proc.new do |ext|
+ File.open File.join(PATH::TXT, sprintf('hyph-%s.%s.txt', bcp47, ext)), 'w'
+ end
+
+ # patterns
+ patterns = language.extract_apostrophes
+ file = outfile.('pat')
+ patterns[:plain].each do |pattern|
+ file.puts pattern
+ end
+
+ # apostrophes if applicable
+ with_apostrophe = patterns[:with_apostrophe]
+ if with_apostrophe
+ file = File.open File.join(PATH::QUOTE, sprintf('hyph-quote-%s.tex', bcp47)), 'w'
+ file.printf "\\bgroup\n\\lccode`\\’=`\\’\n\\patterns{\n"
+ with_apostrophe.each do |pattern|
+ file.printf "%s\n", pattern
+ end
+ file.puts "}\n\\egroup\n"
+ end
+
+ # exceptions
+ unless language.exceptions == ""
+ file = outfile.('hyp') # This ensure a file is created, even if it may be empty
+ file.puts language.exceptions
+ end
+
+ ### FIXME Do issue #33 instead
+ ### # characters
+ ### file = outfile.('chr')
+ ### language.extract_characters.each do |character|
+ ### file.puts character
+ ### end
+
+ ### FIXME Do something else instead
+ ### # comments and licence
+ ### file = outfile.('lic')
+ ### file.puts language.comments_and_licence
+
+ ### file.close
+end
+puts
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb
new file mode 100755
index 0000000000..0f50706dc0
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb
@@ -0,0 +1,82 @@
+#!/usr/bin/env ruby
+
+require_relative 'hyph-utf8'
+require_relative 'lib/tex/hyphen'
+require_relative 'lib/tex/hyphen/texlive'
+include TeX::Hyphen
+include TeXLive
+
+# this file generates patterns for pTeX out of the plain ones
+
+# load encodings
+encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"]
+encodings = Hash.new
+encodings_list.each do |encoding_name|
+ encodings[encoding_name] = HyphEncoding.new(encoding_name)
+end
+
+header = <<-HEADER
+%% pTeX-friendly hyphenation patterns
+%%
+%% language: %s (%s)
+%% encoding: %s
+%%
+%% This file has been auto-generated from hyph-%s.tex
+%% with a script [texmf]/scripts/generic/hyph-utf8/generate-ptex-patterns.rb
+%% See the original file for details about author, licence etc.
+%%
+HEADER
+
+print 'Generating pTeX patterns for (skipped # reason): '
+Language.all.sort.each do |language|
+ if language.use_old_patterns_comment || !language.encoding || language.encoding == 'ascii'
+ if language.use_old_patterns_comment
+ reason = 'legacy patterns'
+ else
+ reason = sprintf 'encoding: %s', language.encoding.inspect
+ end
+ print '(', language.bcp47, ' # ', reason, ') '
+
+ next
+ else
+ encoding = encodings[language.encoding]
+ end
+
+ bcp47 = language.bcp47
+
+ print bcp47, ' '
+ File.open(File.join(PATH::PTEX, sprintf('hyph-%s.%s.tex', bcp47, language.encoding)), 'w') do |file_ptex|
+ patterns = language.patterns
+ exceptions = language.exceptions
+
+ characters = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq
+
+ if language.encoding != 'ascii' then
+ patterns = encoding.convert_to_escaped_characters(patterns)
+ exceptions = encoding.convert_to_escaped_characters(exceptions)
+ end
+
+ file_ptex.printf(header, language.babelname, language.bcp47, language.encoding, language.bcp47)
+
+ file_ptex.puts("\\bgroup")
+ # setting lccodes for letters
+ characters.each do |c|
+ if (c == 0x01FD or c == 0x0301) and language.bcp47 == 'la-x-liturgic'
+ # skip
+ elsif c >= 128 then
+ code = encoding.unicode_characters[c].code_enc
+ file_ptex.printf("\\lccode\"%02X=\"%02X\n", code, code)
+ end
+ end
+ # patterns
+ if patterns.length > 0 then
+ file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}")
+ end
+ # exceptions
+ if exceptions.length > 0 then
+ file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}")
+ end
+ file_ptex.puts("\\egroup")
+ end
+end
+puts
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb
new file mode 100755
index 0000000000..7460658f02
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb
@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+# encoding: utf-8
+
+# this file auto-generates tlpsrc files for hyphenation patterns - to be improved
+
+require_relative 'lib/tex/hyphen'
+require_relative 'lib/tex/hyphen/texlive'
+include TeX::Hyphen
+include TeXLive
+
+#--------#
+# TLPSRC #
+#--------#
+
+print 'Generating .tlpsrc files for packages: '
+Package.all.sort.each do |package|
+ tlpsrcname = File.join(PATH::TLPSRC, sprintf('hyphen-%s.tlpsrc', package.name.safe))
+ file_tlpsrc = File.open(tlpsrcname, 'w')
+ print package.name.safe, ' '
+
+ file_tlpsrc.puts "category TLCore"
+ package.list_dependencies.each do |dependency|
+ file_tlpsrc.puts dependency
+ end
+
+ # FIXME Still doesn’t work well for Latin
+ file_tlpsrc.printf "shortdesc %s.\n", package.description_s
+ package.description.split("\n").each do |line|
+ file_tlpsrc.printf "longdesc %s\n", line
+ end
+ package.languages.each do |language|
+ file_tlpsrc.printf "execute AddHyphen \\\n\tname=%s%s \\\n", language.babelname, language.list_synonyms
+ file_tlpsrc.printf "\t%s \\\n\t%s", language.list_hyphenmins, language.list_loader
+ if language.patterns_line + language.exceptions_line != ""
+ file_tlpsrc.printf " \\\n\t%s \\\n\t%s", language.patterns_line, language.exceptions_line
+ end
+ if language.bcp47 == "mn-cyrl-x-lmc" then
+ file_tlpsrc.printf " \\\n\tluaspecial=\"disabled:only for 8bit montex with lmc encoding\""
+ end
+ # end-of-line
+ file_tlpsrc.puts
+ end
+
+ # documentation
+ package.list_support_files('doc').sort.each do |filename|
+ file_tlpsrc.printf "docpattern d texmf-dist/%s\n", filename
+ end
+
+ # sources
+ package.list_support_files('source').sort.each do |filename|
+ file_tlpsrc.printf "srcpattern d texmf-dist/%s\n", filename
+ end
+
+ package.list_run_files.sort.uniq.each do |filename|
+ file_tlpsrc.printf "runpattern f texmf-dist/%s\n", filename
+ end
+ file_tlpsrc.close
+end
+puts
+
+#--------------------------#
+# language.dat and friends #
+#--------------------------#
+system sprintf "tlmgr generate --dest %s language.dat", File.join(PATH::LANGUAGE_DAT, 'language.dat')
+system sprintf "tlmgr generate --dest %s language.dat.lua", File.join(PATH::LANGUAGE_DAT, 'language.dat.lua')
+
+# Revert changes if only line with date was changed
+diff_lines = `git diff #{PATH::LANGUAGE_DAT}/language.dat* | egrep '^[+-](%|--) Generated' | wc -l`.strip
+`git checkout HEAD -- #{PATH::LANGUAGE_DAT}/language.dat*` if diff_lines == '4'
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb b/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb
new file mode 100644
index 0000000000..60005af380
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb
@@ -0,0 +1,116 @@
+# this is a Unicode character represented in some particular encoding
+class UnicodeCharacter
+ # unicode code
+ # code in that particular encoding
+ # character name (like 'eacute')
+ def initialize(code_uni, code_enc, name)
+ @code_uni = code_uni
+ @code_enc = code_enc
+ @bytes = [code_uni].pack('U').unpack('C*')
+ @name = name
+ end
+
+ attr_reader :code_uni, :code_enc, :bytes, :name
+end
+
+class UnicodeCharacters < Hash
+ # a hash based on the first character
+ def add_new_character_first_byte(code_uni, code_enc, name)
+ first_byte = [code_uni].pack('U').unpack('H2').first
+ if self[first_byte] == nil then
+ self[first_byte] = Array.new
+ end
+ self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name))
+ end
+ # a hash based on the whole unicode codepoint
+ def add_new_character(code_uni, code_enc, name)
+ self[code_uni] = UnicodeCharacter.new(code_uni, code_enc, name)
+ end
+end
+
+class HyphEncoding
+ def initialize(encoding_name)
+ @encoding_name = encoding_name
+ @unicode_characters_first_byte = UnicodeCharacters.new
+ @unicode_characters = UnicodeCharacters.new
+ @lowercase_characters = Array.new
+
+ if encoding_name != 'ascii' then
+ read_data
+ end
+ end
+
+ def convert_to_escaped_characters(str)
+ if str.kind_of?(Array) then
+ str.each_index do |i|
+ str[i] = convert_string_to_escaped_characters(str[i])
+ end
+ elsif str.kind_of?(String) then
+ str = convert_string_to_escaped_characters(str)
+ end
+ return str
+ end
+
+ attr_reader :encoding_name, :unicode_characters, :unicode_characters_first_byte, :lowercase_characters
+
+ def convert_string_to_escaped_characters(str)
+ skip_this_string = false
+ characters = str.unpack('U*')
+ new_string = Array.new(characters.length)
+ characters.each_index do |i|
+ c = characters[i] # character code on position i
+ # check if unicode entry with that number exists
+ uc = @unicode_characters[c]
+ if uc == nil then
+ if c < 128 then
+ new_string[i] = [c].pack('U')
+ elsif c == 8217 # ’
+ new_string[i] = "'"
+ elsif (c == 0x01FD or c == 0x0301) and @encoding_name == 'ec'
+ skip_this_string = true
+ new_string[i] = sprintf("[U+%04X]", c)
+ else
+ puts sprintf("There must be an error: character U+%04X in string '%s' is not ASCII or %s.", c, str, @encoding_name.upcase)
+ end
+ # an unicode character
+ else
+ new_string[i] = sprintf("^^%x", uc.code_enc)
+ end
+ end
+ if skip_this_string
+ new_string.unshift("% ")
+ end
+ return new_string.join('')
+ end
+
+private
+ def read_data
+ # fetch the characters
+ encoding_data_dir = File.expand_path("../data/encodings", __FILE__)
+ filename = "#{encoding_data_dir}/#{@encoding_name}.dat"
+
+ if File.exists?(filename) then
+ File.open(filename).grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z0-9\.]*)$/) do |line|
+ # puts line
+ code_enc = $1.hex
+ code_uni = $2.hex
+ if $3.length > 0
+ type = $3.to_i
+ else
+ type = 0
+ end
+ name = $4
+ if type == 1 then
+ @unicode_characters_first_byte.add_new_character_first_byte(code_uni, code_enc, name)
+ @unicode_characters.add_new_character(code_uni, code_enc, name)
+ @lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name))
+ end
+ @lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc}
+ end
+ else
+ # TODO: throw an error
+ puts "Invalid encoding name '#{@encoding_name}'."
+ puts "File '#{filename}' doesn't exist."
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb b/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb
new file mode 100644
index 0000000000..d4f387c072
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb
@@ -0,0 +1,327 @@
+class OldLanguage
+# "use_new_loader"
+# => true - create a new file and use that one
+# => false - use "filename_old_patterns" in language.dat
+# "filename_old_patterns"
+# => [string] - the name used in language.dat if "use_new_loader" is false
+# "eightbitfilename"
+# => [string] - if set, load another file for 8-bit engines
+# "code"
+# => [string] - used in filenames, needs to conform to the standard
+# "name"
+# => [string] -
+# "synonyms" => [],
+# "hyphenmin" => [],
+# "encoding" => nil,
+# "exceptions" => false,
+# "message" => nil,
+
+ @@language_data = [
+{
+ "code" => "el-monoton",
+ "type" => "rules",
+},
+{
+ "code" => "el-polyton",
+ "type" => "rules",
+},
+{
+ "code" => "grc",
+ "type" => "rules",
+},
+{
+ "code" => "cop",
+ "type" => "rules",
+},
+{
+ "code" => "de-1901",
+ "type" => "dictionary",
+},
+{
+ "code" => "de-1996",
+ "type" => "dictionary",
+},
+{
+ "code" => "de-ch-1901",
+ "type" => "dictionary",
+},
+{
+ "code" => "ru",
+ "type" => "dictionary",
+},
+{
+ "code" => "uk",
+ # "type" => "rules", # TODO: it is not really clear
+},
+{
+ "code" => "af",
+ "type" => "dictionary",
+},
+{
+ "code" => "ca",
+ "type" => "rules", # not only rules, also patgen, but it is a good approximation
+},
+{
+ "code" => "cs",
+ "filename_old_patterns_other" => ["czhyphen.tex","czhyphen.ex"],
+ "type" => "dictionary",
+ # Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3
+ # Typographical rules allow \righthyphenmin=2 when typesetting in a
+ # narrow column (newspapers etc.).
+ # (used to be 2,2)
+
+ # guessing based on CTAN/macros/cstex/base/csplain.tar.gz:
+ # 1998-12-17 (patterns)
+ # 1995-08-23 (exceptions)
+ # but patterns claim 1995
+},
+{
+ "code" => "sk",
+ "type" => "dictionary",
+ "filename_old_patterns_other" => ["skhyphen.tex","skhyphen.ex"],
+},
+{
+ "code" => "cy",
+ "type" => "dictionary",
+},
+{
+ "code" => "da",
+ "type" => "dictionary",
+ "filename_old_patterns_other" => ["dkcommon.tex", "dkspecial.tex"],
+},
+{
+ "code" => "eo",
+ "type" => "rules",
+},
+{
+ "code" => "es",
+ "type" => "dictionary",
+},
+{
+ "code" => "eu",
+ "type" => "rules",
+},
+{
+ "code" => "fr",
+ "type" => "rules",
+},
+{
+ "code" => "gl",
+ "type" => "rules",
+},
+{
+ "code" => "et",
+ "type" => "dictionary",
+},
+{
+ "code" => "fi",
+ "type" => "rules",
+},
+{
+ "code" => "hr",
+ "type" => "dictionary",
+},
+{
+ "code" => "hu",
+ "type" => "dictionary",
+ "source" => "https://github.com/nagybence/huhyphn/", # FIXME Check and remove from desc below
+},
+{
+ "code" => "hy",
+ "type" => "rules",
+},
+{
+ "code" => "ia",
+ "type" => "dictionary",
+},
+{
+ "code" => "id",
+ "type" => "rules",
+},
+{
+ "code" => "is",
+ "type" => "dictionary",
+ # TODO: I'm not sure that the last two names are relevant, I don't find the source of Marteinn Sverrisson
+ "authors" => ["jorgen_pind", "marteinn_sverrisson", "kristinn_gylfason"],
+},
+{
+ "code" => "ga",
+ "type" => "dictionary",
+},
+{
+ "code" => "it",
+ "type" => "rules", # TODO: we might want to check that, but it seems unlikely that patgen was used
+},
+{
+ "code" => "rm",
+ "type" => "rules",
+},
+{
+ "code" => "fur",
+ "type" => "rules",
+},
+{
+ "code" => "pms",
+ "type" => "rules",
+},
+{
+ "code" => "kmr",
+ "type" => "dictionary",
+},
+{
+ "code" => "la",
+ "type" => "rules",
+},
+{
+ "code" => "la-x-classic",
+ "type" => "rules",
+},
+{
+ "code" => "la-x-liturgic",
+ "version" => "1.040",
+ "last_modified" => "2016-05-19",
+ "type" => "rules",
+},
+{
+ "code" => "lt",
+ "description_l" => [
+ #......................................................................#
+ "Hyphenation patterns for Lithuanian in L7X and UTF-8 encodings.",
+ # "Designed for \\lefthyphenmin and \\righthyphenmin set to 2.",
+ "\\lefthyphenmin and \\righthyphenmin have to be at least 2.",
+ # "Changing them to 1 according to grammatical rules from 1997",
+ # "would require to review and maybe rebuild the patterns."
+ ],
+},
+{
+ "code" => "lv",
+ "type" => "dictionary",
+},
+{
+ "code" => "nl",
+ "type" => "dictionary",
+ # quoting Hans Hagen:
+ # patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -)
+},
+{
+ "code" => "oc",
+ "type" => "rules",
+},
+{
+ "code" => "pl",
+ "type" => "dictionary",
+ #{}"hyphenmin" => [1,1],
+},
+{
+ "code" => "pt",
+ "type" => "rules", # TODO: we could create a generating script
+},
+{
+ "code" => "zh-latn-pinyin",
+ "type" => "rules", # TODO: we could create a generating script
+},
+{
+ "code" => "ro",
+ "type" => "dictionary",
+},
+{
+ "code" => "sl",
+ "type" => "dictionary",
+},
+{
+ "code" => "hsb",
+ "type" => "dictionary",
+},
+{
+ "code" => "sv",
+ "type" => "dictionary",
+},
+{
+ "code" => "tk",
+ "type" => "dictionary",
+},
+{
+ "code" => "tr",
+ "type" => "rules",
+},
+{
+ "code" => "en-gb",
+ "type" => "dictionary",
+},
+{
+ "code" => "en-us",
+},
+# TODO: FIXME!!!
+{
+ "code" => "sh-latn",
+ "type" => "dictionary",
+},
+{
+ "code" => "sh-cyrl",
+},
+{
+ "code" => "mn-cyrl",
+ "type" => "dictionary",
+ "last_modified" => "2010-04-03", # FIXME: ?
+},
+{
+ "code" => "sa",
+ "type" => "rules",
+},
+{
+ "code" => "nb",
+ "type" => "dictionary",
+ "last_modified" => "2012-05-18", # FIXME: ?
+},
+{
+ "code" => "as",
+ # this is true for all Indic patterns
+ "type" => "rules",
+},
+# thai
+{
+ "code" => "th",
+ "type" => "dictionary",
+ "last_modified" => "2015-05-07", # FIXME: ?
+},
+# lao
+#{
+# "code" => "lo",
+# "name" => "lao",
+# "use_new_loader" => true,
+# "use_old_patterns" => false,
+# "hyphenmin" => [1,1], # TODO
+# "encoding" => nil, # no patterns for 8-bit engines
+# "exceptions" => false,
+# "message" => "Lao hyphenation patterns",
+#
+# "version" => nil,
+# "last_modified" => "2010-05-19",
+# "type" => "rules",
+# "authors" => [ "brian_wilson", "arthur_reutenauer", "mojca_miklavec" ],
+# "licence" => "other-free",
+# "description_s" => "Lao hyphenation patterns",
+# "description_l" => [
+# #......................................................................#
+# "Hyphenation patterns for Lao language for Unicode engines.",
+# "Current version is experimental and gives bad results.",
+# ],
+#},
+{
+ "code" => "ka",
+ "type" => "dictionary",
+},
+{
+ "code" => "cu",
+ "type" => "machine learning",
+},
+{
+ "code" => "be",
+ "type" => "rules",
+},
+{
+ "code" => "pi",
+ "type" => "rules",
+},
+ ]
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages.rb
new file mode 100644
index 0000000000..6083339833
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages.rb
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+require_relative 'language-data'
+require_relative '../../../../lib/tex/hyphen/language'
+
+class OldLanguage
+ def initialize(language_hash)
+ @use_old_patterns = language_hash["use_old_patterns"]
+ @use_old_patterns_comment = language_hash["use_old_patterns_comment"]
+ @use_old_loader = language_hash["use_old_loader"]
+ @filename_old_patterns = language_hash["filename_old_patterns"]
+ @filename_old_patterns_other = language_hash["filename_old_patterns_other"]
+ @code = language_hash["code"]
+ @name = language_hash["name"]
+ @synonyms = language_hash["synonyms"]
+ @hyphenmin = language_hash["hyphenmin"]
+ @encoding = language_hash["encoding"]
+ @message = language_hash["message"]
+
+ @description_l = language_hash["description_l"]
+ @version = language_hash["version"]
+
+ @licence = language_hash["licence"]
+ @authors = language_hash["authors"]
+
+ @synonyms = [] unless @synonyms
+ end
+
+ def <=>(other)
+ code <=> other.code
+ end
+
+ def self.all
+ @@languages ||= @@language_data.map do |language_data|
+ next if language_data['code'] == 'sr-cyrl'
+ new language_data
+ end.compact
+ end
+
+ # TODO self.find
+
+ # def lc_characters
+ # if @lc_characters == nil
+ # lc_characters = Hash.new
+ # p = self.patterns
+ # p.each do |pattern|
+ # end
+ # end
+ # return @lc_characters
+ # end
+
+ attr_reader :use_old_loader, :use_old_patterns_comment, :filename_old_patterns
+ attr_reader :code, :name, :message
+ attr_reader :description_l
+
+ # Convenience methods related to TeX Live and the .tlpsrc files
+ module TeXLive
+ end
+
+ include TeXLive
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/es/README b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/README
new file mode 100644
index 0000000000..9529767ad4
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/README
@@ -0,0 +1,83 @@
+DIVISI'ON DE PALABRAS
+~~~~~~~~~~~~~~~~~~~~~
+
+The file eshyph.tex under /base is the base for the Spanish patters.
+It is intended mainly for backward compatibility. New systems are
+best based on CTAN:language/hyph-utf8/, which has the same patterns.
+
+Why 4.x? Well, I know at least other three files with the same name,
+so this one is the fourth (there were at least 6 or 7 patterns files).
+The others should vanish as soon as posible.
+
+(c) Javier Bezos 1993 1997.
+(c) Javier Bezos and CervanTeX 2001-2009
+Some parts, (c) by Francesc Carmona
+Licence: LPPL
+
+- division.pdf is a draft of an article (in Spanish) explaining the
+rules to be applied and how they are being translated into TeX in a
+unified set of patterns (somewhat outdated).
+- eshyph-make.lua generates the patterns, with eshyph.src for prefixes
+and special cases.
+- eshyph-test.tex makes a comparison with strict syllabic rules. It
+requires a file spanish-words.txt (not supplied) with a list of word,
+one per line. You can (should) filter the words.
+
+For bug reports and comments:
+
+ http://www.tex-tipografia.com/spanish_hyphen.html
+
+I would like to thanks Francesc Carmona for his permission
+to steal parts of his work without restrictions.
+
+The contrib directory, as its name implies, is not part of the
+official bundle. It has a different set of patterns, but it does not
+follow the rules by the Spanish Academy (despite its claims).
+
+What's new in 4.5 (2009-08-01)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Missing group -gl- added (a major bug). A few minor improvements
+for "des-" and "in-".
+
+What's new in 4.4 (2009-05-19)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some bad patterns fixed.
+
+What's new in 4.3 (2009-05-14)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+Some patterns fixed, particularly for "familia", "superar", "sub-" and
+final consonants. Patterns are now generated with the help of a few
+lua/luatex files (whose status is alpha).
+
+What's new in 4.2
+~~~~~~~~~~~~~~~~~
+
+The encoding is UTF-8, so that it can be used with LuaTeX. Yet, it
+works without changes with standard TeX.
+
+What's new in 4.1
+~~~~~~~~~~~~~~~~~
+
+- Thanks to a list of about 750000 words and extensive
+tests, wrong hyphenations have been reduced dramatically
+and the number of patterns has been almost duplicated.
+
+- Since the Spanish Academy rules of 1999 are very vague,
+incomplete and even contradictory, they had to be
+completed with some traditional ones. The Academy has
+published new rules in November 2005 including many of
+the traditional ones employed here and therefore only
+minor adjustments have been necessary.
+
+- Patterns for a few verbal endings are necessary, and
+now the "voseo" forms (used in many countries of Central
+and South America) have been included.
+
+2009-05-19
+_____________________________________________________________
+Javier Bezos | http://www.cervantex.es/
+.............................................................
+TeX y tipografia | http://www.tex-tipografia.com/ \ No newline at end of file
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua
new file mode 100644
index 0000000000..17ccdf2dc3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua
@@ -0,0 +1,136 @@
+-- (encoding:utf-8)
+
+-- GUIONIZADO DE PALABRAS
+-- ~~~~~~~~~~~~~~~~~~~~~
+-- v 4.7
+--
+-- License: MIT/X11
+--
+-- Copyright (c) 1993, 1997 Javier Bezos
+-- Copyright (c) 2001-2015 Javier Bezos and CervanTeX
+--
+-- Permission is hereby granted, free of charge, to any person obtaining a copy
+-- of this software and associated documentation files (the "Software"), to deal
+-- in the Software without restriction, including without limitation the rights
+-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+-- copies of the Software, and to permit persons to whom the Software is
+-- furnished to do so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in
+-- all copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+-- SOFTWARE.
+--
+-- For further info, bug reports and comments:
+--
+-- http://www.tex-tipografia.com/spanish_hyphen.html
+
+----------
+
+-- ethymologic, trailing no, bad words, prefixes, consonant + h, tl
+
+patfile = io.open('eshyph.tex', 'w')
+patfile:write('\\patterns{\n')
+
+-- Basic patters
+-- Using the characters iterator in luatex
+
+digraphs = 'ch ll'
+liquids = 'bl cl fl gl kl pl vl br cr dr fr gr kr pr rr tr vr'
+avoid = 'tl ts tx tz'
+silent = 'h'
+chrl = 'chr chl'
+letters = 'bcdfghjklmnpqrstvwxyz'
+
+for n in letters:gmatch('.') do
+ if silent:find(n) then
+ patfile:write(' 4' .. n .. '.')
+ else
+ patfile:write('1' .. n .. ' 4' .. n .. '. .' .. n .. '2')
+ end
+ for m in letters:gmatch('.') do
+ pat = n .. m
+ if digraphs:find(pat) then
+ patfile:write(' ')
+ elseif liquids:find(pat) then
+ patfile:write(' ')
+ elseif avoid:find(pat) then
+ patfile:write(' ')
+ elseif silent:find(m) then
+ patfile:write(' 2' .. n .. '1' .. m)
+ else
+ patfile:write(' 2' .. pat)
+ end
+ end
+ patfile:write('\n')
+end
+
+patfile:write('1ñ 4ñ.\n')
+
+for n in digraphs:gmatch('%S+') do
+ patfile:write(n:sub(1,1) .. '4' .. n:sub(2,2) .. ' 4' .. n .. '.')
+ for m in letters:gmatch('.') do
+ pat = n .. m
+ if chrl:find(pat) then
+ patfile:write(' ' .. n .. '2' .. m)
+ else
+ patfile:write(' 2' .. pat)
+ end
+ end
+ patfile:write('\n')
+end
+
+for n in liquids:gmatch('%S+') do
+ patfile:write(n:sub(1,1) .. '2' .. n:sub(2,2) .. ' 4' .. n .. '.')
+ for m in letters:gmatch('.') do
+ patfile:write(' 2' .. n .. '2' .. m)
+ end
+ patfile:write('\n')
+end
+
+letters = 'bcdlmnrstxy'
+etim = 'pt ct cn ps mn gn ft pn cz tz ts'
+
+for n in etim:gmatch('%S+') do
+ for m in letters:gmatch('.') do
+ patfile:write('2' .. m .. '3' .. n:sub(1,1) .. '2' .. n:sub(2,2) .. ' ')
+ end
+ patfile:write('4' .. n .. '.\n')
+end
+
+src = io.open('eshyph.src')
+
+function prefix(p)
+ if p:match('r$') then
+ p = p:sub(1,-2) .. '2' .. p:sub(-1) .. '1'
+ patfile:write(p:sub(1,-2) .. '3r\n')
+ elseif p:match('[aeiou]$') then
+ p = p .. '1'
+ patfile:write(p .. 'h\n')
+ end
+ patfile:write(p .. 'a2 ' .. p .. 'e2 ' .. p .. 'i2 ' .. p .. 'o2 ' .. p .. 'u2\n')
+ patfile:write(p .. 'á2 ' .. p .. 'é2 ' .. p .. 'í2 ' .. p .. 'ó2 ' .. p .. 'ú2\n')
+
+end
+
+for ln in src:lines() do
+ ln = ln:match('[^%%]*')
+ for p in ln:gmatch('%S+') do
+ if p:match('/(.*)/') then
+ prefix(p:match('/(.*)/'))
+ elseif p:sub(1,1) == '*' then
+ patfile:write('de2s3' .. p:sub(2) .. '\n')
+ else
+ patfile:write(p .. '\n')
+ end
+ end
+end
+
+patfile:write('}')
+patfile:close()
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src
new file mode 100644
index 0000000000..bf78e9ec71
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src
@@ -0,0 +1,948 @@
+% v 4.7 License MIT/X11. See eshyph-make.lua
+san4c5t
+plan4c5t
+%
+o1eu2
+a1ae2
+a1aó2
+eo1á2
+eo1e2
+o1ae2
+o1eó2
+%
+2no.
+2t2l
+2t2s
+2t2x
+2t2z
+tz3s4ch
+4caca4 % caca-huete
+4cago4 4caga4 4cagas. % Chi-cago
+4teta. 4tetas. % es-teta
+4puta4 4puto4 % com-putar y derivados, reputa-ción
+.hu4mea .hu4meo .he4mee
+4meo. % Tolo-meo
+4meable. 4meables. % imper-meable
+4pedo4 % bí-pedo % tubér-culo %
+4culo4
+5mente.
+% -aer aero deca
+% ieron etc. no en aer.
+% iera iese no en aer
+% no en aer (el acento!) (contrá)eme ete ele elo ela enos elos
+4i3go. 4es. 4és 4e. 4e3mos. 4éis. 4en.
+4ía. 4ías. 4ía3mos. 4íais. 4ían.
+4í. 4í4s3te. 4í4s3tes. 4í3tes. 4í3mos. 4ís3teis.
+4e3ré. 4e3rás. 4e3rés. 4e3rís. 4e3rá. 4e3re3mos. 4e3réis. 4e3rán.
+4i3ga. 4i3gas. 4i3gás. 4i3gamos. 4i3gáis. 4a4i3gan.
+4e3ría. 4e3rías. 4e3ríamos. 4e3ríais. 4e3rían.
+%
+4i3gá3mosme.
+4i3gá3mosmele. 4i3gá3mosmelo. 4i3gá3mos3mela.
+4i3gá3mosmeles. 4i3gá3mosmelos. 4i3gá3mos3melas.
+4i3gá3moste.
+4i3gá3mostele. 4i3gá3mostelo. 4i3gá3mos3tela.
+4i3gá3mosteles. 4i3gá3mostelos. 4i3gá3mos3telas.
+4i3gá3mosle. 4i3gá3mosla. 4i3gá3moslo.
+4i3gá3mosele. 4i3gá3moselo. 4i3gá3mosela.
+4i3gá3moseles. 4i3gá3moselos. 4i3gá3moselas.
+4i3gá3monos.
+4i3gá3monosle. 4i3gá3monoslo. 4i3gá3monosla.
+4i3gá3monosles. 4i3gá3monoslos. 4i3gá3monoslas.
+4i3gá3moos.
+4i3gá3moosle. 4i3gá3mooslo. 4i3gá3moosla.
+4i3gá3moosles. 4i3gá3mooslos. 4i3gá3mooslas.
+4i3gá3mosles. 4i3gá3moslas. 4i3gá3moslos.
+4ed. 4é.
+4edme.
+4édmele. 4édmelo. 4éd3mela.
+4édmeles. 4édmelos. 4éd3melas.
+4edte.
+4édtele. 4édtelo. 4éd3tela.
+4édteles. 4édtelos. 4éd3telas.
+4edle. 4eedla. 4edlo.
+4édsele. 4édselo. 4édsela.
+4édseles. 4édselos. 4édselas.
+4ednos.
+4édnosle. 4édnoslo. 4édnosla.
+4édnosles. 4édnoslos. 4édnoslas.
+4eos.
+4éosle. 4éoslo. 4éosla.
+4éosles. 4éoslos. 4éoslas.
+4edles. 4edlas. 4edlos.
+4er.
+4erme.
+4érmele. 4érmelo. 4ér3mela.
+4érmeles. 4érmelos. 4ér3melas.
+4erte.
+4értele. 4értelo. 4ér3tela.
+4érteles. 4értelos. 4ér3telas.
+4erle. 4erla. 4erlo.
+4erse.
+4érsele. 4érselo. 4érsela.
+4érseles. 4érselos. 4érselas.
+4ernos.
+4érnosle. 4érnoslo. 4érnosla.
+4érnosles. 4érnoslos. 4érnoslas.
+4e3ros.
+4é3rosle. 4é3roslo. 4é3rosla.
+4é3rosles. 4é3roslos. 4é3roslas.
+4erles. 4erlas. 4erlos.
+4í3do. 4í3da. 4í3dos. 4í3das.
+%-ear: norte, ante, super
+4o. 4as. 4a. 4ás. 4a3mos. 4áis. 4an.
+4aste. 4astes. 4ó. 4ates. 4asteis. 4a3ron.
+4a3ba. 4a3bas. 4á3bamos. 4a3bais. 4a3ban.
+4a3ría. 4a3rías. 4a3ríamos. 4a3ríais 4a3rían.
+4a3ré. 4a3rás. 4a3rés. 4a3rís. 4a3rá. 4a3remos. 4a3réis. 4a3rán.
+4a3ra. 4a3ras. 4á3ramos. 4a3rais. 4a3ran.
+4a3re. 4a3res. 4á3remos. 4a3reis. 4a3ren.
+4a3se. 4a3ses. 4á3semos. 4a3seis. 4a3sen.
+4ad.
+%
+e5r4as. e5r4a3mos. e5r4áis. e5r4an.
+e5r4aste. e5r4astes. e5r4ates. e5r4asteis. e5r4a3ron.
+e5r4a3ba. e5r4a3bas. e5r4á3bamos. e5r4a3bais. e5r4a3ban.
+e5r4a3ría. e5r4a3rías. e5r4a3ríamos. e5r4a3ríais e5r4a3rían.
+e5r4a3ré. e5r4a3rás. e5r4a3rés. e5r4a3rís. e5r4a3rá. e5r4a3remos. e5r4a3réis. e5r4a3rán.
+e5r4a3ra. e5r4a3ras. e5r4á3ramos. e5r4a3rais. e5r4a3ran.
+e5r4a3re. e5r4a3res. e5r4á3remos. e5r4a3reis. e5r4a3ren.
+e5r4a3se. e5r4a3ses. e5r4á3semos. e5r4a3seis. e5r4a3sen.
+e5r4ad.
+%
+4adme.
+4ádmele. 4ádmelo. 4ád3mela.
+4ádmeles. 4ádmelos. 4ád3melas.
+4adte.
+4ádtele. 4ádtelo. 4ád3tela.
+4ádteles. 4ádtelos. 4ád3telas.
+4adle. 4eadla. 4adlo.
+4ádsele. 4ádselo. 4ádsela.
+4ádseles. 4ádselos. 4ádselas.
+4adnos.
+4ádnosle. 4ádnoslo. 4ádnosla.
+4ádnosles. 4ádnoslos. 4ádnoslas.
+4aos.
+4áosle. 4áoslo. 4áosla.
+4áosles. 4áoslos. 4áoslas.
+4adles. 4adlas. 4adlos.
+4ar.
+4a4rme.
+4á4rmele. 4á4rmelo. 4á4r3mela.
+4á4r3meles. 4á4r3melos. 4á4r3melas.
+4a4r3te.
+4á4r3tele. 4á4r3telo. 4á4r3tela.
+4á4r3teles. 4á4r3telos. 4á4r3telas.
+4a4r3le. 4a4r3la. 4a4r3lo.
+4a4r3se.
+4á4r3sele. 4á4r3selo. 4á4r3sela.
+4á4r3seles. 4á4r3selos. 4á4r3selas.
+4a4r3nos.
+4á4r3nosle. 4á4r3noslo. 4á4r3nosla.
+4á4r3nosles. 4á4r3noslos. 4á4r3noslas.
+4a3ros.
+4árosle. 4ároslo. 4árosla.
+4árosles. 4ároslos. 4ároslas.
+4a4r3les. 4a4r3las. 4a4r3los.
+%
+4a3do. 4a3da. 4a3dos. 4a3das.
+%
+e5r4a3do. e5r4a3da. e5r4a3dos. e5r4a3das.
+%
+4ando
+4ándole. 4ándolo. 4ándola. 4ándoles. 4ándolos. 4ándolas.
+4ándonos. 4ándoos.
+4ándome. 4ándomelo. 4ándomela. 4ándomele.
+4ándomelos. 4ándomelas. 4ándomeles.
+4ándote. 4ándoteme.
+4ándotelo. 4ándotela. 4ándotele.
+4ándotelos. 4ándotelas. 4ándoteles.
+4ándotenos.
+4ándose. 4ándoseme.
+4ándoselo. 4ándosela. 4ándosele.
+4ándoselos. 4ándoselas. 4ándoseles.
+4ándosenos.
+%
+4a3dor. 4a3dora. 4a3dores. 4a3doras.
+%
+e5r4a3dor. e5r4a3dora. e5r4a3dores. e5r4a3doras.
+%
+/acto/ % (gal)acto, (l)acto
+/aero/
+/afro/
+.a2 % a, an (prevent)
+.an2a2 .an2e2 .an2i2 .an2o2 .an2u2
+.an2á2 .an2é2 .an2í2 .an2ó2 .an2ú2.
+ana3lí
+.aná3li
+.ana3li
+.an3aero
+.an3e2pigr
+.ane3xa .ane3xá .ane3xe .ane3xé .ane3xio .ane3xió
+.an3h
+.ani3mad .ani3mád
+.ani3dar
+.ani3ll
+.ani3m
+.aniña
+.ani3q
+.an3i2so .an3i2só
+.ani3vel
+.ano5che
+.ano5din
+.ano5mal
+.ano5nad
+.anó3nim
+.anó5mal
+.ano5nim
+.ano5ta .ano3tá
+.anua3l
+.anua4lm
+.anu3bl
+.anu3da
+.anu3l
+asu3b2
+/anfi/
+/anglo/
+/ante/
+.ante2o3je
+acante2
+4ísmo. 4ísmos. 4ísta. 4ístas.
+4ístico. 4ísticos. 4ística. 4ísticas.
+pante4o3n
+.anteo3nes
+mante4a
+e4a3miento
+e4ándo
+tras3antea
+/.anti/
+ti2o3qu ti2o3co
+/archi/
+/auto/
+.bi1anual
+.bi1aur
+.bi1ox
+.bi1ó2x
+.bi1un
+/biblio/
+bien2 bien3h bien3v bien3q bien3m bien3t
+b4ien3do. .su3b4ien b4ien3das.
+.bie4n3and
+.bie4n3a4pa
+.bie4n3a4ve
+.bie4n3est
+.bie4n3int
+.bie4n3o4lie
+/bio/
+bi1u2ní
+/cardio/
+/cefalo/
+/centi/
+centi5área
+/ciclo/
+o4i3dea. o4i3deas. o4i3dal. o4i3dales.
+4o2i3de. 4o2i3des. 4i2dal. 4i2dales.
+4i3deo. 4i3deos.
+/cito/
+/cnico/ % (té)cnico
+%?.co?
+.co2a2 .co2e2 .co2i2 .co3o4 .co2u2 % Note coo
+.co2á2 .co2é2 .co2í2 .co2ó2 .co2ú2
+.co3acree
+.coá3gul
+.co3agen
+.coa3gul
+.coa3lic
+.co3arrend
+.co3auto
+.co3edic
+.co3edit
+.co3educ
+.co3efici
+.coe3tá
+.co3exis
+.co3imput
+.coi3to
+.co4o3per
+.co4o3pér
+.co4orden
+.co4ordin
+.co4ordín
+.co4opt
+.co4ópt
+.co2nurb
+/cripto/
+/crono/
+/contra/ % -aer =
+/deca/ % deca, (en)deca, (do)deca, -aer
+4e3dro. 4e3dros. 4é3drico. 4é3dricos. 4é3drica.
+4é3dricas.
+4í3ble. 4í3bles.
+/.de2s/
+deca2i3mient
+3sa. 3sas.
+*órde
+*orde
+*abast
+*aboll
+*aboto
+*abr
+des4a3brid
+*abroch
+*aceit
+*aceler
+desa3cert
+desa3ciert
+*acobar
+*acomod
+*acomp
+*acons
+*acopl
+*acorr
+*acostum
+*acot
+desa3craliz
+*acredit
+*activ
+*acuart
+*aderez
+*adeud
+*adorar
+*adormec
+*adorn
+*advert
+*aferr
+*afic
+*afil
+*afin
+*afor
+*agu
+desa3gú
+desa3garr
+*agraci
+*agrad
+*agravi
+*agreg
+*agrup
+des4a3guis
+*aherr
+*ahij
+*ajust
+*alagar
+*alent
+*alfom
+*alfor
+*alien
+*a4line
+*a4liné
+*aliñ
+desa3liv
+*alm
+*almid
+*aloj
+*alquil
+*alter
+*alumbr
+desa3marr
+desa3mobl
+*amold
+*amort
+*amuebl
+*ampa
+*and
+*angel
+de3sangr
+*anid
+*anim
+*aním
+*anud
+desa3pañ
+desa3pacib
+*apadr
+*apare
+*aparec
+*aparic
+*apeg
+*apercib
+*apes
+*aplic
+*apolill
+*apoy
+*aprend
+*apret
+*apriet
+*aprob
+*apropi
+*aprovech
+*arbol
+*aren
+*arm
+des4arme
+*arraig
+*arregl
+*arrend
+*arrim
+desa3rroll
+*arrop
+*arrug
+*articul
+*asent
+*asist
+*asn
+% desa3soseg
+% desa3sosieg
+*atenc
+*atend
+*atiend
+*atent
+desa3tin
+*atorn
+*atranc
+*autor
+*avis
+desa3yun
+desa3zón
+desa3zon
+*embal
+*embál
+*embar
+*embár
+*embarg
+*embols
+*emborr
+*embosc
+*embot
+*embrag
+*embrág
+*embrave
+*embráve
+*embroll
+*embróll
+*embruj
+*embrúj
+de3semej
+*empañ
+*empáñ
+*empac
+*empaquet
+*empaquét
+*emparej
+*emparéj
+*emparent
+*empat
+*empé
+*empedr
+*empeg
+*empeor
+*emperez
+*empern
+*emple
+*empolv
+*empotr
+*empoz
+*enam
+*encab
+*encad
+*encaj
+*encáj
+*encall
+*encáll
+*encam
+de3sencant
+*encap
+*encar
+*encár
+*ench
+*encl
+*enco
+*encr
+*encu
+*end
+de3senfad
+de3senfád
+*enfi
+*enfo
+*enfó
+de3senfren
+*enfund
+*enfur
+de3sengañ
+de3sengáñ
+*enganch
+*engar
+*engas
+*engom
+*engoz
+*engra
+*enhebr
+*enj
+*enlad
+*enlaz
+*enlo
+*enm
+*enr
+*ens
+*enta
+de3sentend
+de3sentien
+de3sentién
+*enter
+*entier
+*entiér
+*ento
+*entr
+*entu
+*envain
+de3senvolvim
+de3seo
+*eq
+de3s4erci
+de3s4ert
+de3s4ért
+*espa
+de3sesperac
+*esperanz
+*estabil
+*estim
+de3sider
+de3sidia
+de3sidio
+de3siert
+de3sign
+de3sigual
+de3silusi
+*imagin
+*iman
+*impon
+*impres
+*incent
+*inclin
+*incorp
+*incrust
+de3sinenc
+de3sinfec
+de3su3dar de3su3das de3su3dan
+*inflam
+*infl
+*inform
+*inhib
+*insect
+*instal
+ini3ci
+iní3ci
+de3s4integr
+de3s4inter
+*intox
+*inver
+de3sisten
+de3isti
+*obedec
+*oblig
+*obstr
+de3socup
+*odor
+de3solac
+de3solad
+de3soll
+*orej
+*orient
+de3sortij
+*organi
+de3suell
+de3sonce
+*ovi
+*oxi
+*oye
+*oyé
+de3s4ubstan
+de3s4ustan
+de3s4oseg
+*ub4ic
+*unir
+*unier
+*unim
+.diecio2
+/ecano/ % (m)ecano
+/eco/
+/ectro/ % (esp)ectro, (el)ectro
+/.e2n2/
+.ene3mist .ene3míst
+.eno3jar
+.enu3mera .enu3merá
+.enu3mere
+.en3aceit
+.en3arb
+4emboca
+4o3lógico. 4o3lógica. 4o3lógicos. 4o3lógicas.
+4o3lógicamente. 4o3logía. 4o3logías.
+4ó3logo. 4ó3loga. 4ó3logos. 4ó3logas.
+/endo/
+/ento/ % (ci)ento
+/entre/
+/euco/ % (l)euco
+/euro/ % euro, (n)euro
+/.e2x2/
+3gono. 3gonos. 3gonal. 3gonales.
+exá3men
+exá3ri
+exé3ge
+exó3ti
+exa3cerb
+exa3ger
+exa3min
+exe3cr
+exe3géti
+exe3quia
+exi3ja exi3já exí3ja
+exi3ge exí3ge
+exi3gi exi3gí
+exi3jo exí3jo
+3gua. 3guas. 3guo. 3guos.
+exi3lia exi3liá exí3lia
+exi3lie exi3lié exí3lie
+exi3lio exi3lió exí3lio
+exi3ma exi3má exí3ma
+exi3mi exi3mí exí3mi
+exi3me exi3mé exí3me
+exi3mo exi3mó exí3mo
+exi3tos
+exi3lar
+exo3ner
+exo3crin
+exo3tér
+exo3tic
+exo3tiq
+exo3tism
+exo3gami
+exo3gámi
+exu3ber
+exu3dar
+exu3dat
+exu3dac
+/extra/
+u4teri
+.cau5t
+.deu5t
+/fono/
+/foto/
+/gastro/
+/geo/
+/giga/
+/gluco/
+/hecto/
+/helio/
+/hemato/
+/hemi/
+/hemo/
+2al. 2ales.
+.hepta1e
+/hexa/
+/hidro/
+/hiper/
+pe3r4e3mia
+hipe3r4i3cíne
+hipe3r4o3nimi
+hipe3r4o3ními
+hipe3r4ó3nimo
+hipe3r4o3xia
+/hipo/
+/spano/ % (hi)spano
+/histo/
+/homo/
+/ibero/
+/icono/
+/.i2n2/
+.in3abord
+.in3abarc
+.in3acent
+.in3aguant
+.in3adapt
+.ina3movib
+.in3analiz
+.ina3nic
+.in3anim
+.iná3nim
+.in3apel
+.in3aplic
+.in3aprens
+.in3apreci
+.in3arrug
+.in3asist
+.iné3dit
+.in3efic
+.in3efici
+.in3eludi
+.ine3narr
+ini3cia iní3cia ini3ciá ini3cie
+.rei3na
+re3ini3cia re3iní3cia re3ini3ciá re3ini3cie
+.ini3cuo .ini3cua
+.ino3cuo .ino3cua
+.ino3cula .ino3culá .ino3cule
+.inú3til
+.inu3tiliz
+/infra/
+/.inter/
+.in3ter2e3sa .in3ter2e3se .in3ter2e3so
+.in3ter2e3sá .in3ter2e3sé .in3ter2e3só
+.de3s4in3ter2e3sa .de3s4in3ter2e3se .de3s4in3ter2e3so
+.de3s4in3ter2e3sá .de3s4in3ter2e3sé .de3s4in3ter2e3só
+4n3te3ri3n 4n4te4r5i4nsu
+.in3te3r4rog
+.in3te3r4rupc .in3te3r4rupt .in3te3r4rump
+inter4és
+inte3r4esar
+inte4r4i4n4
+inter5ins5t
+inte5r4regno
+inte3r4ior4
+4i4ano. 4i4ana. 4i4anos. 4i4anas.
+/intra/
+/iso/
+/kilo/
+/macro/
+mal2 ma4l3h .ma4l3e4du mal3b mal3c mal3d mal3f mal3g
+mal3m mal3p mal3q mal3s mal3t mal3v
+.mal1acon
+.mal1acos
+.mala1e
+.mal1andant
+.mal1andanz
+.mal1est
+.mal1int
+/maxi/
+/megalo/
+/mega/
+/melano/
+/micro/
+/.mili/
+familia3ri ia5res.
+amili6a
+a3rio a3ria
+li5área
+mili4ar
+/mini/
+2os. 2o3so. 2o3sos. 2o3sa. 2o3sas. 2o3samente.
+mini4a5tur
+/multi/
+/miria/
+/mono/
+2i3co. 2i3cos. 2i3ca. 2i3cas.
+/namo/ % (di)namo
+/necro/
+/neo/ % (cra)neo
+3c2neor
+/neto/ % (mag)neto
+/norte/
+/octo/
+/octa/
+/oligo/
+/omni/
+i2o. i2os.
+/paleo/
+.pa4n1a4meri
+.pa4n1europ
+.pa4n1afri
+.pa4n1hisp
+.pa4n1ópti
+/para/
+para2is. aí5so. aí5sos.
+para4íso
+para4ulata
+/penta/
+/piezo/
+/pluri/
+/poli/
+poli4u3r
+poli4o5mie
+poli4arq poli4árq
+poli4éste
+poli4andr
+poli4antea
+poli4arca
+expoli4
+/.pos2t2/
+.pos3tin .pos3tín
+pos3ta. pos3tas.
+s3te. s3tes. s3tal. s3ta3les.
+s3ti3lla. s3ti3llas. s3ti3llón. s3ti3llones.
+s3tor. s3tora. s3toras. s3tores.
+.pos3tó3ni
+.pos3terg
+.pos3te3ri
+.pos3ti3go
+.pos3ti3la
+.pos3ti3ne
+.pos3ti3za .pos3ti3zo
+.pos3tu3ra
+.pos3tu3la .pos3tu3lá .pos3tu3le .pos3tu3lé
+.post3elec
+.post3impr
+.post3ind
+.post3ope
+.pos2t3rev
+.pos2t3rom
+.pre2a2 .pre2e2 .pre2i2 .pre2o2 .pre2u2
+.pre2á2 .pre2é2 .pre2í2 .pre2ó2 .pre2ú2
+.pre1h2
+pre3elij pre3elig
+pre3exis
+pre3emin
+preo3cup preo2cúp
+pre3olí
+pre3opin
+/.pro2/
+.pro3abort
+/proto/
+/psico/
+3p2sic
+3p2siq
+/quete/ % (re)quete
+/radio/
+/ranco/ % (f)ranco
+.re2a2 .re3e4 .re2i2 .re2o2 .re2u2
+.re2á2 .re2é2 .re2í2 .re2ó2 .re2ú2
+ea3cio. ea3cios. ea3cia. ea3cias.
+.re3abr .re3ábr
+.re3afirm .re3afírm
+.re3ajust .rea3júst
+.rea3liza .rea3lizá .rea3líza
+.re3alim
+.rea3lism .rea3list
+.re3anim .re3aním
+.re3aparec
+.re3ubica .re3ubíca
+.reu3mati .reu3máti
+.re3unir .re3unír
+.re3usar .re3usár
+.re3utiliz .re3utilíz
+.re3a2eg
+.re3a2q
+.re3a2z
+.re3a2grup
+.re3i2m
+.re3inc
+.re3ing
+.re3ins
+.re3int
+.re3o2b
+.re3oc
+.re3oj
+.re3orga
+.re3unt
+/rmano/ % (ge)rmano
+/retro/
+/romo/ % (c)romo
+/semi/
+i2a. i2as.
+2ótic emi2o2
+/2seudo/ % (p)seudo
+o2os.
+.so3a4s % soasar
+/sobre/
+/socio/
+a3rio. a3rios.
+3logía
+4ón. 4ones.
+4i4er.
+4o2ico. 4o2icos. 4o2ica. 4o2icas.
+.su2b2a2 .su2b2e2 .su2b2i2 .su2b2o2 .su2b2u2
+.su2b2á2 .su2b2é2 .su2b2í2 .su2b2ó2 .su2b2ú2
+.sub2i3ll
+.sub2i3mien
+.sub3índ
+.sub3ími
+.su4b3ray
+.sub3aflue
+.sub3arr
+.sub3enten
+.sub3estim .sub3estím
+.sub3ofici
+.sub3urba
+.sub3alter
+.sub3insp
+.su3bién
+.su3bir
+.su3bam
+.su3bordin .su3bordín
+.sub3acuá
+.sub3espe
+.sub3esta
+.su3burbi
+.su4b5rein
+.su2d1a2fr
+.su2d1a2me
+.su2d1este
+su4d3oes
+.su2r1a2me
+.su2r1est
+.su2r1oes
+/super/ % -ar
+ra5ra ra5rá ra5re ra5ré ra5rí ra5ro
+supe3r4a4r supe3r4á4r
+supe3r4á3vit. supe3r4á3vits.
+super4ior
+4a3ción. 4a3ciones.
+4e3rior. 4e3riores. 4e3riora. 4e3rioras. 4e3riormente.
+4e3rioridad. 4e3rioridades.
+4e3r4a3ble. 4e3r4a3bles. 4e3r4a3blemente.
+pe5r4ante
+perpon5d6r
+/supra/
+sup6ra %
+/talmo/ % (of)talmo
+/tele/
+4ósteo. 4ósteos.
+.tele1imp
+.tele1obj
+.tele4o3lót
+/termo/
+/tetra/
+/topo/
+/.tra2s/
+% .tra2s1a2
+% .tra2s1o2
+.tra3s2o3ñ
+/tran2s/
+tran2s1alp
+tran2s1and
+tran2s1atl
+tran2s1oce
+tran2s1ur
+tran3sacci
+tran3seún
+tran3se3xu
+tran3si3cion tran3si3ción
+tran3si3gi tran3si3gí tran3si3ge
+tran3si3ja tran3si3já tran3si3jo
+tran3sisto
+tran3si3ta tran3si3tá tran3si3te tran3si3té tran3si3to tran3si3tó
+tran3si3tiv
+tran3si3tori
+tran3subst tran3sust
+.tri1ó2x
+/tropo/ % (ant)ropo
+/ultra/
+/vice/
+/video/
+/xeno/
+/zoo/
+wa3s4h
+3hablante
+3habiente
+3aficionad
+%.pan1esl
+%.pan1isl
+% pos (impedir) =
+% tri = (faltan ex)
+% uni = (faltan excep)
+%conjugar ir
+% /miriá/
+% des3ojar/VX
+% desistir dejar
+% desoír/QWY
+% des3escombrar/VX
+% desenvoltu evitar
+% desenvolver
+% de3senlace % evitar
+% desempeñar % evitar \ No newline at end of file
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb
new file mode 100755
index 0000000000..61e88d8991
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb
@@ -0,0 +1,137 @@
+#!/usr/bin/env ruby
+#
+# First version: 97/02/20
+#
+# This script generates TeX hyphenation patterns for Basque
+# and was written by (c) Juan M. Aguirregabiria, 1997
+# based on the shyphen.sh file by Julio Sanchez, which
+# generates the hyphenation patterns for Spanish
+# The copyright notice below applies to this script as well,
+# read it before using this software.
+#
+# Rewritten into ruby by Mojca Miklavec <mojca.miklavec[.lists] at gmail.com>
+# in June 2008. Modifications:
+# - added support for Unicode
+# - dropped support for every other "encoding"
+# - added ñ by default
+# - removed part of copyright that was not relevant any more
+# (actually, nothing here resembles the work for Spanish patterns in any way any more)
+#
+# open file for writing the patterns
+# $file = File.new("hyph-eu.tex", "w")
+# in TDS
+$file = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-eu.tex", "w")
+
+# write comments into the file
+def add_comment(str)
+ $file.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n")
+end
+
+# h is not here.
+consonants=%w{b c d f g j k l m n ñ p q r s t v w x y z}
+# Open vowels: a e o
+vop=%w{a e o}
+# Closed vowels: i u
+vcl=%w{i u}
+#
+# In June 2008 the following conversation took place.
+# In case that anyone figures out how the situation might be improved ...
+#
+# Mojca:
+# Why do you need open and closed vewels separately?
+# Juan:
+# Open and closed vowels are very different from the point of view of
+# hyphenation in Basque (and Spanish):
+# you may write "...e-o...", but not "...u-i..."
+# Mojca:
+# But the script generates the same patterns for both sets of vowels.
+# Juan:
+# After eleven years I don't remember *anything* about all this. Maybe I
+# intended to make a difference between open and closed vowels and then
+# thought it would be safer to avoid all breaking between vowels, because in
+# some cases a rule I don't remember now prevents some of them. Maybe some
+# legal breaks are ignored with current patterns, but a least I have never
+# seen an illegal hyphen and never get a report of such a break.
+#
+# I guess a couple of extra lines in the script don't hurt and maybe some day
+# someone will use the difference to allow more breaks.
+#
+vowels=[].concat(vop).concat(vcl)
+# Groups that cannot be broken.
+legal=%w{ll rr ts tx tz bl br dr fl fr gl gr kl kr pl pr tr}
+# echo -n Usage: `basename $0`
+add_comment(
+"Hyphenation patterns for Basque.
+
+This file has first been written by Juan M. Aguirregabiria
+(juanmari.aguirregabiria@ehu.es) on February 1997 based on the
+shyphen.sh script that generates the Spanish patterns as compiled
+by Julio Sanchez (jsanchez@gmv.es) on September 1991.
+
+In June 2008 the generating script has been rewritten into ruby and
+adapted for native UTF-8 TeX engines. Patterns became part of hyph-utf8
+package and were renamed from bahyph.tex into hyph-eu.tex.
+Functionality should not change apart from adding ñ by default.
+
+The original Copyright followed and applied also to precessor of this file
+whose last version will be always available by anonymous ftp
+from tp.lc.ehu.es or by poynting your Web browser to
+ http://tp.lc.ehu.es/jma/basque.html
+
+For more information about the new UTF-8 hyphenation patterns and
+links to this file see
+ http://www.tug.org/tex-hyphen/
+
+ COPYRIGHT NOTICE
+
+These patterns and the generating script are Copyright (c) JMA 1997, 2008
+These patterns are made public in the hope that they will benefit others.
+You can use this software for any purpose.
+However, this is given for free and WITHOUT ANY WARRANTY.
+
+You are kindly requested to send any changes to the author.
+If you change the generating script, you must include code
+in it such that any output is clearly labeled as generated
+by a modified script.
+
+ END OF COPYRIGHT NOTICE
+
+Open vowels: #{vop.join(" ")}
+Closed vowels: #{vcl.join(" ")}
+Consonants: #{consonants.join(" ")}
+
+Some of the patterns below represent combinations that never
+happen in Basque. Would they happen, they would be hyphenated
+according to the rules.
+")
+
+$file.puts '\patterns{'
+
+add_comment(
+"Rule SR1
+Vowels are kept together by the defaults
+Rule SR2
+Attach vowel groups to left consonant")
+consonants.each do |consonant|
+ patterns = []
+ vowels.each do |vowel|
+ patterns.push "1#{consonant}#{vowel}"
+ end
+ $file.puts patterns.join(" ")
+end
+
+add_comment(
+"Rule SR3
+Build legal consonant groups, leave other consonants bound to
+the previous group. This overrides part of the SR2 pattern group.")
+legal.each do |pair|
+ patterns = []
+ vowels.each do |vowel|
+ patterns.push "1#{pair[0,1]}2#{pair[1,1]}#{vowel}"
+ end
+ $file.puts patterns.join(" ")
+end
+
+add_comment("We now avoid some problematic breaks.")
+$file.puts "su2b2r su2b2l"
+$file.puts "}"
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README
new file mode 100644
index 0000000000..418910cbd5
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README
@@ -0,0 +1,25 @@
+This directory contains seven files + README, licensed all but one under lppl 1.3:
+
+glpatter-utf8.tex:
+ The source file, version 2.4. When processed under the program
+ mkpatter ---by typing 'tex -ini -8bit glpatter-utf8.tex' (without the quotes)
+ at the command line, provided you have the utility
+ mkpattern installed--- it generates the file hyph-utf8-gl.tex.
+
+(hyph-gl.tex):
+ The generated file containing the patterns in UTF-8 (belongs to TEXMF/tex).
+
+
+glhyextr.tex: ##This file is in the public domain##
+ A configuration file, to include additional patterns other
+ than the ones in glpatter-utf8.tex. It is not \input by glpatter-utf8.tex
+ (iin contrast with the single-byte encoded version glpatter.tex)
+ as modern tools allow the addition of patterns at run time.
+
+glhybiox.tex, glhymed.tex, glhyquim.tex, glhytec.tex & glhyxeog.tex:
+ Files with aditional patterns drawn from serveral fields of
+ the language, so that you may input them from glhyextr.tex.
+
+The files for specific patterns are very incomplete and feedback is welcome.
+
+--Javier A. Múgica
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex
new file mode 100644
index 0000000000..461751ecc8
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex
@@ -0,0 +1,25 @@
+% This is the file glhybiox.tex.
+% This is a subset of the galician hyphenation patterns.
+% (c) Javier A. Múgica, 2007
+% License: LPPL version 1.3
+%
+% LPPL maintenance status: maintained
+% Current Maintainer: Javier A. Múgica
+%
+% Prefixos para xeoloxía e bioloxía.
+% Esta lista está aínda moi incompleta. Para aportar
+% novos prefixos ou para calquera outra suxerencia,
+% contactar en javier @ digi21.eu.
+
+%Os prefixos comentados xa están na lista principal en glpatter.tex
+\begin{prefixos}
+%paleo termo
+ciano crio halo .lito
+.oro .uro
+\end{prefixos}
+
+.crio2u .malcrio2u
+\begin{3a_ppi}
+inhalo
+\end{3a_ppi}
+\endinput
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex
new file mode 100644
index 0000000000..37ad86c0b9
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex
@@ -0,0 +1,7 @@
+%glhyextr.tex. This file is in the public domain
+%\mkinput{glhyquim.tex}
+%\mkinput{glhymed.tex}
+%\mkinput{glhybiox.tex}
+%\mkinput{glhyxeog.tex}
+%\mkinput{glhytec.tex}
+\endinput \ No newline at end of file
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex
new file mode 100644
index 0000000000..cd3d6a4997
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex
@@ -0,0 +1,19 @@
+% This is the file glhymed.tex.
+% This is a subset of the galician hyphenation patterns.
+% (c) Javier A. Múgica, 2007
+% License: LPPL version 1.3
+%
+% LPPL maintenance status: maintained
+% Current Maintainer: Javier A. Múgica
+%
+% Prefixos para mediciña.
+% Esta lista está aínda moi incompleta. Para aportar
+% novos prefixos ou para calquera outra suxerencia,
+% contactar en javier @ digi21.eu.
+
+%Os prefixos comentados xa están na lista principal en glpatter.tex
+\begin{prefixos}
+%cardio hemato
+oftalmo pleuro oligo
+\end{prefixos}
+\endinput
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex
new file mode 100644
index 0000000000..573bed5cd6
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex
@@ -0,0 +1,19 @@
+% This is the file glhyquim.tex.
+% This is a subset of the galician hyphenation patterns.
+% (c) Javier A. Múgica, 2007
+% License: LPPL version 1.3
+%
+% LPPL maintenance status: maintained
+% Current Maintainer: Javier A. Múgica
+%
+% Prefixos para química.
+% Esta lista está aínda moi incompleta. Para aportar
+% novos prefixos ou para calquera outra suxerencia,
+% contactar en javier @ digi21.eu.
+
+%Os prefixos comentados xa están na lista principal en glpatter.tex
+\begin{prefixos}
+%hidro
+cloro fluoro gluco lacto lipo
+\end{prefixos}
+\endinput
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex
new file mode 100644
index 0000000000..f36a7007a9
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex
@@ -0,0 +1,19 @@
+% This is the file glhytec.tex.
+% This is a subset of the galician hyphenation patterns.
+% (c) Javier A. Múgica, 2007
+% License: LPPL version 1.3
+%
+% LPPL maintenance status: maintained
+% Current Maintainer: Javier A. Múgica
+%
+% Prefixos para términos técnicos.
+% Esta lista está aínda moi incompleta. Para aportar
+% novos prefixos ou para calquera outra suxerencia,
+% contactar en javier @ digi21.eu.
+
+%Os prefixos comentados xa están na lista principal en glpatter.tex
+\begin{prefixos}
+%hidro termo electro magneto
+alumino metalo piezo
+\end{prefixos}
+\endinput
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex
new file mode 100644
index 0000000000..c5105af19c
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex
@@ -0,0 +1,25 @@
+% This is the file glhyxeog.tex.
+% This is a subset of the galician hyphenation patterns.
+% (c) Javier A. Múgica, 2007
+% License: LPPL version 1.3
+%
+% LPPL maintenance status: maintained
+% Current Maintainer: Javier A. Múgica
+%
+% Prefixos para xeografía.
+% Esta lista está aínda moi incompleta. Para aportar
+% novos prefixos ou para calquera outra suxerencia,
+% contactar en javier @ digi21.eu.
+
+%Os prefixos comentados xa están na lista principal en glpatter.tex
+\begin{prefixos}
+%anglo franco ibero .indo
+alto baixo
+greco italo xermano .sino .uralo
+\end{prefixos}
+
+\exceptions{.baixo2u3}{.baixo2u}
+\begin{3a_ppi}
+asalto .baixo .rebaixo
+\end{3a_ppi}
+\endinput
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex
new file mode 100644
index 0000000000..4b116eede4
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex
@@ -0,0 +1,459 @@
+% This is the file glpatter-utf8.tex, version 2.4
+% It is the source file for the Galician patterns
+%
+% (c) Javier A. Múgica; 2006, 2007, 2008, 2010
+% License: LPPL version 1.3
+%
+% LPPL maintenance status: maintained
+% Current Maintainer: Javier A. Múgica
+%
+%
+% The patterns follows approximately the scheme developed by Javier Bezos for esphyph.tex
+% In order to generate the patterns just type
+% tex -ini -8bit glpatter-utf8.tex
+%
+% For bug reports and comments:
+%
+% Javier Múgica, javier at digi21.eu
+\input mkpatter.tex
+\filename{hyph-gl.tex}
+\tracingexceptions=1
+
+&Va={a e o}
+&Vp={i u}
+&Vi={&Va u}
+&Va+={&Va á}
+&V={a e i o u á é í ó ú}
+&Vv={a e i}
+&Vai={a i}
+&Vvac={á é í}
+&L={l r}
+% ñ non se inclúe nos seguintes grupos
+&C={b c d f g h k l m n p r s t v x z} % Consoantes habituais
+&Cl={b c d f g k p t v} % As que van unidas á líquida, agás r e l
+&G={b c d l m n r s t x} % Posibles consoante precedente a certos grupos pouco frecuentes
+
+\templatedef{prefixos}{#1&{V}2}
+\templatedef{3a_ppi}{#2u3 \newline} % 3a. persoa do perfecto de indicativo, rematadas en -ou, -eu, -iu.
+ % O confilcto cun prefixo máis habitual é na primeira conxugación.
+%
+% Uncomment the %\letters line if you want those characters to be catcode as letters
+% and the correspoding caplital ones to be assigned the right \lcode. In that case an utf-8
+% engine (luatex or xetex) to process this file is mandatory.
+%
+% \letters{á é í ó ú ñ ü ï}
+
+% The commented lines that follow will be placed in the generated file,
+% and do not apply to this file.
+\expandafter\let\expandafter\lee\csname ~~lee\endcsname
+\keepcomments
+% title: Hyphenation patterns for Galician, written in the utf8 encoding.
+% copyright: (c) Javier A. Múgica; 2006, 2007, 2008, 2010
+% notice: This file is part of the hyph-utf8 package.
+% See http://www.hyphenation.org for more information.
+% language:
+% name: Galician
+% tag: gl
+% licence:
+% name: LPPL
+% version: 1.3
+% status: maintained
+% maintainer: Javier A. Múgica
+% url: https://latex-project.org/lppl/lppl-1-3.html
+%
+% ==========================================
+% Generated with the mkpattern utility (v. 1.2), on 2010/04/23
+% The original source file were glpatter-utf8.tex
+% This is a generated file
+%
+% For bug reports and comments:
+%
+% Javier Múgica, javier at digi21.eu
+\let\lee\relax
+\halfcomments
+\iwrite{%}
+\iwrite{% Note that there is no 'j' nor 'y' in Galician}
+\nocomments
+
+\begin{pseudopatterns}
+
+3ï3a \newline
+a1a e1e o1o zo2o a1á e1é o1ó 2&{Va}&{Vp}.
+&{Vi}1i&{Va+} 2i&{Va+}.
+
+\newline
+1ñ
+1&{C}&{V}
+1qu2 1c2h 2ch. 1g2h 2gh. 2k2h \newline % gh é normalmente usado para representar a gheada
+\newline
+\exceptions{1t2l 2tl.}{2t2l,}
+1&{Cl}2&L 2&{Cl}&L.
+
+1l4l 2ll. 1r2r 2rr.
+
+\template{&G1#}
+p2t c2t c2n p2s m2n g2n f2t p2n c2z t2s
+\end{template}
+san2c3t plan2c3t
+
+\template{2#.}
+pt ct cn ps mn gn ft pn cz ts
+\end{template}
+
+un2ha 2non. 3mente. o2hib alde2h \newline
+\newline
+% palabras malsonates
+4caca4 4cago4 4caga4 4cagas. 4puta4 4puto4 4meo. 4mea. \newline
+4meable. 4meables. 4peido4
+
+\exceptions{.co1á2}{}
+\exceptions{.pre1á2 .su2b1i2 .su2b1í2}{} % If duplicate patterns were simply added...
+\exceptions{.pre2á}{.pre2á2} % This is error prone. I think that writting
+\exceptions{.sub2i .sub2í}{.su2b2i2,.su2b2í2} % all the exceptions here together is the best that can be done.
+\begin{prefixos} % Prefixos
+acto afro aero anfi anglo .ante .anti .arqui auto biblio bio
+cardio cefalo ciclo cito .co contra cripto cromo crono
+deca .deza dinamo ecano eco electro endo ento entre
+euco euro extra fono foto franco gastro xeo
+hecto helio hemato hemi hexa hidro hipe2r histo homeo homo
+ibero icono .in .indo infra .inte2r intra .iso kilo
+macro magneto maxi mega megalo melano micro mili mini
+multi miria mono .nano necro .neo norte octo octa
+omni paleo para penta piezo pluri poli .pos2t .pre
+.pro proto pseudo radio .re retro sobre semi
+socio .su2b supe2r supra .tele termo tetra topo
+.tri tropo ultra xeno
+\end{prefixos}
+
+\template{#3r} % prefixos rematados en r
+hiper .inter super
+\end{template}
+
+\newline
+ti2o3qu ti2o3co
+bi1u2ní
+o2i3de o2i3dal
+2al. 2a2is.
+pe3r2e3mia \newline
+
+\begin{3a_ppi}
+\put{% Terceiras persoas do pasado perfecto}\newline
+libero atopo enaxeno
+\end{3a_ppi}
+
+\newline
+2os. 2o3so. 2o3sos. 2o3sa. 2o3sas. 2o3samente. \newline
+2i3co. 2i3cos. 2i3ca. 2i3cas. \newline
+
+\newline
+\put{% Excepcións ó prefixo co}
+% Pode non ser prefixo (expecto os derivados de coar)
+\template{.co2# \newline}
+\put{% Formas do verbo coar con pronomes enclíticos}\newline ar á2 abá
+acerv andro ano añar año art etan enci erci inci ira iro ita
+\end{template}
+
+% Non é prefixo (os que non son da forma &V3 non necesitan ir un por un)
+\template{co2# \newline} % Non é necesario coia, coiote
+a3gul á3gul a3la. a3las. a3lescen a3lición. a3licions. a3na. a3nas.
+antriñ a3ñadeir a3tí. a3tís. e3ficien e3lernos. e3llo. e3lla. e3llos. e3llas.
+e3lleir enll enxía e3sita. e3sitas. e3táne e3vo. e3va. e3vos. e3vas.
+i3dado iei imbra imbrá intreau. í3ña i3ña i3ñei i3pú. i3pús.
+i3ra. i3ras. i3raza i3ro. i3ros. i3ta. i3tas. i3tado. i3to. i3tos.
+i3tel i3tío. i3tus. u3c u3lomb u3try u3qui u3rel
+u3sa. u3sas. u3so. u3sos. u3selo u3tad u3to. u3tos. u3vini u3z
+\end{template}
+
+deca2e3ment
+
+\newline
+\put{% O prefixo des}
+.de2s1&{V}2
+3se. 3s2es. 3sa. 3s2as. de3s2outr 3s2emos. 3s2edes. 3s2en.
+
+\exceptions{de3s2esper}{de3s2esper de4s3esperanz}
+\template{de3s2# \newline} % Prefixos .de (máis ou menos)
+a3crali a3guisa a3lini a3ngr a3ñ a3rrollis astr a3zo e3c e3que e3guid e3la
+ensib e3ñ ert ért esper e3pér e3x é3x i3der ign ígn i3nenc ingr iste isti o3lac o3lad old
+o3lidari uetud sulf
+\end{template}
+
+\template{.des2# \newline} % Prefixos dubidosos ou que sa non se notan. Non se inclúen os que por ser moi breves poderían coincidir con prefixos .des formados ad hoc, xa que impedirían a división de palabras comúns (desecar, deselar).
+abor afia afía afío air emboc embóc empeñ empéñ enlac enlaz enlác enláz
+envol envól idia ora
+\end{template}
+
+\keepcomments
+
+\newline
+% Excepcións ó prefixo in
+\template{.in2# \newline}
+a3misib. a3mov a3ne. a3nic a3nid á3nime antes. au e3dia é3dit e3fab e3narr
+epc ept erc ert erm erv e3siv e3xora i3ci i3cu i3mig i3miza i3qui
+o3cen o3cui o3cuo o3cul ó3cul o3pia. o3sili o3sit o3tróp o3trop uit u3lase u3lina
+unda u3sita ú3til
+\end{template}
+in2o4cular \newline
+
+\newline
+% Excepcións ó prefixo inter
+\template{.inte3r2#}
+és. e3sa é3sa e3sá e3so é3so e3só ior i3no. i3nos. i3na. i3nas. i3nid
+\end{template}
+
+\newline
+% Os prefixos mal e ben
+\exceptions{.be2n1e2 .be2n1é2 .ma2l1e2 .ma2l1é2}{.be2ne2, ,.ma2le2, }
+
+\halfcomments
+.be2n1&{V}2
+\template{be3n2# \newline} % Non é prefixo (ou non se nota)
+ign i3merí i3nés i3nes i3toíta
+\end{template}
+
+\newline
+.ma2l1&{V}2
+\template{.mal2# \newline} % Pode non ser prefixo
+abar abár aco acó armad ogr ura axa
+\end{template}
+\template{ma3l2# \newline} % Non é prefixo. Non é neceario malos, malas
+a3cía a3citan a3gueñ aio aia andrín andrin a3quita ar. a3res. a3ria.
+a3to. a3tos. a3ta. a3tas. a3tión aui eabl eabil eico eolar e3ta. e3tas. e3tín e3teiro
+eza ia. ian i3cia i3cios ign i3kita inke ó3fago ó3nic o3nato o3nilue
+\end{template}
+ma4l3ianq
+\template{.mal1# \newline} % É prefixo e comenza por e
+educ encar ensin entend
+\end{template}
+
+\newline
+\put{% Excepcións ó prefixo mili}
+famili2a familia3r
+\template{mili#}
+2a3rio 2a3ria
+\end{template}
+
+mini2a3tur
+
+\newline
+\put{% Excepcións ó prefixo para}
+\template{para2#}
+u3gas í3so
+\end{template}
+
+\newline
+\put{% Excepcións ó prefixo poli}
+\template{poli2# \newline}
+u3r o3me arq árq éste andr antea
+\end{template}
+expoli2
+
+\newline
+\put{% Excepcións ó prefixo post}
+\template{.pos3t2# \newline}
+a. as. al. ais. a3llo e. es. ear. e3la. e3las. er. erg e3rid e3rior i3go i3la
+illón ín. i3te. i3zo. i3zos. i3za. i3zas. o. os. oiro ó3ni u3la u3lo u3le u3ra. u3ras.
+\end{template}
+
+\newline
+\put{% Excepcións ó prefixo pre}
+\template{.pre2# \newline} % Poden non ser prefixos
+amar \put{% Formas do verbo prear con pronomes enclíticos}\newline ar á abá
+\end{template}
+\template{pre2# \newline} % Non son prefixos
+as. a3da. a3das. á2 abá i3t o3cup o3cúp
+\end{template}
+
+\newline
+\put{% Excepcións ó prefixo pro}
+\template{pro2# \newline} % Non son prefixos. Non é necesario proer
+e3za í3do ust
+\end{template}
+
+\newline
+\put{% Excepcións ó prefixo re} % Faltan excepcións
+\template{.re2#}
+al a3liz a3lidade in i3no. i3nos i3nante iter \newline
+i3xa os. as. u3ma ú3ma ún uni
+\end{template}
+
+\put{% Verbo reunir}
+% No é tan problemáticos coma subir
+\template{re2u3n#}
+ión. ións. ir írmo íren \newline % ir inclúe varias formas. Penso que re3u4nirr no paga a pena
+ido. idos. ida. idas. ind \newline
+o. es. e. imos. ímo en. \newline % ides. incuído en ide, enbaixo
+ía i3amos. i3ades. \newline
+ín. iches. iu. istes. íst iron. \newline
+a. as. amos. ades. an. \newline
+ise íse \newline
+ide íde ámo áde
+\end{template}
+
+\put{% Verbo reinar}
+% No é tan problemáticos coma subir
+\template{re2i3n#}
+ar ado. ados. and \newline
+a. as. amos. ades. an. \newline % ides. incuído en ide, enbaixo
+aba. abas. abamos. ábamos. abades. ábades. \newline % hay moitas palabras que comenzan por inaba
+ei. aches. ou. astes. aron. \newline
+e. es. emos. edes. en. \newline
+ase. ases. asemo. asedes. áse asen. \newline % idem., inase
+ade.
+\end{template}
+
+\newline
+\put{% O prefixo sub: l, r e excepcións}\newline
+.su2b3l .su2b3r
+
+\template{.sub2# \newline} % Poden non ser prefixos
+\put{% Formas do verbo subir con pronomes enclíticos}\newline i í
+eriz orna
+\end{template}
+\newline
+sub3índic sub3indic sub3indiz
+
+\template{.sub4# \newline}
+lev lim
+\end{template}
+
+\template{su3b2# \newline} % Non son prefixos
+e3la. e3las. é3rico e3rina. e3rinas. eroso iote ulado orno. ornos. urbio
+\end{template}
+\template{su3b4# \newline}
+liminar repción reptici
+\end{template}
+
+\newline
+\put{% Excepcións ó prefixo tri}
+\template{tri2# \newline} % Non son prefixos (ou non se nota)
+a3ga. a3gas. al. a3les. angul á3sico. estin % (De Trieste)
+unf unvir
+\end{template}
+
+\newline
+\put{% Excepcións a varios prefixos que son terminacións verbais}\newline
+\put{% Inclúense as variantes de acentuación abamos/ábamos, etc.}
+% As combinacións con pronomes enclíticos (son centos)
+% deberanse evitar nos patróns de cada prefixo cando sexa necesario
+% Inclúense as combinacións máis habituais
+\template{2&{Vai}#.}
+3do 3da 3dos 3das ndo r % inf. xer. part.
+\end{template}
+
+\template{2&{Vv}#.}
+3res rmos rdes 3ren \newline % inf. conx.
+rme rte rlle rnos rvos rlles % inf. con pron. indir.
+\end{template}
+
+\template{2&{Vv}3#.}
+dor dora dores doiro doiros doira doiras deiro deiros deira deiras % subst. deriv.
+lo los la las % inf. con pron. dir.
+rei rás rá remos redes rán \newline % futuro
+ría rías ri1amos ríamos ri1ades ríades rían \newline % cond.
+de % imp.
+\end{template}
+\template{2&{Vvac}3#.}
+deo dea deos deas % imp. con pron. dir.
+\end{template}
+
+\template{2#.}
+% coar, inar, entrenar, prear, superar
+as a3mos a3des an \newline
+a3ba a3bas a3bamos á3bamos a3bades á3bades a3ban \newline
+a3ches astes a3ron \newline
+es e3mos e3des en \newline
+a3se a3ses á3semos á3sedes a3sen \newline
+\end{template}
+% pero permitimos a ruptura para coar, prear, voar, cear, capear...
+% cando o "a" ou "e" temático é tónico (ou casi).
+\template{o3#.}
+ar ado ada ados adas ando \newline
+ares armos ardes aren \newline
+arme arte arlle arnos arvos arlles \newline
+alo alos ala alas \newline % incúe algunhas outras (coala)
+ade ádeo ádea ádeos ádeas \newline
+as amos ades an \newline
+aba abas abamos ábamos abades ábades aban \newline
+aches astes aron \newline
+es emos edes en \newline
+ase ases ásemos ásedes asen \newline
+\end{template}
+\template{e3#.}
+ar ado ada ados adas ando \newline
+ares armos ardes aren \newline
+arme arte arlle arnos arvos arlles \newline
+alo alos ala alas \newline % incúe algunhas outras (coala)
+ade ádeo ádea ádeos ádeas \newline
+as amos ades an \newline
+aba abas abamos ábamos abades ábades aban \newline
+aches astes aron \newline
+es emos edes en \newline
+ase ases ásemos ásedes asen \newline
+\end{template}
+
+\template{2#.}
+% subir
+i3mos i3des \newline
+ía ías íamos íades ían \newline % 2iamos, 2iades estropearía moitos verbos (ca-ia-mos... ) ...
+ín i3ches iu istes i3ron \newline
+% O pres. subx. coincide con pres. ind. da 1ª
+i3se i3ses í3semos í3sedes i3sen
+% contraer, decaer, extraer
+% Pres. subx. xa se divide ben
+\end{template}
+\newline
+í3do í3da í3dos í3das
+
+% Conxugación completa do verbo subir
+\template{.su3b#.}
+ir indo ido ida idos idas \newline
+ires irmos irdes iren \newline
+o es e imos ides en \newline
+ía ías 2i3amos íamos 2i3ades íades ían \newline % ... por eso incluímolos só aquí.
+ín iches iu istes iron \newline
+irei irás irá iremos iredes irán \newline
+iría irías iriamos iríamos iriades iríades irían \newline
+a as amos ades an \newline
+ise ises ísemos ísedes isen \newline
+ide ídeo ídeos ídea ídeas \newline
+ador adora adores adoras % posibles derivados
+\end{template}
+
+% Conxugación completa do verbo superar
+\template{.supe3r#.}
+ar ando ado ada ados adas \newline
+ares armos ardes aren \newline
+a as amos ades an \newline
+aba abas abamos ábamos abades ábades aban \newline
+ei aches ou astes aron \newline
+arei arás ará eremos eredes arán \newline
+aría arías ariamos aríamos ariades aríades arían \newline
+e es emos edes en \newline
+ase ases ásemos ásedes asen \newline
+ade ádeo ádeos ádea ádeas \newline
+ador adora adores adoras ación % derivados
+\end{template}
+
+supe3r2ior supe3r2a3ble supe3r2a3bilidade
+
+% para verbos rematados en -aer, permitimos a división
+% cando a vocal temática tónica e aberta e, salvo inf.,
+% non é a última sílaba.
+\template{a3#.}
+er endo \newline
+eres ermos erdes eren \newline
+erme erte erlle ernos ervos erlles \newline
+elo elos ela elas \newline
+ede édeo édea édeos édeas \newline
+emos edes eron \newline
+ese eses esemos ésemos esedes ésedes esen
+\end{template}
+
+% par-ti-a-mos, di-ci-amos, fa-ci-amos, etc., pero aso-cia-mos, etc.
+
+% The new programs allow to input patterns at run time, so including
+% the extras in the format is not right.
+% \mkinput{glhyextr.tex}
+\end{pseudopatterns}
+\end{}
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb
new file mode 100755
index 0000000000..b2431bcf6b
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb
@@ -0,0 +1,69 @@
+#!/usr/bin/env ruby
+#
+# This script generates TeX hyphenation patterns for Armenian
+#
+# Rules written by Sahak Petrosyan <sahak at mit.edu> (author)
+#
+# These are just primitive rules that says that if there is a
+# combination like <vowel><consonant><vowel>, then it should hyphenate
+# like this: <vowel> - <consonant><vowel>.
+#
+# There are more rules to implement
+#
+# Script written by Mojca Miklavec <mojca.miklavec.lists at gmail.com>
+# (who is not the author of patterns)
+#
+$file = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-hy.tex", "w")
+
+# write comments into the file
+def add_comment(str)
+ $file.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n")
+end
+
+vowels=%w{ա ե է ը ի ո օ}
+consonants=%w{բ գ դ զ թ ժ լ խ ծ կ հ ձ ղ ճ մ յ ն շ չ պ ջ ռ ս վ տ ր ց փ ք}
+
+add_comment(
+"Hyphenation patterns for Armenian.
+
+Written by Sahak Petrosyan <sahak at mit.edu>
+ for Hyphenator.js (http://code.google.com/p/hyphenator/)
+ and later adapted for TeX
+
+Licence: LGPL
+Version: May 2010
+
+These are just primitive rules that hyphenate combinations like
+<vowel> - <consonant><vowel>.
+
+File auto-generated by a script (see source/generic/hyph-utf8/languages/hy)
+
+Vowels: #{vowels.join(" ")}
+Consonants: #{consonants.join(" ")}
+
+Some of the patterns below represent combinations that never
+appear in Armenian, but they would be hyphenated according to the rules.
+")
+
+$file.puts '\patterns{'
+
+add_comment(
+"և-<vowel>")
+vowels.each do |vowel|
+ $file.puts "և1#{vowel}"
+end
+
+add_comment(
+"<vowel>-<consonant><vowel>")
+vowels.each do |v1|
+ consonants.each do |c|
+ patterns = []
+ vowels.each do |v2|
+ patterns.push "#{v1}1#{c}#{v2}"
+ end
+ $file.puts patterns.join(" ")
+ end
+ $file.puts "%"
+end
+
+$file.puts "}"
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua b/language/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua
new file mode 100755
index 0000000000..e3a13e1907
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua
@@ -0,0 +1,154 @@
+#!/usr/bin/env texlua
+
+-- Extract Ethiopic syllables from the Unicode Character Database.
+-- Copyright (c) 2011, 2016 Arthur Reutenauer
+-- http://www.hyphenation.org/
+-- === Text of the MIT licence ===
+-- This file is available under the terms of the MIT licence.
+-- Permission is hereby granted, free of charge, to any person obtaining a copy
+-- of this software and associated documentation files (the “Software”), to deal
+-- in the Software without restriction, including without limitation the rights to
+-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies
+-- of the Software, and to permit persons to whom the Software is furnished to do
+-- so, subject to the following conditions:
+--
+-- The above copyright notice and this permission notice shall be included in all
+-- copies or substantial portions of the Software.
+--
+-- THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+-- SOFTWARE.
+-- === End of MIT licence ===
+-- Version 0.2 2016-05-22
+
+-- Use with TeXLua.
+
+local function extract_mit_licence()
+ reading_mit_licence = false
+ bolicence = lpeg.P'=== Text of the MIT licence ==='
+ eolicence = lpeg.P'=== End of MIT licence ==='
+ mit_licence = { }
+ for line in io.lines(arg[0]) do
+ line = line:gsub('^%-%- ', ''):gsub('^%-%-', '')
+ if eolicence:match(line) then
+ reading_mit_licence = false
+ end
+
+ if reading_mit_licence then
+ mit_licence[#mit_licence + 1] = line
+ end
+
+ if bolicence:match(line) then
+ reading_mit_licence = true
+ end
+ end
+end
+
+-- Small library to convert a hexadecimal string to a number
+local function hex_digit_to_num(h)
+ local c = string.byte(h)
+ if c >= 48 and c < 58
+ then return c - 48
+ elseif c >= 65 and c < 71
+ then return c - 55
+ elseif c >= 97 and c < 103
+ then return c - 87
+ end
+ return 0
+end
+
+local function hex_string_to_num(hex)
+ local n = 0
+ for i = 1, string.len(hex) do
+ n = 16 * n + hex_digit_to_num(string.sub(hex, i, i + 1))
+ end
+ return n
+end
+
+local ucd_file = "UnicodeData.txt"
+
+local error_install_ucd = [[
+Error: the Unicode Character Database could not be found.
+Please download it from http://www.unicode.org/Public/UNIDATA/UnicodeData.txt
+and install it in the current directory.]]
+
+if not lfs.attributes(ucd_file) then
+ print(error_install_ucd)
+ return -1
+end
+
+local P, R, C, match = lpeg.P, lpeg.R, lpeg.C, lpeg.match
+
+local digit = R"09" + R"AF"
+local colon = P";"
+local field = (1 - colon)^0
+local eth_syll = P"ETHIOPIC SYLLABLE " * C(field)
+local ucd_entry = C(digit^4) * colon * eth_syll
+
+local pattfile = assert(io.open("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-mul-ethi.tex", "w"))
+
+
+extract_mit_licence()
+
+pattfile:write[[
+% title: Experimental pattern file for languages written using the Ethiopic script.
+% copyright: Copyright (c) 2011, 2016 Arthur Reutenauer
+% notice: This file is part of the hyph-utf8 package.
+% See http://www.hyphenation.org for more information
+% language:
+% name: Multiple languages using the Ethiopic scripts
+% tag: mul-ethi
+% version: 0.2 2016-05-22
+% licence:
+% - name: MIT
+% - url: https://opensource.org/licenses/MIT
+% - text: >
+]]
+
+for _, line in ipairs(mit_licence) do
+ if lpeg.match(lpeg.P(-1), line) then
+ pattfile:write('%\n')
+ else
+ pattfile:write('% ', line, '\n')
+ end
+end
+
+pattfile:write[[
+% ==========================================
+% This is a generated file. If you wish to edit it, consider adapting the
+% generating programme
+% (https://github.com/hyphenation/tex-hyphen/blob/master/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua).
+%
+% The BCP 47 language tag for that file is “mul-ethi” to reflect the fact that
+% it can be used by multiple languages (and a single script, Ethiopic). It is,
+% though, not supposed to be linguistically relevant and should, for proper
+% typography, be replaced by files tailored to individual languages. What we
+% do for the moment is to simply allow break on either sides of Ethiopic
+% syllables, and to forbid it before some punctuation marks particular to
+% the Ethiopic script (which we thus make letters for this purpose).
+%
+]]
+
+pattfile:write"\\patterns{%\n"
+for ucd_line in io.lines(ucd_file) do
+ local usv, char_name = match(ucd_entry, ucd_line)
+ -- print(string.format("%04X", usv))
+ -- if usv then print(usv, char_name) end
+ if usv then
+ local hex = hex_string_to_num(usv)
+ local hex = hex_string_to_num
+ pattfile:write("1", unicode.utf8.char(hex_string_to_num(usv)), "1 ")
+ pattfile:write("% U+", usv, " ", char_name:lower(), "\n")
+ end
+end
+
+pattfile:write("2", unicode.utf8.char(0x1361), "1 % U+1361 ETHIOPIC WORDSPACE\n")
+pattfile:write("2", unicode.utf8.char(0x1362), "1 % U+1362 ETHIOPIC FULL STOP\n")
+
+pattfile:write"}"
+
+assert(io.close(pattfile))
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb
new file mode 100755
index 0000000000..9e6b3d9558
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb
@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+#
+# This script generates hyphenation patterns for Turkmen
+#
+# This script has been written by Mojca Miklavec <mojca dot miklavec dot lists at gmail dot com>
+
+# open file for writing the patterns
+# $tr = File.new("hyph-tk.tex", "w")
+# in TDS
+$tr = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-tk.tex", "w")
+
+# write comments into the file
+def add_comment(str)
+ $tr.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n")
+end
+
+# define a class of vowels and consonants
+# vowels are split into so that unnecessary permutations are not generated
+front_vowels = %w{ä e i ö ü}
+back_vowels = %w{a y o u}
+consonants = %w{b ç d f g h j k l m n p r s t w ý z ň ž ş}
+# This is to eliminate impossible combinations
+common_suffix_consonants = %w{b ç d g j k l m n p s t ý z ş}
+
+
+# start the file
+add_comment(
+"Hyphenation patterns for Turkmen (hyph-tk.tex)
+
+Author: Nazar Annagurban <nazartm at gmail.com>
+License: Public domain
+Version: 0.1
+Date: 16 March 2010
+
+----------------------------------------------------------------------
+
+The file has been auto-generated from generate_patterns_tk.rb
+that is part of hyph-utf8.
+
+For more information about UTF-8 hyphenation patterns for TeX and
+links to this file see
+ http://www.tug.org/tex-hyphen/
+")
+
+# we have the following comment for Basque:
+#
+# Some of the patterns below represent combinations that never
+# happen in Turkmen. Would they happen, they would be hyphenated
+# according to the rules.
+
+$tr.puts '\patterns{'
+add_comment("Some suffixes are added through a hyphen. When hyphenating these words, a hyphen is added before the hyphen so that the line ends with a hyphen and the new line starts with a hyphen.")
+$tr.puts "1-4"
+
+add_comment("Allow hyphen after a vowel if and only if there is a single consonant before next the vowel")
+front_vowels.each do |v1|
+ consonants.each do |c|
+ front_vowels.each do |v2|
+ $tr.puts "#{v1}1#{c}#{v2}"
+ end
+ end
+end
+
+back_vowels.each do |v1|
+ consonants.each do |c|
+ back_vowels.each do |v2|
+ $tr.puts "#{v1}1#{c}#{v2}"
+ end
+ end
+end
+
+add_comment("These combinations occur in words of foreign origin or joined words")
+consonants.each do |c|
+ $tr.puts "a1#{c}i"
+ $tr.puts "a1#{c}e"
+ $tr.puts "y1#{c}ä"
+ $tr.puts "y1#{c}i"
+ $tr.puts "y1#{c}e"
+ $tr.puts "o1#{c}i"
+ $tr.puts "o1#{c}e"
+ $tr.puts "u1#{c}i"
+ $tr.puts "u1#{c}e"
+ $tr.puts "i1#{c}a"
+ $tr.puts "i1#{c}o"
+ $tr.puts "e1#{c}a"
+ $tr.puts "e1#{c}o"
+ $tr.puts "ä1#{c}o"
+ $tr.puts "ä1#{c}a"
+ $tr.puts "ö1#{c}a"
+end
+
+add_comment("Allow hyphen between two consonants (if there is only two of them), except when they are at the begining of the word")
+consonants.each do |c1|
+ consonants.each do |c2|
+ $tr.puts "#{c1}1#{c2}"
+ $tr.puts ".#{c1}2#{c2}"
+ end
+end
+
+add_comment("Patterns for triple consonants. There may be additions to this category, as this list is not exhaustive.")
+common_suffix_consonants.each do |c|
+ $tr.puts "ý2t1#{c}"
+ $tr.puts "ý2n1#{c}"
+ $tr.puts "ý2d1#{c}"
+ $tr.puts "r2t1#{c}"
+ $tr.puts "ý2p1#{c}"
+ $tr.puts "l2p1#{c}"
+ $tr.puts "l2t1#{c}"
+ $tr.puts "g2t1#{c}"
+ $tr.puts "n2t1#{c}"
+ $tr.puts "r2k1#{c}"
+ $tr.puts "r2p1#{c}"
+ $tr.puts "k2t1#{c}"
+ $tr.puts "r2h1#{c}"
+ $tr.puts "s2t1#{c}"
+ $tr.puts "l2k1#{c}"
+ $tr.puts "w2p1#{c}"
+ $tr.puts "n2s1#{c}"
+ $tr.puts "r2s1#{c}"
+ $tr.puts "l2m1#{c}"
+end
+
+add_comment("Exceptions and single word occurence patterns for words of foreign origin i.e. Russian")
+$tr.puts "s2k1d"
+$tr.puts "l1s2k"
+$tr.puts "l1s2t"
+$tr.puts "s1t2r"
+$tr.puts "n2g1l"
+$tr.puts "n1g2r"
+$tr.puts "s2k1w"
+
+# end the file
+$tr.puts '}'
+$tr.close
+
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README
new file mode 100644
index 0000000000..c56aff64b3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README
@@ -0,0 +1,61 @@
+
+See:
+- http://mirror.ctan.org/language/turkish/hyphen/turk_hyf.c
+- http://www.tug.org/TUGboat/Articles/tb09-1/tb20mackay.pdf
+
+-------
+Original encoding:
+
+a
+a= â
+e
+i ı
+i= î
+i: i
+o
+o= ô
+o: ö
+u
+u: ü
+
+-------
+
+character / UTF-8 letter / EC code / Unicode / Unicode name
+
+@ -- arabic hamza
+# -- ayn
+b
+c
+c: ç E7 U+00E7 LATIN SMALL LETTER C WITH CEDILLA
+d
+d! ḍ -- U+1E0D LATIN SMALL LETTER D WITH DOT BELOW
+d= ḏ -- U+1E0F LATIN SMALL LETTER D WITH LINE BELOW
+f
+g
+g= ğ A7 U+011F LATIN SMALL LETTER G WITH BREVE
+g: ġ -- U+0121 LATIN SMALL LETTER G WITH DOT ABOVE
+h
+h! ḥ -- U+1E25 LATIN SMALL LETTER H WITH DOT BELOW
+h= ẖ -- U+1E96 LATIN SMALL LETTER H WITH LINE BELOW
+j
+k
+k! ḳ -- U+1E33 LATIN SMALL LETTER K WITH DOT BELOW
+l
+m
+n
+n= ñ F1 U+00F1 LATIN SMALL LETTER N WITH TILDE
+p
+r
+s
+s! ṣ -- U+1E63 LATIN SMALL LETTER S WITH DOT BELOW
+s= -- s with line below - not even in unicode
+s: ş B3 U+015F LATIN SMALL LETTER S WITH CEDILLA
+t
+t! ṭ -- U+1E6D LATIN SMALL LETTER T WITH DOT BELOW
+t= ṯ -- U+1E6F LATIN SMALL LETTER T WITH LINE BELOW
+v
+y
+z
+z! ẓ -- U+1E93 LATIN SMALL LETTER Z WITH DOT BELOW
+z= ẕ -- U+1E95 LATIN SMALL LETTER Z WITH LINE BELOW
+z: ż BB U+017C LATIN SMALL LETTER Z WITH DOT ABOVE
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb
new file mode 100755
index 0000000000..ec7610d8d3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb
@@ -0,0 +1,120 @@
+#!/usr/bin/env ruby
+#
+# This script generates hyphenation patterns for modern Turkish
+#
+# "Algorithm" originally developed by:
+#
+# /****************************************************************************
+# * *
+# * turk_hyf.c -- a program to produce PatGen-like hyphenation *
+# * patterns for the Turkish Language. *
+# * *
+# * Copyright 1987, by Pierre A. MacKay. *
+# * *
+# * Humanities and Arts Computing Center *
+# * Mail-Stop DW-10 *
+# * University of Washington *
+# * Seattle, Washington 98105 *
+# * *
+# ****************************************************************************/
+#
+# Script written in June 2008 by Mojca Miklavec
+# as part of "Unicode Hyphenation Patterns" project
+#
+# Thanks to previous work, modifications and suggestions by:
+# - H. Turgut Uyar <uyar at itu.edu.tr>
+# - S. Ekin Kocabas <kocabas at stanford.edu>
+
+# TODO: add a function to generate patterns for Ottoman Turkish
+
+# open file for writing the patterns
+$tr = File.new("hyph-tr.tex", "w")
+# in TDS
+$tr = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-tr.tex", "w")
+
+# write comments into the file
+def add_comment(str)
+ $tr.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n")
+end
+
+# The patterns (and algorithm for generating them) were first meant
+# for supporting Ottoman Turkish, but adapted for modern Turkish later.
+# Turkish alphabet includes some special vowels (ıiöü) and consonants (çğş),
+# but the patterns also support another three vowels (âîû):
+# - a with circumflex is a necessity, it must be supported
+# - ı and u with circumflex could be there, not as pressing as "a" but still
+# Some letters such as z, k, s or t represent multiple letters in the old
+# alphabet, so some transcriptions from Ottoman to Turkish use some accents on
+# these for phonetic reasons, but there is no need to support them.
+#
+# Comment posted by: H. Turgut Uyar <uyar at itu.edu.tr>
+
+# define a class of vowels and consonants
+vowels = %w{a â e ı i î o ö u ü û}
+consonants = %w{b c ç d f g ğ h j k l m n p r s ş t v y z}
+
+# start the file
+add_comment(
+ "hyph-tr.tex\n\n" +
+ "Turkish hyphenation patterns\n\n" +
+ "This file is auto-generated from source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb that is part of hyph-utf8.\n" +
+ "Please don't modify this file; modify the generating script instead.\n\n" +
+ "Copyright (C) 1987 Pierre A. MacKay\n" +
+ " 2008, 2011 TUG\n\n" +
+ "This program can redistributed and/or modified under the terms\n" +
+ "of the LaTeX Project Public License Distributed from CTAN\n" +
+ "archives in directory macros/latex/base/lppl.txt; either\n" +
+ "version 1 of the License, or (at your option) any later version.\n\n" +
+ "Credits:\n" +
+ "- algorithm developed by P. A. MacKay for the Ottoman Texts Project in 1987\n" +
+ "- rules adapted for modern Turkish by H. Turgut Uyar <uyar at itu.edu.tr>\n" +
+ "- initiative to improve Turkish patterns by S. Ekin Kocabas <kocabas at stanford.edu>\n" +
+ "- script written by Mojca Miklavec <mojca.miklavec.lists at gmail.com> in June 2008\n\n" +
+ "See also:\n" +
+ "- http://mirror.ctan.org/language/turkish/hyphen/turk_hyf.c\n" +
+ "- http://www.tug.org/TUGboat/Articles/tb09-1/tb20mackay.pdf\n\n" +
+ "Differences with Ottoman patterns:\n" +
+ "- adapted for the use on modern TeX engines, using UTF-8 charactes\n" +
+ "- only letters for Modern Turkish + âîû (the first one often needed, the other two don't hurt)\n" +
+ "- (if needed, support for Ottoman Turkish might be provided separately under language code 'ota')\n\n" +
+ "Changes:\n" +
+ "- 2008-06-25/27/28 - create this file by adapting Ottoman rules for modern Turkish\n" +
+ "- 2011-08-10 - add LPPL licence with permission of Pierre A. MacKay\n"
+)
+$tr.puts '\patterns{'
+
+# rules for hyphenation
+add_comment("prohibit hyphen before vowels, allow after")
+vowels.each do |vowel|
+ $tr.puts "2#{vowel}1"
+end
+
+add_comment("allow hyphen either side of consonants")
+consonants.each do |cons|
+ $tr.puts "1#{cons}1"
+end
+
+add_comment("prevent e-cek at end of word")
+$tr.puts "2e2cek."
+
+add_comment("prohibit hyphen before pair of consonants\nmany pairs generated here are impossible anyway")
+consonants.each do |c1|
+ consonants.each do |c2|
+ $tr.puts "2#{c1}#{c2}"
+ end
+end
+
+add_comment("allow hyphen between vowels, but not after second vowel of pair\nseveral phonetically impossible pairs here")
+vowels.each do |v1|
+ vowels.each do |v2|
+ $tr.puts "#{v1}3#{v2}2"
+ end
+end
+
+add_comment("a couple of consonant-clusters")
+$tr.puts 'tu4r4k'
+$tr.puts 'm1t4rak'
+
+# end the file
+$tr.puts '}'
+$tr.close
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb
new file mode 100644
index 0000000000..2259f2ff1d
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb
@@ -0,0 +1 @@
+require_relative 'hyphen/language'
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml
new file mode 100644
index 0000000000..f96530da39
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml
@@ -0,0 +1,452 @@
+---
+donald_knuth: !ruby/object:TeX::Hyphen::Author
+ name: Donald
+ surname: Knuth
+ email:
+ contacted1: false
+ contacted2: false
+peter_heslin: !ruby/object:TeX::Hyphen::Author
+ name: Peter
+ surname: Heslin
+ email:
+ contacted1: false
+ contacted2: false
+dimitrios_filippou: !ruby/object:TeX::Hyphen::Author
+ name: Dimitrios
+ surname: Filippou
+ email: dimitrios{dot}filippou{at}riotinto{dot}com
+ contacted1: true
+ contacted2: true
+claudio_beccari: !ruby/object:TeX::Hyphen::Author
+ name: Claudio
+ surname: Beccari
+ email: claudio{dot}beccari{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+juan_aguirregabiria: !ruby/object:TeX::Hyphen::Author
+ name: Juan M.
+ surname: Aguirregabiria
+ email: juanmari{dot}aguirregabiria{at}ehu.es
+ contacted1: true
+ contacted2: true
+igor_marinovic: !ruby/object:TeX::Hyphen::Author
+ name: Igor
+ surname: Marinović
+ email: marinowski{at}gmail.com
+ contacted1: true
+ contacted2: true
+tilla_fick: !ruby/object:TeX::Hyphen::Author
+ name: Tilla
+ surname: Fick
+ email: fick{dot}tilla{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+chris_swanepoel: !ruby/object:TeX::Hyphen::Author
+ name: Chris
+ surname: Swanepoel
+ email: cj{at}swanepoel{dot}net
+ contacted1: true
+ contacted2: true
+matjaz_vrecko: !ruby/object:TeX::Hyphen::Author
+ name: Matjaž
+ surname: Vrečko
+ email: matjaz{at}mg-soft{dot}si
+ contacted1: true
+ contacted2: true
+goncal_badenes: !ruby/object:TeX::Hyphen::Author
+ name: Gonçal
+ surname: Badenes
+ email: g{dot}badenes{at}ieee.org
+ contacted1: false
+ contacted2: false
+pavel_sevecek: !ruby/object:TeX::Hyphen::Author
+ name: Pavel
+ surname: Ševeček
+ email: pavel{at}lingea{dot}cz
+ contacted1: false
+ contacted2: false
+# email doesn't work
+jana_chlebikova: !ruby/object:TeX::Hyphen::Author
+ name: Jana
+ surname: Chlebíková
+ email: chlebikj{at}dcs{dot}fmph{dot}uniba{dot}sk
+ contacted1: false
+ contacted2: false
+yannis_haralambous: !ruby/object:TeX::Hyphen::Author
+ name: Yannis
+ surname: Haralambous
+ email: yannis{dot}haralambous{at}telecom-bretagne{dot}eu
+ contacted1: true
+ contacted2: false
+frank_jensen: !ruby/object:TeX::Hyphen::Author
+ name: Frank
+ surname: Jensen
+ email: frank{dot}jensen{at}hugin{dot}com
+ contacted1: true
+ contacted2: true
+sergei_pokrovsky: !ruby/object:TeX::Hyphen::Author
+ name: Sergei
+ surname: Pokrovsky
+ email: sergio{dot}pokrovskij{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+javier_bezos: !ruby/object:TeX::Hyphen::Author
+ name: Javier
+ surname: Bezos
+ email: jbezos{at}tex-tipografia{dot}com
+ contacted1: true
+ contacted2: true
+een_saar: !ruby/object:TeX::Hyphen::Author
+ name: Enn
+ surname: Saar
+ email: saar{at}aai{dot}ee
+ contacted1: false
+ contacted2: false
+dejan_muhamedagic: !ruby/object:TeX::Hyphen::Author
+ name: Dejan
+ surname: Muhamedagić
+ email: dejan{at}hello-penguin{dot}com
+ contacted1: true
+ contacted2: true
+brian_wilson: !ruby/object:TeX::Hyphen::Author
+ name: Brian
+ surname: Wilson
+ email: bountonw{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+arthur_reutenauer: !ruby/object:TeX::Hyphen::Author
+ name: Arthur
+ surname: Reutenauer
+ email: arthur{dot}reutenauer{at}normalesup{dot}org
+ contacted1: true
+ contacted2: true
+mojca_miklavec: !ruby/object:TeX::Hyphen::Author
+ name: Mojca
+ surname: Miklavec
+ email: mojca{dot}miklavec{dot}lists{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+santhosh_thottingal: !ruby/object:TeX::Hyphen::Author
+ name: Santhosh
+ surname: Thottingal
+ email: santhosh{dot}thottingal{at}gmail{dot}com>
+ contacted1: true
+ contacted2: true
+# email doesn't work
+yves_codet: !ruby/object:TeX::Hyphen::Author
+ name: Yves
+ surname: Codet
+ email: ycodet{at}club-internet{dot}fr
+ contacted1: true
+ contacted2: true
+rune_kleveland: !ruby/object:TeX::Hyphen::Author
+ name: Rune
+ surname: Kleveland
+ email:
+ contacted1: false
+ contacted2: false
+# email doesn't work
+ole_michael_selberg: !ruby/object:TeX::Hyphen::Author
+ name: Ole Michael
+ surname: Selberg
+ email: o{dot}m{dot}selberg{at}c2i{dot}net
+ contacted1: true
+ contacted2: true
+dorjgotov_batmunkh: !ruby/object:TeX::Hyphen::Author
+ name: Dorjgotov
+ surname: Batmunkh
+ email: bataak{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+nazar_annagurban: !ruby/object:TeX::Hyphen::Author
+ name: Nazar
+ surname: Annagurban
+ email: nazartm{at}gmail{dot}com
+ contacted1: false
+ contacted2: false
+jan_michael_rynning: !ruby/object:TeX::Hyphen::Author
+ name: Jan Michael
+ surname: Rynning
+ email:
+ contacted1: false
+ contacted2: false
+eduard_werner: !ruby/object:TeX::Hyphen::Author
+ name: Eduard
+ surname: Werner
+ email: edi{dot}werner{at}gmx{dot}de
+ contacted1: false
+ contacted2: false
+werner_lemberg: !ruby/object:TeX::Hyphen::Author
+ name: Werner
+ surname: Lemberg
+ email: wl{at}gnu{dot}org
+ contacted1: true
+ contacted2: true
+# email doesn't work
+pedro_j_de_rezende: !ruby/object:TeX::Hyphen::Author
+ name: Pedro J.
+ surname: de Rezende
+ email: rezende{at}ddc{dot}unicamp{dot}br
+ contacted1: false
+ contacted2: false
+j_joao_dias_almeida: !ruby/object:TeX::Hyphen::Author
+ name: J. Joao
+ surname: Dias Almeida
+ email: jj{at}di{dot}uminho{dot}pt
+ contacted1:
+ contacted2:
+# email doesn't work
+piet_tutelaers: !ruby/object:TeX::Hyphen::Author
+ name: Piet
+ surname: Tutelaers
+ email: p{dot}t{dot}h{dot}tutelaers{at}tue{dot}nl
+ contacted1: false
+ contacted2: false
+vytas_statulevicius: !ruby/object:TeX::Hyphen::Author
+ name: Vytas
+ surname: Statulevičius
+ email: vytas{at}vtex{dot}nl
+ contacted1: false
+ contacted2: false
+sigitas_tolusis: !ruby/object:TeX::Hyphen::Author
+ name: Sigitas
+ surname: Tolušis
+ email: sigitas{at}vtex{dot}lt
+ contacted1: false
+ contacted2: false
+janis_vilims: !ruby/object:TeX::Hyphen::Author
+ name: Janis
+ surname: Vilims
+ email: jvilims{at}apollo{dot}lv
+ contacted1: false
+ contacted2: false
+joerg_knappen: !ruby/object:TeX::Hyphen::Author
+ name: Jörg
+ surname: Knappen
+ email: jknappen{at}web{dot}de
+ contacted1: true
+ contacted2: true
+medeni_shemde: !ruby/object:TeX::Hyphen::Author
+ name: Medeni
+ surname: Shemdê
+ email:
+ contacted1: false
+ contacted2: false
+terry_mart: !ruby/object:TeX::Hyphen::Author
+ name: Terry
+ surname: Mart
+ email: mart{at}kph{dot}uni-mainz{dot}de
+ contacted1: false
+ contacted2: false
+# email doesn't work
+jorgen_pind: !ruby/object:TeX::Hyphen::Author
+ name: Jorgen
+ surname: Pind
+ email: jorgen{at}lexis{dot}hi{dot}is
+ contacted1: false
+ contacted2: false
+marteinn_sverrisson: !ruby/object:TeX::Hyphen::Author
+ name: Marteinn
+ surname: Sverrisson
+ email:
+ contacted1: false
+ contacted2: false
+# email doesn't work
+kristinn_gylfason: !ruby/object:TeX::Hyphen::Author
+ name: Kristinn
+ surname: Gylfason
+ email: kristgy{at}ieee{dot}org
+ contacted1: false
+ contacted2: false
+# email doesn't work
+kevin_p_scannell: !ruby/object:TeX::Hyphen::Author
+ name: Kevin P.
+ surname: Scannell
+ email: scanell{at}slu{dot}edu
+ contacted1: false
+ contacted2: false
+# email doesn't work
+peter_kleiweg: !ruby/object:TeX::Hyphen::Author
+ name: Peter
+ surname: Kleiweg
+ email: p{dot}c{dot}c{dot}kleiweg{at}rug{dot}nl
+ contacted1: false
+ contacted2: false
+hanna_kolodziejska: !ruby/object:TeX::Hyphen::Author
+ name: Hanna
+ surname: Kołodziejska
+ email:
+ contacted1: false
+ contacted2: false
+boguslaw_jackowski: !ruby/object:TeX::Hyphen::Author
+ name: Bogusław
+ surname: Jackowski
+ email:
+ contacted1: true
+ contacted2: true
+marek_rycko: !ruby/object:TeX::Hyphen::Author
+ name: Marek
+ surname: Ryćko
+ email:
+ contacted1: false
+ contacted2: false
+vladimir_volovich: !ruby/object:TeX::Hyphen::Author
+ name: Vladimir
+ surname: Volovich
+ email: # TODO add e-mail address
+ contacted1: true
+ contacted2: true
+alexander_i_lebedev: !ruby/object:TeX::Hyphen::Author
+ name: Alexander I. # Not sure were 'I' belongs
+ surname: Lebedev
+ email: swan{at}scon155{dot}phys{dot}msu{dot}su
+ contacted1: false
+ contacted2: false
+# first email doesn't work
+maksym_polyakov: !ruby/object:TeX::Hyphen::Author
+ name: Maksym
+ surname: Polyakov
+ email: polyama{at}auburn{dot}edu # Second e-mail address in ukrhypmp.tex: mpoliak@i.com.ua
+ contacted1: false
+ contacted2: false
+adrian_rezus: !ruby/object:TeX::Hyphen::Author
+ name: Adrian
+ surname: Rezus
+ email: adriaan{at}{sci,cs}{dot}kun{dot}nl
+ contacted1: false
+ contacted2: false
+# email doesn't work
+sahak_petrosyan: !ruby/object:TeX::Hyphen::Author
+ name: Sahak
+ surname: Petrosyan
+ email: sahak{at}mit{dot}edu
+ contacted1: true
+ contacted2: true # I think "true, true" is right. Arthur
+dominik_wujastyk: !ruby/object:TeX::Hyphen::Author
+ name: Dominik
+ surname: Wujastyk
+ email: wujastyk{at}gmail{dot}com
+ contacted1: false
+ contacted2: false
+graham_toal: !ruby/object:TeX::Hyphen::Author
+ name: Graham
+ surname: Toal
+ email:
+ contacted1: false
+ contacted2: false
+donald_e_knuth: !ruby/object:TeX::Hyphen::Author
+ name: Donald E.
+ surname: Knuth
+ email: # Don doesn't use e-mail ;-)
+ contacted1: false
+ contacted2: false
+gerard_d_c_kuiken: !ruby/object:TeX::Hyphen::Author
+ name: Gerard D.C.
+ surname: Kuiken
+ email:
+ contacted1: false
+ contacted2: false
+pierre_mackay: !ruby/object:TeX::Hyphen::Author
+ name: P. A.
+ surname: MacKay
+ email:
+ contacted1: true
+ contacted2: true
+h_turgut_uyar: !ruby/object:TeX::Hyphen::Author
+ name: H. Turgut
+ surname: Uyar
+ email: uyar{at}itu{dot}edu{tr}
+ contacted1: true
+ contacted2: true
+# email doesn't work
+s_ekin_kocabas: !ruby/object:TeX::Hyphen::Author
+ name: S. Ekin
+ surname: Kocabas
+ email: kocabas{at}stanford{dot}edu
+ contacted1: true
+ contacted2: true
+bence_nagy: !ruby/object:TeX::Hyphen::Author
+ name: Bence
+ surname: Nagy
+ email: nagybence{at}tipogral{dot}hu
+ contacted1: true
+ contacted2: true
+kauko_saarinen: !ruby/object:TeX::Hyphen::Author
+ name: Kauko
+ surname: Saarinen
+ email:
+ contacted1: false
+ contacted2: false
+fred_karlsson: !ruby/object:TeX::Hyphen::Author
+ name: Fred
+ surname: Karlsson
+ email:
+ contacted1: false
+ contacted2: false
+rene_bastian: !ruby/object:TeX::Hyphen::Author # TODO make contact
+ name: René
+ surname: Bastian
+ email:
+ contacted1: false
+ contacted2: false
+daniel_flipo: !ruby/object:TeX::Hyphen::Author # TODO make contact
+ name: Daniel
+ surname: Flipo
+ email:
+ contacted1: false
+ contacted2: false
+bernard_gaulle: !ruby/object:TeX::Hyphen::Author # Deceased...
+ name: Bernard
+ surname: Gaulle
+ email:
+ contacted1: false
+ contacted2: false
+theppitak_karoonboonyanan: !ruby/object:TeX::Hyphen::Author
+ name: Theppitak
+ surname: Karoonboonyanan
+ email: theppitak{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+levan_shoshiashvili: !ruby/object:TeX::Hyphen::Author
+ name: Levan
+ surname: Shoshiashvili
+ email: shoshia{at}hotmail{dot}com
+ contacted1: true
+ contacted2: true
+# email doesn't work
+javier_mugica: !ruby/object:TeX::Hyphen::Author
+ name: Javier
+ surname: Múgica
+ email: javier{at}digi21{dot}eu
+ contacted1: true
+ contacted2: true
+georgi_boshnakov: !ruby/object:TeX::Hyphen::Author
+ name: Georgi
+ surname: Boshnakov
+ email: georgi{dot}boshnakov{at}manchester{dot}ac{dot}uk
+ contacted1: true
+ contacted2: true
+mike_kroutikov: !ruby/object:TeX::Hyphen::Author
+ name: Mike
+ surname: Kroutikov
+ email: pgmmpk{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
+aleksandr_andreev: !ruby/object:TeX::Hyphen::Author
+ name: Aleksandr
+ surname: Andreev
+ email: ''
+ contacted1: true
+ contacted2: true
+maksim_salau: !ruby/object:TeX::Hyphen::Author
+ name: Maksim
+ surname: Salau
+ email: maksim{dot}salau{at}gmail{dot}com
+ contacted1: true
+ contacted2: false
+wie_ming_ang: !ruby/object:TeX::Hyphen::Author
+ name: Wie-Ming
+ surname: Cittānurakkho Bhikkhu
+ email: wiemingang{at}gmail{dot}com
+ contacted1: true
+ contacted2: true
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
new file mode 100644
index 0000000000..3d3437d74b
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
@@ -0,0 +1,364 @@
+require 'yaml'
+require 'hydra'
+require 'byebug' unless ENV['RACK_ENV'] == "production"
+
+require_relative 'path'
+
+class String
+ def superstrip
+ strip.gsub /%.*$/, ''
+ end
+
+ def supersplit
+ strip.gsub(/\s+/m,"\n").split("\n")
+ end
+
+ def safe
+ gsub /[\s-]/, ''
+ end
+
+ def titlecase
+ split.map(&:capitalize).join(' ')
+ end
+end
+
+module TeX
+ module Hyphen
+ class InvalidMetadata < StandardError; end
+ class NoAuthor < InvalidMetadata; end
+ class NoLicence < InvalidMetadata; end
+
+ include PATH
+
+ class Author
+ def initialize(name,surname,email,contacted1,contacted2)
+ @name = name
+ @surname = surname
+ @email = email
+ # this mostly means if email has been recently checked
+ @contacted1 = contacted1
+ # this means if we made more cooperation with author,
+ # exchanging patches etc.
+ @contacted2 = contacted2
+ end
+
+ attr_reader :name, :surname, :email
+
+ def self.authors
+ @@authors ||= YAML::load File.read File.expand_path 'authors.yml', __dir__
+ end
+
+ def self.all
+ authors.values
+ end
+
+ def self.[] a
+ authors[a]
+ end
+ end
+
+ class Language
+ attr_reader :bcp47
+
+ @@eohmarker = '=' * 42
+
+ def iso639
+ @bcp47.split('-').first
+ end
+
+ # FIXME Find something better than all that extract_metadata unless @blah nonsens
+ def babelname
+ extract_metadata unless @babelname
+ @babelname
+ end
+
+ def description
+ extract_metadata unless @description
+ @description
+ end
+
+ def use_old_loader
+ extract_metadata unless @use_old_loader
+ @use_old_loader
+ end
+
+ def use_old_patterns_comment
+ extract_metadata unless @use_old_patterns_comment
+ @use_old_patterns_comment
+ end
+
+ def legacy_patterns
+ extract_metadata unless @legacy_patterns
+ @legacy_patterns
+ end
+
+ def message
+ extract_metadata unless @message
+ @message
+ end
+
+ def known_bugs
+ extract_metadata unless @known_bugs
+ @known_bugs
+ end
+
+ def initialize(bcp47 = nil)
+ @bcp47 = bcp47
+ end
+
+ def self.languages
+ @@languages ||= Dir.glob(File.join(PATH::TEX, 'hyph-*.tex')).inject [] do |languages, texfile|
+ bcp47 = texfile.gsub /^.*\/hyph-(.*)\.tex$/, '\1'
+ # languages << [bcp47, Language.new(bcp47)]
+ languages << [bcp47, nil]
+ end.to_h
+ end
+ @@languages = Language.languages
+
+ def self.all
+ @@all ||= languages.map do |bcp47, language|
+ next if bcp47 == 'sr-cyrl' # FIXME Remove later
+ @@languages[bcp47] ||= Language.new(bcp47)
+ end.compact
+ end
+
+ def self.all_by_iso639
+ all.inject(Hash.new) do |maintags, language|
+ (maintags[language.iso639] ||= Array.new) << language
+ maintags
+ end
+ end
+
+ def self.find_by_bcp47(bcp47)
+ languages[bcp47]
+ end
+
+ def private_use?
+ @bcp47.length == 3 || @bcp47[3] == '-' and @bcp47[0] == 'q' and ('a'..'t').include? @bcp47[1]
+ end
+
+ # TODO This should probably become “macrolanguage name” or something similar
+ # @@displaynames = {
+ # 'el' => 'Greek',
+ # 'nb' => 'Norwegian',
+ # 'nn' => 'Norwegian',
+ # 'sh' => 'Serbian',
+ # }
+
+ def licences
+ extract_metadata unless @licences
+ @licences
+ end
+
+ def lefthyphenmin
+ extract_metadata unless @lefthyphenmin
+ @lefthyphenmin
+ end
+
+ def righthyphenmin
+ extract_metadata unless @righthyphenmin
+ @righthyphenmin
+ end
+
+ # Strictly speaking a misnomer, because grc-x-ibycus should also return true.
+ # But useful for a number of apostrophe-related routines
+ def isgreek?
+ ['grc', 'el-polyton', 'el-monoton'].include? @bcp47
+ end
+
+ def serbian?
+ @bcp47 =~ /^sh-/
+ end
+
+ def italic?
+ ['it', 'pms', 'rm'].include? @bcp47
+ end
+
+ def has_apostrophes?
+ begin
+ !isgreek? && patterns.any? { |p| p =~ /'/ }
+ rescue Errno::ENOENT
+ false
+ end
+ end
+
+ def has_hyphens?
+ begin
+ patterns.any? { |p| p =~ /-/ }
+ rescue Errno::ENOENT
+ false
+ end
+ end
+
+ def extract_apostrophes
+ plain, with_apostrophe = Array.new, nil
+
+ patterns.each do |pattern|
+ plain << pattern
+ if pattern =~ /'/ && !isgreek?
+ pattern_with_apostrophe = pattern.gsub(/'/,"’")
+ plain << pattern_with_apostrophe
+ (with_apostrophe ||= []) << pattern_with_apostrophe
+ end
+ end
+
+ { plain: plain, with_apostrophe: with_apostrophe }
+ end
+
+ def extract_characters # FIXME Turkish and others iİ; check Church Slavonic
+ characters = Array.new
+
+ characters_indexes = patterns.join.gsub(/[.0-9]/,'').unpack('U*').sort.uniq
+ characters_indexes.each do |c|
+ ch = [c].pack('U')
+ characters << ch + ch.upcase
+ characters << "’’" if ch == "'" && !isgreek?
+ end
+
+ characters
+ end
+
+ def comments_and_licence # Major TODO extract everything into YAML, and write .yml
+ @comments_and_licence ||= readtexfile.gsub(/(.*)\\patterns.*/m,'\1')
+ end
+
+ def authors
+ extract_metadata unless @authors
+ @authors
+ end
+
+ def github_link
+ sprintf 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-%s.tex', @bcp47
+ end
+
+ def <=>(other)
+ # TODO Remove when practical
+ a = self.babelname
+ b = other.babelname
+
+ if a == 'serbian' && b == 'serbianc'
+ return -1
+ elsif a == 'serbianc' && b == 'serbian'
+ return 1
+ end
+
+ self.bcp47 <=> other.bcp47
+ end
+
+ @@texfile = Hash.new
+ def readtexfile(bcp47 = @bcp47)
+ begin
+ @@texfile[bcp47] ||= File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', bcp47)))
+ rescue Errno::ENOENT
+ @@texfile[bcp47] = ""
+ end
+ end
+
+ def patterns
+ @patterns ||= if @bcp47 == 'eo' then
+ readtexfile.superstrip.
+ gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1').
+ #
+ gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.').
+ gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.').
+ gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.').
+ #
+ supersplit
+ else
+ readtexfile(if ['nb', 'nn'].include? @bcp47 then 'no' else @bcp47 end).superstrip.
+ gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1').
+ supersplit
+ end
+ end
+
+ def exceptions
+ unless @exceptions
+ if readtexfile.superstrip.index('\hyphenation')
+ @exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit
+ if @exceptions != ""
+ @hyphenation = @exceptions.inject [] do |exceptions, exception|
+ # byebug unless exception
+ exceptions << [exception.gsub('-', ''), exception]
+ end.to_h
+ else
+ @hyphenation = { }
+ end
+ else
+ @exceptions = ""
+ @hyphenation = { }
+ end
+ end
+
+ @exceptions
+ end
+
+ def hyphenate(word)
+ exceptions
+ return @hyphenation[word] if @hyphenation[word] # FIXME Better name
+
+ unless @hydra
+ begin
+ # byebug
+ metadata = extract_metadata
+ @hydra = Hydra.new patterns, :lax, '', metadata
+ rescue InvalidMetadata
+ @hydra = Hydra.new patterns
+ end
+ end
+ @hydra.showhyphens(word)
+ end
+
+ def extract_metadata
+ header = ""
+ File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', @bcp47))).each_line do |line|
+ break if line =~ /\\patterns|#{@@eohmarker}/
+ header += line.gsub(/^% /, '').gsub(/%.*/, '')
+ end
+ begin
+ metadata = YAML::load header
+ raise InvalidMetadata unless metadata.is_a? Hash
+ rescue Psych::SyntaxError
+ raise InvalidMetadata
+ end
+
+ @name = metadata.dig('language', 'name')
+ @lefthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'left') || metadata.dig('hyphenmins', 'generation', 'left')
+ @righthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'right') || metadata.dig('hyphenmins', 'generation', 'right')
+ # FIXME That’s not nice
+ if respond_to? :synonyms
+ @synonyms = metadata.dig('texlive', 'synonyms') || []
+ @encoding = metadata.dig('texlive', 'encoding')
+ end
+ @message = metadata.dig('texlive', 'message')
+ @legacy_patterns = metadata.dig('texlive', 'legacy_patterns')
+ @use_old_loader = metadata.dig('texlive', 'use_old_loader')
+ @use_old_patterns_comment = metadata.dig('texlive', 'use_old_patterns_comment')
+ @description = metadata.dig('texlive', 'description')
+ @babelname = metadata.dig('texlive', 'babelname')
+ @package = metadata.dig('texlive', 'package')
+ licences = metadata.dig('licence')
+ raise NoLicence unless licences
+ licences = [licences] unless licences.is_a? Array
+ @licences = licences.map do |licence|
+ raise bcp47 if licence.is_a? String
+ next if licence.values == [nil]
+ licence.dig('name') || 'custom'
+ end.compact
+ authors = metadata.dig('authors')
+ if authors
+ @authors = authors.map do |author|
+ author['name']
+ end
+ else
+ @authors = nil
+ raise NoAuthor.new # FIXME
+ end
+ @known_bugs = metadata.dig('known_bugs')
+
+ # raise NoAuthor unless @authors && @authors.count > 0 # TODO Later ;-) AR 2018-09-13
+
+ metadata
+ end
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
new file mode 100644
index 0000000000..bc02973938
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
@@ -0,0 +1,94 @@
+german:
+ dependency: dehyph
+ description: |-
+ Hyphenation patterns for German in T1/EC and UTF-8 encodings,
+ for traditional and reformed spelling, including Swiss German.
+ The package includes the latest patterns from dehyph-exptl
+ (known to TeX under names 'german', 'ngerman' and 'swissgerman'),
+ however 8-bit engines still load old versions of patterns
+ for 'german' and 'ngerman' for backward-compatibility reasons.
+ Swiss German patterns are suitable for Swiss Standard German
+ (Hochdeutsch) not the Alemannic dialects spoken in Switzerland
+ (Schwyzerduetsch).
+ There are no known patterns for written Schwyzerduetsch.
+# for Russian and Ukrainian (until we implement the new functionality at least)
+russian:
+ dependency: ruhyphen
+ukrainian:
+ dependency: ukrhyph
+greek:
+ doc: doc/generic/elhyphen
+ shortdesc: Modern Greek
+ description: |-
+ Hyphenation patterns for Modern Greek in monotonic and polytonic
+ spelling in LGR and UTF-8 encodings. Patterns in UTF-8 use two code
+ positions for each of the vowels with acute accent (a.k.a tonos,
+ oxia), e.g., U+03AC, U+1F71 for alpha.
+hungarian:
+ doc: doc/generic/huhyphen
+arabic:
+ shortdesc: (No) Arabic
+farsi:
+ shortdesc: (No) Persian
+chinese:
+ shortdesc: Chinese pinyin
+ description: |-
+ Hyphenation patterns for unaccented transliterated Mandarin Chinese
+ (pinyin) in T1/EC and UTF-8 encodings. The latter can hyphenate pinyin
+ with or without tone markers; the former only without.
+norwegian:
+ shortdesc: Norwegian Bokmal and Nynorsk
+ description: |-
+ Hyphenation patterns for Norwegian Bokmal and Nynorsk in T1/EC and
+ UTF-8 encodings.
+churchslavonic:
+ shortdesc: Church Slavonic
+uppersorbian:
+ shortdesc: Upper Sorbian
+ethiopic:
+ shortdesc_full: Hyphenation patterns for Ethiopic scripts
+mongolian:
+ shortdesc_full: Mongolian hyphenation patterns in Cyrillic script
+ description: |-
+ Hyphenation patterns for Mongolian in T2A, LMC and UTF-8 encodings.
+ LMC encoding is used in MonTeX. The package includes two sets of
+ patterns that will hopefully be merged in future.
+latin:
+ description: |-
+ Hyphenation patterns for Latin in T1/EC and UTF-8 encodings,
+ mainly in modern spelling (u when u is needed and v when v is needed),
+ medieval spelling with the ligatures \ae and \oe and the (uncial)
+ lowercase 'v' written as a 'u' is also supported. Apparently
+ there is no conflict between the patterns of modern Latin and
+ those of medieval Latin.
+ Hyphenation patterns for the Classical Latin in T1/EC and UTF-8
+ encodings. Classical Latin hyphenation patterns are different from
+ those of 'plain' Latin, the latter being more adapted to modern Latin.
+ Hyphenation patterns for the Liturgical Latin in T1/EC and UTF-8
+ encodings.
+english:
+ description: |-
+ Additional hyphenation patterns for American and British
+ English in ASCII encoding. The American English patterns
+ (usenglishmax) greatly extend the standard patterns from Knuth
+ to find many additional hyphenation points. British English
+ hyphenation is completely different from US English, so has its
+ own set of patterns.
+indic:
+ description: |-
+ Hyphenation patterns for Assamese, Bengali, Gujarati, Hindi, Kannada,
+ Malayalam, Marathi, Oriya, Panjabi, Tamil and Telugu for Unicode
+ engines.
+"ancient greek":
+ description: |-
+ Hyphenation patterns for Ancient Greek in LGR and UTF-8 encodings,
+ including support for (obsolete) Ibycus font encoding.
+ Patterns in UTF-8 use two code positions for each of the vowels with
+ acute accent (a.k.a tonos, oxia), e.g., U+03AE, U+1F75 for eta.
+serbian:
+ description: |-
+ Hyphenation patterns for Serbian in T1/EC, T2A and UTF-8 encodings.
+ For 8-bit engines the patterns are available separately as 'serbian'
+ in T1/EC encoding for Latin script and 'serbianc' in T2A encoding for
+ Cyrillic script. Unicode engines should only use 'serbian'
+ which has patterns in both scripts combined.
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb
new file mode 100644
index 0000000000..fda422b11a
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb
@@ -0,0 +1,25 @@
+module TeX
+ module Hyphen
+ module PATH
+ ROOT = File.expand_path('../../../../../../../..', __FILE__)
+
+ TeXROOT = File.join(ROOT, 'hyph-utf8')
+ TeX_GENERIC = File.join(TeXROOT, 'tex', 'generic')
+ PAT = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns')
+ TXT = File.join(PAT, 'txt')
+ TEX = File.join(PAT, 'tex')
+ PTEX = File.join(PAT, 'ptex')
+ QUOTE = File.join(PAT, 'quote')
+ LOADER = File.join(TeX_GENERIC, 'hyph-utf8', 'loadhyph')
+
+ SUPPORT = File.join(TeXROOT, '%s', 'generic', 'hyph-utf8', 'languages', '*')
+
+ HYPHU8 = File.join('tex', 'generic', 'hyph-utf8')
+
+ TL = File.join(ROOT, 'TL')
+ LANGUAGE_DAT = File.join(PATH::TL, 'texmf-dist', 'tex', 'generic', 'config')
+ # hyphen-foo.tlpsrc for TeX Live
+ TLPSRC = File.join(PATH::TL, 'tlpkg', 'tlpsrc')
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb
new file mode 100644
index 0000000000..45dd33da04
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb
@@ -0,0 +1,22 @@
+require_relative 'texlive/source'
+require_relative 'texlive/loader'
+require_relative 'texlive/package'
+
+module TeX
+ module Hyphen
+ module TeXLive
+ def synonyms
+ extract_metadata unless @synonyms
+ @synonyms
+ end
+
+ def encoding
+ extract_metadata unless @encoding
+ @encoding
+ end
+
+ include Source
+ include Loader
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
new file mode 100644
index 0000000000..2fcb3945d3
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
@@ -0,0 +1,193 @@
+module TeX
+ module Hyphen
+ module TeXLive
+ module Loader
+
+ def unicode_only? # TODO Spec out
+ ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47
+ end
+
+ def string_enc
+ (encoding == nil) ? "" : encoding.upcase + " "
+ end
+
+ def lcchars
+ if isgreek?
+ # some catcodes for XeTeX
+ return {
+ lccode: {
+ 0x0027 => '\'',
+ 0x2019 => '’',
+ 0x02BC => 'ʼ',
+ 0x1FBD => '᾽',
+ 0x1FBF => '᾿',
+ }
+ }
+ end
+
+ return { } unless unicode_only?
+
+ default = {
+ comment: 'Set \lccode for ZWNJ and ZWJ.',
+ lccode: {
+ 0x200C => false,
+ 0x200D => false,
+ }
+ }
+
+ {
+ 'mul-ethi' => {
+ comment: 'Set \lccode for Ethiopian word space.',
+ lccode: {
+ 0x1361 => false,
+ 0x1362 => false,
+ }
+ },
+ 'cu' => {
+ comment: 'fix lccodes for some characters (they were recently included in Unicode)',
+ lccode: {
+ 0x1C82 => 'sharp o in lowercase "uk"',
+ 0x1DF6 => false,
+ 0x1DF7 => false,
+ 0x1DF8 => false,
+ 0x1DF9 => false,
+ 0xA69E => false,
+ 0x1C86 => false,
+ 0xA67E => false,
+ 0xFE2E => false,
+ 0xFE2F => false,
+ }
+ },
+ 'sa' => {
+ comment: 'Set \lccode for ZWNJ and ZWJ.',
+ lccode: {
+ 0x200C => false,
+ 0x200D => "\n% Set \\lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.",
+ 0x0CF1 => false,
+ 0x0CF2 => false,
+ }
+ },
+ }[@bcp47] || default
+ end
+
+ def utf8_engine_message
+ if serbian?
+ 'UTF-8 Serbian hyphenation patterns'
+ else
+ sprintf('UTF-8 %s', message)
+ end
+ end
+
+ def non_utf8_engine_message
+ if unicode_only?
+ sprintf('No %s - only for Unicode engines', message)
+ else
+ sprintf('%s%s', string_enc, message)
+ end
+ end
+
+ def engine_message(engine)
+ if engine == 'UTF-8'
+ {
+ comment: 'Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument',
+ message: utf8_engine_message
+ }
+ else # engine is 8-bit or pTeX
+ {
+ comment: engine +
+ if engine == '8-bit' then " engine (such as TeX or pdfTeX)" else "" end,
+ message: non_utf8_engine_message
+ }
+ end
+ end
+
+ def input_8bit_file
+ if @bcp47 == 'la-x-liturgic'
+ {
+ input: [pTeX_patterns]
+ }
+ elsif use_old_patterns_comment
+ # explain why we are still using the old patterns
+ {
+ comment: use_old_patterns_comment,
+ input: [legacy_patterns]
+ }
+ elsif !italic?
+ {
+ input: [sprintf('conv-utf8-%s.tex', encoding), sprintf('hyph-%s.tex', @bcp47)]
+ }
+ else
+ {
+ input: [sprintf('hyph-%s.tex', @bcp47)]
+ }
+ end
+ end
+
+ def input_utf8_line
+ if serbian?
+ {
+ comment: 'We load both scripts at the same time to simplify usage',
+ input: ['hyph-sh-latn.tex', 'hyph-sh-cyrl.tex']
+ }
+ else
+ {
+ input: [sprintf('hyph-%s.tex', @bcp47)]
+ }
+ end
+ end
+
+ def format_inputs(specification)
+ if specification.is_a? Array
+ return specification.map do |hash|
+ format_inputs(hash)
+ end
+ end
+
+ byebug unless specification.is_a? Hash
+ comment = specification[:comment]
+ message = specification[:message]
+ if comment then [sprintf('%% %s', comment)] else [] end +
+ if message then [sprintf('\\message{%s}', message)] else [] end +
+ (specification[:lccode] || []).map do |code, comment|
+ sprintf '\\lccode"%04X="%04X%s', code, code, if comment then sprintf ' %% %s', comment else '' end
+ end +
+ (specification[:input] || []).map do |input|
+ sprintf '\\input %s', input
+ end
+ end
+
+ def input_line(engine)
+ if engine == '8-bit'
+ input_8bit_file
+ elsif engine == 'pTeX'
+ { input: [pTeX_patterns] }
+ elsif engine == 'UTF-8'
+ input_utf8_line
+ end
+ end
+
+ def utf8_chunk
+ [
+ engine_message('UTF-8'),
+ # lccodes
+ lcchars,
+ input_line('UTF-8'),
+ if has_apostrophes? then { input: [sprintf('hyph-quote-%s.tex', bcp47)] } end
+ ].compact
+ end
+
+ def nonutf8_chunk(engine)
+ [engine_message(engine), unless unicode_only? then input_line(engine) end].compact
+ end
+
+ def loadhyph
+ if use_old_loader
+ legacy_patterns
+ else
+ sprintf 'loadhyph-%s.tex', @bcp47.gsub(/^sh-/, 'sr-')
+ end
+ end
+ end
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
new file mode 100644
index 0000000000..69481a4177
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
@@ -0,0 +1,126 @@
+module TeX
+ module Hyphen
+ module TeXLive
+ class Package
+ attr_reader :name
+
+ @@metadata = YAML::load File.read File.expand_path '../packages.yml', __dir__
+
+ def initialize(name)
+ @name = name
+ @languages = []
+ end
+
+ def add_language(language)
+ @languages << language
+ @languages.sort!
+ end
+
+ def languages
+ @languages
+ end
+
+ def self.packages
+ # a hash with the names of TeX Live packages, either individual language names,
+ # or an array of languages as the value
+ @@packages ||= Language.all.inject(Hash.new) do |packages, language|
+ if name = language.package || language.babelname # Package name, otherwise single language
+ packages[name] ||= Package.new(name) # Create new package if necessary
+ packages[name].add_language language
+ end
+
+ packages
+ end
+ end
+
+ def self.all
+ packages.values
+ end
+
+ def description_s
+ override = @@metadata.dig(@name, 'shortdesc_full')
+ return override if override
+
+ leader = @@metadata.dig(@name, 'shortdesc') || @name.titlecase
+ sprintf '%s hyphenation patterns', leader
+ end
+
+ def description
+ description = @@metadata.dig(@name, 'description')
+ if description
+ description
+ elsif @languages.count == 1
+ @languages.first.description
+ else
+ ''
+ end
+ end
+
+ def has_dependency?
+ @@metadata.dig(name, 'dependency')
+ end
+
+ def list_dependencies
+ dependencies = [
+ "depend hyphen-base",
+ "depend hyph-utf8",
+ ]
+
+ # external dependencies
+ if dependency = has_dependency?
+ dependencies << sprintf("depend %s", dependency)
+ end
+
+ dependencies
+ end
+
+ def list_support_files(type) # type is ‘doc’ or ‘source’
+ # Cache directory contents
+ (@dirlist ||= { })[type] ||= Dir.glob(sprintf(PATH::SUPPORT, type)).select do |file|
+ File.directory?(file)
+ end.map do |dir|
+ dir.gsub /^.*\//, ''
+ end
+
+ files = (languages.map(&:bcp47) & @dirlist[type]).map do |bcp47|
+ sprintf("%s/generic/hyph-utf8/languages/%s", type, bcp47)
+ end
+
+ if special = @@metadata.dig(name, type)
+ files << special
+ end
+
+ files
+ end
+
+ def list_run_files
+ files = []
+ files << "tex/generic/hyph-utf8/patterns/tex/hyph-no.tex" if name == "norwegian"
+
+ files = languages.inject(files) do |files, language|
+ files + language.list_run_files
+ end
+
+ unless has_dependency?
+ languages.each do |language|
+ if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop'
+ files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns)
+ end
+ end
+ end
+
+ files
+ end
+
+ def <=>(other)
+ name <=> other.name
+ end
+
+ # TODO Spec out
+ def self.find(name)
+ packages[name]
+ end
+ end
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
new file mode 100644
index 0000000000..61292dcd3b
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
@@ -0,0 +1,117 @@
+module TeX
+ module Hyphen
+ module TeXLive
+ module Source
+ def list_synonyms
+ if synonyms && synonyms.length > 0
+ sprintf " synonyms=%s", synonyms.join(',')
+ else
+ ''
+ end
+ end
+
+ def list_hyphenmins
+ lmin = lefthyphenmin
+ rmin = righthyphenmin
+ sprintf "lefthyphenmin=%s \\\n\trighthyphenmin=%s", lmin, rmin
+ end
+
+ # ext: 'pat' or 'hyp'
+ # filetype: 'patterns' or 'exceptions'
+ def plain_text_line(ext, filetype) # TODO Figure out if we will sr-cyrl to be generated again
+ return "" if ['ar', 'fa', 'grc-x-ibycus', 'mn-cyrl-x-lmc'].include? @bcp47
+
+ if @bcp47 =~ /^sh-/
+ # TODO Warning AR 2018-09-12
+ filename = sprintf 'hyph-sh-latn.%s.txt,hyph-sh-cyrl.%s.txt', ext, ext
+ else
+ filename = sprintf 'hyph-%s.%s.txt', @bcp47, ext
+ filepath = File.join(PATH::TXT, filename)
+ # check for existence of file and that it’s not empty
+ unless File.file?(filepath) && File.read(filepath).length > 0
+ # if the file we were looking for was a pattern file, something’s wrong
+ if ext == 'pat'
+ raise sprintf("There is some problem with plain patterns for language [%s]!!!", @bcp47)
+ else # the file is simply an exception file and we’re happy
+ filename = '' # And we return and empty file name after all
+ end
+ end
+ end
+
+ sprintf "file_%s=%s", filetype, filename
+ end
+
+ def exceptions_line
+ plain_text_line('hyp', 'exceptions')
+ end
+
+ def patterns_line
+ plain_text_line('pat', 'patterns')
+ end
+
+ def pTeX_patterns
+ if italic?
+ sprintf 'hyph-%s.tex', @bcp47
+ elsif encoding
+ sprintf 'hyph-%s.%s.tex', @bcp47, encoding
+ else
+ legacy_patterns
+ end
+ end
+
+ def list_loader
+ # which loader to use
+ if ['ar', 'fa'].include? @bcp47
+ sprintf "file=%s \\\n\tfile_patterns=", loadhyph
+ elsif @bcp47 == 'grc-x-ibycus'
+ # TODO: fix this
+ sprintf "file=%s \\\n\tluaspecial=\"disabled:8-bit only\"", loadhyph
+ else
+ sprintf "file=%s", loadhyph
+ end
+ end
+
+ def list_run_files
+ return [] if use_old_loader
+
+ files = []
+
+ files << File.join(PATH::HYPHU8, 'loadhyph', loadhyph)
+ if has_apostrophes?
+ files << File.join(PATH::HYPHU8, 'patterns', 'quote', sprintf("hyph-quote-%s.tex", bcp47))
+ end
+
+ files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47))
+ if encoding && encoding != "ascii" then
+ files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding))
+ elsif bcp47 == "cop"
+ files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns)
+ end
+
+ # we skip the mongolian language for luatex files
+ return files if bcp47 == "mn-cyrl-x-lmc"
+
+ ['chr', 'pat', 'hyp', 'lic'].each do |t|
+ files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-%s.%s.txt', bcp47, t))
+ end
+
+ if bcp47 =~ /^sh-/
+ # duplicate entries (will be removed later)
+ files << File.join(PATH::HYPHU8, 'patterns', 'tex', 'hyph-sr-cyrl.tex')
+ ['chr', 'pat', 'hyp', 'lic'].each do |t|
+ # duplicate entries (will be removed later)
+ files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-sr-cyrl.%s.txt', t))
+ end
+ end
+
+ files
+ end
+
+ def package
+ extract_metadata
+ @package
+ end
+ end
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb b/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb
new file mode 100644
index 0000000000..da5b425214
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb
@@ -0,0 +1,7 @@
+$LOAD_PATH << File.expand_path('../../lib', __FILE__)
+require 'tex/hyphen/language'
+require 'tex/hyphen/texlive/package'
+require 'tex/hyphen/texlive'
+require 'tex/hyphen/texlive/loader'
+
+RSpec::Expectations.configuration.on_potential_false_positives = :nothing
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
new file mode 100644
index 0000000000..4805b1878d
--- /dev/null
+++ b/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
@@ -0,0 +1,985 @@
+require_relative '../../spec_helper'
+
+include TeX::Hyphen
+include Language::TeXLive
+include TeXLive
+include Loader
+
+describe String do
+ describe '#superstrip' do
+ it "calls String#strip" do
+ str = " foo bar "
+ expect(str).to receive(:strip).and_return "foo bar"
+ str.superstrip
+ end
+
+ it "strips TeX comments" do
+ expect("foo % bar".superstrip).to eq "foo "
+ end
+ end
+
+ describe '#supersplit' do
+ it "calls String#strip" do
+ str = "foo bar "
+ expect(str).to receive(:strip).and_return "foo bar"
+ str.supersplit
+ end
+
+ it "splits across whitespace ranges" do
+ expect("foo bar".supersplit).to eq ["foo", "bar"]
+ end
+ end
+
+ describe '#safe' do
+ it "strips hyphens" do
+ expect("foo-bar".safe).to eq "foobar"
+ end
+
+ it "strips spaces" do
+ expect("ancient greek".safe).to eq "ancientgreek"
+ end
+ end
+
+ describe '#titlecase' do
+ it "capitalises each word" do
+ expect("modern greek".titlecase).to eq "Modern Greek"
+ end
+ end
+end
+
+# TODO? Spec out PATH module
+
+describe Author do
+ describe 'Class methods' do
+ describe '.authors' do
+ it "returns a hash of authors" do
+ expect(Author.authors).to be_a Hash
+ end
+ end
+
+ describe '.all' do
+ it "returns the array of authors" do
+ expect(Author.all).to be_an Array
+ end
+ end
+
+ describe '.[]' do
+ it "returns the author for the key" do
+ expect(Author['donald_knuth']).to be_an Author
+ end
+ end
+ end
+
+ describe 'Instance methods' do
+ let(:dek) { Author['donald_knuth'] }
+
+ describe '#name' do
+ it "returns the author’s name" do
+ expect(dek.name).to eq 'Donald'
+ end
+ end
+
+ describe '#surname' do
+ it "returns the author’s surname" do
+ expect(dek.surname).to eq 'Knuth'
+ end
+ end
+
+ describe '#email' do
+ it "returns the email" do
+ expect(dek.email).to be_nil
+ end
+ end
+ end
+end
+
+describe Language do
+ describe 'Class variables' do
+ it "has an end-of-header marker" do
+ expect(Language.class_variable_get :@@eohmarker).to match /^\={42}$/
+ end
+ end
+
+ describe '.new' do
+ it "creates a new Language instance" do
+ expect(Language.new).to be_a Language
+ end
+
+ it "takes an optional BCP47 tag as argument" do
+ language = Language.new('ro')
+ expect(language.instance_variable_get :@bcp47).to eq 'ro'
+ end
+
+# it "calls .languages" do
+# expect(Language).to receive(:languages).and_return({ 'pa' => nil })
+# Language.new('pa')
+# end
+ end
+
+ describe '.languages' do
+ it "sets the @@languages class variable" do
+ Language.languages
+ expect(Language.class_variable_get :@@languages).to be_a Hash
+ end
+
+ it "lists all languages" do
+ # All the TeX files. Note [no] and [mn-cyrl-x-lmc] don’t have corresponding plain text files.
+ expect(Language.languages.count).to eq 82 # Was 79; 3 more “TeX Live dummies” [ar] [fa] [grc-x-ibycus] TODO Maybe remove
+ end
+ end
+
+ describe '.all' do
+ it "returns the list of languages as an array" do
+ expect(Language.all).to be_an Array
+ end
+
+ it "returns 81 languages" do # That’s all of them except for [sr-cyrl]
+ expect(Language.all.count).to eq 81
+ end
+ end
+
+ describe '.all_by_iso639' do
+ it "returns a hash of arrays" do
+ expect(Language.all_by_iso639).to be_a Hash
+ expect(Language.all_by_iso639.values.map(&:class).uniq).to eq [Array]
+ end
+
+ it "groups languages by ISO 639 code elements" do
+ expect(Language.all_by_iso639['en'].count).to eq 2
+ end
+ end
+
+ describe '.find_by_bcp47' do
+ it "finds the language for that BCP47 tag" do
+ language = Language.find_by_bcp47 'bn'
+ expect(language).to be_a Language
+ end
+
+ it "calls .languages first" do
+ expect(Language).to receive(:languages).and_return({ 'cy' => Language.new('cy') })
+ Language.find_by_bcp47('cy')
+ end
+ end
+
+ describe '#bcp47' do # TODO Add #8bitenc
+ it "returns the BCP47 tag of the language" do
+ language = Language.new('oc')
+ expect(language.bcp47).to eq 'oc'
+ end
+ end
+
+ describe '#iso639' do
+ it "returns the ISO 639 primary tag" do
+ expect(Language.new('en-gb').iso639).to eq 'en'
+ end
+
+ it "works on 3-character tags" do
+ expect(Language.new('grc-x-ibycus').iso639).to eq 'grc'
+ end
+ end
+
+# describe '#name' do
+# let(:new_orthography_german) { Language.new('de-1996') }
+#
+# it "returns the name" do
+# expect(new_orthography_german.name).to eq 'German, reformed spelling'
+# end
+#
+# it "calls #extract_metadata first if necessary" do
+# expect(new_orthography_german).to receive(:extract_metadata).and_return({ 'name' => 'German, reformed spelling' })
+# new_orthography_german.name
+# end
+#
+# it "doesn’t call #extract_metadata if @name is already set" do
+# new_orthography_german.instance_variable_set :@name, 'Deutch in neuer Rechtschreibung'
+# expect(new_orthography_german).not_to receive :extract_metadata
+# new_orthography_german.name
+# end
+# end
+
+ # describe '#displayname' do
+ # it "returns @name most of the name" do
+ # romansh = Language.new('rm')
+ # expect(romansh.displayname).to eq 'Romansh'
+ # end
+
+ # it "strips to the language subtag part" do
+ # polytonic_greek = Language.new('el-polyton')
+ # expect(polytonic_greek.displayname).to eq 'Greek'
+ # end
+
+ # it "returns Norwegian for [nb] and [no]" do
+ # bokmål = Language.new('nb')
+ # expect(bokmål.displayname).to eq 'Norwegian'
+ # end
+
+ # it "returns Serbian for [sh]" do
+ # serbocroatian_cyrillic = Language.new('sh-cyrl')
+ # expect(serbocroatian_cyrillic.displayname).to eq 'Serbian'
+ # end
+
+ # it "calls #extract_metadata if needed" do
+ # friulan = Language.new('fur')
+ # expect(friulan).to receive :extract_metadata
+ # friulan.displayname
+ # end
+ # end
+
+ describe '#babelname' do
+ it "returns the Babel name" do
+ expect(Language.new('de-1996').babelname).to eq 'ngerman'
+ end
+
+ it "calls #extract_metadata first" do
+ german_CH = Language.new('de-ch-1901')
+ expect(german_CH).to receive :extract_metadata
+ german_CH.babelname
+ end
+ end
+
+ # describe '#packagename' do
+ # it "returns the package name for TeX Live" do
+ # expect(Language.new('gu').packagename).to eq 'indic'
+ # end
+
+ # it "uses some questionable choices" do
+ # expect(Language.new('grc').packagename).to eq 'ancient greek'
+ # end
+ # end
+
+ describe '#licences' do
+ let(:church_slavonic) { Language.new('cu') }
+
+ it "returns the licences as an array" do
+ expect(church_slavonic.licences).to eq ['MIT']
+ end
+
+ it "call #extract_metadata first if necessary" do
+ expect(church_slavonic).to receive(:extract_metadata)
+ church_slavonic.licences
+ end
+
+ it "doesn’t call #extract_metadata if @licences is already set" do
+ church_slavonic.instance_variable_set :@licences, ['MIT licence']
+ expect(church_slavonic).not_to receive :extract_metadata
+ church_slavonic.licences
+ end
+
+ it "raises an exception if @licences is nil or empty" do
+ nolicence = Language.new('qnl')
+ allow(File).to receive(:read).and_return("code: qnl\nauthors:\n - me")
+ expect { nolicence.licences }.to raise_exception NoLicence
+ end
+ end
+
+ describe '#lefthyphenmin' do
+ let(:swiss_spelling_german) { Language.new('de-ch-1901') }
+
+ it "returns the left hyphenmin value for typesetting" do
+ expect(swiss_spelling_german.lefthyphenmin).to eq 2
+ end
+
+ it "calls #extract_metadata first if necessary" do
+ expect(swiss_spelling_german).to receive :extract_metadata
+ swiss_spelling_german.lefthyphenmin
+ end
+
+ it "doesn’t call #extract_metadata if @lefthyphenmin is already set" do
+ swiss_spelling_german.instance_variable_set :@lefthyphenmin, 1
+ expect(swiss_spelling_german).not_to receive :extract_metadata
+ swiss_spelling_german.lefthyphenmin
+ end
+
+ it "uses the generation value if typesetting is missing" do
+ ethiopic = Language.new('mul-ethi')
+ expect(ethiopic.lefthyphenmin).to eq 1
+ end
+ end
+
+ describe '#righthyphenmin' do
+ let(:french) { Language.new('fr') }
+
+ it "returns the right hyphenmin value for typesetting" do
+ expect(french.righthyphenmin).to eq 2
+ end
+
+ it "call #extract_metadata first if necessary" do
+ expect(french).to receive :extract_metadata
+ french.righthyphenmin
+ end
+
+ it "doesn’t call #extract_metadata if @righthyphenmin is already set" do
+ french.instance_variable_set :@righthyphenmin, 2
+ expect(french).not_to receive :extract_metadata
+ french.righthyphenmin
+ end
+
+ it "uses the generation value if typesetting is missing" do
+ ethiopic = Language.new('mul-ethi')
+ expect(ethiopic.righthyphenmin).to eq 1
+ end
+ end
+
+ describe '#authors' do
+ let(:traditional_orthography_german) { Language.new('de-1901') }
+
+ it "returns the authors as an array" do
+ expect(traditional_orthography_german.authors).to eq ['Deutschsprachige Trennmustermannschaft']
+ end
+
+ it "calls #extract_metadata first if necessary" do
+ expect(traditional_orthography_german).to receive(:extract_metadata).and_return({ authors: ['Werner Lemberg', 'others'] })
+ traditional_orthography_german.authors
+ end
+
+ it "doesn’t call #extract_metadata if @authors is already set" do
+ traditional_orthography_german.instance_variable_set :@authors, ['German hyphenation patterns team']
+ expect(traditional_orthography_german).not_to receive :extract_metadata
+ traditional_orthography_german.authors
+ end
+ end
+
+ describe '#synonyms' do
+ it "returns the synonyms as an array" do
+ expect(Language.new('sl').synonyms).to eq ['slovene']
+ end
+
+ it "returns an empty array instead of nil if there are no synonyms" do
+ expect(Language.new('sk').synonyms).to eq []
+ end
+ end
+
+ describe '#encoding' do
+ it "returns the encoding" do
+ expect(Language.new('sh-cyrl').encoding).to eq 't2a'
+ end
+
+ it "returns “ascii” if applicable" do
+ expect(Language.new('rm').encoding).to eq 'ascii'
+ end
+
+ it "returns nil if patterns are Unicode-only" do
+ expect(Language.new('sa').encoding).to be_nil
+ end
+ end
+
+ describe '#github_link' do
+ it "returns the GitHub link" do
+ upper_sorbian = Language.new('hsb')
+ expect(upper_sorbian.github_link).to eq 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-hsb.tex'
+ end
+ end
+
+ describe '#<=>' do
+# it "compares Language’s using @name’s" do
+# expect(Language.new('de-1996') <=> Language.new('fr')).to eq 1
+# end
+
+ it "uses BCP47 codes if names are not available" do
+ expect(Language.new('zh-latn-pinyin') <=> Language.new('cu')).to eq 1
+ end
+ end
+
+ describe '#readtexfile' do
+ let(:basque) { Language.new('eu') }
+
+ it "reads the TeX file" do
+ expect(File).to receive(:read)
+ basque.readtexfile
+ end
+
+ it "stores the contents into the @@texfile class variable" do
+ basque.readtexfile
+ expect(Language.class_variable_get(:@@texfile)['eu']).to match /1ba.*1ko.*1t2xe.*su2b2r/m
+ end
+
+ it "recovers gracefully from nonexistent files" do
+ expect { Language.new('kl').readtexfile }.not_to raise_exception
+ end
+ end
+
+ describe '#patterns' do
+ it "returns the patterns" do
+ danish = Language.new('da')
+ expect(['.ae3', '.an3k', '.an1s'].all? { |p| danish.patterns.include? p }).to be_truthy
+ end
+
+ it "calls .languages first" do
+ language = Language.new('eu')
+ expect(Language).to receive(:languages).and_return({ 'eu' => Language.new('eu') })
+ pending "Needs pondering"
+ language.patterns
+ end
+
+ it "loads the patterns" do
+ language = Language.new('fi')
+ # byebug
+ expect(language.patterns[151..154]).to eq ['uu1a2', 'uu1e2', 'uu1o2', 'uu1i2']
+ end
+
+ it "doesn’t crash on inexistent patterns" do
+ expect { Language.new('zu').patterns }.not_to raise_exception
+ end
+
+ it "caches the list of patterns" do
+ language = Language.new('ru')
+ language.patterns
+ expect(language.instance_variable_get(:@patterns)[0..2]).to eq ['.аб1р', '.аг1ро', '.ади2']
+ end
+
+ it "uses the [no] patterns for [nb]" do
+ expect(Language.new('nb').patterns).to eq Language.new('no').patterns
+ end
+
+ it "expands the Esperanto patterns" do
+ esperanto = Language.new('eo')
+ expect(['.di3s2a.', '.di3s2aj.', '.di3s2ajn.', '.di3s2an.', '.di3s2e.'].any? { |p| esperanto.patterns.include? p }).to be_truthy
+ end
+ end
+
+ describe '#exceptions' do
+ it "returns the hyphenation exceptions" do
+ language = Language.new('ga')
+ # byebug
+ expect(language.exceptions[0..2]).to eq ['bhrachtaí', 'mbrachtaí', 'cháintí']
+ end
+
+ it "calls .languages first" do
+ language = Language.new('hu')
+ pending "Needs examining"
+ expect(Language).to receive(:all).and_return({ 'hu' => Language.new('hu') })
+ language.patterns
+ end
+
+ it "loads the exceptions" do
+ language = Language.new('is')
+ expect(File).to receive(:read).and_return("alc-un alc-u-nis-si-me alc-un-men-te")
+ language.patterns
+ end
+
+ it "doesn’t crash on inexistent patterns" do
+ expect { Language.new('iu').exceptions}.not_to raise_exception
+ end
+
+ it "caches the exceptions" do
+ language = Language.new('sk')
+ language.exceptions
+ expect(language.instance_variable_get(:@exceptions)[0..2]).to match ['dosť', 'me-tó-da', 'me-tó-dy']
+ end
+
+ it "hashes the exceptions" do
+ language = Language.new('en-gb')
+ language.exceptions
+ hyphenation = language.instance_variable_get :@hyphenation
+ expect(hyphenation.count).to eq 8
+ expect(hyphenation['however']).to eq 'how-ever'
+ end
+ end
+
+ describe '#hyphenate' do
+ it "hyphenates with the appropriate patterns" do
+ czech = Language.new('cs')
+ expect(czech.hyphenate('ubrousek')).to eq 'ubrou-sek'
+ end
+
+ it "takes hyphenmins in account if available" do
+ language = Language.new('de-1996')
+ expect(language.hyphenate('Zwangsvollstreckungsmaßnahme')).to eq 'zwangs-voll-stre-ckungs-maß-nah-me'
+ end
+
+ it "takes exceptions in account if available" do
+ american_english = Language.new('en-us')
+ expect(american_english.hyphenate('project')).to eq 'project'
+ end
+
+ it "initialises the hydra if needed" do
+ language = Language.new('de-1901')
+ language.hyphenate('Zwangsvollstreckungsmaßnahme')
+ expect(language.instance_variable_get(:@hydra)).to be_a Hydra
+ end
+
+ it "calls #exceptions" do
+ esperanto = Language.new('eo')
+ expect(esperanto).to receive(:exceptions)
+ esperanto.instance_variable_set :@hyphenation, { 'ŝtatregosciencon' => 'ŝta-tre-go-scien-con' }
+ esperanto.hyphenate('ŝtatregosciencon')
+ end
+ end
+
+ describe '#private_use?' do
+ it "returns true for languages with private use BCP 47 tags" do
+ expect(Language.new('qls').private_use?).to be_truthy
+ end
+
+ it "returns true otherwise" do
+ expect(Language.new('qyz').private_use?).to be_falsey
+ end
+
+ it "doesn’t crash on two-letter tags" do
+ expect { Language.new('qa').private_use? }.not_to raise_exception
+ end
+
+ it "returns false on two-letter tags" do
+ expect(Language.new('qt').private_use?).to be_falsey
+ end
+
+ it "works correctly on four-letter tags" do
+ expect(Language.new('qaaa').private_use?).to be_falsey
+ end
+
+ it "works correctly in the presence of subtags" do
+ expect(Language.new('qtz-GB').private_use?).to be_truthy
+ end
+ end
+
+ describe '#extract_metadata' do
+ it "returns a hash with the metadata" do
+ language = Language.new('bg')
+ expect(language.extract_metadata).to be_a Hash
+ end
+
+ it "raises an exception if the metadata is just a string" do
+ language = Language.new('qls')
+ allow(File).to receive(:read).and_return("just a string")
+ expect { language.extract_metadata }.to raise_exception InvalidMetadata
+ end
+
+ it "raises an exception if the licence is missing" do
+ language = Language.new('qlv')
+ allow(File).to receive(:read).and_return("name: language virtual\ncode: qlv")
+ expect { language.extract_metadata }.to raise_exception InvalidMetadata
+ end
+
+ it "raises an exception if @authors is nil or empty" do
+ not_church_slavonic = Language.new('qcu')
+ allow(File).to receive(:read).and_return "code: qcu\nlicence:\n name:\n MIT"
+ expect { not_church_slavonic.authors }.to raise_exception NoAuthor
+ end
+
+ it "doesn’t crash on invalid licence entries" do
+ syntax_error = Language.new('qse')
+ allow(File).to receive(:read).and_return "foo:\nbar"
+ expect { syntax_error.extract_metadata }.not_to raise_exception Psych::SyntaxError
+ end
+
+ it "sets the language name" do
+ language = Language.new('th')
+ language.extract_metadata
+ expect(language.instance_variable_get :@name).to eq 'Thai'
+ end
+
+ it "sets the licence list" do
+ language = Language.new('la')
+ language.extract_metadata
+ expect(language.instance_variable_get :@licences).to eq ['MIT', 'LPPL']
+ end
+
+ it "sets lefthyphenmin" do
+ pali = Language.new('pi')
+ pali.extract_metadata
+ expect(pali.instance_variable_get :@lefthyphenmin).to eq 1
+ end
+
+ it "sets righthyphenmin" do
+ german = Language.new('de-1996')
+ german.extract_metadata
+ expect(german.instance_variable_get :@righthyphenmin).to eq 2
+ end
+
+ it "sets the list of authors" do
+ liturgical_latin = Language.new('la-x-liturgic')
+ liturgical_latin.extract_metadata
+ expect(liturgical_latin.instance_variable_get :@authors).to eq ['Claudio Beccari', 'Monastery of Solesmes', 'Élie Roux']
+ end
+
+ context "With Swedish as an example" do
+ let(:swedish) { Language.new('sv') }
+
+ it "sets the message" do
+ swedish.extract_metadata
+ expect(swedish.instance_variable_get :@message).to eq "Swedish hyphenation patterns"
+ end
+
+ it "sets the old pattern file name" do
+ swedish.extract_metadata
+ expect(swedish.instance_variable_get :@legacy_patterns).to eq "svhyph.tex"
+ end
+
+ it "leaves @use_old_loader to nil in most cases" do
+ swedish.extract_metadata
+ expect(swedish.instance_variable_get :@use_old_loader).to be_nil
+ end
+
+ it "sets the @use_old_loader boolean to true if applicable" do
+ norwegian = Language.new('no')
+ norwegian.extract_metadata
+ expect(norwegian.instance_variable_get :@use_old_loader).to be_truthy
+ end
+
+ it "leaves @use_old_patterns_comment to nil in most cases" do
+ swedish.extract_metadata
+ expect(swedish.instance_variable_get :@use_old_patterns_comment).to be_nil
+ end
+
+ it "sets the @use_old_patterns_comment to string if applicable" do
+ german_AR = Language.new('de-1901')
+ german_AR.extract_metadata
+ expect(german_AR.instance_variable_get :@use_old_patterns_comment).to eq "Kept for the sake of backward compatibility, but newer and better patterns by WL are available."
+ end
+
+ it "sets the long description" do
+ swedish.extract_metadata
+ expect(swedish.instance_variable_get :@description).to eq "Hyphenation patterns for Swedish in T1/EC and UTF-8 encodings."
+ end
+
+ it "sets the Babel name" do
+ swedish.extract_metadata
+ expect(swedish.instance_variable_get :@babelname).to eq "swedish"
+ end
+
+ it "sets the Babel name even if it is slightly silly ;-)" do
+ german_NR = Language.new('de-1996')
+ german_NR.extract_metadata
+ expect(german_NR.instance_variable_get :@babelname).to eq "ngerman"
+ end
+
+ it "sets @package to nil in most cases" do
+ expect(swedish.instance_variable_get :@package).to be_nil
+ end
+
+ it "sets @package for a few languages" do
+ gujarati = Language.new('gu')
+ gujarati.extract_metadata
+ expect(gujarati.instance_variable_get :@package).to eq 'indic'
+ end
+ end
+ end
+
+ describe '#has_apostrophes?' do
+ it "returns if patterns have apostrophes" do
+ expect(Language.new('be').has_apostrophes?).to be_truthy
+ end
+
+ it "returns false otherwise" do
+ expect(Language.new('bn').has_apostrophes?).to be_falsey
+ end
+ end
+
+ describe '#has_hyphens?' do
+ it "returns true if patterns have dashes" do
+ expect(Language.new('uk').has_hyphens?).to be_truthy
+ end
+
+ it "returns false otherwise" do
+ expect(Language.new('tr').has_hyphens?).to be_falsey
+ end
+ end
+
+ describe '#isgreek?' do
+ it "returns true if language is Greek (sort of)" do
+ expect(Language.new('grc').isgreek?).to be_truthy
+ end
+
+ it "returns false if not" do
+ expect(Language.new('la').isgreek?).to be_falsey
+ end
+
+ it "exceptionally returns false for Ibycus" do
+ expect(Language.new('grc-x-ibycus').isgreek?).to be_falsey
+ end
+ end
+
+ describe '#message' do
+ it "returns the message to be displayed on the terminal" do
+ expect(Language.new('af').message).to eq 'Afrikaans hyphenation patterns'
+ end
+ end
+
+ describe '#known_bugs' do
+ it "returns the known bugs" do
+ american_english = Language.new('en-us')
+ expect(american_english.known_bugs.keys).to eq ['de-mo-c-rat']
+ end
+ end
+
+ describe TeXLive do
+ describe '#loadhyph' do
+ it "returns the name of the pattern loader file" do
+ expect(Language.new('cy').loadhyph).to eq 'loadhyph-cy.tex'
+ end
+
+ it "replaces the main sh subtag by sr" do
+ expect(Language.new('sh-latn').loadhyph).to eq 'loadhyph-sr-latn.tex'
+ end
+
+ it "returns the old loader name if applicable" do
+ expect(Language.new('grc-x-ibycus').loadhyph).to eq 'ibyhyph.tex'
+ end
+ end
+
+ describe '#list_loader' do
+ it "returns the tlpsrc line with the loader" do
+ expect(Language.new('hr').list_loader).to eq 'file=loadhyph-hr.tex'
+ end
+
+ it "includes an empty line for Arabic and Farsi" do
+ expect(Language.new('ar').list_loader).to eq "file=zerohyph.tex \\\n\tfile_patterns="
+ end
+
+ it "includes a Lua special line for Ibycus" do
+ expect(Language.new('grc-x-ibycus').list_loader).to eq "file=ibyhyph.tex \\\n\tluaspecial=\"disabled:8-bit only\""
+ end
+ end
+
+ describe '#list_run_files' do
+ it "returns the list of TeX file" do
+ # puts Language.new('ka').list_run_files
+ expect(Language.new('ka').list_run_files).to eq ['tex/generic/hyph-utf8/loadhyph/loadhyph-ka.tex',
+ 'tex/generic/hyph-utf8/patterns/tex/hyph-ka.tex',
+ 'tex/generic/hyph-utf8/patterns/ptex/hyph-ka.t8m.tex',
+ 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.chr.txt',
+ 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.pat.txt',
+ 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.hyp.txt',
+ 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.lic.txt']
+ end
+ end
+
+ describe '#patterns_line' do
+ it "returns the patterns line for TLPSRC" do
+ expect(Language.new('tk').patterns_line).to eq "file_patterns=hyph-tk.pat.txt"
+ end
+
+ it "returns two files for Serbian" do
+ expect(Language.new('sh-cyrl').patterns_line).to eq "file_patterns=hyph-sh-latn.pat.txt,hyph-sh-cyrl.pat.txt"
+ end
+ end
+
+ describe '#exceptions_line' do
+ it "returns the exceptions line for TLPSRC" do
+ expect(Language.new('nn').exceptions_line).to eq "file_exceptions=hyph-nn.hyp.txt"
+ end
+
+ it "returns two files for Serbian" do
+ expect(Language.new('sh-latn').exceptions_line).to eq "file_exceptions=hyph-sh-latn.hyp.txt,hyph-sh-cyrl.hyp.txt"
+ end
+ end
+
+ describe '#extract_apostrophes' do
+ it "returns the list of patterns with apostrophes" do
+ expect(Language.new('af').extract_apostrophes[:with_apostrophe]).to eq ['.af6ro’', '.a7fro’s', '.l’7etji', '.m’7etji', '.r’7etji', 's’9ie.', 'x’9ie.']
+ end
+
+ it "returns nil otherwise" do
+ expect(Language.new('nl').extract_apostrophes[:with_apostrophe]).to be_nil
+ end
+ end
+
+ describe '#extract_characters' do
+ it "extracts the list of characters with in lowercase and uppercase" do
+ expect(Language.new('id').extract_characters).to eq (('a'..'z').to_a - ['x']).map { |c| c + c.upcase }
+ end
+ end
+
+ describe '#comments_and_licence' do
+ it "extracts the header" do
+ expect(Language.new('kmr').comments_and_licence).to match /^% title: Hyphenation patterns for Kurmanji \(Northern Kurdish\).*The patterns are generated by patgen from a word list of approx\. 2500\n% hyphenated words provided by Medeni Shemdê$/m
+ end
+ end
+
+ describe '#list_synonyms' do
+ it "returns a list of synonyms" do
+ expect(Language.new('es').list_synonyms).to eq ' synonyms=espanol'
+ end
+ end
+
+ describe '#list_hyphenmins' do
+ it "returns the hyphenmins" do
+ expect(Language.new('gl').list_hyphenmins).to eq "lefthyphenmin=2 \\\n\trighthyphenmin=2"
+ end
+ end
+
+ describe '#message' do
+ it "returns the message to be displayed to TeX users" do
+ expect(Language.new('de-1996').message).to eq 'German hyphenation patterns (reformed orthography)'
+ end
+ end
+
+ describe '#legacy_patterns' do
+ it "returns the file name of the legacy patterns" do
+ expect(Language.new('de-1996').legacy_patterns).to eq 'dehyphn.tex'
+ end
+ end
+
+ describe '#use_old_loader' do
+ it "says whether to use old loader or not" do
+ expect(Language.new('de-1996').use_old_loader).to be_nil
+ end
+
+ it "returns true if applicable" do
+ # pending "Refactor in progress"
+ expect(Language.new('no').use_old_loader).to be_truthy
+ end
+ end
+
+ describe '#use_old_patterns_comment' do
+ it "returns true if language uses old patterns" do
+ expect(Language.new('de-1996').use_old_patterns_comment).to be_truthy
+ end
+
+ it "returns nil otherwise" do
+ expect(Language.new('no').use_old_patterns_comment).to be_nil
+ end
+
+ it "is a string" do
+ expect(Language.new('cop').use_old_patterns_comment).to be_a String
+ end
+ end
+
+ describe '#description' do
+ it "returns the long description" do
+ text = <<-EOD
+ Hyphenation patterns for Dutch in T1/EC and UTF-8 encodings.
+ These patterns don't handle cases like 'menuutje' > 'menu-tje',
+ and don't hyphenate words that have different hyphenations according
+ to their meaning.
+ EOD
+ description = text.split("\n").map(&:strip).join("\n")
+ expect(Language.new('nl').description).to eq description
+ end
+
+ it "returns nil for new spelling German, for some reason" do
+ expect(Language.new('de-1996').description).to be_nil
+ end
+ end
+
+ describe '#package' do
+ it "returns the package name if applicable" do
+ expect(Language.new('bn').package).to eq 'indic'
+ end
+
+ it "returns nil in most cases" do
+ expect(Language.new('bg').package).to be_nil
+ end
+
+ it "calls #extract_metadata first" do
+ hindi = Language.new('hi')
+ expect(hindi).to receive :extract_metadata
+ hindi.package
+ end
+ end
+ end
+end
+
+describe Package do
+ let(:latin) { Package.find('latin') }
+ let(:german) { Package.find('german') }
+ let(:hungarian) { Package.find('hungarian') }
+ let(:norwegian) { Package.find('norwegian') }
+
+ describe "Instance variables" do
+ it "has a @name" do
+ expect(latin.name).to eq 'latin'
+ end
+ end
+
+ describe '.new' do
+ it "initialises @languages to an empty array" do
+ package = Package.new('mongolian')
+ expect(package.instance_variable_get :@languages).to eq []
+ end
+ end
+
+ describe "#description_s" do
+ it "returns the short description" do
+ expect(latin.description_s).to eq 'Latin hyphenation patterns'
+ end
+ end
+
+ describe "#description" do
+ it "returns the long description" do
+ expect(latin.description).to match /^Hyphenation patterns for.*modern spelling.*medieval spelling/m
+ expect(latin.description).to match /Classical Latin/
+ expect(latin.description).to match /Liturgical Latin/
+ end
+
+ it "returns the long package description" do
+ text = <<-EOD
+ Hyphenation patterns for German in T1/EC and UTF-8 encodings,
+ for traditional and reformed spelling, including Swiss German.
+ The package includes the latest patterns from dehyph-exptl
+ (known to TeX under names 'german', 'ngerman' and 'swissgerman'),
+ however 8-bit engines still load old versions of patterns
+ for 'german' and 'ngerman' for backward-compatibility reasons.
+ Swiss German patterns are suitable for Swiss Standard German
+ (Hochdeutsch) not the Alemannic dialects spoken in Switzerland
+ (Schwyzerduetsch).
+ There are no known patterns for written Schwyzerduetsch.
+ EOD
+ description = text.split("\n").map(&:strip).join("\n")
+ expect(Package.find('german').description).to match description
+ end
+ end
+
+ describe '#add_language' do
+ it "adds a language to the package" do
+ package = Package.new('indic')
+ assamese = Language.new('as')
+ package.add_language(assamese)
+ expect(package.instance_variable_get(:@languages).first).to eq assamese
+ end
+ end
+
+ describe "#languages" do
+ it "returns the list of languages" do
+ expect(latin.languages.map(&:bcp47).sort).to eq ['la', 'la-x-classic', 'la-x-liturgic']
+ end
+ end
+
+ describe "#has_dependency?" do
+ it "returns the external dependencies" do
+ expect(german.has_dependency?).to eq "dehyph"
+ end
+
+ it "returns nil for most packages" do
+ expect(hungarian.has_dependency?).to be_nil
+ end
+ end
+
+ describe "#list_dependencies" do
+ it "lists the dependencies" do
+ # FIXME Should return ['depend hyphen-base', 'depend hyph-utf8', 'depend dehyph'] or nothing
+ expect(german.has_dependency?).to eq 'dehyph'
+ end
+ end
+
+ describe "#list_support_files" do # FIXME? list_non_run_files
+ it "lists doc and source files" do
+ expect(hungarian.list_support_files('doc')).to eq ['doc/generic/hyph-utf8/languages/hu', 'doc/generic/huhyphen']
+ # FIXME Should return ['texmf-dist/doc/generic/huhyphen', 'texmf-dist/doc/generic/hyph-utf8/languages/hu'] or nothing
+ end
+ end
+
+ describe "#list_run_files" do
+ it "lists the run-time files" do
+ # pending "it crashes ;-)"
+ norwegian_run = norwegian.list_run_files
+ expect(norwegian_run.count).to eq 15
+ expect(norwegian_run.select { |f| f =~ /tex\/hyph-[^\.]*\.tex$/ }).to eq ['tex/generic/hyph-utf8/patterns/tex/hyph-no.tex',
+ 'tex/generic/hyph-utf8/patterns/tex/hyph-nb.tex',
+ 'tex/generic/hyph-utf8/patterns/tex/hyph-nn.tex']
+ end
+ end
+
+ describe '#<=>' do
+ it "compares two Package’s" do
+ # puts hungarian.class, german.class
+ expect(hungarian.<=> german).to eq 1
+ end
+ end
+
+ describe '#find' do
+ it "returns the package with that name" do
+ expect(Package.find('norwegian')).to eq norwegian
+ end
+ end
+end
diff --git a/language/hyph-utf8/source/luatex/hyph-utf8/Makefile b/language/hyph-utf8/source/luatex/hyph-utf8/Makefile
new file mode 100644
index 0000000000..2ae8bd67a1
--- /dev/null
+++ b/language/hyph-utf8/source/luatex/hyph-utf8/Makefile
@@ -0,0 +1,35 @@
+# Makefile for luatex-hyphen in tex-hyphen/hyph-utf8
+
+NAME = luatex-hyphen
+DTX = $(NAME).dtx
+
+# Generated files
+COMPILED = $(NAME).pdf
+UNPACKED = $(NAME).lua hyphen.cfg
+
+# Installation locations
+RUNDIR = ../../../tex/luatex/hyph-utf8
+DOCDIR = ../../../doc/luatex/hyph-utf8
+
+# Final files
+RUNFILES = $(patsubst %, $(RUNDIR)/%, $(UNPACKED))
+DOCFILE = $(DOCDIR)/$(COMPILED)
+
+all: $(RUNFILES) $(DOCFILE)
+
+DO_TEX = tex --interaction=batchmode $< >/dev/null
+DO_PDFLATEX = pdflatex --interaction=batchmode $< >/dev/null
+
+$(DOCFILE): $(DTX)
+ $(DO_PDFLATEX)
+ $(DO_PDFLATEX)
+ mv $(COMPILED) $(DOCDIR)
+
+$(RUNFILES): $(DTX)
+ $(DO_TEX)
+ for file in $(UNPACKED); do mv $$file $(RUNDIR); done
+
+clean:
+ @$(RM) -- *.log *.aux *.toc *.idx *.ind *.ilg
+
+.PHONY: clean
diff --git a/language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx b/language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx
new file mode 100644
index 0000000000..ca1761ef2b
--- /dev/null
+++ b/language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx
@@ -0,0 +1,337 @@
+% \iffalse meta-comment
+%
+% This is a module for hyphenation patterns loading for LuaTeX. It is under
+% the CC0 license (http://creativecommons.org/publicdomain/zero/1.0/).
+%
+%<*ignore>
+\begingroup
+ \def\x{LaTeX2e}%
+\expandafter\endgroup
+\ifcase 0\ifx\install y1\fi\expandafter
+ \ifx\csname processbatchFile\endcsname\relax\else1\fi
+ \ifx\fmtname\x\else 1\fi\relax
+\else\csname fi\endcsname
+%</ignore>
+%<*install>
+\input docstrip.tex
+
+\keepsilent
+\askforoverwritefalse
+
+\def\MetaPrefix{-- }
+
+\def\luapostamble{%
+ \MetaPrefix^^J%
+ \MetaPrefix\space End of File `\outFileName'.%
+}
+
+\def\currentpostamble{\luapostamble}%
+
+\preamble
+
+This is a generated file (source: luatex-hyphen.dtx).
+
+Copyright (C) 2012-2013 by The LuaLaTeX development team.
+
+This work is under the CC0 license.
+
+\endpreamble
+
+\generate{%
+ \file{luatex-hyphen.lua}{\from{luatex-hyphen.dtx}{lua}}%
+}
+
+\endbatchfile
+%</install>
+%<*ignore>
+\fi
+%</ignore>
+%<*driver>
+\NeedsTeXFormat{LaTeX2e}
+\ProvidesFile{luatex-hyphen.drv}
+ [2013/05/16 v1.6 Hyphenation pattern loading for LuaTeX]
+\documentclass{ltxdoc}
+\usepackage[
+ pdftitle={The luatex-hyphen package},
+ pdfsubject={Hyphenation pattern loading for LuaTeX},
+ pdfauthor={Manuel Pégourié-Gonnard, Khaled Hosny & Elie Roux},
+ pdfkeywords={luatex, lualatex, unicode, hyphenation}
+]{hyperref}
+\usepackage{fontspec}
+\setmainfont[
+ Ligatures = TeX,
+ BoldFont = {Linux Libertine O Bold},
+ ItalicFont = {Linux Libertine O Italic},
+ SlantedFont = {Linux Libertine O Italic},
+]{Linux Libertine O}
+\setmonofont[Ligatures=TeX,Scale=MatchLowercase]{Liberation Mono}
+\setsansfont[Ligatures=TeX,Scale=MatchLowercase]{Iwona Medium}
+\EnableCrossrefs
+\CodelineIndex
+\begin{document}
+ \DocInput{luatex-hyphen.dtx}%
+\end{document}
+%</driver>
+% \fi
+%
+% \CheckSum{0}
+%
+% \CharacterTable
+% {Upper-case \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z
+% Lower-case \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z
+% Digits \0\1\2\3\4\5\6\7\8\9
+% Exclamation \! Double quote \" Hash (number) \#
+% Dollar \$ Percent \% Ampersand \&
+% Acute accent \' Left paren \( Right paren \)
+% Asterisk \* Plus \+ Comma \,
+% Minus \- Point \. Solidus \/
+% Colon \: Semicolon \; Less than \<
+% Equals \= Greater than \> Question mark \?
+% Commercial at \@ Left bracket \[ Backslash \\
+% Right bracket \] Circumflex \^ Underscore \_
+% Grave accent \` Left brace \{ Vertical bar \|
+% Right brace \} Tilde \~}
+%
+% \GetFileInfo{luatex-hyphen.drv}
+%
+% \title{The \texttt{hyphen.cfg} file for Lua\TeX }
+% \date{2013/05/16 v1.6}
+% \author{Khaled Hosny, Élie Roux, and Manuel Pégourié-Gonnard\\
+% \texttt{khaledhosny@eglug.org} \\
+% \texttt{elie.roux@telecom-bretagne.eu} \\
+% \texttt{mpg@elzevir.fr}}
+%
+% \maketitle
+%
+% \begin{abstract}
+% This package is mainly a Lua module, to be used by \textsf{Babel} and
+% \textsf{polyglossia} to adapt their hyphenation patterns loading mechanism to
+% Lua\TeX's dynamic pattern loading capabilities. It makes use of a
+% \texttt{language.dat.lua} file (whose format is described below) that should
+% be present in the distribution, in addition to the regular
+% \texttt{language.dat} file.
+%
+% \textsf{Babel} needed to be updated -- this used to be the goal of this
+% package -- before version \textsf{3.9} (\TeX Live 2013) and
+% \textsf{polyglossia} handles Lua\TeX\ since version \textsf{1.3} (\TeX Live
+% 2013).
+%
+% There is a version of \texttt{etex.src} modified for the same reasons using
+% similar code, which also makes use of the \texttt{luatex-hyphen.lua} and
+% \texttt{language.dat.lua} files described here.
+% \end{abstract}
+%
+% \section{Documentation}
+%
+% Hyphenation patterns should be loaded at runtime with Lua\TeX: if they appear
+% in the format, they will be rehashed when the format is loaded anyway, which
+% makes the format quite long to load (many seconds even on modern machines)
+% and provides for bad user experience. Hence, it is desirable to load as few
+% patterns as possible in the format, and load on-demand the needed patterns
+% at runtime.
+%
+% This package provides a modified version of hyphen.cfg adapted to Lua\TeX,
+% as well as a supporting Lua module. Since a lot of things, especially the
+% catcodes, are not as predictable at runtime than at format creation time, we
+% don't \verb+\input+ the usual pattern files, but rather load the patterns
+% using the Lua interface, using a special plain text version of the pattern
+% files if available.
+%
+% The existence and file name of such a version cannot be guessed, so we need
+% a specific database: the file \texttt{language.dat.lua}. This file should be
+% loadable by Lua and return a table whose keys are the canonical language
+% names as found in \texttt{language.dat}, and the values are Lua tables
+% consisting of:
+% \begin{enumerate}
+% \item A fixed part with one mandatory field:
+% \begin{quote}
+% \verb+synonyms = { <string> alternative name, ...}+
+% \end{quote}
+% This field's value must be the same as in \texttt{language.dat}.
+% \item A variable part consisting of either:
+% \begin{itemize}
+% \item For most languages:
+% \begin{quote}
+% \verb+patterns = <string> filenames for patterns+\\
+% \verb+hyphenation = <string> filenames for exceptions+\\
+% \end{quote}
+% Each string contains a coma-separated list of file names (whitespace
+% before or after the coma is not accepted).
+% The files given by \verb+patterns+ (resp. \verb+hypenation+) must be
+% plain text files encoded in UTF-8, with only patterns (resp.
+% exceptions) and not even comments: their content will be used
+% directly without being parsed by \TeX. If one of these keys is
+% missing or is the empty string, it is ignored and no patterns (resp.
+% exceptions) are loaded for this language.
+% \item Special cases are supported by a field \verb+special+. Currently,
+% the following kind of values are recognized:
+% \begin{description}
+% \item[\texttt{'disabled:<reason>'}] allows to disable specific
+% languages: when the user tries to load this language, an error
+% will be issued, with the \verb+<reason>+.
+% \item[\texttt{'language0'}] only \texttt{english} should use this
+% type of special, to indicate it is normally dumped in the format
+% as \verb+\language0+ (see below).
+% \end{description}
+% Special languages may have \texttt{*hyphenmin} information when it
+% makes sense (mostly \verb+\language0+).
+% \end{itemize}
+% \item Optional fields may be added. For example:
+% \begin{quote}
+% \verb+loader = <string> name of the TeX loader+\\
+% \verb+lefthyphenmin = <number> value for \lefthyphenmin+\\
+% \verb+righthyphenmin = <number> value for \righthyphenmin+
+% \end{quote}
+% Those fields are present in \texttt{language.dat.lua} as generated by
+% \texttt{tlmgr}, for example, but they \emph{are not} used by the present
+% code in any way.
+% \end{enumerate}
+% Languages that are mentioned in \texttt{language.dat} but not in
+% \texttt{language.dat.lua} will be loaded in the format. So, if the
+% \texttt{language.dat.lua} file is missing or incomplete, languages will just
+% go back to the ``old'' behaviour, resulting in longer startup time, which
+% seems less bad than complete breakage.
+%
+% For backward compatibility, Knuth's original patterns for US English are
+% always loaded in the format, as \verb+\language0+.\footnote{It is assumed
+% to be the first entry in \texttt{language.dat}.}
+%
+% \StopEventually{
+% }
+%
+% \section{Implementation}
+%
+% \begin{macrocode}
+%<*lua>
+% \end{macrocode}
+%
+% Start a Lua module, two functions for error and information reporting.
+%
+% \begin{macrocode}
+luatexhyphen = luatexhyphen or {}
+local luatexhyphen = luatexhyphen
+local function wlog(msg, ...)
+ texio.write_nl('log', 'luatex-hyphen: '..msg:format(...))
+end
+local function err(msg, ...)
+ error('luatex-hyphen: '..msg:format(...), 2)
+end
+% \end{macrocode}
+%
+% Load the \texttt{language.dat.lua} file with the Lua version of the
+% language database.
+%
+% \begin{macrocode}
+local dbname = "language.dat.lua"
+local language_dat
+local dbfile = kpse.find_file(dbname, 'lua')
+if not dbfile then
+ err("file not found: "..dbname)
+else
+ wlog('using data file: %s', dbfile)
+ language_dat = dofile(dbfile)
+end
+% \end{macrocode}
+%
+% Look up a language in the database, and return the associated
+% information, as well as the canonical name of the language.
+%
+% \begin{macrocode}
+local function lookupname(name)
+ if language_dat[name] then
+ return language_dat[name], name
+ else
+ for canon, data in pairs(language_dat) do
+ for _,syn in ipairs(data.synonyms) do
+ if syn == name then
+ return data, canon
+ end
+ end
+ end
+ end
+end
+luatexhyphen.lookupname = lookupname
+% \end{macrocode}
+%
+% Set hyphenation patterns and exceptions for a language given by its name
+% (in the database) and number (value of \verb+\language+). Doesn't return
+% anything, but will call \verb+error()+ if things go wrong.
+%
+% \begin{macrocode}
+local function loadlanguage(lname, id)
+ if id == 0 then
+ return
+ end
+ local msg = "loading%s patterns and exceptions for: %s (\\language%d)"
+% \end{macrocode}
+%
+% Lookup the language in the database.
+%
+% \begin{macrocode}
+ local ldata, cname = lookupname(lname)
+ if not ldata then
+ err("no entry in %s for this language: %s", dbname, lname)
+ end
+% \end{macrocode}
+%
+% Handle special languages.
+%
+% \begin{macrocode}
+ if ldata.special then
+ if ldata.special:find('^disabled:') then
+ err("language disabled by %s: %s (%s)", dbname, cname,
+ ldata.special:gsub('^disabled:', ''))
+ elseif ldata.special == 'language0' then
+ err("\\language0 should be dumped in the format")
+ else
+ err("bad entry in %s for language %s")
+ end
+ end
+% \end{macrocode}
+%
+% The generic case: load hyphenation patterns and exceptions from files
+% given by the language code.
+%
+% \begin{macrocode}
+ wlog(msg, '', cname, id)
+ for _, item in ipairs{'patterns', 'hyphenation'} do
+ local filelist = ldata[item]
+ if filelist ~= nil and filelist ~= '' then
+ for _, file in ipairs(filelist:explode(',')) do
+ local file = kpse.find_file(file) or err("file not found: %s", file)
+ local fh = io.open(file, 'r')
+ local data = fh:read('*a') or err("file not readable: %s", f)
+ fh:close()
+ lang[item](lang.new(id), data)
+ end
+ else
+ if item == 'hyphenation' then item = item..' exceptions' end
+ wlog("info: no %s for this language", item)
+ end
+ end
+end
+luatexhyphen.loadlanguage = loadlanguage
+% \end{macrocode}
+%
+% Add \textsf{Babel}'s ``dialects'' as synonyms.
+%
+% \begin{macrocode}
+local function adddialect(dialect, language)
+ if dialect ~= '0' then
+ dialect = dialect:gsub('l@', '')
+ language = language:gsub('l@', '')
+ data = lookupname(language)
+ if data then
+ data.synonyms[#data.synonyms+1] = dialect
+ end
+ end
+end
+luatexhyphen.adddialect = adddialect
+% \end{macrocode}
+%
+% \begin{macrocode}
+%</lua>
+% \end{macrocode}
+%
+% \Finale
+\endinput