diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /language/hyph-utf8/source |
Initial commit
Diffstat (limited to 'language/hyph-utf8/source')
50 files changed, 8155 insertions, 0 deletions
diff --git a/language/hyph-utf8/source/generic/hyph-utf8/README b/language/hyph-utf8/source/generic/hyph-utf8/README new file mode 100644 index 0000000000..97f8737876 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/README @@ -0,0 +1,57 @@ +generate-converters.rb +====================== +INPUT: +- source/generic/hyph-utf8/data/encodings/*.dat +OUTPUT: +- tex/generic/hyph-utf8/conversions/conv-utf8-*.dat + +Auto-generates conversions from UTF-8 to some particular encoding. + +Needs to be run only in case that a new encoding is added or an old encoding is fixed. + +generate-pattern-loaders.rb +=========================== +INPUT: +- language.rb with data +OUTPUT: +- tex/generic/hyph-utf8/loadhyph/loadhyph-*.tex + +Auto-generates pattern loaders for languages that should be used in TeX Live or other distros. + +Needs to be run when a new language is added, a bug needs to be fixed or a strategy is changed. + +generate-tl-files.rb +==================== +Has been used once for the very first TL script generation. +Out-of-date and not maintaned; will probably be removed. + +generate-webpage.rb +=================== +Used to generated webpage with language overview. + +languages.rb +============ +Database with language information used for almost every script. +Needs some clean-up and some updates. + + +languages/eu/generate_patterns_eu.rb +==================================== +Generator for patterns for Basque. +Author: Juan M. Aguirregabiria, adapted by Mojca Miklavec + +languages/gl +============ +Generator for patterns for Galician. +Author: Javier A. Múgica + +languages/tr/generate_patterns_tr.rb +==================================== +Generator for patterns for Turkish. +Author: See file, modified by Mojca Miklavec + + +contributed/make-exhyph.pl +========================== +Imported script for hack related to breaking compound words. + diff --git a/language/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl b/language/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl new file mode 100755 index 0000000000..3fe44239d2 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/contributed/make-exhyph.pl @@ -0,0 +1,50 @@ +#!/usr/bin/perl +# -*- coding: utf-8; -*- +# Copyright (C) 2008 Vladimir Volovich. +# You may freely use, modify and/or distribute this file. +# +# Generate additional patterns (for some language), to be used in case +# there is an alternative hyphen character available. +# +# It enables the hyphenation of words containing explicit hyphens +# when using fonts with \hyphenchar\font <> `\- (e.g. T1 or T2A encoding). +# +# sample usage: perl make-exhyph.pl ru > exhyph-ru.tex + +use strict; +use utf8; +binmode(STDOUT, ":utf8"); + +my %alphabet = ( + 'ru' => [ qw( а б в г д е ё ж з и й к л м н о п р с т у ф х ц ч ш щ ъ ы ь э ю я ) ], + 'uk' => [ qw( а б в г ґ д е є ж з и і ї й к л м н о п р с т у ф х ц ч ш щ ь ю я ' ) ] +); + +my $lang = $ARGV[0]; +die "unspecified lang" if ! defined $lang; +die "invalid lang" if ! exists $alphabet{$lang}; + +my @alphabet = @{$alphabet{$lang}}; + +print <<\EOF; +\begingroup +\lccode45=45 % Make hyphen "-" a word constituent +\patterns{ +8-7 +--8 +EOF + +for my $l (@alphabet) { + print ".${l}-8\n"; +} + +for my $l1 (@alphabet) { + for my $l2 (@alphabet) { + print "-${l1}8${l2}8\n8${l1}8${l2}-\n"; + } +} + +print <<\EOF; +} +\endgroup +EOF diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat new file mode 100644 index 0000000000..905d5b1d04 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/ec.dat @@ -0,0 +1,257 @@ +0x00 U+0060 grave +0x01 U+00B4 acute +0x02 U+02C6 circumflex +0x03 U+02DC tilde +0x04 U+00A8 dieresis +0x05 U+02DD hungarumlaut +0x06 U+02DA ring +0x07 U+02C7 caron +0x08 U+02D8 breve +0x09 U+00AF macron +0x0A U+02D9 dotaccent +0x0B U+00B8 cedilla +0x0C U+02DB ogonek +0x0D U+201A quotesinglbase +0x0E U+2039 guilsinglleft +0x0F U+203A guilsinglright +0x10 U+201C quotedblleft +0x11 U+201D quotedblright +0x12 U+201E quotedblbase +0x13 U+00AB guillemotleft +0x14 U+00BB guillemotright +0x15 U+2013 endash +0x16 U+2014 emdash +0x17 U+200B cwm +0x18 U+.... perthousandzero +0x19 U+0131 1 dotlessi +0x1A U+0237 1 dotlessj +0x1B U+FB00 ff +0x1C U+FB01 fi +0x1D U+FB02 fl +0x1E U+FB03 ffi +0x1F U+FB04 ffl +0x20 U+2423 visiblespace +0x21 U+0021 exclam +0x22 U+0022 quotedbl +0x23 U+0023 numbersign +0x24 U+0024 dollar +0x25 U+0025 percent +0x26 U+0026 ampersand +0x27 U+2019 quoteright +0x28 U+0028 parenleft +0x29 U+0029 parenright +0x2A U+002A asterisk +0x2B U+002B plus +0x2C U+002C comma +0x2D U+002D hyphen +0x2E U+002E period +0x2F U+002F slash +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x3A U+003A colon +0x3B U+003B semicolon +0x3C U+003C less +0x3D U+003D equal +0x3E U+003E greater +0x3F U+003F question +0x40 U+0040 at +0x41 U+0041 A +0x42 U+0042 B +0x43 U+0043 C +0x44 U+0044 D +0x45 U+0045 E +0x46 U+0046 F +0x47 U+0047 G +0x48 U+0048 H +0x49 U+0049 I +0x4A U+004A J +0x4B U+004B K +0x4C U+004C L +0x4D U+004D M +0x4E U+004E N +0x4F U+004F O +0x50 U+0050 P +0x51 U+0051 Q +0x52 U+0052 R +0x53 U+0053 S +0x54 U+0054 T +0x55 U+0055 U +0x56 U+0056 V +0x57 U+0057 W +0x58 U+0058 X +0x59 U+0059 Y +0x5A U+005A Z +0x5B U+005B bracketleft +0x5C U+005C backslash +0x5D U+005D bracketright +0x5E U+005E asciicircum +0x5F U+005F underscore +0x60 U+2018 quoteleft +0x61 U+0061 a +0x62 U+0062 b +0x63 U+0063 c +0x64 U+0064 d +0x65 U+0065 e +0x66 U+0066 f +0x67 U+0067 g +0x68 U+0068 h +0x69 U+0069 i +0x6A U+006A j +0x6B U+006B k +0x6C U+006C l +0x6D U+006D m +0x6E U+006E n +0x6F U+006F o +0x70 U+0070 p +0x71 U+0071 q +0x72 U+0072 r +0x73 U+0073 s +0x74 U+0074 t +0x75 U+0075 u +0x76 U+0076 v +0x77 U+0077 w +0x78 U+0078 x +0x79 U+0079 y +0x7A U+007A z +0x7B U+007B braceleft +0x7C U+007C bar +0x7D U+007D braceright +0x7E U+007E asciitilde +0x7F U+002D hyphenchar +0x80 U+0102 Abreve +0x81 U+0104 Aogonek +0x82 U+0106 Cacute +0x83 U+010C Ccaron +0x84 U+010E Dcaron +0x85 U+011A Ecaron +0x86 U+0118 Eogonek +0x87 U+011E Gbreve +0x88 U+0139 Lacute +0x89 U+013D Lcaron +0x8A U+0141 Lslash +0x8B U+0143 Nacute +0x8C U+0147 Ncaron +0x8D U+014A Eng +0x8E U+0150 Ohungarumlaut +0x8F U+0154 Racute +0x90 U+0158 Rcaron +0x91 U+015A Sacute +0x92 U+0160 Scaron +0x93 U+015E Scedilla +0x94 U+0164 Tcaron +0x95 U+0162 Tcedilla +0x96 U+0170 Uhungarumlaut +0x97 U+016E Uring +0x98 U+0178 Ydieresis +0x99 U+0179 Zacute +0x9A U+017D Zcaron +0x9B U+017B Zdotaccent +0x9C U+0132 IJ +0x9D U+0130 Idotaccent +0x9E U+0111 1 dcroat +0x9F U+00A7 section +0xA0 U+0103 1 abreve +0xA1 U+0105 1 aogonek +0xA2 U+0107 1 cacute +0xA3 U+010D 1 ccaron +0xA4 U+010F 1 dcaron +0xA5 U+011B 1 ecaron +0xA6 U+0119 1 eogonek +0xA7 U+011F 1 gbreve +0xA8 U+013A 1 lacute +0xA9 U+013E 1 lcaron +0xAA U+0142 1 lslash +0xAB U+0144 1 nacute +0xAC U+0148 1 ncaron +0xAD U+014B 1 eng +0xAE U+0151 1 ohungarumlaut +0xAF U+0155 1 racute +0xB0 U+0159 1 rcaron +0xB1 U+015B 1 sacute +0xB2 U+0161 1 scaron +0xB3 U+015F 1 scedilla +0xB3 U+0219 1 scommaaccent +0xB4 U+0165 1 tcaron +0xB5 U+021B 1 tcommaaccent +0xB6 U+0171 1 uhungarumlaut +0xB7 U+016F 1 uring +0xB8 U+00FF 1 ydieresis +0xB9 U+017A 1 zacute +0xBA U+017E 1 zcaron +0xBB U+017C 1 zdotaccent +0xBC U+0133 1 ij +0xBD U+00A1 exclamdown +0xBE U+00BF questiondown +0xBF U+00A3 sterling +0xC0 U+00C0 Agrave +0xC1 U+00C1 Aacute +0xC2 U+00C2 Acircumflex +0xC3 U+00C3 Atilde +0xC4 U+00C4 Adieresis +0xC5 U+00C5 Aring +0xC6 U+00C6 AE +0xC7 U+00C7 Ccedilla +0xC8 U+00C8 Egrave +0xC9 U+00C9 Eacute +0xCA U+00CA Ecircumflex +0xCB U+00CB Edieresis +0xCC U+00CC Igrave +0xCD U+00CD Iacute +0xCE U+00CE Icircumflex +0xCF U+00CF Idieresis +0xD0 U+00D0 Eth +0xD1 U+00D1 Ntilde +0xD2 U+00D2 Ograve +0xD3 U+00D3 Oacute +0xD4 U+00D4 Ocircumflex +0xD5 U+00D5 Otilde +0xD6 U+00D6 Odieresis +0xD7 U+0152 OE +0xD8 U+00D8 Oslash +0xD9 U+00D9 Ugrave +0xDA U+00DA Uacute +0xDB U+00DB Ucircumflex +0xDC U+00DC Udieresis +0xDD U+00DD Yacute +0xDE U+00DE Thorn +0xDF U+1E9E Germandbls +0xE0 U+00E0 1 agrave +0xE1 U+00E1 1 aacute +0xE2 U+00E2 1 acircumflex +0xE3 U+00E3 1 atilde +0xE4 U+00E4 1 adieresis +0xE5 U+00E5 1 aring +0xE6 U+00E6 1 ae +0xE7 U+00E7 1 ccedilla +0xE8 U+00E8 1 egrave +0xE9 U+00E9 1 eacute +0xEA U+00EA 1 ecircumflex +0xEB U+00EB 1 edieresis +0xEC U+00EC 1 igrave +0xED U+00ED 1 iacute +0xEE U+00EE 1 icircumflex +0xEF U+00EF 1 idieresis +0xF0 U+00F0 1 eth +0xF1 U+00F1 1 ntilde +0xF2 U+00F2 1 ograve +0xF3 U+00F3 1 oacute +0xF4 U+00F4 1 ocircumflex +0xF5 U+00F5 1 otilde +0xF6 U+00F6 1 odieresis +0xF7 U+0153 1 oe +0xF8 U+00F8 1 oslash +0xF9 U+00F9 1 ugrave +0xFA U+00FA 1 uacute +0xFB U+00FB 1 ucircumflex +0xFC U+00FC 1 udieresis +0xFD U+00FD 1 yacute +0xFE U+00FE 1 thorn +0xFF U+00DF 1 germandbls diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat new file mode 100644 index 0000000000..421230ecc3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il2.dat @@ -0,0 +1,128 @@ +0x80 U+0080 +0x81 U+0081 +0x82 U+0082 +0x83 U+0083 +0x84 U+0084 +0x85 U+0085 +0x86 U+0086 +0x87 U+0087 +0x88 U+0088 +0x89 U+0089 +0x8A U+008A +0x8B U+008B +0x8C U+008C +0x8D U+008D +0x8E U+008E +0x8F U+008F +0x90 U+0090 +0x91 U+0091 +0x92 U+0092 +0x93 U+0093 +0x94 U+0094 +0x95 U+0095 +0x96 U+0096 +0x97 U+0097 +0x98 U+0098 +0x99 U+0099 +0x9A U+009A +0x9B U+009B +0x9C U+009C +0x9D U+009D +0x9E U+009E +0x9F U+009F +0xA0 U+00A0 +0xA1 U+0104 Aogonek +0xA2 U+02D8 breve +0xA3 U+0141 Lslash +0xA4 U+00A4 currency +0xA5 U+013D Lcaron +0xA6 U+015A Sacute +0xA7 U+00A7 section +0xA8 U+00A8 dieresis +0xA9 U+0160 Scaron +0xAA U+015E Scedilla +0xAB U+0164 Tcaron +0xAC U+0179 Zacute +0xAD U+00AD +0xAE U+017D Zcaron +0xAF U+017B Zdotaccent +0xB0 U+00B0 degree +0xB1 U+0105 1 aogonek +0xB2 U+02DB ogonek +0xB3 U+0142 1 lslash +0xB4 U+00B4 acute +0xB5 U+013E 1 lcaron +0xB6 U+015B 1 sacute +0xB7 U+02C7 caron +0xB8 U+00B8 cedilla +0xB9 U+0161 1 scaron +0xBA U+015F 1 scedilla +0xBB U+0165 1 tcaron +0xBC U+017A 1 zacute +0xBD U+02DD hungarumlaut +0xBE U+017E 1 zcaron +0xBF U+017C 1 zdotaccent +0xC0 U+0154 Racute +0xC1 U+00C1 Aacute +0xC2 U+00C2 Acircumflex +0xC3 U+0102 Abreve +0xC4 U+00C4 Adieresis +0xC5 U+0139 Lacute +0xC6 U+0106 Cacute +0xC7 U+00C7 Ccedilla +0xC8 U+010C Ccaron +0xC9 U+00C9 Eacute +0xCA U+0118 Eogonek +0xCB U+00CB Edieresis +0xCC U+011A Ecaron +0xCD U+00CD Iacute +0xCE U+00CE Icircumflex +0xCF U+010E Dcaron +0xD0 U+0110 Dcroat +0xD1 U+0143 Nacute +0xD2 U+0147 Ncaron +0xD3 U+00D3 Oacute +0xD4 U+00D4 Ocircumflex +0xD5 U+0150 Ohungarumlaut +0xD6 U+00D6 Odieresis +0xD7 U+00D7 multiply +0xD8 U+0158 Rcaron +0xD9 U+016E Uring +0xDA U+00DA Uacute +0xDB U+0170 Uhungarumlaut +0xDC U+00DC Udieresis +0xDD U+00DD Yacute +0xDE U+0162 Tcommaaccent +0xDF U+00DF 1 germandbls +0xE0 U+0155 1 racute +0xE1 U+00E1 1 aacute +0xE2 U+00E2 1 acircumflex +0xE3 U+0103 1 abreve +0xE4 U+00E4 1 adieresis +0xE5 U+013A 1 lacute +0xE6 U+0107 1 cacute +0xE7 U+00E7 1 ccedilla +0xE8 U+010D 1 ccaron +0xE9 U+00E9 1 eacute +0xEA U+0119 1 eogonek +0xEB U+00EB 1 edieresis +0xEC U+011B 1 ecaron +0xED U+00ED 1 iacute +0xEE U+00EE 1 icircumflex +0xEF U+010F 1 dcaron +0xF0 U+0111 1 dcroat +0xF1 U+0144 1 nacute +0xF2 U+0148 1 ncaron +0xF3 U+00F3 1 oacute +0xF4 U+00F4 1 ocircumflex +0xF5 U+0151 1 ohungarumlaut +0xF6 U+00F6 1 odieresis +0xF7 U+00F7 divide +0xF8 U+0159 1 rcaron +0xF9 U+016F 1 uring +0xFA U+00FA 1 uacute +0xFB U+0171 1 uhungarumlaut +0xFC U+00FC 1 udieresis +0xFD U+00FD 1 yacute +0xFE U+0163 1 tcommaaccent +0xFF U+02D9 dotaccent diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat new file mode 100644 index 0000000000..7e7d856a19 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/il3.dat @@ -0,0 +1,6 @@ +0xB6 U+0125 1 hcircumflex +0xBC U+0135 1 jcircumflex +0xE6 U+0109 1 ccircumflex +0xF8 U+011D 1 gcircumflex +0xFD U+016D 1 ubreve +0xFE U+015D 1 scircumflex diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat new file mode 100644 index 0000000000..c3e9fa0890 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/l7x.dat @@ -0,0 +1,256 @@ +0x00 U+0060 grave +0x01 U+00B4 acute +0x02 U+02C6 circumflex +0x03 U+02DC tilde +0x04 U+00A8 dieresis +0x05 U+02DD hungarumlaut +0x06 U+02DA ring +0x07 U+02C7 caron +0x08 U+02D8 breve +0x09 U+00AF macron +0x0A U+02D9 dotaccent +0x0B U+00B8 cedilla +0x0C U+02DB ogonek +0x0D U+201A quotesinglbase +0x0E U+2039 guilsinglleft +0x0F U+203A guilsinglright +0x10 U+201C quotedblleft +0x11 U+201D quotedblright +0x12 U+201E quotedblbase +0x13 U+00AB guillemotleft +0x14 U+00BB guillemotright +0x15 U+2013 endash +0x16 U+2014 emdash +0x17 U+200B cwm +0x18 U+.... perthousandzero +0x19 U+0131 1 dotlessi +0x1A U+0237 1 dotlessj +0x1B U+FB00 ff +0x1C U+FB01 fi +0x1D U+FB02 fl +0x1E U+FB03 ffi +0x1F U+FB04 ffl +0x20 U+0020 space +0x21 U+0021 exclam +0x22 U+0022 quotedbl +0x23 U+0023 numbersign +0x24 U+0024 dollar +0x25 U+0025 percent +0x26 U+0026 ampersand +0x27 U+2019 quoteright +0x28 U+0028 parenleft +0x29 U+0029 parenright +0x2A U+002A asterisk +0x2B U+002B plus +0x2C U+002C comma +0x2D U+002D hyphen +0x2E U+002E period +0x2F U+002F slash +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x3A U+003A colon +0x3B U+003B semicolon +0x3C U+003C less +0x3D U+003D equal +0x3E U+003E greater +0x3F U+003F question +0x40 U+0040 at +0x41 U+0041 A +0x42 U+0042 B +0x43 U+0043 C +0x44 U+0044 D +0x45 U+0045 E +0x46 U+0046 F +0x47 U+0047 G +0x48 U+0048 H +0x49 U+0049 I +0x4A U+004A J +0x4B U+004B K +0x4C U+004C L +0x4D U+004D M +0x4E U+004E N +0x4F U+004F O +0x50 U+0050 P +0x51 U+0051 Q +0x52 U+0052 R +0x53 U+0053 S +0x54 U+0054 T +0x55 U+0055 U +0x56 U+0056 V +0x57 U+0057 W +0x58 U+0058 X +0x59 U+0059 Y +0x5A U+005A Z +0x5B U+005B bracketleft +0x5C U+005C backslash +0x5D U+005D bracketright +0x5E U+005E asciicircum +0x5F U+005F underscore +0x60 U+2018 quoteleft +0x61 U+0061 a +0x62 U+0062 b +0x63 U+0063 c +0x64 U+0064 d +0x65 U+0065 e +0x66 U+0066 f +0x67 U+0067 g +0x68 U+0068 h +0x69 U+0069 i +0x6A U+006A j +0x6B U+006B k +0x6C U+006C l +0x6D U+006D m +0x6E U+006E n +0x6F U+006F o +0x70 U+0070 p +0x71 U+0071 q +0x72 U+0072 r +0x73 U+0073 s +0x74 U+0074 t +0x75 U+0075 u +0x76 U+0076 v +0x77 U+0077 w +0x78 U+0078 x +0x79 U+0079 y +0x7A U+007A z +0x7B U+007B braceleft +0x7C U+007C bar +0x7D U+007D braceright +0x7E U+007E asciitilde +0x7F U+.... .notdef +0x80 U+20AC Euro +0x81 U+.... .notdef +0x82 U+201A quotesinglbase +0x83 U+0192 florin +0x84 U+201E quotedblbase +0x85 U+2026 ellipsis +0x86 U+2020 dagger +0x87 U+2021 daggerdbl +0x88 U+.... .notdef +0x89 U+2030 perthousand +0x8A U+.... .notdef +0x8B U+2039 guilsinglleft +0x8C U+0152 OE +0x8D U+00A8 dieresis +0x8E U+02C7 caron +0x8F U+00B8 cedilla +0x90 U+.... .notdef +0x91 U+2018 quoteleft +0x92 U+2019 quoteright +0x93 U+201C quotedblleft +0x94 U+201D quotedblright +0x95 U+2022 bullet +0x96 U+2013 endash +0x97 U+2014 emdash +0x98 U+.... .notdef +0x99 U+2122 trademark +0x9A U+.... .notdef +0x9B U+203A guilsinglright +0x9C U+0153 1 oe +0x9D U+00AF macron +0x9E U+02DB ogonek +0x9F U+.... .notdef +0xA0 U+0020 space +0xA1 U+201D quotedblright +0xA2 U+00A2 cent +0xA3 U+00A3 sterling +0xA4 U+00A4 currency +0xA5 U+201E quotedblbase +0xA6 U+00A6 brokenbar +0xA7 U+00A7 section +0xA8 U+00D8 Oslash +0xA9 U+00A9 copyright +0xAA U+0156 Rcommaaccent +0xAB U+00AB guillemotleft +0xAC U+00AC logicalnot +0xAD U+002D hyphen +0xAE U+00AE registered +0xAF U+00C6 AE +0xB0 U+00B0 degree +0xB1 U+00B1 plusminus +0xB2 U+00B2 twosuperior +0xB3 U+00B3 threesuperior +0xB4 U+201C quotedblleft +0xB5 U+00B5 mu +0xB6 U+00B6 paragraph +0xB7 U+00B7 periodcentered +0xB8 U+00F8 1 oslash +0xB9 U+00B9 onesuperior +0xBA U+0157 1 rcommaaccent +0xBB U+00BB guillemotright +0xBC U+00BC onequarter +0xBD U+00BD onehalf +0xBE U+00BE threequarters +0xBF U+00E6 1 ae +0xC0 U+0104 Aogonek +0xC1 U+012E Iogonek +0xC2 U+0100 Amacron +0xC3 U+0106 Cacute +0xC4 U+00C4 Adieresis +0xC5 U+00C5 Aring +0xC6 U+0118 Eogonek +0xC7 U+0112 Emacron +0xC8 U+010C Ccaron +0xC9 U+00C9 Eacute +0xCA U+0179 Zacute +0xCB U+0116 Edotaccent +0xCC U+0122 Gcommaaccent +0xCD U+0136 Kcommaaccent +0xCE U+012A Imacron +0xCF U+013B Lcommaaccent +0xD0 U+0160 Scaron +0xD1 U+0143 Nacute +0xD2 U+0145 Ncommaaccent +0xD3 U+00D3 Oacute +0xD4 U+014C Omacron +0xD5 U+00D5 Otilde +0xD6 U+00D6 Odieresis +0xD7 U+00D7 multiply +0xD8 U+0172 Uogonek +0xD9 U+0141 Lslash +0xDA U+015A Sacute +0xDB U+016A Umacron +0xDC U+00DC Udieresis +0xDD U+017B Zdotaccent +0xDE U+017D Zcaron +0xDF U+00DF 1 germandbls +0xE0 U+0105 1 aogonek +0xE1 U+012F 1 iogonek +0xE2 U+0101 1 amacron +0xE3 U+0107 1 cacute +0xE4 U+00E4 1 adieresis +0xE5 U+00E5 1 aring +0xE6 U+0119 1 eogonek +0xE7 U+0113 1 emacron +0xE8 U+010D 1 ccaron +0xE9 U+00E9 1 eacute +0xEA U+017A 1 zacute +0xEB U+0117 1 edotaccent +0xEC U+0123 1 gcommaaccent +0xED U+0137 1 kcommaaccent +0xEE U+012B 1 imacron +0xEF U+013C 1 lcommaaccent +0xF0 U+0161 1 scaron +0xF1 U+0144 1 nacute +0xF2 U+0146 1 ncommaaccent +0xF3 U+00F3 1 oacute +0xF4 U+014D 1 omacron +0xF5 U+00F5 1 otilde +0xF6 U+00F6 1 odieresis +0xF7 U+00F7 divide +0xF8 U+0173 1 uogonek +0xF9 U+0142 1 lslash +0xFA U+015B 1 sacute +0xFB U+016B 1 umacron +0xFC U+00FC 1 udieresis +0xFD U+017C 1 zdotaccent +0xFE U+017E 1 zcaron +0xFF U+2019 quoteright diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat new file mode 100644 index 0000000000..2fefcb77c3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lmc.dat @@ -0,0 +1,70 @@ +0x41 U+0410 CYRA +0x42 U+0411 CYRB +0x43 U+0426 CYRC +0x44 U+0414 CYRD +0x45 U+0415 CYRE +0x46 U+0424 CYRF +0x47 U+0413 CYRG +0x49 U+0418 CYRI +0x4A U+0416 CYRZH +0x4B U+041A CYRK +0x4C U+041B CYRL +0x4D U+041C CYRM +0x4E U+041D CYRN +0x4F U+041E CYRO +0x50 U+041F CYRP +0x51 U+0427 CYRCH +0x52 U+0420 CYRR +0x53 U+0421 CYRS +0x54 U+0422 CYRT +0x55 U+0423 CYRU +0x57 U+0412 CYRV +0x58 U+0425 CYRH +0x59 U+042B CYRERY +0x5A U+0417 CYRZ +0x61 U+0430 1 cyra +0x62 U+0431 1 cyrb +0x63 U+0446 1 cyrc +0x64 U+0434 1 cyrd +0x65 U+0435 1 cyre +0x66 U+0444 1 cyrf +0x67 U+0433 1 cyrg +0x69 U+0438 1 cyri +0x6A U+0436 1 cyrzh +0x6B U+043A 1 cyrk +0x6C U+043B 1 cyrl +0x6D U+043C 1 cyrm +0x6E U+043D 1 cyrn +0x6F U+043E 1 cyro +0x70 U+043F 1 cyrp +0x71 U+0447 1 cyrch +0x72 U+0440 1 cyrr +0x73 U+0441 1 cyrs +0x74 U+0442 1 cyrt +0x75 U+0443 1 cyru +0x77 U+0432 1 cyrv +0x78 U+0445 1 cyrh +0x79 U+044B 1 cyrery +0x7A U+0437 1 cyrz +0x90 U+0429 CYRSHCH +0x92 U+0428 CYRSH +0x93 U+042A CYRHRDSN +0x94 U+042C CYRSFTSN +0x95 U+042E CYRYU +0x96 U+042F CYRYA +0xB0 U+0449 1 cyrshch +0xB2 U+0448 1 cyrsh +0xB3 U+044A 1 cyrhrdsn +0xB4 U+0131 1 i +0xB5 U+044E 1 cyryu +0xB6 U+044F 1 cyrya +0xC4 U+042D CYREREV +0xCB U+0401 CYRYO +0xCF U+0419 CYRISHRT +0xD6 U+04E8 CYROTLD +0xDC U+04AE CYRY +0xE4 U+044D 1 cyrerev +0xEB U+0451 1 cyryo +0xEF U+0439 1 cyrishrt +0xF6 U+04E9 1 cyrotld +0xFC U+04AF 1 cyry diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat new file mode 100644 index 0000000000..4d0839cbb2 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/lth.dat @@ -0,0 +1,87 @@ +0xA1 U+0E01 1 +0xA2 U+0E02 1 +0xA3 U+0E03 1 +0xA4 U+0E04 1 +0xA5 U+0E05 1 +0xA6 U+0E06 1 +0xA7 U+0E07 1 +0xA8 U+0E08 1 +0xA9 U+0E09 1 +0xAA U+0E0A 1 +0xAB U+0E0B 1 +0xAC U+0E0C 1 +0xAD U+0E0D 1 +0xAE U+0E0E 1 +0xAF U+0E0F 1 +0xB0 U+0E10 1 +0xB1 U+0E11 1 +0xB2 U+0E12 1 +0xB3 U+0E13 1 +0xB4 U+0E14 1 +0xB5 U+0E15 1 +0xB6 U+0E16 1 +0xB7 U+0E17 1 +0xB8 U+0E18 1 +0xB9 U+0E19 1 +0xBA U+0E1A 1 +0xBB U+0E1B 1 +0xBC U+0E1C 1 +0xBD U+0E1D 1 +0xBE U+0E1E 1 +0xBF U+0E1F 1 +0xC0 U+0E20 1 +0xC1 U+0E21 1 +0xC2 U+0E22 1 +0xC3 U+0E23 1 +0xC4 U+0E24 1 +0xC5 U+0E25 1 +0xC6 U+0E26 1 +0xC7 U+0E27 1 +0xC8 U+0E28 1 +0xC9 U+0E29 1 +0xCA U+0E2A 1 +0xCB U+0E2B 1 +0xCC U+0E2C 1 +0xCD U+0E2D 1 +0xCE U+0E2E 1 +0xCF U+0E2F 1 +0xD0 U+0E30 1 +0xD1 U+0E31 1 +0xD2 U+0E32 1 +0xD3 U+0E33 1 +0xD4 U+0E34 1 +0xD5 U+0E35 1 +0xD6 U+0E36 1 +0xD7 U+0E37 1 +0xD8 U+0E38 1 +0xD9 U+0E39 1 +0xDA U+0E3A 1 +0xDF U+0E3F +0xE0 U+0E40 1 +0xE1 U+0E41 1 +0xE2 U+0E42 1 +0xE3 U+0E43 1 +0xE4 U+0E44 1 +0xE5 U+0E45 1 +0xE6 U+0E46 +0xE7 U+0E47 1 +0xE8 U+0E48 1 +0xE9 U+0E49 1 +0xEA U+0E4A 1 +0xEB U+0E4B 1 +0xEC U+0E4C 1 +0xED U+0E4D 1 +0xEE U+0E4E 1 +0xEF U+0E4F +0xF0 U+0E50 +0xF1 U+0E51 +0xF2 U+0E52 +0xF3 U+0E53 +0xF4 U+0E54 +0xF5 U+0E55 +0xF6 U+0E56 +0xF7 U+0E57 +0xF8 U+0E58 +0xF9 U+0E59 +0xFA U+0E5A +0xFB U+0E5B diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat new file mode 100644 index 0000000000..57abcac177 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/qx.dat @@ -0,0 +1,256 @@ +0x00 U+.... .notdef +0x01 U+2206 Delta +0x02 U+.... .notdef +0x03 U+.... .notdef +0x04 U+.... .notdef +0x05 U+03A0 Pi +0x06 U+03A3 Sigma +0x07 U+00B5 mu +0x08 U+2026 ellipsis +0x09 U+.... f_k +0x0A U+2126 Omega +0x0B U+FB00 ff +0x0C U+FB01 fi +0x0D U+FB02 fl +0x0E U+FB03 ffi +0x0F U+FB04 ffl +0x10 U+0131 1 dotlessi +0x11 U+0237 1 dotlessj +0x12 U+0060 grave +0x13 U+00B4 acute +0x14 U+02C7 caron +0x15 U+02D8 breve +0x16 U+00AF macron +0x17 U+02DA ring +0x18 U+00B8 cedilla +0x19 U+00DF 1 germandbls +0x1A U+00E6 1 ae +0x1B U+0153 1 oe +0x1C U+00F8 1 oslash +0x1D U+00C6 AE +0x1E U+0152 OE +0x1F U+00D8 Oslash +0x20 U+0020 space +0x21 U+0021 exclam +0x22 U+201D quotedblright +0x23 U+0023 numbersign +0x24 U+0024 dollar +0x25 U+0025 percent +0x26 U+0026 ampersand +0x27 U+2019 quoteright +0x28 U+0028 parenleft +0x29 U+0029 parenright +0x2A U+002A asterisk +0x2B U+002B plus +0x2C U+002C comma +0x2D U+002D hyphen +0x2E U+002E period +0x2F U+002F slash +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x3A U+003A colon +0x3B U+003B semicolon +0x3C U+00A1 exclamdown +0x3D U+003D equal +0x3E U+00BF questiondown +0x3F U+003F question +0x40 U+0040 at +0x41 U+0041 A +0x42 U+0042 B +0x43 U+0043 C +0x44 U+0044 D +0x45 U+0045 E +0x46 U+0046 F +0x47 U+0047 G +0x48 U+0048 H +0x49 U+0049 I +0x4A U+004A J +0x4B U+004B K +0x4C U+004C L +0x4D U+004D M +0x4E U+004E N +0x4F U+004F O +0x50 U+0050 P +0x51 U+0051 Q +0x52 U+0052 R +0x53 U+0053 S +0x54 U+0054 T +0x55 U+0055 U +0x56 U+0056 V +0x57 U+0057 W +0x58 U+0058 X +0x59 U+0059 Y +0x5A U+005A Z +0x5B U+005B bracketleft +0x5C U+201C quotedblleft +0x5D U+005D bracketright +0x5E U+02C6 circumflex +0x5F U+02D9 dotaccent +0x60 U+2018 quoteleft +0x61 U+0061 a +0x62 U+0062 b +0x63 U+0063 c +0x64 U+0064 d +0x65 U+0065 e +0x66 U+0066 f +0x67 U+0067 g +0x68 U+0068 h +0x69 U+0069 i +0x6A U+006A j +0x6B U+006B k +0x6C U+006C l +0x6D U+006D m +0x6E U+006E n +0x6F U+006F o +0x70 U+0070 p +0x71 U+0071 q +0x72 U+0072 r +0x73 U+0073 s +0x74 U+0074 t +0x75 U+0075 u +0x76 U+0076 v +0x77 U+0077 w +0x78 U+0078 x +0x79 U+0079 y +0x7A U+007A z +0x7B U+2013 endash +0x7C U+2014 emdash +0x7D U+02DD hungarumlaut +0x7E U+02DC tilde +0x7F U+00A8 dieresis +0x80 U+20AC Euro +0x81 U+0104 Aogonek +0x82 U+0106 Cacute +0x83 U+003E greater +0x84 U+.... .notdef +0x85 U+.... .notdef +0x86 U+0118 Eogonek +0x87 U+012E Iogonek +0x88 U+003C less +0x89 U+.... .notdef +0x8A U+0141 Lslash +0x8B U+0143 Nacute +0x8C U+007E asciitilde +0x8D U+005E asciicircum +0x8E U+.... .notdef +0x8F U+2020 dagger +0x90 U+2021 daggerdbl +0x91 U+015A Sacute +0x92 U+0160 Scaron +0x93 U+0218 Scommaaccent +0x94 U+00B0 degree +0x95 U+0162 Tcommaaccent +0x96 U+02DB ogonek +0x97 U+0172 Uogonek +0x98 U+0178 Ydieresis +0x99 U+0179 Zacute +0x9A U+017D Zcaron +0x9B U+017B Zdotaccent +0x9C U+0132 IJ +0x9D U+007B braceleft +0x9E U+007D braceright +0x9F U+00A7 section +0xA0 U+.... .notdef +0xA1 U+0105 1 aogonek +0xA2 U+0107 1 cacute +0xA3 U+00AE registered +0xA4 U+00A9 copyright +0xA5 U+00F7 divide +0xA6 U+0119 1 eogonek +0xA7 U+012F 1 iogonek +0xA8 U+2212 minus +0xA9 U+00D7 multiply +0xAA U+0142 1 lslash +0xAB U+0144 1 nacute +0xAC U+00B1 plusminus +0xAD U+221E infinity +0xAE U+00AB guillemotleft +0xAF U+00BB guillemotright +0xB0 U+00B6 paragraph +0xB1 U+015B 1 sacute +0xB2 U+0161 1 scaron +0xB3 U+0219 1 scommaaccent +0xB4 U+2022 bullet +0xB5 U+0163 1 tcommaaccent +0xB6 U+.... threequartersemdash +0xB7 U+0173 1 uogonek +0xB8 U+00FF 1 ydieresis +0xB9 U+017A 1 zacute +0xBA U+017E 1 zcaron +0xBB U+017C 1 zdotaccent +0xBC U+0133 1 ij +0xBD U+00B7 periodcentered +0xBE U+0022 quotedbl +0xBF U+0027 quotesingle +0xC0 U+00C0 Agrave +0xC1 U+00C1 Aacute +0xC2 U+00C2 Acircumflex +0xC3 U+00C3 Atilde +0xC4 U+00C4 Adieresis +0xC5 U+00C5 Aring +0xC6 U+005C backslash +0xC7 U+00C7 Ccedilla +0xC8 U+00C8 Egrave +0xC9 U+00C9 Eacute +0xCA U+00CA Ecircumflex +0xCB U+00CB Edieresis +0xCC U+00CC Igrave +0xCD U+00CD Iacute +0xCE U+00CE Icircumflex +0xCF U+00CF Idieresis +0xD0 U+00D0 Eth +0xD1 U+00D1 Ntilde +0xD2 U+00D2 Ograve +0xD3 U+00D3 Oacute +0xD4 U+00D4 Ocircumflex +0xD5 U+00D5 Otilde +0xD6 U+00D6 Odieresis +0xD7 U+00A4 currency +0xD8 U+2030 perthousand +0xD9 U+00D9 Ugrave +0xDA U+00DA Uacute +0xDB U+00DB Ucircumflex +0xDC U+00DC Udieresis +0xDD U+00DD Yacute +0xDE U+00DE Thorn +0xDF U+007C bar +0xE0 U+00E0 1 agrave +0xE1 U+00E1 1 aacute +0xE2 U+00E2 1 acircumflex +0xE3 U+00E3 1 atilde +0xE4 U+00E4 1 adieresis +0xE5 U+00E5 1 aring +0xE6 U+005F underscore +0xE7 U+00E7 1 ccedilla +0xE8 U+00E8 1 egrave +0xE9 U+00E9 1 eacute +0xEA U+00EA 1 ecircumflex +0xEB U+00EB 1 edieresis +0xEC U+00EC 1 igrave +0xED U+00ED 1 iacute +0xEE U+00EE 1 icircumflex +0xEF U+00EF 1 idieresis +0xF0 U+00F0 1 eth +0xF1 U+00F1 1 ntilde +0xF2 U+00F2 1 ograve +0xF3 U+00F3 1 oacute +0xF4 U+00F4 1 ocircumflex +0xF5 U+00F5 1 otilde +0xF6 U+00F6 1 odieresis +0xF7 U+2222 anglearc +0xF8 U+2300 diameter +0xF9 U+00F9 1 ugrave +0xFA U+00FA 1 uacute +0xFB U+00FB 1 ucircumflex +0xFC U+00FC 1 udieresis +0xFD U+00FD 1 yacute +0xFE U+00FE 1 thorn +0xFF U+201E quotedblbase diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat new file mode 100644 index 0000000000..23b8d8fa4a --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t2a.dat @@ -0,0 +1,256 @@ +0x00 U+0060 +0x01 U+00B4 +0x02 U+02C6 +0x03 U+02DC +0x04 U+00A8 +0x05 U+02DD +0x06 U+02DA +0x07 U+02C7 +0x08 U+02D8 +0x09 U+00AF +0x0A U+02D9 +0x0B U+00B8 +0x0C U+02DB +0x0D U+04C0 CYRpalochka +0x0E U+2329 textlangle +0x0F U+232A textrangle +0x10 U+201C textquotedblleft +0x11 U+201D textquotedblright +0x12 U+F6D5 +0x13 U+F6D6 +0x14 U+F6D4 +0x15 U+2013 textendash +0x16 U+2014 textemdash +# 0x17 U+0x17 Nothing? +# 0x18 U+0x18 ? +0x19 U+0131 1 i +0x1A U+F6BE +0x1B U+FB00 # Hand-edited! +0x1C U+FB01 # +0x1D U+FB02 # +0x1E U+FB03 # +0x1F U+FB04 # +0x20 U+0020 +0x21 U+0021 exclam +0x22 U+0022 quotedbl +0x23 U+0023 numbersign +0x24 U+0024 dollar +0x25 U+0025 percent +0x26 U+0026 ampersand +0x27 U+2019 quoteright +0x28 U+0028 parenleft +0x29 U+0029 parenright +0x2A U+002A asterisk +0x2B U+002B plus +0x2C U+002C comma +0x2D U+002D hyphen +0x2E U+002E period +0x2F U+002F slash +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x3A U+003A colon +0x3B U+003B semicolon +0x3C U+003C less +0x3D U+003D equal +0x3E U+003E greater +0x3F U+003F question +0x40 U+0040 at +0x41 U+0041 A +0x42 U+0042 B +0x43 U+0043 C +0x44 U+0044 D +0x45 U+0045 E +0x46 U+0046 F +0x47 U+0047 G +0x48 U+0048 H +0x49 U+0406 CYRII +0x4A U+004A J +0x4B U+004B K +0x4C U+004C L +0x4D U+004D M +0x4E U+004E N +0x4F U+004F O +0x50 U+0050 P +0x51 U+0051 Q +0x52 U+0052 R +0x53 U+0053 S +0x54 U+0054 T +0x55 U+0055 U +0x56 U+0056 V +0x57 U+0057 W +0x58 U+0058 X +0x59 U+0059 Y +0x5A U+005A Z +0x5B U+005B bracketleft +0x5C U+005C backslash +0x5D U+005D bracketright +0x5E U+02C6 circumflex +0x5F U+005F underscore +0x60 U+2018 quoteleft +0x61 U+0061 a +0x62 U+0062 b +0x63 U+0063 c +0x64 U+0064 d +0x65 U+0065 e +0x66 U+0066 f +0x67 U+0067 g +0x68 U+0068 h +0x69 U+0456 1 cyrii +# Hum ... (fix for Serbian) +0x6A U+0458 1 cyrj +0x6B U+006B k +0x6C U+006C l +0x6D U+006D m +0x6E U+006E n +0x6F U+006F o +0x70 U+0070 p +0x71 U+0071 q +0x72 U+0072 r +0x73 U+0073 s +0x74 U+0074 t +0x75 U+0075 u +0x76 U+0076 v +0x77 U+0077 w +0x78 U+0078 x +0x79 U+0079 y +0x7A U+007A z +0x7B U+007B braceleft +0x7C U+007C bar +0x7D U+007D braceright +0x7E U+02DC tilde +0x80 U+0490 CYRGUP +0x81 U+0492 CYRGHCRS +0x82 U+0402 CYRDJE +0x83 U+040B CYRTSHE +0x84 U+04BA CYRSHHA +0x85 U+0496 CYRZHDSC +0x86 U+0498 CYRZDSC +0x87 U+0409 CYRLJE +0x88 U+0407 CYRYI +0x89 U+049A CYRKDSC +0x8A U+04A0 +0x8B U+049C CYRKVCRS +0x8C U+04D4 CYRAE +0x8D U+04A2 CYRNDSC +0x8E U+04A4 CYRNG +0x8F U+0405 CYRDZE +0x90 U+04E8 CYROTLD +0x91 U+04AA CYRSDSC +0x92 U+040E CYRUSHRT +0x93 U+04AE CYRY +0x94 U+04B0 CYRYHCRS +0x95 U+04B2 CYRHDSC +0x96 U+040F CYRDZHE +0x97 U+04B8 CYRCHVCRS +0x98 U+04B6 CYRCHRDSC +0x99 U+0404 CYRIE +0x9A U+04D8 CYRSCHWA +0x9B U+040A CYRNJE +0x9C U+0401 CYRYO +0x9D U+2116 textnumero +0x9E U+00A4 textcurrency +0x9F U+00A7 textsection +0xA0 U+0491 1 cyrgup +0xA1 U+0493 1 cyrghcrs +0xA2 U+0452 1 cyrdje +0xA3 U+045B 1 cyrtshe +0xA4 U+04BB 1 cyrshha +0xA5 U+0497 1 cyrzhdsc +0xA6 U+0499 1 cyrzdsc +0xA7 U+0459 1 cyrlje +0xA8 U+0457 1 cyryi +0xA9 U+049B 1 cyrkdsc +0xAA U+04A1 1 +0xAB U+049D 1 cyrkvcrs +0xAC U+04D5 1 cyrae +0xAD U+04A3 1 cyrndsc +0xAE U+04A5 1 cyrng +0xAF U+0455 1 cyrdze +0xB0 U+04E9 1 cyrotld +0xB1 U+04AB 1 cyrsdsc +0xB2 U+045E 1 cyrushrt +0xB3 U+04AF 1 cyry +0xB4 U+04B1 1 cyryhcrs +0xB5 U+04B3 1 cyrhdsc +0xB6 U+045F 1 cyrdzhe +0xB7 U+04B9 1 cyrchvcrs +0xB8 U+04B7 1 cyrchrdsc +0xB9 U+0454 1 cyrie +0xBA U+04D9 1 cyrschwa +0xBB U+045A 1 cyrnje +0xBC U+0451 1 cyryo +0xBD U+201E quotedblbase +0xBE U+00AB guillemotleft +0xBF U+00BB guillemotright +0xC0 U+0410 CYRA +0xC1 U+0411 CYRB +0xC2 U+0412 CYRV +0xC3 U+0413 CYRG +0xC4 U+0414 CYRD +0xC5 U+0415 CYRE +0xC6 U+0416 CYRZH +0xC7 U+0417 CYRZ +0xC8 U+0418 CYRI +0xC9 U+0419 CYRISHRT +0xCA U+041A CYRK +0xCB U+041B CYRL +0xCC U+041C CYRM +0xCD U+041D CYRN +0xCE U+041E CYRO +0xCF U+041F CYRP +0xD0 U+0420 CYRR +0xD1 U+0421 CYRS +0xD2 U+0422 CYRT +0xD3 U+0423 CYRU +0xD4 U+0424 CYRF +0xD5 U+0425 CYRH +0xD6 U+0426 CYRC +0xD7 U+0427 CYRCH +0xD8 U+0428 CYRSH +0xD9 U+0429 CYRSHCH +0xDA U+042A CYRHRDSN +0xDB U+042B CYRERY +0xDC U+042C CYRSFTSN +0xDD U+042D CYREREV +0xDE U+042E CYRYU +0xDF U+042F CYRYA +0xE0 U+0430 1 cyra +0xE1 U+0431 1 cyrb +0xE2 U+0432 1 cyrv +0xE3 U+0433 1 cyrg +0xE4 U+0434 1 cyrd +0xE5 U+0435 1 cyre +0xE6 U+0436 1 cyrzh +0xE7 U+0437 1 cyrz +0xE8 U+0438 1 cyri +0xE9 U+0439 1 cyrishrt +0xEA U+043A 1 cyrk +0xEB U+043B 1 cyrl +0xEC U+043C 1 cyrm +0xED U+043D 1 cyrn +0xEE U+043E 1 cyro +0xEF U+043F 1 cyrp +0xF0 U+0440 1 cyrr +0xF1 U+0441 1 cyrs +0xF2 U+0442 1 cyrt +0xF3 U+0443 1 cyru +0xF4 U+0444 1 cyrf +0xF5 U+0445 1 cyrh +0xF6 U+0446 1 cyrc +0xF7 U+0447 1 cyrch +0xF8 U+0448 1 cyrsh +0xF9 U+0449 1 cyrshch +0xFA U+044A 1 cyrhrdsn +0xFB U+044B 1 cyrery +0xFC U+044C 1 cyrsftsn +0xFD U+044D 1 cyrerev +0xFE U+044E 1 cyryu +0xFF U+044F 1 cyrya diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat new file mode 100644 index 0000000000..78e09abc94 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/t8m.dat @@ -0,0 +1,256 @@ +0x00 U+.... .notdef +0x01 U+.... .notdef +0x02 U+.... .notdef +0x03 U+.... .notdef +0x04 U+.... .notdef +0x05 U+.... .notdef +0x06 U+.... .notdef +0x07 U+.... .notdef +0x08 U+.... .notdef +0x09 U+00A9 copyright +0x0A U+00AB guillemotleft +0x0B U+00BB guillemotright +0x0C U+00AE registered +0x0D U+00A3 sterling +0x0E U+2020 dagger +0x0F U+2116 uni2116 +0x10 U+201E quotedblbase +0x11 U+201C quotedblleft +0x12 U+.... .notdef +0x13 U+.... .notdef +0x14 U+.... .notdef +0x15 U+.... .notdef +0x16 U+.... .notdef +0x17 U+.... .notdef +0x18 U+.... .notdef +0x19 U+.... .notdef +0x1A U+.... .notdef +0x1B U+.... .notdef +0x1C U+.... .notdef +0x1D U+.... .notdef +0x1E U+.... .notdef +0x1F U+.... .notdef +0x20 U+0020 space +0x21 U+0021 exclam +0x22 U+0022 quotedbl +0x23 U+0023 numbersign +0x24 U+0024 dollar +0x25 U+0025 percent +0x26 U+0026 ampersand +0x27 U+0027 quotesingle +0x28 U+0028 parenleft +0x29 U+0029 parenright +0x2A U+002A asterisk +0x2B U+002B plus +0x2C U+002C comma +0x2D U+002D hyphen +0x2E U+002E period +0x2F U+002F slash +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x3A U+003A colon +0x3B U+003B semicolon +0x3C U+003C less +0x3D U+003D equal +0x3E U+003E greater +0x3F U+003F question +0x40 U+0040 at +0x41 U+0041 A +0x42 U+0042 B +0x43 U+0043 C +0x44 U+0044 D +0x45 U+0045 E +0x46 U+0046 F +0x47 U+0047 G +0x48 U+0048 H +0x49 U+0049 I +0x4A U+004A J +0x4B U+004B K +0x4C U+004C L +0x4D U+004D M +0x4E U+004E N +0x4F U+004F O +0x50 U+0050 P +0x51 U+0051 Q +0x52 U+0052 R +0x53 U+0053 S +0x54 U+0054 T +0x55 U+0055 U +0x56 U+0056 V +0x57 U+0057 W +0x58 U+0058 X +0x59 U+0059 Y +0x5A U+005A Z +0x5B U+005B bracketleft +0x5C U+005C backslash +0x5D U+005D bracketright +0x5E U+005E asciicircum +0x5F U+005F underscore +0x60 U+0060 grave +0x61 U+0061 a +0x62 U+0062 b +0x63 U+0063 c +0x64 U+0064 d +0x65 U+0065 e +0x66 U+0066 f +0x67 U+0067 g +0x68 U+0068 h +0x69 U+0069 i +0x6A U+006A j +0x6B U+006B k +0x6C U+006C l +0x6D U+006D m +0x6E U+006E n +0x6F U+006F o +0x70 U+0070 p +0x71 U+0071 q +0x72 U+0072 r +0x73 U+0073 s +0x74 U+0074 t +0x75 U+0075 u +0x76 U+0076 v +0x77 U+0077 w +0x78 U+0078 x +0x79 U+0079 y +0x7A U+007A z +0x7B U+007B braceleft +0x7C U+007C bar +0x7D U+007D braceright +0x7E U+007E asciitilde +0x7F U+.... .notdef +0x80 U+10FB uni10FB +0x81 U+10A0 uni10A0 +0x82 U+10A1 uni10A1 +0x83 U+10A2 uni10A2 +0x84 U+10A3 uni10A3 +0x85 U+10A4 uni10A4 +0x86 U+10A5 uni10A5 +0x87 U+10A6 uni10A6 +0x88 U+10A7 uni10A7 +0x89 U+10A8 uni10A8 +0x8A U+10A9 uni10A9 +0x8B U+10AA uni10AA +0x8C U+10AB uni10AB +0x8D U+10AC uni10AC +0x8E U+10AD uni10AD +0x8F U+10AE uni10AE +0x90 U+10AF uni10AF +0x91 U+10B0 uni10B0 +0x92 U+10B1 uni10B1 +0x93 U+10B2 uni10B2 +0x94 U+10B3 uni10B3 +0x95 U+10B4 uni10B4 +0x96 U+10B5 uni10B5 +0x97 U+10B6 uni10B6 +0x98 U+10B7 uni10B7 +0x99 U+10B8 uni10B8 +0x9A U+10B9 uni10B9 +0x9B U+10BA uni10BA +0x9C U+10BB uni10BB +0x9D U+2013 endash +0x9E U+2014 emdash +0x9F U+00A7 section +0xA0 U+00A0 nonbreakingspace +0xA1 U+10D0 1 uni10D0 +0xA2 U+10D1 1 uni10D1 +0xA3 U+10D2 1 uni10D2 +0xA4 U+10D3 1 uni10D3 +0xA5 U+10D4 1 uni10D4 +0xA6 U+10D5 1 uni10D5 +0xA7 U+10D6 1 uni10D6 +0xA8 U+10D7 1 uni10D7 +0xA9 U+10D8 1 uni10D8 +0xAA U+10D9 1 uni10D9 +0xAB U+10DA 1 uni10DA +0xAC U+10DB 1 uni10DB +0xAD U+10DC 1 uni10DC +0xAE U+10DD 1 uni10DD +0xAF U+10DE 1 uni10DE +0xB0 U+10DF 1 uni10DF +0xB1 U+10E0 1 uni10E0 +0xB2 U+10E1 1 uni10E1 +0xB3 U+10E2 1 uni10E2 +0xB4 U+10E3 1 uni10E3 +0xB5 U+10E4 1 uni10E4 +0xB6 U+10E5 1 uni10E5 +0xB7 U+10E6 1 uni10E6 +0xB8 U+10E7 1 uni10E7 +0xB9 U+10E8 1 uni10E8 +0xBA U+10E9 1 uni10E9 +0xBB U+10EA 1 uni10EA +0xBC U+10EB 1 uni10EB +0xBD U+.... .notdef +0xBE U+.... .notdef +0xBF U+.... .notdef +0xC0 U+10BC uni10BC +0xC1 U+10BD uni10BD +0xC2 U+10BE uni10BE +0xC3 U+10BF uni10BF +0xC4 U+10C0 uni10C0 +0xC5 U+10C1 uni10C1 +0xC6 U+10C2 uni10C2 +0xC7 U+10C3 uni10C3 +0xC8 U+10C4 uni10C4 +0xC9 U+10C5 uni10C5 +0xCA U+.... .notdef +0xCB U+.... .notdef +0xCC U+.... .notdef +0xCD U+.... .notdef +0xCE U+.... .notdef +0xCF U+.... .notdef +0xD0 U+.... .notdef +0xD1 U+.... .notdef +0xD2 U+.... .notdef +0xD3 U+.... .notdef +0xD4 U+.... .notdef +0xD5 U+.... .notdef +0xD6 U+.... .notdef +0xD7 U+.... .notdef +0xD8 U+.... .notdef +0xD9 U+.... .notdef +0xDA U+.... .notdef +0xDB U+.... .notdef +0xDC U+.... .notdef +0xDD U+.... .notdef +0xDE U+.... .notdef +0xDF U+.... .notdef +0xE0 U+10EC 1 uni10EC +0xE1 U+10ED 1 uni10ED +0xE2 U+10EE 1 uni10EE +0xE3 U+10EF 1 uni10EF +0xE4 U+10F0 1 uni10F0 +0xE5 U+10F1 1 uni10F1 +0xE6 U+10F2 1 uni10F2 +0xE7 U+10F3 1 uni10F3 +0xE8 U+10F4 1 uni10F4 +0xE9 U+10F5 1 uni10F5 +0xEA U+10F6 1 uni10F6 +0xEB U+10F7 1 uni10F7 +0xEC U+10F8 1 uni10F8 +0xED U+10F9 1 uni10F9 +0xEE U+10FA 1 uni10FA +0xEF U+10FC uni10FC +0xF0 U+.... .notdef +0xF1 U+.... .notdef +0xF2 U+.... .notdef +0xF3 U+.... .notdef +0xF4 U+.... .notdef +0xF5 U+.... .notdef +0xF6 U+.... .notdef +0xF7 U+.... .notdef +0xF8 U+.... .notdef +0xF9 U+.... .notdef +0xFA U+.... .notdef +0xFB U+.... .notdef +0xFC U+.... .notdef +0xFD U+.... .notdef +0xFE U+.... .notdef +0xFF U+.... .notdef diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat new file mode 100644 index 0000000000..96aaea03a0 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/texnansi.dat @@ -0,0 +1,256 @@ +0x00 U+.... .notdef +0x01 U+20AC Euro +0x02 U+.... .notdef +0x03 U+.... .notdef +0x04 U+2044 fraction +0x05 U+02D9 dotaccent +0x06 U+02DD hungarumlaut +0x07 U+02DB ogonek +0x08 U+FB02 fl +0x09 U+.... .notdef +0x0A U+200B cwm +0x0B U+FB00 ff +0x0C U+FB01 fi +0x0D U+.... .notdef +0x0E U+FB03 ffi +0x0F U+FB04 ffl +0x10 U+0131 1 dotlessi +0x11 U+0237 1 dotlessj +0x12 U+0060 grave +0x13 U+00B4 acute +0x14 U+02C7 caron +0x15 U+02D8 breve +0x16 U+00AF macron +0x17 U+02DA ring +0x18 U+00B8 cedilla +0x19 U+00DF 1 germandbls +0x1A U+00E6 1 ae +0x1B U+0153 1 oe +0x1C U+00F8 1 oslash +0x1D U+00C6 AE +0x1E U+0152 OE +0x1F U+00D8 Oslash +0x20 U+0020 space +0x21 U+0021 exclam +0x22 U+0022 quotedbl +0x23 U+0023 numbersign +0x24 U+0024 dollar +0x25 U+0025 percent +0x26 U+0026 ampersand +0x27 U+2019 quoteright +0x28 U+0028 parenleft +0x29 U+0029 parenright +0x2A U+002A asterisk +0x2B U+002B plus +0x2C U+002C comma +0x2D U+002D hyphen +0x2E U+002E period +0x2F U+002F slash +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x3A U+003A colon +0x3B U+003B semicolon +0x3C U+003C less +0x3D U+003D equal +0x3E U+003E greater +0x3F U+003F question +0x40 U+0040 at +0x41 U+0041 A +0x42 U+0042 B +0x43 U+0043 C +0x44 U+0044 D +0x45 U+0045 E +0x46 U+0046 F +0x47 U+0047 G +0x48 U+0048 H +0x49 U+0049 I +0x4A U+004A J +0x4B U+004B K +0x4C U+004C L +0x4D U+004D M +0x4E U+004E N +0x4F U+004F O +0x50 U+0050 P +0x51 U+0051 Q +0x52 U+0052 R +0x53 U+0053 S +0x54 U+0054 T +0x55 U+0055 U +0x56 U+0056 V +0x57 U+0057 W +0x58 U+0058 X +0x59 U+0059 Y +0x5A U+005A Z +0x5B U+005B bracketleft +0x5C U+005C backslash +0x5D U+005D bracketright +0x5E U+02C6 circumflex +0x5F U+005F underscore +0x60 U+2018 quoteleft +0x61 U+0061 a +0x62 U+0062 b +0x63 U+0063 c +0x64 U+0064 d +0x65 U+0065 e +0x66 U+0066 f +0x67 U+0067 g +0x68 U+0068 h +0x69 U+0069 i +0x6A U+006A j +0x6B U+006B k +0x6C U+006C l +0x6D U+006D m +0x6E U+006E n +0x6F U+006F o +0x70 U+0070 p +0x71 U+0071 q +0x72 U+0072 r +0x73 U+0073 s +0x74 U+0074 t +0x75 U+0075 u +0x76 U+0076 v +0x77 U+0077 w +0x78 U+0078 x +0x79 U+0079 y +0x7A U+007A z +0x7B U+007B braceleft +0x7C U+007C bar +0x7D U+007D braceright +0x7E U+02DC tilde +0x7F U+00A8 dieresis +0x80 U+0141 Lslash +0x81 U+0027 quotesingle +0x82 U+201A quotesinglbase +0x83 U+0192 1 florin +0x84 U+201E quotedblbase +0x85 U+2026 ellipsis +0x86 U+2020 dagger +0x87 U+2021 daggerdbl +0x88 U+02C6 circumflex +0x89 U+2030 perthousand +0x8A U+0160 Scaron +0x8B U+2039 guilsinglleft +0x8C U+0152 OE +0x8D U+017D Zcaron +0x8E U+005E asciicircum +0x8F U+2212 minus +0x90 U+0142 1 lslash +0x91 U+2018 quoteleft +0x92 U+2019 quoteright +0x93 U+201C quotedblleft +0x94 U+201D quotedblright +0x95 U+2022 bullet +0x96 U+2013 endash +0x97 U+2014 emdash +0x98 U+02DC tilde +0x99 U+2122 trademark +0x9A U+0161 1 scaron +0x9B U+203A guilsinglright +0x9C U+0153 1 oe +0x9D U+017E 1 zcaron +0x9E U+007E asciitilde +0x9F U+0178 Ydieresis +0xA0 U+00A0 nbspace +0xA1 U+00A1 exclamdown +0xA2 U+00A2 cent +0xA3 U+00A3 sterling +0xA4 U+00A4 currency +0xA5 U+00A5 yen +0xA6 U+00A6 brokenbar +0xA7 U+00A7 section +0xA8 U+00A8 dieresis +0xA9 U+00A9 copyright +0xAA U+00AA 1 ordfeminine +0xAB U+00AB guillemotleft +0xAC U+00AC logicalnot +0xAD U+00AD sfthyphen +0xAE U+00AE registered +0xAF U+00AF macron +0xB0 U+00B0 degree +0xB1 U+00B1 plusminus +0xB2 U+00B2 twosuperior +0xB3 U+00B3 threesuperior +0xB4 U+00B4 acute +0xB5 U+00B5 mu +0xB6 U+00B6 paragraph +0xB7 U+00B7 periodcentered +0xB8 U+00B8 cedilla +0xB9 U+00B9 onesuperior +0xBA U+00BA 1 ordmasculine +0xBB U+00BB guillemotright +0xBC U+00BC onequarter +0xBD U+00BD onehalf +0xBE U+00BE threequarters +0xBF U+00BF questiondown +0xC0 U+00C0 Agrave +0xC1 U+00C1 Aacute +0xC2 U+00C2 Acircumflex +0xC3 U+00C3 Atilde +0xC4 U+00C4 Adieresis +0xC5 U+00C5 Aring +0xC6 U+00C6 AE +0xC7 U+00C7 Ccedilla +0xC8 U+00C8 Egrave +0xC9 U+00C9 Eacute +0xCA U+00CA Ecircumflex +0xCB U+00CB Edieresis +0xCC U+00CC Igrave +0xCD U+00CD Iacute +0xCE U+00CE Icircumflex +0xCF U+00CF Idieresis +0xD0 U+00D0 Eth +0xD1 U+00D1 Ntilde +0xD2 U+00D2 Ograve +0xD3 U+00D3 Oacute +0xD4 U+00D4 Ocircumflex +0xD5 U+00D5 Otilde +0xD6 U+00D6 Odieresis +0xD7 U+00D7 multiply +0xD8 U+00D8 Oslash +0xD9 U+00D9 Ugrave +0xDA U+00DA Uacute +0xDB U+00DB Ucircumflex +0xDC U+00DC Udieresis +0xDD U+00DD Yacute +0xDE U+00DE Thorn +0xDF U+00DF 1 germandbls +0xE0 U+00E0 1 agrave +0xE1 U+00E1 1 aacute +0xE2 U+00E2 1 acircumflex +0xE3 U+00E3 1 atilde +0xE4 U+00E4 1 adieresis +0xE5 U+00E5 1 aring +0xE6 U+00E6 1 ae +0xE7 U+00E7 1 ccedilla +0xE8 U+00E8 1 egrave +0xE9 U+00E9 1 eacute +0xEA U+00EA 1 ecircumflex +0xEB U+00EB 1 edieresis +0xEC U+00EC 1 igrave +0xED U+00ED 1 iacute +0xEE U+00EE 1 icircumflex +0xEF U+00EF 1 idieresis +0xF0 U+00F0 1 eth +0xF1 U+00F1 1 ntilde +0xF2 U+00F2 1 ograve +0xF3 U+00F3 1 oacute +0xF4 U+00F4 1 ocircumflex +0xF5 U+00F5 1 otilde +0xF6 U+00F6 1 odieresis +0xF7 U+00F7 divide +0xF8 U+00F8 1 oslash +0xF9 U+00F9 1 ugrave +0xFA U+00FA 1 uacute +0xFB U+00FB 1 ucircumflex +0xFC U+00FC 1 udieresis +0xFD U+00FD 1 yacute +0xFE U+00FE 1 thorn +0xFF U+00FF 1 ydieresis diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb new file mode 100755 index 0000000000..d76aa29657 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb @@ -0,0 +1,177 @@ +#!/usr/bin/env ruby + +require_relative 'hyph-utf8' + +$path_root=File.expand_path("../../../..", __FILE__) +$encoding_data_dir = File.expand_path("../data/encodings", __FILE__) +$output_data_dir = "#{$path_root}/tex/generic/hyph-utf8/conversions" + +def output_file_name(encoding) + File.join($output_data_dir, sprintf('conv-utf8-%s.tex', encoding)) +end + +$header = <<__EOHEADER__ +%% conv-utf8-%s.tex +%% +%% Conversion from UTF-8 to %s, +%% used before loading hyphenation patterns for 8-bit TeX engines. +%% +%% This file is part of hyph-utf8 package and autogenerated. +%% See http://tug.org/tex-hyphen +%% +%% Copyright 2008-%d TeX Users Group. +%% You may freely use, modify and/or distribute this file. +%% (But consider adapting the scripts if you need modifications.) +__EOHEADER__ + +def output_copyright_notice(outfile, encoding) + outfile.printf $header, encoding, encoding.upcase, Time.new.year +end + +$uniconvmacro1 = <<__EOUNIMAC1__ +% macros adapted from ConTeXt MKII; see unic-ini.mkii +\\def\\unicodechar#1{% + \\ifcsname unichar@\\number#1\\endcsname + \\csname unichar@\\number#1\\endcsname + \\else + \\errmessage{Unicode character [#1] not in encoding.}% + \\fi} +__EOUNIMAC1__ + +$uniconvmacros = [nil, nil] + +$uniconvmacros << <<__EOTWOBYTES__ +\\def\\utftwouniglyph#1#2% + {\\expandafter\\unicodechar\\expandafter + {\\the\\numexpr64*(#1-192)+`#2-128\\relax}} +__EOTWOBYTES__ + +$uniconvmacros << <<__EOTHREEBYTES__ +\\def\\utfthreeuniglyph#1#2#3% + {\\expandafter\\unicodechar\\expandafter + {\\the\\numexpr4096*(#1-224)+64*(`#2-128)+`#3-128\\relax}} +__EOTHREEBYTES__ + +$uniconvmacros << <<__EOFOURBYTES__ +\\def\\utffouruniglyph#1#2#3#4% + {\\expandafter\\unicodechar\\expandafter + {\\the\\numexpr262144*(#1-240)+4096*(`#2-128)+64*(`#3-128)+`#4-128\\relax}} +__EOFOURBYTES__ + +$uniconvmacro2 = <<__EOUNIMAC2__ + +\\def\\addunichar #1 #2 {\\expandafter\\def\\csname unichar@\\number#1\\endcsname{#2}} + +% \\addunichar "unicode_code - ^^font_encoding_code +__EOUNIMAC2__ + +["t8m", "lth"].each do |encoding| + # load encoding + e = HyphEncoding.new(encoding) + + # open file + File.open(output_file_name(encoding), "w") do |file_out| + + # copyright notice + output_copyright_notice(file_out, encoding) + file_out.puts + + # macro to get mapping unicode -> font encoding & error message if screwed up + file_out.puts $uniconvmacro1 + + # minimal and maximal length of characters in the encoding (until now just 2 & 3) + unicode_characters_array = e.unicode_characters.sort + length_min = unicode_characters_array.first[1].bytes.size + length_max = unicode_characters_array.last[1].bytes.size + + # only output the necessary macros for transforming UTF-8 -> Unicode number + (length_min..length_max).each do |nbytes| + file_out.puts $uniconvmacros[nbytes] + end + + # macro to store mapping unicode -> font encoding + file_out.puts $uniconvmacro2 + + # all unicode characters in the encoding + e.unicode_characters.sort.each do |code,c| + file_out.printf("\\addunichar \"%04X ^^%02x \\lccode\"%02X=\"%02X %% %s - %s\n", + c.code_uni, c.code_enc, c.code_enc, c.code_enc, [c.code_uni].pack('U'), c.name) + end + file_out.puts + + # make all the possible first characters active + # output the definition into file + e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| + byte = first_byte_code.hex + size = chars[0].bytes.size + # 2-byte: 0b11000000 <= byte < 0b11100000 + str = case size + when 2 then + "two" + # 3-byte: 0b11100000 <= byte < 0b11110000 + when 3 then + "three" + # 4-byte: 0b11110000 <= byte < 0b11111000 + when 4 then + "four" + end + file_out.printf("\\catcode\"%02X=\\active \\def^^%02x{\\utf%suniglyph{\"%02X}}\n", byte, byte, str, byte) + end + end +end + +["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"].each do |encoding| + # load encoding + e = HyphEncoding.new(encoding) + + # open file + File.open(output_file_name(encoding), "w") do |file_out| + + # copyright notice + output_copyright_notice(file_out, encoding) + file_out.puts '%' + + e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| + # sorting all the second characters alphabetically + chars.sort!{|x,y| x.code_uni <=> y.code_uni } + # make all the possible first characters active + # output the definition into file + file_out.printf("\\catcode\"%02X=\\active\n", first_byte_code.hex) + end + file_out.puts "%" + e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| + first_byte_code = first_byte_code.hex + size = chars[0].bytes.size + if size != 2 then + throw "The encoding #{encoding} uses more than two bytes to encode characters" + else + + file_out.printf("\\def^^%02x#1{%%\n", first_byte_code) + string_fi = "" + for i in 1..(chars.size) + uni_character = chars[i-1] + enc_byte = uni_character.code_enc + enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2') + file_out.printf("\t\\ifx#1^^%02x^^%02x\\else %% %s - U+%04X - %s\n", uni_character.bytes[1], uni_character.code_enc, [uni_character.code_uni].pack('U'), uni_character.code_uni, uni_character.name) + string_fi = string_fi + "\\fi" + end + + # at least three bytes + end + file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}" + file_out.puts string_fi + "}" + end + file_out.puts '%' + file_out.puts '% ensure all the chars above have valid \lccode values' + file_out.puts '%' + e.lowercase_characters.each do |character| + code = [ character.code_enc ].pack("c").unpack("H2").first.upcase + # \lccode"FF="FF + file_out.printf "\\lccode\"%s=\"%s %% %s - U+%04X - %s\n", code, code, [character.code_uni].pack('U'), character.code_uni, character.name + end + + file_out.puts + end +end + + diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb new file mode 100755 index 0000000000..c0781932a3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-pattern-loaders.rb @@ -0,0 +1,122 @@ +#!/usr/bin/env ruby +# encoding: utf-8 + +# this file auto-generates loaders for hyphenation patterns - to be improved # No shit, Sherlock -- AR 2018-11-27 + +require_relative 'lib/tex/hyphen' +require_relative 'lib/tex/hyphen/texlive' +include TeX::Hyphen +include TeXLive + +#text_if_native_utf = "\input pattern-loader.tex\n\\ifNativeUtfEightPatterns" + +def output(file, string, indent = 0) + if string.is_a? Enumerable + string.each { |line| output(file, line, indent) } + else + file.print ' ' * indent + file.puts(string) + end +end + +print 'Generating loaders for ' +Language.all.each do |language| + +# puts language.bcp47 + +################ +# Header texts # +################ + +# a message about auto-generation +# TODO: write a more comprehensive one + text_header = <<-EOHEADER +% filename: loadhyph-#{language.bcp47}.tex +% language: #{language.babelname} +% +% Loader for hyphenation patterns, generated by +% source/generic/hyph-utf8/generate-pattern-loaders.rb +% See also http://tug.org/tex-hyphen +% +% Copyright 2008-#{Time.now.year} TeX Users Group. +% You may freely use, modify and/or distribute this file. +% (But consider adapting the scripts if you need modifications.) +% +% Once it turns out that more than a simple definition is needed, +% these lines may be moved to a separate file. +% + EOHEADER + +########### +# lccodes # +########### + + lccodes_common = [] + if language.has_apostrophes? then + lccodes_common.push("\\lccode`\\'=`\\'") + end + if language.has_hyphens? then + lccodes_common.push("\\lccode`\\-=`\\-") + end + + next if language.use_old_loader + print language.bcp47, ' ' + + filename = File.join(PATH::LOADER, language.loadhyph) + File.open(filename, "w") do |file| + # puts language.bcp47 + file.puts text_header + file.puts('\begingroup') + + if lccodes_common.length > 0 then + file.puts lccodes_common.join("\n") + end + + # for ASCII encoding, we don't load any special support files, but simply load everything + if language.encoding == 'ascii' && !language.italic? + file.puts "% ASCII patterns - no additional support is needed" + file.puts "\\message{ASCII #{language.message}}" + file.puts "\\input hyph-#{language.bcp47}.tex" + else + file.puts '% Test for pTeX +\\ifx\\kanjiskip\\undefined +% Test for native UTF-8 (which gets only a single argument) +% That\'s Tau (as in Taco or ΤΕΧ, Tau-Epsilon-Chi), a 2-byte UTF-8 character +\\def\\testengine#1#2!{\\def\\secondarg{#2}}\\testengine Τ!\\relax +\\ifx\\secondarg\\empty' + output(file, language.format_inputs(language.utf8_chunk), 2) + file.puts("\\else\n") + output(file, language.format_inputs(language.nonutf8_chunk('8-bit')), 2) + file.puts("\\fi\\else\n") + output(file, language.format_inputs(language.nonutf8_chunk('pTeX')), 2) + file.puts("\\fi\n") + end + +######################################## +# GROUP nr. 1 - ONLY USABLE WITH UTF-8 # +######################################## + # some special cases first + # + # some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines + # TODO - maybe consider doing something similar for ibycus + +####################### +# GROUP nr. 2 - ASCII # +####################### + +#################################### +# GROUP nr. 3 - different patterns # +#################################### + # when lanugage uses old patterns for 8-bit engines, load two different patterns rather than using the converter + # greek, coptic +######################### +# GROUP nr. 4 - regular # +######################### +####### +# end # +####### + file.puts('\endgroup') + end +end + +puts diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb new file mode 100755 index 0000000000..eaa9cbf608 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb @@ -0,0 +1,68 @@ +#!/usr/bin/env ruby +# encoding: utf-8 + +# this file generates plain patterns (one-per-line) out of TeX source + +# use 'gem install unicode' if unicode is missing on your computer +require 'unicode' + +require_relative 'lib/tex/hyphen/language.rb' +# include OldLanguage::TeXLive +include TeX::Hyphen + +# FIXME Close files! +# FIXME sr-cyrl? +print 'Generating plain files for (parenthesised tags are skipped) ' +Language.all.sort.each do |language| + bcp47 = language.bcp47 + + if language.use_old_loader || bcp47 == 'mn-cyrl-x-lmc' + print '(', language.bcp47, ') ' + next + else + print bcp47, ' ' + end + + outfile = Proc.new do |ext| + File.open File.join(PATH::TXT, sprintf('hyph-%s.%s.txt', bcp47, ext)), 'w' + end + + # patterns + patterns = language.extract_apostrophes + file = outfile.('pat') + patterns[:plain].each do |pattern| + file.puts pattern + end + + # apostrophes if applicable + with_apostrophe = patterns[:with_apostrophe] + if with_apostrophe + file = File.open File.join(PATH::QUOTE, sprintf('hyph-quote-%s.tex', bcp47)), 'w' + file.printf "\\bgroup\n\\lccode`\\’=`\\’\n\\patterns{\n" + with_apostrophe.each do |pattern| + file.printf "%s\n", pattern + end + file.puts "}\n\\egroup\n" + end + + # exceptions + unless language.exceptions == "" + file = outfile.('hyp') # This ensure a file is created, even if it may be empty + file.puts language.exceptions + end + + ### FIXME Do issue #33 instead + ### # characters + ### file = outfile.('chr') + ### language.extract_characters.each do |character| + ### file.puts character + ### end + + ### FIXME Do something else instead + ### # comments and licence + ### file = outfile.('lic') + ### file.puts language.comments_and_licence + + ### file.close +end +puts diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb new file mode 100755 index 0000000000..0f50706dc0 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-ptex-patterns.rb @@ -0,0 +1,82 @@ +#!/usr/bin/env ruby + +require_relative 'hyph-utf8' +require_relative 'lib/tex/hyphen' +require_relative 'lib/tex/hyphen/texlive' +include TeX::Hyphen +include TeXLive + +# this file generates patterns for pTeX out of the plain ones + +# load encodings +encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"] +encodings = Hash.new +encodings_list.each do |encoding_name| + encodings[encoding_name] = HyphEncoding.new(encoding_name) +end + +header = <<-HEADER +%% pTeX-friendly hyphenation patterns +%% +%% language: %s (%s) +%% encoding: %s +%% +%% This file has been auto-generated from hyph-%s.tex +%% with a script [texmf]/scripts/generic/hyph-utf8/generate-ptex-patterns.rb +%% See the original file for details about author, licence etc. +%% +HEADER + +print 'Generating pTeX patterns for (skipped # reason): ' +Language.all.sort.each do |language| + if language.use_old_patterns_comment || !language.encoding || language.encoding == 'ascii' + if language.use_old_patterns_comment + reason = 'legacy patterns' + else + reason = sprintf 'encoding: %s', language.encoding.inspect + end + print '(', language.bcp47, ' # ', reason, ') ' + + next + else + encoding = encodings[language.encoding] + end + + bcp47 = language.bcp47 + + print bcp47, ' ' + File.open(File.join(PATH::PTEX, sprintf('hyph-%s.%s.tex', bcp47, language.encoding)), 'w') do |file_ptex| + patterns = language.patterns + exceptions = language.exceptions + + characters = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq + + if language.encoding != 'ascii' then + patterns = encoding.convert_to_escaped_characters(patterns) + exceptions = encoding.convert_to_escaped_characters(exceptions) + end + + file_ptex.printf(header, language.babelname, language.bcp47, language.encoding, language.bcp47) + + file_ptex.puts("\\bgroup") + # setting lccodes for letters + characters.each do |c| + if (c == 0x01FD or c == 0x0301) and language.bcp47 == 'la-x-liturgic' + # skip + elsif c >= 128 then + code = encoding.unicode_characters[c].code_enc + file_ptex.printf("\\lccode\"%02X=\"%02X\n", code, code) + end + end + # patterns + if patterns.length > 0 then + file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}") + end + # exceptions + if exceptions.length > 0 then + file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}") + end + file_ptex.puts("\\egroup") + end +end +puts diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb new file mode 100755 index 0000000000..7460658f02 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb @@ -0,0 +1,69 @@ +#!/usr/bin/env ruby +# encoding: utf-8 + +# this file auto-generates tlpsrc files for hyphenation patterns - to be improved + +require_relative 'lib/tex/hyphen' +require_relative 'lib/tex/hyphen/texlive' +include TeX::Hyphen +include TeXLive + +#--------# +# TLPSRC # +#--------# + +print 'Generating .tlpsrc files for packages: ' +Package.all.sort.each do |package| + tlpsrcname = File.join(PATH::TLPSRC, sprintf('hyphen-%s.tlpsrc', package.name.safe)) + file_tlpsrc = File.open(tlpsrcname, 'w') + print package.name.safe, ' ' + + file_tlpsrc.puts "category TLCore" + package.list_dependencies.each do |dependency| + file_tlpsrc.puts dependency + end + + # FIXME Still doesn’t work well for Latin + file_tlpsrc.printf "shortdesc %s.\n", package.description_s + package.description.split("\n").each do |line| + file_tlpsrc.printf "longdesc %s\n", line + end + package.languages.each do |language| + file_tlpsrc.printf "execute AddHyphen \\\n\tname=%s%s \\\n", language.babelname, language.list_synonyms + file_tlpsrc.printf "\t%s \\\n\t%s", language.list_hyphenmins, language.list_loader + if language.patterns_line + language.exceptions_line != "" + file_tlpsrc.printf " \\\n\t%s \\\n\t%s", language.patterns_line, language.exceptions_line + end + if language.bcp47 == "mn-cyrl-x-lmc" then + file_tlpsrc.printf " \\\n\tluaspecial=\"disabled:only for 8bit montex with lmc encoding\"" + end + # end-of-line + file_tlpsrc.puts + end + + # documentation + package.list_support_files('doc').sort.each do |filename| + file_tlpsrc.printf "docpattern d texmf-dist/%s\n", filename + end + + # sources + package.list_support_files('source').sort.each do |filename| + file_tlpsrc.printf "srcpattern d texmf-dist/%s\n", filename + end + + package.list_run_files.sort.uniq.each do |filename| + file_tlpsrc.printf "runpattern f texmf-dist/%s\n", filename + end + file_tlpsrc.close +end +puts + +#--------------------------# +# language.dat and friends # +#--------------------------# +system sprintf "tlmgr generate --dest %s language.dat", File.join(PATH::LANGUAGE_DAT, 'language.dat') +system sprintf "tlmgr generate --dest %s language.dat.lua", File.join(PATH::LANGUAGE_DAT, 'language.dat.lua') + +# Revert changes if only line with date was changed +diff_lines = `git diff #{PATH::LANGUAGE_DAT}/language.dat* | egrep '^[+-](%|--) Generated' | wc -l`.strip +`git checkout HEAD -- #{PATH::LANGUAGE_DAT}/language.dat*` if diff_lines == '4' diff --git a/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb b/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb new file mode 100644 index 0000000000..60005af380 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb @@ -0,0 +1,116 @@ +# this is a Unicode character represented in some particular encoding +class UnicodeCharacter + # unicode code + # code in that particular encoding + # character name (like 'eacute') + def initialize(code_uni, code_enc, name) + @code_uni = code_uni + @code_enc = code_enc + @bytes = [code_uni].pack('U').unpack('C*') + @name = name + end + + attr_reader :code_uni, :code_enc, :bytes, :name +end + +class UnicodeCharacters < Hash + # a hash based on the first character + def add_new_character_first_byte(code_uni, code_enc, name) + first_byte = [code_uni].pack('U').unpack('H2').first + if self[first_byte] == nil then + self[first_byte] = Array.new + end + self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name)) + end + # a hash based on the whole unicode codepoint + def add_new_character(code_uni, code_enc, name) + self[code_uni] = UnicodeCharacter.new(code_uni, code_enc, name) + end +end + +class HyphEncoding + def initialize(encoding_name) + @encoding_name = encoding_name + @unicode_characters_first_byte = UnicodeCharacters.new + @unicode_characters = UnicodeCharacters.new + @lowercase_characters = Array.new + + if encoding_name != 'ascii' then + read_data + end + end + + def convert_to_escaped_characters(str) + if str.kind_of?(Array) then + str.each_index do |i| + str[i] = convert_string_to_escaped_characters(str[i]) + end + elsif str.kind_of?(String) then + str = convert_string_to_escaped_characters(str) + end + return str + end + + attr_reader :encoding_name, :unicode_characters, :unicode_characters_first_byte, :lowercase_characters + + def convert_string_to_escaped_characters(str) + skip_this_string = false + characters = str.unpack('U*') + new_string = Array.new(characters.length) + characters.each_index do |i| + c = characters[i] # character code on position i + # check if unicode entry with that number exists + uc = @unicode_characters[c] + if uc == nil then + if c < 128 then + new_string[i] = [c].pack('U') + elsif c == 8217 # ’ + new_string[i] = "'" + elsif (c == 0x01FD or c == 0x0301) and @encoding_name == 'ec' + skip_this_string = true + new_string[i] = sprintf("[U+%04X]", c) + else + puts sprintf("There must be an error: character U+%04X in string '%s' is not ASCII or %s.", c, str, @encoding_name.upcase) + end + # an unicode character + else + new_string[i] = sprintf("^^%x", uc.code_enc) + end + end + if skip_this_string + new_string.unshift("% ") + end + return new_string.join('') + end + +private + def read_data + # fetch the characters + encoding_data_dir = File.expand_path("../data/encodings", __FILE__) + filename = "#{encoding_data_dir}/#{@encoding_name}.dat" + + if File.exists?(filename) then + File.open(filename).grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z0-9\.]*)$/) do |line| + # puts line + code_enc = $1.hex + code_uni = $2.hex + if $3.length > 0 + type = $3.to_i + else + type = 0 + end + name = $4 + if type == 1 then + @unicode_characters_first_byte.add_new_character_first_byte(code_uni, code_enc, name) + @unicode_characters.add_new_character(code_uni, code_enc, name) + @lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name)) + end + @lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc} + end + else + # TODO: throw an error + puts "Invalid encoding name '#{@encoding_name}'." + puts "File '#{filename}' doesn't exist." + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb b/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb new file mode 100644 index 0000000000..d4f387c072 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb @@ -0,0 +1,327 @@ +class OldLanguage +# "use_new_loader" +# => true - create a new file and use that one +# => false - use "filename_old_patterns" in language.dat +# "filename_old_patterns" +# => [string] - the name used in language.dat if "use_new_loader" is false +# "eightbitfilename" +# => [string] - if set, load another file for 8-bit engines +# "code" +# => [string] - used in filenames, needs to conform to the standard +# "name" +# => [string] - +# "synonyms" => [], +# "hyphenmin" => [], +# "encoding" => nil, +# "exceptions" => false, +# "message" => nil, + + @@language_data = [ +{ + "code" => "el-monoton", + "type" => "rules", +}, +{ + "code" => "el-polyton", + "type" => "rules", +}, +{ + "code" => "grc", + "type" => "rules", +}, +{ + "code" => "cop", + "type" => "rules", +}, +{ + "code" => "de-1901", + "type" => "dictionary", +}, +{ + "code" => "de-1996", + "type" => "dictionary", +}, +{ + "code" => "de-ch-1901", + "type" => "dictionary", +}, +{ + "code" => "ru", + "type" => "dictionary", +}, +{ + "code" => "uk", + # "type" => "rules", # TODO: it is not really clear +}, +{ + "code" => "af", + "type" => "dictionary", +}, +{ + "code" => "ca", + "type" => "rules", # not only rules, also patgen, but it is a good approximation +}, +{ + "code" => "cs", + "filename_old_patterns_other" => ["czhyphen.tex","czhyphen.ex"], + "type" => "dictionary", + # Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 + # Typographical rules allow \righthyphenmin=2 when typesetting in a + # narrow column (newspapers etc.). + # (used to be 2,2) + + # guessing based on CTAN/macros/cstex/base/csplain.tar.gz: + # 1998-12-17 (patterns) + # 1995-08-23 (exceptions) + # but patterns claim 1995 +}, +{ + "code" => "sk", + "type" => "dictionary", + "filename_old_patterns_other" => ["skhyphen.tex","skhyphen.ex"], +}, +{ + "code" => "cy", + "type" => "dictionary", +}, +{ + "code" => "da", + "type" => "dictionary", + "filename_old_patterns_other" => ["dkcommon.tex", "dkspecial.tex"], +}, +{ + "code" => "eo", + "type" => "rules", +}, +{ + "code" => "es", + "type" => "dictionary", +}, +{ + "code" => "eu", + "type" => "rules", +}, +{ + "code" => "fr", + "type" => "rules", +}, +{ + "code" => "gl", + "type" => "rules", +}, +{ + "code" => "et", + "type" => "dictionary", +}, +{ + "code" => "fi", + "type" => "rules", +}, +{ + "code" => "hr", + "type" => "dictionary", +}, +{ + "code" => "hu", + "type" => "dictionary", + "source" => "https://github.com/nagybence/huhyphn/", # FIXME Check and remove from desc below +}, +{ + "code" => "hy", + "type" => "rules", +}, +{ + "code" => "ia", + "type" => "dictionary", +}, +{ + "code" => "id", + "type" => "rules", +}, +{ + "code" => "is", + "type" => "dictionary", + # TODO: I'm not sure that the last two names are relevant, I don't find the source of Marteinn Sverrisson + "authors" => ["jorgen_pind", "marteinn_sverrisson", "kristinn_gylfason"], +}, +{ + "code" => "ga", + "type" => "dictionary", +}, +{ + "code" => "it", + "type" => "rules", # TODO: we might want to check that, but it seems unlikely that patgen was used +}, +{ + "code" => "rm", + "type" => "rules", +}, +{ + "code" => "fur", + "type" => "rules", +}, +{ + "code" => "pms", + "type" => "rules", +}, +{ + "code" => "kmr", + "type" => "dictionary", +}, +{ + "code" => "la", + "type" => "rules", +}, +{ + "code" => "la-x-classic", + "type" => "rules", +}, +{ + "code" => "la-x-liturgic", + "version" => "1.040", + "last_modified" => "2016-05-19", + "type" => "rules", +}, +{ + "code" => "lt", + "description_l" => [ + #......................................................................# + "Hyphenation patterns for Lithuanian in L7X and UTF-8 encodings.", + # "Designed for \\lefthyphenmin and \\righthyphenmin set to 2.", + "\\lefthyphenmin and \\righthyphenmin have to be at least 2.", + # "Changing them to 1 according to grammatical rules from 1997", + # "would require to review and maybe rebuild the patterns." + ], +}, +{ + "code" => "lv", + "type" => "dictionary", +}, +{ + "code" => "nl", + "type" => "dictionary", + # quoting Hans Hagen: + # patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) +}, +{ + "code" => "oc", + "type" => "rules", +}, +{ + "code" => "pl", + "type" => "dictionary", + #{}"hyphenmin" => [1,1], +}, +{ + "code" => "pt", + "type" => "rules", # TODO: we could create a generating script +}, +{ + "code" => "zh-latn-pinyin", + "type" => "rules", # TODO: we could create a generating script +}, +{ + "code" => "ro", + "type" => "dictionary", +}, +{ + "code" => "sl", + "type" => "dictionary", +}, +{ + "code" => "hsb", + "type" => "dictionary", +}, +{ + "code" => "sv", + "type" => "dictionary", +}, +{ + "code" => "tk", + "type" => "dictionary", +}, +{ + "code" => "tr", + "type" => "rules", +}, +{ + "code" => "en-gb", + "type" => "dictionary", +}, +{ + "code" => "en-us", +}, +# TODO: FIXME!!! +{ + "code" => "sh-latn", + "type" => "dictionary", +}, +{ + "code" => "sh-cyrl", +}, +{ + "code" => "mn-cyrl", + "type" => "dictionary", + "last_modified" => "2010-04-03", # FIXME: ? +}, +{ + "code" => "sa", + "type" => "rules", +}, +{ + "code" => "nb", + "type" => "dictionary", + "last_modified" => "2012-05-18", # FIXME: ? +}, +{ + "code" => "as", + # this is true for all Indic patterns + "type" => "rules", +}, +# thai +{ + "code" => "th", + "type" => "dictionary", + "last_modified" => "2015-05-07", # FIXME: ? +}, +# lao +#{ +# "code" => "lo", +# "name" => "lao", +# "use_new_loader" => true, +# "use_old_patterns" => false, +# "hyphenmin" => [1,1], # TODO +# "encoding" => nil, # no patterns for 8-bit engines +# "exceptions" => false, +# "message" => "Lao hyphenation patterns", +# +# "version" => nil, +# "last_modified" => "2010-05-19", +# "type" => "rules", +# "authors" => [ "brian_wilson", "arthur_reutenauer", "mojca_miklavec" ], +# "licence" => "other-free", +# "description_s" => "Lao hyphenation patterns", +# "description_l" => [ +# #......................................................................# +# "Hyphenation patterns for Lao language for Unicode engines.", +# "Current version is experimental and gives bad results.", +# ], +#}, +{ + "code" => "ka", + "type" => "dictionary", +}, +{ + "code" => "cu", + "type" => "machine learning", +}, +{ + "code" => "be", + "type" => "rules", +}, +{ + "code" => "pi", + "type" => "rules", +}, + ] +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages.rb new file mode 100644 index 0000000000..6083339833 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages.rb @@ -0,0 +1,60 @@ +# -*- coding: utf-8 -*- +require_relative 'language-data' +require_relative '../../../../lib/tex/hyphen/language' + +class OldLanguage + def initialize(language_hash) + @use_old_patterns = language_hash["use_old_patterns"] + @use_old_patterns_comment = language_hash["use_old_patterns_comment"] + @use_old_loader = language_hash["use_old_loader"] + @filename_old_patterns = language_hash["filename_old_patterns"] + @filename_old_patterns_other = language_hash["filename_old_patterns_other"] + @code = language_hash["code"] + @name = language_hash["name"] + @synonyms = language_hash["synonyms"] + @hyphenmin = language_hash["hyphenmin"] + @encoding = language_hash["encoding"] + @message = language_hash["message"] + + @description_l = language_hash["description_l"] + @version = language_hash["version"] + + @licence = language_hash["licence"] + @authors = language_hash["authors"] + + @synonyms = [] unless @synonyms + end + + def <=>(other) + code <=> other.code + end + + def self.all + @@languages ||= @@language_data.map do |language_data| + next if language_data['code'] == 'sr-cyrl' + new language_data + end.compact + end + + # TODO self.find + + # def lc_characters + # if @lc_characters == nil + # lc_characters = Hash.new + # p = self.patterns + # p.each do |pattern| + # end + # end + # return @lc_characters + # end + + attr_reader :use_old_loader, :use_old_patterns_comment, :filename_old_patterns + attr_reader :code, :name, :message + attr_reader :description_l + + # Convenience methods related to TeX Live and the .tlpsrc files + module TeXLive + end + + include TeXLive +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/es/README b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/README new file mode 100644 index 0000000000..9529767ad4 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/README @@ -0,0 +1,83 @@ +DIVISI'ON DE PALABRAS +~~~~~~~~~~~~~~~~~~~~~ + +The file eshyph.tex under /base is the base for the Spanish patters. +It is intended mainly for backward compatibility. New systems are +best based on CTAN:language/hyph-utf8/, which has the same patterns. + +Why 4.x? Well, I know at least other three files with the same name, +so this one is the fourth (there were at least 6 or 7 patterns files). +The others should vanish as soon as posible. + +(c) Javier Bezos 1993 1997. +(c) Javier Bezos and CervanTeX 2001-2009 +Some parts, (c) by Francesc Carmona +Licence: LPPL + +- division.pdf is a draft of an article (in Spanish) explaining the +rules to be applied and how they are being translated into TeX in a +unified set of patterns (somewhat outdated). +- eshyph-make.lua generates the patterns, with eshyph.src for prefixes +and special cases. +- eshyph-test.tex makes a comparison with strict syllabic rules. It +requires a file spanish-words.txt (not supplied) with a list of word, +one per line. You can (should) filter the words. + +For bug reports and comments: + + http://www.tex-tipografia.com/spanish_hyphen.html + +I would like to thanks Francesc Carmona for his permission +to steal parts of his work without restrictions. + +The contrib directory, as its name implies, is not part of the +official bundle. It has a different set of patterns, but it does not +follow the rules by the Spanish Academy (despite its claims). + +What's new in 4.5 (2009-08-01) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Missing group -gl- added (a major bug). A few minor improvements +for "des-" and "in-". + +What's new in 4.4 (2009-05-19) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some bad patterns fixed. + +What's new in 4.3 (2009-05-14) +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Some patterns fixed, particularly for "familia", "superar", "sub-" and +final consonants. Patterns are now generated with the help of a few +lua/luatex files (whose status is alpha). + +What's new in 4.2 +~~~~~~~~~~~~~~~~~ + +The encoding is UTF-8, so that it can be used with LuaTeX. Yet, it +works without changes with standard TeX. + +What's new in 4.1 +~~~~~~~~~~~~~~~~~ + +- Thanks to a list of about 750000 words and extensive +tests, wrong hyphenations have been reduced dramatically +and the number of patterns has been almost duplicated. + +- Since the Spanish Academy rules of 1999 are very vague, +incomplete and even contradictory, they had to be +completed with some traditional ones. The Academy has +published new rules in November 2005 including many of +the traditional ones employed here and therefore only +minor adjustments have been necessary. + +- Patterns for a few verbal endings are necessary, and +now the "voseo" forms (used in many countries of Central +and South America) have been included. + +2009-05-19 +_____________________________________________________________ +Javier Bezos | http://www.cervantex.es/ +............................................................. +TeX y tipografia | http://www.tex-tipografia.com/
\ No newline at end of file diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua new file mode 100644 index 0000000000..17ccdf2dc3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph-make.lua @@ -0,0 +1,136 @@ +-- (encoding:utf-8) + +-- GUIONIZADO DE PALABRAS +-- ~~~~~~~~~~~~~~~~~~~~~ +-- v 4.7 +-- +-- License: MIT/X11 +-- +-- Copyright (c) 1993, 1997 Javier Bezos +-- Copyright (c) 2001-2015 Javier Bezos and CervanTeX +-- +-- Permission is hereby granted, free of charge, to any person obtaining a copy +-- of this software and associated documentation files (the "Software"), to deal +-- in the Software without restriction, including without limitation the rights +-- to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +-- copies of the Software, and to permit persons to whom the Software is +-- furnished to do so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in +-- all copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. +-- +-- For further info, bug reports and comments: +-- +-- http://www.tex-tipografia.com/spanish_hyphen.html + +---------- + +-- ethymologic, trailing no, bad words, prefixes, consonant + h, tl + +patfile = io.open('eshyph.tex', 'w') +patfile:write('\\patterns{\n') + +-- Basic patters +-- Using the characters iterator in luatex + +digraphs = 'ch ll' +liquids = 'bl cl fl gl kl pl vl br cr dr fr gr kr pr rr tr vr' +avoid = 'tl ts tx tz' +silent = 'h' +chrl = 'chr chl' +letters = 'bcdfghjklmnpqrstvwxyz' + +for n in letters:gmatch('.') do + if silent:find(n) then + patfile:write(' 4' .. n .. '.') + else + patfile:write('1' .. n .. ' 4' .. n .. '. .' .. n .. '2') + end + for m in letters:gmatch('.') do + pat = n .. m + if digraphs:find(pat) then + patfile:write(' ') + elseif liquids:find(pat) then + patfile:write(' ') + elseif avoid:find(pat) then + patfile:write(' ') + elseif silent:find(m) then + patfile:write(' 2' .. n .. '1' .. m) + else + patfile:write(' 2' .. pat) + end + end + patfile:write('\n') +end + +patfile:write('1ñ 4ñ.\n') + +for n in digraphs:gmatch('%S+') do + patfile:write(n:sub(1,1) .. '4' .. n:sub(2,2) .. ' 4' .. n .. '.') + for m in letters:gmatch('.') do + pat = n .. m + if chrl:find(pat) then + patfile:write(' ' .. n .. '2' .. m) + else + patfile:write(' 2' .. pat) + end + end + patfile:write('\n') +end + +for n in liquids:gmatch('%S+') do + patfile:write(n:sub(1,1) .. '2' .. n:sub(2,2) .. ' 4' .. n .. '.') + for m in letters:gmatch('.') do + patfile:write(' 2' .. n .. '2' .. m) + end + patfile:write('\n') +end + +letters = 'bcdlmnrstxy' +etim = 'pt ct cn ps mn gn ft pn cz tz ts' + +for n in etim:gmatch('%S+') do + for m in letters:gmatch('.') do + patfile:write('2' .. m .. '3' .. n:sub(1,1) .. '2' .. n:sub(2,2) .. ' ') + end + patfile:write('4' .. n .. '.\n') +end + +src = io.open('eshyph.src') + +function prefix(p) + if p:match('r$') then + p = p:sub(1,-2) .. '2' .. p:sub(-1) .. '1' + patfile:write(p:sub(1,-2) .. '3r\n') + elseif p:match('[aeiou]$') then + p = p .. '1' + patfile:write(p .. 'h\n') + end + patfile:write(p .. 'a2 ' .. p .. 'e2 ' .. p .. 'i2 ' .. p .. 'o2 ' .. p .. 'u2\n') + patfile:write(p .. 'á2 ' .. p .. 'é2 ' .. p .. 'í2 ' .. p .. 'ó2 ' .. p .. 'ú2\n') + +end + +for ln in src:lines() do + ln = ln:match('[^%%]*') + for p in ln:gmatch('%S+') do + if p:match('/(.*)/') then + prefix(p:match('/(.*)/')) + elseif p:sub(1,1) == '*' then + patfile:write('de2s3' .. p:sub(2) .. '\n') + else + patfile:write(p .. '\n') + end + end +end + +patfile:write('}') +patfile:close() diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src new file mode 100644 index 0000000000..bf78e9ec71 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/es/eshyph.src @@ -0,0 +1,948 @@ +% v 4.7 License MIT/X11. See eshyph-make.lua +san4c5t +plan4c5t +% +o1eu2 +a1ae2 +a1aó2 +eo1á2 +eo1e2 +o1ae2 +o1eó2 +% +2no. +2t2l +2t2s +2t2x +2t2z +tz3s4ch +4caca4 % caca-huete +4cago4 4caga4 4cagas. % Chi-cago +4teta. 4tetas. % es-teta +4puta4 4puto4 % com-putar y derivados, reputa-ción +.hu4mea .hu4meo .he4mee +4meo. % Tolo-meo +4meable. 4meables. % imper-meable +4pedo4 % bí-pedo % tubér-culo % +4culo4 +5mente. +% -aer aero deca +% ieron etc. no en aer. +% iera iese no en aer +% no en aer (el acento!) (contrá)eme ete ele elo ela enos elos +4i3go. 4es. 4és 4e. 4e3mos. 4éis. 4en. +4ía. 4ías. 4ía3mos. 4íais. 4ían. +4í. 4í4s3te. 4í4s3tes. 4í3tes. 4í3mos. 4ís3teis. +4e3ré. 4e3rás. 4e3rés. 4e3rís. 4e3rá. 4e3re3mos. 4e3réis. 4e3rán. +4i3ga. 4i3gas. 4i3gás. 4i3gamos. 4i3gáis. 4a4i3gan. +4e3ría. 4e3rías. 4e3ríamos. 4e3ríais. 4e3rían. +% +4i3gá3mosme. +4i3gá3mosmele. 4i3gá3mosmelo. 4i3gá3mos3mela. +4i3gá3mosmeles. 4i3gá3mosmelos. 4i3gá3mos3melas. +4i3gá3moste. +4i3gá3mostele. 4i3gá3mostelo. 4i3gá3mos3tela. +4i3gá3mosteles. 4i3gá3mostelos. 4i3gá3mos3telas. +4i3gá3mosle. 4i3gá3mosla. 4i3gá3moslo. +4i3gá3mosele. 4i3gá3moselo. 4i3gá3mosela. +4i3gá3moseles. 4i3gá3moselos. 4i3gá3moselas. +4i3gá3monos. +4i3gá3monosle. 4i3gá3monoslo. 4i3gá3monosla. +4i3gá3monosles. 4i3gá3monoslos. 4i3gá3monoslas. +4i3gá3moos. +4i3gá3moosle. 4i3gá3mooslo. 4i3gá3moosla. +4i3gá3moosles. 4i3gá3mooslos. 4i3gá3mooslas. +4i3gá3mosles. 4i3gá3moslas. 4i3gá3moslos. +4ed. 4é. +4edme. +4édmele. 4édmelo. 4éd3mela. +4édmeles. 4édmelos. 4éd3melas. +4edte. +4édtele. 4édtelo. 4éd3tela. +4édteles. 4édtelos. 4éd3telas. +4edle. 4eedla. 4edlo. +4édsele. 4édselo. 4édsela. +4édseles. 4édselos. 4édselas. +4ednos. +4édnosle. 4édnoslo. 4édnosla. +4édnosles. 4édnoslos. 4édnoslas. +4eos. +4éosle. 4éoslo. 4éosla. +4éosles. 4éoslos. 4éoslas. +4edles. 4edlas. 4edlos. +4er. +4erme. +4érmele. 4érmelo. 4ér3mela. +4érmeles. 4érmelos. 4ér3melas. +4erte. +4értele. 4értelo. 4ér3tela. +4érteles. 4értelos. 4ér3telas. +4erle. 4erla. 4erlo. +4erse. +4érsele. 4érselo. 4érsela. +4érseles. 4érselos. 4érselas. +4ernos. +4érnosle. 4érnoslo. 4érnosla. +4érnosles. 4érnoslos. 4érnoslas. +4e3ros. +4é3rosle. 4é3roslo. 4é3rosla. +4é3rosles. 4é3roslos. 4é3roslas. +4erles. 4erlas. 4erlos. +4í3do. 4í3da. 4í3dos. 4í3das. +%-ear: norte, ante, super +4o. 4as. 4a. 4ás. 4a3mos. 4áis. 4an. +4aste. 4astes. 4ó. 4ates. 4asteis. 4a3ron. +4a3ba. 4a3bas. 4á3bamos. 4a3bais. 4a3ban. +4a3ría. 4a3rías. 4a3ríamos. 4a3ríais 4a3rían. +4a3ré. 4a3rás. 4a3rés. 4a3rís. 4a3rá. 4a3remos. 4a3réis. 4a3rán. +4a3ra. 4a3ras. 4á3ramos. 4a3rais. 4a3ran. +4a3re. 4a3res. 4á3remos. 4a3reis. 4a3ren. +4a3se. 4a3ses. 4á3semos. 4a3seis. 4a3sen. +4ad. +% +e5r4as. e5r4a3mos. e5r4áis. e5r4an. +e5r4aste. e5r4astes. e5r4ates. e5r4asteis. e5r4a3ron. +e5r4a3ba. e5r4a3bas. e5r4á3bamos. e5r4a3bais. e5r4a3ban. +e5r4a3ría. e5r4a3rías. e5r4a3ríamos. e5r4a3ríais e5r4a3rían. +e5r4a3ré. e5r4a3rás. e5r4a3rés. e5r4a3rís. e5r4a3rá. e5r4a3remos. e5r4a3réis. e5r4a3rán. +e5r4a3ra. e5r4a3ras. e5r4á3ramos. e5r4a3rais. e5r4a3ran. +e5r4a3re. e5r4a3res. e5r4á3remos. e5r4a3reis. e5r4a3ren. +e5r4a3se. e5r4a3ses. e5r4á3semos. e5r4a3seis. e5r4a3sen. +e5r4ad. +% +4adme. +4ádmele. 4ádmelo. 4ád3mela. +4ádmeles. 4ádmelos. 4ád3melas. +4adte. +4ádtele. 4ádtelo. 4ád3tela. +4ádteles. 4ádtelos. 4ád3telas. +4adle. 4eadla. 4adlo. +4ádsele. 4ádselo. 4ádsela. +4ádseles. 4ádselos. 4ádselas. +4adnos. +4ádnosle. 4ádnoslo. 4ádnosla. +4ádnosles. 4ádnoslos. 4ádnoslas. +4aos. +4áosle. 4áoslo. 4áosla. +4áosles. 4áoslos. 4áoslas. +4adles. 4adlas. 4adlos. +4ar. +4a4rme. +4á4rmele. 4á4rmelo. 4á4r3mela. +4á4r3meles. 4á4r3melos. 4á4r3melas. +4a4r3te. +4á4r3tele. 4á4r3telo. 4á4r3tela. +4á4r3teles. 4á4r3telos. 4á4r3telas. +4a4r3le. 4a4r3la. 4a4r3lo. +4a4r3se. +4á4r3sele. 4á4r3selo. 4á4r3sela. +4á4r3seles. 4á4r3selos. 4á4r3selas. +4a4r3nos. +4á4r3nosle. 4á4r3noslo. 4á4r3nosla. +4á4r3nosles. 4á4r3noslos. 4á4r3noslas. +4a3ros. +4árosle. 4ároslo. 4árosla. +4árosles. 4ároslos. 4ároslas. +4a4r3les. 4a4r3las. 4a4r3los. +% +4a3do. 4a3da. 4a3dos. 4a3das. +% +e5r4a3do. e5r4a3da. e5r4a3dos. e5r4a3das. +% +4ando +4ándole. 4ándolo. 4ándola. 4ándoles. 4ándolos. 4ándolas. +4ándonos. 4ándoos. +4ándome. 4ándomelo. 4ándomela. 4ándomele. +4ándomelos. 4ándomelas. 4ándomeles. +4ándote. 4ándoteme. +4ándotelo. 4ándotela. 4ándotele. +4ándotelos. 4ándotelas. 4ándoteles. +4ándotenos. +4ándose. 4ándoseme. +4ándoselo. 4ándosela. 4ándosele. +4ándoselos. 4ándoselas. 4ándoseles. +4ándosenos. +% +4a3dor. 4a3dora. 4a3dores. 4a3doras. +% +e5r4a3dor. e5r4a3dora. e5r4a3dores. e5r4a3doras. +% +/acto/ % (gal)acto, (l)acto +/aero/ +/afro/ +.a2 % a, an (prevent) +.an2a2 .an2e2 .an2i2 .an2o2 .an2u2 +.an2á2 .an2é2 .an2í2 .an2ó2 .an2ú2. +ana3lí +.aná3li +.ana3li +.an3aero +.an3e2pigr +.ane3xa .ane3xá .ane3xe .ane3xé .ane3xio .ane3xió +.an3h +.ani3mad .ani3mád +.ani3dar +.ani3ll +.ani3m +.aniña +.ani3q +.an3i2so .an3i2só +.ani3vel +.ano5che +.ano5din +.ano5mal +.ano5nad +.anó3nim +.anó5mal +.ano5nim +.ano5ta .ano3tá +.anua3l +.anua4lm +.anu3bl +.anu3da +.anu3l +asu3b2 +/anfi/ +/anglo/ +/ante/ +.ante2o3je +acante2 +4ísmo. 4ísmos. 4ísta. 4ístas. +4ístico. 4ísticos. 4ística. 4ísticas. +pante4o3n +.anteo3nes +mante4a +e4a3miento +e4ándo +tras3antea +/.anti/ +ti2o3qu ti2o3co +/archi/ +/auto/ +.bi1anual +.bi1aur +.bi1ox +.bi1ó2x +.bi1un +/biblio/ +bien2 bien3h bien3v bien3q bien3m bien3t +b4ien3do. .su3b4ien b4ien3das. +.bie4n3and +.bie4n3a4pa +.bie4n3a4ve +.bie4n3est +.bie4n3int +.bie4n3o4lie +/bio/ +bi1u2ní +/cardio/ +/cefalo/ +/centi/ +centi5área +/ciclo/ +o4i3dea. o4i3deas. o4i3dal. o4i3dales. +4o2i3de. 4o2i3des. 4i2dal. 4i2dales. +4i3deo. 4i3deos. +/cito/ +/cnico/ % (té)cnico +%?.co? +.co2a2 .co2e2 .co2i2 .co3o4 .co2u2 % Note coo +.co2á2 .co2é2 .co2í2 .co2ó2 .co2ú2 +.co3acree +.coá3gul +.co3agen +.coa3gul +.coa3lic +.co3arrend +.co3auto +.co3edic +.co3edit +.co3educ +.co3efici +.coe3tá +.co3exis +.co3imput +.coi3to +.co4o3per +.co4o3pér +.co4orden +.co4ordin +.co4ordín +.co4opt +.co4ópt +.co2nurb +/cripto/ +/crono/ +/contra/ % -aer = +/deca/ % deca, (en)deca, (do)deca, -aer +4e3dro. 4e3dros. 4é3drico. 4é3dricos. 4é3drica. +4é3dricas. +4í3ble. 4í3bles. +/.de2s/ +deca2i3mient +3sa. 3sas. +*órde +*orde +*abast +*aboll +*aboto +*abr +des4a3brid +*abroch +*aceit +*aceler +desa3cert +desa3ciert +*acobar +*acomod +*acomp +*acons +*acopl +*acorr +*acostum +*acot +desa3craliz +*acredit +*activ +*acuart +*aderez +*adeud +*adorar +*adormec +*adorn +*advert +*aferr +*afic +*afil +*afin +*afor +*agu +desa3gú +desa3garr +*agraci +*agrad +*agravi +*agreg +*agrup +des4a3guis +*aherr +*ahij +*ajust +*alagar +*alent +*alfom +*alfor +*alien +*a4line +*a4liné +*aliñ +desa3liv +*alm +*almid +*aloj +*alquil +*alter +*alumbr +desa3marr +desa3mobl +*amold +*amort +*amuebl +*ampa +*and +*angel +de3sangr +*anid +*anim +*aním +*anud +desa3pañ +desa3pacib +*apadr +*apare +*aparec +*aparic +*apeg +*apercib +*apes +*aplic +*apolill +*apoy +*aprend +*apret +*apriet +*aprob +*apropi +*aprovech +*arbol +*aren +*arm +des4arme +*arraig +*arregl +*arrend +*arrim +desa3rroll +*arrop +*arrug +*articul +*asent +*asist +*asn +% desa3soseg +% desa3sosieg +*atenc +*atend +*atiend +*atent +desa3tin +*atorn +*atranc +*autor +*avis +desa3yun +desa3zón +desa3zon +*embal +*embál +*embar +*embár +*embarg +*embols +*emborr +*embosc +*embot +*embrag +*embrág +*embrave +*embráve +*embroll +*embróll +*embruj +*embrúj +de3semej +*empañ +*empáñ +*empac +*empaquet +*empaquét +*emparej +*emparéj +*emparent +*empat +*empé +*empedr +*empeg +*empeor +*emperez +*empern +*emple +*empolv +*empotr +*empoz +*enam +*encab +*encad +*encaj +*encáj +*encall +*encáll +*encam +de3sencant +*encap +*encar +*encár +*ench +*encl +*enco +*encr +*encu +*end +de3senfad +de3senfád +*enfi +*enfo +*enfó +de3senfren +*enfund +*enfur +de3sengañ +de3sengáñ +*enganch +*engar +*engas +*engom +*engoz +*engra +*enhebr +*enj +*enlad +*enlaz +*enlo +*enm +*enr +*ens +*enta +de3sentend +de3sentien +de3sentién +*enter +*entier +*entiér +*ento +*entr +*entu +*envain +de3senvolvim +de3seo +*eq +de3s4erci +de3s4ert +de3s4ért +*espa +de3sesperac +*esperanz +*estabil +*estim +de3sider +de3sidia +de3sidio +de3siert +de3sign +de3sigual +de3silusi +*imagin +*iman +*impon +*impres +*incent +*inclin +*incorp +*incrust +de3sinenc +de3sinfec +de3su3dar de3su3das de3su3dan +*inflam +*infl +*inform +*inhib +*insect +*instal +ini3ci +iní3ci +de3s4integr +de3s4inter +*intox +*inver +de3sisten +de3isti +*obedec +*oblig +*obstr +de3socup +*odor +de3solac +de3solad +de3soll +*orej +*orient +de3sortij +*organi +de3suell +de3sonce +*ovi +*oxi +*oye +*oyé +de3s4ubstan +de3s4ustan +de3s4oseg +*ub4ic +*unir +*unier +*unim +.diecio2 +/ecano/ % (m)ecano +/eco/ +/ectro/ % (esp)ectro, (el)ectro +/.e2n2/ +.ene3mist .ene3míst +.eno3jar +.enu3mera .enu3merá +.enu3mere +.en3aceit +.en3arb +4emboca +4o3lógico. 4o3lógica. 4o3lógicos. 4o3lógicas. +4o3lógicamente. 4o3logía. 4o3logías. +4ó3logo. 4ó3loga. 4ó3logos. 4ó3logas. +/endo/ +/ento/ % (ci)ento +/entre/ +/euco/ % (l)euco +/euro/ % euro, (n)euro +/.e2x2/ +3gono. 3gonos. 3gonal. 3gonales. +exá3men +exá3ri +exé3ge +exó3ti +exa3cerb +exa3ger +exa3min +exe3cr +exe3géti +exe3quia +exi3ja exi3já exí3ja +exi3ge exí3ge +exi3gi exi3gí +exi3jo exí3jo +3gua. 3guas. 3guo. 3guos. +exi3lia exi3liá exí3lia +exi3lie exi3lié exí3lie +exi3lio exi3lió exí3lio +exi3ma exi3má exí3ma +exi3mi exi3mí exí3mi +exi3me exi3mé exí3me +exi3mo exi3mó exí3mo +exi3tos +exi3lar +exo3ner +exo3crin +exo3tér +exo3tic +exo3tiq +exo3tism +exo3gami +exo3gámi +exu3ber +exu3dar +exu3dat +exu3dac +/extra/ +u4teri +.cau5t +.deu5t +/fono/ +/foto/ +/gastro/ +/geo/ +/giga/ +/gluco/ +/hecto/ +/helio/ +/hemato/ +/hemi/ +/hemo/ +2al. 2ales. +.hepta1e +/hexa/ +/hidro/ +/hiper/ +pe3r4e3mia +hipe3r4i3cíne +hipe3r4o3nimi +hipe3r4o3ními +hipe3r4ó3nimo +hipe3r4o3xia +/hipo/ +/spano/ % (hi)spano +/histo/ +/homo/ +/ibero/ +/icono/ +/.i2n2/ +.in3abord +.in3abarc +.in3acent +.in3aguant +.in3adapt +.ina3movib +.in3analiz +.ina3nic +.in3anim +.iná3nim +.in3apel +.in3aplic +.in3aprens +.in3apreci +.in3arrug +.in3asist +.iné3dit +.in3efic +.in3efici +.in3eludi +.ine3narr +ini3cia iní3cia ini3ciá ini3cie +.rei3na +re3ini3cia re3iní3cia re3ini3ciá re3ini3cie +.ini3cuo .ini3cua +.ino3cuo .ino3cua +.ino3cula .ino3culá .ino3cule +.inú3til +.inu3tiliz +/infra/ +/.inter/ +.in3ter2e3sa .in3ter2e3se .in3ter2e3so +.in3ter2e3sá .in3ter2e3sé .in3ter2e3só +.de3s4in3ter2e3sa .de3s4in3ter2e3se .de3s4in3ter2e3so +.de3s4in3ter2e3sá .de3s4in3ter2e3sé .de3s4in3ter2e3só +4n3te3ri3n 4n4te4r5i4nsu +.in3te3r4rog +.in3te3r4rupc .in3te3r4rupt .in3te3r4rump +inter4és +inte3r4esar +inte4r4i4n4 +inter5ins5t +inte5r4regno +inte3r4ior4 +4i4ano. 4i4ana. 4i4anos. 4i4anas. +/intra/ +/iso/ +/kilo/ +/macro/ +mal2 ma4l3h .ma4l3e4du mal3b mal3c mal3d mal3f mal3g +mal3m mal3p mal3q mal3s mal3t mal3v +.mal1acon +.mal1acos +.mala1e +.mal1andant +.mal1andanz +.mal1est +.mal1int +/maxi/ +/megalo/ +/mega/ +/melano/ +/micro/ +/.mili/ +familia3ri ia5res. +amili6a +a3rio a3ria +li5área +mili4ar +/mini/ +2os. 2o3so. 2o3sos. 2o3sa. 2o3sas. 2o3samente. +mini4a5tur +/multi/ +/miria/ +/mono/ +2i3co. 2i3cos. 2i3ca. 2i3cas. +/namo/ % (di)namo +/necro/ +/neo/ % (cra)neo +3c2neor +/neto/ % (mag)neto +/norte/ +/octo/ +/octa/ +/oligo/ +/omni/ +i2o. i2os. +/paleo/ +.pa4n1a4meri +.pa4n1europ +.pa4n1afri +.pa4n1hisp +.pa4n1ópti +/para/ +para2is. aí5so. aí5sos. +para4íso +para4ulata +/penta/ +/piezo/ +/pluri/ +/poli/ +poli4u3r +poli4o5mie +poli4arq poli4árq +poli4éste +poli4andr +poli4antea +poli4arca +expoli4 +/.pos2t2/ +.pos3tin .pos3tín +pos3ta. pos3tas. +s3te. s3tes. s3tal. s3ta3les. +s3ti3lla. s3ti3llas. s3ti3llón. s3ti3llones. +s3tor. s3tora. s3toras. s3tores. +.pos3tó3ni +.pos3terg +.pos3te3ri +.pos3ti3go +.pos3ti3la +.pos3ti3ne +.pos3ti3za .pos3ti3zo +.pos3tu3ra +.pos3tu3la .pos3tu3lá .pos3tu3le .pos3tu3lé +.post3elec +.post3impr +.post3ind +.post3ope +.pos2t3rev +.pos2t3rom +.pre2a2 .pre2e2 .pre2i2 .pre2o2 .pre2u2 +.pre2á2 .pre2é2 .pre2í2 .pre2ó2 .pre2ú2 +.pre1h2 +pre3elij pre3elig +pre3exis +pre3emin +preo3cup preo2cúp +pre3olí +pre3opin +/.pro2/ +.pro3abort +/proto/ +/psico/ +3p2sic +3p2siq +/quete/ % (re)quete +/radio/ +/ranco/ % (f)ranco +.re2a2 .re3e4 .re2i2 .re2o2 .re2u2 +.re2á2 .re2é2 .re2í2 .re2ó2 .re2ú2 +ea3cio. ea3cios. ea3cia. ea3cias. +.re3abr .re3ábr +.re3afirm .re3afírm +.re3ajust .rea3júst +.rea3liza .rea3lizá .rea3líza +.re3alim +.rea3lism .rea3list +.re3anim .re3aním +.re3aparec +.re3ubica .re3ubíca +.reu3mati .reu3máti +.re3unir .re3unír +.re3usar .re3usár +.re3utiliz .re3utilíz +.re3a2eg +.re3a2q +.re3a2z +.re3a2grup +.re3i2m +.re3inc +.re3ing +.re3ins +.re3int +.re3o2b +.re3oc +.re3oj +.re3orga +.re3unt +/rmano/ % (ge)rmano +/retro/ +/romo/ % (c)romo +/semi/ +i2a. i2as. +2ótic emi2o2 +/2seudo/ % (p)seudo +o2os. +.so3a4s % soasar +/sobre/ +/socio/ +a3rio. a3rios. +3logía +4ón. 4ones. +4i4er. +4o2ico. 4o2icos. 4o2ica. 4o2icas. +.su2b2a2 .su2b2e2 .su2b2i2 .su2b2o2 .su2b2u2 +.su2b2á2 .su2b2é2 .su2b2í2 .su2b2ó2 .su2b2ú2 +.sub2i3ll +.sub2i3mien +.sub3índ +.sub3ími +.su4b3ray +.sub3aflue +.sub3arr +.sub3enten +.sub3estim .sub3estím +.sub3ofici +.sub3urba +.sub3alter +.sub3insp +.su3bién +.su3bir +.su3bam +.su3bordin .su3bordín +.sub3acuá +.sub3espe +.sub3esta +.su3burbi +.su4b5rein +.su2d1a2fr +.su2d1a2me +.su2d1este +su4d3oes +.su2r1a2me +.su2r1est +.su2r1oes +/super/ % -ar +ra5ra ra5rá ra5re ra5ré ra5rí ra5ro +supe3r4a4r supe3r4á4r +supe3r4á3vit. supe3r4á3vits. +super4ior +4a3ción. 4a3ciones. +4e3rior. 4e3riores. 4e3riora. 4e3rioras. 4e3riormente. +4e3rioridad. 4e3rioridades. +4e3r4a3ble. 4e3r4a3bles. 4e3r4a3blemente. +pe5r4ante +perpon5d6r +/supra/ +sup6ra % +/talmo/ % (of)talmo +/tele/ +4ósteo. 4ósteos. +.tele1imp +.tele1obj +.tele4o3lót +/termo/ +/tetra/ +/topo/ +/.tra2s/ +% .tra2s1a2 +% .tra2s1o2 +.tra3s2o3ñ +/tran2s/ +tran2s1alp +tran2s1and +tran2s1atl +tran2s1oce +tran2s1ur +tran3sacci +tran3seún +tran3se3xu +tran3si3cion tran3si3ción +tran3si3gi tran3si3gí tran3si3ge +tran3si3ja tran3si3já tran3si3jo +tran3sisto +tran3si3ta tran3si3tá tran3si3te tran3si3té tran3si3to tran3si3tó +tran3si3tiv +tran3si3tori +tran3subst tran3sust +.tri1ó2x +/tropo/ % (ant)ropo +/ultra/ +/vice/ +/video/ +/xeno/ +/zoo/ +wa3s4h +3hablante +3habiente +3aficionad +%.pan1esl +%.pan1isl +% pos (impedir) = +% tri = (faltan ex) +% uni = (faltan excep) +%conjugar ir +% /miriá/ +% des3ojar/VX +% desistir dejar +% desoír/QWY +% des3escombrar/VX +% desenvoltu evitar +% desenvolver +% de3senlace % evitar +% desempeñar % evitar
\ No newline at end of file diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb new file mode 100755 index 0000000000..61e88d8991 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/eu/generate_patterns_eu.rb @@ -0,0 +1,137 @@ +#!/usr/bin/env ruby +# +# First version: 97/02/20 +# +# This script generates TeX hyphenation patterns for Basque +# and was written by (c) Juan M. Aguirregabiria, 1997 +# based on the shyphen.sh file by Julio Sanchez, which +# generates the hyphenation patterns for Spanish +# The copyright notice below applies to this script as well, +# read it before using this software. +# +# Rewritten into ruby by Mojca Miklavec <mojca.miklavec[.lists] at gmail.com> +# in June 2008. Modifications: +# - added support for Unicode +# - dropped support for every other "encoding" +# - added ñ by default +# - removed part of copyright that was not relevant any more +# (actually, nothing here resembles the work for Spanish patterns in any way any more) +# +# open file for writing the patterns +# $file = File.new("hyph-eu.tex", "w") +# in TDS +$file = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-eu.tex", "w") + +# write comments into the file +def add_comment(str) + $file.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n") +end + +# h is not here. +consonants=%w{b c d f g j k l m n ñ p q r s t v w x y z} +# Open vowels: a e o +vop=%w{a e o} +# Closed vowels: i u +vcl=%w{i u} +# +# In June 2008 the following conversation took place. +# In case that anyone figures out how the situation might be improved ... +# +# Mojca: +# Why do you need open and closed vewels separately? +# Juan: +# Open and closed vowels are very different from the point of view of +# hyphenation in Basque (and Spanish): +# you may write "...e-o...", but not "...u-i..." +# Mojca: +# But the script generates the same patterns for both sets of vowels. +# Juan: +# After eleven years I don't remember *anything* about all this. Maybe I +# intended to make a difference between open and closed vowels and then +# thought it would be safer to avoid all breaking between vowels, because in +# some cases a rule I don't remember now prevents some of them. Maybe some +# legal breaks are ignored with current patterns, but a least I have never +# seen an illegal hyphen and never get a report of such a break. +# +# I guess a couple of extra lines in the script don't hurt and maybe some day +# someone will use the difference to allow more breaks. +# +vowels=[].concat(vop).concat(vcl) +# Groups that cannot be broken. +legal=%w{ll rr ts tx tz bl br dr fl fr gl gr kl kr pl pr tr} +# echo -n Usage: `basename $0` +add_comment( +"Hyphenation patterns for Basque. + +This file has first been written by Juan M. Aguirregabiria +(juanmari.aguirregabiria@ehu.es) on February 1997 based on the +shyphen.sh script that generates the Spanish patterns as compiled +by Julio Sanchez (jsanchez@gmv.es) on September 1991. + +In June 2008 the generating script has been rewritten into ruby and +adapted for native UTF-8 TeX engines. Patterns became part of hyph-utf8 +package and were renamed from bahyph.tex into hyph-eu.tex. +Functionality should not change apart from adding ñ by default. + +The original Copyright followed and applied also to precessor of this file +whose last version will be always available by anonymous ftp +from tp.lc.ehu.es or by poynting your Web browser to + http://tp.lc.ehu.es/jma/basque.html + +For more information about the new UTF-8 hyphenation patterns and +links to this file see + http://www.tug.org/tex-hyphen/ + + COPYRIGHT NOTICE + +These patterns and the generating script are Copyright (c) JMA 1997, 2008 +These patterns are made public in the hope that they will benefit others. +You can use this software for any purpose. +However, this is given for free and WITHOUT ANY WARRANTY. + +You are kindly requested to send any changes to the author. +If you change the generating script, you must include code +in it such that any output is clearly labeled as generated +by a modified script. + + END OF COPYRIGHT NOTICE + +Open vowels: #{vop.join(" ")} +Closed vowels: #{vcl.join(" ")} +Consonants: #{consonants.join(" ")} + +Some of the patterns below represent combinations that never +happen in Basque. Would they happen, they would be hyphenated +according to the rules. +") + +$file.puts '\patterns{' + +add_comment( +"Rule SR1 +Vowels are kept together by the defaults +Rule SR2 +Attach vowel groups to left consonant") +consonants.each do |consonant| + patterns = [] + vowels.each do |vowel| + patterns.push "1#{consonant}#{vowel}" + end + $file.puts patterns.join(" ") +end + +add_comment( +"Rule SR3 +Build legal consonant groups, leave other consonants bound to +the previous group. This overrides part of the SR2 pattern group.") +legal.each do |pair| + patterns = [] + vowels.each do |vowel| + patterns.push "1#{pair[0,1]}2#{pair[1,1]}#{vowel}" + end + $file.puts patterns.join(" ") +end + +add_comment("We now avoid some problematic breaks.") +$file.puts "su2b2r su2b2l" +$file.puts "}" diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README new file mode 100644 index 0000000000..418910cbd5 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/README @@ -0,0 +1,25 @@ +This directory contains seven files + README, licensed all but one under lppl 1.3: + +glpatter-utf8.tex: + The source file, version 2.4. When processed under the program + mkpatter ---by typing 'tex -ini -8bit glpatter-utf8.tex' (without the quotes) + at the command line, provided you have the utility + mkpattern installed--- it generates the file hyph-utf8-gl.tex. + +(hyph-gl.tex): + The generated file containing the patterns in UTF-8 (belongs to TEXMF/tex). + + +glhyextr.tex: ##This file is in the public domain## + A configuration file, to include additional patterns other + than the ones in glpatter-utf8.tex. It is not \input by glpatter-utf8.tex + (iin contrast with the single-byte encoded version glpatter.tex) + as modern tools allow the addition of patterns at run time. + +glhybiox.tex, glhymed.tex, glhyquim.tex, glhytec.tex & glhyxeog.tex: + Files with aditional patterns drawn from serveral fields of + the language, so that you may input them from glhyextr.tex. + +The files for specific patterns are very incomplete and feedback is welcome. + +--Javier A. Múgica diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex new file mode 100644 index 0000000000..461751ecc8 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhybiox.tex @@ -0,0 +1,25 @@ +% This is the file glhybiox.tex. +% This is a subset of the galician hyphenation patterns. +% (c) Javier A. Múgica, 2007 +% License: LPPL version 1.3 +% +% LPPL maintenance status: maintained +% Current Maintainer: Javier A. Múgica +% +% Prefixos para xeoloxía e bioloxía. +% Esta lista está aínda moi incompleta. Para aportar +% novos prefixos ou para calquera outra suxerencia, +% contactar en javier @ digi21.eu. + +%Os prefixos comentados xa están na lista principal en glpatter.tex +\begin{prefixos} +%paleo termo +ciano crio halo .lito +.oro .uro +\end{prefixos} + +.crio2u .malcrio2u +\begin{3a_ppi} +inhalo +\end{3a_ppi} +\endinput diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex new file mode 100644 index 0000000000..37ad86c0b9 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyextr.tex @@ -0,0 +1,7 @@ +%glhyextr.tex. This file is in the public domain +%\mkinput{glhyquim.tex} +%\mkinput{glhymed.tex} +%\mkinput{glhybiox.tex} +%\mkinput{glhyxeog.tex} +%\mkinput{glhytec.tex} +\endinput
\ No newline at end of file diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex new file mode 100644 index 0000000000..cd3d6a4997 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhymed.tex @@ -0,0 +1,19 @@ +% This is the file glhymed.tex. +% This is a subset of the galician hyphenation patterns. +% (c) Javier A. Múgica, 2007 +% License: LPPL version 1.3 +% +% LPPL maintenance status: maintained +% Current Maintainer: Javier A. Múgica +% +% Prefixos para mediciña. +% Esta lista está aínda moi incompleta. Para aportar +% novos prefixos ou para calquera outra suxerencia, +% contactar en javier @ digi21.eu. + +%Os prefixos comentados xa están na lista principal en glpatter.tex +\begin{prefixos} +%cardio hemato +oftalmo pleuro oligo +\end{prefixos} +\endinput diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex new file mode 100644 index 0000000000..573bed5cd6 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyquim.tex @@ -0,0 +1,19 @@ +% This is the file glhyquim.tex. +% This is a subset of the galician hyphenation patterns. +% (c) Javier A. Múgica, 2007 +% License: LPPL version 1.3 +% +% LPPL maintenance status: maintained +% Current Maintainer: Javier A. Múgica +% +% Prefixos para química. +% Esta lista está aínda moi incompleta. Para aportar +% novos prefixos ou para calquera outra suxerencia, +% contactar en javier @ digi21.eu. + +%Os prefixos comentados xa están na lista principal en glpatter.tex +\begin{prefixos} +%hidro +cloro fluoro gluco lacto lipo +\end{prefixos} +\endinput diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex new file mode 100644 index 0000000000..f36a7007a9 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhytec.tex @@ -0,0 +1,19 @@ +% This is the file glhytec.tex. +% This is a subset of the galician hyphenation patterns. +% (c) Javier A. Múgica, 2007 +% License: LPPL version 1.3 +% +% LPPL maintenance status: maintained +% Current Maintainer: Javier A. Múgica +% +% Prefixos para términos técnicos. +% Esta lista está aínda moi incompleta. Para aportar +% novos prefixos ou para calquera outra suxerencia, +% contactar en javier @ digi21.eu. + +%Os prefixos comentados xa están na lista principal en glpatter.tex +\begin{prefixos} +%hidro termo electro magneto +alumino metalo piezo +\end{prefixos} +\endinput diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex new file mode 100644 index 0000000000..c5105af19c --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glhyxeog.tex @@ -0,0 +1,25 @@ +% This is the file glhyxeog.tex. +% This is a subset of the galician hyphenation patterns. +% (c) Javier A. Múgica, 2007 +% License: LPPL version 1.3 +% +% LPPL maintenance status: maintained +% Current Maintainer: Javier A. Múgica +% +% Prefixos para xeografía. +% Esta lista está aínda moi incompleta. Para aportar +% novos prefixos ou para calquera outra suxerencia, +% contactar en javier @ digi21.eu. + +%Os prefixos comentados xa están na lista principal en glpatter.tex +\begin{prefixos} +%anglo franco ibero .indo +alto baixo +greco italo xermano .sino .uralo +\end{prefixos} + +\exceptions{.baixo2u3}{.baixo2u} +\begin{3a_ppi} +asalto .baixo .rebaixo +\end{3a_ppi} +\endinput diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex new file mode 100644 index 0000000000..4b116eede4 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/gl/glpatter-utf8.tex @@ -0,0 +1,459 @@ +% This is the file glpatter-utf8.tex, version 2.4 +% It is the source file for the Galician patterns +% +% (c) Javier A. Múgica; 2006, 2007, 2008, 2010 +% License: LPPL version 1.3 +% +% LPPL maintenance status: maintained +% Current Maintainer: Javier A. Múgica +% +% +% The patterns follows approximately the scheme developed by Javier Bezos for esphyph.tex +% In order to generate the patterns just type +% tex -ini -8bit glpatter-utf8.tex +% +% For bug reports and comments: +% +% Javier Múgica, javier at digi21.eu +\input mkpatter.tex +\filename{hyph-gl.tex} +\tracingexceptions=1 + +&Va={a e o} +&Vp={i u} +&Vi={&Va u} +&Va+={&Va á} +&V={a e i o u á é í ó ú} +&Vv={a e i} +&Vai={a i} +&Vvac={á é í} +&L={l r} +% ñ non se inclúe nos seguintes grupos +&C={b c d f g h k l m n p r s t v x z} % Consoantes habituais +&Cl={b c d f g k p t v} % As que van unidas á líquida, agás r e l +&G={b c d l m n r s t x} % Posibles consoante precedente a certos grupos pouco frecuentes + +\templatedef{prefixos}{#1&{V}2} +\templatedef{3a_ppi}{#2u3 \newline} % 3a. persoa do perfecto de indicativo, rematadas en -ou, -eu, -iu. + % O confilcto cun prefixo máis habitual é na primeira conxugación. +% +% Uncomment the %\letters line if you want those characters to be catcode as letters +% and the correspoding caplital ones to be assigned the right \lcode. In that case an utf-8 +% engine (luatex or xetex) to process this file is mandatory. +% +% \letters{á é í ó ú ñ ü ï} + +% The commented lines that follow will be placed in the generated file, +% and do not apply to this file. +\expandafter\let\expandafter\lee\csname ~~lee\endcsname +\keepcomments +% title: Hyphenation patterns for Galician, written in the utf8 encoding. +% copyright: (c) Javier A. Múgica; 2006, 2007, 2008, 2010 +% notice: This file is part of the hyph-utf8 package. +% See http://www.hyphenation.org for more information. +% language: +% name: Galician +% tag: gl +% licence: +% name: LPPL +% version: 1.3 +% status: maintained +% maintainer: Javier A. Múgica +% url: https://latex-project.org/lppl/lppl-1-3.html +% +% ========================================== +% Generated with the mkpattern utility (v. 1.2), on 2010/04/23 +% The original source file were glpatter-utf8.tex +% This is a generated file +% +% For bug reports and comments: +% +% Javier Múgica, javier at digi21.eu +\let\lee\relax +\halfcomments +\iwrite{%} +\iwrite{% Note that there is no 'j' nor 'y' in Galician} +\nocomments + +\begin{pseudopatterns} + +3ï3a \newline +a1a e1e o1o zo2o a1á e1é o1ó 2&{Va}&{Vp}. +&{Vi}1i&{Va+} 2i&{Va+}. + +\newline +1ñ +1&{C}&{V} +1qu2 1c2h 2ch. 1g2h 2gh. 2k2h \newline % gh é normalmente usado para representar a gheada +\newline +\exceptions{1t2l 2tl.}{2t2l,} +1&{Cl}2&L 2&{Cl}&L. + +1l4l 2ll. 1r2r 2rr. + +\template{&G1#} +p2t c2t c2n p2s m2n g2n f2t p2n c2z t2s +\end{template} +san2c3t plan2c3t + +\template{2#.} +pt ct cn ps mn gn ft pn cz ts +\end{template} + +un2ha 2non. 3mente. o2hib alde2h \newline +\newline +% palabras malsonates +4caca4 4cago4 4caga4 4cagas. 4puta4 4puto4 4meo. 4mea. \newline +4meable. 4meables. 4peido4 + +\exceptions{.co1á2}{} +\exceptions{.pre1á2 .su2b1i2 .su2b1í2}{} % If duplicate patterns were simply added... +\exceptions{.pre2á}{.pre2á2} % This is error prone. I think that writting +\exceptions{.sub2i .sub2í}{.su2b2i2,.su2b2í2} % all the exceptions here together is the best that can be done. +\begin{prefixos} % Prefixos +acto afro aero anfi anglo .ante .anti .arqui auto biblio bio +cardio cefalo ciclo cito .co contra cripto cromo crono +deca .deza dinamo ecano eco electro endo ento entre +euco euro extra fono foto franco gastro xeo +hecto helio hemato hemi hexa hidro hipe2r histo homeo homo +ibero icono .in .indo infra .inte2r intra .iso kilo +macro magneto maxi mega megalo melano micro mili mini +multi miria mono .nano necro .neo norte octo octa +omni paleo para penta piezo pluri poli .pos2t .pre +.pro proto pseudo radio .re retro sobre semi +socio .su2b supe2r supra .tele termo tetra topo +.tri tropo ultra xeno +\end{prefixos} + +\template{#3r} % prefixos rematados en r +hiper .inter super +\end{template} + +\newline +ti2o3qu ti2o3co +bi1u2ní +o2i3de o2i3dal +2al. 2a2is. +pe3r2e3mia \newline + +\begin{3a_ppi} +\put{% Terceiras persoas do pasado perfecto}\newline +libero atopo enaxeno +\end{3a_ppi} + +\newline +2os. 2o3so. 2o3sos. 2o3sa. 2o3sas. 2o3samente. \newline +2i3co. 2i3cos. 2i3ca. 2i3cas. \newline + +\newline +\put{% Excepcións ó prefixo co} +% Pode non ser prefixo (expecto os derivados de coar) +\template{.co2# \newline} +\put{% Formas do verbo coar con pronomes enclíticos}\newline ar á2 abá +acerv andro ano añar año art etan enci erci inci ira iro ita +\end{template} + +% Non é prefixo (os que non son da forma &V3 non necesitan ir un por un) +\template{co2# \newline} % Non é necesario coia, coiote +a3gul á3gul a3la. a3las. a3lescen a3lición. a3licions. a3na. a3nas. +antriñ a3ñadeir a3tí. a3tís. e3ficien e3lernos. e3llo. e3lla. e3llos. e3llas. +e3lleir enll enxía e3sita. e3sitas. e3táne e3vo. e3va. e3vos. e3vas. +i3dado iei imbra imbrá intreau. í3ña i3ña i3ñei i3pú. i3pús. +i3ra. i3ras. i3raza i3ro. i3ros. i3ta. i3tas. i3tado. i3to. i3tos. +i3tel i3tío. i3tus. u3c u3lomb u3try u3qui u3rel +u3sa. u3sas. u3so. u3sos. u3selo u3tad u3to. u3tos. u3vini u3z +\end{template} + +deca2e3ment + +\newline +\put{% O prefixo des} +.de2s1&{V}2 +3se. 3s2es. 3sa. 3s2as. de3s2outr 3s2emos. 3s2edes. 3s2en. + +\exceptions{de3s2esper}{de3s2esper de4s3esperanz} +\template{de3s2# \newline} % Prefixos .de (máis ou menos) +a3crali a3guisa a3lini a3ngr a3ñ a3rrollis astr a3zo e3c e3que e3guid e3la +ensib e3ñ ert ért esper e3pér e3x é3x i3der ign ígn i3nenc ingr iste isti o3lac o3lad old +o3lidari uetud sulf +\end{template} + +\template{.des2# \newline} % Prefixos dubidosos ou que sa non se notan. Non se inclúen os que por ser moi breves poderían coincidir con prefixos .des formados ad hoc, xa que impedirían a división de palabras comúns (desecar, deselar). +abor afia afía afío air emboc embóc empeñ empéñ enlac enlaz enlác enláz +envol envól idia ora +\end{template} + +\keepcomments + +\newline +% Excepcións ó prefixo in +\template{.in2# \newline} +a3misib. a3mov a3ne. a3nic a3nid á3nime antes. au e3dia é3dit e3fab e3narr +epc ept erc ert erm erv e3siv e3xora i3ci i3cu i3mig i3miza i3qui +o3cen o3cui o3cuo o3cul ó3cul o3pia. o3sili o3sit o3tróp o3trop uit u3lase u3lina +unda u3sita ú3til +\end{template} +in2o4cular \newline + +\newline +% Excepcións ó prefixo inter +\template{.inte3r2#} +és. e3sa é3sa e3sá e3so é3so e3só ior i3no. i3nos. i3na. i3nas. i3nid +\end{template} + +\newline +% Os prefixos mal e ben +\exceptions{.be2n1e2 .be2n1é2 .ma2l1e2 .ma2l1é2}{.be2ne2, ,.ma2le2, } + +\halfcomments +.be2n1&{V}2 +\template{be3n2# \newline} % Non é prefixo (ou non se nota) +ign i3merí i3nés i3nes i3toíta +\end{template} + +\newline +.ma2l1&{V}2 +\template{.mal2# \newline} % Pode non ser prefixo +abar abár aco acó armad ogr ura axa +\end{template} +\template{ma3l2# \newline} % Non é prefixo. Non é neceario malos, malas +a3cía a3citan a3gueñ aio aia andrín andrin a3quita ar. a3res. a3ria. +a3to. a3tos. a3ta. a3tas. a3tión aui eabl eabil eico eolar e3ta. e3tas. e3tín e3teiro +eza ia. ian i3cia i3cios ign i3kita inke ó3fago ó3nic o3nato o3nilue +\end{template} +ma4l3ianq +\template{.mal1# \newline} % É prefixo e comenza por e +educ encar ensin entend +\end{template} + +\newline +\put{% Excepcións ó prefixo mili} +famili2a familia3r +\template{mili#} +2a3rio 2a3ria +\end{template} + +mini2a3tur + +\newline +\put{% Excepcións ó prefixo para} +\template{para2#} +u3gas í3so +\end{template} + +\newline +\put{% Excepcións ó prefixo poli} +\template{poli2# \newline} +u3r o3me arq árq éste andr antea +\end{template} +expoli2 + +\newline +\put{% Excepcións ó prefixo post} +\template{.pos3t2# \newline} +a. as. al. ais. a3llo e. es. ear. e3la. e3las. er. erg e3rid e3rior i3go i3la +illón ín. i3te. i3zo. i3zos. i3za. i3zas. o. os. oiro ó3ni u3la u3lo u3le u3ra. u3ras. +\end{template} + +\newline +\put{% Excepcións ó prefixo pre} +\template{.pre2# \newline} % Poden non ser prefixos +amar \put{% Formas do verbo prear con pronomes enclíticos}\newline ar á abá +\end{template} +\template{pre2# \newline} % Non son prefixos +as. a3da. a3das. á2 abá i3t o3cup o3cúp +\end{template} + +\newline +\put{% Excepcións ó prefixo pro} +\template{pro2# \newline} % Non son prefixos. Non é necesario proer +e3za í3do ust +\end{template} + +\newline +\put{% Excepcións ó prefixo re} % Faltan excepcións +\template{.re2#} +al a3liz a3lidade in i3no. i3nos i3nante iter \newline +i3xa os. as. u3ma ú3ma ún uni +\end{template} + +\put{% Verbo reunir} +% No é tan problemáticos coma subir +\template{re2u3n#} +ión. ións. ir írmo íren \newline % ir inclúe varias formas. Penso que re3u4nirr no paga a pena +ido. idos. ida. idas. ind \newline +o. es. e. imos. ímo en. \newline % ides. incuído en ide, enbaixo +ía i3amos. i3ades. \newline +ín. iches. iu. istes. íst iron. \newline +a. as. amos. ades. an. \newline +ise íse \newline +ide íde ámo áde +\end{template} + +\put{% Verbo reinar} +% No é tan problemáticos coma subir +\template{re2i3n#} +ar ado. ados. and \newline +a. as. amos. ades. an. \newline % ides. incuído en ide, enbaixo +aba. abas. abamos. ábamos. abades. ábades. \newline % hay moitas palabras que comenzan por inaba +ei. aches. ou. astes. aron. \newline +e. es. emos. edes. en. \newline +ase. ases. asemo. asedes. áse asen. \newline % idem., inase +ade. +\end{template} + +\newline +\put{% O prefixo sub: l, r e excepcións}\newline +.su2b3l .su2b3r + +\template{.sub2# \newline} % Poden non ser prefixos +\put{% Formas do verbo subir con pronomes enclíticos}\newline i í +eriz orna +\end{template} +\newline +sub3índic sub3indic sub3indiz + +\template{.sub4# \newline} +lev lim +\end{template} + +\template{su3b2# \newline} % Non son prefixos +e3la. e3las. é3rico e3rina. e3rinas. eroso iote ulado orno. ornos. urbio +\end{template} +\template{su3b4# \newline} +liminar repción reptici +\end{template} + +\newline +\put{% Excepcións ó prefixo tri} +\template{tri2# \newline} % Non son prefixos (ou non se nota) +a3ga. a3gas. al. a3les. angul á3sico. estin % (De Trieste) +unf unvir +\end{template} + +\newline +\put{% Excepcións a varios prefixos que son terminacións verbais}\newline +\put{% Inclúense as variantes de acentuación abamos/ábamos, etc.} +% As combinacións con pronomes enclíticos (son centos) +% deberanse evitar nos patróns de cada prefixo cando sexa necesario +% Inclúense as combinacións máis habituais +\template{2&{Vai}#.} +3do 3da 3dos 3das ndo r % inf. xer. part. +\end{template} + +\template{2&{Vv}#.} +3res rmos rdes 3ren \newline % inf. conx. +rme rte rlle rnos rvos rlles % inf. con pron. indir. +\end{template} + +\template{2&{Vv}3#.} +dor dora dores doiro doiros doira doiras deiro deiros deira deiras % subst. deriv. +lo los la las % inf. con pron. dir. +rei rás rá remos redes rán \newline % futuro +ría rías ri1amos ríamos ri1ades ríades rían \newline % cond. +de % imp. +\end{template} +\template{2&{Vvac}3#.} +deo dea deos deas % imp. con pron. dir. +\end{template} + +\template{2#.} +% coar, inar, entrenar, prear, superar +as a3mos a3des an \newline +a3ba a3bas a3bamos á3bamos a3bades á3bades a3ban \newline +a3ches astes a3ron \newline +es e3mos e3des en \newline +a3se a3ses á3semos á3sedes a3sen \newline +\end{template} +% pero permitimos a ruptura para coar, prear, voar, cear, capear... +% cando o "a" ou "e" temático é tónico (ou casi). +\template{o3#.} +ar ado ada ados adas ando \newline +ares armos ardes aren \newline +arme arte arlle arnos arvos arlles \newline +alo alos ala alas \newline % incúe algunhas outras (coala) +ade ádeo ádea ádeos ádeas \newline +as amos ades an \newline +aba abas abamos ábamos abades ábades aban \newline +aches astes aron \newline +es emos edes en \newline +ase ases ásemos ásedes asen \newline +\end{template} +\template{e3#.} +ar ado ada ados adas ando \newline +ares armos ardes aren \newline +arme arte arlle arnos arvos arlles \newline +alo alos ala alas \newline % incúe algunhas outras (coala) +ade ádeo ádea ádeos ádeas \newline +as amos ades an \newline +aba abas abamos ábamos abades ábades aban \newline +aches astes aron \newline +es emos edes en \newline +ase ases ásemos ásedes asen \newline +\end{template} + +\template{2#.} +% subir +i3mos i3des \newline +ía ías íamos íades ían \newline % 2iamos, 2iades estropearía moitos verbos (ca-ia-mos... ) ... +ín i3ches iu istes i3ron \newline +% O pres. subx. coincide con pres. ind. da 1ª +i3se i3ses í3semos í3sedes i3sen +% contraer, decaer, extraer +% Pres. subx. xa se divide ben +\end{template} +\newline +í3do í3da í3dos í3das + +% Conxugación completa do verbo subir +\template{.su3b#.} +ir indo ido ida idos idas \newline +ires irmos irdes iren \newline +o es e imos ides en \newline +ía ías 2i3amos íamos 2i3ades íades ían \newline % ... por eso incluímolos só aquí. +ín iches iu istes iron \newline +irei irás irá iremos iredes irán \newline +iría irías iriamos iríamos iriades iríades irían \newline +a as amos ades an \newline +ise ises ísemos ísedes isen \newline +ide ídeo ídeos ídea ídeas \newline +ador adora adores adoras % posibles derivados +\end{template} + +% Conxugación completa do verbo superar +\template{.supe3r#.} +ar ando ado ada ados adas \newline +ares armos ardes aren \newline +a as amos ades an \newline +aba abas abamos ábamos abades ábades aban \newline +ei aches ou astes aron \newline +arei arás ará eremos eredes arán \newline +aría arías ariamos aríamos ariades aríades arían \newline +e es emos edes en \newline +ase ases ásemos ásedes asen \newline +ade ádeo ádeos ádea ádeas \newline +ador adora adores adoras ación % derivados +\end{template} + +supe3r2ior supe3r2a3ble supe3r2a3bilidade + +% para verbos rematados en -aer, permitimos a división +% cando a vocal temática tónica e aberta e, salvo inf., +% non é a última sílaba. +\template{a3#.} +er endo \newline +eres ermos erdes eren \newline +erme erte erlle ernos ervos erlles \newline +elo elos ela elas \newline +ede édeo édea édeos édeas \newline +emos edes eron \newline +ese eses esemos ésemos esedes ésedes esen +\end{template} + +% par-ti-a-mos, di-ci-amos, fa-ci-amos, etc., pero aso-cia-mos, etc. + +% The new programs allow to input patterns at run time, so including +% the extras in the format is not right. +% \mkinput{glhyextr.tex} +\end{pseudopatterns} +\end{} diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb new file mode 100755 index 0000000000..b2431bcf6b --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/hy/generate_patterns_hy.rb @@ -0,0 +1,69 @@ +#!/usr/bin/env ruby +# +# This script generates TeX hyphenation patterns for Armenian +# +# Rules written by Sahak Petrosyan <sahak at mit.edu> (author) +# +# These are just primitive rules that says that if there is a +# combination like <vowel><consonant><vowel>, then it should hyphenate +# like this: <vowel> - <consonant><vowel>. +# +# There are more rules to implement +# +# Script written by Mojca Miklavec <mojca.miklavec.lists at gmail.com> +# (who is not the author of patterns) +# +$file = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-hy.tex", "w") + +# write comments into the file +def add_comment(str) + $file.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n") +end + +vowels=%w{ա ե է ը ի ո օ} +consonants=%w{բ գ դ զ թ ժ լ խ ծ կ հ ձ ղ ճ մ յ ն շ չ պ ջ ռ ս վ տ ր ց փ ք} + +add_comment( +"Hyphenation patterns for Armenian. + +Written by Sahak Petrosyan <sahak at mit.edu> + for Hyphenator.js (http://code.google.com/p/hyphenator/) + and later adapted for TeX + +Licence: LGPL +Version: May 2010 + +These are just primitive rules that hyphenate combinations like +<vowel> - <consonant><vowel>. + +File auto-generated by a script (see source/generic/hyph-utf8/languages/hy) + +Vowels: #{vowels.join(" ")} +Consonants: #{consonants.join(" ")} + +Some of the patterns below represent combinations that never +appear in Armenian, but they would be hyphenated according to the rules. +") + +$file.puts '\patterns{' + +add_comment( +"և-<vowel>") +vowels.each do |vowel| + $file.puts "և1#{vowel}" +end + +add_comment( +"<vowel>-<consonant><vowel>") +vowels.each do |v1| + consonants.each do |c| + patterns = [] + vowels.each do |v2| + patterns.push "#{v1}1#{c}#{v2}" + end + $file.puts patterns.join(" ") + end + $file.puts "%" +end + +$file.puts "}" diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua b/language/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua new file mode 100755 index 0000000000..e3a13e1907 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua @@ -0,0 +1,154 @@ +#!/usr/bin/env texlua + +-- Extract Ethiopic syllables from the Unicode Character Database. +-- Copyright (c) 2011, 2016 Arthur Reutenauer +-- http://www.hyphenation.org/ +-- === Text of the MIT licence === +-- This file is available under the terms of the MIT licence. +-- Permission is hereby granted, free of charge, to any person obtaining a copy +-- of this software and associated documentation files (the “Software”), to deal +-- in the Software without restriction, including without limitation the rights to +-- use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +-- of the Software, and to permit persons to whom the Software is furnished to do +-- so, subject to the following conditions: +-- +-- The above copyright notice and this permission notice shall be included in all +-- copies or substantial portions of the Software. +-- +-- THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +-- IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +-- FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +-- AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +-- LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +-- OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +-- SOFTWARE. +-- === End of MIT licence === +-- Version 0.2 2016-05-22 + +-- Use with TeXLua. + +local function extract_mit_licence() + reading_mit_licence = false + bolicence = lpeg.P'=== Text of the MIT licence ===' + eolicence = lpeg.P'=== End of MIT licence ===' + mit_licence = { } + for line in io.lines(arg[0]) do + line = line:gsub('^%-%- ', ''):gsub('^%-%-', '') + if eolicence:match(line) then + reading_mit_licence = false + end + + if reading_mit_licence then + mit_licence[#mit_licence + 1] = line + end + + if bolicence:match(line) then + reading_mit_licence = true + end + end +end + +-- Small library to convert a hexadecimal string to a number +local function hex_digit_to_num(h) + local c = string.byte(h) + if c >= 48 and c < 58 + then return c - 48 + elseif c >= 65 and c < 71 + then return c - 55 + elseif c >= 97 and c < 103 + then return c - 87 + end + return 0 +end + +local function hex_string_to_num(hex) + local n = 0 + for i = 1, string.len(hex) do + n = 16 * n + hex_digit_to_num(string.sub(hex, i, i + 1)) + end + return n +end + +local ucd_file = "UnicodeData.txt" + +local error_install_ucd = [[ +Error: the Unicode Character Database could not be found. +Please download it from http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +and install it in the current directory.]] + +if not lfs.attributes(ucd_file) then + print(error_install_ucd) + return -1 +end + +local P, R, C, match = lpeg.P, lpeg.R, lpeg.C, lpeg.match + +local digit = R"09" + R"AF" +local colon = P";" +local field = (1 - colon)^0 +local eth_syll = P"ETHIOPIC SYLLABLE " * C(field) +local ucd_entry = C(digit^4) * colon * eth_syll + +local pattfile = assert(io.open("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-mul-ethi.tex", "w")) + + +extract_mit_licence() + +pattfile:write[[ +% title: Experimental pattern file for languages written using the Ethiopic script. +% copyright: Copyright (c) 2011, 2016 Arthur Reutenauer +% notice: This file is part of the hyph-utf8 package. +% See http://www.hyphenation.org for more information +% language: +% name: Multiple languages using the Ethiopic scripts +% tag: mul-ethi +% version: 0.2 2016-05-22 +% licence: +% - name: MIT +% - url: https://opensource.org/licenses/MIT +% - text: > +]] + +for _, line in ipairs(mit_licence) do + if lpeg.match(lpeg.P(-1), line) then + pattfile:write('%\n') + else + pattfile:write('% ', line, '\n') + end +end + +pattfile:write[[ +% ========================================== +% This is a generated file. If you wish to edit it, consider adapting the +% generating programme +% (https://github.com/hyphenation/tex-hyphen/blob/master/hyph-utf8/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua). +% +% The BCP 47 language tag for that file is “mul-ethi” to reflect the fact that +% it can be used by multiple languages (and a single script, Ethiopic). It is, +% though, not supposed to be linguistically relevant and should, for proper +% typography, be replaced by files tailored to individual languages. What we +% do for the moment is to simply allow break on either sides of Ethiopic +% syllables, and to forbid it before some punctuation marks particular to +% the Ethiopic script (which we thus make letters for this purpose). +% +]] + +pattfile:write"\\patterns{%\n" +for ucd_line in io.lines(ucd_file) do + local usv, char_name = match(ucd_entry, ucd_line) + -- print(string.format("%04X", usv)) + -- if usv then print(usv, char_name) end + if usv then + local hex = hex_string_to_num(usv) + local hex = hex_string_to_num + pattfile:write("1", unicode.utf8.char(hex_string_to_num(usv)), "1 ") + pattfile:write("% U+", usv, " ", char_name:lower(), "\n") + end +end + +pattfile:write("2", unicode.utf8.char(0x1361), "1 % U+1361 ETHIOPIC WORDSPACE\n") +pattfile:write("2", unicode.utf8.char(0x1362), "1 % U+1362 ETHIOPIC FULL STOP\n") + +pattfile:write"}" + +assert(io.close(pattfile)) diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb new file mode 100755 index 0000000000..9e6b3d9558 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb @@ -0,0 +1,135 @@ +#!/usr/bin/env ruby +# +# This script generates hyphenation patterns for Turkmen +# +# This script has been written by Mojca Miklavec <mojca dot miklavec dot lists at gmail dot com> + +# open file for writing the patterns +# $tr = File.new("hyph-tk.tex", "w") +# in TDS +$tr = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-tk.tex", "w") + +# write comments into the file +def add_comment(str) + $tr.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n") +end + +# define a class of vowels and consonants +# vowels are split into so that unnecessary permutations are not generated +front_vowels = %w{ä e i ö ü} +back_vowels = %w{a y o u} +consonants = %w{b ç d f g h j k l m n p r s t w ý z ň ž ş} +# This is to eliminate impossible combinations +common_suffix_consonants = %w{b ç d g j k l m n p s t ý z ş} + + +# start the file +add_comment( +"Hyphenation patterns for Turkmen (hyph-tk.tex) + +Author: Nazar Annagurban <nazartm at gmail.com> +License: Public domain +Version: 0.1 +Date: 16 March 2010 + +---------------------------------------------------------------------- + +The file has been auto-generated from generate_patterns_tk.rb +that is part of hyph-utf8. + +For more information about UTF-8 hyphenation patterns for TeX and +links to this file see + http://www.tug.org/tex-hyphen/ +") + +# we have the following comment for Basque: +# +# Some of the patterns below represent combinations that never +# happen in Turkmen. Would they happen, they would be hyphenated +# according to the rules. + +$tr.puts '\patterns{' +add_comment("Some suffixes are added through a hyphen. When hyphenating these words, a hyphen is added before the hyphen so that the line ends with a hyphen and the new line starts with a hyphen.") +$tr.puts "1-4" + +add_comment("Allow hyphen after a vowel if and only if there is a single consonant before next the vowel") +front_vowels.each do |v1| + consonants.each do |c| + front_vowels.each do |v2| + $tr.puts "#{v1}1#{c}#{v2}" + end + end +end + +back_vowels.each do |v1| + consonants.each do |c| + back_vowels.each do |v2| + $tr.puts "#{v1}1#{c}#{v2}" + end + end +end + +add_comment("These combinations occur in words of foreign origin or joined words") +consonants.each do |c| + $tr.puts "a1#{c}i" + $tr.puts "a1#{c}e" + $tr.puts "y1#{c}ä" + $tr.puts "y1#{c}i" + $tr.puts "y1#{c}e" + $tr.puts "o1#{c}i" + $tr.puts "o1#{c}e" + $tr.puts "u1#{c}i" + $tr.puts "u1#{c}e" + $tr.puts "i1#{c}a" + $tr.puts "i1#{c}o" + $tr.puts "e1#{c}a" + $tr.puts "e1#{c}o" + $tr.puts "ä1#{c}o" + $tr.puts "ä1#{c}a" + $tr.puts "ö1#{c}a" +end + +add_comment("Allow hyphen between two consonants (if there is only two of them), except when they are at the begining of the word") +consonants.each do |c1| + consonants.each do |c2| + $tr.puts "#{c1}1#{c2}" + $tr.puts ".#{c1}2#{c2}" + end +end + +add_comment("Patterns for triple consonants. There may be additions to this category, as this list is not exhaustive.") +common_suffix_consonants.each do |c| + $tr.puts "ý2t1#{c}" + $tr.puts "ý2n1#{c}" + $tr.puts "ý2d1#{c}" + $tr.puts "r2t1#{c}" + $tr.puts "ý2p1#{c}" + $tr.puts "l2p1#{c}" + $tr.puts "l2t1#{c}" + $tr.puts "g2t1#{c}" + $tr.puts "n2t1#{c}" + $tr.puts "r2k1#{c}" + $tr.puts "r2p1#{c}" + $tr.puts "k2t1#{c}" + $tr.puts "r2h1#{c}" + $tr.puts "s2t1#{c}" + $tr.puts "l2k1#{c}" + $tr.puts "w2p1#{c}" + $tr.puts "n2s1#{c}" + $tr.puts "r2s1#{c}" + $tr.puts "l2m1#{c}" +end + +add_comment("Exceptions and single word occurence patterns for words of foreign origin i.e. Russian") +$tr.puts "s2k1d" +$tr.puts "l1s2k" +$tr.puts "l1s2t" +$tr.puts "s1t2r" +$tr.puts "n2g1l" +$tr.puts "n1g2r" +$tr.puts "s2k1w" + +# end the file +$tr.puts '}' +$tr.close + diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README new file mode 100644 index 0000000000..c56aff64b3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/README @@ -0,0 +1,61 @@ + +See: +- http://mirror.ctan.org/language/turkish/hyphen/turk_hyf.c +- http://www.tug.org/TUGboat/Articles/tb09-1/tb20mackay.pdf + +------- +Original encoding: + +a +a= â +e +i ı +i= î +i: i +o +o= ô +o: ö +u +u: ü + +------- + +character / UTF-8 letter / EC code / Unicode / Unicode name + +@ -- arabic hamza +# -- ayn +b +c +c: ç E7 U+00E7 LATIN SMALL LETTER C WITH CEDILLA +d +d! ḍ -- U+1E0D LATIN SMALL LETTER D WITH DOT BELOW +d= ḏ -- U+1E0F LATIN SMALL LETTER D WITH LINE BELOW +f +g +g= ğ A7 U+011F LATIN SMALL LETTER G WITH BREVE +g: ġ -- U+0121 LATIN SMALL LETTER G WITH DOT ABOVE +h +h! ḥ -- U+1E25 LATIN SMALL LETTER H WITH DOT BELOW +h= ẖ -- U+1E96 LATIN SMALL LETTER H WITH LINE BELOW +j +k +k! ḳ -- U+1E33 LATIN SMALL LETTER K WITH DOT BELOW +l +m +n +n= ñ F1 U+00F1 LATIN SMALL LETTER N WITH TILDE +p +r +s +s! ṣ -- U+1E63 LATIN SMALL LETTER S WITH DOT BELOW +s= -- s with line below - not even in unicode +s: ş B3 U+015F LATIN SMALL LETTER S WITH CEDILLA +t +t! ṭ -- U+1E6D LATIN SMALL LETTER T WITH DOT BELOW +t= ṯ -- U+1E6F LATIN SMALL LETTER T WITH LINE BELOW +v +y +z +z! ẓ -- U+1E93 LATIN SMALL LETTER Z WITH DOT BELOW +z= ẕ -- U+1E95 LATIN SMALL LETTER Z WITH LINE BELOW +z: ż BB U+017C LATIN SMALL LETTER Z WITH DOT ABOVE diff --git a/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb new file mode 100755 index 0000000000..ec7610d8d3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb @@ -0,0 +1,120 @@ +#!/usr/bin/env ruby +# +# This script generates hyphenation patterns for modern Turkish +# +# "Algorithm" originally developed by: +# +# /**************************************************************************** +# * * +# * turk_hyf.c -- a program to produce PatGen-like hyphenation * +# * patterns for the Turkish Language. * +# * * +# * Copyright 1987, by Pierre A. MacKay. * +# * * +# * Humanities and Arts Computing Center * +# * Mail-Stop DW-10 * +# * University of Washington * +# * Seattle, Washington 98105 * +# * * +# ****************************************************************************/ +# +# Script written in June 2008 by Mojca Miklavec +# as part of "Unicode Hyphenation Patterns" project +# +# Thanks to previous work, modifications and suggestions by: +# - H. Turgut Uyar <uyar at itu.edu.tr> +# - S. Ekin Kocabas <kocabas at stanford.edu> + +# TODO: add a function to generate patterns for Ottoman Turkish + +# open file for writing the patterns +$tr = File.new("hyph-tr.tex", "w") +# in TDS +$tr = File.new("../../../../../tex/generic/hyph-utf8/patterns/tex/hyph-tr.tex", "w") + +# write comments into the file +def add_comment(str) + $tr.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n") +end + +# The patterns (and algorithm for generating them) were first meant +# for supporting Ottoman Turkish, but adapted for modern Turkish later. +# Turkish alphabet includes some special vowels (ıiöü) and consonants (çğş), +# but the patterns also support another three vowels (âîû): +# - a with circumflex is a necessity, it must be supported +# - ı and u with circumflex could be there, not as pressing as "a" but still +# Some letters such as z, k, s or t represent multiple letters in the old +# alphabet, so some transcriptions from Ottoman to Turkish use some accents on +# these for phonetic reasons, but there is no need to support them. +# +# Comment posted by: H. Turgut Uyar <uyar at itu.edu.tr> + +# define a class of vowels and consonants +vowels = %w{a â e ı i î o ö u ü û} +consonants = %w{b c ç d f g ğ h j k l m n p r s ş t v y z} + +# start the file +add_comment( + "hyph-tr.tex\n\n" + + "Turkish hyphenation patterns\n\n" + + "This file is auto-generated from source/generic/hyph-utf8/languages/tr/generate_patterns_tr.rb that is part of hyph-utf8.\n" + + "Please don't modify this file; modify the generating script instead.\n\n" + + "Copyright (C) 1987 Pierre A. MacKay\n" + + " 2008, 2011 TUG\n\n" + + "This program can redistributed and/or modified under the terms\n" + + "of the LaTeX Project Public License Distributed from CTAN\n" + + "archives in directory macros/latex/base/lppl.txt; either\n" + + "version 1 of the License, or (at your option) any later version.\n\n" + + "Credits:\n" + + "- algorithm developed by P. A. MacKay for the Ottoman Texts Project in 1987\n" + + "- rules adapted for modern Turkish by H. Turgut Uyar <uyar at itu.edu.tr>\n" + + "- initiative to improve Turkish patterns by S. Ekin Kocabas <kocabas at stanford.edu>\n" + + "- script written by Mojca Miklavec <mojca.miklavec.lists at gmail.com> in June 2008\n\n" + + "See also:\n" + + "- http://mirror.ctan.org/language/turkish/hyphen/turk_hyf.c\n" + + "- http://www.tug.org/TUGboat/Articles/tb09-1/tb20mackay.pdf\n\n" + + "Differences with Ottoman patterns:\n" + + "- adapted for the use on modern TeX engines, using UTF-8 charactes\n" + + "- only letters for Modern Turkish + âîû (the first one often needed, the other two don't hurt)\n" + + "- (if needed, support for Ottoman Turkish might be provided separately under language code 'ota')\n\n" + + "Changes:\n" + + "- 2008-06-25/27/28 - create this file by adapting Ottoman rules for modern Turkish\n" + + "- 2011-08-10 - add LPPL licence with permission of Pierre A. MacKay\n" +) +$tr.puts '\patterns{' + +# rules for hyphenation +add_comment("prohibit hyphen before vowels, allow after") +vowels.each do |vowel| + $tr.puts "2#{vowel}1" +end + +add_comment("allow hyphen either side of consonants") +consonants.each do |cons| + $tr.puts "1#{cons}1" +end + +add_comment("prevent e-cek at end of word") +$tr.puts "2e2cek." + +add_comment("prohibit hyphen before pair of consonants\nmany pairs generated here are impossible anyway") +consonants.each do |c1| + consonants.each do |c2| + $tr.puts "2#{c1}#{c2}" + end +end + +add_comment("allow hyphen between vowels, but not after second vowel of pair\nseveral phonetically impossible pairs here") +vowels.each do |v1| + vowels.each do |v2| + $tr.puts "#{v1}3#{v2}2" + end +end + +add_comment("a couple of consonant-clusters") +$tr.puts 'tu4r4k' +$tr.puts 'm1t4rak' + +# end the file +$tr.puts '}' +$tr.close diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb new file mode 100644 index 0000000000..2259f2ff1d --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen.rb @@ -0,0 +1 @@ +require_relative 'hyphen/language' diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml new file mode 100644 index 0000000000..f96530da39 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml @@ -0,0 +1,452 @@ +--- +donald_knuth: !ruby/object:TeX::Hyphen::Author + name: Donald + surname: Knuth + email: + contacted1: false + contacted2: false +peter_heslin: !ruby/object:TeX::Hyphen::Author + name: Peter + surname: Heslin + email: + contacted1: false + contacted2: false +dimitrios_filippou: !ruby/object:TeX::Hyphen::Author + name: Dimitrios + surname: Filippou + email: dimitrios{dot}filippou{at}riotinto{dot}com + contacted1: true + contacted2: true +claudio_beccari: !ruby/object:TeX::Hyphen::Author + name: Claudio + surname: Beccari + email: claudio{dot}beccari{at}gmail{dot}com + contacted1: true + contacted2: true +juan_aguirregabiria: !ruby/object:TeX::Hyphen::Author + name: Juan M. + surname: Aguirregabiria + email: juanmari{dot}aguirregabiria{at}ehu.es + contacted1: true + contacted2: true +igor_marinovic: !ruby/object:TeX::Hyphen::Author + name: Igor + surname: Marinović + email: marinowski{at}gmail.com + contacted1: true + contacted2: true +tilla_fick: !ruby/object:TeX::Hyphen::Author + name: Tilla + surname: Fick + email: fick{dot}tilla{at}gmail{dot}com + contacted1: true + contacted2: true +chris_swanepoel: !ruby/object:TeX::Hyphen::Author + name: Chris + surname: Swanepoel + email: cj{at}swanepoel{dot}net + contacted1: true + contacted2: true +matjaz_vrecko: !ruby/object:TeX::Hyphen::Author + name: Matjaž + surname: Vrečko + email: matjaz{at}mg-soft{dot}si + contacted1: true + contacted2: true +goncal_badenes: !ruby/object:TeX::Hyphen::Author + name: Gonçal + surname: Badenes + email: g{dot}badenes{at}ieee.org + contacted1: false + contacted2: false +pavel_sevecek: !ruby/object:TeX::Hyphen::Author + name: Pavel + surname: Ševeček + email: pavel{at}lingea{dot}cz + contacted1: false + contacted2: false +# email doesn't work +jana_chlebikova: !ruby/object:TeX::Hyphen::Author + name: Jana + surname: Chlebíková + email: chlebikj{at}dcs{dot}fmph{dot}uniba{dot}sk + contacted1: false + contacted2: false +yannis_haralambous: !ruby/object:TeX::Hyphen::Author + name: Yannis + surname: Haralambous + email: yannis{dot}haralambous{at}telecom-bretagne{dot}eu + contacted1: true + contacted2: false +frank_jensen: !ruby/object:TeX::Hyphen::Author + name: Frank + surname: Jensen + email: frank{dot}jensen{at}hugin{dot}com + contacted1: true + contacted2: true +sergei_pokrovsky: !ruby/object:TeX::Hyphen::Author + name: Sergei + surname: Pokrovsky + email: sergio{dot}pokrovskij{at}gmail{dot}com + contacted1: true + contacted2: true +javier_bezos: !ruby/object:TeX::Hyphen::Author + name: Javier + surname: Bezos + email: jbezos{at}tex-tipografia{dot}com + contacted1: true + contacted2: true +een_saar: !ruby/object:TeX::Hyphen::Author + name: Enn + surname: Saar + email: saar{at}aai{dot}ee + contacted1: false + contacted2: false +dejan_muhamedagic: !ruby/object:TeX::Hyphen::Author + name: Dejan + surname: Muhamedagić + email: dejan{at}hello-penguin{dot}com + contacted1: true + contacted2: true +brian_wilson: !ruby/object:TeX::Hyphen::Author + name: Brian + surname: Wilson + email: bountonw{at}gmail{dot}com + contacted1: true + contacted2: true +arthur_reutenauer: !ruby/object:TeX::Hyphen::Author + name: Arthur + surname: Reutenauer + email: arthur{dot}reutenauer{at}normalesup{dot}org + contacted1: true + contacted2: true +mojca_miklavec: !ruby/object:TeX::Hyphen::Author + name: Mojca + surname: Miklavec + email: mojca{dot}miklavec{dot}lists{at}gmail{dot}com + contacted1: true + contacted2: true +santhosh_thottingal: !ruby/object:TeX::Hyphen::Author + name: Santhosh + surname: Thottingal + email: santhosh{dot}thottingal{at}gmail{dot}com> + contacted1: true + contacted2: true +# email doesn't work +yves_codet: !ruby/object:TeX::Hyphen::Author + name: Yves + surname: Codet + email: ycodet{at}club-internet{dot}fr + contacted1: true + contacted2: true +rune_kleveland: !ruby/object:TeX::Hyphen::Author + name: Rune + surname: Kleveland + email: + contacted1: false + contacted2: false +# email doesn't work +ole_michael_selberg: !ruby/object:TeX::Hyphen::Author + name: Ole Michael + surname: Selberg + email: o{dot}m{dot}selberg{at}c2i{dot}net + contacted1: true + contacted2: true +dorjgotov_batmunkh: !ruby/object:TeX::Hyphen::Author + name: Dorjgotov + surname: Batmunkh + email: bataak{at}gmail{dot}com + contacted1: true + contacted2: true +nazar_annagurban: !ruby/object:TeX::Hyphen::Author + name: Nazar + surname: Annagurban + email: nazartm{at}gmail{dot}com + contacted1: false + contacted2: false +jan_michael_rynning: !ruby/object:TeX::Hyphen::Author + name: Jan Michael + surname: Rynning + email: + contacted1: false + contacted2: false +eduard_werner: !ruby/object:TeX::Hyphen::Author + name: Eduard + surname: Werner + email: edi{dot}werner{at}gmx{dot}de + contacted1: false + contacted2: false +werner_lemberg: !ruby/object:TeX::Hyphen::Author + name: Werner + surname: Lemberg + email: wl{at}gnu{dot}org + contacted1: true + contacted2: true +# email doesn't work +pedro_j_de_rezende: !ruby/object:TeX::Hyphen::Author + name: Pedro J. + surname: de Rezende + email: rezende{at}ddc{dot}unicamp{dot}br + contacted1: false + contacted2: false +j_joao_dias_almeida: !ruby/object:TeX::Hyphen::Author + name: J. Joao + surname: Dias Almeida + email: jj{at}di{dot}uminho{dot}pt + contacted1: + contacted2: +# email doesn't work +piet_tutelaers: !ruby/object:TeX::Hyphen::Author + name: Piet + surname: Tutelaers + email: p{dot}t{dot}h{dot}tutelaers{at}tue{dot}nl + contacted1: false + contacted2: false +vytas_statulevicius: !ruby/object:TeX::Hyphen::Author + name: Vytas + surname: Statulevičius + email: vytas{at}vtex{dot}nl + contacted1: false + contacted2: false +sigitas_tolusis: !ruby/object:TeX::Hyphen::Author + name: Sigitas + surname: Tolušis + email: sigitas{at}vtex{dot}lt + contacted1: false + contacted2: false +janis_vilims: !ruby/object:TeX::Hyphen::Author + name: Janis + surname: Vilims + email: jvilims{at}apollo{dot}lv + contacted1: false + contacted2: false +joerg_knappen: !ruby/object:TeX::Hyphen::Author + name: Jörg + surname: Knappen + email: jknappen{at}web{dot}de + contacted1: true + contacted2: true +medeni_shemde: !ruby/object:TeX::Hyphen::Author + name: Medeni + surname: Shemdê + email: + contacted1: false + contacted2: false +terry_mart: !ruby/object:TeX::Hyphen::Author + name: Terry + surname: Mart + email: mart{at}kph{dot}uni-mainz{dot}de + contacted1: false + contacted2: false +# email doesn't work +jorgen_pind: !ruby/object:TeX::Hyphen::Author + name: Jorgen + surname: Pind + email: jorgen{at}lexis{dot}hi{dot}is + contacted1: false + contacted2: false +marteinn_sverrisson: !ruby/object:TeX::Hyphen::Author + name: Marteinn + surname: Sverrisson + email: + contacted1: false + contacted2: false +# email doesn't work +kristinn_gylfason: !ruby/object:TeX::Hyphen::Author + name: Kristinn + surname: Gylfason + email: kristgy{at}ieee{dot}org + contacted1: false + contacted2: false +# email doesn't work +kevin_p_scannell: !ruby/object:TeX::Hyphen::Author + name: Kevin P. + surname: Scannell + email: scanell{at}slu{dot}edu + contacted1: false + contacted2: false +# email doesn't work +peter_kleiweg: !ruby/object:TeX::Hyphen::Author + name: Peter + surname: Kleiweg + email: p{dot}c{dot}c{dot}kleiweg{at}rug{dot}nl + contacted1: false + contacted2: false +hanna_kolodziejska: !ruby/object:TeX::Hyphen::Author + name: Hanna + surname: Kołodziejska + email: + contacted1: false + contacted2: false +boguslaw_jackowski: !ruby/object:TeX::Hyphen::Author + name: Bogusław + surname: Jackowski + email: + contacted1: true + contacted2: true +marek_rycko: !ruby/object:TeX::Hyphen::Author + name: Marek + surname: Ryćko + email: + contacted1: false + contacted2: false +vladimir_volovich: !ruby/object:TeX::Hyphen::Author + name: Vladimir + surname: Volovich + email: # TODO add e-mail address + contacted1: true + contacted2: true +alexander_i_lebedev: !ruby/object:TeX::Hyphen::Author + name: Alexander I. # Not sure were 'I' belongs + surname: Lebedev + email: swan{at}scon155{dot}phys{dot}msu{dot}su + contacted1: false + contacted2: false +# first email doesn't work +maksym_polyakov: !ruby/object:TeX::Hyphen::Author + name: Maksym + surname: Polyakov + email: polyama{at}auburn{dot}edu # Second e-mail address in ukrhypmp.tex: mpoliak@i.com.ua + contacted1: false + contacted2: false +adrian_rezus: !ruby/object:TeX::Hyphen::Author + name: Adrian + surname: Rezus + email: adriaan{at}{sci,cs}{dot}kun{dot}nl + contacted1: false + contacted2: false +# email doesn't work +sahak_petrosyan: !ruby/object:TeX::Hyphen::Author + name: Sahak + surname: Petrosyan + email: sahak{at}mit{dot}edu + contacted1: true + contacted2: true # I think "true, true" is right. Arthur +dominik_wujastyk: !ruby/object:TeX::Hyphen::Author + name: Dominik + surname: Wujastyk + email: wujastyk{at}gmail{dot}com + contacted1: false + contacted2: false +graham_toal: !ruby/object:TeX::Hyphen::Author + name: Graham + surname: Toal + email: + contacted1: false + contacted2: false +donald_e_knuth: !ruby/object:TeX::Hyphen::Author + name: Donald E. + surname: Knuth + email: # Don doesn't use e-mail ;-) + contacted1: false + contacted2: false +gerard_d_c_kuiken: !ruby/object:TeX::Hyphen::Author + name: Gerard D.C. + surname: Kuiken + email: + contacted1: false + contacted2: false +pierre_mackay: !ruby/object:TeX::Hyphen::Author + name: P. A. + surname: MacKay + email: + contacted1: true + contacted2: true +h_turgut_uyar: !ruby/object:TeX::Hyphen::Author + name: H. Turgut + surname: Uyar + email: uyar{at}itu{dot}edu{tr} + contacted1: true + contacted2: true +# email doesn't work +s_ekin_kocabas: !ruby/object:TeX::Hyphen::Author + name: S. Ekin + surname: Kocabas + email: kocabas{at}stanford{dot}edu + contacted1: true + contacted2: true +bence_nagy: !ruby/object:TeX::Hyphen::Author + name: Bence + surname: Nagy + email: nagybence{at}tipogral{dot}hu + contacted1: true + contacted2: true +kauko_saarinen: !ruby/object:TeX::Hyphen::Author + name: Kauko + surname: Saarinen + email: + contacted1: false + contacted2: false +fred_karlsson: !ruby/object:TeX::Hyphen::Author + name: Fred + surname: Karlsson + email: + contacted1: false + contacted2: false +rene_bastian: !ruby/object:TeX::Hyphen::Author # TODO make contact + name: René + surname: Bastian + email: + contacted1: false + contacted2: false +daniel_flipo: !ruby/object:TeX::Hyphen::Author # TODO make contact + name: Daniel + surname: Flipo + email: + contacted1: false + contacted2: false +bernard_gaulle: !ruby/object:TeX::Hyphen::Author # Deceased... + name: Bernard + surname: Gaulle + email: + contacted1: false + contacted2: false +theppitak_karoonboonyanan: !ruby/object:TeX::Hyphen::Author + name: Theppitak + surname: Karoonboonyanan + email: theppitak{at}gmail{dot}com + contacted1: true + contacted2: true +levan_shoshiashvili: !ruby/object:TeX::Hyphen::Author + name: Levan + surname: Shoshiashvili + email: shoshia{at}hotmail{dot}com + contacted1: true + contacted2: true +# email doesn't work +javier_mugica: !ruby/object:TeX::Hyphen::Author + name: Javier + surname: Múgica + email: javier{at}digi21{dot}eu + contacted1: true + contacted2: true +georgi_boshnakov: !ruby/object:TeX::Hyphen::Author + name: Georgi + surname: Boshnakov + email: georgi{dot}boshnakov{at}manchester{dot}ac{dot}uk + contacted1: true + contacted2: true +mike_kroutikov: !ruby/object:TeX::Hyphen::Author + name: Mike + surname: Kroutikov + email: pgmmpk{at}gmail{dot}com + contacted1: true + contacted2: true +aleksandr_andreev: !ruby/object:TeX::Hyphen::Author + name: Aleksandr + surname: Andreev + email: '' + contacted1: true + contacted2: true +maksim_salau: !ruby/object:TeX::Hyphen::Author + name: Maksim + surname: Salau + email: maksim{dot}salau{at}gmail{dot}com + contacted1: true + contacted2: false +wie_ming_ang: !ruby/object:TeX::Hyphen::Author + name: Wie-Ming + surname: Cittānurakkho Bhikkhu + email: wiemingang{at}gmail{dot}com + contacted1: true + contacted2: true diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb new file mode 100644 index 0000000000..3d3437d74b --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/language.rb @@ -0,0 +1,364 @@ +require 'yaml' +require 'hydra' +require 'byebug' unless ENV['RACK_ENV'] == "production" + +require_relative 'path' + +class String + def superstrip + strip.gsub /%.*$/, '' + end + + def supersplit + strip.gsub(/\s+/m,"\n").split("\n") + end + + def safe + gsub /[\s-]/, '' + end + + def titlecase + split.map(&:capitalize).join(' ') + end +end + +module TeX + module Hyphen + class InvalidMetadata < StandardError; end + class NoAuthor < InvalidMetadata; end + class NoLicence < InvalidMetadata; end + + include PATH + + class Author + def initialize(name,surname,email,contacted1,contacted2) + @name = name + @surname = surname + @email = email + # this mostly means if email has been recently checked + @contacted1 = contacted1 + # this means if we made more cooperation with author, + # exchanging patches etc. + @contacted2 = contacted2 + end + + attr_reader :name, :surname, :email + + def self.authors + @@authors ||= YAML::load File.read File.expand_path 'authors.yml', __dir__ + end + + def self.all + authors.values + end + + def self.[] a + authors[a] + end + end + + class Language + attr_reader :bcp47 + + @@eohmarker = '=' * 42 + + def iso639 + @bcp47.split('-').first + end + + # FIXME Find something better than all that extract_metadata unless @blah nonsens + def babelname + extract_metadata unless @babelname + @babelname + end + + def description + extract_metadata unless @description + @description + end + + def use_old_loader + extract_metadata unless @use_old_loader + @use_old_loader + end + + def use_old_patterns_comment + extract_metadata unless @use_old_patterns_comment + @use_old_patterns_comment + end + + def legacy_patterns + extract_metadata unless @legacy_patterns + @legacy_patterns + end + + def message + extract_metadata unless @message + @message + end + + def known_bugs + extract_metadata unless @known_bugs + @known_bugs + end + + def initialize(bcp47 = nil) + @bcp47 = bcp47 + end + + def self.languages + @@languages ||= Dir.glob(File.join(PATH::TEX, 'hyph-*.tex')).inject [] do |languages, texfile| + bcp47 = texfile.gsub /^.*\/hyph-(.*)\.tex$/, '\1' + # languages << [bcp47, Language.new(bcp47)] + languages << [bcp47, nil] + end.to_h + end + @@languages = Language.languages + + def self.all + @@all ||= languages.map do |bcp47, language| + next if bcp47 == 'sr-cyrl' # FIXME Remove later + @@languages[bcp47] ||= Language.new(bcp47) + end.compact + end + + def self.all_by_iso639 + all.inject(Hash.new) do |maintags, language| + (maintags[language.iso639] ||= Array.new) << language + maintags + end + end + + def self.find_by_bcp47(bcp47) + languages[bcp47] + end + + def private_use? + @bcp47.length == 3 || @bcp47[3] == '-' and @bcp47[0] == 'q' and ('a'..'t').include? @bcp47[1] + end + + # TODO This should probably become “macrolanguage name” or something similar + # @@displaynames = { + # 'el' => 'Greek', + # 'nb' => 'Norwegian', + # 'nn' => 'Norwegian', + # 'sh' => 'Serbian', + # } + + def licences + extract_metadata unless @licences + @licences + end + + def lefthyphenmin + extract_metadata unless @lefthyphenmin + @lefthyphenmin + end + + def righthyphenmin + extract_metadata unless @righthyphenmin + @righthyphenmin + end + + # Strictly speaking a misnomer, because grc-x-ibycus should also return true. + # But useful for a number of apostrophe-related routines + def isgreek? + ['grc', 'el-polyton', 'el-monoton'].include? @bcp47 + end + + def serbian? + @bcp47 =~ /^sh-/ + end + + def italic? + ['it', 'pms', 'rm'].include? @bcp47 + end + + def has_apostrophes? + begin + !isgreek? && patterns.any? { |p| p =~ /'/ } + rescue Errno::ENOENT + false + end + end + + def has_hyphens? + begin + patterns.any? { |p| p =~ /-/ } + rescue Errno::ENOENT + false + end + end + + def extract_apostrophes + plain, with_apostrophe = Array.new, nil + + patterns.each do |pattern| + plain << pattern + if pattern =~ /'/ && !isgreek? + pattern_with_apostrophe = pattern.gsub(/'/,"’") + plain << pattern_with_apostrophe + (with_apostrophe ||= []) << pattern_with_apostrophe + end + end + + { plain: plain, with_apostrophe: with_apostrophe } + end + + def extract_characters # FIXME Turkish and others iİ; check Church Slavonic + characters = Array.new + + characters_indexes = patterns.join.gsub(/[.0-9]/,'').unpack('U*').sort.uniq + characters_indexes.each do |c| + ch = [c].pack('U') + characters << ch + ch.upcase + characters << "’’" if ch == "'" && !isgreek? + end + + characters + end + + def comments_and_licence # Major TODO extract everything into YAML, and write .yml + @comments_and_licence ||= readtexfile.gsub(/(.*)\\patterns.*/m,'\1') + end + + def authors + extract_metadata unless @authors + @authors + end + + def github_link + sprintf 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-%s.tex', @bcp47 + end + + def <=>(other) + # TODO Remove when practical + a = self.babelname + b = other.babelname + + if a == 'serbian' && b == 'serbianc' + return -1 + elsif a == 'serbianc' && b == 'serbian' + return 1 + end + + self.bcp47 <=> other.bcp47 + end + + @@texfile = Hash.new + def readtexfile(bcp47 = @bcp47) + begin + @@texfile[bcp47] ||= File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', bcp47))) + rescue Errno::ENOENT + @@texfile[bcp47] = "" + end + end + + def patterns + @patterns ||= if @bcp47 == 'eo' then + readtexfile.superstrip. + gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1'). + # + gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.'). + gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.'). + gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.'). + # + supersplit + else + readtexfile(if ['nb', 'nn'].include? @bcp47 then 'no' else @bcp47 end).superstrip. + gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1'). + supersplit + end + end + + def exceptions + unless @exceptions + if readtexfile.superstrip.index('\hyphenation') + @exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit + if @exceptions != "" + @hyphenation = @exceptions.inject [] do |exceptions, exception| + # byebug unless exception + exceptions << [exception.gsub('-', ''), exception] + end.to_h + else + @hyphenation = { } + end + else + @exceptions = "" + @hyphenation = { } + end + end + + @exceptions + end + + def hyphenate(word) + exceptions + return @hyphenation[word] if @hyphenation[word] # FIXME Better name + + unless @hydra + begin + # byebug + metadata = extract_metadata + @hydra = Hydra.new patterns, :lax, '', metadata + rescue InvalidMetadata + @hydra = Hydra.new patterns + end + end + @hydra.showhyphens(word) + end + + def extract_metadata + header = "" + File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', @bcp47))).each_line do |line| + break if line =~ /\\patterns|#{@@eohmarker}/ + header += line.gsub(/^% /, '').gsub(/%.*/, '') + end + begin + metadata = YAML::load header + raise InvalidMetadata unless metadata.is_a? Hash + rescue Psych::SyntaxError + raise InvalidMetadata + end + + @name = metadata.dig('language', 'name') + @lefthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'left') || metadata.dig('hyphenmins', 'generation', 'left') + @righthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'right') || metadata.dig('hyphenmins', 'generation', 'right') + # FIXME That’s not nice + if respond_to? :synonyms + @synonyms = metadata.dig('texlive', 'synonyms') || [] + @encoding = metadata.dig('texlive', 'encoding') + end + @message = metadata.dig('texlive', 'message') + @legacy_patterns = metadata.dig('texlive', 'legacy_patterns') + @use_old_loader = metadata.dig('texlive', 'use_old_loader') + @use_old_patterns_comment = metadata.dig('texlive', 'use_old_patterns_comment') + @description = metadata.dig('texlive', 'description') + @babelname = metadata.dig('texlive', 'babelname') + @package = metadata.dig('texlive', 'package') + licences = metadata.dig('licence') + raise NoLicence unless licences + licences = [licences] unless licences.is_a? Array + @licences = licences.map do |licence| + raise bcp47 if licence.is_a? String + next if licence.values == [nil] + licence.dig('name') || 'custom' + end.compact + authors = metadata.dig('authors') + if authors + @authors = authors.map do |author| + author['name'] + end + else + @authors = nil + raise NoAuthor.new # FIXME + end + @known_bugs = metadata.dig('known_bugs') + + # raise NoAuthor unless @authors && @authors.count > 0 # TODO Later ;-) AR 2018-09-13 + + metadata + end + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml new file mode 100644 index 0000000000..bc02973938 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml @@ -0,0 +1,94 @@ +german: + dependency: dehyph + description: |- + Hyphenation patterns for German in T1/EC and UTF-8 encodings, + for traditional and reformed spelling, including Swiss German. + The package includes the latest patterns from dehyph-exptl + (known to TeX under names 'german', 'ngerman' and 'swissgerman'), + however 8-bit engines still load old versions of patterns + for 'german' and 'ngerman' for backward-compatibility reasons. + Swiss German patterns are suitable for Swiss Standard German + (Hochdeutsch) not the Alemannic dialects spoken in Switzerland + (Schwyzerduetsch). + There are no known patterns for written Schwyzerduetsch. +# for Russian and Ukrainian (until we implement the new functionality at least) +russian: + dependency: ruhyphen +ukrainian: + dependency: ukrhyph +greek: + doc: doc/generic/elhyphen + shortdesc: Modern Greek + description: |- + Hyphenation patterns for Modern Greek in monotonic and polytonic + spelling in LGR and UTF-8 encodings. Patterns in UTF-8 use two code + positions for each of the vowels with acute accent (a.k.a tonos, + oxia), e.g., U+03AC, U+1F71 for alpha. +hungarian: + doc: doc/generic/huhyphen +arabic: + shortdesc: (No) Arabic +farsi: + shortdesc: (No) Persian +chinese: + shortdesc: Chinese pinyin + description: |- + Hyphenation patterns for unaccented transliterated Mandarin Chinese + (pinyin) in T1/EC and UTF-8 encodings. The latter can hyphenate pinyin + with or without tone markers; the former only without. +norwegian: + shortdesc: Norwegian Bokmal and Nynorsk + description: |- + Hyphenation patterns for Norwegian Bokmal and Nynorsk in T1/EC and + UTF-8 encodings. +churchslavonic: + shortdesc: Church Slavonic +uppersorbian: + shortdesc: Upper Sorbian +ethiopic: + shortdesc_full: Hyphenation patterns for Ethiopic scripts +mongolian: + shortdesc_full: Mongolian hyphenation patterns in Cyrillic script + description: |- + Hyphenation patterns for Mongolian in T2A, LMC and UTF-8 encodings. + LMC encoding is used in MonTeX. The package includes two sets of + patterns that will hopefully be merged in future. +latin: + description: |- + Hyphenation patterns for Latin in T1/EC and UTF-8 encodings, + mainly in modern spelling (u when u is needed and v when v is needed), + medieval spelling with the ligatures \ae and \oe and the (uncial) + lowercase 'v' written as a 'u' is also supported. Apparently + there is no conflict between the patterns of modern Latin and + those of medieval Latin. + Hyphenation patterns for the Classical Latin in T1/EC and UTF-8 + encodings. Classical Latin hyphenation patterns are different from + those of 'plain' Latin, the latter being more adapted to modern Latin. + Hyphenation patterns for the Liturgical Latin in T1/EC and UTF-8 + encodings. +english: + description: |- + Additional hyphenation patterns for American and British + English in ASCII encoding. The American English patterns + (usenglishmax) greatly extend the standard patterns from Knuth + to find many additional hyphenation points. British English + hyphenation is completely different from US English, so has its + own set of patterns. +indic: + description: |- + Hyphenation patterns for Assamese, Bengali, Gujarati, Hindi, Kannada, + Malayalam, Marathi, Oriya, Panjabi, Tamil and Telugu for Unicode + engines. +"ancient greek": + description: |- + Hyphenation patterns for Ancient Greek in LGR and UTF-8 encodings, + including support for (obsolete) Ibycus font encoding. + Patterns in UTF-8 use two code positions for each of the vowels with + acute accent (a.k.a tonos, oxia), e.g., U+03AE, U+1F75 for eta. +serbian: + description: |- + Hyphenation patterns for Serbian in T1/EC, T2A and UTF-8 encodings. + For 8-bit engines the patterns are available separately as 'serbian' + in T1/EC encoding for Latin script and 'serbianc' in T2A encoding for + Cyrillic script. Unicode engines should only use 'serbian' + which has patterns in both scripts combined. diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb new file mode 100644 index 0000000000..fda422b11a --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/path.rb @@ -0,0 +1,25 @@ +module TeX + module Hyphen + module PATH + ROOT = File.expand_path('../../../../../../../..', __FILE__) + + TeXROOT = File.join(ROOT, 'hyph-utf8') + TeX_GENERIC = File.join(TeXROOT, 'tex', 'generic') + PAT = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns') + TXT = File.join(PAT, 'txt') + TEX = File.join(PAT, 'tex') + PTEX = File.join(PAT, 'ptex') + QUOTE = File.join(PAT, 'quote') + LOADER = File.join(TeX_GENERIC, 'hyph-utf8', 'loadhyph') + + SUPPORT = File.join(TeXROOT, '%s', 'generic', 'hyph-utf8', 'languages', '*') + + HYPHU8 = File.join('tex', 'generic', 'hyph-utf8') + + TL = File.join(ROOT, 'TL') + LANGUAGE_DAT = File.join(PATH::TL, 'texmf-dist', 'tex', 'generic', 'config') + # hyphen-foo.tlpsrc for TeX Live + TLPSRC = File.join(PATH::TL, 'tlpkg', 'tlpsrc') + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb new file mode 100644 index 0000000000..45dd33da04 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb @@ -0,0 +1,22 @@ +require_relative 'texlive/source' +require_relative 'texlive/loader' +require_relative 'texlive/package' + +module TeX + module Hyphen + module TeXLive + def synonyms + extract_metadata unless @synonyms + @synonyms + end + + def encoding + extract_metadata unless @encoding + @encoding + end + + include Source + include Loader + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb new file mode 100644 index 0000000000..2fcb3945d3 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb @@ -0,0 +1,193 @@ +module TeX + module Hyphen + module TeXLive + module Loader + + def unicode_only? # TODO Spec out + ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47 + end + + def string_enc + (encoding == nil) ? "" : encoding.upcase + " " + end + + def lcchars + if isgreek? + # some catcodes for XeTeX + return { + lccode: { + 0x0027 => '\'', + 0x2019 => '’', + 0x02BC => 'ʼ', + 0x1FBD => '᾽', + 0x1FBF => '᾿', + } + } + end + + return { } unless unicode_only? + + default = { + comment: 'Set \lccode for ZWNJ and ZWJ.', + lccode: { + 0x200C => false, + 0x200D => false, + } + } + + { + 'mul-ethi' => { + comment: 'Set \lccode for Ethiopian word space.', + lccode: { + 0x1361 => false, + 0x1362 => false, + } + }, + 'cu' => { + comment: 'fix lccodes for some characters (they were recently included in Unicode)', + lccode: { + 0x1C82 => 'sharp o in lowercase "uk"', + 0x1DF6 => false, + 0x1DF7 => false, + 0x1DF8 => false, + 0x1DF9 => false, + 0xA69E => false, + 0x1C86 => false, + 0xA67E => false, + 0xFE2E => false, + 0xFE2F => false, + } + }, + 'sa' => { + comment: 'Set \lccode for ZWNJ and ZWJ.', + lccode: { + 0x200C => false, + 0x200D => "\n% Set \\lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.", + 0x0CF1 => false, + 0x0CF2 => false, + } + }, + }[@bcp47] || default + end + + def utf8_engine_message + if serbian? + 'UTF-8 Serbian hyphenation patterns' + else + sprintf('UTF-8 %s', message) + end + end + + def non_utf8_engine_message + if unicode_only? + sprintf('No %s - only for Unicode engines', message) + else + sprintf('%s%s', string_enc, message) + end + end + + def engine_message(engine) + if engine == 'UTF-8' + { + comment: 'Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument', + message: utf8_engine_message + } + else # engine is 8-bit or pTeX + { + comment: engine + + if engine == '8-bit' then " engine (such as TeX or pdfTeX)" else "" end, + message: non_utf8_engine_message + } + end + end + + def input_8bit_file + if @bcp47 == 'la-x-liturgic' + { + input: [pTeX_patterns] + } + elsif use_old_patterns_comment + # explain why we are still using the old patterns + { + comment: use_old_patterns_comment, + input: [legacy_patterns] + } + elsif !italic? + { + input: [sprintf('conv-utf8-%s.tex', encoding), sprintf('hyph-%s.tex', @bcp47)] + } + else + { + input: [sprintf('hyph-%s.tex', @bcp47)] + } + end + end + + def input_utf8_line + if serbian? + { + comment: 'We load both scripts at the same time to simplify usage', + input: ['hyph-sh-latn.tex', 'hyph-sh-cyrl.tex'] + } + else + { + input: [sprintf('hyph-%s.tex', @bcp47)] + } + end + end + + def format_inputs(specification) + if specification.is_a? Array + return specification.map do |hash| + format_inputs(hash) + end + end + + byebug unless specification.is_a? Hash + comment = specification[:comment] + message = specification[:message] + if comment then [sprintf('%% %s', comment)] else [] end + + if message then [sprintf('\\message{%s}', message)] else [] end + + (specification[:lccode] || []).map do |code, comment| + sprintf '\\lccode"%04X="%04X%s', code, code, if comment then sprintf ' %% %s', comment else '' end + end + + (specification[:input] || []).map do |input| + sprintf '\\input %s', input + end + end + + def input_line(engine) + if engine == '8-bit' + input_8bit_file + elsif engine == 'pTeX' + { input: [pTeX_patterns] } + elsif engine == 'UTF-8' + input_utf8_line + end + end + + def utf8_chunk + [ + engine_message('UTF-8'), + # lccodes + lcchars, + input_line('UTF-8'), + if has_apostrophes? then { input: [sprintf('hyph-quote-%s.tex', bcp47)] } end + ].compact + end + + def nonutf8_chunk(engine) + [engine_message(engine), unless unicode_only? then input_line(engine) end].compact + end + + def loadhyph + if use_old_loader + legacy_patterns + else + sprintf 'loadhyph-%s.tex', @bcp47.gsub(/^sh-/, 'sr-') + end + end + end + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb new file mode 100644 index 0000000000..69481a4177 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb @@ -0,0 +1,126 @@ +module TeX + module Hyphen + module TeXLive + class Package + attr_reader :name + + @@metadata = YAML::load File.read File.expand_path '../packages.yml', __dir__ + + def initialize(name) + @name = name + @languages = [] + end + + def add_language(language) + @languages << language + @languages.sort! + end + + def languages + @languages + end + + def self.packages + # a hash with the names of TeX Live packages, either individual language names, + # or an array of languages as the value + @@packages ||= Language.all.inject(Hash.new) do |packages, language| + if name = language.package || language.babelname # Package name, otherwise single language + packages[name] ||= Package.new(name) # Create new package if necessary + packages[name].add_language language + end + + packages + end + end + + def self.all + packages.values + end + + def description_s + override = @@metadata.dig(@name, 'shortdesc_full') + return override if override + + leader = @@metadata.dig(@name, 'shortdesc') || @name.titlecase + sprintf '%s hyphenation patterns', leader + end + + def description + description = @@metadata.dig(@name, 'description') + if description + description + elsif @languages.count == 1 + @languages.first.description + else + '' + end + end + + def has_dependency? + @@metadata.dig(name, 'dependency') + end + + def list_dependencies + dependencies = [ + "depend hyphen-base", + "depend hyph-utf8", + ] + + # external dependencies + if dependency = has_dependency? + dependencies << sprintf("depend %s", dependency) + end + + dependencies + end + + def list_support_files(type) # type is ‘doc’ or ‘source’ + # Cache directory contents + (@dirlist ||= { })[type] ||= Dir.glob(sprintf(PATH::SUPPORT, type)).select do |file| + File.directory?(file) + end.map do |dir| + dir.gsub /^.*\//, '' + end + + files = (languages.map(&:bcp47) & @dirlist[type]).map do |bcp47| + sprintf("%s/generic/hyph-utf8/languages/%s", type, bcp47) + end + + if special = @@metadata.dig(name, type) + files << special + end + + files + end + + def list_run_files + files = [] + files << "tex/generic/hyph-utf8/patterns/tex/hyph-no.tex" if name == "norwegian" + + files = languages.inject(files) do |files, language| + files + language.list_run_files + end + + unless has_dependency? + languages.each do |language| + if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop' + files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns) + end + end + end + + files + end + + def <=>(other) + name <=> other.name + end + + # TODO Spec out + def self.find(name) + packages[name] + end + end + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb new file mode 100644 index 0000000000..61292dcd3b --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb @@ -0,0 +1,117 @@ +module TeX + module Hyphen + module TeXLive + module Source + def list_synonyms + if synonyms && synonyms.length > 0 + sprintf " synonyms=%s", synonyms.join(',') + else + '' + end + end + + def list_hyphenmins + lmin = lefthyphenmin + rmin = righthyphenmin + sprintf "lefthyphenmin=%s \\\n\trighthyphenmin=%s", lmin, rmin + end + + # ext: 'pat' or 'hyp' + # filetype: 'patterns' or 'exceptions' + def plain_text_line(ext, filetype) # TODO Figure out if we will sr-cyrl to be generated again + return "" if ['ar', 'fa', 'grc-x-ibycus', 'mn-cyrl-x-lmc'].include? @bcp47 + + if @bcp47 =~ /^sh-/ + # TODO Warning AR 2018-09-12 + filename = sprintf 'hyph-sh-latn.%s.txt,hyph-sh-cyrl.%s.txt', ext, ext + else + filename = sprintf 'hyph-%s.%s.txt', @bcp47, ext + filepath = File.join(PATH::TXT, filename) + # check for existence of file and that it’s not empty + unless File.file?(filepath) && File.read(filepath).length > 0 + # if the file we were looking for was a pattern file, something’s wrong + if ext == 'pat' + raise sprintf("There is some problem with plain patterns for language [%s]!!!", @bcp47) + else # the file is simply an exception file and we’re happy + filename = '' # And we return and empty file name after all + end + end + end + + sprintf "file_%s=%s", filetype, filename + end + + def exceptions_line + plain_text_line('hyp', 'exceptions') + end + + def patterns_line + plain_text_line('pat', 'patterns') + end + + def pTeX_patterns + if italic? + sprintf 'hyph-%s.tex', @bcp47 + elsif encoding + sprintf 'hyph-%s.%s.tex', @bcp47, encoding + else + legacy_patterns + end + end + + def list_loader + # which loader to use + if ['ar', 'fa'].include? @bcp47 + sprintf "file=%s \\\n\tfile_patterns=", loadhyph + elsif @bcp47 == 'grc-x-ibycus' + # TODO: fix this + sprintf "file=%s \\\n\tluaspecial=\"disabled:8-bit only\"", loadhyph + else + sprintf "file=%s", loadhyph + end + end + + def list_run_files + return [] if use_old_loader + + files = [] + + files << File.join(PATH::HYPHU8, 'loadhyph', loadhyph) + if has_apostrophes? + files << File.join(PATH::HYPHU8, 'patterns', 'quote', sprintf("hyph-quote-%s.tex", bcp47)) + end + + files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47)) + if encoding && encoding != "ascii" then + files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding)) + elsif bcp47 == "cop" + files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns) + end + + # we skip the mongolian language for luatex files + return files if bcp47 == "mn-cyrl-x-lmc" + + ['chr', 'pat', 'hyp', 'lic'].each do |t| + files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-%s.%s.txt', bcp47, t)) + end + + if bcp47 =~ /^sh-/ + # duplicate entries (will be removed later) + files << File.join(PATH::HYPHU8, 'patterns', 'tex', 'hyph-sr-cyrl.tex') + ['chr', 'pat', 'hyp', 'lic'].each do |t| + # duplicate entries (will be removed later) + files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-sr-cyrl.%s.txt', t)) + end + end + + files + end + + def package + extract_metadata + @package + end + end + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb b/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb new file mode 100644 index 0000000000..da5b425214 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb @@ -0,0 +1,7 @@ +$LOAD_PATH << File.expand_path('../../lib', __FILE__) +require 'tex/hyphen/language' +require 'tex/hyphen/texlive/package' +require 'tex/hyphen/texlive' +require 'tex/hyphen/texlive/loader' + +RSpec::Expectations.configuration.on_potential_false_positives = :nothing diff --git a/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb new file mode 100644 index 0000000000..4805b1878d --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb @@ -0,0 +1,985 @@ +require_relative '../../spec_helper' + +include TeX::Hyphen +include Language::TeXLive +include TeXLive +include Loader + +describe String do + describe '#superstrip' do + it "calls String#strip" do + str = " foo bar " + expect(str).to receive(:strip).and_return "foo bar" + str.superstrip + end + + it "strips TeX comments" do + expect("foo % bar".superstrip).to eq "foo " + end + end + + describe '#supersplit' do + it "calls String#strip" do + str = "foo bar " + expect(str).to receive(:strip).and_return "foo bar" + str.supersplit + end + + it "splits across whitespace ranges" do + expect("foo bar".supersplit).to eq ["foo", "bar"] + end + end + + describe '#safe' do + it "strips hyphens" do + expect("foo-bar".safe).to eq "foobar" + end + + it "strips spaces" do + expect("ancient greek".safe).to eq "ancientgreek" + end + end + + describe '#titlecase' do + it "capitalises each word" do + expect("modern greek".titlecase).to eq "Modern Greek" + end + end +end + +# TODO? Spec out PATH module + +describe Author do + describe 'Class methods' do + describe '.authors' do + it "returns a hash of authors" do + expect(Author.authors).to be_a Hash + end + end + + describe '.all' do + it "returns the array of authors" do + expect(Author.all).to be_an Array + end + end + + describe '.[]' do + it "returns the author for the key" do + expect(Author['donald_knuth']).to be_an Author + end + end + end + + describe 'Instance methods' do + let(:dek) { Author['donald_knuth'] } + + describe '#name' do + it "returns the author’s name" do + expect(dek.name).to eq 'Donald' + end + end + + describe '#surname' do + it "returns the author’s surname" do + expect(dek.surname).to eq 'Knuth' + end + end + + describe '#email' do + it "returns the email" do + expect(dek.email).to be_nil + end + end + end +end + +describe Language do + describe 'Class variables' do + it "has an end-of-header marker" do + expect(Language.class_variable_get :@@eohmarker).to match /^\={42}$/ + end + end + + describe '.new' do + it "creates a new Language instance" do + expect(Language.new).to be_a Language + end + + it "takes an optional BCP47 tag as argument" do + language = Language.new('ro') + expect(language.instance_variable_get :@bcp47).to eq 'ro' + end + +# it "calls .languages" do +# expect(Language).to receive(:languages).and_return({ 'pa' => nil }) +# Language.new('pa') +# end + end + + describe '.languages' do + it "sets the @@languages class variable" do + Language.languages + expect(Language.class_variable_get :@@languages).to be_a Hash + end + + it "lists all languages" do + # All the TeX files. Note [no] and [mn-cyrl-x-lmc] don’t have corresponding plain text files. + expect(Language.languages.count).to eq 82 # Was 79; 3 more “TeX Live dummies” [ar] [fa] [grc-x-ibycus] TODO Maybe remove + end + end + + describe '.all' do + it "returns the list of languages as an array" do + expect(Language.all).to be_an Array + end + + it "returns 81 languages" do # That’s all of them except for [sr-cyrl] + expect(Language.all.count).to eq 81 + end + end + + describe '.all_by_iso639' do + it "returns a hash of arrays" do + expect(Language.all_by_iso639).to be_a Hash + expect(Language.all_by_iso639.values.map(&:class).uniq).to eq [Array] + end + + it "groups languages by ISO 639 code elements" do + expect(Language.all_by_iso639['en'].count).to eq 2 + end + end + + describe '.find_by_bcp47' do + it "finds the language for that BCP47 tag" do + language = Language.find_by_bcp47 'bn' + expect(language).to be_a Language + end + + it "calls .languages first" do + expect(Language).to receive(:languages).and_return({ 'cy' => Language.new('cy') }) + Language.find_by_bcp47('cy') + end + end + + describe '#bcp47' do # TODO Add #8bitenc + it "returns the BCP47 tag of the language" do + language = Language.new('oc') + expect(language.bcp47).to eq 'oc' + end + end + + describe '#iso639' do + it "returns the ISO 639 primary tag" do + expect(Language.new('en-gb').iso639).to eq 'en' + end + + it "works on 3-character tags" do + expect(Language.new('grc-x-ibycus').iso639).to eq 'grc' + end + end + +# describe '#name' do +# let(:new_orthography_german) { Language.new('de-1996') } +# +# it "returns the name" do +# expect(new_orthography_german.name).to eq 'German, reformed spelling' +# end +# +# it "calls #extract_metadata first if necessary" do +# expect(new_orthography_german).to receive(:extract_metadata).and_return({ 'name' => 'German, reformed spelling' }) +# new_orthography_german.name +# end +# +# it "doesn’t call #extract_metadata if @name is already set" do +# new_orthography_german.instance_variable_set :@name, 'Deutch in neuer Rechtschreibung' +# expect(new_orthography_german).not_to receive :extract_metadata +# new_orthography_german.name +# end +# end + + # describe '#displayname' do + # it "returns @name most of the name" do + # romansh = Language.new('rm') + # expect(romansh.displayname).to eq 'Romansh' + # end + + # it "strips to the language subtag part" do + # polytonic_greek = Language.new('el-polyton') + # expect(polytonic_greek.displayname).to eq 'Greek' + # end + + # it "returns Norwegian for [nb] and [no]" do + # bokmål = Language.new('nb') + # expect(bokmål.displayname).to eq 'Norwegian' + # end + + # it "returns Serbian for [sh]" do + # serbocroatian_cyrillic = Language.new('sh-cyrl') + # expect(serbocroatian_cyrillic.displayname).to eq 'Serbian' + # end + + # it "calls #extract_metadata if needed" do + # friulan = Language.new('fur') + # expect(friulan).to receive :extract_metadata + # friulan.displayname + # end + # end + + describe '#babelname' do + it "returns the Babel name" do + expect(Language.new('de-1996').babelname).to eq 'ngerman' + end + + it "calls #extract_metadata first" do + german_CH = Language.new('de-ch-1901') + expect(german_CH).to receive :extract_metadata + german_CH.babelname + end + end + + # describe '#packagename' do + # it "returns the package name for TeX Live" do + # expect(Language.new('gu').packagename).to eq 'indic' + # end + + # it "uses some questionable choices" do + # expect(Language.new('grc').packagename).to eq 'ancient greek' + # end + # end + + describe '#licences' do + let(:church_slavonic) { Language.new('cu') } + + it "returns the licences as an array" do + expect(church_slavonic.licences).to eq ['MIT'] + end + + it "call #extract_metadata first if necessary" do + expect(church_slavonic).to receive(:extract_metadata) + church_slavonic.licences + end + + it "doesn’t call #extract_metadata if @licences is already set" do + church_slavonic.instance_variable_set :@licences, ['MIT licence'] + expect(church_slavonic).not_to receive :extract_metadata + church_slavonic.licences + end + + it "raises an exception if @licences is nil or empty" do + nolicence = Language.new('qnl') + allow(File).to receive(:read).and_return("code: qnl\nauthors:\n - me") + expect { nolicence.licences }.to raise_exception NoLicence + end + end + + describe '#lefthyphenmin' do + let(:swiss_spelling_german) { Language.new('de-ch-1901') } + + it "returns the left hyphenmin value for typesetting" do + expect(swiss_spelling_german.lefthyphenmin).to eq 2 + end + + it "calls #extract_metadata first if necessary" do + expect(swiss_spelling_german).to receive :extract_metadata + swiss_spelling_german.lefthyphenmin + end + + it "doesn’t call #extract_metadata if @lefthyphenmin is already set" do + swiss_spelling_german.instance_variable_set :@lefthyphenmin, 1 + expect(swiss_spelling_german).not_to receive :extract_metadata + swiss_spelling_german.lefthyphenmin + end + + it "uses the generation value if typesetting is missing" do + ethiopic = Language.new('mul-ethi') + expect(ethiopic.lefthyphenmin).to eq 1 + end + end + + describe '#righthyphenmin' do + let(:french) { Language.new('fr') } + + it "returns the right hyphenmin value for typesetting" do + expect(french.righthyphenmin).to eq 2 + end + + it "call #extract_metadata first if necessary" do + expect(french).to receive :extract_metadata + french.righthyphenmin + end + + it "doesn’t call #extract_metadata if @righthyphenmin is already set" do + french.instance_variable_set :@righthyphenmin, 2 + expect(french).not_to receive :extract_metadata + french.righthyphenmin + end + + it "uses the generation value if typesetting is missing" do + ethiopic = Language.new('mul-ethi') + expect(ethiopic.righthyphenmin).to eq 1 + end + end + + describe '#authors' do + let(:traditional_orthography_german) { Language.new('de-1901') } + + it "returns the authors as an array" do + expect(traditional_orthography_german.authors).to eq ['Deutschsprachige Trennmustermannschaft'] + end + + it "calls #extract_metadata first if necessary" do + expect(traditional_orthography_german).to receive(:extract_metadata).and_return({ authors: ['Werner Lemberg', 'others'] }) + traditional_orthography_german.authors + end + + it "doesn’t call #extract_metadata if @authors is already set" do + traditional_orthography_german.instance_variable_set :@authors, ['German hyphenation patterns team'] + expect(traditional_orthography_german).not_to receive :extract_metadata + traditional_orthography_german.authors + end + end + + describe '#synonyms' do + it "returns the synonyms as an array" do + expect(Language.new('sl').synonyms).to eq ['slovene'] + end + + it "returns an empty array instead of nil if there are no synonyms" do + expect(Language.new('sk').synonyms).to eq [] + end + end + + describe '#encoding' do + it "returns the encoding" do + expect(Language.new('sh-cyrl').encoding).to eq 't2a' + end + + it "returns “ascii” if applicable" do + expect(Language.new('rm').encoding).to eq 'ascii' + end + + it "returns nil if patterns are Unicode-only" do + expect(Language.new('sa').encoding).to be_nil + end + end + + describe '#github_link' do + it "returns the GitHub link" do + upper_sorbian = Language.new('hsb') + expect(upper_sorbian.github_link).to eq 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-hsb.tex' + end + end + + describe '#<=>' do +# it "compares Language’s using @name’s" do +# expect(Language.new('de-1996') <=> Language.new('fr')).to eq 1 +# end + + it "uses BCP47 codes if names are not available" do + expect(Language.new('zh-latn-pinyin') <=> Language.new('cu')).to eq 1 + end + end + + describe '#readtexfile' do + let(:basque) { Language.new('eu') } + + it "reads the TeX file" do + expect(File).to receive(:read) + basque.readtexfile + end + + it "stores the contents into the @@texfile class variable" do + basque.readtexfile + expect(Language.class_variable_get(:@@texfile)['eu']).to match /1ba.*1ko.*1t2xe.*su2b2r/m + end + + it "recovers gracefully from nonexistent files" do + expect { Language.new('kl').readtexfile }.not_to raise_exception + end + end + + describe '#patterns' do + it "returns the patterns" do + danish = Language.new('da') + expect(['.ae3', '.an3k', '.an1s'].all? { |p| danish.patterns.include? p }).to be_truthy + end + + it "calls .languages first" do + language = Language.new('eu') + expect(Language).to receive(:languages).and_return({ 'eu' => Language.new('eu') }) + pending "Needs pondering" + language.patterns + end + + it "loads the patterns" do + language = Language.new('fi') + # byebug + expect(language.patterns[151..154]).to eq ['uu1a2', 'uu1e2', 'uu1o2', 'uu1i2'] + end + + it "doesn’t crash on inexistent patterns" do + expect { Language.new('zu').patterns }.not_to raise_exception + end + + it "caches the list of patterns" do + language = Language.new('ru') + language.patterns + expect(language.instance_variable_get(:@patterns)[0..2]).to eq ['.аб1р', '.аг1ро', '.ади2'] + end + + it "uses the [no] patterns for [nb]" do + expect(Language.new('nb').patterns).to eq Language.new('no').patterns + end + + it "expands the Esperanto patterns" do + esperanto = Language.new('eo') + expect(['.di3s2a.', '.di3s2aj.', '.di3s2ajn.', '.di3s2an.', '.di3s2e.'].any? { |p| esperanto.patterns.include? p }).to be_truthy + end + end + + describe '#exceptions' do + it "returns the hyphenation exceptions" do + language = Language.new('ga') + # byebug + expect(language.exceptions[0..2]).to eq ['bhrachtaí', 'mbrachtaí', 'cháintí'] + end + + it "calls .languages first" do + language = Language.new('hu') + pending "Needs examining" + expect(Language).to receive(:all).and_return({ 'hu' => Language.new('hu') }) + language.patterns + end + + it "loads the exceptions" do + language = Language.new('is') + expect(File).to receive(:read).and_return("alc-un alc-u-nis-si-me alc-un-men-te") + language.patterns + end + + it "doesn’t crash on inexistent patterns" do + expect { Language.new('iu').exceptions}.not_to raise_exception + end + + it "caches the exceptions" do + language = Language.new('sk') + language.exceptions + expect(language.instance_variable_get(:@exceptions)[0..2]).to match ['dosť', 'me-tó-da', 'me-tó-dy'] + end + + it "hashes the exceptions" do + language = Language.new('en-gb') + language.exceptions + hyphenation = language.instance_variable_get :@hyphenation + expect(hyphenation.count).to eq 8 + expect(hyphenation['however']).to eq 'how-ever' + end + end + + describe '#hyphenate' do + it "hyphenates with the appropriate patterns" do + czech = Language.new('cs') + expect(czech.hyphenate('ubrousek')).to eq 'ubrou-sek' + end + + it "takes hyphenmins in account if available" do + language = Language.new('de-1996') + expect(language.hyphenate('Zwangsvollstreckungsmaßnahme')).to eq 'zwangs-voll-stre-ckungs-maß-nah-me' + end + + it "takes exceptions in account if available" do + american_english = Language.new('en-us') + expect(american_english.hyphenate('project')).to eq 'project' + end + + it "initialises the hydra if needed" do + language = Language.new('de-1901') + language.hyphenate('Zwangsvollstreckungsmaßnahme') + expect(language.instance_variable_get(:@hydra)).to be_a Hydra + end + + it "calls #exceptions" do + esperanto = Language.new('eo') + expect(esperanto).to receive(:exceptions) + esperanto.instance_variable_set :@hyphenation, { 'ŝtatregosciencon' => 'ŝta-tre-go-scien-con' } + esperanto.hyphenate('ŝtatregosciencon') + end + end + + describe '#private_use?' do + it "returns true for languages with private use BCP 47 tags" do + expect(Language.new('qls').private_use?).to be_truthy + end + + it "returns true otherwise" do + expect(Language.new('qyz').private_use?).to be_falsey + end + + it "doesn’t crash on two-letter tags" do + expect { Language.new('qa').private_use? }.not_to raise_exception + end + + it "returns false on two-letter tags" do + expect(Language.new('qt').private_use?).to be_falsey + end + + it "works correctly on four-letter tags" do + expect(Language.new('qaaa').private_use?).to be_falsey + end + + it "works correctly in the presence of subtags" do + expect(Language.new('qtz-GB').private_use?).to be_truthy + end + end + + describe '#extract_metadata' do + it "returns a hash with the metadata" do + language = Language.new('bg') + expect(language.extract_metadata).to be_a Hash + end + + it "raises an exception if the metadata is just a string" do + language = Language.new('qls') + allow(File).to receive(:read).and_return("just a string") + expect { language.extract_metadata }.to raise_exception InvalidMetadata + end + + it "raises an exception if the licence is missing" do + language = Language.new('qlv') + allow(File).to receive(:read).and_return("name: language virtual\ncode: qlv") + expect { language.extract_metadata }.to raise_exception InvalidMetadata + end + + it "raises an exception if @authors is nil or empty" do + not_church_slavonic = Language.new('qcu') + allow(File).to receive(:read).and_return "code: qcu\nlicence:\n name:\n MIT" + expect { not_church_slavonic.authors }.to raise_exception NoAuthor + end + + it "doesn’t crash on invalid licence entries" do + syntax_error = Language.new('qse') + allow(File).to receive(:read).and_return "foo:\nbar" + expect { syntax_error.extract_metadata }.not_to raise_exception Psych::SyntaxError + end + + it "sets the language name" do + language = Language.new('th') + language.extract_metadata + expect(language.instance_variable_get :@name).to eq 'Thai' + end + + it "sets the licence list" do + language = Language.new('la') + language.extract_metadata + expect(language.instance_variable_get :@licences).to eq ['MIT', 'LPPL'] + end + + it "sets lefthyphenmin" do + pali = Language.new('pi') + pali.extract_metadata + expect(pali.instance_variable_get :@lefthyphenmin).to eq 1 + end + + it "sets righthyphenmin" do + german = Language.new('de-1996') + german.extract_metadata + expect(german.instance_variable_get :@righthyphenmin).to eq 2 + end + + it "sets the list of authors" do + liturgical_latin = Language.new('la-x-liturgic') + liturgical_latin.extract_metadata + expect(liturgical_latin.instance_variable_get :@authors).to eq ['Claudio Beccari', 'Monastery of Solesmes', 'Élie Roux'] + end + + context "With Swedish as an example" do + let(:swedish) { Language.new('sv') } + + it "sets the message" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@message).to eq "Swedish hyphenation patterns" + end + + it "sets the old pattern file name" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@legacy_patterns).to eq "svhyph.tex" + end + + it "leaves @use_old_loader to nil in most cases" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@use_old_loader).to be_nil + end + + it "sets the @use_old_loader boolean to true if applicable" do + norwegian = Language.new('no') + norwegian.extract_metadata + expect(norwegian.instance_variable_get :@use_old_loader).to be_truthy + end + + it "leaves @use_old_patterns_comment to nil in most cases" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@use_old_patterns_comment).to be_nil + end + + it "sets the @use_old_patterns_comment to string if applicable" do + german_AR = Language.new('de-1901') + german_AR.extract_metadata + expect(german_AR.instance_variable_get :@use_old_patterns_comment).to eq "Kept for the sake of backward compatibility, but newer and better patterns by WL are available." + end + + it "sets the long description" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@description).to eq "Hyphenation patterns for Swedish in T1/EC and UTF-8 encodings." + end + + it "sets the Babel name" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@babelname).to eq "swedish" + end + + it "sets the Babel name even if it is slightly silly ;-)" do + german_NR = Language.new('de-1996') + german_NR.extract_metadata + expect(german_NR.instance_variable_get :@babelname).to eq "ngerman" + end + + it "sets @package to nil in most cases" do + expect(swedish.instance_variable_get :@package).to be_nil + end + + it "sets @package for a few languages" do + gujarati = Language.new('gu') + gujarati.extract_metadata + expect(gujarati.instance_variable_get :@package).to eq 'indic' + end + end + end + + describe '#has_apostrophes?' do + it "returns if patterns have apostrophes" do + expect(Language.new('be').has_apostrophes?).to be_truthy + end + + it "returns false otherwise" do + expect(Language.new('bn').has_apostrophes?).to be_falsey + end + end + + describe '#has_hyphens?' do + it "returns true if patterns have dashes" do + expect(Language.new('uk').has_hyphens?).to be_truthy + end + + it "returns false otherwise" do + expect(Language.new('tr').has_hyphens?).to be_falsey + end + end + + describe '#isgreek?' do + it "returns true if language is Greek (sort of)" do + expect(Language.new('grc').isgreek?).to be_truthy + end + + it "returns false if not" do + expect(Language.new('la').isgreek?).to be_falsey + end + + it "exceptionally returns false for Ibycus" do + expect(Language.new('grc-x-ibycus').isgreek?).to be_falsey + end + end + + describe '#message' do + it "returns the message to be displayed on the terminal" do + expect(Language.new('af').message).to eq 'Afrikaans hyphenation patterns' + end + end + + describe '#known_bugs' do + it "returns the known bugs" do + american_english = Language.new('en-us') + expect(american_english.known_bugs.keys).to eq ['de-mo-c-rat'] + end + end + + describe TeXLive do + describe '#loadhyph' do + it "returns the name of the pattern loader file" do + expect(Language.new('cy').loadhyph).to eq 'loadhyph-cy.tex' + end + + it "replaces the main sh subtag by sr" do + expect(Language.new('sh-latn').loadhyph).to eq 'loadhyph-sr-latn.tex' + end + + it "returns the old loader name if applicable" do + expect(Language.new('grc-x-ibycus').loadhyph).to eq 'ibyhyph.tex' + end + end + + describe '#list_loader' do + it "returns the tlpsrc line with the loader" do + expect(Language.new('hr').list_loader).to eq 'file=loadhyph-hr.tex' + end + + it "includes an empty line for Arabic and Farsi" do + expect(Language.new('ar').list_loader).to eq "file=zerohyph.tex \\\n\tfile_patterns=" + end + + it "includes a Lua special line for Ibycus" do + expect(Language.new('grc-x-ibycus').list_loader).to eq "file=ibyhyph.tex \\\n\tluaspecial=\"disabled:8-bit only\"" + end + end + + describe '#list_run_files' do + it "returns the list of TeX file" do + # puts Language.new('ka').list_run_files + expect(Language.new('ka').list_run_files).to eq ['tex/generic/hyph-utf8/loadhyph/loadhyph-ka.tex', + 'tex/generic/hyph-utf8/patterns/tex/hyph-ka.tex', + 'tex/generic/hyph-utf8/patterns/ptex/hyph-ka.t8m.tex', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.chr.txt', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.pat.txt', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.hyp.txt', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.lic.txt'] + end + end + + describe '#patterns_line' do + it "returns the patterns line for TLPSRC" do + expect(Language.new('tk').patterns_line).to eq "file_patterns=hyph-tk.pat.txt" + end + + it "returns two files for Serbian" do + expect(Language.new('sh-cyrl').patterns_line).to eq "file_patterns=hyph-sh-latn.pat.txt,hyph-sh-cyrl.pat.txt" + end + end + + describe '#exceptions_line' do + it "returns the exceptions line for TLPSRC" do + expect(Language.new('nn').exceptions_line).to eq "file_exceptions=hyph-nn.hyp.txt" + end + + it "returns two files for Serbian" do + expect(Language.new('sh-latn').exceptions_line).to eq "file_exceptions=hyph-sh-latn.hyp.txt,hyph-sh-cyrl.hyp.txt" + end + end + + describe '#extract_apostrophes' do + it "returns the list of patterns with apostrophes" do + expect(Language.new('af').extract_apostrophes[:with_apostrophe]).to eq ['.af6ro’', '.a7fro’s', '.l’7etji', '.m’7etji', '.r’7etji', 's’9ie.', 'x’9ie.'] + end + + it "returns nil otherwise" do + expect(Language.new('nl').extract_apostrophes[:with_apostrophe]).to be_nil + end + end + + describe '#extract_characters' do + it "extracts the list of characters with in lowercase and uppercase" do + expect(Language.new('id').extract_characters).to eq (('a'..'z').to_a - ['x']).map { |c| c + c.upcase } + end + end + + describe '#comments_and_licence' do + it "extracts the header" do + expect(Language.new('kmr').comments_and_licence).to match /^% title: Hyphenation patterns for Kurmanji \(Northern Kurdish\).*The patterns are generated by patgen from a word list of approx\. 2500\n% hyphenated words provided by Medeni Shemdê$/m + end + end + + describe '#list_synonyms' do + it "returns a list of synonyms" do + expect(Language.new('es').list_synonyms).to eq ' synonyms=espanol' + end + end + + describe '#list_hyphenmins' do + it "returns the hyphenmins" do + expect(Language.new('gl').list_hyphenmins).to eq "lefthyphenmin=2 \\\n\trighthyphenmin=2" + end + end + + describe '#message' do + it "returns the message to be displayed to TeX users" do + expect(Language.new('de-1996').message).to eq 'German hyphenation patterns (reformed orthography)' + end + end + + describe '#legacy_patterns' do + it "returns the file name of the legacy patterns" do + expect(Language.new('de-1996').legacy_patterns).to eq 'dehyphn.tex' + end + end + + describe '#use_old_loader' do + it "says whether to use old loader or not" do + expect(Language.new('de-1996').use_old_loader).to be_nil + end + + it "returns true if applicable" do + # pending "Refactor in progress" + expect(Language.new('no').use_old_loader).to be_truthy + end + end + + describe '#use_old_patterns_comment' do + it "returns true if language uses old patterns" do + expect(Language.new('de-1996').use_old_patterns_comment).to be_truthy + end + + it "returns nil otherwise" do + expect(Language.new('no').use_old_patterns_comment).to be_nil + end + + it "is a string" do + expect(Language.new('cop').use_old_patterns_comment).to be_a String + end + end + + describe '#description' do + it "returns the long description" do + text = <<-EOD + Hyphenation patterns for Dutch in T1/EC and UTF-8 encodings. + These patterns don't handle cases like 'menuutje' > 'menu-tje', + and don't hyphenate words that have different hyphenations according + to their meaning. + EOD + description = text.split("\n").map(&:strip).join("\n") + expect(Language.new('nl').description).to eq description + end + + it "returns nil for new spelling German, for some reason" do + expect(Language.new('de-1996').description).to be_nil + end + end + + describe '#package' do + it "returns the package name if applicable" do + expect(Language.new('bn').package).to eq 'indic' + end + + it "returns nil in most cases" do + expect(Language.new('bg').package).to be_nil + end + + it "calls #extract_metadata first" do + hindi = Language.new('hi') + expect(hindi).to receive :extract_metadata + hindi.package + end + end + end +end + +describe Package do + let(:latin) { Package.find('latin') } + let(:german) { Package.find('german') } + let(:hungarian) { Package.find('hungarian') } + let(:norwegian) { Package.find('norwegian') } + + describe "Instance variables" do + it "has a @name" do + expect(latin.name).to eq 'latin' + end + end + + describe '.new' do + it "initialises @languages to an empty array" do + package = Package.new('mongolian') + expect(package.instance_variable_get :@languages).to eq [] + end + end + + describe "#description_s" do + it "returns the short description" do + expect(latin.description_s).to eq 'Latin hyphenation patterns' + end + end + + describe "#description" do + it "returns the long description" do + expect(latin.description).to match /^Hyphenation patterns for.*modern spelling.*medieval spelling/m + expect(latin.description).to match /Classical Latin/ + expect(latin.description).to match /Liturgical Latin/ + end + + it "returns the long package description" do + text = <<-EOD + Hyphenation patterns for German in T1/EC and UTF-8 encodings, + for traditional and reformed spelling, including Swiss German. + The package includes the latest patterns from dehyph-exptl + (known to TeX under names 'german', 'ngerman' and 'swissgerman'), + however 8-bit engines still load old versions of patterns + for 'german' and 'ngerman' for backward-compatibility reasons. + Swiss German patterns are suitable for Swiss Standard German + (Hochdeutsch) not the Alemannic dialects spoken in Switzerland + (Schwyzerduetsch). + There are no known patterns for written Schwyzerduetsch. + EOD + description = text.split("\n").map(&:strip).join("\n") + expect(Package.find('german').description).to match description + end + end + + describe '#add_language' do + it "adds a language to the package" do + package = Package.new('indic') + assamese = Language.new('as') + package.add_language(assamese) + expect(package.instance_variable_get(:@languages).first).to eq assamese + end + end + + describe "#languages" do + it "returns the list of languages" do + expect(latin.languages.map(&:bcp47).sort).to eq ['la', 'la-x-classic', 'la-x-liturgic'] + end + end + + describe "#has_dependency?" do + it "returns the external dependencies" do + expect(german.has_dependency?).to eq "dehyph" + end + + it "returns nil for most packages" do + expect(hungarian.has_dependency?).to be_nil + end + end + + describe "#list_dependencies" do + it "lists the dependencies" do + # FIXME Should return ['depend hyphen-base', 'depend hyph-utf8', 'depend dehyph'] or nothing + expect(german.has_dependency?).to eq 'dehyph' + end + end + + describe "#list_support_files" do # FIXME? list_non_run_files + it "lists doc and source files" do + expect(hungarian.list_support_files('doc')).to eq ['doc/generic/hyph-utf8/languages/hu', 'doc/generic/huhyphen'] + # FIXME Should return ['texmf-dist/doc/generic/huhyphen', 'texmf-dist/doc/generic/hyph-utf8/languages/hu'] or nothing + end + end + + describe "#list_run_files" do + it "lists the run-time files" do + # pending "it crashes ;-)" + norwegian_run = norwegian.list_run_files + expect(norwegian_run.count).to eq 15 + expect(norwegian_run.select { |f| f =~ /tex\/hyph-[^\.]*\.tex$/ }).to eq ['tex/generic/hyph-utf8/patterns/tex/hyph-no.tex', + 'tex/generic/hyph-utf8/patterns/tex/hyph-nb.tex', + 'tex/generic/hyph-utf8/patterns/tex/hyph-nn.tex'] + end + end + + describe '#<=>' do + it "compares two Package’s" do + # puts hungarian.class, german.class + expect(hungarian.<=> german).to eq 1 + end + end + + describe '#find' do + it "returns the package with that name" do + expect(Package.find('norwegian')).to eq norwegian + end + end +end diff --git a/language/hyph-utf8/source/luatex/hyph-utf8/Makefile b/language/hyph-utf8/source/luatex/hyph-utf8/Makefile new file mode 100644 index 0000000000..2ae8bd67a1 --- /dev/null +++ b/language/hyph-utf8/source/luatex/hyph-utf8/Makefile @@ -0,0 +1,35 @@ +# Makefile for luatex-hyphen in tex-hyphen/hyph-utf8 + +NAME = luatex-hyphen +DTX = $(NAME).dtx + +# Generated files +COMPILED = $(NAME).pdf +UNPACKED = $(NAME).lua hyphen.cfg + +# Installation locations +RUNDIR = ../../../tex/luatex/hyph-utf8 +DOCDIR = ../../../doc/luatex/hyph-utf8 + +# Final files +RUNFILES = $(patsubst %, $(RUNDIR)/%, $(UNPACKED)) +DOCFILE = $(DOCDIR)/$(COMPILED) + +all: $(RUNFILES) $(DOCFILE) + +DO_TEX = tex --interaction=batchmode $< >/dev/null +DO_PDFLATEX = pdflatex --interaction=batchmode $< >/dev/null + +$(DOCFILE): $(DTX) + $(DO_PDFLATEX) + $(DO_PDFLATEX) + mv $(COMPILED) $(DOCDIR) + +$(RUNFILES): $(DTX) + $(DO_TEX) + for file in $(UNPACKED); do mv $$file $(RUNDIR); done + +clean: + @$(RM) -- *.log *.aux *.toc *.idx *.ind *.ilg + +.PHONY: clean diff --git a/language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx b/language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx new file mode 100644 index 0000000000..ca1761ef2b --- /dev/null +++ b/language/hyph-utf8/source/luatex/hyph-utf8/luatex-hyphen.dtx @@ -0,0 +1,337 @@ +% \iffalse meta-comment +% +% This is a module for hyphenation patterns loading for LuaTeX. It is under +% the CC0 license (http://creativecommons.org/publicdomain/zero/1.0/). +% +%<*ignore> +\begingroup + \def\x{LaTeX2e}% +\expandafter\endgroup +\ifcase 0\ifx\install y1\fi\expandafter + \ifx\csname processbatchFile\endcsname\relax\else1\fi + \ifx\fmtname\x\else 1\fi\relax +\else\csname fi\endcsname +%</ignore> +%<*install> +\input docstrip.tex + +\keepsilent +\askforoverwritefalse + +\def\MetaPrefix{-- } + +\def\luapostamble{% + \MetaPrefix^^J% + \MetaPrefix\space End of File `\outFileName'.% +} + +\def\currentpostamble{\luapostamble}% + +\preamble + +This is a generated file (source: luatex-hyphen.dtx). + +Copyright (C) 2012-2013 by The LuaLaTeX development team. + +This work is under the CC0 license. + +\endpreamble + +\generate{% + \file{luatex-hyphen.lua}{\from{luatex-hyphen.dtx}{lua}}% +} + +\endbatchfile +%</install> +%<*ignore> +\fi +%</ignore> +%<*driver> +\NeedsTeXFormat{LaTeX2e} +\ProvidesFile{luatex-hyphen.drv} + [2013/05/16 v1.6 Hyphenation pattern loading for LuaTeX] +\documentclass{ltxdoc} +\usepackage[ + pdftitle={The luatex-hyphen package}, + pdfsubject={Hyphenation pattern loading for LuaTeX}, + pdfauthor={Manuel Pégourié-Gonnard, Khaled Hosny & Elie Roux}, + pdfkeywords={luatex, lualatex, unicode, hyphenation} +]{hyperref} +\usepackage{fontspec} +\setmainfont[ + Ligatures = TeX, + BoldFont = {Linux Libertine O Bold}, + ItalicFont = {Linux Libertine O Italic}, + SlantedFont = {Linux Libertine O Italic}, +]{Linux Libertine O} +\setmonofont[Ligatures=TeX,Scale=MatchLowercase]{Liberation Mono} +\setsansfont[Ligatures=TeX,Scale=MatchLowercase]{Iwona Medium} +\EnableCrossrefs +\CodelineIndex +\begin{document} + \DocInput{luatex-hyphen.dtx}% +\end{document} +%</driver> +% \fi +% +% \CheckSum{0} +% +% \CharacterTable +% {Upper-case \A\B\C\D\E\F\G\H\I\J\K\L\M\N\O\P\Q\R\S\T\U\V\W\X\Y\Z +% Lower-case \a\b\c\d\e\f\g\h\i\j\k\l\m\n\o\p\q\r\s\t\u\v\w\x\y\z +% Digits \0\1\2\3\4\5\6\7\8\9 +% Exclamation \! Double quote \" Hash (number) \# +% Dollar \$ Percent \% Ampersand \& +% Acute accent \' Left paren \( Right paren \) +% Asterisk \* Plus \+ Comma \, +% Minus \- Point \. Solidus \/ +% Colon \: Semicolon \; Less than \< +% Equals \= Greater than \> Question mark \? +% Commercial at \@ Left bracket \[ Backslash \\ +% Right bracket \] Circumflex \^ Underscore \_ +% Grave accent \` Left brace \{ Vertical bar \| +% Right brace \} Tilde \~} +% +% \GetFileInfo{luatex-hyphen.drv} +% +% \title{The \texttt{hyphen.cfg} file for Lua\TeX } +% \date{2013/05/16 v1.6} +% \author{Khaled Hosny, Élie Roux, and Manuel Pégourié-Gonnard\\ +% \texttt{khaledhosny@eglug.org} \\ +% \texttt{elie.roux@telecom-bretagne.eu} \\ +% \texttt{mpg@elzevir.fr}} +% +% \maketitle +% +% \begin{abstract} +% This package is mainly a Lua module, to be used by \textsf{Babel} and +% \textsf{polyglossia} to adapt their hyphenation patterns loading mechanism to +% Lua\TeX's dynamic pattern loading capabilities. It makes use of a +% \texttt{language.dat.lua} file (whose format is described below) that should +% be present in the distribution, in addition to the regular +% \texttt{language.dat} file. +% +% \textsf{Babel} needed to be updated -- this used to be the goal of this +% package -- before version \textsf{3.9} (\TeX Live 2013) and +% \textsf{polyglossia} handles Lua\TeX\ since version \textsf{1.3} (\TeX Live +% 2013). +% +% There is a version of \texttt{etex.src} modified for the same reasons using +% similar code, which also makes use of the \texttt{luatex-hyphen.lua} and +% \texttt{language.dat.lua} files described here. +% \end{abstract} +% +% \section{Documentation} +% +% Hyphenation patterns should be loaded at runtime with Lua\TeX: if they appear +% in the format, they will be rehashed when the format is loaded anyway, which +% makes the format quite long to load (many seconds even on modern machines) +% and provides for bad user experience. Hence, it is desirable to load as few +% patterns as possible in the format, and load on-demand the needed patterns +% at runtime. +% +% This package provides a modified version of hyphen.cfg adapted to Lua\TeX, +% as well as a supporting Lua module. Since a lot of things, especially the +% catcodes, are not as predictable at runtime than at format creation time, we +% don't \verb+\input+ the usual pattern files, but rather load the patterns +% using the Lua interface, using a special plain text version of the pattern +% files if available. +% +% The existence and file name of such a version cannot be guessed, so we need +% a specific database: the file \texttt{language.dat.lua}. This file should be +% loadable by Lua and return a table whose keys are the canonical language +% names as found in \texttt{language.dat}, and the values are Lua tables +% consisting of: +% \begin{enumerate} +% \item A fixed part with one mandatory field: +% \begin{quote} +% \verb+synonyms = { <string> alternative name, ...}+ +% \end{quote} +% This field's value must be the same as in \texttt{language.dat}. +% \item A variable part consisting of either: +% \begin{itemize} +% \item For most languages: +% \begin{quote} +% \verb+patterns = <string> filenames for patterns+\\ +% \verb+hyphenation = <string> filenames for exceptions+\\ +% \end{quote} +% Each string contains a coma-separated list of file names (whitespace +% before or after the coma is not accepted). +% The files given by \verb+patterns+ (resp. \verb+hypenation+) must be +% plain text files encoded in UTF-8, with only patterns (resp. +% exceptions) and not even comments: their content will be used +% directly without being parsed by \TeX. If one of these keys is +% missing or is the empty string, it is ignored and no patterns (resp. +% exceptions) are loaded for this language. +% \item Special cases are supported by a field \verb+special+. Currently, +% the following kind of values are recognized: +% \begin{description} +% \item[\texttt{'disabled:<reason>'}] allows to disable specific +% languages: when the user tries to load this language, an error +% will be issued, with the \verb+<reason>+. +% \item[\texttt{'language0'}] only \texttt{english} should use this +% type of special, to indicate it is normally dumped in the format +% as \verb+\language0+ (see below). +% \end{description} +% Special languages may have \texttt{*hyphenmin} information when it +% makes sense (mostly \verb+\language0+). +% \end{itemize} +% \item Optional fields may be added. For example: +% \begin{quote} +% \verb+loader = <string> name of the TeX loader+\\ +% \verb+lefthyphenmin = <number> value for \lefthyphenmin+\\ +% \verb+righthyphenmin = <number> value for \righthyphenmin+ +% \end{quote} +% Those fields are present in \texttt{language.dat.lua} as generated by +% \texttt{tlmgr}, for example, but they \emph{are not} used by the present +% code in any way. +% \end{enumerate} +% Languages that are mentioned in \texttt{language.dat} but not in +% \texttt{language.dat.lua} will be loaded in the format. So, if the +% \texttt{language.dat.lua} file is missing or incomplete, languages will just +% go back to the ``old'' behaviour, resulting in longer startup time, which +% seems less bad than complete breakage. +% +% For backward compatibility, Knuth's original patterns for US English are +% always loaded in the format, as \verb+\language0+.\footnote{It is assumed +% to be the first entry in \texttt{language.dat}.} +% +% \StopEventually{ +% } +% +% \section{Implementation} +% +% \begin{macrocode} +%<*lua> +% \end{macrocode} +% +% Start a Lua module, two functions for error and information reporting. +% +% \begin{macrocode} +luatexhyphen = luatexhyphen or {} +local luatexhyphen = luatexhyphen +local function wlog(msg, ...) + texio.write_nl('log', 'luatex-hyphen: '..msg:format(...)) +end +local function err(msg, ...) + error('luatex-hyphen: '..msg:format(...), 2) +end +% \end{macrocode} +% +% Load the \texttt{language.dat.lua} file with the Lua version of the +% language database. +% +% \begin{macrocode} +local dbname = "language.dat.lua" +local language_dat +local dbfile = kpse.find_file(dbname, 'lua') +if not dbfile then + err("file not found: "..dbname) +else + wlog('using data file: %s', dbfile) + language_dat = dofile(dbfile) +end +% \end{macrocode} +% +% Look up a language in the database, and return the associated +% information, as well as the canonical name of the language. +% +% \begin{macrocode} +local function lookupname(name) + if language_dat[name] then + return language_dat[name], name + else + for canon, data in pairs(language_dat) do + for _,syn in ipairs(data.synonyms) do + if syn == name then + return data, canon + end + end + end + end +end +luatexhyphen.lookupname = lookupname +% \end{macrocode} +% +% Set hyphenation patterns and exceptions for a language given by its name +% (in the database) and number (value of \verb+\language+). Doesn't return +% anything, but will call \verb+error()+ if things go wrong. +% +% \begin{macrocode} +local function loadlanguage(lname, id) + if id == 0 then + return + end + local msg = "loading%s patterns and exceptions for: %s (\\language%d)" +% \end{macrocode} +% +% Lookup the language in the database. +% +% \begin{macrocode} + local ldata, cname = lookupname(lname) + if not ldata then + err("no entry in %s for this language: %s", dbname, lname) + end +% \end{macrocode} +% +% Handle special languages. +% +% \begin{macrocode} + if ldata.special then + if ldata.special:find('^disabled:') then + err("language disabled by %s: %s (%s)", dbname, cname, + ldata.special:gsub('^disabled:', '')) + elseif ldata.special == 'language0' then + err("\\language0 should be dumped in the format") + else + err("bad entry in %s for language %s") + end + end +% \end{macrocode} +% +% The generic case: load hyphenation patterns and exceptions from files +% given by the language code. +% +% \begin{macrocode} + wlog(msg, '', cname, id) + for _, item in ipairs{'patterns', 'hyphenation'} do + local filelist = ldata[item] + if filelist ~= nil and filelist ~= '' then + for _, file in ipairs(filelist:explode(',')) do + local file = kpse.find_file(file) or err("file not found: %s", file) + local fh = io.open(file, 'r') + local data = fh:read('*a') or err("file not readable: %s", f) + fh:close() + lang[item](lang.new(id), data) + end + else + if item == 'hyphenation' then item = item..' exceptions' end + wlog("info: no %s for this language", item) + end + end +end +luatexhyphen.loadlanguage = loadlanguage +% \end{macrocode} +% +% Add \textsf{Babel}'s ``dialects'' as synonyms. +% +% \begin{macrocode} +local function adddialect(dialect, language) + if dialect ~= '0' then + dialect = dialect:gsub('l@', '') + language = language:gsub('l@', '') + data = lookupname(language) + if data then + data.synonyms[#data.synonyms+1] = dialect + end + end +end +luatexhyphen.adddialect = adddialect +% \end{macrocode} +% +% \begin{macrocode} +%</lua> +% \end{macrocode} +% +% \Finale +\endinput |