From 6c0ddfe655a39b71d1ab6a4a95d57a7ea0d94243 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 27 Mar 2020 03:01:30 +0000 Subject: CTAN sync 202003270301 --- .../hyph-utf8/data/encodings/macedonian.dat | 42 ++ .../generic/hyph-utf8/generate-converters.rb | 6 +- .../generic/hyph-utf8/generate-plain-patterns.rb | 6 +- .../source/generic/hyph-utf8/generate-tl-files.rb | 24 +- .../source/generic/hyph-utf8/hyph-utf8.rb | 2 +- .../source/generic/hyph-utf8/language-data.rb | 614 ++++++++++----------- .../generic/hyph-utf8/lib/tex/hyphen/converter.rb | 39 ++ .../hyph-utf8/lib/tex/hyphen/texlive/loader.rb | 2 +- .../hyph-utf8/lib/tex/hyphen/texlive/package.rb | 6 +- .../hyph-utf8/lib/tex/hyphen/texlive/source.rb | 6 +- .../source/generic/hyph-utf8/spec/spec_helper.rb | 1 + .../hyph-utf8/spec/tex/hyphen/language_spec.rb | 28 +- 12 files changed, 442 insertions(+), 334 deletions(-) create mode 100644 language/hyph-utf8/source/generic/hyph-utf8/data/encodings/macedonian.dat create mode 100644 language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb (limited to 'language/hyph-utf8/source') diff --git a/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/macedonian.dat b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/macedonian.dat new file mode 100644 index 0000000000..e8c64b4f20 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/data/encodings/macedonian.dat @@ -0,0 +1,42 @@ +0x2E U+002E period +0x30 U+0030 zero +0x31 U+0031 one +0x32 U+0032 two +0x33 U+0033 three +0x34 U+0034 four +0x35 U+0035 five +0x36 U+0036 six +0x37 U+0037 seven +0x38 U+0038 eight +0x39 U+0039 nine +0x83 U+0453 cyrgj +0x9A U+0459 cyrlje +0x9C U+045A cyrnje +0x9D U+045C cyrkje +0x9F U+045F cyrdzh +0xBC U+0458 cyrj +0xBE U+0455 cyrdze +0xE0 U+0430 cyra +0xE1 U+0431 cyrb +0xE2 U+0432 cyrv +0xE3 U+0433 cyrg +0xE4 U+0434 cyrd +0xE5 U+0435 cyre +0xE6 U+0436 cyrzh +0xE7 U+0437 cyrz +0xE8 U+0438 cyri +0xEA U+043A cyrk +0xEB U+043B cyrl +0xEC U+043C cyrm +0xED U+043D cyrn +0xEE U+043E cyro +0xEF U+043F cyrp +0xF0 U+0440 cyrr +0xF1 U+0441 cyrs +0xF2 U+0442 cyrt +0xF3 U+0443 cyru +0xF4 U+0444 cyrf +0xF5 U+0445 cyrh +0xF6 U+0446 cyrc +0xF7 U+0447 cyrch +0xF8 U+0448 cyrsh diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb index d76aa29657..17f2b515f2 100755 --- a/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-converters.rb @@ -32,9 +32,9 @@ $uniconvmacro1 = <<__EOUNIMAC1__ % macros adapted from ConTeXt MKII; see unic-ini.mkii \\def\\unicodechar#1{% \\ifcsname unichar@\\number#1\\endcsname - \\csname unichar@\\number#1\\endcsname + \\csname unichar@\\number#1\\endcsname \\else - \\errmessage{Unicode character [#1] not in encoding.}% + \\errmessage{Unicode character [#1] not in encoding.}% \\fi} __EOUNIMAC1__ @@ -173,5 +173,3 @@ end file_out.puts end end - - diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb index eaa9cbf608..0265374c5b 100755 --- a/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-plain-patterns.rb @@ -49,16 +49,16 @@ Language.all.sort.each do |language| unless language.exceptions == "" file = outfile.('hyp') # This ensure a file is created, even if it may be empty file.puts language.exceptions - end + end - ### FIXME Do issue #33 instead + ### FIXME Do issue #33 instead ### # characters ### file = outfile.('chr') ### language.extract_characters.each do |character| ### file.puts character ### end - ### FIXME Do something else instead + ### FIXME Do something else instead ### # comments and licence ### file = outfile.('lic') ### file.puts language.comments_and_licence diff --git a/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb b/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb index 7460658f02..2acb294027 100755 --- a/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/generate-tl-files.rb @@ -26,8 +26,8 @@ Package.all.sort.each do |package| # FIXME Still doesn’t work well for Latin file_tlpsrc.printf "shortdesc %s.\n", package.description_s package.description.split("\n").each do |line| - file_tlpsrc.printf "longdesc %s\n", line - end + file_tlpsrc.printf "longdesc %s\n", line + end package.languages.each do |language| file_tlpsrc.printf "execute AddHyphen \\\n\tname=%s%s \\\n", language.babelname, language.list_synonyms file_tlpsrc.printf "\t%s \\\n\t%s", language.list_hyphenmins, language.list_loader @@ -61,9 +61,17 @@ puts #--------------------------# # language.dat and friends # #--------------------------# -system sprintf "tlmgr generate --dest %s language.dat", File.join(PATH::LANGUAGE_DAT, 'language.dat') -system sprintf "tlmgr generate --dest %s language.dat.lua", File.join(PATH::LANGUAGE_DAT, 'language.dat.lua') - -# Revert changes if only line with date was changed -diff_lines = `git diff #{PATH::LANGUAGE_DAT}/language.dat* | egrep '^[+-](%|--) Generated' | wc -l`.strip -`git checkout HEAD -- #{PATH::LANGUAGE_DAT}/language.dat*` if diff_lines == '4' +ldatfile = File.join PATH::LANGUAGE_DAT, 'language.dat' +File.open(ldatfile, 'w') do |file| + puts 'Generating language.dat' + Package.all.sort.each do |package| + package.languages.each do |language| + # Main language name + file.printf "%s\t%s\n", language.babelname, language.loadhyph + # Synonyms + language.synonyms.each do |synonym| + file.printf "=%s\n", synonym + end + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb b/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb index 60005af380..9ca036fa8e 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/hyph-utf8.rb @@ -9,7 +9,7 @@ class UnicodeCharacter @bytes = [code_uni].pack('U').unpack('C*') @name = name end - + attr_reader :code_uni, :code_enc, :bytes, :name end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb b/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb index d4f387c072..6470911010 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/language-data.rb @@ -16,312 +16,312 @@ class OldLanguage # "exceptions" => false, # "message" => nil, - @@language_data = [ -{ - "code" => "el-monoton", - "type" => "rules", -}, -{ - "code" => "el-polyton", - "type" => "rules", -}, -{ - "code" => "grc", - "type" => "rules", -}, -{ - "code" => "cop", - "type" => "rules", -}, -{ - "code" => "de-1901", - "type" => "dictionary", -}, -{ - "code" => "de-1996", - "type" => "dictionary", -}, -{ - "code" => "de-ch-1901", - "type" => "dictionary", -}, -{ - "code" => "ru", - "type" => "dictionary", -}, -{ - "code" => "uk", - # "type" => "rules", # TODO: it is not really clear -}, -{ - "code" => "af", - "type" => "dictionary", -}, -{ - "code" => "ca", - "type" => "rules", # not only rules, also patgen, but it is a good approximation -}, -{ - "code" => "cs", - "filename_old_patterns_other" => ["czhyphen.tex","czhyphen.ex"], - "type" => "dictionary", - # Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 - # Typographical rules allow \righthyphenmin=2 when typesetting in a - # narrow column (newspapers etc.). - # (used to be 2,2) +@@language_data = [ + { + "code" => "el-monoton", + "type" => "rules", + }, + { + "code" => "el-polyton", + "type" => "rules", + }, + { + "code" => "grc", + "type" => "rules", + }, + { + "code" => "cop", + "type" => "rules", + }, + { + "code" => "de-1901", + "type" => "dictionary", + }, + { + "code" => "de-1996", + "type" => "dictionary", + }, + { + "code" => "de-ch-1901", + "type" => "dictionary", + }, + { + "code" => "ru", + "type" => "dictionary", + }, + { + "code" => "uk", + # "type" => "rules", # TODO: it is not really clear + }, + { + "code" => "af", + "type" => "dictionary", + }, + { + "code" => "ca", + "type" => "rules", # not only rules, also patgen, but it is a good approximation + }, + { + "code" => "cs", + "filename_old_patterns_other" => ["czhyphen.tex", "czhyphen.ex"], + "type" => "dictionary", + # Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 + # Typographical rules allow \righthyphenmin=2 when typesetting in a + # narrow column (newspapers etc.). + # (used to be 2,2) - # guessing based on CTAN/macros/cstex/base/csplain.tar.gz: - # 1998-12-17 (patterns) - # 1995-08-23 (exceptions) - # but patterns claim 1995 -}, -{ - "code" => "sk", - "type" => "dictionary", - "filename_old_patterns_other" => ["skhyphen.tex","skhyphen.ex"], -}, -{ - "code" => "cy", - "type" => "dictionary", -}, -{ - "code" => "da", - "type" => "dictionary", - "filename_old_patterns_other" => ["dkcommon.tex", "dkspecial.tex"], -}, -{ - "code" => "eo", - "type" => "rules", -}, -{ - "code" => "es", - "type" => "dictionary", -}, -{ - "code" => "eu", - "type" => "rules", -}, -{ - "code" => "fr", - "type" => "rules", -}, -{ - "code" => "gl", - "type" => "rules", -}, -{ - "code" => "et", - "type" => "dictionary", -}, -{ - "code" => "fi", - "type" => "rules", -}, -{ - "code" => "hr", - "type" => "dictionary", -}, -{ - "code" => "hu", - "type" => "dictionary", - "source" => "https://github.com/nagybence/huhyphn/", # FIXME Check and remove from desc below -}, -{ - "code" => "hy", - "type" => "rules", -}, -{ - "code" => "ia", - "type" => "dictionary", -}, -{ - "code" => "id", - "type" => "rules", -}, -{ - "code" => "is", - "type" => "dictionary", - # TODO: I'm not sure that the last two names are relevant, I don't find the source of Marteinn Sverrisson - "authors" => ["jorgen_pind", "marteinn_sverrisson", "kristinn_gylfason"], -}, -{ - "code" => "ga", - "type" => "dictionary", -}, -{ - "code" => "it", - "type" => "rules", # TODO: we might want to check that, but it seems unlikely that patgen was used -}, -{ - "code" => "rm", - "type" => "rules", -}, -{ - "code" => "fur", - "type" => "rules", -}, -{ - "code" => "pms", - "type" => "rules", -}, -{ - "code" => "kmr", - "type" => "dictionary", -}, -{ - "code" => "la", - "type" => "rules", -}, -{ - "code" => "la-x-classic", - "type" => "rules", -}, -{ - "code" => "la-x-liturgic", - "version" => "1.040", - "last_modified" => "2016-05-19", - "type" => "rules", -}, -{ - "code" => "lt", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Lithuanian in L7X and UTF-8 encodings.", - # "Designed for \\lefthyphenmin and \\righthyphenmin set to 2.", - "\\lefthyphenmin and \\righthyphenmin have to be at least 2.", - # "Changing them to 1 according to grammatical rules from 1997", - # "would require to review and maybe rebuild the patterns." - ], -}, -{ - "code" => "lv", - "type" => "dictionary", -}, -{ - "code" => "nl", - "type" => "dictionary", - # quoting Hans Hagen: - # patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) -}, -{ - "code" => "oc", - "type" => "rules", -}, -{ - "code" => "pl", - "type" => "dictionary", - #{}"hyphenmin" => [1,1], -}, -{ - "code" => "pt", - "type" => "rules", # TODO: we could create a generating script -}, -{ - "code" => "zh-latn-pinyin", - "type" => "rules", # TODO: we could create a generating script -}, -{ - "code" => "ro", - "type" => "dictionary", -}, -{ - "code" => "sl", - "type" => "dictionary", -}, -{ - "code" => "hsb", - "type" => "dictionary", -}, -{ - "code" => "sv", - "type" => "dictionary", -}, -{ - "code" => "tk", - "type" => "dictionary", -}, -{ - "code" => "tr", - "type" => "rules", -}, -{ - "code" => "en-gb", - "type" => "dictionary", -}, -{ - "code" => "en-us", -}, -# TODO: FIXME!!! -{ - "code" => "sh-latn", - "type" => "dictionary", -}, -{ - "code" => "sh-cyrl", -}, -{ - "code" => "mn-cyrl", - "type" => "dictionary", - "last_modified" => "2010-04-03", # FIXME: ? -}, -{ - "code" => "sa", - "type" => "rules", -}, -{ - "code" => "nb", - "type" => "dictionary", - "last_modified" => "2012-05-18", # FIXME: ? -}, -{ - "code" => "as", - # this is true for all Indic patterns - "type" => "rules", -}, -# thai -{ - "code" => "th", - "type" => "dictionary", - "last_modified" => "2015-05-07", # FIXME: ? -}, -# lao -#{ -# "code" => "lo", -# "name" => "lao", -# "use_new_loader" => true, -# "use_old_patterns" => false, -# "hyphenmin" => [1,1], # TODO -# "encoding" => nil, # no patterns for 8-bit engines -# "exceptions" => false, -# "message" => "Lao hyphenation patterns", -# -# "version" => nil, -# "last_modified" => "2010-05-19", -# "type" => "rules", -# "authors" => [ "brian_wilson", "arthur_reutenauer", "mojca_miklavec" ], -# "licence" => "other-free", -# "description_s" => "Lao hyphenation patterns", -# "description_l" => [ -# #......................................................................# -# "Hyphenation patterns for Lao language for Unicode engines.", -# "Current version is experimental and gives bad results.", -# ], -#}, -{ - "code" => "ka", - "type" => "dictionary", -}, -{ - "code" => "cu", - "type" => "machine learning", -}, -{ - "code" => "be", - "type" => "rules", -}, -{ - "code" => "pi", - "type" => "rules", -}, - ] + # guessing based on CTAN/macros/cstex/base/csplain.tar.gz: + # 1998-12-17 (patterns) + # 1995-08-23 (exceptions) + # but patterns claim 1995 + }, + { + "code" => "sk", + "type" => "dictionary", + "filename_old_patterns_other" => ["skhyphen.tex", "skhyphen.ex"], + }, + { + "code" => "cy", + "type" => "dictionary", + }, + { + "code" => "da", + "type" => "dictionary", + "filename_old_patterns_other" => ["dkcommon.tex", "dkspecial.tex"], + }, + { + "code" => "eo", + "type" => "rules", + }, + { + "code" => "es", + "type" => "dictionary", + }, + { + "code" => "eu", + "type" => "rules", + }, + { + "code" => "fr", + "type" => "rules", + }, + { + "code" => "gl", + "type" => "rules", + }, + { + "code" => "et", + "type" => "dictionary", + }, + { + "code" => "fi", + "type" => "rules", + }, + { + "code" => "hr", + "type" => "dictionary", + }, + { + "code" => "hu", + "type" => "dictionary", + "source" => "https://github.com/nagybence/huhyphn/", # FIXME Check and remove from desc below + }, + { + "code" => "hy", + "type" => "rules", + }, + { + "code" => "ia", + "type" => "dictionary", + }, + { + "code" => "id", + "type" => "rules", + }, + { + "code" => "is", + "type" => "dictionary", + # TODO: I'm not sure that the last two names are relevant, I don't find the source of Marteinn Sverrisson + "authors" => ["jorgen_pind", "marteinn_sverrisson", "kristinn_gylfason"], + }, + { + "code" => "ga", + "type" => "dictionary", + }, + { + "code" => "it", + "type" => "rules", # TODO: we might want to check that, but it seems unlikely that patgen was used + }, + { + "code" => "rm", + "type" => "rules", + }, + { + "code" => "fur", + "type" => "rules", + }, + { + "code" => "pms", + "type" => "rules", + }, + { + "code" => "kmr", + "type" => "dictionary", + }, + { + "code" => "la", + "type" => "rules", + }, + { + "code" => "la-x-classic", + "type" => "rules", + }, + { + "code" => "la-x-liturgic", + "type" => "rules", + "version" => "1.040", + "last_modified" => "2016-05-19", + }, + { + "code" => "lt", + "description_l" => [ + #......................................................................# + "Hyphenation patterns for Lithuanian in L7X and UTF-8 encodings.", + # "Designed for \\lefthyphenmin and \\righthyphenmin set to 2.", + "\\lefthyphenmin and \\righthyphenmin have to be at least 2.", + # "Changing them to 1 according to grammatical rules from 1997", + # "would require to review and maybe rebuild the patterns." + ], + }, + { + "code" => "lv", + "type" => "dictionary", + }, + { + "code" => "nl", + "type" => "dictionary", + # quoting Hans Hagen: + # patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) + }, + { + "code" => "oc", + "type" => "rules", + }, + { + "code" => "pl", + "type" => "dictionary", + #{}"hyphenmin" => [1,1], + }, + { + "code" => "pt", + "type" => "rules", # TODO: we could create a generating script + }, + { + "code" => "zh-latn-pinyin", + "type" => "rules", # TODO: we could create a generating script + }, + { + "code" => "ro", + "type" => "dictionary", + }, + { + "code" => "sl", + "type" => "dictionary", + }, + { + "code" => "hsb", + "type" => "dictionary", + }, + { + "code" => "sv", + "type" => "dictionary", + }, + { + "code" => "tk", + "type" => "dictionary", + }, + { + "code" => "tr", + "type" => "rules", + }, + { + "code" => "en-gb", + "type" => "dictionary", + }, + { + "code" => "en-us", + }, + # TODO: FIXME!!! + { + "code" => "sh-latn", + "type" => "dictionary", + }, + { + "code" => "sh-cyrl", + }, + { + "code" => "mn-cyrl", + "type" => "dictionary", + "last_modified" => "2010-04-03", # FIXME: ? + }, + { + "code" => "sa", + "type" => "rules", + }, + { + "code" => "nb", + "type" => "dictionary", + "last_modified" => "2012-05-18", # FIXME: ? + }, + { + "code" => "as", + # this is true for all Indic patterns + "type" => "rules", + }, + # thai + { + "code" => "th", + "type" => "dictionary", + "last_modified" => "2015-05-07", # FIXME: ? + }, + # lao + #{ + # "code" => "lo", + # "name" => "lao", + # "use_new_loader" => true, + # "use_old_patterns" => false, + # "hyphenmin" => [1,1], # TODO + # "encoding" => nil, # no patterns for 8-bit engines + # "exceptions" => false, + # "message" => "Lao hyphenation patterns", + # + # "version" => nil, + # "last_modified" => "2010-05-19", + # "type" => "rules", + # "authors" => [ "brian_wilson", "arthur_reutenauer", "mojca_miklavec" ], + # "licence" => "other-free", + # "description_s" => "Lao hyphenation patterns", + # "description_l" => [ + # #......................................................................# + # "Hyphenation patterns for Lao language for Unicode engines.", + # "Current version is experimental and gives bad results.", + # ], + #}, + { + "code" => "ka", + "type" => "dictionary", + }, + { + "code" => "cu", + "type" => "machine learning", + }, + { + "code" => "be", + "type" => "rules", + }, + { + "code" => "pi", + "type" => "rules", + }, +] end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb new file mode 100644 index 0000000000..3b966d45a4 --- /dev/null +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb @@ -0,0 +1,39 @@ +# encoding: UTF-8 + +require 'scanf' +require 'byebug' + +module TeX + module Hyphen + class Converter + def read(conversion) + @mapping = { } + File.read(conversion).each_line do |line| + next if line =~ /^#/ + + eightbit, usv = line.scanf "0x%02X\tU+%04X" + @mapping[eightbit] = usv.chr(Encoding::UTF_8) + end + end + + def convert(filename) + raise "Please define the encoding mapping first with #read" unless @mapping + doconvert = false + File.open(filename, external_encoding: Encoding::ASCII_8BIT).each_line do |line| + if line =~ /\\patterns{/ + doconvert = true + next + end + doconvert = false if doconvert && line =~ /}/ + + if doconvert + puts (line.strip.each_byte.map do |byte| + byebug unless @mapping[byte] + @mapping[byte] + end || '').join + end + end + end + end + end +end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb index 2fcb3945d3..878bf0ef0d 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb @@ -4,7 +4,7 @@ module TeX module Loader def unicode_only? # TODO Spec out - ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47 + ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi', 'mk'].include? @bcp47 end def string_enc diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb index 69481a4177..1d40dc512e 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb @@ -104,7 +104,11 @@ module TeX unless has_dependency? languages.each do |language| if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop' - files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns) + if language.bcp47 == 'la-x-classic' + files << sprintf("tex/generic/hyph-utf8/patterns/tex-8bit/%s", language.legacy_patterns) + else + files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns) + end end end end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb index 61292dcd3b..dce5be00b4 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb @@ -82,7 +82,7 @@ module TeX end files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47)) - if encoding && encoding != "ascii" then + if encoding && encoding != "ascii" && bcp47 != 'la-x-classic' then files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding)) elsif bcp47 == "cop" files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns) @@ -91,14 +91,14 @@ module TeX # we skip the mongolian language for luatex files return files if bcp47 == "mn-cyrl-x-lmc" - ['chr', 'pat', 'hyp', 'lic'].each do |t| + ['pat', 'hyp'].each do |t| files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-%s.%s.txt', bcp47, t)) end if bcp47 =~ /^sh-/ # duplicate entries (will be removed later) files << File.join(PATH::HYPHU8, 'patterns', 'tex', 'hyph-sr-cyrl.tex') - ['chr', 'pat', 'hyp', 'lic'].each do |t| + ['pat', 'hyp'].each do |t| # duplicate entries (will be removed later) files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-sr-cyrl.%s.txt', t)) end diff --git a/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb b/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb index da5b425214..08dd5b96b7 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/spec/spec_helper.rb @@ -3,5 +3,6 @@ require 'tex/hyphen/language' require 'tex/hyphen/texlive/package' require 'tex/hyphen/texlive' require 'tex/hyphen/texlive/loader' +require 'tex/hyphen/converter' RSpec::Expectations.configuration.on_potential_false_positives = :nothing diff --git a/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb index 4805b1878d..93f42c462d 100644 --- a/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb +++ b/language/hyph-utf8/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb @@ -124,7 +124,7 @@ describe Language do it "lists all languages" do # All the TeX files. Note [no] and [mn-cyrl-x-lmc] don’t have corresponding plain text files. - expect(Language.languages.count).to eq 82 # Was 79; 3 more “TeX Live dummies” [ar] [fa] [grc-x-ibycus] TODO Maybe remove + expect(Language.languages.count).to eq 83 # Was 79; 3 more “TeX Live dummies” [ar] [fa] [grc-x-ibycus] TODO Maybe remove end end @@ -134,7 +134,7 @@ describe Language do end it "returns 81 languages" do # That’s all of them except for [sr-cyrl] - expect(Language.all.count).to eq 81 + expect(Language.all.count).to eq 82 end end @@ -737,10 +737,8 @@ describe Language do expect(Language.new('ka').list_run_files).to eq ['tex/generic/hyph-utf8/loadhyph/loadhyph-ka.tex', 'tex/generic/hyph-utf8/patterns/tex/hyph-ka.tex', 'tex/generic/hyph-utf8/patterns/ptex/hyph-ka.t8m.tex', - 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.chr.txt', 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.pat.txt', - 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.hyp.txt', - 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.lic.txt'] + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.hyp.txt'] end end @@ -963,7 +961,7 @@ describe Package do it "lists the run-time files" do # pending "it crashes ;-)" norwegian_run = norwegian.list_run_files - expect(norwegian_run.count).to eq 15 + expect(norwegian_run.count).to eq 11 expect(norwegian_run.select { |f| f =~ /tex\/hyph-[^\.]*\.tex$/ }).to eq ['tex/generic/hyph-utf8/patterns/tex/hyph-no.tex', 'tex/generic/hyph-utf8/patterns/tex/hyph-nb.tex', 'tex/generic/hyph-utf8/patterns/tex/hyph-nn.tex'] @@ -983,3 +981,21 @@ describe Package do end end end + +describe Converter do + describe '#read' do + it "reads the conversion data" do + converter = Converter.new + converter.read(File.join(File.expand_path(__dir__), '..', '..', '..', 'data', 'encodings', 'macedonian.dat')) + puts converter.instance_variable_get(:@mapping) + end + end + + describe '#convert' do + it "runs one pass through the file" do + converter = Converter.new + converter.read(File.join(File.expand_path(__dir__), '..', '..', '..', 'data', 'encodings', 'macedonian.dat')) + converter.convert(File.join(File.expand_path(__dir__), '..', '..', '..', '..', '..', '..', '..', 'old', 'other', 'mk', 'mkhyphen.tex')) + end + end +end -- cgit v1.2.3