From 28ca80ca0dac99633e98802159d5e05ff2c27eba Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Fri, 5 Apr 2019 22:22:32 +0000 Subject: hyph-utf8 (5apr19) git-svn-id: svn://tug.org/texlive/trunk@50792 c570f23f-e606-0410-a88d-b1316a301751 --- .../source/generic/hyph-utf8/author-data.rb | 92 -- .../generic/hyph-utf8/generate-pattern-loaders.rb | 261 +--- .../generic/hyph-utf8/generate-plain-patterns.rb | 23 +- .../generic/hyph-utf8/generate-ptex-patterns.rb | 92 +- .../source/generic/hyph-utf8/generate-tl-files.rb | 51 +- .../source/generic/hyph-utf8/generate-webpage.rb | 132 -- .../source/generic/hyph-utf8/language-data.rb | 1517 +------------------- .../source/generic/hyph-utf8/languages.rb | 459 +----- .../source/generic/hyph-utf8/lib/tex/hyphen.rb | 1 + .../generic/hyph-utf8/lib/tex/hyphen/authors.yml | 452 ++++++ .../generic/hyph-utf8/lib/tex/hyphen/language.rb | 364 +++++ .../generic/hyph-utf8/lib/tex/hyphen/packages.yml | 94 ++ .../generic/hyph-utf8/lib/tex/hyphen/path.rb | 25 + .../generic/hyph-utf8/lib/tex/hyphen/texlive.rb | 22 + .../hyph-utf8/lib/tex/hyphen/texlive/loader.rb | 193 +++ .../hyph-utf8/lib/tex/hyphen/texlive/package.rb | 126 ++ .../hyph-utf8/lib/tex/hyphen/texlive/source.rb | 117 ++ .../source/generic/hyph-utf8/spec/spec_helper.rb | 7 + .../hyph-utf8/spec/tex/hyphen/language_spec.rb | 985 +++++++++++++ 19 files changed, 2603 insertions(+), 2410 deletions(-) delete mode 100644 Master/texmf-dist/source/generic/hyph-utf8/author-data.rb delete mode 100644 Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/spec/spec_helper.rb create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb (limited to 'Master/texmf-dist/source/generic') diff --git a/Master/texmf-dist/source/generic/hyph-utf8/author-data.rb b/Master/texmf-dist/source/generic/hyph-utf8/author-data.rb deleted file mode 100644 index fa0673c1254..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/author-data.rb +++ /dev/null @@ -1,92 +0,0 @@ -class Author - @@author_data = { -#authors = { - "donald_knuth" => ["Donald", "Knuth", nil, false, false], - "peter_heslin" => ["Peter", "Heslin", nil, false, false], - "dimitrios_filippou" => ["Dimitrios", "Filippou", "dimitrios{dot}filippou{at}riotinto{dot}com", true, true], - "claudio_beccari" => ["Claudio", "Beccari","claudio{dot}beccari{at}gmail{dot}com", true, true], - "juan_aguirregabiria" => ["Juan M.", "Aguirregabiria", "juanmari{dot}aguirregabiria{at}ehu.es", true, true], - "igor_marinovic" => ["Igor", "Marinović", "marinowski{at}gmail.com", true, true], - "tilla_fick" => ["Tilla", "Fick", "fick{dot}tilla{at}gmail{dot}com", true, true], - "chris_swanepoel" => ["Chris", "Swanepoel", "cj{at}swanepoel{dot}net", true, true], - "matjaz_vrecko" => ["Matjaž", "Vrečko", "matjaz{at}mg-soft{dot}si", true, true], - "goncal_badenes" => ["Gonçal", "Badenes", "g{dot}badenes{at}ieee.org", false, false], - "pavel_sevecek" => ["Pavel", "Ševeček", "pavel{at}lingea{dot}cz", false, false], - # email doesn't work - "jana_chlebikova" => ["Jana", "Chlebíková", "chlebikj{at}dcs{dot}fmph{dot}uniba{dot}sk", false, false], - "yannis_haralambous" => ["Yannis", "Haralambous", "yannis{dot}haralambous{at}telecom-bretagne{dot}eu", true, false], - "frank_jensen" => ["Frank", "Jensen", "frank{dot}jensen{at}hugin{dot}com", true, true], - "sergei_pokrovsky" => ["Sergei", "Pokrovsky", "sergio{dot}pokrovskij{at}gmail{dot}com", true, true], - "javier_bezos" => ["Javier", "Bezos", "jbezos{at}tex-tipografia{dot}com", true, true], - "een_saar" => ["Enn", "Saar", "saar{at}aai{dot}ee", false, false], - "dejan_muhamedagic" => ["Dejan", "Muhamedagić", "dejan{at}hello-penguin{dot}com", true, true], - "brian_wilson" => ["Brian", "Wilson", "bountonw{at}gmail{dot}com", true, true], - "arthur_reutenauer" => ["Arthur", "Reutenauer", "arthur{dot}reutenauer{at}normalesup{dot}org", true, true], - "mojca_miklavec" => ["Mojca", "Miklavec", "mojca{dot}miklavec{dot}lists{at}gmail{dot}com", true, true], - "santhosh_thottingal" => ["Santhosh", "Thottingal", "santhosh{dot}thottingal{at}gmail{dot}com>", true, true], - # email doesn't work - "yves_codet" => ["Yves", "Codet", "ycodet{at}club-internet{dot}fr", true, true], - "rune_kleveland" => ["Rune", "Kleveland", nil, false, false], - # email doesn't work - "ole_michael_selberg" => ["Ole Michael", "Selberg", "o{dot}m{dot}selberg{at}c2i{dot}net", true, true], - "dorjgotov_batmunkh" => ["Dorjgotov", "Batmunkh", "bataak{at}gmail{dot}com", true, true], - "nazar_annagurban" => ["Nazar", "Annagurban", "nazartm{at}gmail{dot}com", false, false], - "jan_michael_rynning" => ["Jan Michael", "Rynning", nil, false, false], - "eduard_werner" => ["Eduard", "Werner", "edi{dot}werner{at}gmx{dot}de", false, false], - "werner_lemberg" => ["Werner", "Lemberg", "wl{at}gnu{dot}org", true, true], - # email doesn't work - "pedro_j_de_rezende" => ["Pedro J.", "de Rezende", "rezende{at}ddc{dot}unicamp{dot}br", false, false], - "j_joao_dias_almeida" => ["J. Joao", "Dias Almeida", "jj{at}di{dot}uminho{dot}pt"], - # email doesn't work - "piet_tutelaers" => ["Piet", "Tutelaers", "p{dot}t{dot}h{dot}tutelaers{at}tue{dot}nl", false, false], - "vytas_statulevicius" => ["Vytas", "Statulevičius", "vytas{at}vtex{dot}nl", false, false], - "sigitas_tolusis" => ["Sigitas", "Tolušis", "sigitas{at}vtex{dot}lt", false, false], - "janis_vilims" => ["Janis", "Vilims", "jvilims{at}apollo{dot}lv", false, false], - "joerg_knappen" => ["Jörg", "Knappen", "jknappen{at}web{dot}de", true, true], - "medeni_shemde" => ["Medeni", "Shemdê", nil, false, false], - "terry_mart" => ["Terry", "Mart", "mart{at}kph{dot}uni-mainz{dot}de", false, false], - # email doesn't work - "jorgen_pind" => ["Jorgen", "Pind", "jorgen{at}lexis{dot}hi{dot}is", false, false], - "marteinn_sverrisson" => ["Marteinn", "Sverrisson", nil, false, false], - # email doesn't work - "kristinn_gylfason" => ["Kristinn", "Gylfason", "kristgy{at}ieee{dot}org", false, false], - # email doesn't work - "kevin_p_scannell" => ["Kevin P.", "Scannell", "scanell{at}slu{dot}edu", false, false], - # email doesn't work - "peter_kleiweg" => ["Peter", "Kleiweg", "p{dot}c{dot}c{dot}kleiweg{at}rug{dot}nl", false, false], - "hanna_kolodziejska" => ["Hanna", "Kołodziejska", nil, false, false], - "boguslaw_jackowski" => ["Bogusław", "Jackowski", nil, true, true], - "marek_rycko" => ["Marek", "Ryćko", nil, false, false], - "vladimir_volovich" => ["Vladimir", "Volovich", nil, true, true], # TODO add e-mail address - "alexander_i_lebedev" => ["Alexander I.", "Lebedev", "swan{at}scon155{dot}phys{dot}msu{dot}su", false, false], # Not sure were 'I' belongs - # first email doesn't work - "maksym_polyakov" => ["Maksym", "Polyakov", "polyama{at}auburn{dot}edu", false, false], # Second e-mail address in ukrhypmp.tex: mpoliak@i.com.ua - "adrian_rezus" => ["Adrian", "Rezus", "adriaan{at}\{sci,cs\}{dot}kun{dot}nl", false, false], - # email doesn't work - "sahak_petrosyan" => ["Sahak", "Petrosyan", "sahak{at}mit{dot}edu", true, true], # I think "true, true" is right. Arthur - "dominik_wujastyk" => ["Dominik", "Wujastyk", "wujastyk{at}gmail{dot}com", false, false], - "graham_toal" => ["Graham", "Toal", nil, false, false], - "donald_e_knuth" => ["Donald E.", "Knuth", nil, false, false], # Don doesn't use e-mail ;-) - "gerard_d_c_kuiken" => ["Gerard D.C.", "Kuiken", nil, false, false], - "pierre_mackay" => ["P. A.", "MacKay", nil, true, true], - "h_turgut_uyar" => ["H. Turgut", "Uyar", "uyar{at}itu{dot}edu{tr}", true, true], - # email doesn't work - "s_ekin_kocabas" => ["S. Ekin", "Kocabas", "kocabas{at}stanford{dot}edu", true, true], - "bence_nagy" => ["Bence", "Nagy", "nagybence{at}tipogral{dot}hu", true, true], - "kauko_saarinen" => ["Kauko", "Saarinen", nil, false, false], - "fred_karlsson" => ["Fred", "Karlsson", nil, false, false], - "rene_bastian" => ["René", "Bastian", nil, false, false], # TODO make contact - "daniel_flipo" => ["Daniel", "Flipo", nil, false, false], # TODO make contact - "bernard_gaulle" => ["Bernard", "Gaulle", nil, false, false], # Deceased... - "theppitak_karoonboonyanan" => ["Theppitak", "Karoonboonyanan", "theppitak{at}gmail{dot}com", true, true], - "levan_shoshiashvili" => ["Levan", "Shoshiashvili", "shoshia{at}hotmail{dot}com", true, true], - # email doesn't work - "javier_mugica" => ["Javier", "Múgica", "javier{at}digi21{dot}eu", true, true], - "georgi_boshnakov" => ["Georgi", "Boshnakov", "georgi{dot}boshnakov{at}manchester{dot}ac{dot}uk", true, true], - "mike_kroutikov" => ["Mike", "Kroutikov", "pgmmpk{at}gmail{dot}com", true, true], - "aleksandr_andreev" => ["Aleksandr", "Andreev", "", true, true], - "maksim_salau" => ["Maksim", "Salau", "maksim{dot}salau{at}gmail{dot}com", true, false], - "wie_ming_ang" => ["Wie-Ming", "Cittānurakkho Bhikkhu", "wiemingang{at}gmail{dot}com", true, true] -} -# -end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb index 4b9d8a62c3b..c0781932a38 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb @@ -1,24 +1,28 @@ #!/usr/bin/env ruby # encoding: utf-8 -# this file auto-generates loaders for hyphenation patterns - to be improved +# this file auto-generates loaders for hyphenation patterns - to be improved # No shit, Sherlock -- AR 2018-11-27 -require_relative 'languages.rb' - -$package_name="hyph-utf8" - -# TODO - make this a bit less hard-coded -$path_tex_generic=File.expand_path("../../../../tex/generic", __FILE__) -$path_loadhyph="#{$path_tex_generic}/#{$package_name}/loadhyph" - -# TODO: should be singleton -languages = Language.all +require_relative 'lib/tex/hyphen' +require_relative 'lib/tex/hyphen/texlive' +include TeX::Hyphen +include TeXLive #text_if_native_utf = "\input pattern-loader.tex\n\\ifNativeUtfEightPatterns" -languages.each do |language| +def output(file, string, indent = 0) + if string.is_a? Enumerable + string.each { |line| output(file, line, indent) } + else + file.print ' ' * indent + file.puts(string) + end +end -string_enc = (language.encoding == nil) ? "" : language.encoding.upcase + " " +print 'Generating loaders for ' +Language.all.each do |language| + +# puts language.bcp47 ################ # Header texts # @@ -26,212 +30,93 @@ string_enc = (language.encoding == nil) ? "" : language.encoding.upcase + " " # a message about auto-generation # TODO: write a more comprehensive one -text_header = -"% filename: loadhyph-#{language.code}.tex -% language: #{language.name.safe} + text_header = <<-EOHEADER +% filename: loadhyph-#{language.bcp47}.tex +% language: #{language.babelname} % % Loader for hyphenation patterns, generated by % source/generic/hyph-utf8/generate-pattern-loaders.rb % See also http://tug.org/tex-hyphen % -% Copyright 2008-2016 TeX Users Group. +% Copyright 2008-#{Time.now.year} TeX Users Group. % You may freely use, modify and/or distribute this file. % (But consider adapting the scripts if you need modifications.) % % Once it turns out that more than a simple definition is needed, % these lines may be moved to a separate file. -%" +% + EOHEADER -text_if_native_utf = -'% Test for pTeX +########### +# lccodes # +########### + + lccodes_common = [] + if language.has_apostrophes? then + lccodes_common.push("\\lccode`\\'=`\\'") + end + if language.has_hyphens? then + lccodes_common.push("\\lccode`\\-=`\\-") + end + + next if language.use_old_loader + print language.bcp47, ' ' + + filename = File.join(PATH::LOADER, language.loadhyph) + File.open(filename, "w") do |file| + # puts language.bcp47 + file.puts text_header + file.puts('\begingroup') + + if lccodes_common.length > 0 then + file.puts lccodes_common.join("\n") + end + + # for ASCII encoding, we don't load any special support files, but simply load everything + if language.encoding == 'ascii' && !language.italic? + file.puts "% ASCII patterns - no additional support is needed" + file.puts "\\message{ASCII #{language.message}}" + file.puts "\\input hyph-#{language.bcp47}.tex" + else + file.puts '% Test for pTeX \\ifx\\kanjiskip\\undefined % Test for native UTF-8 (which gets only a single argument) % That\'s Tau (as in Taco or ΤΕΧ, Tau-Epsilon-Chi), a 2-byte UTF-8 character \\def\\testengine#1#2!{\\def\\secondarg{#2}}\\testengine Τ!\\relax \\ifx\\secondarg\\empty' - -comment_engine_utf8 = "% Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument" -comment_engine_8bit = "% 8-bit engine (such as TeX or pdfTeX)" -comment_engine_ptex = "% pTeX" - -text_engine_ascii = ["% ASCII patterns - no additional support is needed", - "\\message{ASCII #{language.message}}", - "\\input hyph-#{language.code}.tex"] -text_engine_utf8 = [" #{comment_engine_utf8}", - " \\message{UTF-8 #{language.message}}"] -text_engine_8bit = [" #{comment_engine_8bit}", - " \\message{#{string_enc}#{language.message}}"] -text_engine_ptex = [" #{comment_engine_ptex}", - " \\message{#{string_enc}#{language.message}}"] -text_engine_8bit_no = [" #{comment_engine_8bit}", - " \\message{No #{language.message} - only for Unicode engines}", - " %\\input zerohyph.tex"] -text_engine_ptex_no = [" #{comment_engine_ptex}", - " \\message{No #{language.message} - only for Unicode engines}", - " %\\input zerohyph.tex"] -text_patterns = " \\input hyph-#{language.code}.tex" -text_patterns_ptex = " \\input hyph-#{language.code}.#{language.encoding}.tex" -text_patterns_old = " \\input #{language.filename_old_patterns}" -text_patterns_conv = " \\input conv-utf8-#{language.encoding}.tex" -text_patterns_utf8 = text_patterns - -text_patterns_quote = " \\input hyph-quote-#{language.code}.tex" - -########### -# lccodes # -########### - -lccodes_common = [] -if language.has_apostrophes then - lccodes_common.push("\\lccode`\\'=`\\'") -end -if language.has_dashes then - lccodes_common.push("\\lccode`\\-=`\\-") -end - -if ['sh-latn', 'sh-cyrl'].include?(language.code) then - text_patterns_utf8 = [" \\input hyph-sh-latn.tex", - " \\input hyph-sh-cyrl.tex"] - text_engine_utf8 = [" #{comment_engine_utf8}", - " \\message{UTF-8 Serbian hyphenation patterns}", - " % We load both scripts at the same time to simplify usage"] -end - - next if language.use_old_loader - - if language.code == 'sh-latn' then - filename = "#{$path_loadhyph}/loadhyph-sr-latn.tex" - elsif language.code == 'sh-cyrl' then - filename = "#{$path_loadhyph}/loadhyph-sr-cyrl.tex" - else - filename = "#{$path_loadhyph}/loadhyph-#{language.code}.tex" - end - puts "generating '#{filename}'" - - File.open(filename, "w") do |file| - file.puts text_header - file.puts('\begingroup') - - if lccodes_common.length > 0 then - file.puts lccodes_common.join("\n") - end + output(file, language.format_inputs(language.utf8_chunk), 2) + file.puts("\\else\n") + output(file, language.format_inputs(language.nonutf8_chunk('8-bit')), 2) + file.puts("\\fi\\else\n") + output(file, language.format_inputs(language.nonutf8_chunk('pTeX')), 2) + file.puts("\\fi\n") + end ######################################## # GROUP nr. 1 - ONLY USABLE WITH UTF-8 # ######################################## - # some special cases first - # - # some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines - # TODO - maybe consider doing something similar for ibycus - if ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include?(language.code) then - file.puts(text_if_native_utf) - file.puts(text_engine_utf8) - # lccodes - if language.code == 'mul-ethi' then - file.puts(' % Set \lccode for Ethiopian word space.') - file.puts(' \lccode"1361="1361') - file.puts(' \lccode"1362="1362') - elsif language.code == "cu" then - file.puts(' % fix lccodes for some characters (they were recently included in Unicode)') - file.puts(' \lccode"1C82="1C82 % sharp o in lowercase "uk"') - ['1DF6', '1DF7', '1DF8', '1DF9', 'A69E', '1C86', 'A67E', 'FE2E', 'FE2F'].each do |l| - file.puts(" \\lccode\"#{l}=\"#{l}") - end - elsif language.code != 'lo' then - file.puts(' % Set \lccode for ZWNJ and ZWJ.') - file.puts(' \lccode"200C="200C') - file.puts(' \lccode"200D="200D') - if language.code == 'sa' then - file.puts(' % Set \lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.') - file.puts(' \lccode"0CF1="0CF1') - file.puts(' \lccode"0CF2="0CF2') - end - end - file.puts(text_patterns) - file.puts('\else') - file.puts(text_engine_8bit_no) - file.puts('\fi\else') - file.puts(text_engine_ptex_no) - file.puts('\fi') + # some special cases first + # + # some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines + # TODO - maybe consider doing something similar for ibycus ####################### # GROUP nr. 2 - ASCII # ####################### - elsif ['it', 'pms', 'rm'].include?(language.code) - file.puts(text_if_native_utf) - file.puts(text_engine_utf8) - file.puts(text_patterns) - file.puts(text_patterns_quote) - file.puts('\else') - file.puts(text_engine_8bit) - file.puts(text_patterns) - file.puts('\fi\else') - file.puts(text_engine_ptex) - file.puts(text_patterns) - file.puts('\fi') - # for ASCII encoding, we don't load any special support files, but simply load everything - elsif language.encoding == 'ascii' then - file.puts(text_engine_ascii) + #################################### # GROUP nr. 3 - different patterns # #################################### - # when lanugage uses old patterns for 8-bit engines, load two different patterns rather than using the converter - elsif language.use_old_patterns then - file.puts(text_if_native_utf) - file.puts(text_engine_utf8) - # some catcodes for XeTeX - if language.code == 'grc' or language.code.slice(0,2) == 'el' then - file.puts(" \\lccode`'=`'\\lccode`’=`’\\lccode`ʼ=`ʼ\\lccode`᾽=`᾽\\lccode`᾿=`᾿") - end - file.puts(text_patterns) - if language.code == 'uk' then - file.puts(text_patterns_quote) - end - file.puts('\else') - file.puts(text_engine_8bit) - # explain why we are still using the old patterns - if language.use_old_patterns_comment != nil then - file.puts(" % #{language.use_old_patterns_comment}") - else - puts "Missing comment for #{language.name}" - file.puts(' % we still load old patterns for 8-bit TeX') - end - file.puts(text_patterns_old) - file.puts('\fi\else') - file.puts(text_engine_ptex) - # greek, coptic - if language.encoding == nil then - file.puts(text_patterns_old) - else - file.puts(text_patterns_ptex) - end - file.puts('\fi') + # when lanugage uses old patterns for 8-bit engines, load two different patterns rather than using the converter + # greek, coptic ######################### # GROUP nr. 4 - regular # ######################### - else - file.puts(text_if_native_utf) - file.puts(text_engine_utf8) - file.puts(text_patterns_utf8) - if language.has_apostrophes - file.puts(text_patterns_quote) - end - file.puts('\else') - file.puts(text_engine_8bit) - if ['la-x-liturgic'].include?(language.code) then - file.puts(text_patterns_ptex) - else - file.puts(text_patterns_conv) - file.puts(text_patterns) - end - file.puts('\fi\else') - file.puts(text_engine_ptex) - file.puts(text_patterns_ptex) - file.puts('\fi') - end ####### # end # ####### - file.puts('\endgroup') - end + file.puts('\endgroup') + end end + +puts diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb index fe64c38364b..39f0bc19a6a 100755 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb @@ -6,22 +6,24 @@ # use 'gem install unicode' if unicode is missing on your computer require 'unicode' -require_relative 'languages.rb' -include Language::TeXLive +require_relative 'lib/tex/hyphen/language.rb' +# include OldLanguage::TeXLive +include TeX::Hyphen # FIXME sr-cyrl? +print 'Generating plain files for (parenthesised tags are skipped) ' Language.all.sort.each do |language| - code = language.code + bcp47 = language.bcp47 - if language.use_old_loader || code == 'mn-cyrl-x-lmc' - puts "(skipping #{language.code})" + if language.use_old_loader || bcp47 == 'mn-cyrl-x-lmc' + print '(', language.bcp47, ') ' next + else + print bcp47, ' ' end - puts "generating #{code}" - outfile = Proc.new do |ext| - File.open File.join(PATH::TXT, sprintf('hyph-%s.%s.txt', code, ext)), 'w' + File.open File.join(PATH::TXT, sprintf('hyph-%s.%s.txt', bcp47, ext)), 'w' end # patterns @@ -34,7 +36,7 @@ Language.all.sort.each do |language| # apostrophes if applicable with_apostrophe = patterns[:with_apostrophe] if with_apostrophe - file = File.open File.join(PATH::QUOTE, sprintf('hyph-quote-%s.tex', code)), 'w' + file = File.open File.join(PATH::QUOTE, sprintf('hyph-quote-%s.tex', bcp47)), 'w' file.printf "\\bgroup\n\\lccode`\\’=`\\’\n\\patterns{\n" with_apostrophe.each do |pattern| file.printf "%s\n", pattern @@ -54,7 +56,8 @@ Language.all.sort.each do |language| # comments and licence file = outfile.('lic') - file.puts language.get_comments_and_licence + file.puts language.comments_and_licence file.close end +puts diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb index e537fe43ba7..0f50706dc01 100755 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb @@ -1,7 +1,10 @@ #!/usr/bin/env ruby require_relative 'hyph-utf8' -require_relative 'languages' +require_relative 'lib/tex/hyphen' +require_relative 'lib/tex/hyphen/texlive' +include TeX::Hyphen +include TeXLive # this file generates patterns for pTeX out of the plain ones @@ -9,7 +12,7 @@ require_relative 'languages' encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"] encodings = Hash.new encodings_list.each do |encoding_name| - encodings[encoding_name] = HyphEncoding.new(encoding_name) + encodings[encoding_name] = HyphEncoding.new(encoding_name) end header = <<-HEADER @@ -24,53 +27,56 @@ header = <<-HEADER %% HEADER +print 'Generating pTeX patterns for (skipped # reason): ' Language.all.sort.each do |language| - if language.use_old_loader - puts "(skipping #{language.code} # loader)" - next - end + if language.use_old_patterns_comment || !language.encoding || language.encoding == 'ascii' + if language.use_old_patterns_comment + reason = 'legacy patterns' + else + reason = sprintf 'encoding: %s', language.encoding.inspect + end + print '(', language.bcp47, ' # ', reason, ') ' - if language.encoding == nil || language.encoding == 'ascii' - puts "(skipping #{language.code} # #{if language.encoding then 'ascii' else 'encoding' end})" - next - else - encoding = encodings[language.encoding] - end + next + else + encoding = encodings[language.encoding] + end - code = language.code + bcp47 = language.bcp47 - puts ">> generating #{code} (#{language.name.safe})" - File.open(File.join(PATH::PTEX, sprintf('hyph-%s.%s.tex', code, language.encoding)), 'w') do |file_ptex| - patterns = language.patterns - exceptions = language.exceptions + print bcp47, ' ' + File.open(File.join(PATH::PTEX, sprintf('hyph-%s.%s.tex', bcp47, language.encoding)), 'w') do |file_ptex| + patterns = language.patterns + exceptions = language.exceptions - characters = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq + characters = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq - if language.encoding != 'ascii' then - patterns = encoding.convert_to_escaped_characters(patterns) - exceptions = encoding.convert_to_escaped_characters(exceptions) - end + if language.encoding != 'ascii' then + patterns = encoding.convert_to_escaped_characters(patterns) + exceptions = encoding.convert_to_escaped_characters(exceptions) + end - file_ptex.printf(header, language.name.safe, language.code, language.encoding, language.code) + file_ptex.printf(header, language.babelname, language.bcp47, language.encoding, language.bcp47) - file_ptex.puts("\\bgroup") - # setting lccodes for letters - characters.each do |c| - if (c == 0x01FD or c == 0x0301) and language.code == 'la-x-liturgic' - # skip - elsif c >= 128 then - code = encoding.unicode_characters[c].code_enc - file_ptex.printf("\\lccode\"%02X=\"%02X\n", code, code) - end - end - # patterns - if patterns.length > 0 then - file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}") - end - # exceptions - if exceptions.length > 0 then - file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}") - end - file_ptex.puts("\\egroup") - end + file_ptex.puts("\\bgroup") + # setting lccodes for letters + characters.each do |c| + if (c == 0x01FD or c == 0x0301) and language.bcp47 == 'la-x-liturgic' + # skip + elsif c >= 128 then + code = encoding.unicode_characters[c].code_enc + file_ptex.printf("\\lccode\"%02X=\"%02X\n", code, code) + end + end + # patterns + if patterns.length > 0 then + file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}") + end + # exceptions + if exceptions.length > 0 then + file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}") + end + file_ptex.puts("\\egroup") + end end +puts diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb index 223b3f04579..7460658f020 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb @@ -3,18 +3,20 @@ # this file auto-generates tlpsrc files for hyphenation patterns - to be improved -require_relative 'languages.rb' -include Language::TeXLive +require_relative 'lib/tex/hyphen' +require_relative 'lib/tex/hyphen/texlive' +include TeX::Hyphen include TeXLive #--------# # TLPSRC # #--------# +print 'Generating .tlpsrc files for packages: ' Package.all.sort.each do |package| tlpsrcname = File.join(PATH::TLPSRC, sprintf('hyphen-%s.tlpsrc', package.name.safe)) file_tlpsrc = File.open(tlpsrcname, 'w') - printf "generating %s\n", tlpsrcname + print package.name.safe, ' ' file_tlpsrc.puts "category TLCore" package.list_dependencies.each do |dependency| @@ -23,23 +25,16 @@ Package.all.sort.each do |package| # FIXME Still doesn’t work well for Latin file_tlpsrc.printf "shortdesc %s.\n", package.description_s - package.description_l.each do |line| - file_tlpsrc.printf "longdesc %s\n", line - end + package.description.split("\n").each do |line| + file_tlpsrc.printf "longdesc %s\n", line + end package.languages.each do |language| - if language.description_s && language.description_l then - # file_tlpsrc.printf "shortdesc %s.\n", language.description_s - # language.description_l.each do |line| - # file_tlpsrc.printf "longdesc %s\n", line - # end - end - - file_tlpsrc.printf "execute AddHyphen \\\n\tname=%s%s \\\n", language.name.safe, language.list_synonyms + file_tlpsrc.printf "execute AddHyphen \\\n\tname=%s%s \\\n", language.babelname, language.list_synonyms file_tlpsrc.printf "\t%s \\\n\t%s", language.list_hyphenmins, language.list_loader if language.patterns_line + language.exceptions_line != "" file_tlpsrc.printf " \\\n\t%s \\\n\t%s", language.patterns_line, language.exceptions_line end - if language.code == "mn-cyrl-x-lmc" then + if language.bcp47 == "mn-cyrl-x-lmc" then file_tlpsrc.printf " \\\n\tluaspecial=\"disabled:only for 8bit montex with lmc encoding\"" end # end-of-line @@ -61,22 +56,14 @@ Package.all.sort.each do |package| end file_tlpsrc.close end +puts -#--------------# -# language.dat # -#--------------# -language_dat_filename = File.join PATH::LANGUAGE_DAT, 'language.dat' -File.open(language_dat_filename, 'w') do |file_language_dat| - printf "Generating %s\n", language_dat_filename - Package.all.sort.each do |package| - package.languages.each do |language| - # main language name - file_language_dat.printf "%s\t%s\n", language.name.safe, language.loadhyph +#--------------------------# +# language.dat and friends # +#--------------------------# +system sprintf "tlmgr generate --dest %s language.dat", File.join(PATH::LANGUAGE_DAT, 'language.dat') +system sprintf "tlmgr generate --dest %s language.dat.lua", File.join(PATH::LANGUAGE_DAT, 'language.dat.lua') - # synonyms - language.synonyms.each do |synonym| - file_language_dat.printf "=%s\n", synonym - end - end - end -end +# Revert changes if only line with date was changed +diff_lines = `git diff #{PATH::LANGUAGE_DAT}/language.dat* | egrep '^[+-](%|--) Generated' | wc -l`.strip +`git checkout HEAD -- #{PATH::LANGUAGE_DAT}/language.dat*` if diff_lines == '4' diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb deleted file mode 100644 index 20607a45109..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb +++ /dev/null @@ -1,132 +0,0 @@ -#!/usr/bin/env ruby - -require_relative 'languages.rb' - -include TeXLive -include Language::TeXLive - -# this file auto-generates loaders for hyphenation patterns - to be improved - -$package_name="hyph-utf8" - -# TODO - make this a bit less hard-coded -$path_tex_generic="../../../tex/generic" - -language_default = Language.new({ - "code" => "(default)", - "name" => "english", "synonyms" => ["usenglish", "USenglish", "american"], - "hyphenmin" => [2,3], - "encoding" => "ascii", - "type" => "dictionary", - "authors" => [ "donald_knuth" ], - "licence" => "", -}) - - -# add english to the list -# $l["default"] = language_default -# TODO: should be singleton -languages = Language.all.sort{|a,b| a.name <=> b.name} - -$a = Author - -language_grouping = { - 'english' => ['default', 'en-gb', 'en-us'], - 'latin' => ['la', 'la-x-classic'], - 'norwegian' => ['nb', 'nn'], - 'german' => ['de-1901', 'de-1996','de-ch-1901'], - 'mongolian' => ['mn-cyrl', 'mn-cyrl-x-lmc'], - 'greek' => ['el-monoton', 'el-polyton'], - 'ancientgreek' => ['grc', 'grc-x-ibycus'], - 'chinese' => ['zh-latn-pinyin'], - # TODO - until someone tells what to do - #'serbian' => ['sr-latn', 'sr-cyrl'], - 'serbian' => ['sh-latn', 'sh-cyrl'], -} - -space_leading = " " -space_tr = " " - -Package.all.sort.each do |package| - first_line_printed = false - package.languages.each do |language| - if language != nil then - puts "#{space_leading}" - - line_content = "" - if not first_line_printed then - line_content = "#{package.name.capitalize}" - first_line_printed = true; - else - line_content = " " - end - puts "#{space_leading}#{space_tr}#{line_content}" - - # synonyms - if language.synonyms != nil and language.synonyms.length > 0 then - synonyms=", #{language.synonyms.join(', ')}" - else - synonyms="" - end - puts "#{space_leading}#{space_tr}#{language.name}#{synonyms}" - - # if language.use_old_patterns == false then - if language.use_old_loader then - url_patterns = "" - code = language.code - else - url_patterns = "#{$ctan_url}/patterns/tex/hyph-#{language.code}.tex" - code = "#{language.code}" - end - - puts "#{space_leading}#{space_tr}#{code}" - - # lefthyphenmin/righthyphenmin - if language.hyphenmin == nil or language.hyphenmin.length == 0 then - lmin = '' - rmin = '' - elsif language.filename_old_patterns == "zerohyph.tex" then - lmin = '' - rmin = '' - else - lmin = language.hyphenmin[0] - rmin = language.hyphenmin[1] - end - puts "#{space_leading}#{space_tr}(#{lmin},#{rmin})" - # which file to use - if language.use_old_loader then - file = "#{language.filename_old_patterns}" - else - file = "loadhyph-#{language.code}.tex" - end - #puts "\t#{file}" - if language.encoding == nil then - encoding = "" - else - encoding = language.encoding.upcase - end - puts "#{space_leading}#{space_tr}#{encoding}" - - # licence - licence = language.licence - puts "#{space_leading}#{space_tr}#{licence}" - - authors = [] - if language.authors != nil then - language.authors.each do |a| - author = $a[a] - if author != nil then - authors.push("#{author.name} #{author.surname}") - else - puts "author is nil!!!" - end - end - end - puts "#{space_leading}#{space_tr}#{authors.join('
')}" - - # finish the language definition - puts "#{space_leading}\n" - end - end -end - diff --git a/Master/texmf-dist/source/generic/hyph-utf8/language-data.rb b/Master/texmf-dist/source/generic/hyph-utf8/language-data.rb index 368eb8b37c7..d4f387c0725 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/language-data.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/language-data.rb @@ -1,4 +1,4 @@ -class Language +class OldLanguage # "use_new_loader" # => true - create a new file and use that one # => false - use "filename_old_patterns" in language.dat @@ -17,971 +17,173 @@ class Language # "message" => nil, @@language_data = [ -# -------------------------------------- -# languages with no hyphenation patterns -# -------------------------------------- -# arabic -{ - "code" => "ar", - "name" => "arabic", - "use_old_loader" => true, - "filename_old_patterns" => "zerohyph.tex", -# "hyphenmin" => [], # not needed - "encoding" => nil, - "exceptions" => false, - "message" => nil, - - # TODO - "description_l" => [ - #......................................................................# - "Prevent hyphenation in Arabic.", - ], -}, -# farsi -# =persian -{ - "code" => "fa", - "name" => "farsi", "synonyms" => ["persian"], - "use_old_loader" => true, - "filename_old_patterns" => "zerohyph.tex", -# "hyphenmin" => [], # not needed - "encoding" => nil, - "exceptions" => false, - "message" => nil, - - # TODO - "description_l" => [ - #......................................................................# - "Prevent hyphenation in Persian.", - ], -}, -# ---------------------------- -# languages using old patterns -# ---------------------------- -# greek -# =polygreek -# monogreek { "code" => "el-monoton", - "name" => "monogreek", - "use_old_patterns" => true, - "use_old_patterns_comment" => "Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.", - "filename_old_patterns" => "grmhyph5.tex", - "hyphenmin" => [1,1], # polyglossia - "encoding" => nil, - "exceptions" => true, - "message" => "Hyphenation patterns for uni-accent (monotonic) Modern Greek", - - "version" => "5.0", - "last_modified" => "2008-06-06", "type" => "rules", - "authors" => ["dimitrios_filippou"], - "licence" => "LPPL", - # "description_s" => "Monotonic Modern Greek hyphenation patterns", - # "description_l" => [ - # #......................................................................# - # "Hyphenation patterns for Modern Greek in monotonic spelling.", - # "The pattern file used for 8-bit engines is grmhyph5.tex, in Babel's LGR encoding,", - # "that is not part of hyph-utf8.", - # "Patterns in UTF-8 use two code positions for each of the vowels with acute accent", - # "(a.k.a tonos, oxia), e.g., U+03AD, U+1F73 for epsilon.", - # ], - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Modern Greek in monotonic and polytonic", - "spelling in LGR and UTF-8 encodings. Patterns in UTF-8 use two code", - "positions for each of the vowels with acute accent (a.k.a tonos,", - "oxia), e.g., U+03AC, U+1F71 for alpha.", - ], }, { "code" => "el-polyton", - "name" => "greek", "synonyms" => ["polygreek"], - "use_old_patterns" => true, - "use_old_patterns_comment" => "Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.", - "filename_old_patterns" => "grphyph5.tex", - # left/right hyphen min for Greek can be as low as one (1), - # but for aesthetic reasons keep them at 2/2. - # Dimitrios Filippou - "hyphenmin" => [1,1], # polyglossia - "encoding" => nil, - "exceptions" => true, - "message" => "Hyphenation patterns for multi-accent (polytonic) Modern Greek", - - "version" => "5.0", - "last_modified" => "2008-06-06", "type" => "rules", - "authors" => ["dimitrios_filippou"], - "licence" => "LPPL", - # "description_s" => "Polytonic Modern Greek hyphenation patterns", - # # #.....................................................................# - # "description_l" => [ - # #......................................................................# - # "Hyphenation patterns for Modern Greek in polytonic spelling.", - # "The pattern file used for 8-bit engines is grphyph5.tex that is", - # "not part of hyph-utf8. Patterns in UTF-8 use two code positions for", - # "each of the vowels with acute accent (a.k.a tonos, oxia), e.g.,", - # "U+03AC, U+1F71 for alpha.", - # ], }, -# ancientgreek { "code" => "grc", - "name" => "ancientgreek", - "use_old_patterns" => true, - "use_old_patterns_comment" => "Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.", - "filename_old_patterns" => "grahyph5.tex", - "hyphenmin" => [1,1], # polyglossia - "encoding" => nil, - "exceptions" => false, - "message" => "Hyphenation patterns for Ancient Greek", - - "version" => "5.0", - "last_modified" => "2016-05-12", "type" => "rules", - "authors" => ["dimitrios_filippou"], - "licence" => "LPPL", - # "description_s" => "Ancient Greek hyphenation patterns", - # "description_l" => [ - # #......................................................................# - # "Hyphenation patterns for Ancient Greek.", - # "The pattern file used for 8-bit engines is grahyph5.tex, in Babel's LGR encoding,", - # "that is not part of hyph-utf8.", - # "Patterns in UTF-8 use two code positions for each of the vowels with acute accent", - # "(a.k.a tonos, oxia), e.g., U+03AE, U+1F75 for eta.", - # ], - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Ancient Greek in LGR and UTF-8 encodings,", - "including support for (obsolete) Ibycus font encoding.", - "Patterns in UTF-8 use two code positions for each of the vowels with", - "acute accent (a.k.a tonos, oxia), e.g., U+03AE, U+1F75 for eta.", - ], }, -# ------------------------------- -# special patterns, not converted -# ------------------------------- -# ibycus ibyhyph.tex -{ - "code" => "grc-x-ibycus", - "name" => "ibycus", - "use_old_patterns" => true, - "use_old_loader" => true, - "filename_old_patterns" => "ibyhyph.tex", - "hyphenmin" => [2,2], - "encoding" => nil, - "exceptions" => false, - "message" => "Ancient Greek hyphenation patterns for Ibycus encoding", - - # "authors" => ["peter_heslin"], - "version" => 3.0, -}, -# ---------------------------------------------------------- -# more languages using old patterns (see el-* and grc above) -# ---------------------------------------------------------- -# coptic { "code" => "cop", - "name" => "coptic", - "use_old_patterns" => true, - "use_old_patterns_comment" => "TODO: automatic conversion could be done, but was too complicated; leave for later.", - "filename_old_patterns" => "copthyph.tex", - "hyphenmin" => [1,1], # polyglossia TODO: no documentation found - "encoding" => nil, - "exceptions" => false, - "message" => "Coptic hyphenation patterns", - - "version" => nil, - "last_modified" => "2004-10-03", "type" => "rules", - "authors" => [ "claudio_beccari" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Coptic in UTF-8 encoding", - "as well as in ASCII-based encoding for 8-bit engines.", - "The latter can only be used with special Coptic fonts (like CBcoptic).", - "The patterns are considered experimental.", - ], }, -# german { "code" => "de-1901", - "name" => "german", - "use_old_patterns" => true, - "use_old_patterns_comment" => "Kept for the sake of backward compatibility, but newer and better patterns by WL are available.", - "filename_old_patterns" => "dehypht.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "German hyphenation patterns (traditional orthography)", - - "version" => "0.40", - "last_modified" => "2014-05-21", # For the active project - "type" => "dictionary", - "authors" => ["werner_lemberg"], - "licence" => "LPPL", - # "description_s" => "German hyphenation patterns in traditional spelling", - # "description_l" => [ - # "Hyphenation patterns for German in traditional spelling, in T1/EC and UTF-8 encoding.", - # "Patterns encoded in UTF-8 are provided by a separate package.", - # ], - - "description_l" => [ - #......................................................................# - "Hyphenation patterns for German in T1/EC and UTF-8 encodings,", - "for traditional and reformed spelling, including Swiss German.", - "The package includes the latest patterns from dehyph-exptl", - "(known to TeX under names 'german', 'ngerman' and 'swissgerman'),", - "however 8-bit engines still load old versions of patterns", - "for 'german' and 'ngerman' for backward-compatibility reasons.", - "Swiss German patterns are suitable for Swiss Standard German", - "(Hochdeutsch) not the Alemannic dialects spoken in Switzerland", - "(Schwyzerduetsch).", - "There are no known patterns for written Schwyzerduetsch.", - ] + "type" => "dictionary", }, -# ngerman { "code" => "de-1996", - "name" => "ngerman", - "use_old_patterns" => true, - "use_old_patterns_comment" => "Kept for the sake of backward compatibility, but newer and better patterns by WL are available.", - "filename_old_patterns" => "dehyphn.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "German hyphenation patterns (reformed orthography)", - - "version" => "0.40", - "last_modified" => "2014-05-21", # For the active project "type" => "dictionary", - "authors" => ["werner_lemberg"], - "licence" => "LPPL", - # "description_s" => "German hyphenation patterns in reformed spelling", - # "description_l" => [ - # "Hyphenation patterns for German in reformed spelling, in T1/EC and UTF-8 encoding.", - # "Patterns encoded in UTF-8 are provided by a separate package.", - # ], }, -# swissgerman { "code" => "de-ch-1901", - "name" => "swissgerman", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Swiss-German hyphenation patterns (traditional orthography)", - - "version" => "0.40", - "last_modified" => "2014-05-21", # For the active project - "type" => "dictionary", - "authors" => ["werner_lemberg"], - "licence" => "LPPL", - # "description_l" => "German hyphenation patterns, traditional spelling in Switzerland.", - # "description_s" => [ - # "Hyphenation patterns for German in traditional spelling as used in Switzerland, in T1/EC and UTF-8.", - # "These patterns are suitable for Standard German (Hochdeutsch), not the Alemannic dialects", - # "spoken in Switzerland (Schwyzerdütsch). There are no patterns for written Schwyzerdütsch", - # "we know of.", - # ], + "type" => "dictionary", }, -# russian { "code" => "ru", - "name" => "russian", - "use_old_patterns" => true, - "use_old_patterns_comment" => "The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.", - "filename_old_patterns" => "ruhyphen.tex", - "hyphenmin" => [2,2], - "encoding" => "t2a", - "exceptions" => false, - "message" => "Russian hyphenation patterns", - - "version" => nil, - "last_modified" => "2003-03-10", # Date of the patterns proper, not the support files - "type" => "dictionary", - "authors" => ["alexander_i_lebedev", "werner_lemberg", "vladimir_volovich"], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Russian in T2A and UTF-8 encodings.", - "For 8-bit engines, the 'ruhyphen' package provides a number of", - "different pattern sets, as well as different (8-bit) encodings, that", - "can be chosen at format-generation time. The UTF-8 version only", - "provides the default pattern set. A mechanism similar to the one used", - "for 8-bit patterns may be implemented in the future.", - ], + "type" => "dictionary", }, -# ukrainian { "code" => "uk", - "name" => "ukrainian", - "use_old_patterns" => true, - "use_old_patterns_comment" => "The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.", - "filename_old_patterns" => "ukrhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "t2a", - "exceptions" => false, - "message" => "Ukrainian hyphenation patterns", - - "version" => nil, - "last_modified" => "2001-05-10", # Date of the patterns proper # "type" => "rules", # TODO: it is not really clear - "authors" => ["maksym_polyakov", "werner_lemberg", "vladimir_volovich"], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Ukrainian in T2A and UTF-8 encodings.", - "For 8-bit engines, the 'ukrhyph' package provides a number of", - "different pattern sets, as well as different (8-bit) encodings, that", - "can be chosen at format-generation time. The UTF-8 version only", - "provides the default pattern set. A mechanism similar to the one used", - "for 8-bit patterns may be implemented in the future.", - ], }, -# ---------------------------- -# languages using new patterns -# ---------------------------- -# afrikaans { "code" => "af", - "name" => "afrikaans", - "filename_old_patterns" => nil, - "hyphenmin" => [1,2], # in babel: 2,2 - "encoding" => "ec", - "exceptions" => true, - "message" => "Afrikaans hyphenation patterns", - - "version" => "1.0", - "last_modified" => "2013-10-08", - "type" => "dictionary", - "authors" => [ "tilla_fick", "chris_swanepoel" ], - "email" => [ "hyphen{at}rekenaar{dot}net" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Afrikaans in T1/EC and UTF-8 encodings.", - "OpenOffice includes older patterns created by a different author,", - "but the patterns packaged with TeX are considered superior in quality.", - # "Word list used to generate patterns with opatgen might be released in future.", - ], + "type" => "dictionary", }, -# catalan { "code" => "ca", - "name" => "catalan", - "filename_old_patterns" => "cahyph.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => true, - "message" => "Catalan hyphenation patterns", - - "version" => "1.11", - "last_modified" => "2003-07-15", - "type" => "rules", # not only rules, also patgen, but it is a good approximation - "authors" => [ "goncal_badenes" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Catalan in T1/EC and UTF-8 encodings.", - ], + "type" => "rules", # not only rules, also patgen, but it is a good approximation }, -# czech { "code" => "cs", - "name" => "czech", - "filename_old_patterns" => "czhyph.tex", "filename_old_patterns_other" => ["czhyphen.tex","czhyphen.ex"], + "type" => "dictionary", # Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 # Typographical rules allow \righthyphenmin=2 when typesetting in a # narrow column (newspapers etc.). # (used to be 2,2) - "hyphenmin" => [2,3], - "encoding" => "ec", - "exceptions" => true, - "message" => "Czech hyphenation patterns", - "version" => "3", # guessing based on CTAN/macros/cstex/base/csplain.tar.gz: # 1998-12-17 (patterns) # 1995-08-23 (exceptions) # but patterns claim 1995 - "last_modified" => "1995", # todo: no date - "type" => "dictionary", - "authors" => [ "pavel_sevecek" ], - "licence" => "GPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Czech in T1/EC and UTF-8 encodings.", - "Original patterns 'czhyphen' are still distributed in the 'csplain'", - "package and loaded with ISO Latin 2 encoding (IL2).", - # however hyph-utf8 could also be used for that - ], }, -# slovak { "code" => "sk", - "name" => "slovak", - "filename_old_patterns" => "skhyph.tex", - "filename_old_patterns_other" => ["skhyphen.tex","skhyphen.ex"], - # see czech - "hyphenmin" => [2,3], - "encoding" => "ec", - "exceptions" => true, - "message" => "Slovak hyphenation patterns", - - "version" => "2", - "last_modified" => "1992-04-24", "type" => "dictionary", - "authors" => [ "jana_chlebikova" ], - "licence" => "GPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Slovak in T1/EC and UTF-8 encodings.", - "Original patterns 'skhyphen' are still distributed in the 'csplain'", - "package and loaded with ISO Latin 2 encoding (IL2).", - # however hyph-utf8 could also be used for that - ], + "filename_old_patterns_other" => ["skhyphen.tex","skhyphen.ex"], }, -# welsh { "code" => "cy", - "name" => "welsh", - "filename_old_patterns" => "cyhyph.tex", - "hyphenmin" => [2,3], - "encoding" => "ec", - "exceptions" => false, - "message" => "Welsh hyphenation patterns", - - "version" => nil, - "last_modified" => "1996", - "type" => "dictionary", - "authors" => [ "yannis_haralambous" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Welsh in T1/EC and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# danish { "code" => "da", - "name" => "danish", - "filename_old_patterns" => "dkhyph.tex", - "filename_old_patterns_other" => ["dkcommon.tex", "dkspecial.tex"], - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Danish hyphenation patterns", - - "version" => nil, - "last_modified" => "2011-01-11", "type" => "dictionary", - "authors" => [ "frank_jensen" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Danish in T1/EC and UTF-8 encodings.", - ], + "filename_old_patterns_other" => ["dkcommon.tex", "dkspecial.tex"], }, -# esperanto { "code" => "eo", - "name" => "esperanto", - "filename_old_patterns" => "eohyph.tex", - "hyphenmin" => [2,2], - "encoding" => "il3", # TODO - "exceptions" => false, - "message" => "Esperanto hyphenation patterns", - - "version" => nil, - "last_modified" => "1999-08-10", - "type" => "rules", - "authors" => [ "sergei_pokrovsky" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Esperanto ISO Latin 3 and UTF-8 encodings.", - "Note that TeX distributions don't ship any suitable fonts in Latin 3", - "encoding, so unless you create your own font support or want to use", - "MlTeX, using native Unicode engines is highly recommended.", - # "Hyphenation patterns for Esperanto ISO Latin 3 and UTF-8 encodings.", - # "Note that TeX distributions usually don't ship any suitable fonts in", - # "Latin 3 encoding, so unless you create your own font support or want", - # "to use MlTeX, using native UTF-8 engines is highly recommended.", - ], + "type" => "rules", }, -# spanish -# =espanol { "code" => "es", - "name" => "spanish", "synonyms" => ["espanol"], - "filename_old_patterns" => "eshyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Spanish hyphenation patterns", - - "version" => "4.7", - "last_modified" => "2015-11-12", - "type" => "dictionary", - "authors" => [ "javier_bezos" ], - "licence" => "MIT/X11", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Spanish in T1/EC and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# basque { "code" => "eu", - "name" => "basque", - "filename_old_patterns" => "bahyph.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Basque hyphenation patterns", - - "version" => nil, - "last_modified" => "2008-06-26", - "type" => "rules", - "authors" => [ "juan_aguirregabiria" ], - "licence" => "other-free", # "public-check", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Basque in T1/EC and UTF-8 encodings.", - # "Generating scripts for these rule-based patterns is included in hyph-utf8." - ], + "type" => "rules", }, -# french -# =patois -# =francais { "code" => "fr", - "name" => "french", "synonyms" => ["patois","francais"], - "filename_old_patterns" => "frhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "French hyphenation patterns", - - "version" => "2.13", - "last_modified" => "2016-05-12", - "type" => "rules", - "authors" => ["daniel_flipo", "bernard_gaulle", "arthur_reutenauer"], - "email" => ["cesure-l{at}gutenberg{dot}eu{dot}org"], - "licence" => "MIT", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for French in T1/EC and UTF-8 encodings.", - ] + "type" => "rules", }, -# galician { "code" => "gl", - "name" => "galician", - "filename_old_patterns" => "glhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Galician hyphenation patterns", - - "version" => "2.4", - "last_modified" => "2010-04-23", "type" => "rules", - "authors" => ["javier_mugica"], - "licence" => "LPPL", # Status maintained - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Galician in T1/EC and UTF-8 encodings.", - # "Generated automatically from the mkpattern utility.", - ] }, -# estonian { "code" => "et", - "name" => "estonian", - "filename_old_patterns" => "ethyph.tex", - "hyphenmin" => [2,3], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Estonian hyphenation patterns", - - "version" => nil, - "last_modified" => "2004-04-13", - "type" => "dictionary", - "authors" => [ "een_saar" ], - "licence" => "LPPL|MIT", # Dual MIT-LPPL - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Estonian in T1/EC and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# finnish { "code" => "fi", - "name" => "finnish", - "filename_old_patterns" => "fihyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Finnish hyphenation patterns", - - "version" => "2.2", - "last_modified" => "1989-03-08", - "type" => "rules", - "authors" => ["kauko_saarinen", "fred_karlsson"], - "licence" => "other-free", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Finnish in T1/EC and UTF-8 encodings.", - ], + "type" => "rules", }, -# croatian { "code" => "hr", - "name" => "croatian", - "filename_old_patterns" => "hrhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Croatian hyphenation patterns", - - "version" => nil, - "last_modified" => "1996-03-19", - "type" => "dictionary", - "authors" => [ "igor_marinovic" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Croatian in T1/EC and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# hungarian { "code" => "hu", - "name" => "hungarian", - "filename_old_patterns" => "huhyphn.tex", - "hyphenmin" => [2,2], # polyglossia - "encoding" => "ec", - "exceptions" => false, - "message" => "Hungarian hyphenation patterns", - - "source" => "https://github.com/nagybence/huhyphn/", # FIXME Check and remove from desc below - "version" => "v20110815", - "last_modified" => "2011-08-15", # actually, it is "2009-06-12" or older for contents; we probably want to check/fix this - "type" => "dictionary", - "authors" => ["bence_nagy"], - "licence" => "MPL 1.1/GPL 2.0/LGPL 2.1", # TODO - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Hungarian in T1/EC and UTF-8 encodings.", - # TODO: same comment as for Irish: I'm slightly reluctant to put URL addresses here. - "From https://github.com/nagybence/huhyphn/." - ], + "type" => "dictionary", + "source" => "https://github.com/nagybence/huhyphn/", # FIXME Check and remove from desc below }, -# armenian -# Sahak Petrosyan { "code" => "hy", - "name" => "armenian", - "filename_old_patterns" => nil, - "hyphenmin" => [1,2], # taken from Hyphenator.js; check the value - "encoding" => nil, - "exceptions" => false, - "message" => "Armenian hyphenation patterns", - - "version" => nil, - "last_modified" => "2010-05", - "type" => "rules", - "authors" => ["sahak_petrosyan"], - "licence" => "LGPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Armenian for Unicode engines.", - # "Auto-generated from a script included in hyph-utf8.", - ], + "type" => "rules", }, -# interlingua { "code" => "ia", - "name" => "interlingua", - "filename_old_patterns" => "iahyphen.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ascii", - "exceptions" => true, - "message" => "Hyphenation patterns for Interlingua", - - "version" => "0.2c", - "last_modified" => "2005-06-28", - "type" => "dictionary", - "authors" => ["peter_kleiweg"], - "licence" => "LPPL", # TODO Status maintained - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Interlingua in ASCII encoding.", - ], + "type" => "dictionary", }, -# indonesian { "code" => "id", - "name" => "indonesian", - "filename_old_patterns" => "inhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ascii", - "exceptions" => true, - "message" => "Indonesian hyphenation patterns", - - "version" => "1.3", - "last_modified" => "1997-09-19", - "type" => "rules", - "authors" => ["joerg_knappen", "terry_mart"], - "licence" => "GPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Indonesian (Bahasa Indonesia) in ASCII", - "encoding. They are probably also usable for Malay (Bahasa Melayu).", - ], + "type" => "rules", }, -# icelandic { "code" => "is", - "name" => "icelandic", - "filename_old_patterns" => "icehyph.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Icelandic hyphenation patterns", - - "version" => nil, - "last_modified" => "2004-03-02", - "type" => "dictionary", + "type" => "dictionary", # TODO: I'm not sure that the last two names are relevant, I don't find the source of Marteinn Sverrisson "authors" => ["jorgen_pind", "marteinn_sverrisson", "kristinn_gylfason"], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Icelandic in T1/EC and UTF-8 encodings.", - ], }, -# irish { "code" => "ga", - "name" => "irish", - "filename_old_patterns" => "gahyph.tex", - "hyphenmin" => [2,3], # babel - "encoding" => "ec", - "exceptions" => true, - "message" => "Irish hyphenation patterns", - - "version" => "1.0", - "last_modified" => "2004-01-22", - "type" => "dictionary", - "authors" => ["kevin_p_scannell"], - "licence" => "GPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Irish (Gaeilge) in T1/EC and UTF-8 encodings.", - "Visit http://borel.slu.edu/fleiscin/index.html for more information.", - # TODO: I'm slightly reluctant to put URL here - ], + "type" => "dictionary", }, -# italian { "code" => "it", - "name" => "italian", - "filename_old_patterns" => "ithyph.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ascii", - "exceptions" => false, - "message" => "Italian hyphenation patterns", - - "version" => "4.9", - "last_modified" => "2014-04-22", "type" => "rules", # TODO: we might want to check that, but it seems unlikely that patgen was used - "authors" => ["claudio_beccari"], - "licence" => "LPPL", # Status: maintained! - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Italian in ASCII encoding.", - # supposed to be ... - "Compliant with the Recommendation UNI 6461 on hyphenation", - "issued by the Italian Standards Institution", - "(Ente Nazionale di Unificazione UNI).", - # "Implements Recommendation UNI 6461 issued by the Italian Standards Institution", - # "(Ente Nazionale di Unificazione UNI).", - ], }, -# romansh { "code" => "rm", - "name" => "romansh", - "filename_old_patterns" => nil, - "hyphenmin" => [2,2], # todo - "encoding" => "ascii", - "exceptions" => false, - "message" => "Romansh hyphenation patterns", - - "version" => "1.1", - "last_modified" => "2012-04-10", - "type" => "rules", - "authors" => ["claudio_beccari"], - "licence" => "LPPL", # Status: maintained! - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Romansh in ASCII encoding.", - "They are supposed to comply with the rules indicated by the Lia", - "Rumantscha (Romansh language society).", - ], + "type" => "rules", }, -# friulan { "code" => "fur", - "name" => "friulan", "synonyms" => [], - "filename_old_patterns" => nil, - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Friulan hyphenation patterns", - - "version" => "1.1", - "last_modified" => "2012-04-10", - "type" => "rules", - "authors" => ["claudio_beccari"], - "licence" => "LPPL", # Status: maintained! - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Friulan in ASCII encoding.", - "They are supposed to comply with the common spelling of the", - "Friulan (Furlan) language as fixed by the Regional Law N.15/96", - "dated November 6, 1996 and its following amendments.", - ], + "type" => "rules", }, -# piedmontese { "code" => "pms", - "name" => "piedmontese", - "filename_old_patterns" => nil, - "hyphenmin" => [2,2], - "encoding" => "ascii", - "exceptions" => false, - "message" => "Piedmontese hyphenation patterns", - - "version" => "1.0", - "last_modified" => "2013-02-14", - "type" => "rules", - "authors" => ["claudio_beccari"], - "licence" => "LPPL", # Status: maintained! - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Piedmontese in ASCII encoding.", - "Compliant with 'Gramatica dla lengua piemonteisa' by Camillo Brero.", - ], + "type" => "rules", }, -# kurmanji { "code" => "kmr", - "name" => "kurmanji", - "filename_old_patterns" => "kmrhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Kurmanji hyphenation patterns", - - "version" => "1.0", - "last_modified" => "2009-06-29", - "type" => "dictionary", - "authors" => ["joerg_knappen", "medeni_shemde"], - "licence" => "LPPL", # Status: maintained! - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Kurmanji (Northern Kurdish) as spoken in", - "Turkey and by the Kurdish diaspora in Europe, in T1/EC and UTF-8", - "encodings." - ], + "type" => "dictionary", }, -# latin { "code" => "la", - "name" => "latin", - "filename_old_patterns" => "lahyph.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Latin hyphenation patterns", - - "version" => "3.2a", - "last_modified" => "2014-06-04", # patterns (behaviour) last modified on 2010-06-01 - "type" => "rules", - "authors" => [ "claudio_beccari" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Latin in T1/EC and UTF-8 encodings,", - "mainly in modern spelling (u when u is needed and v when v is needed),", - "medieval spelling with the ligatures \\ae and \\oe and the (uncial)", - "lowercase 'v' written as a 'u' is also supported. Apparently", - "there is no conflict between the patterns of modern Latin and", - "those of medieval Latin.", - ], + "type" => "rules", }, -# classiclatin { "code" => "la-x-classic", - "name" => "classiclatin", - "filename_old_patterns" => nil, - "hyphenmin" => [2,2], - "encoding" => "ascii", - "exceptions" => false, - "message" => "Classical Latin hyphenation patterns", - - "version" => "1.2", - "last_modified" => "2014-10-06", - "type" => "rules", - "authors" => [ "claudio_beccari" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for the Classical Latin in T1/EC and UTF-8", - "encodings. Classical Latin hyphenation patterns are different from", - "those of 'plain' Latin, the latter being more adapted to modern Latin.", - ], + "type" => "rules", }, -# liturgicallatin { "code" => "la-x-liturgic", - "name" => "liturgicallatin", - "filename_old_patterns" => nil, - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Liturgical Latin hyphenation patterns", - "version" => "1.040", "last_modified" => "2016-05-19", "type" => "rules", - "authors" => [ "claudio_beccari" ], - "licence" => "MIT", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for the Liturgical Latin in T1/EC and UTF-8", - "encodings.", - ], }, -# lithuanian { "code" => "lt", - "name" => "lithuanian", - "hyphenmin" => [2,2], - "encoding" => "l7x", - "exceptions" => false, - "message" => "Lithuanian hyphenation patterns", - - "version" => nil, - "last_modified" => "2002-11-20", # 1992-03-04, for the content? - "authors" => ["vytas_statulevicius", "yannis_haralambous", "sigitas_tolusis"], - # "licence" => "LPPL", "description_l" => [ #......................................................................# "Hyphenation patterns for Lithuanian in L7X and UTF-8 encodings.", @@ -991,626 +193,96 @@ class Language # "would require to review and maybe rebuild the patterns." ], }, -# latvian { "code" => "lv", - "name" => "latvian", - "hyphenmin" => [2,2], - "encoding" => "l7x", - "exceptions" => false, - "message" => "Latvian hyphenation patterns", - - "version" => "0.3", - "last_modified" => "2005-09-14", - "type" => "dictionary", - "authors" => ["janis_vilims"], - "licence" => "LGPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Latvian in L7X and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# dutch { "code" => "nl", - "name" => "dutch", - "filename_old_patterns" => "nehyph96.tex", - # quoting Hans Hagen: - # patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => true, - "message" => "Dutch hyphenation patterns", - - "version" => "1.1", - "last_modified" => "1996-11", - "type" => "dictionary", - "authors" => ["piet_tutelaers"], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Dutch in T1/EC and UTF-8 encodings.", - # "\\lefthyphenmin and \\righthyphenmin must both be > 1.", - "These patterns don't handle cases like 'menuutje' > 'menu-tje',", - "and don't hyphenate words that have different hyphenations according", - "to their meaning." - ], + "type" => "dictionary", + # quoting Hans Hagen: + # patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) }, -# occitan { "code" => "oc", - "name" => "occitan", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Occitan hyphenation patterns", - - "version" => "0.1", - "last_modified" => "2016-02-04", - "type" => "rules", - "authors" => [ "claudio_beccari" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Occitan in T1/EC and UTF-8 encodings.", - "They are supposed to be valid for all the Occitan variants", - "spoken and written in the wide area called 'Occitanie' by the French.", - "It ranges from the Val d'Aran within Catalunya, to the South Western", - "Italian Alps encompassing the southern half of the French pentagon.", - ], + "type" => "rules", }, -# polish { "code" => "pl", - "name" => "polish", - "filename_old_patterns" => "plhyph.tex", - #{}"hyphenmin" => [1,1], - "hyphenmin" => [2,2], - "encoding" => "qx", - "exceptions" => true, - "message" => "Polish hyphenation patterns", - - "version" => "3.0a", - "last_modified" => "1995-06-17", - "type" => "dictionary", - "authors" => ["hanna_kolodziejska", "boguslaw_jackowski", "marek_rycko"], - "licence" => "public", # TODO Knuthian type - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Polish in QX and UTF-8 encodings.", - "These patterns are also used by Polish TeX formats MeX and LaMeX.", - ], + "type" => "dictionary", + #{}"hyphenmin" => [1,1], }, -# portuguese -# =portuges { "code" => "pt", - "name" => "portuguese", "synonyms" => ["portuges"], - "filename_old_patterns" => "pthyph.tex", - "hyphenmin" => [2,3], # babel - "encoding" => "ec", - "exceptions" => true, - "message" => "Portuguese hyphenation patterns", - - "version" => "1.0", - "last_modified" => "1996-07-21", - "type" => "rules", # TODO: we could create a generating script - "authors" => ["pedro_j_de_rezende", "j_joao_dias_almeida"], - "licence" => "BSD-3", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Portuguese in T1/EC and UTF-8 encodings.", - ], + "type" => "rules", # TODO: we could create a generating script }, -# pinyin { "code" => "zh-latn-pinyin", - "name" => "pinyin", - "filename_old_patterns" => "pyhyph.tex", - "hyphenmin" => [1,1], - "encoding" => "ec", - "exceptions" => false, - "message" => "Hyphenation patterns for unaccented pinyin syllables", - - "version" => "4.8.0", - "last_modified" => "2008-05-22", - "type" => "rules", # TODO: we could create a generating script - "authors" => ["werner_lemberg"], - "licence" => "GPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for unaccented transliterated Mandarin Chinese", - "(pinyin) in T1/EC and UTF-8 encodings.", - ], + "type" => "rules", # TODO: we could create a generating script }, -# romanian { "code" => "ro", - "name" => "romanian", - "filename_old_patterns" => "rohyphen.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Romanian hyphenation patterns", # : `rohyphen' 1.1 <29.10.1996> - - "version" => "1.1R", - "last_modified" => "1996-11-07", - "type" => "dictionary", - "authors" => ["adrian_rezus"], - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Romanian in T1/EC and UTF-8 encodings.", - "The UTF-8 patterns use U+0219 for the character 's with comma accent'", - "and U+021B for 't with comma accent', but we may consider using U+015F", - "and U+0163 as well in the future.", - # "Generated by PatGen2-output hyphen-level 9.", - ], + "type" => "dictionary", }, -# slovenian -# =slovene { "code" => "sl", - "name" => "slovenian", "synonyms" => ["slovene"], - "filename_old_patterns" => "sihyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Slovenian hyphenation patterns", - - "version" => "2.3", - "last_modified" => "1997-15-04", "type" => "dictionary", - "authors" => [ "matjaz_vrecko" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Slovenian in T1/EC and UTF-8 encodings.", - ], }, -# uppersorbian { "code" => "hsb", - "name" => "upper sorbian", - "filename_old_patterns" => "sorhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => true, - "message" => "Upper Sorbian hyphenation patterns", -# \message{Hyphenation patterns for Upper Sorbian, E. Werner} -# \message{Completely new revision 1997, March 22} - - "version" => nil, - "last_modified" => "1997-03-22", - "authors" => ["eduard_werner"], - "type" => "dictionary", - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Upper Sorbian in T1/EC and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# swedish { "code" => "sv", - "name" => "swedish", - "filename_old_patterns" => "svhyph.tex", - "hyphenmin" => [2,2], # patters say it could be 1,2; babel says 2,2 - double check - "encoding" => "ec", - "exceptions" => false, - "message" => "Swedish hyphenation patterns", - - # "version" => "1994-03-03", # that is what author used in message - "version" => nil, - "last_modified" => "1994-03-03", "type" => "dictionary", - "authors" => ["jan_michael_rynning"], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Swedish in T1/EC and UTF-8 encodings.", - ], }, -# turkmen { "code" => "tk", - "name" => "turkmen", - "filename_old_patterns" => nil, - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => false, - "message" => "Turkmen hyphenation patterns", - - "version" => "0.1", - "last_modified" => "2010-03-16", - "type" => "dictionary", - "authors" => [ "nazar_annagurban" ], - "licence" => "public", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Turkmen in T1/EC and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# turkish { "code" => "tr", - "name" => "turkish", - "filename_old_patterns" => "tkhyph.tex", - "hyphenmin" => [2,2], # polyglossia - "encoding" => "ec", - "exceptions" => false, - "message" => "Turkish hyphenation patterns", - - "version" => nil, - "last_modified" => "2008-06-28", - "type" => "rules", - "authors" => ["pierre_mackay", "h_turgut_uyar", "s_ekin_kocabas", "mojca_miklavec"], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Turkish in T1/EC and UTF-8 encodings.", - # "Auto-generated from a script included in the distribution.", - "The patterns for Turkish were first produced for the Ottoman Texts", - "Project in 1987 and were suitable for both Modern Turkish and Ottoman", - "Turkish in Latin script, however the required character set didn't fit", - "into EC encoding, so support for Ottoman Turkish had to be dropped to", - "keep compatibility with 8-bit engines.", - ] + "type" => "rules", }, - - - - -# ukenglish { "code" => "en-gb", - "name" => "ukenglish", "synonyms" => ["british", "UKenglish"], - "filename_old_patterns" => "ukhyphen.tex", - "hyphenmin" => [2,3], # confirmed, same as what Knuth says - "encoding" => "ascii", - "exceptions" => true, - "message" => "Hyphenation patterns for British English", # FIXME Inconsistent - - "version" => "1.0a", # FIXME a much older comment says 2.0? - "last_modified" => "2005-10-18", - "type" => "dictionary", - "authors" => ["dominik_wujastyk", "graham_toal"], - "licence" => "other-free", # TODO Knuth-like - # (This will be uncommented during reimplementation) - # "description_s" => "British English hyphenation patterns", - # "description_l" => [ - # "Hyphenation patterns for British English in ASCII encoding.", - # ], - - "description_l" => [ - #......................................................................# - "Additional hyphenation patterns for American and British", - "English in ASCII encoding. The American English patterns", - "(usenglishmax) greatly extend the standard patterns from Knuth", - "to find many additional hyphenation points. British English", - "hyphenation is completely different from US English, so has its", - "own set of patterns.", - ], + "type" => "dictionary", }, -# (US english) -# usenglishmax { "code" => "en-us", - "name" => "usenglishmax", - "filename_old_patterns" => "ushyphmax.tex", - "hyphenmin" => [2,3], # confirmed, same as what Knuth says - "encoding" => "ascii", - "exceptions" => true, - "message" => "Hyphenation patterns for American English", # FIXME Consistent - - "version" => nil, - "last_modified" => "1990-03-01", # 8-bit file also has version 2005-05-30. - "type" => "dictionary", - "authors" => ["donald_e_knuth", "gerard_d_c_kuiken"], - "licence" => "other-free", - # (This will be uncommented during reimplementation) - # "description_s" => "American English hyphenation patterns", - # "description_l" => [ - # "Hyphenation patterns for American English in ASCII encoding.", - # ], }, -# US english -# { -# "code" => "en-us-x-knuth", -# "name" => "english", -# "use_new_loader" => false, -# "use_old_patterns" => false, -# "filename_old_patterns" => "hyphen.tex", -# "hyphenmin" => [2,3], # confirmed, same as what Knuth says -# "encoding" => "ascii", -# "exceptions" => true, -# "message" => "Hyphenation patterns for American English", -# }, # TODO: FIXME!!! -# serbian { "code" => "sh-latn", - "name" => "serbian", - "filename_old_patterns" => "shhyphl.tex", - # It is allowed to leave one character at the end of the row. - # However, if you think that it is graphicaly not very - # pleasant these patterns will work well with \lefthyphenmin=2. - # \lefthyphenmin=1 \righthyphenmin=2 - "hyphenmin" => [2,2], - "encoding" => "ec", - "exceptions" => true, - "message" => "Serbian hyphenation patterns in Latin script", - - # only for serbian - "version" => "2.02", - "last_modified" => "2008-06-22", - "type" => "dictionary", - "authors" => [ "dejan_muhamedagic" ], - "licence" => "LPPL", - # for both scripts - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Serbian in T1/EC, T2A and UTF-8 encodings.", - "For 8-bit engines the patterns are available separately as 'serbian'", - "in T1/EC encoding for Latin script and 'serbianc' in T2A encoding for", - "Cyrillic script. Unicode engines should only use 'serbian'", - "which has patterns in both scripts combined.", - ], + "type" => "dictionary", }, -# serbianc { "code" => "sh-cyrl", - "name" => "serbianc", - # "filename_old_patterns" => "srhyphc.tex", - "hyphenmin" => [2,2], - "encoding" => "t2a", - "exceptions" => true, - "message" => "Serbian hyphenation patterns in Cyrillic script", }, -# mongolian (used to be mongolian2a) { "code" => "mn-cyrl", - "name" => "mongolian", - "filename_old_patterns" => "mnhyphn.tex", - "hyphenmin" => [2,2], - "encoding" => "t2a", - "exceptions" => false, - "message" => "(New) Mongolian hyphenation patterns", - - # only for this one - "version" => "1.2", - "last_modified" => "2010-04-03", - "type" => "dictionary", - "authors" => [ "dorjgotov_batmunkh" ], - "licence" => "LPPL", - # for both - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Mongolian in T2A, LMC and UTF-8 encodings.", - "LMC encoding is used in MonTeX. The package includes two sets of", - "patterns that will hopefully be merged in future.", - ], -}, -# mongolianlmc xu-mnhyph.tex (used to be mongolian) -{ - "code" => "mn-cyrl-x-lmc", - "name" => "mongolianlmc", - "filename_old_patterns" => "mnhyph.tex", - "hyphenmin" => [2,2], - "encoding" => "lmc", - "exceptions" => false, - "message" => "Mongolian hyphenation patterns", -}, -# bulgarian -{ - "code" => "bg", - "name" => "bulgarian", - "filename_old_patterns" => "bghyphen.tex", - "hyphenmin" => [2,2], # babel - "encoding" => "t2a", - "exceptions" => false, - "message" => "Bulgarian hyphenation patterns", - - "version" => "1.7", - "last_modified" => "2008-06", - "type" => "pattern", - "authors" => [ "georgi_boshnakov" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Bulgarian in T2A and UTF-8 encodings.", - ], + "type" => "dictionary", + "last_modified" => "2010-04-03", # FIXME: ? }, -# sanskrit { "code" => "sa", - "name" => "sanskrit", - "hyphenmin" => [1,3], # polyglossia - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Sanskrit hyphenation patterns", - - "version" => "0.6", - "last_modified" => "2011-09-14", - "type" => "rules", - "authors" => ["yves_codet"], - "licence" => "free", # You may freely use, copy, modify and/or distribute this file. - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Sanskrit and Prakrit in transliteration,", - "and in Devanagari, Bengali, Kannada, Malayalam and Telugu scripts", - "for Unicode engines.", - ], -}, -# norwegian -{ - "code" => "no", - "name" => "norwegian", # TODO: fixme - "use_old_loader" => true, - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => false, - "message" => "Norwegian hyphenation patterns", + "type" => "rules", }, -# norsk { "code" => "nb", - "name" => "bokmal", "synonyms" => ["norwegian", "norsk"], - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => true, - "message" => "Norwegian Bokmal hyphenation patterns", - - "version" => nil, - "last_modified" => "2012-05-18", - "type" => "dictionary", - "authors" => [ "rune_kleveland", "ole_michael_selberg" ], - "licence" => "free", # TODO - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Norwegian Bokmal and Nynorsk in T1/EC and", - "UTF-8 encodings.", - ], -}, -# nynorsk -{ - "code" => "nn", - "name" => "nynorsk", - "hyphenmin" => [2,2], # babel - "encoding" => "ec", - "exceptions" => true, - "message" => "Norwegian Nynorsk hyphenation patterns", + "type" => "dictionary", + "last_modified" => "2012-05-18", # FIXME: ? }, -##### -# assamese { "code" => "as", - "name" => "assamese", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Assamese hyphenation patterns", - # this is true for all Indic patterns - "version" => "0.9.0", - "last_modified" => "2016-01-16", "type" => "rules", - "authors" => ["santhosh_thottingal"], - "licence" => "MIT", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Assamese, Bengali, Gujarati, Hindi, Kannada,", - "Malayalam, Marathi, Oriya, Panjabi, Tamil and Telugu for Unicode", - "engines.", - ], -}, -# bengali -{ - "code" => "bn", - "name" => "bengali", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Bengali hyphenation patterns", -}, -# gujarati -{ - "code" => "gu", - "name" => "gujarati", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Gujarati hyphenation patterns", -}, -# hindi -{ - "code" => "hi", - "name" => "hindi", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Hindi hyphenation patterns", -}, -# kannada -{ - "code" => "kn", - "name" => "kannada", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Kannada hyphenation patterns", -}, -# malayalam -{ - "code" => "ml", - "name" => "malayalam", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Malayalam hyphenation patterns", -}, -# marathi -{ - "code" => "mr", - "name" => "marathi", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Marathi hyphenation patterns", -}, -# oriya -{ - "code" => "or", - "name" => "oriya", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Oriya hyphenation patterns", -}, -# panjabi -{ - "code" => "pa", - "name" => "panjabi", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Panjabi hyphenation patterns", -}, -# tamil -{ - "code" => "ta", - "name" => "tamil", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Tamil hyphenation patterns", -}, -# telugu -{ - "code" => "te", - "name" => "telugu", - "hyphenmin" => [1,1], # TODO - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Telugu hyphenation patterns", }, # thai { "code" => "th", - "name" => "thai", - "hyphenmin" => [2,3], - "encoding" => "lth", - "exceptions" => false, - "message" => "Thai hyphenation patterns", - - "version" => nil, - "last_modified" => "2015-05-07", - "type" => "dictionary", - "authors" => [ "theppitak_karoonboonyanan" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Thai in LTH and UTF-8 encodings.", - ], + "type" => "dictionary", + "last_modified" => "2015-05-07", # FIXME: ? }, # lao #{ @@ -1635,100 +307,21 @@ class Language # "Current version is experimental and gives bad results.", # ], #}, -# pan-Ethiopic -{ - "code" => "mul-ethi", - "name" => "ethiopic", "synonyms" => ["amharic", "geez"], - "version" => nil, - "last_modified" => "2011-01-10", - "authors" => ["arthur_reutenauer", "mojca_miklavec"], - "licence" => "public-ask", # TODO - "hyphenmin" => [1,1], - "encoding" => nil, # no patterns for 8-bit engines - "exceptions" => false, - "message" => "Pan-Ethiopic hyphenation patterns", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for languages written using the Ethiopic script", - "for Unicode engines. They are not supposed to be linguistically", - "relevant in all cases and should, for proper typography, be replaced", - "by files tailored to individual languages.", - ], -}, -# georgian { "code" => "ka", - "name" => "georgian", - "hyphenmin" => [1,2], - "encoding" => "t8m", - "exceptions" => false, - "message" => "Georgian hyphenation patterns", - - "version" => "0.3", - "last_modified" => "2013-04-15", - "type" => "dictionary", - "authors" => [ "levan_shoshiashvili" ], - "licence" => "LPPL", - "description_l" => [ - #......................................................................# - "Hyphenation patterns for Georgian in T8M, T8K and UTF-8 encodings.", - ], + "type" => "dictionary", }, -# Church Slavonic { "code" => "cu", - "name" => "church slavonic", - "hyphenmin" => [1, 2], - "encoding" => nil, - "exceptions" => true, - "message" => "Church Slavonic hyphenation patterns", - - "version" => nil, - "last_modified" => "2016-04-16", - "type" => "machine learning", - "authors" => ["mike_kroutikov", "aleksandr_andreev"], - "licence" => "MIT", - "description_l" => [ - "Hyphenation patterns for Church Slavonic in UTF-8 encoding", - ], + "type" => "machine learning", }, -# Belarusian { "code" => "be", - "name" => "belarusian", - "hyphenmin" => [2,2], - "encoding" => "t2a", - "message" => "Belarusian hyphenation patterns", - - "version" => nil, - "last_modified" => "2016-09-29", - "type" => "rules", - "authors" => ["maksim_salau"], - "licence" => "MIT", - "description_l" => [ - "Belarusian hyphenation patterns in T2A and UTF-8 encodings" - ], + "type" => "rules", }, -# Pali { "code" => "pi", - "name" => "pali", - "hyphenmin" => [1, 2], - # no encoding - "message" => "Pali hyphenation patterns", - # no version number - "last_modified" => "2018-06-29", "type" => "rules", - "authors" => ["wie_ming_ang"], - "licence" => "MIT", - "description_l" => [ - "Pali hyphenation patterns in UTF-8 encoding", - ], }, -# dumylang -> dumyhyph.tex -# nohyphenation -> zerohyph.tex -# arabic -> zerohyph.tex -# farsi zerohyph.tex -# =persian ] end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb index 46ae947460f..60833398335 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb @@ -1,74 +1,8 @@ # -*- coding: utf-8 -*- -require_relative 'author-data' require_relative 'language-data' +require_relative '../../../../lib/tex/hyphen/language' -module PATH - ROOT = File.expand_path('../../../../..', __FILE__) - - TeXROOT = File.join(ROOT, 'hyph-utf8') - TeX_GENERIC = File.join(TeXROOT, 'tex', 'generic') - TXT = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns', 'txt') - TEX = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns', 'tex') - PTEX = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns', 'ptex') - QUOTE = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns', 'quote') - - HYPHU8 = File.join('tex', 'generic', 'hyph-utf8') - - TL = File.join(ROOT, 'TL') - LANGUAGE_DAT = File.join(PATH::TL, 'texmf-dist', 'tex', 'generic', 'config') - # hyphen-foo.tlpsrc for TeX Live - TLPSRC = File.join(PATH::TL, 'tlpkg', 'tlpsrc') -end - -class String - def superstrip - strip.gsub /%.*$/, '' - end - - def supersplit - strip.gsub(/\s+/m,"\n").split("\n") - end - - def safe - gsub /[\s-]/, '' - end - - def titlecase - split.map(&:capitalize).join(' ') - end -end - -class Author - def initialize(name,surname,email,contacted1,contacted2) - @name = name - @surname = surname - @email = email - # this mostly means if email has been recently checked - @contacted1 = contacted1 - # this means if we made more cooperation with author, - # exchanging patches etc. - @contacted2 = contacted2 - end - - attr_reader :name, :surname, :email - - def self.authors - @@authors ||= @@author_data.map do |id, details| - author = Author.new(details[0], details[1], details[2], details[3], details[4]) - [id, author] - end.to_h - end - - def self.all - authors.values - end - - def self.[] a - authors[a] - end -end - -class Language +class OldLanguage def initialize(language_hash) @use_old_patterns = language_hash["use_old_patterns"] @use_old_patterns_comment = language_hash["use_old_patterns_comment"] @@ -97,46 +31,13 @@ class Language def self.all @@languages ||= @@language_data.map do |language_data| + next if language_data['code'] == 'sr-cyrl' new language_data - end + end.compact end # TODO self.find - @@texfile = Hash.new - def readtexfile(code = @code) - @@texfile[code] ||= File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', code))) - end - - def exceptions - @exceptions ||= if readtexfile.superstrip.index('\hyphenation') - readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit - else - "" - end - end - - def patterns - @patterns ||= if @code == 'eo' then - readtexfile.superstrip. - gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1'). - # - gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.'). - gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.'). - gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.'). - # - supersplit - else - readtexfile(if ['nb', 'nn'].include? @code then 'no' else @code end).superstrip. - gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1'). - supersplit - end - end - - def get_comments_and_licence - @comments_and_licence ||= readtexfile.gsub(/(.*)\\patterns.*/m,'\1') - end - # def lc_characters # if @lc_characters == nil # lc_characters = Hash.new @@ -147,357 +48,13 @@ class Language # return @lc_characters # end - attr_reader :use_old_loader, :use_old_patterns, :use_old_patterns_comment, :filename_old_patterns - attr_reader :code, :name, :synonyms, :hyphenmin, :encoding, :message - attr_reader :description_l, :version - attr_reader :licence, :authors - - # def message - # @name.titlecase + ' hyphenation patterns' - # end - - def description_s - @message - end - - # Strictly speaking a misnomer, because grc-x-ibycus should also return true. - # But useful for a number of apostrophe-related routines - def isgreek? - ['grc', 'el-polyton', 'el-monoton'].include? @code - end - - def has_apostrophes - begin - !isgreek? && patterns.any? { |p| p =~ /'/ } - rescue Errno::ENOENT - false - end - end - - def has_dashes - begin - patterns.any? { |p| p =~ /-/ } - rescue Errno::ENOENT - false - end - end + attr_reader :use_old_loader, :use_old_patterns_comment, :filename_old_patterns + attr_reader :code, :name, :message + attr_reader :description_l # Convenience methods related to TeX Live and the .tlpsrc files module TeXLive - def loadhyph - return filename_old_patterns if use_old_loader - - code = @code - code = @code.gsub 'sh-', 'sr-' if @code =~ /^sh-/ - sprintf 'loadhyph-%s.tex', code - end - - # ext: 'pat' or 'hyp' - # filetype: 'patterns' or 'exceptions' - def plain_text_line(ext, filetype) - return "" if ['ar', 'fa', 'grc-x-ibycus', 'mn-cyrl-x-lmc'].include? @code - - if @code =~ /^sh-/ - # TODO Warning AR 2018-09-12 - filename = sprintf 'hyph-sh-latn.%s.txt,hyph-sh-cyrl.%s.txt', ext, ext - else - filename = sprintf 'hyph-%s.%s.txt', @code, ext - filepath = File.join(PATH::TXT, filename) - # check for existence of file and that it’s not empty - unless File.file?(filepath) && File.read(filepath).length > 0 - # if the file we were looking for was a pattern file, something’s wrong - if ext == 'pat' - raise sprintf("There is some problem with plain patterns for language [%s]!!!", @code) - else # the file is simply an exception file and we’re happy - filename = '' # And we return and empty file name after all - end - end - end - - sprintf "file_%s=%s", filetype, filename - end - - def exceptions_line - plain_text_line('hyp', 'exceptions') - end - - def patterns_line - plain_text_line('pat', 'patterns') - end - - def extract_apostrophes - plain, with_apostrophe = Array.new, nil - - patterns.each do |pattern| - plain << pattern - if pattern =~ /'/ && !isgreek? - pattern_with_apostrophe = pattern.gsub(/'/,"’") - plain << pattern_with_apostrophe - (with_apostrophe ||= []) << pattern_with_apostrophe - end - end - - { plain: plain, with_apostrophe: with_apostrophe } - end - - def extract_characters - characters = Array.new - - characters_indexes = patterns.join.gsub(/[.0-9]/,'').unpack('U*').sort.uniq - characters_indexes.each do |c| - ch = [c].pack('U') - characters << ch + Unicode.upcase(ch) - characters << "’’" if ch == "'" && !isgreek? - end - - characters - end - end - - def list_synonyms - # synonyms - if synonyms && synonyms.length > 0 - sprintf " synonyms=%s", synonyms.join(',') - else - '' - end - end - - def list_hyphenmins - # lefthyphenmin/righthyphenmin - lmin = (hyphenmin || [])[0] - rmin = (hyphenmin || [])[1] - sprintf "lefthyphenmin=%s \\\n\trighthyphenmin=%s", lmin, rmin - end - - def list_loader - # which loader to use - file = sprintf "file=%s", loadhyph - return file unless use_old_loader - - if ['ar', 'fa'].include? code - file = file + " \\\n\tfile_patterns=" - elsif code == 'grc-x-ibycus' - # TODO: fix this - file = file + " \\\n\tluaspecial=\"disabled:8-bit only\"" - end - end - - - def list_run_files - return [] if use_old_loader - - files = [] - - files << File.join(PATH::HYPHU8, 'loadhyph', loadhyph) - if has_apostrophes - files << File.join(PATH::HYPHU8, 'patterns', 'quote', sprintf("hyph-quote-%s.tex", code)) - end - - files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', code)) - if encoding && encoding != "ascii" then - files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', code, encoding)) - elsif code == "cop" - files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', filename_old_patterns) - end - - # we skip the mongolian language for luatex files - return files if code == "mn-cyrl-x-lmc" - - ['chr', 'pat', 'hyp', 'lic'].each do |t| - files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-%s.%s.txt', code, t)) - end - - if code =~ /^sh-/ - # duplicate entries (will be removed later) - files << File.join(PATH::HYPHU8, 'patterns', 'tex', 'hyph-sr-cyrl.tex') - ['chr', 'pat', 'hyp', 'lic'].each do |t| - # duplicate entries (will be removed later) - files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-sr-cyrl.%s.txt', t)) - end - end - - files end -end - -module TeXLive - class Package - attr_reader :name - - def initialize(name) - @name = name - end - - @@package_mappings = { - "en-gb"=>"english", - "en-us"=>"english", - "nb"=>"norwegian", - "nn"=>"norwegian", - "de-1901"=>"german", - "de-1996"=>"german", - "de-ch-1901"=>"german", - "mn-cyrl"=>"mongolian", - "mn-cyrl-x-lmc"=>"mongolian", - "el-monoton"=>"greek", - "el-polyton"=>"greek", - "grc"=>"ancient greek", - "grc-x-ibycus"=>"ancient greek", - "zh-latn-pinyin"=>"chinese", - "as"=>"indic", - "bn"=>"indic", - "gu"=>"indic", - "hi"=>"indic", - "kn"=>"indic", - "ml"=>"indic", - "mr"=>"indic", - "or"=>"indic", - "pa"=>"indic", - "ta"=>"indic", - "te"=>"indic", - "sh-latn"=>"serbian", - "sh-cyrl"=>"serbian", - "la"=>"latin", - "la-x-classic"=>"latin", - "la-x-liturgic"=>"latin" - } - - def self.make_mappings - @@package_names = @@package_mappings.values.uniq.map do |package_name| - [package_name, new(package_name)] - end.to_h - - # a hash with the names of TeX Live packages, either individual language names, - # or an array of languages as the value - @@packages = Hash.new - Language.all.each do |language| - package_name = @@package_mappings[language.code] - next if !package_name && @@package_names.include?(language.name) - package_name ||= language.name - package = @@package_names[package_name] || new(package_name) - (@@packages[package] ||= []) << language - end - - @@packages - end - - @@packages = make_mappings - def self.all - @@packages.keys - end - - # FIXME That’s oh-so-awful - def description_s - return 'Hyphenation patterns for Ethiopic scripts' if @name == 'ethiopic' - - if @name == 'arabic' - leader = '(No) Arabic' - elsif @name == 'farsi' - leader = '(No) Persian' - elsif @name == 'greek' - leader = 'Modern Greek' - elsif @name == 'chinese' - leader = 'Chinese pinyin' - elsif @name == 'norwegian' - leader = 'Norwegian Bokmal and Nynorsk' - else - leader = @name.titlecase - end - - shortdesc = sprintf '%s hyphenation patterns', leader - - shortdesc += ' in Cyrillic script' if @name == 'mongolian' - - shortdesc - end - - # FIXME This should be at package level from the start - def description_l - languages.inject([]) do |description, language| - description + if language.description_l then language.description_l else [] end - end - end - - def languages - @languages ||= @@packages[self] - end - - def has_dependency? - { - "german" => "dehyph", - # for Russian and Ukrainian (until we implement the new functionality at least) - "russian" => "ruhyphen", - "ukrainian" => "ukrhyph", - }[name] - end - - def list_dependencies - dependencies = [ - "depend hyphen-base", - "depend hyph-utf8", - ] - - # external dependencies - if dependency = has_dependency? - dependencies << sprintf("depend %s", dependency) - end - - dependencies - end - - @@special_support = { - 'doc' => { - 'greek' => 'doc/generic/elhyphen', - 'hungarian' => 'doc/generic/huhyphen', - } - } - - def list_support_files(type) - # Cache directory contents - (@dirlist ||= { })[type] ||= Dir.glob(File.expand_path(sprintf('../../../../%s/generic/hyph-utf8/languages/*', type), __FILE__)).select do |file| - File.directory?(file) - end.map do |dir| - dir.gsub /^.*\//, '' - end - - files = (languages.map(&:code) & @dirlist[type]).map do |code| - sprintf("%s/generic/hyph-utf8/languages/%s", type, code) - end - - if special = @@special_support.dig(type, name) - files << special - end - - files - end - - def list_run_files - files = [] - files << "tex/generic/hyph-utf8/patterns/tex/hyph-no.tex" if name == "norwegian" - - files = languages.inject(files) do |files, language| - files + language.list_run_files - end - - unless has_dependency? - languages.each do |language| - if language.use_old_patterns and language.filename_old_patterns != "zerohyph.tex" and language.code != 'cop' - files << sprintf("tex/generic/hyphen/%s", language.filename_old_patterns) - end - end - end - - files - end - - def <=>(other) - name <=> other.name - end - - # FIXME Change later - def find(name) - @@package_names[self] - end - end + include TeXLive end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb new file mode 100644 index 00000000000..2259f2ff1d4 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb @@ -0,0 +1 @@ +require_relative 'hyphen/language' diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml new file mode 100644 index 00000000000..f96530da39c --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml @@ -0,0 +1,452 @@ +--- +donald_knuth: !ruby/object:TeX::Hyphen::Author + name: Donald + surname: Knuth + email: + contacted1: false + contacted2: false +peter_heslin: !ruby/object:TeX::Hyphen::Author + name: Peter + surname: Heslin + email: + contacted1: false + contacted2: false +dimitrios_filippou: !ruby/object:TeX::Hyphen::Author + name: Dimitrios + surname: Filippou + email: dimitrios{dot}filippou{at}riotinto{dot}com + contacted1: true + contacted2: true +claudio_beccari: !ruby/object:TeX::Hyphen::Author + name: Claudio + surname: Beccari + email: claudio{dot}beccari{at}gmail{dot}com + contacted1: true + contacted2: true +juan_aguirregabiria: !ruby/object:TeX::Hyphen::Author + name: Juan M. + surname: Aguirregabiria + email: juanmari{dot}aguirregabiria{at}ehu.es + contacted1: true + contacted2: true +igor_marinovic: !ruby/object:TeX::Hyphen::Author + name: Igor + surname: Marinović + email: marinowski{at}gmail.com + contacted1: true + contacted2: true +tilla_fick: !ruby/object:TeX::Hyphen::Author + name: Tilla + surname: Fick + email: fick{dot}tilla{at}gmail{dot}com + contacted1: true + contacted2: true +chris_swanepoel: !ruby/object:TeX::Hyphen::Author + name: Chris + surname: Swanepoel + email: cj{at}swanepoel{dot}net + contacted1: true + contacted2: true +matjaz_vrecko: !ruby/object:TeX::Hyphen::Author + name: Matjaž + surname: Vrečko + email: matjaz{at}mg-soft{dot}si + contacted1: true + contacted2: true +goncal_badenes: !ruby/object:TeX::Hyphen::Author + name: Gonçal + surname: Badenes + email: g{dot}badenes{at}ieee.org + contacted1: false + contacted2: false +pavel_sevecek: !ruby/object:TeX::Hyphen::Author + name: Pavel + surname: Ševeček + email: pavel{at}lingea{dot}cz + contacted1: false + contacted2: false +# email doesn't work +jana_chlebikova: !ruby/object:TeX::Hyphen::Author + name: Jana + surname: Chlebíková + email: chlebikj{at}dcs{dot}fmph{dot}uniba{dot}sk + contacted1: false + contacted2: false +yannis_haralambous: !ruby/object:TeX::Hyphen::Author + name: Yannis + surname: Haralambous + email: yannis{dot}haralambous{at}telecom-bretagne{dot}eu + contacted1: true + contacted2: false +frank_jensen: !ruby/object:TeX::Hyphen::Author + name: Frank + surname: Jensen + email: frank{dot}jensen{at}hugin{dot}com + contacted1: true + contacted2: true +sergei_pokrovsky: !ruby/object:TeX::Hyphen::Author + name: Sergei + surname: Pokrovsky + email: sergio{dot}pokrovskij{at}gmail{dot}com + contacted1: true + contacted2: true +javier_bezos: !ruby/object:TeX::Hyphen::Author + name: Javier + surname: Bezos + email: jbezos{at}tex-tipografia{dot}com + contacted1: true + contacted2: true +een_saar: !ruby/object:TeX::Hyphen::Author + name: Enn + surname: Saar + email: saar{at}aai{dot}ee + contacted1: false + contacted2: false +dejan_muhamedagic: !ruby/object:TeX::Hyphen::Author + name: Dejan + surname: Muhamedagić + email: dejan{at}hello-penguin{dot}com + contacted1: true + contacted2: true +brian_wilson: !ruby/object:TeX::Hyphen::Author + name: Brian + surname: Wilson + email: bountonw{at}gmail{dot}com + contacted1: true + contacted2: true +arthur_reutenauer: !ruby/object:TeX::Hyphen::Author + name: Arthur + surname: Reutenauer + email: arthur{dot}reutenauer{at}normalesup{dot}org + contacted1: true + contacted2: true +mojca_miklavec: !ruby/object:TeX::Hyphen::Author + name: Mojca + surname: Miklavec + email: mojca{dot}miklavec{dot}lists{at}gmail{dot}com + contacted1: true + contacted2: true +santhosh_thottingal: !ruby/object:TeX::Hyphen::Author + name: Santhosh + surname: Thottingal + email: santhosh{dot}thottingal{at}gmail{dot}com> + contacted1: true + contacted2: true +# email doesn't work +yves_codet: !ruby/object:TeX::Hyphen::Author + name: Yves + surname: Codet + email: ycodet{at}club-internet{dot}fr + contacted1: true + contacted2: true +rune_kleveland: !ruby/object:TeX::Hyphen::Author + name: Rune + surname: Kleveland + email: + contacted1: false + contacted2: false +# email doesn't work +ole_michael_selberg: !ruby/object:TeX::Hyphen::Author + name: Ole Michael + surname: Selberg + email: o{dot}m{dot}selberg{at}c2i{dot}net + contacted1: true + contacted2: true +dorjgotov_batmunkh: !ruby/object:TeX::Hyphen::Author + name: Dorjgotov + surname: Batmunkh + email: bataak{at}gmail{dot}com + contacted1: true + contacted2: true +nazar_annagurban: !ruby/object:TeX::Hyphen::Author + name: Nazar + surname: Annagurban + email: nazartm{at}gmail{dot}com + contacted1: false + contacted2: false +jan_michael_rynning: !ruby/object:TeX::Hyphen::Author + name: Jan Michael + surname: Rynning + email: + contacted1: false + contacted2: false +eduard_werner: !ruby/object:TeX::Hyphen::Author + name: Eduard + surname: Werner + email: edi{dot}werner{at}gmx{dot}de + contacted1: false + contacted2: false +werner_lemberg: !ruby/object:TeX::Hyphen::Author + name: Werner + surname: Lemberg + email: wl{at}gnu{dot}org + contacted1: true + contacted2: true +# email doesn't work +pedro_j_de_rezende: !ruby/object:TeX::Hyphen::Author + name: Pedro J. + surname: de Rezende + email: rezende{at}ddc{dot}unicamp{dot}br + contacted1: false + contacted2: false +j_joao_dias_almeida: !ruby/object:TeX::Hyphen::Author + name: J. Joao + surname: Dias Almeida + email: jj{at}di{dot}uminho{dot}pt + contacted1: + contacted2: +# email doesn't work +piet_tutelaers: !ruby/object:TeX::Hyphen::Author + name: Piet + surname: Tutelaers + email: p{dot}t{dot}h{dot}tutelaers{at}tue{dot}nl + contacted1: false + contacted2: false +vytas_statulevicius: !ruby/object:TeX::Hyphen::Author + name: Vytas + surname: Statulevičius + email: vytas{at}vtex{dot}nl + contacted1: false + contacted2: false +sigitas_tolusis: !ruby/object:TeX::Hyphen::Author + name: Sigitas + surname: Tolušis + email: sigitas{at}vtex{dot}lt + contacted1: false + contacted2: false +janis_vilims: !ruby/object:TeX::Hyphen::Author + name: Janis + surname: Vilims + email: jvilims{at}apollo{dot}lv + contacted1: false + contacted2: false +joerg_knappen: !ruby/object:TeX::Hyphen::Author + name: Jörg + surname: Knappen + email: jknappen{at}web{dot}de + contacted1: true + contacted2: true +medeni_shemde: !ruby/object:TeX::Hyphen::Author + name: Medeni + surname: Shemdê + email: + contacted1: false + contacted2: false +terry_mart: !ruby/object:TeX::Hyphen::Author + name: Terry + surname: Mart + email: mart{at}kph{dot}uni-mainz{dot}de + contacted1: false + contacted2: false +# email doesn't work +jorgen_pind: !ruby/object:TeX::Hyphen::Author + name: Jorgen + surname: Pind + email: jorgen{at}lexis{dot}hi{dot}is + contacted1: false + contacted2: false +marteinn_sverrisson: !ruby/object:TeX::Hyphen::Author + name: Marteinn + surname: Sverrisson + email: + contacted1: false + contacted2: false +# email doesn't work +kristinn_gylfason: !ruby/object:TeX::Hyphen::Author + name: Kristinn + surname: Gylfason + email: kristgy{at}ieee{dot}org + contacted1: false + contacted2: false +# email doesn't work +kevin_p_scannell: !ruby/object:TeX::Hyphen::Author + name: Kevin P. + surname: Scannell + email: scanell{at}slu{dot}edu + contacted1: false + contacted2: false +# email doesn't work +peter_kleiweg: !ruby/object:TeX::Hyphen::Author + name: Peter + surname: Kleiweg + email: p{dot}c{dot}c{dot}kleiweg{at}rug{dot}nl + contacted1: false + contacted2: false +hanna_kolodziejska: !ruby/object:TeX::Hyphen::Author + name: Hanna + surname: Kołodziejska + email: + contacted1: false + contacted2: false +boguslaw_jackowski: !ruby/object:TeX::Hyphen::Author + name: Bogusław + surname: Jackowski + email: + contacted1: true + contacted2: true +marek_rycko: !ruby/object:TeX::Hyphen::Author + name: Marek + surname: Ryćko + email: + contacted1: false + contacted2: false +vladimir_volovich: !ruby/object:TeX::Hyphen::Author + name: Vladimir + surname: Volovich + email: # TODO add e-mail address + contacted1: true + contacted2: true +alexander_i_lebedev: !ruby/object:TeX::Hyphen::Author + name: Alexander I. # Not sure were 'I' belongs + surname: Lebedev + email: swan{at}scon155{dot}phys{dot}msu{dot}su + contacted1: false + contacted2: false +# first email doesn't work +maksym_polyakov: !ruby/object:TeX::Hyphen::Author + name: Maksym + surname: Polyakov + email: polyama{at}auburn{dot}edu # Second e-mail address in ukrhypmp.tex: mpoliak@i.com.ua + contacted1: false + contacted2: false +adrian_rezus: !ruby/object:TeX::Hyphen::Author + name: Adrian + surname: Rezus + email: adriaan{at}{sci,cs}{dot}kun{dot}nl + contacted1: false + contacted2: false +# email doesn't work +sahak_petrosyan: !ruby/object:TeX::Hyphen::Author + name: Sahak + surname: Petrosyan + email: sahak{at}mit{dot}edu + contacted1: true + contacted2: true # I think "true, true" is right. Arthur +dominik_wujastyk: !ruby/object:TeX::Hyphen::Author + name: Dominik + surname: Wujastyk + email: wujastyk{at}gmail{dot}com + contacted1: false + contacted2: false +graham_toal: !ruby/object:TeX::Hyphen::Author + name: Graham + surname: Toal + email: + contacted1: false + contacted2: false +donald_e_knuth: !ruby/object:TeX::Hyphen::Author + name: Donald E. + surname: Knuth + email: # Don doesn't use e-mail ;-) + contacted1: false + contacted2: false +gerard_d_c_kuiken: !ruby/object:TeX::Hyphen::Author + name: Gerard D.C. + surname: Kuiken + email: + contacted1: false + contacted2: false +pierre_mackay: !ruby/object:TeX::Hyphen::Author + name: P. A. + surname: MacKay + email: + contacted1: true + contacted2: true +h_turgut_uyar: !ruby/object:TeX::Hyphen::Author + name: H. Turgut + surname: Uyar + email: uyar{at}itu{dot}edu{tr} + contacted1: true + contacted2: true +# email doesn't work +s_ekin_kocabas: !ruby/object:TeX::Hyphen::Author + name: S. Ekin + surname: Kocabas + email: kocabas{at}stanford{dot}edu + contacted1: true + contacted2: true +bence_nagy: !ruby/object:TeX::Hyphen::Author + name: Bence + surname: Nagy + email: nagybence{at}tipogral{dot}hu + contacted1: true + contacted2: true +kauko_saarinen: !ruby/object:TeX::Hyphen::Author + name: Kauko + surname: Saarinen + email: + contacted1: false + contacted2: false +fred_karlsson: !ruby/object:TeX::Hyphen::Author + name: Fred + surname: Karlsson + email: + contacted1: false + contacted2: false +rene_bastian: !ruby/object:TeX::Hyphen::Author # TODO make contact + name: René + surname: Bastian + email: + contacted1: false + contacted2: false +daniel_flipo: !ruby/object:TeX::Hyphen::Author # TODO make contact + name: Daniel + surname: Flipo + email: + contacted1: false + contacted2: false +bernard_gaulle: !ruby/object:TeX::Hyphen::Author # Deceased... + name: Bernard + surname: Gaulle + email: + contacted1: false + contacted2: false +theppitak_karoonboonyanan: !ruby/object:TeX::Hyphen::Author + name: Theppitak + surname: Karoonboonyanan + email: theppitak{at}gmail{dot}com + contacted1: true + contacted2: true +levan_shoshiashvili: !ruby/object:TeX::Hyphen::Author + name: Levan + surname: Shoshiashvili + email: shoshia{at}hotmail{dot}com + contacted1: true + contacted2: true +# email doesn't work +javier_mugica: !ruby/object:TeX::Hyphen::Author + name: Javier + surname: Múgica + email: javier{at}digi21{dot}eu + contacted1: true + contacted2: true +georgi_boshnakov: !ruby/object:TeX::Hyphen::Author + name: Georgi + surname: Boshnakov + email: georgi{dot}boshnakov{at}manchester{dot}ac{dot}uk + contacted1: true + contacted2: true +mike_kroutikov: !ruby/object:TeX::Hyphen::Author + name: Mike + surname: Kroutikov + email: pgmmpk{at}gmail{dot}com + contacted1: true + contacted2: true +aleksandr_andreev: !ruby/object:TeX::Hyphen::Author + name: Aleksandr + surname: Andreev + email: '' + contacted1: true + contacted2: true +maksim_salau: !ruby/object:TeX::Hyphen::Author + name: Maksim + surname: Salau + email: maksim{dot}salau{at}gmail{dot}com + contacted1: true + contacted2: false +wie_ming_ang: !ruby/object:TeX::Hyphen::Author + name: Wie-Ming + surname: Cittānurakkho Bhikkhu + email: wiemingang{at}gmail{dot}com + contacted1: true + contacted2: true diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb new file mode 100644 index 00000000000..3d3437d74b5 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb @@ -0,0 +1,364 @@ +require 'yaml' +require 'hydra' +require 'byebug' unless ENV['RACK_ENV'] == "production" + +require_relative 'path' + +class String + def superstrip + strip.gsub /%.*$/, '' + end + + def supersplit + strip.gsub(/\s+/m,"\n").split("\n") + end + + def safe + gsub /[\s-]/, '' + end + + def titlecase + split.map(&:capitalize).join(' ') + end +end + +module TeX + module Hyphen + class InvalidMetadata < StandardError; end + class NoAuthor < InvalidMetadata; end + class NoLicence < InvalidMetadata; end + + include PATH + + class Author + def initialize(name,surname,email,contacted1,contacted2) + @name = name + @surname = surname + @email = email + # this mostly means if email has been recently checked + @contacted1 = contacted1 + # this means if we made more cooperation with author, + # exchanging patches etc. + @contacted2 = contacted2 + end + + attr_reader :name, :surname, :email + + def self.authors + @@authors ||= YAML::load File.read File.expand_path 'authors.yml', __dir__ + end + + def self.all + authors.values + end + + def self.[] a + authors[a] + end + end + + class Language + attr_reader :bcp47 + + @@eohmarker = '=' * 42 + + def iso639 + @bcp47.split('-').first + end + + # FIXME Find something better than all that extract_metadata unless @blah nonsens + def babelname + extract_metadata unless @babelname + @babelname + end + + def description + extract_metadata unless @description + @description + end + + def use_old_loader + extract_metadata unless @use_old_loader + @use_old_loader + end + + def use_old_patterns_comment + extract_metadata unless @use_old_patterns_comment + @use_old_patterns_comment + end + + def legacy_patterns + extract_metadata unless @legacy_patterns + @legacy_patterns + end + + def message + extract_metadata unless @message + @message + end + + def known_bugs + extract_metadata unless @known_bugs + @known_bugs + end + + def initialize(bcp47 = nil) + @bcp47 = bcp47 + end + + def self.languages + @@languages ||= Dir.glob(File.join(PATH::TEX, 'hyph-*.tex')).inject [] do |languages, texfile| + bcp47 = texfile.gsub /^.*\/hyph-(.*)\.tex$/, '\1' + # languages << [bcp47, Language.new(bcp47)] + languages << [bcp47, nil] + end.to_h + end + @@languages = Language.languages + + def self.all + @@all ||= languages.map do |bcp47, language| + next if bcp47 == 'sr-cyrl' # FIXME Remove later + @@languages[bcp47] ||= Language.new(bcp47) + end.compact + end + + def self.all_by_iso639 + all.inject(Hash.new) do |maintags, language| + (maintags[language.iso639] ||= Array.new) << language + maintags + end + end + + def self.find_by_bcp47(bcp47) + languages[bcp47] + end + + def private_use? + @bcp47.length == 3 || @bcp47[3] == '-' and @bcp47[0] == 'q' and ('a'..'t').include? @bcp47[1] + end + + # TODO This should probably become “macrolanguage name” or something similar + # @@displaynames = { + # 'el' => 'Greek', + # 'nb' => 'Norwegian', + # 'nn' => 'Norwegian', + # 'sh' => 'Serbian', + # } + + def licences + extract_metadata unless @licences + @licences + end + + def lefthyphenmin + extract_metadata unless @lefthyphenmin + @lefthyphenmin + end + + def righthyphenmin + extract_metadata unless @righthyphenmin + @righthyphenmin + end + + # Strictly speaking a misnomer, because grc-x-ibycus should also return true. + # But useful for a number of apostrophe-related routines + def isgreek? + ['grc', 'el-polyton', 'el-monoton'].include? @bcp47 + end + + def serbian? + @bcp47 =~ /^sh-/ + end + + def italic? + ['it', 'pms', 'rm'].include? @bcp47 + end + + def has_apostrophes? + begin + !isgreek? && patterns.any? { |p| p =~ /'/ } + rescue Errno::ENOENT + false + end + end + + def has_hyphens? + begin + patterns.any? { |p| p =~ /-/ } + rescue Errno::ENOENT + false + end + end + + def extract_apostrophes + plain, with_apostrophe = Array.new, nil + + patterns.each do |pattern| + plain << pattern + if pattern =~ /'/ && !isgreek? + pattern_with_apostrophe = pattern.gsub(/'/,"’") + plain << pattern_with_apostrophe + (with_apostrophe ||= []) << pattern_with_apostrophe + end + end + + { plain: plain, with_apostrophe: with_apostrophe } + end + + def extract_characters # FIXME Turkish and others iİ; check Church Slavonic + characters = Array.new + + characters_indexes = patterns.join.gsub(/[.0-9]/,'').unpack('U*').sort.uniq + characters_indexes.each do |c| + ch = [c].pack('U') + characters << ch + ch.upcase + characters << "’’" if ch == "'" && !isgreek? + end + + characters + end + + def comments_and_licence # Major TODO extract everything into YAML, and write .yml + @comments_and_licence ||= readtexfile.gsub(/(.*)\\patterns.*/m,'\1') + end + + def authors + extract_metadata unless @authors + @authors + end + + def github_link + sprintf 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-%s.tex', @bcp47 + end + + def <=>(other) + # TODO Remove when practical + a = self.babelname + b = other.babelname + + if a == 'serbian' && b == 'serbianc' + return -1 + elsif a == 'serbianc' && b == 'serbian' + return 1 + end + + self.bcp47 <=> other.bcp47 + end + + @@texfile = Hash.new + def readtexfile(bcp47 = @bcp47) + begin + @@texfile[bcp47] ||= File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', bcp47))) + rescue Errno::ENOENT + @@texfile[bcp47] = "" + end + end + + def patterns + @patterns ||= if @bcp47 == 'eo' then + readtexfile.superstrip. + gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1'). + # + gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.'). + gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.'). + gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.'). + # + supersplit + else + readtexfile(if ['nb', 'nn'].include? @bcp47 then 'no' else @bcp47 end).superstrip. + gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1'). + supersplit + end + end + + def exceptions + unless @exceptions + if readtexfile.superstrip.index('\hyphenation') + @exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit + if @exceptions != "" + @hyphenation = @exceptions.inject [] do |exceptions, exception| + # byebug unless exception + exceptions << [exception.gsub('-', ''), exception] + end.to_h + else + @hyphenation = { } + end + else + @exceptions = "" + @hyphenation = { } + end + end + + @exceptions + end + + def hyphenate(word) + exceptions + return @hyphenation[word] if @hyphenation[word] # FIXME Better name + + unless @hydra + begin + # byebug + metadata = extract_metadata + @hydra = Hydra.new patterns, :lax, '', metadata + rescue InvalidMetadata + @hydra = Hydra.new patterns + end + end + @hydra.showhyphens(word) + end + + def extract_metadata + header = "" + File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', @bcp47))).each_line do |line| + break if line =~ /\\patterns|#{@@eohmarker}/ + header += line.gsub(/^% /, '').gsub(/%.*/, '') + end + begin + metadata = YAML::load header + raise InvalidMetadata unless metadata.is_a? Hash + rescue Psych::SyntaxError + raise InvalidMetadata + end + + @name = metadata.dig('language', 'name') + @lefthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'left') || metadata.dig('hyphenmins', 'generation', 'left') + @righthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'right') || metadata.dig('hyphenmins', 'generation', 'right') + # FIXME That’s not nice + if respond_to? :synonyms + @synonyms = metadata.dig('texlive', 'synonyms') || [] + @encoding = metadata.dig('texlive', 'encoding') + end + @message = metadata.dig('texlive', 'message') + @legacy_patterns = metadata.dig('texlive', 'legacy_patterns') + @use_old_loader = metadata.dig('texlive', 'use_old_loader') + @use_old_patterns_comment = metadata.dig('texlive', 'use_old_patterns_comment') + @description = metadata.dig('texlive', 'description') + @babelname = metadata.dig('texlive', 'babelname') + @package = metadata.dig('texlive', 'package') + licences = metadata.dig('licence') + raise NoLicence unless licences + licences = [licences] unless licences.is_a? Array + @licences = licences.map do |licence| + raise bcp47 if licence.is_a? String + next if licence.values == [nil] + licence.dig('name') || 'custom' + end.compact + authors = metadata.dig('authors') + if authors + @authors = authors.map do |author| + author['name'] + end + else + @authors = nil + raise NoAuthor.new # FIXME + end + @known_bugs = metadata.dig('known_bugs') + + # raise NoAuthor unless @authors && @authors.count > 0 # TODO Later ;-) AR 2018-09-13 + + metadata + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml new file mode 100644 index 00000000000..bc029739388 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml @@ -0,0 +1,94 @@ +german: + dependency: dehyph + description: |- + Hyphenation patterns for German in T1/EC and UTF-8 encodings, + for traditional and reformed spelling, including Swiss German. + The package includes the latest patterns from dehyph-exptl + (known to TeX under names 'german', 'ngerman' and 'swissgerman'), + however 8-bit engines still load old versions of patterns + for 'german' and 'ngerman' for backward-compatibility reasons. + Swiss German patterns are suitable for Swiss Standard German + (Hochdeutsch) not the Alemannic dialects spoken in Switzerland + (Schwyzerduetsch). + There are no known patterns for written Schwyzerduetsch. +# for Russian and Ukrainian (until we implement the new functionality at least) +russian: + dependency: ruhyphen +ukrainian: + dependency: ukrhyph +greek: + doc: doc/generic/elhyphen + shortdesc: Modern Greek + description: |- + Hyphenation patterns for Modern Greek in monotonic and polytonic + spelling in LGR and UTF-8 encodings. Patterns in UTF-8 use two code + positions for each of the vowels with acute accent (a.k.a tonos, + oxia), e.g., U+03AC, U+1F71 for alpha. +hungarian: + doc: doc/generic/huhyphen +arabic: + shortdesc: (No) Arabic +farsi: + shortdesc: (No) Persian +chinese: + shortdesc: Chinese pinyin + description: |- + Hyphenation patterns for unaccented transliterated Mandarin Chinese + (pinyin) in T1/EC and UTF-8 encodings. The latter can hyphenate pinyin + with or without tone markers; the former only without. +norwegian: + shortdesc: Norwegian Bokmal and Nynorsk + description: |- + Hyphenation patterns for Norwegian Bokmal and Nynorsk in T1/EC and + UTF-8 encodings. +churchslavonic: + shortdesc: Church Slavonic +uppersorbian: + shortdesc: Upper Sorbian +ethiopic: + shortdesc_full: Hyphenation patterns for Ethiopic scripts +mongolian: + shortdesc_full: Mongolian hyphenation patterns in Cyrillic script + description: |- + Hyphenation patterns for Mongolian in T2A, LMC and UTF-8 encodings. + LMC encoding is used in MonTeX. The package includes two sets of + patterns that will hopefully be merged in future. +latin: + description: |- + Hyphenation patterns for Latin in T1/EC and UTF-8 encodings, + mainly in modern spelling (u when u is needed and v when v is needed), + medieval spelling with the ligatures \ae and \oe and the (uncial) + lowercase 'v' written as a 'u' is also supported. Apparently + there is no conflict between the patterns of modern Latin and + those of medieval Latin. + Hyphenation patterns for the Classical Latin in T1/EC and UTF-8 + encodings. Classical Latin hyphenation patterns are different from + those of 'plain' Latin, the latter being more adapted to modern Latin. + Hyphenation patterns for the Liturgical Latin in T1/EC and UTF-8 + encodings. +english: + description: |- + Additional hyphenation patterns for American and British + English in ASCII encoding. The American English patterns + (usenglishmax) greatly extend the standard patterns from Knuth + to find many additional hyphenation points. British English + hyphenation is completely different from US English, so has its + own set of patterns. +indic: + description: |- + Hyphenation patterns for Assamese, Bengali, Gujarati, Hindi, Kannada, + Malayalam, Marathi, Oriya, Panjabi, Tamil and Telugu for Unicode + engines. +"ancient greek": + description: |- + Hyphenation patterns for Ancient Greek in LGR and UTF-8 encodings, + including support for (obsolete) Ibycus font encoding. + Patterns in UTF-8 use two code positions for each of the vowels with + acute accent (a.k.a tonos, oxia), e.g., U+03AE, U+1F75 for eta. +serbian: + description: |- + Hyphenation patterns for Serbian in T1/EC, T2A and UTF-8 encodings. + For 8-bit engines the patterns are available separately as 'serbian' + in T1/EC encoding for Latin script and 'serbianc' in T2A encoding for + Cyrillic script. Unicode engines should only use 'serbian' + which has patterns in both scripts combined. diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb new file mode 100644 index 00000000000..fda422b11a8 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb @@ -0,0 +1,25 @@ +module TeX + module Hyphen + module PATH + ROOT = File.expand_path('../../../../../../../..', __FILE__) + + TeXROOT = File.join(ROOT, 'hyph-utf8') + TeX_GENERIC = File.join(TeXROOT, 'tex', 'generic') + PAT = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns') + TXT = File.join(PAT, 'txt') + TEX = File.join(PAT, 'tex') + PTEX = File.join(PAT, 'ptex') + QUOTE = File.join(PAT, 'quote') + LOADER = File.join(TeX_GENERIC, 'hyph-utf8', 'loadhyph') + + SUPPORT = File.join(TeXROOT, '%s', 'generic', 'hyph-utf8', 'languages', '*') + + HYPHU8 = File.join('tex', 'generic', 'hyph-utf8') + + TL = File.join(ROOT, 'TL') + LANGUAGE_DAT = File.join(PATH::TL, 'texmf-dist', 'tex', 'generic', 'config') + # hyphen-foo.tlpsrc for TeX Live + TLPSRC = File.join(PATH::TL, 'tlpkg', 'tlpsrc') + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb new file mode 100644 index 00000000000..45dd33da047 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb @@ -0,0 +1,22 @@ +require_relative 'texlive/source' +require_relative 'texlive/loader' +require_relative 'texlive/package' + +module TeX + module Hyphen + module TeXLive + def synonyms + extract_metadata unless @synonyms + @synonyms + end + + def encoding + extract_metadata unless @encoding + @encoding + end + + include Source + include Loader + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb new file mode 100644 index 00000000000..2fcb3945d30 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb @@ -0,0 +1,193 @@ +module TeX + module Hyphen + module TeXLive + module Loader + + def unicode_only? # TODO Spec out + ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47 + end + + def string_enc + (encoding == nil) ? "" : encoding.upcase + " " + end + + def lcchars + if isgreek? + # some catcodes for XeTeX + return { + lccode: { + 0x0027 => '\'', + 0x2019 => '’', + 0x02BC => 'ʼ', + 0x1FBD => '᾽', + 0x1FBF => '᾿', + } + } + end + + return { } unless unicode_only? + + default = { + comment: 'Set \lccode for ZWNJ and ZWJ.', + lccode: { + 0x200C => false, + 0x200D => false, + } + } + + { + 'mul-ethi' => { + comment: 'Set \lccode for Ethiopian word space.', + lccode: { + 0x1361 => false, + 0x1362 => false, + } + }, + 'cu' => { + comment: 'fix lccodes for some characters (they were recently included in Unicode)', + lccode: { + 0x1C82 => 'sharp o in lowercase "uk"', + 0x1DF6 => false, + 0x1DF7 => false, + 0x1DF8 => false, + 0x1DF9 => false, + 0xA69E => false, + 0x1C86 => false, + 0xA67E => false, + 0xFE2E => false, + 0xFE2F => false, + } + }, + 'sa' => { + comment: 'Set \lccode for ZWNJ and ZWJ.', + lccode: { + 0x200C => false, + 0x200D => "\n% Set \\lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.", + 0x0CF1 => false, + 0x0CF2 => false, + } + }, + }[@bcp47] || default + end + + def utf8_engine_message + if serbian? + 'UTF-8 Serbian hyphenation patterns' + else + sprintf('UTF-8 %s', message) + end + end + + def non_utf8_engine_message + if unicode_only? + sprintf('No %s - only for Unicode engines', message) + else + sprintf('%s%s', string_enc, message) + end + end + + def engine_message(engine) + if engine == 'UTF-8' + { + comment: 'Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument', + message: utf8_engine_message + } + else # engine is 8-bit or pTeX + { + comment: engine + + if engine == '8-bit' then " engine (such as TeX or pdfTeX)" else "" end, + message: non_utf8_engine_message + } + end + end + + def input_8bit_file + if @bcp47 == 'la-x-liturgic' + { + input: [pTeX_patterns] + } + elsif use_old_patterns_comment + # explain why we are still using the old patterns + { + comment: use_old_patterns_comment, + input: [legacy_patterns] + } + elsif !italic? + { + input: [sprintf('conv-utf8-%s.tex', encoding), sprintf('hyph-%s.tex', @bcp47)] + } + else + { + input: [sprintf('hyph-%s.tex', @bcp47)] + } + end + end + + def input_utf8_line + if serbian? + { + comment: 'We load both scripts at the same time to simplify usage', + input: ['hyph-sh-latn.tex', 'hyph-sh-cyrl.tex'] + } + else + { + input: [sprintf('hyph-%s.tex', @bcp47)] + } + end + end + + def format_inputs(specification) + if specification.is_a? Array + return specification.map do |hash| + format_inputs(hash) + end + end + + byebug unless specification.is_a? Hash + comment = specification[:comment] + message = specification[:message] + if comment then [sprintf('%% %s', comment)] else [] end + + if message then [sprintf('\\message{%s}', message)] else [] end + + (specification[:lccode] || []).map do |code, comment| + sprintf '\\lccode"%04X="%04X%s', code, code, if comment then sprintf ' %% %s', comment else '' end + end + + (specification[:input] || []).map do |input| + sprintf '\\input %s', input + end + end + + def input_line(engine) + if engine == '8-bit' + input_8bit_file + elsif engine == 'pTeX' + { input: [pTeX_patterns] } + elsif engine == 'UTF-8' + input_utf8_line + end + end + + def utf8_chunk + [ + engine_message('UTF-8'), + # lccodes + lcchars, + input_line('UTF-8'), + if has_apostrophes? then { input: [sprintf('hyph-quote-%s.tex', bcp47)] } end + ].compact + end + + def nonutf8_chunk(engine) + [engine_message(engine), unless unicode_only? then input_line(engine) end].compact + end + + def loadhyph + if use_old_loader + legacy_patterns + else + sprintf 'loadhyph-%s.tex', @bcp47.gsub(/^sh-/, 'sr-') + end + end + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb new file mode 100644 index 00000000000..69481a41774 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb @@ -0,0 +1,126 @@ +module TeX + module Hyphen + module TeXLive + class Package + attr_reader :name + + @@metadata = YAML::load File.read File.expand_path '../packages.yml', __dir__ + + def initialize(name) + @name = name + @languages = [] + end + + def add_language(language) + @languages << language + @languages.sort! + end + + def languages + @languages + end + + def self.packages + # a hash with the names of TeX Live packages, either individual language names, + # or an array of languages as the value + @@packages ||= Language.all.inject(Hash.new) do |packages, language| + if name = language.package || language.babelname # Package name, otherwise single language + packages[name] ||= Package.new(name) # Create new package if necessary + packages[name].add_language language + end + + packages + end + end + + def self.all + packages.values + end + + def description_s + override = @@metadata.dig(@name, 'shortdesc_full') + return override if override + + leader = @@metadata.dig(@name, 'shortdesc') || @name.titlecase + sprintf '%s hyphenation patterns', leader + end + + def description + description = @@metadata.dig(@name, 'description') + if description + description + elsif @languages.count == 1 + @languages.first.description + else + '' + end + end + + def has_dependency? + @@metadata.dig(name, 'dependency') + end + + def list_dependencies + dependencies = [ + "depend hyphen-base", + "depend hyph-utf8", + ] + + # external dependencies + if dependency = has_dependency? + dependencies << sprintf("depend %s", dependency) + end + + dependencies + end + + def list_support_files(type) # type is ‘doc’ or ‘source’ + # Cache directory contents + (@dirlist ||= { })[type] ||= Dir.glob(sprintf(PATH::SUPPORT, type)).select do |file| + File.directory?(file) + end.map do |dir| + dir.gsub /^.*\//, '' + end + + files = (languages.map(&:bcp47) & @dirlist[type]).map do |bcp47| + sprintf("%s/generic/hyph-utf8/languages/%s", type, bcp47) + end + + if special = @@metadata.dig(name, type) + files << special + end + + files + end + + def list_run_files + files = [] + files << "tex/generic/hyph-utf8/patterns/tex/hyph-no.tex" if name == "norwegian" + + files = languages.inject(files) do |files, language| + files + language.list_run_files + end + + unless has_dependency? + languages.each do |language| + if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop' + files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns) + end + end + end + + files + end + + def <=>(other) + name <=> other.name + end + + # TODO Spec out + def self.find(name) + packages[name] + end + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb new file mode 100644 index 00000000000..61292dcd3bb --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb @@ -0,0 +1,117 @@ +module TeX + module Hyphen + module TeXLive + module Source + def list_synonyms + if synonyms && synonyms.length > 0 + sprintf " synonyms=%s", synonyms.join(',') + else + '' + end + end + + def list_hyphenmins + lmin = lefthyphenmin + rmin = righthyphenmin + sprintf "lefthyphenmin=%s \\\n\trighthyphenmin=%s", lmin, rmin + end + + # ext: 'pat' or 'hyp' + # filetype: 'patterns' or 'exceptions' + def plain_text_line(ext, filetype) # TODO Figure out if we will sr-cyrl to be generated again + return "" if ['ar', 'fa', 'grc-x-ibycus', 'mn-cyrl-x-lmc'].include? @bcp47 + + if @bcp47 =~ /^sh-/ + # TODO Warning AR 2018-09-12 + filename = sprintf 'hyph-sh-latn.%s.txt,hyph-sh-cyrl.%s.txt', ext, ext + else + filename = sprintf 'hyph-%s.%s.txt', @bcp47, ext + filepath = File.join(PATH::TXT, filename) + # check for existence of file and that it’s not empty + unless File.file?(filepath) && File.read(filepath).length > 0 + # if the file we were looking for was a pattern file, something’s wrong + if ext == 'pat' + raise sprintf("There is some problem with plain patterns for language [%s]!!!", @bcp47) + else # the file is simply an exception file and we’re happy + filename = '' # And we return and empty file name after all + end + end + end + + sprintf "file_%s=%s", filetype, filename + end + + def exceptions_line + plain_text_line('hyp', 'exceptions') + end + + def patterns_line + plain_text_line('pat', 'patterns') + end + + def pTeX_patterns + if italic? + sprintf 'hyph-%s.tex', @bcp47 + elsif encoding + sprintf 'hyph-%s.%s.tex', @bcp47, encoding + else + legacy_patterns + end + end + + def list_loader + # which loader to use + if ['ar', 'fa'].include? @bcp47 + sprintf "file=%s \\\n\tfile_patterns=", loadhyph + elsif @bcp47 == 'grc-x-ibycus' + # TODO: fix this + sprintf "file=%s \\\n\tluaspecial=\"disabled:8-bit only\"", loadhyph + else + sprintf "file=%s", loadhyph + end + end + + def list_run_files + return [] if use_old_loader + + files = [] + + files << File.join(PATH::HYPHU8, 'loadhyph', loadhyph) + if has_apostrophes? + files << File.join(PATH::HYPHU8, 'patterns', 'quote', sprintf("hyph-quote-%s.tex", bcp47)) + end + + files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47)) + if encoding && encoding != "ascii" then + files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding)) + elsif bcp47 == "cop" + files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns) + end + + # we skip the mongolian language for luatex files + return files if bcp47 == "mn-cyrl-x-lmc" + + ['chr', 'pat', 'hyp', 'lic'].each do |t| + files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-%s.%s.txt', bcp47, t)) + end + + if bcp47 =~ /^sh-/ + # duplicate entries (will be removed later) + files << File.join(PATH::HYPHU8, 'patterns', 'tex', 'hyph-sr-cyrl.tex') + ['chr', 'pat', 'hyp', 'lic'].each do |t| + # duplicate entries (will be removed later) + files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-sr-cyrl.%s.txt', t)) + end + end + + files + end + + def package + extract_metadata + @package + end + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/spec_helper.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/spec_helper.rb new file mode 100644 index 00000000000..da5b4252148 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/spec_helper.rb @@ -0,0 +1,7 @@ +$LOAD_PATH << File.expand_path('../../lib', __FILE__) +require 'tex/hyphen/language' +require 'tex/hyphen/texlive/package' +require 'tex/hyphen/texlive' +require 'tex/hyphen/texlive/loader' + +RSpec::Expectations.configuration.on_potential_false_positives = :nothing diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb new file mode 100644 index 00000000000..4805b1878db --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb @@ -0,0 +1,985 @@ +require_relative '../../spec_helper' + +include TeX::Hyphen +include Language::TeXLive +include TeXLive +include Loader + +describe String do + describe '#superstrip' do + it "calls String#strip" do + str = " foo bar " + expect(str).to receive(:strip).and_return "foo bar" + str.superstrip + end + + it "strips TeX comments" do + expect("foo % bar".superstrip).to eq "foo " + end + end + + describe '#supersplit' do + it "calls String#strip" do + str = "foo bar " + expect(str).to receive(:strip).and_return "foo bar" + str.supersplit + end + + it "splits across whitespace ranges" do + expect("foo bar".supersplit).to eq ["foo", "bar"] + end + end + + describe '#safe' do + it "strips hyphens" do + expect("foo-bar".safe).to eq "foobar" + end + + it "strips spaces" do + expect("ancient greek".safe).to eq "ancientgreek" + end + end + + describe '#titlecase' do + it "capitalises each word" do + expect("modern greek".titlecase).to eq "Modern Greek" + end + end +end + +# TODO? Spec out PATH module + +describe Author do + describe 'Class methods' do + describe '.authors' do + it "returns a hash of authors" do + expect(Author.authors).to be_a Hash + end + end + + describe '.all' do + it "returns the array of authors" do + expect(Author.all).to be_an Array + end + end + + describe '.[]' do + it "returns the author for the key" do + expect(Author['donald_knuth']).to be_an Author + end + end + end + + describe 'Instance methods' do + let(:dek) { Author['donald_knuth'] } + + describe '#name' do + it "returns the author’s name" do + expect(dek.name).to eq 'Donald' + end + end + + describe '#surname' do + it "returns the author’s surname" do + expect(dek.surname).to eq 'Knuth' + end + end + + describe '#email' do + it "returns the email" do + expect(dek.email).to be_nil + end + end + end +end + +describe Language do + describe 'Class variables' do + it "has an end-of-header marker" do + expect(Language.class_variable_get :@@eohmarker).to match /^\={42}$/ + end + end + + describe '.new' do + it "creates a new Language instance" do + expect(Language.new).to be_a Language + end + + it "takes an optional BCP47 tag as argument" do + language = Language.new('ro') + expect(language.instance_variable_get :@bcp47).to eq 'ro' + end + +# it "calls .languages" do +# expect(Language).to receive(:languages).and_return({ 'pa' => nil }) +# Language.new('pa') +# end + end + + describe '.languages' do + it "sets the @@languages class variable" do + Language.languages + expect(Language.class_variable_get :@@languages).to be_a Hash + end + + it "lists all languages" do + # All the TeX files. Note [no] and [mn-cyrl-x-lmc] don’t have corresponding plain text files. + expect(Language.languages.count).to eq 82 # Was 79; 3 more “TeX Live dummies” [ar] [fa] [grc-x-ibycus] TODO Maybe remove + end + end + + describe '.all' do + it "returns the list of languages as an array" do + expect(Language.all).to be_an Array + end + + it "returns 81 languages" do # That’s all of them except for [sr-cyrl] + expect(Language.all.count).to eq 81 + end + end + + describe '.all_by_iso639' do + it "returns a hash of arrays" do + expect(Language.all_by_iso639).to be_a Hash + expect(Language.all_by_iso639.values.map(&:class).uniq).to eq [Array] + end + + it "groups languages by ISO 639 code elements" do + expect(Language.all_by_iso639['en'].count).to eq 2 + end + end + + describe '.find_by_bcp47' do + it "finds the language for that BCP47 tag" do + language = Language.find_by_bcp47 'bn' + expect(language).to be_a Language + end + + it "calls .languages first" do + expect(Language).to receive(:languages).and_return({ 'cy' => Language.new('cy') }) + Language.find_by_bcp47('cy') + end + end + + describe '#bcp47' do # TODO Add #8bitenc + it "returns the BCP47 tag of the language" do + language = Language.new('oc') + expect(language.bcp47).to eq 'oc' + end + end + + describe '#iso639' do + it "returns the ISO 639 primary tag" do + expect(Language.new('en-gb').iso639).to eq 'en' + end + + it "works on 3-character tags" do + expect(Language.new('grc-x-ibycus').iso639).to eq 'grc' + end + end + +# describe '#name' do +# let(:new_orthography_german) { Language.new('de-1996') } +# +# it "returns the name" do +# expect(new_orthography_german.name).to eq 'German, reformed spelling' +# end +# +# it "calls #extract_metadata first if necessary" do +# expect(new_orthography_german).to receive(:extract_metadata).and_return({ 'name' => 'German, reformed spelling' }) +# new_orthography_german.name +# end +# +# it "doesn’t call #extract_metadata if @name is already set" do +# new_orthography_german.instance_variable_set :@name, 'Deutch in neuer Rechtschreibung' +# expect(new_orthography_german).not_to receive :extract_metadata +# new_orthography_german.name +# end +# end + + # describe '#displayname' do + # it "returns @name most of the name" do + # romansh = Language.new('rm') + # expect(romansh.displayname).to eq 'Romansh' + # end + + # it "strips to the language subtag part" do + # polytonic_greek = Language.new('el-polyton') + # expect(polytonic_greek.displayname).to eq 'Greek' + # end + + # it "returns Norwegian for [nb] and [no]" do + # bokmål = Language.new('nb') + # expect(bokmål.displayname).to eq 'Norwegian' + # end + + # it "returns Serbian for [sh]" do + # serbocroatian_cyrillic = Language.new('sh-cyrl') + # expect(serbocroatian_cyrillic.displayname).to eq 'Serbian' + # end + + # it "calls #extract_metadata if needed" do + # friulan = Language.new('fur') + # expect(friulan).to receive :extract_metadata + # friulan.displayname + # end + # end + + describe '#babelname' do + it "returns the Babel name" do + expect(Language.new('de-1996').babelname).to eq 'ngerman' + end + + it "calls #extract_metadata first" do + german_CH = Language.new('de-ch-1901') + expect(german_CH).to receive :extract_metadata + german_CH.babelname + end + end + + # describe '#packagename' do + # it "returns the package name for TeX Live" do + # expect(Language.new('gu').packagename).to eq 'indic' + # end + + # it "uses some questionable choices" do + # expect(Language.new('grc').packagename).to eq 'ancient greek' + # end + # end + + describe '#licences' do + let(:church_slavonic) { Language.new('cu') } + + it "returns the licences as an array" do + expect(church_slavonic.licences).to eq ['MIT'] + end + + it "call #extract_metadata first if necessary" do + expect(church_slavonic).to receive(:extract_metadata) + church_slavonic.licences + end + + it "doesn’t call #extract_metadata if @licences is already set" do + church_slavonic.instance_variable_set :@licences, ['MIT licence'] + expect(church_slavonic).not_to receive :extract_metadata + church_slavonic.licences + end + + it "raises an exception if @licences is nil or empty" do + nolicence = Language.new('qnl') + allow(File).to receive(:read).and_return("code: qnl\nauthors:\n - me") + expect { nolicence.licences }.to raise_exception NoLicence + end + end + + describe '#lefthyphenmin' do + let(:swiss_spelling_german) { Language.new('de-ch-1901') } + + it "returns the left hyphenmin value for typesetting" do + expect(swiss_spelling_german.lefthyphenmin).to eq 2 + end + + it "calls #extract_metadata first if necessary" do + expect(swiss_spelling_german).to receive :extract_metadata + swiss_spelling_german.lefthyphenmin + end + + it "doesn’t call #extract_metadata if @lefthyphenmin is already set" do + swiss_spelling_german.instance_variable_set :@lefthyphenmin, 1 + expect(swiss_spelling_german).not_to receive :extract_metadata + swiss_spelling_german.lefthyphenmin + end + + it "uses the generation value if typesetting is missing" do + ethiopic = Language.new('mul-ethi') + expect(ethiopic.lefthyphenmin).to eq 1 + end + end + + describe '#righthyphenmin' do + let(:french) { Language.new('fr') } + + it "returns the right hyphenmin value for typesetting" do + expect(french.righthyphenmin).to eq 2 + end + + it "call #extract_metadata first if necessary" do + expect(french).to receive :extract_metadata + french.righthyphenmin + end + + it "doesn’t call #extract_metadata if @righthyphenmin is already set" do + french.instance_variable_set :@righthyphenmin, 2 + expect(french).not_to receive :extract_metadata + french.righthyphenmin + end + + it "uses the generation value if typesetting is missing" do + ethiopic = Language.new('mul-ethi') + expect(ethiopic.righthyphenmin).to eq 1 + end + end + + describe '#authors' do + let(:traditional_orthography_german) { Language.new('de-1901') } + + it "returns the authors as an array" do + expect(traditional_orthography_german.authors).to eq ['Deutschsprachige Trennmustermannschaft'] + end + + it "calls #extract_metadata first if necessary" do + expect(traditional_orthography_german).to receive(:extract_metadata).and_return({ authors: ['Werner Lemberg', 'others'] }) + traditional_orthography_german.authors + end + + it "doesn’t call #extract_metadata if @authors is already set" do + traditional_orthography_german.instance_variable_set :@authors, ['German hyphenation patterns team'] + expect(traditional_orthography_german).not_to receive :extract_metadata + traditional_orthography_german.authors + end + end + + describe '#synonyms' do + it "returns the synonyms as an array" do + expect(Language.new('sl').synonyms).to eq ['slovene'] + end + + it "returns an empty array instead of nil if there are no synonyms" do + expect(Language.new('sk').synonyms).to eq [] + end + end + + describe '#encoding' do + it "returns the encoding" do + expect(Language.new('sh-cyrl').encoding).to eq 't2a' + end + + it "returns “ascii” if applicable" do + expect(Language.new('rm').encoding).to eq 'ascii' + end + + it "returns nil if patterns are Unicode-only" do + expect(Language.new('sa').encoding).to be_nil + end + end + + describe '#github_link' do + it "returns the GitHub link" do + upper_sorbian = Language.new('hsb') + expect(upper_sorbian.github_link).to eq 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-hsb.tex' + end + end + + describe '#<=>' do +# it "compares Language’s using @name’s" do +# expect(Language.new('de-1996') <=> Language.new('fr')).to eq 1 +# end + + it "uses BCP47 codes if names are not available" do + expect(Language.new('zh-latn-pinyin') <=> Language.new('cu')).to eq 1 + end + end + + describe '#readtexfile' do + let(:basque) { Language.new('eu') } + + it "reads the TeX file" do + expect(File).to receive(:read) + basque.readtexfile + end + + it "stores the contents into the @@texfile class variable" do + basque.readtexfile + expect(Language.class_variable_get(:@@texfile)['eu']).to match /1ba.*1ko.*1t2xe.*su2b2r/m + end + + it "recovers gracefully from nonexistent files" do + expect { Language.new('kl').readtexfile }.not_to raise_exception + end + end + + describe '#patterns' do + it "returns the patterns" do + danish = Language.new('da') + expect(['.ae3', '.an3k', '.an1s'].all? { |p| danish.patterns.include? p }).to be_truthy + end + + it "calls .languages first" do + language = Language.new('eu') + expect(Language).to receive(:languages).and_return({ 'eu' => Language.new('eu') }) + pending "Needs pondering" + language.patterns + end + + it "loads the patterns" do + language = Language.new('fi') + # byebug + expect(language.patterns[151..154]).to eq ['uu1a2', 'uu1e2', 'uu1o2', 'uu1i2'] + end + + it "doesn’t crash on inexistent patterns" do + expect { Language.new('zu').patterns }.not_to raise_exception + end + + it "caches the list of patterns" do + language = Language.new('ru') + language.patterns + expect(language.instance_variable_get(:@patterns)[0..2]).to eq ['.аб1р', '.аг1ро', '.ади2'] + end + + it "uses the [no] patterns for [nb]" do + expect(Language.new('nb').patterns).to eq Language.new('no').patterns + end + + it "expands the Esperanto patterns" do + esperanto = Language.new('eo') + expect(['.di3s2a.', '.di3s2aj.', '.di3s2ajn.', '.di3s2an.', '.di3s2e.'].any? { |p| esperanto.patterns.include? p }).to be_truthy + end + end + + describe '#exceptions' do + it "returns the hyphenation exceptions" do + language = Language.new('ga') + # byebug + expect(language.exceptions[0..2]).to eq ['bhrachtaí', 'mbrachtaí', 'cháintí'] + end + + it "calls .languages first" do + language = Language.new('hu') + pending "Needs examining" + expect(Language).to receive(:all).and_return({ 'hu' => Language.new('hu') }) + language.patterns + end + + it "loads the exceptions" do + language = Language.new('is') + expect(File).to receive(:read).and_return("alc-un alc-u-nis-si-me alc-un-men-te") + language.patterns + end + + it "doesn’t crash on inexistent patterns" do + expect { Language.new('iu').exceptions}.not_to raise_exception + end + + it "caches the exceptions" do + language = Language.new('sk') + language.exceptions + expect(language.instance_variable_get(:@exceptions)[0..2]).to match ['dosť', 'me-tó-da', 'me-tó-dy'] + end + + it "hashes the exceptions" do + language = Language.new('en-gb') + language.exceptions + hyphenation = language.instance_variable_get :@hyphenation + expect(hyphenation.count).to eq 8 + expect(hyphenation['however']).to eq 'how-ever' + end + end + + describe '#hyphenate' do + it "hyphenates with the appropriate patterns" do + czech = Language.new('cs') + expect(czech.hyphenate('ubrousek')).to eq 'ubrou-sek' + end + + it "takes hyphenmins in account if available" do + language = Language.new('de-1996') + expect(language.hyphenate('Zwangsvollstreckungsmaßnahme')).to eq 'zwangs-voll-stre-ckungs-maß-nah-me' + end + + it "takes exceptions in account if available" do + american_english = Language.new('en-us') + expect(american_english.hyphenate('project')).to eq 'project' + end + + it "initialises the hydra if needed" do + language = Language.new('de-1901') + language.hyphenate('Zwangsvollstreckungsmaßnahme') + expect(language.instance_variable_get(:@hydra)).to be_a Hydra + end + + it "calls #exceptions" do + esperanto = Language.new('eo') + expect(esperanto).to receive(:exceptions) + esperanto.instance_variable_set :@hyphenation, { 'ŝtatregosciencon' => 'ŝta-tre-go-scien-con' } + esperanto.hyphenate('ŝtatregosciencon') + end + end + + describe '#private_use?' do + it "returns true for languages with private use BCP 47 tags" do + expect(Language.new('qls').private_use?).to be_truthy + end + + it "returns true otherwise" do + expect(Language.new('qyz').private_use?).to be_falsey + end + + it "doesn’t crash on two-letter tags" do + expect { Language.new('qa').private_use? }.not_to raise_exception + end + + it "returns false on two-letter tags" do + expect(Language.new('qt').private_use?).to be_falsey + end + + it "works correctly on four-letter tags" do + expect(Language.new('qaaa').private_use?).to be_falsey + end + + it "works correctly in the presence of subtags" do + expect(Language.new('qtz-GB').private_use?).to be_truthy + end + end + + describe '#extract_metadata' do + it "returns a hash with the metadata" do + language = Language.new('bg') + expect(language.extract_metadata).to be_a Hash + end + + it "raises an exception if the metadata is just a string" do + language = Language.new('qls') + allow(File).to receive(:read).and_return("just a string") + expect { language.extract_metadata }.to raise_exception InvalidMetadata + end + + it "raises an exception if the licence is missing" do + language = Language.new('qlv') + allow(File).to receive(:read).and_return("name: language virtual\ncode: qlv") + expect { language.extract_metadata }.to raise_exception InvalidMetadata + end + + it "raises an exception if @authors is nil or empty" do + not_church_slavonic = Language.new('qcu') + allow(File).to receive(:read).and_return "code: qcu\nlicence:\n name:\n MIT" + expect { not_church_slavonic.authors }.to raise_exception NoAuthor + end + + it "doesn’t crash on invalid licence entries" do + syntax_error = Language.new('qse') + allow(File).to receive(:read).and_return "foo:\nbar" + expect { syntax_error.extract_metadata }.not_to raise_exception Psych::SyntaxError + end + + it "sets the language name" do + language = Language.new('th') + language.extract_metadata + expect(language.instance_variable_get :@name).to eq 'Thai' + end + + it "sets the licence list" do + language = Language.new('la') + language.extract_metadata + expect(language.instance_variable_get :@licences).to eq ['MIT', 'LPPL'] + end + + it "sets lefthyphenmin" do + pali = Language.new('pi') + pali.extract_metadata + expect(pali.instance_variable_get :@lefthyphenmin).to eq 1 + end + + it "sets righthyphenmin" do + german = Language.new('de-1996') + german.extract_metadata + expect(german.instance_variable_get :@righthyphenmin).to eq 2 + end + + it "sets the list of authors" do + liturgical_latin = Language.new('la-x-liturgic') + liturgical_latin.extract_metadata + expect(liturgical_latin.instance_variable_get :@authors).to eq ['Claudio Beccari', 'Monastery of Solesmes', 'Élie Roux'] + end + + context "With Swedish as an example" do + let(:swedish) { Language.new('sv') } + + it "sets the message" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@message).to eq "Swedish hyphenation patterns" + end + + it "sets the old pattern file name" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@legacy_patterns).to eq "svhyph.tex" + end + + it "leaves @use_old_loader to nil in most cases" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@use_old_loader).to be_nil + end + + it "sets the @use_old_loader boolean to true if applicable" do + norwegian = Language.new('no') + norwegian.extract_metadata + expect(norwegian.instance_variable_get :@use_old_loader).to be_truthy + end + + it "leaves @use_old_patterns_comment to nil in most cases" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@use_old_patterns_comment).to be_nil + end + + it "sets the @use_old_patterns_comment to string if applicable" do + german_AR = Language.new('de-1901') + german_AR.extract_metadata + expect(german_AR.instance_variable_get :@use_old_patterns_comment).to eq "Kept for the sake of backward compatibility, but newer and better patterns by WL are available." + end + + it "sets the long description" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@description).to eq "Hyphenation patterns for Swedish in T1/EC and UTF-8 encodings." + end + + it "sets the Babel name" do + swedish.extract_metadata + expect(swedish.instance_variable_get :@babelname).to eq "swedish" + end + + it "sets the Babel name even if it is slightly silly ;-)" do + german_NR = Language.new('de-1996') + german_NR.extract_metadata + expect(german_NR.instance_variable_get :@babelname).to eq "ngerman" + end + + it "sets @package to nil in most cases" do + expect(swedish.instance_variable_get :@package).to be_nil + end + + it "sets @package for a few languages" do + gujarati = Language.new('gu') + gujarati.extract_metadata + expect(gujarati.instance_variable_get :@package).to eq 'indic' + end + end + end + + describe '#has_apostrophes?' do + it "returns if patterns have apostrophes" do + expect(Language.new('be').has_apostrophes?).to be_truthy + end + + it "returns false otherwise" do + expect(Language.new('bn').has_apostrophes?).to be_falsey + end + end + + describe '#has_hyphens?' do + it "returns true if patterns have dashes" do + expect(Language.new('uk').has_hyphens?).to be_truthy + end + + it "returns false otherwise" do + expect(Language.new('tr').has_hyphens?).to be_falsey + end + end + + describe '#isgreek?' do + it "returns true if language is Greek (sort of)" do + expect(Language.new('grc').isgreek?).to be_truthy + end + + it "returns false if not" do + expect(Language.new('la').isgreek?).to be_falsey + end + + it "exceptionally returns false for Ibycus" do + expect(Language.new('grc-x-ibycus').isgreek?).to be_falsey + end + end + + describe '#message' do + it "returns the message to be displayed on the terminal" do + expect(Language.new('af').message).to eq 'Afrikaans hyphenation patterns' + end + end + + describe '#known_bugs' do + it "returns the known bugs" do + american_english = Language.new('en-us') + expect(american_english.known_bugs.keys).to eq ['de-mo-c-rat'] + end + end + + describe TeXLive do + describe '#loadhyph' do + it "returns the name of the pattern loader file" do + expect(Language.new('cy').loadhyph).to eq 'loadhyph-cy.tex' + end + + it "replaces the main sh subtag by sr" do + expect(Language.new('sh-latn').loadhyph).to eq 'loadhyph-sr-latn.tex' + end + + it "returns the old loader name if applicable" do + expect(Language.new('grc-x-ibycus').loadhyph).to eq 'ibyhyph.tex' + end + end + + describe '#list_loader' do + it "returns the tlpsrc line with the loader" do + expect(Language.new('hr').list_loader).to eq 'file=loadhyph-hr.tex' + end + + it "includes an empty line for Arabic and Farsi" do + expect(Language.new('ar').list_loader).to eq "file=zerohyph.tex \\\n\tfile_patterns=" + end + + it "includes a Lua special line for Ibycus" do + expect(Language.new('grc-x-ibycus').list_loader).to eq "file=ibyhyph.tex \\\n\tluaspecial=\"disabled:8-bit only\"" + end + end + + describe '#list_run_files' do + it "returns the list of TeX file" do + # puts Language.new('ka').list_run_files + expect(Language.new('ka').list_run_files).to eq ['tex/generic/hyph-utf8/loadhyph/loadhyph-ka.tex', + 'tex/generic/hyph-utf8/patterns/tex/hyph-ka.tex', + 'tex/generic/hyph-utf8/patterns/ptex/hyph-ka.t8m.tex', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.chr.txt', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.pat.txt', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.hyp.txt', + 'tex/generic/hyph-utf8/patterns/txt/hyph-ka.lic.txt'] + end + end + + describe '#patterns_line' do + it "returns the patterns line for TLPSRC" do + expect(Language.new('tk').patterns_line).to eq "file_patterns=hyph-tk.pat.txt" + end + + it "returns two files for Serbian" do + expect(Language.new('sh-cyrl').patterns_line).to eq "file_patterns=hyph-sh-latn.pat.txt,hyph-sh-cyrl.pat.txt" + end + end + + describe '#exceptions_line' do + it "returns the exceptions line for TLPSRC" do + expect(Language.new('nn').exceptions_line).to eq "file_exceptions=hyph-nn.hyp.txt" + end + + it "returns two files for Serbian" do + expect(Language.new('sh-latn').exceptions_line).to eq "file_exceptions=hyph-sh-latn.hyp.txt,hyph-sh-cyrl.hyp.txt" + end + end + + describe '#extract_apostrophes' do + it "returns the list of patterns with apostrophes" do + expect(Language.new('af').extract_apostrophes[:with_apostrophe]).to eq ['.af6ro’', '.a7fro’s', '.l’7etji', '.m’7etji', '.r’7etji', 's’9ie.', 'x’9ie.'] + end + + it "returns nil otherwise" do + expect(Language.new('nl').extract_apostrophes[:with_apostrophe]).to be_nil + end + end + + describe '#extract_characters' do + it "extracts the list of characters with in lowercase and uppercase" do + expect(Language.new('id').extract_characters).to eq (('a'..'z').to_a - ['x']).map { |c| c + c.upcase } + end + end + + describe '#comments_and_licence' do + it "extracts the header" do + expect(Language.new('kmr').comments_and_licence).to match /^% title: Hyphenation patterns for Kurmanji \(Northern Kurdish\).*The patterns are generated by patgen from a word list of approx\. 2500\n% hyphenated words provided by Medeni Shemdê$/m + end + end + + describe '#list_synonyms' do + it "returns a list of synonyms" do + expect(Language.new('es').list_synonyms).to eq ' synonyms=espanol' + end + end + + describe '#list_hyphenmins' do + it "returns the hyphenmins" do + expect(Language.new('gl').list_hyphenmins).to eq "lefthyphenmin=2 \\\n\trighthyphenmin=2" + end + end + + describe '#message' do + it "returns the message to be displayed to TeX users" do + expect(Language.new('de-1996').message).to eq 'German hyphenation patterns (reformed orthography)' + end + end + + describe '#legacy_patterns' do + it "returns the file name of the legacy patterns" do + expect(Language.new('de-1996').legacy_patterns).to eq 'dehyphn.tex' + end + end + + describe '#use_old_loader' do + it "says whether to use old loader or not" do + expect(Language.new('de-1996').use_old_loader).to be_nil + end + + it "returns true if applicable" do + # pending "Refactor in progress" + expect(Language.new('no').use_old_loader).to be_truthy + end + end + + describe '#use_old_patterns_comment' do + it "returns true if language uses old patterns" do + expect(Language.new('de-1996').use_old_patterns_comment).to be_truthy + end + + it "returns nil otherwise" do + expect(Language.new('no').use_old_patterns_comment).to be_nil + end + + it "is a string" do + expect(Language.new('cop').use_old_patterns_comment).to be_a String + end + end + + describe '#description' do + it "returns the long description" do + text = <<-EOD + Hyphenation patterns for Dutch in T1/EC and UTF-8 encodings. + These patterns don't handle cases like 'menuutje' > 'menu-tje', + and don't hyphenate words that have different hyphenations according + to their meaning. + EOD + description = text.split("\n").map(&:strip).join("\n") + expect(Language.new('nl').description).to eq description + end + + it "returns nil for new spelling German, for some reason" do + expect(Language.new('de-1996').description).to be_nil + end + end + + describe '#package' do + it "returns the package name if applicable" do + expect(Language.new('bn').package).to eq 'indic' + end + + it "returns nil in most cases" do + expect(Language.new('bg').package).to be_nil + end + + it "calls #extract_metadata first" do + hindi = Language.new('hi') + expect(hindi).to receive :extract_metadata + hindi.package + end + end + end +end + +describe Package do + let(:latin) { Package.find('latin') } + let(:german) { Package.find('german') } + let(:hungarian) { Package.find('hungarian') } + let(:norwegian) { Package.find('norwegian') } + + describe "Instance variables" do + it "has a @name" do + expect(latin.name).to eq 'latin' + end + end + + describe '.new' do + it "initialises @languages to an empty array" do + package = Package.new('mongolian') + expect(package.instance_variable_get :@languages).to eq [] + end + end + + describe "#description_s" do + it "returns the short description" do + expect(latin.description_s).to eq 'Latin hyphenation patterns' + end + end + + describe "#description" do + it "returns the long description" do + expect(latin.description).to match /^Hyphenation patterns for.*modern spelling.*medieval spelling/m + expect(latin.description).to match /Classical Latin/ + expect(latin.description).to match /Liturgical Latin/ + end + + it "returns the long package description" do + text = <<-EOD + Hyphenation patterns for German in T1/EC and UTF-8 encodings, + for traditional and reformed spelling, including Swiss German. + The package includes the latest patterns from dehyph-exptl + (known to TeX under names 'german', 'ngerman' and 'swissgerman'), + however 8-bit engines still load old versions of patterns + for 'german' and 'ngerman' for backward-compatibility reasons. + Swiss German patterns are suitable for Swiss Standard German + (Hochdeutsch) not the Alemannic dialects spoken in Switzerland + (Schwyzerduetsch). + There are no known patterns for written Schwyzerduetsch. + EOD + description = text.split("\n").map(&:strip).join("\n") + expect(Package.find('german').description).to match description + end + end + + describe '#add_language' do + it "adds a language to the package" do + package = Package.new('indic') + assamese = Language.new('as') + package.add_language(assamese) + expect(package.instance_variable_get(:@languages).first).to eq assamese + end + end + + describe "#languages" do + it "returns the list of languages" do + expect(latin.languages.map(&:bcp47).sort).to eq ['la', 'la-x-classic', 'la-x-liturgic'] + end + end + + describe "#has_dependency?" do + it "returns the external dependencies" do + expect(german.has_dependency?).to eq "dehyph" + end + + it "returns nil for most packages" do + expect(hungarian.has_dependency?).to be_nil + end + end + + describe "#list_dependencies" do + it "lists the dependencies" do + # FIXME Should return ['depend hyphen-base', 'depend hyph-utf8', 'depend dehyph'] or nothing + expect(german.has_dependency?).to eq 'dehyph' + end + end + + describe "#list_support_files" do # FIXME? list_non_run_files + it "lists doc and source files" do + expect(hungarian.list_support_files('doc')).to eq ['doc/generic/hyph-utf8/languages/hu', 'doc/generic/huhyphen'] + # FIXME Should return ['texmf-dist/doc/generic/huhyphen', 'texmf-dist/doc/generic/hyph-utf8/languages/hu'] or nothing + end + end + + describe "#list_run_files" do + it "lists the run-time files" do + # pending "it crashes ;-)" + norwegian_run = norwegian.list_run_files + expect(norwegian_run.count).to eq 15 + expect(norwegian_run.select { |f| f =~ /tex\/hyph-[^\.]*\.tex$/ }).to eq ['tex/generic/hyph-utf8/patterns/tex/hyph-no.tex', + 'tex/generic/hyph-utf8/patterns/tex/hyph-nb.tex', + 'tex/generic/hyph-utf8/patterns/tex/hyph-nn.tex'] + end + end + + describe '#<=>' do + it "compares two Package’s" do + # puts hungarian.class, german.class + expect(hungarian.<=> german).to eq 1 + end + end + + describe '#find' do + it "returns the package with that name" do + expect(Package.find('norwegian')).to eq norwegian + end + end +end -- cgit v1.2.3