diff options
Diffstat (limited to 'Master/texmf-dist/source/generic/hyph-utf8/lib')
9 files changed, 1394 insertions, 0 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb new file mode 100644 index 00000000000..2259f2ff1d4 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen.rb @@ -0,0 +1 @@ +require_relative 'hyphen/language' diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml new file mode 100644 index 00000000000..f96530da39c --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/authors.yml @@ -0,0 +1,452 @@ +--- +donald_knuth: !ruby/object:TeX::Hyphen::Author + name: Donald + surname: Knuth + email: + contacted1: false + contacted2: false +peter_heslin: !ruby/object:TeX::Hyphen::Author + name: Peter + surname: Heslin + email: + contacted1: false + contacted2: false +dimitrios_filippou: !ruby/object:TeX::Hyphen::Author + name: Dimitrios + surname: Filippou + email: dimitrios{dot}filippou{at}riotinto{dot}com + contacted1: true + contacted2: true +claudio_beccari: !ruby/object:TeX::Hyphen::Author + name: Claudio + surname: Beccari + email: claudio{dot}beccari{at}gmail{dot}com + contacted1: true + contacted2: true +juan_aguirregabiria: !ruby/object:TeX::Hyphen::Author + name: Juan M. + surname: Aguirregabiria + email: juanmari{dot}aguirregabiria{at}ehu.es + contacted1: true + contacted2: true +igor_marinovic: !ruby/object:TeX::Hyphen::Author + name: Igor + surname: Marinović + email: marinowski{at}gmail.com + contacted1: true + contacted2: true +tilla_fick: !ruby/object:TeX::Hyphen::Author + name: Tilla + surname: Fick + email: fick{dot}tilla{at}gmail{dot}com + contacted1: true + contacted2: true +chris_swanepoel: !ruby/object:TeX::Hyphen::Author + name: Chris + surname: Swanepoel + email: cj{at}swanepoel{dot}net + contacted1: true + contacted2: true +matjaz_vrecko: !ruby/object:TeX::Hyphen::Author + name: Matjaž + surname: Vrečko + email: matjaz{at}mg-soft{dot}si + contacted1: true + contacted2: true +goncal_badenes: !ruby/object:TeX::Hyphen::Author + name: Gonçal + surname: Badenes + email: g{dot}badenes{at}ieee.org + contacted1: false + contacted2: false +pavel_sevecek: !ruby/object:TeX::Hyphen::Author + name: Pavel + surname: Ševeček + email: pavel{at}lingea{dot}cz + contacted1: false + contacted2: false +# email doesn't work +jana_chlebikova: !ruby/object:TeX::Hyphen::Author + name: Jana + surname: Chlebíková + email: chlebikj{at}dcs{dot}fmph{dot}uniba{dot}sk + contacted1: false + contacted2: false +yannis_haralambous: !ruby/object:TeX::Hyphen::Author + name: Yannis + surname: Haralambous + email: yannis{dot}haralambous{at}telecom-bretagne{dot}eu + contacted1: true + contacted2: false +frank_jensen: !ruby/object:TeX::Hyphen::Author + name: Frank + surname: Jensen + email: frank{dot}jensen{at}hugin{dot}com + contacted1: true + contacted2: true +sergei_pokrovsky: !ruby/object:TeX::Hyphen::Author + name: Sergei + surname: Pokrovsky + email: sergio{dot}pokrovskij{at}gmail{dot}com + contacted1: true + contacted2: true +javier_bezos: !ruby/object:TeX::Hyphen::Author + name: Javier + surname: Bezos + email: jbezos{at}tex-tipografia{dot}com + contacted1: true + contacted2: true +een_saar: !ruby/object:TeX::Hyphen::Author + name: Enn + surname: Saar + email: saar{at}aai{dot}ee + contacted1: false + contacted2: false +dejan_muhamedagic: !ruby/object:TeX::Hyphen::Author + name: Dejan + surname: Muhamedagić + email: dejan{at}hello-penguin{dot}com + contacted1: true + contacted2: true +brian_wilson: !ruby/object:TeX::Hyphen::Author + name: Brian + surname: Wilson + email: bountonw{at}gmail{dot}com + contacted1: true + contacted2: true +arthur_reutenauer: !ruby/object:TeX::Hyphen::Author + name: Arthur + surname: Reutenauer + email: arthur{dot}reutenauer{at}normalesup{dot}org + contacted1: true + contacted2: true +mojca_miklavec: !ruby/object:TeX::Hyphen::Author + name: Mojca + surname: Miklavec + email: mojca{dot}miklavec{dot}lists{at}gmail{dot}com + contacted1: true + contacted2: true +santhosh_thottingal: !ruby/object:TeX::Hyphen::Author + name: Santhosh + surname: Thottingal + email: santhosh{dot}thottingal{at}gmail{dot}com> + contacted1: true + contacted2: true +# email doesn't work +yves_codet: !ruby/object:TeX::Hyphen::Author + name: Yves + surname: Codet + email: ycodet{at}club-internet{dot}fr + contacted1: true + contacted2: true +rune_kleveland: !ruby/object:TeX::Hyphen::Author + name: Rune + surname: Kleveland + email: + contacted1: false + contacted2: false +# email doesn't work +ole_michael_selberg: !ruby/object:TeX::Hyphen::Author + name: Ole Michael + surname: Selberg + email: o{dot}m{dot}selberg{at}c2i{dot}net + contacted1: true + contacted2: true +dorjgotov_batmunkh: !ruby/object:TeX::Hyphen::Author + name: Dorjgotov + surname: Batmunkh + email: bataak{at}gmail{dot}com + contacted1: true + contacted2: true +nazar_annagurban: !ruby/object:TeX::Hyphen::Author + name: Nazar + surname: Annagurban + email: nazartm{at}gmail{dot}com + contacted1: false + contacted2: false +jan_michael_rynning: !ruby/object:TeX::Hyphen::Author + name: Jan Michael + surname: Rynning + email: + contacted1: false + contacted2: false +eduard_werner: !ruby/object:TeX::Hyphen::Author + name: Eduard + surname: Werner + email: edi{dot}werner{at}gmx{dot}de + contacted1: false + contacted2: false +werner_lemberg: !ruby/object:TeX::Hyphen::Author + name: Werner + surname: Lemberg + email: wl{at}gnu{dot}org + contacted1: true + contacted2: true +# email doesn't work +pedro_j_de_rezende: !ruby/object:TeX::Hyphen::Author + name: Pedro J. + surname: de Rezende + email: rezende{at}ddc{dot}unicamp{dot}br + contacted1: false + contacted2: false +j_joao_dias_almeida: !ruby/object:TeX::Hyphen::Author + name: J. Joao + surname: Dias Almeida + email: jj{at}di{dot}uminho{dot}pt + contacted1: + contacted2: +# email doesn't work +piet_tutelaers: !ruby/object:TeX::Hyphen::Author + name: Piet + surname: Tutelaers + email: p{dot}t{dot}h{dot}tutelaers{at}tue{dot}nl + contacted1: false + contacted2: false +vytas_statulevicius: !ruby/object:TeX::Hyphen::Author + name: Vytas + surname: Statulevičius + email: vytas{at}vtex{dot}nl + contacted1: false + contacted2: false +sigitas_tolusis: !ruby/object:TeX::Hyphen::Author + name: Sigitas + surname: Tolušis + email: sigitas{at}vtex{dot}lt + contacted1: false + contacted2: false +janis_vilims: !ruby/object:TeX::Hyphen::Author + name: Janis + surname: Vilims + email: jvilims{at}apollo{dot}lv + contacted1: false + contacted2: false +joerg_knappen: !ruby/object:TeX::Hyphen::Author + name: Jörg + surname: Knappen + email: jknappen{at}web{dot}de + contacted1: true + contacted2: true +medeni_shemde: !ruby/object:TeX::Hyphen::Author + name: Medeni + surname: Shemdê + email: + contacted1: false + contacted2: false +terry_mart: !ruby/object:TeX::Hyphen::Author + name: Terry + surname: Mart + email: mart{at}kph{dot}uni-mainz{dot}de + contacted1: false + contacted2: false +# email doesn't work +jorgen_pind: !ruby/object:TeX::Hyphen::Author + name: Jorgen + surname: Pind + email: jorgen{at}lexis{dot}hi{dot}is + contacted1: false + contacted2: false +marteinn_sverrisson: !ruby/object:TeX::Hyphen::Author + name: Marteinn + surname: Sverrisson + email: + contacted1: false + contacted2: false +# email doesn't work +kristinn_gylfason: !ruby/object:TeX::Hyphen::Author + name: Kristinn + surname: Gylfason + email: kristgy{at}ieee{dot}org + contacted1: false + contacted2: false +# email doesn't work +kevin_p_scannell: !ruby/object:TeX::Hyphen::Author + name: Kevin P. + surname: Scannell + email: scanell{at}slu{dot}edu + contacted1: false + contacted2: false +# email doesn't work +peter_kleiweg: !ruby/object:TeX::Hyphen::Author + name: Peter + surname: Kleiweg + email: p{dot}c{dot}c{dot}kleiweg{at}rug{dot}nl + contacted1: false + contacted2: false +hanna_kolodziejska: !ruby/object:TeX::Hyphen::Author + name: Hanna + surname: Kołodziejska + email: + contacted1: false + contacted2: false +boguslaw_jackowski: !ruby/object:TeX::Hyphen::Author + name: Bogusław + surname: Jackowski + email: + contacted1: true + contacted2: true +marek_rycko: !ruby/object:TeX::Hyphen::Author + name: Marek + surname: Ryćko + email: + contacted1: false + contacted2: false +vladimir_volovich: !ruby/object:TeX::Hyphen::Author + name: Vladimir + surname: Volovich + email: # TODO add e-mail address + contacted1: true + contacted2: true +alexander_i_lebedev: !ruby/object:TeX::Hyphen::Author + name: Alexander I. # Not sure were 'I' belongs + surname: Lebedev + email: swan{at}scon155{dot}phys{dot}msu{dot}su + contacted1: false + contacted2: false +# first email doesn't work +maksym_polyakov: !ruby/object:TeX::Hyphen::Author + name: Maksym + surname: Polyakov + email: polyama{at}auburn{dot}edu # Second e-mail address in ukrhypmp.tex: mpoliak@i.com.ua + contacted1: false + contacted2: false +adrian_rezus: !ruby/object:TeX::Hyphen::Author + name: Adrian + surname: Rezus + email: adriaan{at}{sci,cs}{dot}kun{dot}nl + contacted1: false + contacted2: false +# email doesn't work +sahak_petrosyan: !ruby/object:TeX::Hyphen::Author + name: Sahak + surname: Petrosyan + email: sahak{at}mit{dot}edu + contacted1: true + contacted2: true # I think "true, true" is right. Arthur +dominik_wujastyk: !ruby/object:TeX::Hyphen::Author + name: Dominik + surname: Wujastyk + email: wujastyk{at}gmail{dot}com + contacted1: false + contacted2: false +graham_toal: !ruby/object:TeX::Hyphen::Author + name: Graham + surname: Toal + email: + contacted1: false + contacted2: false +donald_e_knuth: !ruby/object:TeX::Hyphen::Author + name: Donald E. + surname: Knuth + email: # Don doesn't use e-mail ;-) + contacted1: false + contacted2: false +gerard_d_c_kuiken: !ruby/object:TeX::Hyphen::Author + name: Gerard D.C. + surname: Kuiken + email: + contacted1: false + contacted2: false +pierre_mackay: !ruby/object:TeX::Hyphen::Author + name: P. A. + surname: MacKay + email: + contacted1: true + contacted2: true +h_turgut_uyar: !ruby/object:TeX::Hyphen::Author + name: H. Turgut + surname: Uyar + email: uyar{at}itu{dot}edu{tr} + contacted1: true + contacted2: true +# email doesn't work +s_ekin_kocabas: !ruby/object:TeX::Hyphen::Author + name: S. Ekin + surname: Kocabas + email: kocabas{at}stanford{dot}edu + contacted1: true + contacted2: true +bence_nagy: !ruby/object:TeX::Hyphen::Author + name: Bence + surname: Nagy + email: nagybence{at}tipogral{dot}hu + contacted1: true + contacted2: true +kauko_saarinen: !ruby/object:TeX::Hyphen::Author + name: Kauko + surname: Saarinen + email: + contacted1: false + contacted2: false +fred_karlsson: !ruby/object:TeX::Hyphen::Author + name: Fred + surname: Karlsson + email: + contacted1: false + contacted2: false +rene_bastian: !ruby/object:TeX::Hyphen::Author # TODO make contact + name: René + surname: Bastian + email: + contacted1: false + contacted2: false +daniel_flipo: !ruby/object:TeX::Hyphen::Author # TODO make contact + name: Daniel + surname: Flipo + email: + contacted1: false + contacted2: false +bernard_gaulle: !ruby/object:TeX::Hyphen::Author # Deceased... + name: Bernard + surname: Gaulle + email: + contacted1: false + contacted2: false +theppitak_karoonboonyanan: !ruby/object:TeX::Hyphen::Author + name: Theppitak + surname: Karoonboonyanan + email: theppitak{at}gmail{dot}com + contacted1: true + contacted2: true +levan_shoshiashvili: !ruby/object:TeX::Hyphen::Author + name: Levan + surname: Shoshiashvili + email: shoshia{at}hotmail{dot}com + contacted1: true + contacted2: true +# email doesn't work +javier_mugica: !ruby/object:TeX::Hyphen::Author + name: Javier + surname: Múgica + email: javier{at}digi21{dot}eu + contacted1: true + contacted2: true +georgi_boshnakov: !ruby/object:TeX::Hyphen::Author + name: Georgi + surname: Boshnakov + email: georgi{dot}boshnakov{at}manchester{dot}ac{dot}uk + contacted1: true + contacted2: true +mike_kroutikov: !ruby/object:TeX::Hyphen::Author + name: Mike + surname: Kroutikov + email: pgmmpk{at}gmail{dot}com + contacted1: true + contacted2: true +aleksandr_andreev: !ruby/object:TeX::Hyphen::Author + name: Aleksandr + surname: Andreev + email: '' + contacted1: true + contacted2: true +maksim_salau: !ruby/object:TeX::Hyphen::Author + name: Maksim + surname: Salau + email: maksim{dot}salau{at}gmail{dot}com + contacted1: true + contacted2: false +wie_ming_ang: !ruby/object:TeX::Hyphen::Author + name: Wie-Ming + surname: Cittānurakkho Bhikkhu + email: wiemingang{at}gmail{dot}com + contacted1: true + contacted2: true diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb new file mode 100644 index 00000000000..3d3437d74b5 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb @@ -0,0 +1,364 @@ +require 'yaml' +require 'hydra' +require 'byebug' unless ENV['RACK_ENV'] == "production" + +require_relative 'path' + +class String + def superstrip + strip.gsub /%.*$/, '' + end + + def supersplit + strip.gsub(/\s+/m,"\n").split("\n") + end + + def safe + gsub /[\s-]/, '' + end + + def titlecase + split.map(&:capitalize).join(' ') + end +end + +module TeX + module Hyphen + class InvalidMetadata < StandardError; end + class NoAuthor < InvalidMetadata; end + class NoLicence < InvalidMetadata; end + + include PATH + + class Author + def initialize(name,surname,email,contacted1,contacted2) + @name = name + @surname = surname + @email = email + # this mostly means if email has been recently checked + @contacted1 = contacted1 + # this means if we made more cooperation with author, + # exchanging patches etc. + @contacted2 = contacted2 + end + + attr_reader :name, :surname, :email + + def self.authors + @@authors ||= YAML::load File.read File.expand_path 'authors.yml', __dir__ + end + + def self.all + authors.values + end + + def self.[] a + authors[a] + end + end + + class Language + attr_reader :bcp47 + + @@eohmarker = '=' * 42 + + def iso639 + @bcp47.split('-').first + end + + # FIXME Find something better than all that extract_metadata unless @blah nonsens + def babelname + extract_metadata unless @babelname + @babelname + end + + def description + extract_metadata unless @description + @description + end + + def use_old_loader + extract_metadata unless @use_old_loader + @use_old_loader + end + + def use_old_patterns_comment + extract_metadata unless @use_old_patterns_comment + @use_old_patterns_comment + end + + def legacy_patterns + extract_metadata unless @legacy_patterns + @legacy_patterns + end + + def message + extract_metadata unless @message + @message + end + + def known_bugs + extract_metadata unless @known_bugs + @known_bugs + end + + def initialize(bcp47 = nil) + @bcp47 = bcp47 + end + + def self.languages + @@languages ||= Dir.glob(File.join(PATH::TEX, 'hyph-*.tex')).inject [] do |languages, texfile| + bcp47 = texfile.gsub /^.*\/hyph-(.*)\.tex$/, '\1' + # languages << [bcp47, Language.new(bcp47)] + languages << [bcp47, nil] + end.to_h + end + @@languages = Language.languages + + def self.all + @@all ||= languages.map do |bcp47, language| + next if bcp47 == 'sr-cyrl' # FIXME Remove later + @@languages[bcp47] ||= Language.new(bcp47) + end.compact + end + + def self.all_by_iso639 + all.inject(Hash.new) do |maintags, language| + (maintags[language.iso639] ||= Array.new) << language + maintags + end + end + + def self.find_by_bcp47(bcp47) + languages[bcp47] + end + + def private_use? + @bcp47.length == 3 || @bcp47[3] == '-' and @bcp47[0] == 'q' and ('a'..'t').include? @bcp47[1] + end + + # TODO This should probably become “macrolanguage name” or something similar + # @@displaynames = { + # 'el' => 'Greek', + # 'nb' => 'Norwegian', + # 'nn' => 'Norwegian', + # 'sh' => 'Serbian', + # } + + def licences + extract_metadata unless @licences + @licences + end + + def lefthyphenmin + extract_metadata unless @lefthyphenmin + @lefthyphenmin + end + + def righthyphenmin + extract_metadata unless @righthyphenmin + @righthyphenmin + end + + # Strictly speaking a misnomer, because grc-x-ibycus should also return true. + # But useful for a number of apostrophe-related routines + def isgreek? + ['grc', 'el-polyton', 'el-monoton'].include? @bcp47 + end + + def serbian? + @bcp47 =~ /^sh-/ + end + + def italic? + ['it', 'pms', 'rm'].include? @bcp47 + end + + def has_apostrophes? + begin + !isgreek? && patterns.any? { |p| p =~ /'/ } + rescue Errno::ENOENT + false + end + end + + def has_hyphens? + begin + patterns.any? { |p| p =~ /-/ } + rescue Errno::ENOENT + false + end + end + + def extract_apostrophes + plain, with_apostrophe = Array.new, nil + + patterns.each do |pattern| + plain << pattern + if pattern =~ /'/ && !isgreek? + pattern_with_apostrophe = pattern.gsub(/'/,"’") + plain << pattern_with_apostrophe + (with_apostrophe ||= []) << pattern_with_apostrophe + end + end + + { plain: plain, with_apostrophe: with_apostrophe } + end + + def extract_characters # FIXME Turkish and others iİ; check Church Slavonic + characters = Array.new + + characters_indexes = patterns.join.gsub(/[.0-9]/,'').unpack('U*').sort.uniq + characters_indexes.each do |c| + ch = [c].pack('U') + characters << ch + ch.upcase + characters << "’’" if ch == "'" && !isgreek? + end + + characters + end + + def comments_and_licence # Major TODO extract everything into YAML, and write .yml + @comments_and_licence ||= readtexfile.gsub(/(.*)\\patterns.*/m,'\1') + end + + def authors + extract_metadata unless @authors + @authors + end + + def github_link + sprintf 'https://github.com/hyphenation/tex-hyphen/tree/master/hyph-utf8/tex/generic/hyph-utf8/patterns/tex/hyph-%s.tex', @bcp47 + end + + def <=>(other) + # TODO Remove when practical + a = self.babelname + b = other.babelname + + if a == 'serbian' && b == 'serbianc' + return -1 + elsif a == 'serbianc' && b == 'serbian' + return 1 + end + + self.bcp47 <=> other.bcp47 + end + + @@texfile = Hash.new + def readtexfile(bcp47 = @bcp47) + begin + @@texfile[bcp47] ||= File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', bcp47))) + rescue Errno::ENOENT + @@texfile[bcp47] = "" + end + end + + def patterns + @patterns ||= if @bcp47 == 'eo' then + readtexfile.superstrip. + gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1'). + # + gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.'). + gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.'). + gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.'). + # + supersplit + else + readtexfile(if ['nb', 'nn'].include? @bcp47 then 'no' else @bcp47 end).superstrip. + gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1'). + supersplit + end + end + + def exceptions + unless @exceptions + if readtexfile.superstrip.index('\hyphenation') + @exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit + if @exceptions != "" + @hyphenation = @exceptions.inject [] do |exceptions, exception| + # byebug unless exception + exceptions << [exception.gsub('-', ''), exception] + end.to_h + else + @hyphenation = { } + end + else + @exceptions = "" + @hyphenation = { } + end + end + + @exceptions + end + + def hyphenate(word) + exceptions + return @hyphenation[word] if @hyphenation[word] # FIXME Better name + + unless @hydra + begin + # byebug + metadata = extract_metadata + @hydra = Hydra.new patterns, :lax, '', metadata + rescue InvalidMetadata + @hydra = Hydra.new patterns + end + end + @hydra.showhyphens(word) + end + + def extract_metadata + header = "" + File.read(File.join(PATH::TEX, sprintf('hyph-%s.tex', @bcp47))).each_line do |line| + break if line =~ /\\patterns|#{@@eohmarker}/ + header += line.gsub(/^% /, '').gsub(/%.*/, '') + end + begin + metadata = YAML::load header + raise InvalidMetadata unless metadata.is_a? Hash + rescue Psych::SyntaxError + raise InvalidMetadata + end + + @name = metadata.dig('language', 'name') + @lefthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'left') || metadata.dig('hyphenmins', 'generation', 'left') + @righthyphenmin = metadata.dig('hyphenmins', 'typesetting', 'right') || metadata.dig('hyphenmins', 'generation', 'right') + # FIXME That’s not nice + if respond_to? :synonyms + @synonyms = metadata.dig('texlive', 'synonyms') || [] + @encoding = metadata.dig('texlive', 'encoding') + end + @message = metadata.dig('texlive', 'message') + @legacy_patterns = metadata.dig('texlive', 'legacy_patterns') + @use_old_loader = metadata.dig('texlive', 'use_old_loader') + @use_old_patterns_comment = metadata.dig('texlive', 'use_old_patterns_comment') + @description = metadata.dig('texlive', 'description') + @babelname = metadata.dig('texlive', 'babelname') + @package = metadata.dig('texlive', 'package') + licences = metadata.dig('licence') + raise NoLicence unless licences + licences = [licences] unless licences.is_a? Array + @licences = licences.map do |licence| + raise bcp47 if licence.is_a? String + next if licence.values == [nil] + licence.dig('name') || 'custom' + end.compact + authors = metadata.dig('authors') + if authors + @authors = authors.map do |author| + author['name'] + end + else + @authors = nil + raise NoAuthor.new # FIXME + end + @known_bugs = metadata.dig('known_bugs') + + # raise NoAuthor unless @authors && @authors.count > 0 # TODO Later ;-) AR 2018-09-13 + + metadata + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml new file mode 100644 index 00000000000..bc029739388 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml @@ -0,0 +1,94 @@ +german: + dependency: dehyph + description: |- + Hyphenation patterns for German in T1/EC and UTF-8 encodings, + for traditional and reformed spelling, including Swiss German. + The package includes the latest patterns from dehyph-exptl + (known to TeX under names 'german', 'ngerman' and 'swissgerman'), + however 8-bit engines still load old versions of patterns + for 'german' and 'ngerman' for backward-compatibility reasons. + Swiss German patterns are suitable for Swiss Standard German + (Hochdeutsch) not the Alemannic dialects spoken in Switzerland + (Schwyzerduetsch). + There are no known patterns for written Schwyzerduetsch. +# for Russian and Ukrainian (until we implement the new functionality at least) +russian: + dependency: ruhyphen +ukrainian: + dependency: ukrhyph +greek: + doc: doc/generic/elhyphen + shortdesc: Modern Greek + description: |- + Hyphenation patterns for Modern Greek in monotonic and polytonic + spelling in LGR and UTF-8 encodings. Patterns in UTF-8 use two code + positions for each of the vowels with acute accent (a.k.a tonos, + oxia), e.g., U+03AC, U+1F71 for alpha. +hungarian: + doc: doc/generic/huhyphen +arabic: + shortdesc: (No) Arabic +farsi: + shortdesc: (No) Persian +chinese: + shortdesc: Chinese pinyin + description: |- + Hyphenation patterns for unaccented transliterated Mandarin Chinese + (pinyin) in T1/EC and UTF-8 encodings. The latter can hyphenate pinyin + with or without tone markers; the former only without. +norwegian: + shortdesc: Norwegian Bokmal and Nynorsk + description: |- + Hyphenation patterns for Norwegian Bokmal and Nynorsk in T1/EC and + UTF-8 encodings. +churchslavonic: + shortdesc: Church Slavonic +uppersorbian: + shortdesc: Upper Sorbian +ethiopic: + shortdesc_full: Hyphenation patterns for Ethiopic scripts +mongolian: + shortdesc_full: Mongolian hyphenation patterns in Cyrillic script + description: |- + Hyphenation patterns for Mongolian in T2A, LMC and UTF-8 encodings. + LMC encoding is used in MonTeX. The package includes two sets of + patterns that will hopefully be merged in future. +latin: + description: |- + Hyphenation patterns for Latin in T1/EC and UTF-8 encodings, + mainly in modern spelling (u when u is needed and v when v is needed), + medieval spelling with the ligatures \ae and \oe and the (uncial) + lowercase 'v' written as a 'u' is also supported. Apparently + there is no conflict between the patterns of modern Latin and + those of medieval Latin. + Hyphenation patterns for the Classical Latin in T1/EC and UTF-8 + encodings. Classical Latin hyphenation patterns are different from + those of 'plain' Latin, the latter being more adapted to modern Latin. + Hyphenation patterns for the Liturgical Latin in T1/EC and UTF-8 + encodings. +english: + description: |- + Additional hyphenation patterns for American and British + English in ASCII encoding. The American English patterns + (usenglishmax) greatly extend the standard patterns from Knuth + to find many additional hyphenation points. British English + hyphenation is completely different from US English, so has its + own set of patterns. +indic: + description: |- + Hyphenation patterns for Assamese, Bengali, Gujarati, Hindi, Kannada, + Malayalam, Marathi, Oriya, Panjabi, Tamil and Telugu for Unicode + engines. +"ancient greek": + description: |- + Hyphenation patterns for Ancient Greek in LGR and UTF-8 encodings, + including support for (obsolete) Ibycus font encoding. + Patterns in UTF-8 use two code positions for each of the vowels with + acute accent (a.k.a tonos, oxia), e.g., U+03AE, U+1F75 for eta. +serbian: + description: |- + Hyphenation patterns for Serbian in T1/EC, T2A and UTF-8 encodings. + For 8-bit engines the patterns are available separately as 'serbian' + in T1/EC encoding for Latin script and 'serbianc' in T2A encoding for + Cyrillic script. Unicode engines should only use 'serbian' + which has patterns in both scripts combined. diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb new file mode 100644 index 00000000000..fda422b11a8 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/path.rb @@ -0,0 +1,25 @@ +module TeX + module Hyphen + module PATH + ROOT = File.expand_path('../../../../../../../..', __FILE__) + + TeXROOT = File.join(ROOT, 'hyph-utf8') + TeX_GENERIC = File.join(TeXROOT, 'tex', 'generic') + PAT = File.join(TeX_GENERIC, 'hyph-utf8', 'patterns') + TXT = File.join(PAT, 'txt') + TEX = File.join(PAT, 'tex') + PTEX = File.join(PAT, 'ptex') + QUOTE = File.join(PAT, 'quote') + LOADER = File.join(TeX_GENERIC, 'hyph-utf8', 'loadhyph') + + SUPPORT = File.join(TeXROOT, '%s', 'generic', 'hyph-utf8', 'languages', '*') + + HYPHU8 = File.join('tex', 'generic', 'hyph-utf8') + + TL = File.join(ROOT, 'TL') + LANGUAGE_DAT = File.join(PATH::TL, 'texmf-dist', 'tex', 'generic', 'config') + # hyphen-foo.tlpsrc for TeX Live + TLPSRC = File.join(PATH::TL, 'tlpkg', 'tlpsrc') + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb new file mode 100644 index 00000000000..45dd33da047 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive.rb @@ -0,0 +1,22 @@ +require_relative 'texlive/source' +require_relative 'texlive/loader' +require_relative 'texlive/package' + +module TeX + module Hyphen + module TeXLive + def synonyms + extract_metadata unless @synonyms + @synonyms + end + + def encoding + extract_metadata unless @encoding + @encoding + end + + include Source + include Loader + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb new file mode 100644 index 00000000000..2fcb3945d30 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb @@ -0,0 +1,193 @@ +module TeX + module Hyphen + module TeXLive + module Loader + + def unicode_only? # TODO Spec out + ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47 + end + + def string_enc + (encoding == nil) ? "" : encoding.upcase + " " + end + + def lcchars + if isgreek? + # some catcodes for XeTeX + return { + lccode: { + 0x0027 => '\'', + 0x2019 => '’', + 0x02BC => 'ʼ', + 0x1FBD => '᾽', + 0x1FBF => '᾿', + } + } + end + + return { } unless unicode_only? + + default = { + comment: 'Set \lccode for ZWNJ and ZWJ.', + lccode: { + 0x200C => false, + 0x200D => false, + } + } + + { + 'mul-ethi' => { + comment: 'Set \lccode for Ethiopian word space.', + lccode: { + 0x1361 => false, + 0x1362 => false, + } + }, + 'cu' => { + comment: 'fix lccodes for some characters (they were recently included in Unicode)', + lccode: { + 0x1C82 => 'sharp o in lowercase "uk"', + 0x1DF6 => false, + 0x1DF7 => false, + 0x1DF8 => false, + 0x1DF9 => false, + 0xA69E => false, + 0x1C86 => false, + 0xA67E => false, + 0xFE2E => false, + 0xFE2F => false, + } + }, + 'sa' => { + comment: 'Set \lccode for ZWNJ and ZWJ.', + lccode: { + 0x200C => false, + 0x200D => "\n% Set \\lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.", + 0x0CF1 => false, + 0x0CF2 => false, + } + }, + }[@bcp47] || default + end + + def utf8_engine_message + if serbian? + 'UTF-8 Serbian hyphenation patterns' + else + sprintf('UTF-8 %s', message) + end + end + + def non_utf8_engine_message + if unicode_only? + sprintf('No %s - only for Unicode engines', message) + else + sprintf('%s%s', string_enc, message) + end + end + + def engine_message(engine) + if engine == 'UTF-8' + { + comment: 'Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument', + message: utf8_engine_message + } + else # engine is 8-bit or pTeX + { + comment: engine + + if engine == '8-bit' then " engine (such as TeX or pdfTeX)" else "" end, + message: non_utf8_engine_message + } + end + end + + def input_8bit_file + if @bcp47 == 'la-x-liturgic' + { + input: [pTeX_patterns] + } + elsif use_old_patterns_comment + # explain why we are still using the old patterns + { + comment: use_old_patterns_comment, + input: [legacy_patterns] + } + elsif !italic? + { + input: [sprintf('conv-utf8-%s.tex', encoding), sprintf('hyph-%s.tex', @bcp47)] + } + else + { + input: [sprintf('hyph-%s.tex', @bcp47)] + } + end + end + + def input_utf8_line + if serbian? + { + comment: 'We load both scripts at the same time to simplify usage', + input: ['hyph-sh-latn.tex', 'hyph-sh-cyrl.tex'] + } + else + { + input: [sprintf('hyph-%s.tex', @bcp47)] + } + end + end + + def format_inputs(specification) + if specification.is_a? Array + return specification.map do |hash| + format_inputs(hash) + end + end + + byebug unless specification.is_a? Hash + comment = specification[:comment] + message = specification[:message] + if comment then [sprintf('%% %s', comment)] else [] end + + if message then [sprintf('\\message{%s}', message)] else [] end + + (specification[:lccode] || []).map do |code, comment| + sprintf '\\lccode"%04X="%04X%s', code, code, if comment then sprintf ' %% %s', comment else '' end + end + + (specification[:input] || []).map do |input| + sprintf '\\input %s', input + end + end + + def input_line(engine) + if engine == '8-bit' + input_8bit_file + elsif engine == 'pTeX' + { input: [pTeX_patterns] } + elsif engine == 'UTF-8' + input_utf8_line + end + end + + def utf8_chunk + [ + engine_message('UTF-8'), + # lccodes + lcchars, + input_line('UTF-8'), + if has_apostrophes? then { input: [sprintf('hyph-quote-%s.tex', bcp47)] } end + ].compact + end + + def nonutf8_chunk(engine) + [engine_message(engine), unless unicode_only? then input_line(engine) end].compact + end + + def loadhyph + if use_old_loader + legacy_patterns + else + sprintf 'loadhyph-%s.tex', @bcp47.gsub(/^sh-/, 'sr-') + end + end + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb new file mode 100644 index 00000000000..69481a41774 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb @@ -0,0 +1,126 @@ +module TeX + module Hyphen + module TeXLive + class Package + attr_reader :name + + @@metadata = YAML::load File.read File.expand_path '../packages.yml', __dir__ + + def initialize(name) + @name = name + @languages = [] + end + + def add_language(language) + @languages << language + @languages.sort! + end + + def languages + @languages + end + + def self.packages + # a hash with the names of TeX Live packages, either individual language names, + # or an array of languages as the value + @@packages ||= Language.all.inject(Hash.new) do |packages, language| + if name = language.package || language.babelname # Package name, otherwise single language + packages[name] ||= Package.new(name) # Create new package if necessary + packages[name].add_language language + end + + packages + end + end + + def self.all + packages.values + end + + def description_s + override = @@metadata.dig(@name, 'shortdesc_full') + return override if override + + leader = @@metadata.dig(@name, 'shortdesc') || @name.titlecase + sprintf '%s hyphenation patterns', leader + end + + def description + description = @@metadata.dig(@name, 'description') + if description + description + elsif @languages.count == 1 + @languages.first.description + else + '' + end + end + + def has_dependency? + @@metadata.dig(name, 'dependency') + end + + def list_dependencies + dependencies = [ + "depend hyphen-base", + "depend hyph-utf8", + ] + + # external dependencies + if dependency = has_dependency? + dependencies << sprintf("depend %s", dependency) + end + + dependencies + end + + def list_support_files(type) # type is ‘doc’ or ‘source’ + # Cache directory contents + (@dirlist ||= { })[type] ||= Dir.glob(sprintf(PATH::SUPPORT, type)).select do |file| + File.directory?(file) + end.map do |dir| + dir.gsub /^.*\//, '' + end + + files = (languages.map(&:bcp47) & @dirlist[type]).map do |bcp47| + sprintf("%s/generic/hyph-utf8/languages/%s", type, bcp47) + end + + if special = @@metadata.dig(name, type) + files << special + end + + files + end + + def list_run_files + files = [] + files << "tex/generic/hyph-utf8/patterns/tex/hyph-no.tex" if name == "norwegian" + + files = languages.inject(files) do |files, language| + files + language.list_run_files + end + + unless has_dependency? + languages.each do |language| + if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop' + files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns) + end + end + end + + files + end + + def <=>(other) + name <=> other.name + end + + # TODO Spec out + def self.find(name) + packages[name] + end + end + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb new file mode 100644 index 00000000000..61292dcd3bb --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb @@ -0,0 +1,117 @@ +module TeX + module Hyphen + module TeXLive + module Source + def list_synonyms + if synonyms && synonyms.length > 0 + sprintf " synonyms=%s", synonyms.join(',') + else + '' + end + end + + def list_hyphenmins + lmin = lefthyphenmin + rmin = righthyphenmin + sprintf "lefthyphenmin=%s \\\n\trighthyphenmin=%s", lmin, rmin + end + + # ext: 'pat' or 'hyp' + # filetype: 'patterns' or 'exceptions' + def plain_text_line(ext, filetype) # TODO Figure out if we will sr-cyrl to be generated again + return "" if ['ar', 'fa', 'grc-x-ibycus', 'mn-cyrl-x-lmc'].include? @bcp47 + + if @bcp47 =~ /^sh-/ + # TODO Warning AR 2018-09-12 + filename = sprintf 'hyph-sh-latn.%s.txt,hyph-sh-cyrl.%s.txt', ext, ext + else + filename = sprintf 'hyph-%s.%s.txt', @bcp47, ext + filepath = File.join(PATH::TXT, filename) + # check for existence of file and that it’s not empty + unless File.file?(filepath) && File.read(filepath).length > 0 + # if the file we were looking for was a pattern file, something’s wrong + if ext == 'pat' + raise sprintf("There is some problem with plain patterns for language [%s]!!!", @bcp47) + else # the file is simply an exception file and we’re happy + filename = '' # And we return and empty file name after all + end + end + end + + sprintf "file_%s=%s", filetype, filename + end + + def exceptions_line + plain_text_line('hyp', 'exceptions') + end + + def patterns_line + plain_text_line('pat', 'patterns') + end + + def pTeX_patterns + if italic? + sprintf 'hyph-%s.tex', @bcp47 + elsif encoding + sprintf 'hyph-%s.%s.tex', @bcp47, encoding + else + legacy_patterns + end + end + + def list_loader + # which loader to use + if ['ar', 'fa'].include? @bcp47 + sprintf "file=%s \\\n\tfile_patterns=", loadhyph + elsif @bcp47 == 'grc-x-ibycus' + # TODO: fix this + sprintf "file=%s \\\n\tluaspecial=\"disabled:8-bit only\"", loadhyph + else + sprintf "file=%s", loadhyph + end + end + + def list_run_files + return [] if use_old_loader + + files = [] + + files << File.join(PATH::HYPHU8, 'loadhyph', loadhyph) + if has_apostrophes? + files << File.join(PATH::HYPHU8, 'patterns', 'quote', sprintf("hyph-quote-%s.tex", bcp47)) + end + + files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47)) + if encoding && encoding != "ascii" then + files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding)) + elsif bcp47 == "cop" + files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns) + end + + # we skip the mongolian language for luatex files + return files if bcp47 == "mn-cyrl-x-lmc" + + ['chr', 'pat', 'hyp', 'lic'].each do |t| + files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-%s.%s.txt', bcp47, t)) + end + + if bcp47 =~ /^sh-/ + # duplicate entries (will be removed later) + files << File.join(PATH::HYPHU8, 'patterns', 'tex', 'hyph-sr-cyrl.tex') + ['chr', 'pat', 'hyp', 'lic'].each do |t| + # duplicate entries (will be removed later) + files << File.join(PATH::HYPHU8, 'patterns', 'txt', sprintf('hyph-sr-cyrl.%s.txt', t)) + end + end + + files + end + + def package + extract_metadata + @package + end + end + end + end +end |