hyph-utf main (21mar21)

git-svn-id: svn://tug.org/texlive/trunk@58609 c570f23f-e606-0410-a88d-b1316a301751

11 files changed, 103 insertions, 60 deletions

diff --git a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat
index e8c64b4f20b..54dc6cf1f2d 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat
+++ b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat
@@ -1,42 +1,42 @@
-0x2E	U+002E		period
-0x30	U+0030		zero
-0x31	U+0031		one
-0x32	U+0032		two
-0x33	U+0033		three
-0x34	U+0034		four
-0x35	U+0035		five
-0x36	U+0036		six
-0x37	U+0037		seven
-0x38	U+0038		eight
-0x39	U+0039		nine
-0x83	U+0453		cyrgj
-0x9A	U+0459		cyrlje
-0x9C	U+045A		cyrnje
-0x9D	U+045C		cyrkje
-0x9F	U+045F		cyrdzh
-0xBC	U+0458		cyrj
-0xBE	U+0455		cyrdze
-0xE0	U+0430		cyra
-0xE1	U+0431		cyrb
-0xE2	U+0432		cyrv
-0xE3	U+0433		cyrg
-0xE4	U+0434		cyrd
-0xE5	U+0435		cyre
-0xE6	U+0436		cyrzh
-0xE7	U+0437		cyrz
-0xE8	U+0438		cyri
-0xEA	U+043A		cyrk
-0xEB	U+043B		cyrl
-0xEC	U+043C		cyrm
-0xED	U+043D		cyrn
-0xEE	U+043E		cyro
-0xEF	U+043F		cyrp
-0xF0	U+0440		cyrr
-0xF1	U+0441		cyrs
-0xF2	U+0442		cyrt
-0xF3	U+0443		cyru
-0xF4	U+0444		cyrf
-0xF5	U+0445		cyrh
-0xF6	U+0446		cyrc
-0xF7	U+0447		cyrch
-0xF8	U+0448		cyrsh
+0x2E	U+002E	1	period
+0x30	U+0030	1	zero
+0x31	U+0031	1	one
+0x32	U+0032	1	two
+0x33	U+0033	1	three
+0x34	U+0034	1	four
+0x35	U+0035	1	five
+0x36	U+0036	1	six
+0x37	U+0037	1	seven
+0x38	U+0038	1	eight
+0x39	U+0039	1	nine
+0x83	U+0453	1	cyrgj
+0x9A	U+0459	1	cyrlje
+0x9C	U+045A	1	cyrnje
+0x9D	U+045C	1	cyrkje
+0x9F	U+045F	1	cyrdzh
+0xBC	U+0458	1	cyrj
+0xBE	U+0455	1	cyrdze
+0xE0	U+0430	1	cyra
+0xE1	U+0431	1	cyrb
+0xE2	U+0432	1	cyrv
+0xE3	U+0433	1	cyrg
+0xE4	U+0434	1	cyrd
+0xE5	U+0435	1	cyre
+0xE6	U+0436	1	cyrzh
+0xE7	U+0437	1	cyrz
+0xE8	U+0438	1	cyri
+0xEA	U+043A	1	cyrk
+0xEB	U+043B	1	cyrl
+0xEC	U+043C	1	cyrm
+0xED	U+043D	1	cyrn
+0xEE	U+043E	1	cyro
+0xEF	U+043F	1	cyrp
+0xF0	U+0440	1	cyrr
+0xF1	U+0441	1	cyrs
+0xF2	U+0442	1	cyrt
+0xF3	U+0443	1	cyru
+0xF4	U+0444	1	cyrf
+0xF5	U+0445	1	cyrh
+0xF6	U+0446	1	cyrc
+0xF7	U+0447	1	cyrch
+0xF8	U+0448	1	cyrsh
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
index 0265374c5b5..85d1d99dbf9 100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
@@ -4,7 +4,6 @@
 # this file generates plain patterns (one-per-line) out of TeX source
 
 # use 'gem install unicode' if unicode is missing on your computer
-require 'unicode'
 
 require_relative 'lib/tex/hyphen/language.rb'
 # include OldLanguage::TeXLive
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
index 0f50706dc01..a7b5c8e8055 100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
@@ -9,7 +9,10 @@ include TeXLive
 # this file generates patterns for pTeX out of the plain ones
 
 # load encodings
-encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"]
+encodings_list =
+  Dir.glob(File.expand_path(File.join(__dir__, 'data/encodings/*.dat'))).map do |encdat|
+    encdat.gsub /^.*\/(.*)\.dat$/, '\1'
+end
 encodings = Hash.new
 encodings_list.each do |encoding_name|
   encodings[encoding_name] = HyphEncoding.new(encoding_name)
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb
index 3b966d45a40..d270767c1b8 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb
@@ -1,18 +1,22 @@
 # encoding: UTF-8
 
-require 'scanf'
-require 'byebug'
-
 module TeX
   module Hyphen
     class Converter
+      def self.scan_line(line)
+        m = line.match /^(0x\h{2})\tU\+(\h{4})(?:\t(1?)(?:\t(\w+)))?$/
+        return nil unless m
+
+        return m[1].to_i(16), m[2].to_i(16).chr(Encoding::UTF_8), m[3].to_i, m[4]
+      end
+
       def read(conversion)
         @mapping = { }
         File.read(conversion).each_line do |line|
           next if line =~ /^#/
 
-          eightbit, usv = line.scanf "0x%02X\tU+%04X"
-          @mapping[eightbit] = usv.chr(Encoding::UTF_8)
+          eightbit, usv = Converter.scan_line line
+          @mapping[eightbit] = usv.chr
         end
       end
 
@@ -28,7 +32,6 @@ module TeX
 
           if doconvert
             puts (line.strip.each_byte.map do |byte|
-              byebug unless @mapping[byte]
               @mapping[byte]
             end || '').join
           end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
index 3d3437d74b5..df6daa39478 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
@@ -1,6 +1,5 @@
 require 'yaml'
 require 'hydra'
-require 'byebug' unless ENV['RACK_ENV'] == "production"
 
 require_relative 'path'
 
@@ -277,7 +276,6 @@ module TeX
             @exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit
             if @exceptions != ""
               @hyphenation = @exceptions.inject [] do |exceptions, exception|
-                # byebug unless exception
                 exceptions << [exception.gsub('-', ''), exception]
               end.to_h
             else
@@ -298,7 +296,6 @@ module TeX
 
         unless @hydra
           begin
-            # byebug
             metadata = extract_metadata
             @hydra = Hydra.new patterns, :lax, '', metadata
           rescue InvalidMetadata
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
index bc029739388..68a70fea0ed 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
@@ -1,3 +1,8 @@
+finish:
+  description:
+    Hyphenation patterns for Finnish in T1 and UTF-8 encodings.
+    The older set, labelled just “fi”, tries to implement etymological rules,
+    while the newer ones (fi-x-school) implements the simpler rules taught at Finnish school.
 german:
   dependency: dehyph
   description: |-
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
index 878bf0ef0d3..1efcb1ee4bf 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
@@ -4,7 +4,7 @@ module TeX
       module Loader
 
         def unicode_only? # TODO Spec out
-          ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi', 'mk'].include? @bcp47
+          ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47
         end
 
         def string_enc
@@ -143,7 +143,6 @@ module TeX
             end
           end
 
-          byebug unless specification.is_a? Hash
           comment = specification[:comment]
           message = specification[:message]
           if comment then [sprintf('%% %s', comment)] else [] end +
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
index 1d40dc512e8..3de7d69805e 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
@@ -104,7 +104,7 @@ module TeX
           unless has_dependency?
             languages.each do |language|
               if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop'
-                if language.bcp47 == 'la-x-classic'
+                if ['la-x-classic', 'mk'].include? language.bcp47 # FIXME.  Yes, fix it ;-)
                   files << sprintf("tex/generic/hyph-utf8/patterns/tex-8bit/%s", language.legacy_patterns)
                 else
                   files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns)
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
index dce5be00b41..172b9b0a60b 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
@@ -82,9 +82,10 @@ module TeX
           end
 
           files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47))
-          if encoding && encoding != "ascii" && bcp47 != 'la-x-classic' then
+          # FIXME That line is awful -- AR 2020-11-22
+          if encoding && encoding != "ascii" && !['la-x-classic', 'mk'].include?( bcp47) then
             files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding))
-          elsif bcp47 == "cop"
+          elsif ['cop', 'mk'].include? bcp47 # FIXME That one too!
             files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns)
           end
 
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb
new file mode 100644
index 00000000000..ff8c258f44a
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb
@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+include TeX::Hyphen
+
+describe Converter do
+  describe '.scan_line' do
+    it "scans the line" do
+      eightbits, usv = Converter.scan_line "0xDF\tU+042F"
+      expect(eightbits).to eq 223
+      expect(usv).to eq 'Я'
+    end
+
+    it "can read an optional name" do
+      eightbits, usv, bool, name = Converter.scan_line "0x2E\tU+002E\t\tperiod"
+      expect(eightbits).to eq 46
+      expect(usv).to eq '.'
+      expect(bool).to eq 0
+      expect(name).to eq 'period'
+    end
+
+    it "doesn’t crash if there is no match" do
+      expect { Converter.scan_line "" }.not_to raise_exception
+    end
+
+    it "returns nil if there are no matches" do
+      b = Converter.scan_line "0x2F"
+      expect(b).to be_nil
+    end
+
+    it "can also have a useless strange entry" do
+      eightbits, usv, bool, name = Converter.scan_line "0x19\tU+0131\t1\tdotlessi"
+      expect(eightbits).to eq 25
+      expect(usv).to eq 'ı'
+      expect(bool).to eq 1
+      expect(name).to eq 'dotlessi'
+    end
+  end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
index 93f42c462d1..666ee1b43f9 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
@@ -413,7 +413,6 @@ describe Language do
 
     it "loads the patterns" do
       language = Language.new('fi')
-      # byebug
       expect(language.patterns[151..154]).to eq ['uu1a2', 'uu1e2', 'uu1o2', 'uu1i2']
     end
 
@@ -440,7 +439,6 @@ describe Language do
   describe '#exceptions' do
     it "returns the hyphenation exceptions" do
       language = Language.new('ga')
-      # byebug
       expect(language.exceptions[0..2]).to eq ['bhrachtaí', 'mbrachtaí', 'cháintí']
     end