From 0ed350c60624a3a8d3de8982910e34f8b306b6d2 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sun, 21 Mar 2021 22:18:35 +0000 Subject: hyph-utf main (21mar21) git-svn-id: svn://tug.org/texlive/trunk@58609 c570f23f-e606-0410-a88d-b1316a301751 --- .../hyph-utf8/data/encodings/macedonian.dat | 84 +++++++++++----------- .../generic/hyph-utf8/generate-plain-patterns.rb | 1 - .../generic/hyph-utf8/generate-ptex-patterns.rb | 5 +- .../generic/hyph-utf8/lib/tex/hyphen/converter.rb | 15 ++-- .../generic/hyph-utf8/lib/tex/hyphen/language.rb | 3 - .../generic/hyph-utf8/lib/tex/hyphen/packages.yml | 5 ++ .../hyph-utf8/lib/tex/hyphen/texlive/loader.rb | 3 +- .../hyph-utf8/lib/tex/hyphen/texlive/package.rb | 2 +- .../hyph-utf8/lib/tex/hyphen/texlive/source.rb | 5 +- .../hyph-utf8/spec/tex/hyphen/converter_spec.rb | 38 ++++++++++ .../hyph-utf8/spec/tex/hyphen/language_spec.rb | 2 - 11 files changed, 103 insertions(+), 60 deletions(-) create mode 100644 Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb (limited to 'Master/texmf-dist/source/generic/hyph-utf8') diff --git a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat index e8c64b4f20b..54dc6cf1f2d 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat +++ b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat @@ -1,42 +1,42 @@ -0x2E U+002E period -0x30 U+0030 zero -0x31 U+0031 one -0x32 U+0032 two -0x33 U+0033 three -0x34 U+0034 four -0x35 U+0035 five -0x36 U+0036 six -0x37 U+0037 seven -0x38 U+0038 eight -0x39 U+0039 nine -0x83 U+0453 cyrgj -0x9A U+0459 cyrlje -0x9C U+045A cyrnje -0x9D U+045C cyrkje -0x9F U+045F cyrdzh -0xBC U+0458 cyrj -0xBE U+0455 cyrdze -0xE0 U+0430 cyra -0xE1 U+0431 cyrb -0xE2 U+0432 cyrv -0xE3 U+0433 cyrg -0xE4 U+0434 cyrd -0xE5 U+0435 cyre -0xE6 U+0436 cyrzh -0xE7 U+0437 cyrz -0xE8 U+0438 cyri -0xEA U+043A cyrk -0xEB U+043B cyrl -0xEC U+043C cyrm -0xED U+043D cyrn -0xEE U+043E cyro -0xEF U+043F cyrp -0xF0 U+0440 cyrr -0xF1 U+0441 cyrs -0xF2 U+0442 cyrt -0xF3 U+0443 cyru -0xF4 U+0444 cyrf -0xF5 U+0445 cyrh -0xF6 U+0446 cyrc -0xF7 U+0447 cyrch -0xF8 U+0448 cyrsh +0x2E U+002E 1 period +0x30 U+0030 1 zero +0x31 U+0031 1 one +0x32 U+0032 1 two +0x33 U+0033 1 three +0x34 U+0034 1 four +0x35 U+0035 1 five +0x36 U+0036 1 six +0x37 U+0037 1 seven +0x38 U+0038 1 eight +0x39 U+0039 1 nine +0x83 U+0453 1 cyrgj +0x9A U+0459 1 cyrlje +0x9C U+045A 1 cyrnje +0x9D U+045C 1 cyrkje +0x9F U+045F 1 cyrdzh +0xBC U+0458 1 cyrj +0xBE U+0455 1 cyrdze +0xE0 U+0430 1 cyra +0xE1 U+0431 1 cyrb +0xE2 U+0432 1 cyrv +0xE3 U+0433 1 cyrg +0xE4 U+0434 1 cyrd +0xE5 U+0435 1 cyre +0xE6 U+0436 1 cyrzh +0xE7 U+0437 1 cyrz +0xE8 U+0438 1 cyri +0xEA U+043A 1 cyrk +0xEB U+043B 1 cyrl +0xEC U+043C 1 cyrm +0xED U+043D 1 cyrn +0xEE U+043E 1 cyro +0xEF U+043F 1 cyrp +0xF0 U+0440 1 cyrr +0xF1 U+0441 1 cyrs +0xF2 U+0442 1 cyrt +0xF3 U+0443 1 cyru +0xF4 U+0444 1 cyrf +0xF5 U+0445 1 cyrh +0xF6 U+0446 1 cyrc +0xF7 U+0447 1 cyrch +0xF8 U+0448 1 cyrsh diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb index 0265374c5b5..85d1d99dbf9 100755 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb @@ -4,7 +4,6 @@ # this file generates plain patterns (one-per-line) out of TeX source # use 'gem install unicode' if unicode is missing on your computer -require 'unicode' require_relative 'lib/tex/hyphen/language.rb' # include OldLanguage::TeXLive diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb index 0f50706dc01..a7b5c8e8055 100755 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb @@ -9,7 +9,10 @@ include TeXLive # this file generates patterns for pTeX out of the plain ones # load encodings -encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"] +encodings_list = + Dir.glob(File.expand_path(File.join(__dir__, 'data/encodings/*.dat'))).map do |encdat| + encdat.gsub /^.*\/(.*)\.dat$/, '\1' +end encodings = Hash.new encodings_list.each do |encoding_name| encodings[encoding_name] = HyphEncoding.new(encoding_name) diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb index 3b966d45a40..d270767c1b8 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb @@ -1,18 +1,22 @@ # encoding: UTF-8 -require 'scanf' -require 'byebug' - module TeX module Hyphen class Converter + def self.scan_line(line) + m = line.match /^(0x\h{2})\tU\+(\h{4})(?:\t(1?)(?:\t(\w+)))?$/ + return nil unless m + + return m[1].to_i(16), m[2].to_i(16).chr(Encoding::UTF_8), m[3].to_i, m[4] + end + def read(conversion) @mapping = { } File.read(conversion).each_line do |line| next if line =~ /^#/ - eightbit, usv = line.scanf "0x%02X\tU+%04X" - @mapping[eightbit] = usv.chr(Encoding::UTF_8) + eightbit, usv = Converter.scan_line line + @mapping[eightbit] = usv.chr end end @@ -28,7 +32,6 @@ module TeX if doconvert puts (line.strip.each_byte.map do |byte| - byebug unless @mapping[byte] @mapping[byte] end || '').join end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb index 3d3437d74b5..df6daa39478 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb @@ -1,6 +1,5 @@ require 'yaml' require 'hydra' -require 'byebug' unless ENV['RACK_ENV'] == "production" require_relative 'path' @@ -277,7 +276,6 @@ module TeX @exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit if @exceptions != "" @hyphenation = @exceptions.inject [] do |exceptions, exception| - # byebug unless exception exceptions << [exception.gsub('-', ''), exception] end.to_h else @@ -298,7 +296,6 @@ module TeX unless @hydra begin - # byebug metadata = extract_metadata @hydra = Hydra.new patterns, :lax, '', metadata rescue InvalidMetadata diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml index bc029739388..68a70fea0ed 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml @@ -1,3 +1,8 @@ +finish: + description: + Hyphenation patterns for Finnish in T1 and UTF-8 encodings. + The older set, labelled just “fi”, tries to implement etymological rules, + while the newer ones (fi-x-school) implements the simpler rules taught at Finnish school. german: dependency: dehyph description: |- diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb index 878bf0ef0d3..1efcb1ee4bf 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb @@ -4,7 +4,7 @@ module TeX module Loader def unicode_only? # TODO Spec out - ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi', 'mk'].include? @bcp47 + ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47 end def string_enc @@ -143,7 +143,6 @@ module TeX end end - byebug unless specification.is_a? Hash comment = specification[:comment] message = specification[:message] if comment then [sprintf('%% %s', comment)] else [] end + diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb index 1d40dc512e8..3de7d69805e 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb @@ -104,7 +104,7 @@ module TeX unless has_dependency? languages.each do |language| if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop' - if language.bcp47 == 'la-x-classic' + if ['la-x-classic', 'mk'].include? language.bcp47 # FIXME. Yes, fix it ;-) files << sprintf("tex/generic/hyph-utf8/patterns/tex-8bit/%s", language.legacy_patterns) else files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns) diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb index dce5be00b41..172b9b0a60b 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb @@ -82,9 +82,10 @@ module TeX end files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47)) - if encoding && encoding != "ascii" && bcp47 != 'la-x-classic' then + # FIXME That line is awful -- AR 2020-11-22 + if encoding && encoding != "ascii" && !['la-x-classic', 'mk'].include?( bcp47) then files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding)) - elsif bcp47 == "cop" + elsif ['cop', 'mk'].include? bcp47 # FIXME That one too! files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns) end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb new file mode 100644 index 00000000000..ff8c258f44a --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb @@ -0,0 +1,38 @@ +require 'spec_helper' + +include TeX::Hyphen + +describe Converter do + describe '.scan_line' do + it "scans the line" do + eightbits, usv = Converter.scan_line "0xDF\tU+042F" + expect(eightbits).to eq 223 + expect(usv).to eq 'Я' + end + + it "can read an optional name" do + eightbits, usv, bool, name = Converter.scan_line "0x2E\tU+002E\t\tperiod" + expect(eightbits).to eq 46 + expect(usv).to eq '.' + expect(bool).to eq 0 + expect(name).to eq 'period' + end + + it "doesn’t crash if there is no match" do + expect { Converter.scan_line "" }.not_to raise_exception + end + + it "returns nil if there are no matches" do + b = Converter.scan_line "0x2F" + expect(b).to be_nil + end + + it "can also have a useless strange entry" do + eightbits, usv, bool, name = Converter.scan_line "0x19\tU+0131\t1\tdotlessi" + expect(eightbits).to eq 25 + expect(usv).to eq 'ı' + expect(bool).to eq 1 + expect(name).to eq 'dotlessi' + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb index 93f42c462d1..666ee1b43f9 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb @@ -413,7 +413,6 @@ describe Language do it "loads the patterns" do language = Language.new('fi') - # byebug expect(language.patterns[151..154]).to eq ['uu1a2', 'uu1e2', 'uu1o2', 'uu1i2'] end @@ -440,7 +439,6 @@ describe Language do describe '#exceptions' do it "returns the hyphenation exceptions" do language = Language.new('ga') - # byebug expect(language.exceptions[0..2]).to eq ['bhrachtaí', 'mbrachtaí', 'cháintí'] end -- cgit v1.2.3