summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/generic
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2021-03-21 22:18:35 +0000
committerKarl Berry <karl@freefriends.org>2021-03-21 22:18:35 +0000
commit0ed350c60624a3a8d3de8982910e34f8b306b6d2 (patch)
tree58ff50b634e1f80f22619664753141eb76a65805 /Master/texmf-dist/source/generic
parent0f67bb16431a1ef46f820429fa885994b04cabc1 (diff)
hyph-utf main (21mar21)
git-svn-id: svn://tug.org/texlive/trunk@58609 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/generic')
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat84
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb1
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb5
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb15
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb3
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml5
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb3
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb2
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb5
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb38
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb2
11 files changed, 103 insertions, 60 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat
index e8c64b4f20b..54dc6cf1f2d 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat
+++ b/Master/texmf-dist/source/generic/hyph-utf8/data/encodings/macedonian.dat
@@ -1,42 +1,42 @@
-0x2E U+002E period
-0x30 U+0030 zero
-0x31 U+0031 one
-0x32 U+0032 two
-0x33 U+0033 three
-0x34 U+0034 four
-0x35 U+0035 five
-0x36 U+0036 six
-0x37 U+0037 seven
-0x38 U+0038 eight
-0x39 U+0039 nine
-0x83 U+0453 cyrgj
-0x9A U+0459 cyrlje
-0x9C U+045A cyrnje
-0x9D U+045C cyrkje
-0x9F U+045F cyrdzh
-0xBC U+0458 cyrj
-0xBE U+0455 cyrdze
-0xE0 U+0430 cyra
-0xE1 U+0431 cyrb
-0xE2 U+0432 cyrv
-0xE3 U+0433 cyrg
-0xE4 U+0434 cyrd
-0xE5 U+0435 cyre
-0xE6 U+0436 cyrzh
-0xE7 U+0437 cyrz
-0xE8 U+0438 cyri
-0xEA U+043A cyrk
-0xEB U+043B cyrl
-0xEC U+043C cyrm
-0xED U+043D cyrn
-0xEE U+043E cyro
-0xEF U+043F cyrp
-0xF0 U+0440 cyrr
-0xF1 U+0441 cyrs
-0xF2 U+0442 cyrt
-0xF3 U+0443 cyru
-0xF4 U+0444 cyrf
-0xF5 U+0445 cyrh
-0xF6 U+0446 cyrc
-0xF7 U+0447 cyrch
-0xF8 U+0448 cyrsh
+0x2E U+002E 1 period
+0x30 U+0030 1 zero
+0x31 U+0031 1 one
+0x32 U+0032 1 two
+0x33 U+0033 1 three
+0x34 U+0034 1 four
+0x35 U+0035 1 five
+0x36 U+0036 1 six
+0x37 U+0037 1 seven
+0x38 U+0038 1 eight
+0x39 U+0039 1 nine
+0x83 U+0453 1 cyrgj
+0x9A U+0459 1 cyrlje
+0x9C U+045A 1 cyrnje
+0x9D U+045C 1 cyrkje
+0x9F U+045F 1 cyrdzh
+0xBC U+0458 1 cyrj
+0xBE U+0455 1 cyrdze
+0xE0 U+0430 1 cyra
+0xE1 U+0431 1 cyrb
+0xE2 U+0432 1 cyrv
+0xE3 U+0433 1 cyrg
+0xE4 U+0434 1 cyrd
+0xE5 U+0435 1 cyre
+0xE6 U+0436 1 cyrzh
+0xE7 U+0437 1 cyrz
+0xE8 U+0438 1 cyri
+0xEA U+043A 1 cyrk
+0xEB U+043B 1 cyrl
+0xEC U+043C 1 cyrm
+0xED U+043D 1 cyrn
+0xEE U+043E 1 cyro
+0xEF U+043F 1 cyrp
+0xF0 U+0440 1 cyrr
+0xF1 U+0441 1 cyrs
+0xF2 U+0442 1 cyrt
+0xF3 U+0443 1 cyru
+0xF4 U+0444 1 cyrf
+0xF5 U+0445 1 cyrh
+0xF6 U+0446 1 cyrc
+0xF7 U+0447 1 cyrch
+0xF8 U+0448 1 cyrsh
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
index 0265374c5b5..85d1d99dbf9 100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
@@ -4,7 +4,6 @@
# this file generates plain patterns (one-per-line) out of TeX source
# use 'gem install unicode' if unicode is missing on your computer
-require 'unicode'
require_relative 'lib/tex/hyphen/language.rb'
# include OldLanguage::TeXLive
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
index 0f50706dc01..a7b5c8e8055 100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
@@ -9,7 +9,10 @@ include TeXLive
# this file generates patterns for pTeX out of the plain ones
# load encodings
-encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"]
+encodings_list =
+ Dir.glob(File.expand_path(File.join(__dir__, 'data/encodings/*.dat'))).map do |encdat|
+ encdat.gsub /^.*\/(.*)\.dat$/, '\1'
+end
encodings = Hash.new
encodings_list.each do |encoding_name|
encodings[encoding_name] = HyphEncoding.new(encoding_name)
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb
index 3b966d45a40..d270767c1b8 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/converter.rb
@@ -1,18 +1,22 @@
# encoding: UTF-8
-require 'scanf'
-require 'byebug'
-
module TeX
module Hyphen
class Converter
+ def self.scan_line(line)
+ m = line.match /^(0x\h{2})\tU\+(\h{4})(?:\t(1?)(?:\t(\w+)))?$/
+ return nil unless m
+
+ return m[1].to_i(16), m[2].to_i(16).chr(Encoding::UTF_8), m[3].to_i, m[4]
+ end
+
def read(conversion)
@mapping = { }
File.read(conversion).each_line do |line|
next if line =~ /^#/
- eightbit, usv = line.scanf "0x%02X\tU+%04X"
- @mapping[eightbit] = usv.chr(Encoding::UTF_8)
+ eightbit, usv = Converter.scan_line line
+ @mapping[eightbit] = usv.chr
end
end
@@ -28,7 +32,6 @@ module TeX
if doconvert
puts (line.strip.each_byte.map do |byte|
- byebug unless @mapping[byte]
@mapping[byte]
end || '').join
end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
index 3d3437d74b5..df6daa39478 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/language.rb
@@ -1,6 +1,5 @@
require 'yaml'
require 'hydra'
-require 'byebug' unless ENV['RACK_ENV'] == "production"
require_relative 'path'
@@ -277,7 +276,6 @@ module TeX
@exceptions = readtexfile.superstrip.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').supersplit
if @exceptions != ""
@hyphenation = @exceptions.inject [] do |exceptions, exception|
- # byebug unless exception
exceptions << [exception.gsub('-', ''), exception]
end.to_h
else
@@ -298,7 +296,6 @@ module TeX
unless @hydra
begin
- # byebug
metadata = extract_metadata
@hydra = Hydra.new patterns, :lax, '', metadata
rescue InvalidMetadata
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
index bc029739388..68a70fea0ed 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/packages.yml
@@ -1,3 +1,8 @@
+finish:
+ description:
+ Hyphenation patterns for Finnish in T1 and UTF-8 encodings.
+ The older set, labelled just “fi”, tries to implement etymological rules,
+ while the newer ones (fi-x-school) implements the simpler rules taught at Finnish school.
german:
dependency: dehyph
description: |-
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
index 878bf0ef0d3..1efcb1ee4bf 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/loader.rb
@@ -4,7 +4,7 @@ module TeX
module Loader
def unicode_only? # TODO Spec out
- ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi', 'mk'].include? @bcp47
+ ['cu', 'sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te', 'pi'].include? @bcp47
end
def string_enc
@@ -143,7 +143,6 @@ module TeX
end
end
- byebug unless specification.is_a? Hash
comment = specification[:comment]
message = specification[:message]
if comment then [sprintf('%% %s', comment)] else [] end +
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
index 1d40dc512e8..3de7d69805e 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/package.rb
@@ -104,7 +104,7 @@ module TeX
unless has_dependency?
languages.each do |language|
if language.use_old_patterns_comment and language.legacy_patterns != "zerohyph.tex" and language.bcp47 != 'cop'
- if language.bcp47 == 'la-x-classic'
+ if ['la-x-classic', 'mk'].include? language.bcp47 # FIXME. Yes, fix it ;-)
files << sprintf("tex/generic/hyph-utf8/patterns/tex-8bit/%s", language.legacy_patterns)
else
files << sprintf("tex/generic/hyphen/%s", language.legacy_patterns)
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
index dce5be00b41..172b9b0a60b 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/lib/tex/hyphen/texlive/source.rb
@@ -82,9 +82,10 @@ module TeX
end
files << File.join(PATH::HYPHU8, 'patterns', 'tex', sprintf('hyph-%s.tex', bcp47))
- if encoding && encoding != "ascii" && bcp47 != 'la-x-classic' then
+ # FIXME That line is awful -- AR 2020-11-22
+ if encoding && encoding != "ascii" && !['la-x-classic', 'mk'].include?( bcp47) then
files << File.join(PATH::HYPHU8, 'patterns', 'ptex', sprintf('hyph-%s.%s.tex', bcp47, encoding))
- elsif bcp47 == "cop"
+ elsif ['cop', 'mk'].include? bcp47 # FIXME That one too!
files << File.join(PATH::HYPHU8, 'patterns', 'tex-8bit', legacy_patterns)
end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb
new file mode 100644
index 00000000000..ff8c258f44a
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/converter_spec.rb
@@ -0,0 +1,38 @@
+require 'spec_helper'
+
+include TeX::Hyphen
+
+describe Converter do
+ describe '.scan_line' do
+ it "scans the line" do
+ eightbits, usv = Converter.scan_line "0xDF\tU+042F"
+ expect(eightbits).to eq 223
+ expect(usv).to eq 'Я'
+ end
+
+ it "can read an optional name" do
+ eightbits, usv, bool, name = Converter.scan_line "0x2E\tU+002E\t\tperiod"
+ expect(eightbits).to eq 46
+ expect(usv).to eq '.'
+ expect(bool).to eq 0
+ expect(name).to eq 'period'
+ end
+
+ it "doesn’t crash if there is no match" do
+ expect { Converter.scan_line "" }.not_to raise_exception
+ end
+
+ it "returns nil if there are no matches" do
+ b = Converter.scan_line "0x2F"
+ expect(b).to be_nil
+ end
+
+ it "can also have a useless strange entry" do
+ eightbits, usv, bool, name = Converter.scan_line "0x19\tU+0131\t1\tdotlessi"
+ expect(eightbits).to eq 25
+ expect(usv).to eq 'ı'
+ expect(bool).to eq 1
+ expect(name).to eq 'dotlessi'
+ end
+ end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
index 93f42c462d1..666ee1b43f9 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/spec/tex/hyphen/language_spec.rb
@@ -413,7 +413,6 @@ describe Language do
it "loads the patterns" do
language = Language.new('fi')
- # byebug
expect(language.patterns[151..154]).to eq ['uu1a2', 'uu1e2', 'uu1o2', 'uu1i2']
end
@@ -440,7 +439,6 @@ describe Language do
describe '#exceptions' do
it "returns the hyphenation exceptions" do
language = Language.new('ga')
- # byebug
expect(language.exceptions[0..2]).to eq ['bhrachtaí', 'mbrachtaí', 'cháintí']
end