summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source
diff options
context:
space:
mode:
authorMojca Miklavec <mojca.miklavec@gmail.com>2010-06-01 01:39:02 +0000
committerMojca Miklavec <mojca.miklavec@gmail.com>2010-06-01 01:39:02 +0000
commit3a5cf8de164c5d0f1fdb01999af33d77f59d526f (patch)
tree1e1c013154793e9a74852ded1537869e3cfabbb0 /Master/texmf-dist/source
parent3205d02db781aec158019161797c2a156ceb0b34 (diff)
update of hyph-utf8: add the forgotten ru and uk extra patterns, remove OT1 support from Latin, updates in luatex sources and docs, removed collaboration files
git-svn-id: svn://tug.org/texlive/trunk@18661 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source')
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/README6
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb204
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java108
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl38
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java385
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl128
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java428
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README38
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java325
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml23
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml434
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb55
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb2
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb108
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb11
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb109
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/languages.rb15
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README50
-rw-r--r--Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx22
19 files changed, 284 insertions, 2205 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/README b/Master/texmf-dist/source/generic/hyph-utf8/README
index 07ba26b49a4..97f87378761 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/README
+++ b/Master/texmf-dist/source/generic/hyph-utf8/README
@@ -9,11 +9,6 @@ Auto-generates conversions from UTF-8 to some particular encoding.
Needs to be run only in case that a new encoding is added or an old encoding is fixed.
-generate-offo.rb
-================
-Temporary; another version maintaned by author in Java; this one will be improved or removed.
-Will be rewritten to take raw patterns as input with no need to parse.
-
generate-pattern-loaders.rb
===========================
INPUT:
@@ -59,3 +54,4 @@ Author: See file, modified by Mojca Miklavec
contributed/make-exhyph.pl
==========================
Imported script for hack related to breaking compound words.
+
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb b/Master/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb
deleted file mode 100755
index 55a44f67f1b..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb
+++ /dev/null
@@ -1,204 +0,0 @@
-#!/usr/bin/env ruby
-
-# This file generates patterns for Hyphenator.js
-# http://code.google.com/p/hyphenator
-#
-# Collaboration with:
-# Mathias Nater, <mathias at mnn.ch>
-
-load '../languages.rb'
-# TODO: should be singleton
-languages = Languages.new
-
-languages["de-CH-1901"] = languages["de-ch-1901"]
-languages["en-GB"] = languages["en-gb"]
-languages["en-US"] = languages["en-us"]
-languages["en-US-x-knuth"] = languages["en-us"]
-languages["mn"] = languages["mn-cyrl"]
-languages["sr-Cyrl"] = languages["sr-cyrl"]
-languages["sr-Latn"] = languages["sh-latn"]
-languages["zh-Latn"] = languages["zh-latn"]
-
-
-$path_root=File.expand_path(Dir.getwd + "/../../../..")
-$path_plain="#{$path_root}/tex/generic/hyph-utf8/patterns/txt"
-$path_js=File.expand_path("#{$path_root}/../collaboration/hyphenator/repo")
-
-# change to current folder and read all the files in it
-Dir.chdir("#{$path_plain}")
-files = Dir.glob("*.pat.txt")
-# files = Dir.glob("*sl.pat.txt")
-
-# we need to escape some characters; for a complete list see
-# http://www.jslint.com/lint.html
-# but at the moment there are only two such characters present anyway
-#
-# this function encapsulates the string into single quotes and uses
-def unescape_string_if_needed(str)
- # unsafe characters - see above for complete list
- unsafeCharacters = [0x200c, 0x200d]
- # let's convert our string into array (to preserve proper unicode numbers)
- str_array=str.unpack("U*")
- # set this to false until the first replacement takes place
- replacement_done = false
-
- # loop over all unsafe character and try to replace all occurencies
- unsafeCharacters.each do |c|
- # find the first occurence of that character
- i = str_array.index(c)
- while i != nil
- # replaces a single character with '%uXXXX', where XXXX is hex code of character
- # this only works with non-math characters, but it should not happen that any bigger number would occur
- str_array[i,1] = sprintf("%%u%4X", c).unpack("U*")
- i = str_array.index(c)
- replacement_done = true
- end
- end
-
- # convert the array back to string
- str = str_array.pack("U*")
-
- if replacement_done
- return "unescape('#{str}')"
- else
- return "'#{str}'"
- end
-end
-
-class Pattern
- # include Enumerable
-
- def initialize(pattern)
- @pattern = pattern.strip
- @pattern_array = @pattern.unpack("U*")
- @length = @pattern_array.length
- end
-
- def <=>(anOther)
- # if @length == anOther.length
- # 0.upto(@length-1) do |i|
- # if @pattern_array[i] != anOther.pattern_array[i]
- # return @pattern_array[i] <=> anOther.pattern_array[i]
- # end
- # end
- # return 1 <=> 1
- # else
- # @length <=> anOther.length
- # end
- @length <=> anOther.length
- end
-
- def js_pattern
- @pattern.gsub(/[.]/, "_")
- end
-
- def to_s
- @pattern
- end
-
- def length_of_letters_only
- return @pattern.gsub(/[0-9]/,'').unpack("U*").length
- end
-
- # def sort_by_length
- attr_reader :pattern, :length, :pattern_array
-end
-
-# TODO: this should be an explicit array of patterns only
-class Patterns < Array
- def length_of_shortest_and_longest_pattern
- # store the minimum and maximum length of pattern
- a = [self.first.length_of_letters_only, self.first.length_of_letters_only]
- #
- # a = [0, 1]
- self.each do |pat|
- a[0] = [a[0], pat.length_of_letters_only].min
- a[1] = [a[1], pat.length_of_letters_only].max
- # a.first = [a.first, pat.length_of_letters_only].min
- # a.last = [a.last, pat.length_of_letters_only].max
- end
- return a
- end
- # TODO: you need to make sure that patterns are sorted according to their length first
- def each_length
- current_length = 0
- first_pattern_with_some_size = Array.new
-
- self.each_index do |i|
- pattern = self[i]
- if pattern.length > current_length
- current_length = pattern.length
- first_pattern_with_some_size.push(i)
- end
- end
-
- first_pattern_with_some_size.each_index do |i|
- i_first = first_pattern_with_some_size[i]
- i_last = nil
- if i < first_pattern_with_some_size.length-1
- i_last = first_pattern_with_some_size[i+1]
- else
- i_last = self.length
- end
- i_len = i_last-i_first
-
- yield self[i_first,i_len]
- end
- end
-end
-
-files.each do |filename|
- code_in = filename.gsub(/hyph-(.*).pat.txt/,'\1')
- code_out = code_in.gsub(/-/,"_")
- language = languages[code_in] # FIXME
- # TODO: handle exceptions
- puts
- puts "Generating Hyphenator.js support for " + code_in
- puts " writing to '#{$path_js}/#{code_out}.js'"
- puts
- patterns = Patterns.new
- File.open(filename,'r') do |f_in|
- f_in.each_line do |line|
- if line.strip.length > 0
- patterns.push(Pattern.new(line))
- end
- end
- end
- patterns.sort!
- # puts patterns
- specialChars = patterns.join('').gsub(/[.0-9a-z]/,'').unpack('U*').sort.uniq.pack('U*')
-
- File.open("#{$path_js}/#{code_out}.js", "w") do |f_out|
- # BOM mark
- # f_out.puts [239,187,191].pack("ccc")
- # f_out.print ["EF","BB","BF"].pack("H2H2H2")
- f_out.putc(239)
- f_out.putc(187)
- f_out.putc(191)
- f_out.puts "Hyphenator.languages.#{code_out} = {"
- f_out.puts "\tleftmin : #{language.hyphenmin[0]},"
- f_out.puts "\trightmin : #{language.hyphenmin[1]},"
- lengths = patterns.length_of_shortest_and_longest_pattern
- f_out.puts "\tshortestPattern : #{lengths.first},"
- f_out.puts "\tlongestPattern : #{lengths.last},"
- # TODO: handle Ux201C, Ux201D
- # if specialChars.gsub!(/.../, ...) ~= nil
- # if specialChars =~ /[]/
- # if has_unsafe_characters(specialChars)
- # end
- unescape_string_if_needed(specialChars)
- f_out.puts "\tspecialChars : #{unescape_string_if_needed(specialChars)},"
- f_out.puts "\tpatterns : {"
-
- # current length of patterns (they are sorted according to their length)
- current_length = 0
- pattern_string = ""
- i_first = i_last = -1
- patterns.each_length do |pats|
- f_out.puts "\t\t#{pats.first.length} : #{unescape_string_if_needed(pats.join(""))}"
- end
-
- f_out.puts "\t}"
- f_out.puts "};"
- end
-end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java
deleted file mode 100644
index a4f70616a36..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java
+++ /dev/null
@@ -1,108 +0,0 @@
-/*
- * Copyright Simon Pepping 2009
- *
- * The copyright owner licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* $Id: ConvertLanguageData.java 304 2009-11-26 07:26:55Z spepping $ */
-
-package org.tug.texhyphen;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.URISyntaxException;
-import java.net.URL;
-
-import javax.xml.transform.Result;
-import javax.xml.transform.Source;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
-
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import org.xml.sax.XMLReader;
-
-/**
- * Convert language data in ruby format to XML format
- */
-public final class ConvertLanguageData {
-
- /**
- * @param languageDataPath
- * @throws IOException
- * @throws TransformerException
- * @throws SAXException
- * @throws URISyntaxException
- */
- public static void convert(String languageDataPath, boolean useStylesheet)
- throws IOException, TransformerException, SAXException, URISyntaxException {
-
- // input
- InputStream inis = new FileInputStream(languageDataPath);
- InputSource input = new InputSource(inis);
- input.setSystemId(languageDataPath);
- input.setEncoding("utf-8");
- XMLReader reader = new LanguageDataParser();
-
- // output
- String outPath = languageDataPath.replaceFirst("\\.rb$", ".xml");
- Result result = new StreamResult(outPath);
-
- // transformation
- TransformerFactory tf = TransformerFactory.newInstance();
- if (!tf.getFeature(SAXTransformerFactory.FEATURE)) {
- throw new TransformerException("TransformerFactory is not a SAXTransformerFactory");
- }
- SAXTransformerFactory stf = (SAXTransformerFactory) tf;
- TransformerHandler th;
- if (useStylesheet) {
- URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertLanguageData.xsl");
- File xsltFile = new File(xsltUrl.toURI());
- InputStream xsltStream = new FileInputStream(xsltFile);
- Source xsltSource = new StreamSource(xsltStream);
- xsltSource.setSystemId(xsltFile.getAbsolutePath());
- th = stf.newTransformerHandler(xsltSource);
- } else {
- th = stf.newTransformerHandler();
- }
-
- // pipeline
- reader.setContentHandler(th);
- reader.setProperty("http://xml.org/sax/properties/lexical-handler", th);
- th.setResult(result);
- reader.parse(input);
- }
-
- /**
- * @param args
- * @throws IOException
- * @throws TransformerException
- * @throws SAXException
- * @throws URISyntaxException
- */
- public static void main(String[] args)
- throws IOException, TransformerException, SAXException, URISyntaxException {
- if (args[0].endsWith("--debug")) {
- convert(args[1], false);
- } else {
- convert(args[0], true);
- }
- }
-
-}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl
deleted file mode 100644
index 98aa733a0c0..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl
+++ /dev/null
@@ -1,38 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<xsl:stylesheet version="1.0"
- xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns:lang="urn:org:tug:texhyphen:languagedata">
-
- <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
-
- <xsl:variable name="language-data" select="document('codemapping.xml')/lang:code-mappings"/>
-
- <xsl:template match="*|@*|node()">
- <xsl:copy>
- <xsl:apply-templates select="*|@*|node()"/>
- </xsl:copy>
- </xsl:template>
-
- <xsl:template match="lang:language">
- <xsl:variable name="code" select="$language-data/lang:code-mapping[@code=current()/@code]"/>
- <xsl:variable name="fop-code">
- <xsl:choose>
- <xsl:when test="$code">
- <xsl:value-of select="$code/@fop-code"/>
- </xsl:when>
- <xsl:otherwise>
- <xsl:value-of select="@code"/>
- </xsl:otherwise>
- </xsl:choose>
- </xsl:variable>
- <xsl:copy>
- <xsl:if test="string($fop-code)">
- <xsl:attribute name="fop-code">
- <xsl:value-of select="$fop-code"/>
- </xsl:attribute>
- </xsl:if>
- <xsl:apply-templates select="*|@*|node()"/>
- </xsl:copy>
- </xsl:template>
-
-</xsl:stylesheet>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java
deleted file mode 100644
index 1ec9cee9474..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java
+++ /dev/null
@@ -1,385 +0,0 @@
-/*
- * Copyright Simon Pepping 2009
- *
- * The copyright owner licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* $Id: ConvertTeXPattern.java 304 2009-11-26 07:26:55Z spepping $ */
-
-package org.tug.texhyphen;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileNotFoundException;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.net.MalformedURLException;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.Map;
-
-import javax.xml.parsers.ParserConfigurationException;
-import javax.xml.parsers.SAXParser;
-import javax.xml.parsers.SAXParserFactory;
-import javax.xml.transform.Result;
-import javax.xml.transform.Source;
-import javax.xml.transform.Transformer;
-import javax.xml.transform.TransformerException;
-import javax.xml.transform.TransformerFactory;
-import javax.xml.transform.sax.SAXTransformerFactory;
-import javax.xml.transform.sax.TransformerHandler;
-import javax.xml.transform.stream.StreamResult;
-import javax.xml.transform.stream.StreamSource;
-
-import org.xml.sax.Attributes;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import org.xml.sax.XMLReader;
-import org.xml.sax.helpers.DefaultHandler;
-
-/**
- * Convert modern UTF8 TeX hyphenation patterns to XML format
- */
-public final class ConvertTeXPattern {
-
- public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet,
- boolean useLanguagedata)
- throws IOException, TransformerException, SAXException, URISyntaxException,
- ParserConfigurationException, CodeMappingException {
- checkCodeMapping();
- Collection<String> languages = codeMapping.keySet();
- convert(texPatterns, outfilePath, useStylesheet, languages);
- }
-
- public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet)
- throws IOException, TransformerException, SAXException, URISyntaxException,
- CodeMappingException {
- convert(texPatterns, outfilePath, useStylesheet, null);
- }
-
- /**
- * infile outfile
- * indir outdir (file protocol only)
- * infiles outdir
- * file and http protocols allowed
- *
- * @param texPatternUri
- * @param outfilePath
- * @param useStylesheet
- * @param texcodes filter of requested tex codes; is allowed to be null
- * @throws IOException
- * @throws TransformerException
- * @throws SAXException
- * @throws URISyntaxException
- * @throws CodeMappingException
- */
- public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet,
- Collection<String> texcodes)
- throws IOException, TransformerException, SAXException, URISyntaxException,
- CodeMappingException {
- File outDir = new File(outfilePath);
- boolean oneTexcode = (texcodes != null && texcodes.size() == 1);
- boolean oneInputfile = (texPatterns.length == 1);
- boolean oneFilteredInput = (oneTexcode || oneInputfile);
- if (!oneFilteredInput && !outDir.isDirectory()) {
- throw new IllegalArgumentException
- ("with multiple input files the output path " + outfilePath + " must be a directory");
- }
- for (String texPattern : texPatterns) {
- URI texPatternUri = makeTexPatternUri(texPattern);
- URI[] texPatternUris = makeTexPatternUris(texPatternUri);
- oneInputfile = (texPatternUris.length == 1);
- oneFilteredInput = (oneTexcode || oneInputfile);
- if (!oneFilteredInput && !outDir.isDirectory()) {
- throw new IllegalArgumentException
- ("with an input directory " + texPattern + " the output path " + outfilePath + " must be a directory");
- }
- for (URI t : texPatternUris) {
- TransformationData transformationData = makeTransformationData(t, outDir, texcodes);
- if (transformationData == null) {
- continue;
- }
- doConvert(t, transformationData, useStylesheet);
- }
- }
- }
-
- /**
- * @param texPattern
- * @return
- * @throws URISyntaxException
- * @throws FileNotFoundException
- */
- private static URI makeTexPatternUri(String texPattern)
- throws URISyntaxException, FileNotFoundException {
- URI texPatternUri;
- texPatternUri = new URI(texPattern);
- String scheme = texPatternUri.getScheme();
- // see if it is a relative file path
- if (scheme == null) {
- File f = new File(texPattern);
- texPatternUri = new URI("file", null, f.getAbsolutePath(), null, null);
- scheme = texPatternUri.getScheme();
- }
- if (scheme == null || !(scheme.equals("http") || scheme.equals("file"))) {
- throw new FileNotFoundException
- ("URI with file or http scheme required for hyphenation pattern file");
- }
- return texPatternUri;
- }
-
- /**
- * @param outfilePath
- * @param outDir
- * @param texPatternUri
- * @param scheme
- * @return
- * @throws URISyntaxException
- */
- private static URI[] makeTexPatternUris(URI texPatternUri) throws URISyntaxException {
- URI[] texPatternUris;
- texPatternUris = new URI[] {texPatternUri};
- String scheme = texPatternUri.getScheme();
- if (scheme.equals("file")) {
- File dir = new File(texPatternUri);
- if (dir.isDirectory()) {
- ArrayList<URI> l = new ArrayList<URI>();
- FilenameFilter filter = new FilenameFilter() {
- public boolean accept(File dir, String name) {
- return name.endsWith(".tex");
- }
- };
- for (File f : dir.listFiles(filter)) {
- l.add(new URI("file", null, f.getAbsolutePath(), null, null));
- }
- texPatternUris = l.toArray(texPatternUris);
- }
- }
- return texPatternUris;
- }
-
- /**
- * @param t
- * @param outDir
- * @param texcodes filter of requested tex codes; is allowed to be null
- * @return
- * @throws CodeMappingException
- */
- private static TransformationData makeTransformationData
- (URI t, File outDir, Collection<String> texcodes) throws CodeMappingException {
- File outFile;
- String path = t.getPath();
- String basename = path.substring(path.lastIndexOf('/') + 1);
- String base = basename.substring(0, basename.lastIndexOf('.'));
- // xmlCode, texCode
- String[] codes = mapCode(base);
- // code mapping lists no xmlCode
- if (codes[0] == null) {
- return null;
- }
- if (texcodes != null && !texcodes.contains(codes[1])) {
- return null;
- }
- if (!outDir.isDirectory()) {
- outFile = outDir;
- } else {
- outFile = new File(outDir, codes[0] + ".xml");
- }
- return new TransformationData(outFile, codes[1]);
- }
-
- private static class TransformationData {
- File outFile;
- String texCode;
- TransformationData(File outFile, String texCode) {
- this.outFile = outFile;
- this.texCode = texCode;
- }
- }
-
- private static class CodeMappingException extends Exception {
- public CodeMappingException(Exception e) {
- super(e);
- }
- public CodeMappingException(String m) {
- super(m);
- }
- }
-
- static Map<String, String> codeMapping;
- static CodeMappingException codeMappingException;
- static {
- try {
- codeMapping = readLanguagedata();
- } catch (ParserConfigurationException e) {
- codeMappingException = new CodeMappingException(e);
- } catch (SAXException e) {
- codeMappingException = new CodeMappingException(e);
- } catch (IOException e) {
- codeMappingException = new CodeMappingException(e);
- }
- }
-
- private static String[] mapCode(String texCode) throws CodeMappingException {
- checkCodeMapping();
- String hyp = "hyph-";
- String xmlCode = texCode;
- if (texCode.startsWith(hyp)) {
- texCode = texCode.substring(hyp.length());
- xmlCode = codeMapping.get(texCode);
- }
- return new String[] {xmlCode, texCode};
- }
-
- /**
- * @throws CodeMappingException
- */
- private static void checkCodeMapping() throws CodeMappingException {
- if (codeMapping == null) {
- if (codeMappingException != null) {
- throw codeMappingException;
- } else {
- throw new CodeMappingException("Failure initializing code mapping");
- }
- }
- }
-
- public static Map<String,String> readLanguagedata()
- throws ParserConfigurationException, SAXException, IOException {
- SAXParserFactory spf = SAXParserFactory.newInstance();
- spf.setNamespaceAware(true);
- SAXParser parser = spf.newSAXParser();
- InputStream is = ConvertTeXPattern.class.getResourceAsStream("languages.xml");
- TexcodeReader texcodeReader = new TexcodeReader();
- parser.parse(is, texcodeReader);
- return texcodeReader.getTexcodes();
- }
-
- private static class TexcodeReader extends DefaultHandler {
-
- private Map<String, String> texcodes = new HashMap<String, String>();
-
- /* (non-Javadoc)
- * @see org.xml.sax.helpers.DefaultHandler#startElement(java.lang.String, java.lang.String, java.lang.String, org.xml.sax.Attributes)
- */
- @Override
- public void startElement(String uri, String localName, String qName,
- Attributes attributes) throws SAXException {
- if (uri.equals(LanguageDataParser.LANG_NAMESPACE) && localName.equals("language")) {
- String texcode = attributes.getValue("code");
- String fopcode = attributes.getValue("fop-code");
- if (fopcode != null) {
- texcodes.put(texcode, fopcode);
- }
- }
- }
-
- /**
- * @return the texcodes
- */
- public Map<String,String> getTexcodes() {
- return texcodes;
- }
-
- }
-
- public static void doConvert(URI texPatternUri, TransformationData outdata, boolean useStylesheet)
- throws TransformerException, SAXException, MalformedURLException, IOException, URISyntaxException {
-
- String scheme = texPatternUri.getScheme();
- InputStream inis = null;
- if (scheme.equals("file")) {
- File in = new File(texPatternUri);
- inis = new FileInputStream(in);
- } else if (scheme.equals("http")) {
- inis = texPatternUri.toURL().openStream();
- } else {
- throw new FileNotFoundException
- ("URI with file or http scheme required for hyphenation pattern file");
- }
-
- InputSource input = new InputSource(inis);
- input.setSystemId(texPatternUri.toString());
- input.setEncoding("utf-8");
- XMLReader reader = new TeXPatternParser();
- Result result = new StreamResult(outdata.outFile);
- TransformerFactory tf = TransformerFactory.newInstance();
- if (!tf.getFeature(SAXTransformerFactory.FEATURE)) {
- throw new TransformerException("TransformerFactory is not a SAXTransformerFactory");
- }
- SAXTransformerFactory stf = (SAXTransformerFactory) tf;
- TransformerHandler th;
- if (useStylesheet) {
- URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertTeXPattern.xsl");
- File xsltFile = new File(xsltUrl.toURI());
- InputStream xsltStream = new FileInputStream(xsltFile);
- Source xsltSource = new StreamSource(xsltStream);
- xsltSource.setSystemId(xsltFile.getAbsolutePath());
- th = stf.newTransformerHandler(xsltSource);
- Transformer tr = th.getTransformer();
- tr.setParameter("tex-code", outdata.texCode);
- } else {
- th = stf.newTransformerHandler();
- }
- reader.setContentHandler(th);
- reader.setProperty("http://xml.org/sax/properties/lexical-handler", th);
- th.setResult(result);
- reader.parse(input);
- }
-
- /**
- * @param args input URI, output file
- * @throws URISyntaxException if the URI is not correct
- * @throws IOException if a file is not found, or contains illegal content
- * @throws TransformerException
- * @throws SAXException
- * @throws ParserConfigurationException
- * @throws CodeMappingException
- */
- public static void main(String[] args)
- throws URISyntaxException, IOException, TransformerException, SAXException,
- ParserConfigurationException, CodeMappingException {
- String prefix = "--";
- int i = 0;
- boolean useStylesheet = true;
- boolean useLanguagedata = false;
- Collection<String> texcodes = null;
- while (args[i].startsWith(prefix)) {
- String option = args[i].substring(prefix.length());
- if (option.equals("debug")) {
- useStylesheet = false;
- } else if (option.equals("uselanguagedata") || option.equals("langdata")) {
- useLanguagedata = true;
- } else if (option.equals("texcodes")) {
- texcodes = Arrays.asList(args[++i].split(","));
- } else {
- throw new IllegalArgumentException("Unknown option: " + option);
- }
- ++i;
- }
- if (texcodes != null) {
- convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1],
- useStylesheet, texcodes);
- } else {
- convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1],
- useStylesheet, useLanguagedata);
- }
- }
-
-}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl
deleted file mode 100644
index 803962b7d5a..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl
+++ /dev/null
@@ -1,128 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<xsl:stylesheet version="1.0"
- xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
- xmlns:tex="urn:org:tug:texhyphen"
- xmlns:lang="urn:org:tug:texhyphen:languagedata"
- exclude-result-prefixes="tex lang">
-
- <xsl:output doctype-system="hyphenation.dtd" indent="yes"/>
-
- <xsl:param name="comment-length" select="72"/>
- <xsl:param name="tex-code"/>
- <xsl:param name="hyphen-min-before-default" select="2"/>
- <xsl:param name="hyphen-min-after-default" select="3"/>
-
- <xsl:template match="/tex:tex">
- <hyphenation-info>
- <xsl:choose>
- <xsl:when test="tex:hyphenation">
- <!-- (comment*), (patterns), (comment*, hyphenation, comment*) => (comment*), hyphen-min, (comment*, exceptions, comment*), (patterns) -->
- <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/>
- <xsl:variable name="set2" select="tex:patterns"/>
- <xsl:variable name="set3" select="node()[preceding-sibling::tex:patterns]"/>
- <xsl:apply-templates select="$set1"/>
- <xsl:call-template name="hyphen-min"/>
- <xsl:apply-templates select="$set3"/>
- <xsl:apply-templates select="$set2"/>
- </xsl:when>
- <xsl:otherwise>
- <!-- (comment*), (patterns, comment*) => (comment*), hyphen-min, (patterns, comment*) -->
- <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/>
- <xsl:variable name="set2" select="node()[preceding-sibling::tex:patterns or self::tex:patterns]"/>
- <xsl:apply-templates select="$set1"/>
- <xsl:call-template name="hyphen-min"/>
- <xsl:apply-templates select="$set2"/>
- </xsl:otherwise>
- </xsl:choose>
- </hyphenation-info>
- </xsl:template>
-
- <xsl:template match="tex:patterns">
- <patterns>
- <xsl:apply-templates />
- </patterns>
- </xsl:template>
-
- <xsl:template match="tex:patterns" mode="call-hyphen-min">
- <xsl:call-template name="hyphen-min"/>
- <patterns>
- <xsl:apply-templates />
- </patterns>
- </xsl:template>
-
- <xsl:template name="hyphen-min">
- <xsl:variable name="hyphen-min"
- select="document('languages.xml')/lang:languages/lang:language[@code=$tex-code]/lang:hyphen-min" />
- <xsl:if test="count($hyphen-min)">
- <hyphen-min before="{$hyphen-min/@before}" after="{$hyphen-min/@after}" />
- </xsl:if>
- </xsl:template>
-
- <xsl:template match="tex:message"/>
-
- <xsl:template match="tex:hyphenation">
- <exceptions>
- <xsl:apply-templates />
- </exceptions>
- </xsl:template>
-
- <!-- Comments in TeX contain the trailing new line. -->
- <!-- Here we keep the trailing new line if the comment is immediately -->
- <!-- preceded or followed by a text node. -->
- <!-- Otherwise we strip the new line and pad the comment -->
- <!-- to the parameter comment-length. -->
- <!-- The XSLT engine uses the same criteria to decide if the comment -->
- <!-- should start on a new line or not. -->
- <!-- This is not quite correct, because we risk adding spaces to -->
- <!-- elements with mixed content. -->
- <!-- The following test would be more appropriate: -->
- <!-- test="preceding-sibling::text() or following-sibling::text()". -->
- <xsl:template match="comment()">
- <xsl:choose>
- <xsl:when
- test="preceding-sibling::node()[1][self::text()]
- or following-sibling::node()[1][self::text()]">
- <xsl:comment>
- <xsl:value-of select="." />
- </xsl:comment>
- </xsl:when>
- <xsl:otherwise>
- <xsl:variable name="length" select="string-length(.)" />
- <xsl:comment>
- <xsl:value-of select="substring(.,1,$length - 1)" />
- <xsl:text> </xsl:text>
- <xsl:call-template name="make-spaces">
- <xsl:with-param name="length"
- select="$comment-length - ($length - 1)" />
- </xsl:call-template>
- </xsl:comment>
- </xsl:otherwise>
- </xsl:choose>
- </xsl:template>
-
- <xsl:template name="make-spaces">
- <xsl:param name="length" select="0" />
- <xsl:choose>
- <xsl:when test="$length >= 10">
- <xsl:text> </xsl:text>
- <xsl:call-template name="make-spaces">
- <xsl:with-param name="length" select="$length - 10" />
- </xsl:call-template>
- </xsl:when>
- <xsl:when test="$length >= 5">
- <xsl:text> </xsl:text>
- <xsl:call-template name="make-spaces">
- <xsl:with-param name="length" select="$length - 5" />
- </xsl:call-template>
- </xsl:when>
- <xsl:when test="$length >= 1">
- <xsl:text> </xsl:text>
- <xsl:call-template name="make-spaces">
- <xsl:with-param name="length" select="$length - 1" />
- </xsl:call-template>
- </xsl:when>
- </xsl:choose>
- <xsl:text></xsl:text>
- </xsl:template>
-
-</xsl:stylesheet> \ No newline at end of file
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java
deleted file mode 100644
index 95c484b25ab..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java
+++ /dev/null
@@ -1,428 +0,0 @@
-/*
- * Copyright Simon Pepping 2009
- *
- * The copyright owner licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* $Id: LanguageDataParser.java 304 2009-11-26 07:26:55Z spepping $ */
-
-package org.tug.texhyphen;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Collection;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Vector;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.xml.sax.Attributes;
-import org.xml.sax.ContentHandler;
-import org.xml.sax.DTDHandler;
-import org.xml.sax.EntityResolver;
-import org.xml.sax.ErrorHandler;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import org.xml.sax.SAXNotRecognizedException;
-import org.xml.sax.SAXNotSupportedException;
-import org.xml.sax.XMLReader;
-import org.xml.sax.ext.LexicalHandler;
-import org.xml.sax.helpers.AttributesImpl;
-
-/**
- * The class TeXParser parses TeX hyphenation pattern files and produces SAX events
- */
-public class LanguageDataParser implements XMLReader {
-
- public static final String LANG_NAMESPACE = "urn:org:tug:texhyphen:languagedata";
- public static int lineLength = 72;
- private static final int TOP_LEVEL = 3, IN_LANG = 4;
- private static final Pattern
- comment = Pattern.compile("#.*"),
- langStart = Pattern.compile("{", Pattern.LITERAL),
- langEnd = Pattern.compile("}", Pattern.LITERAL),
- dataline = Pattern.compile("\"([^\"]+)\" ?=> ?\"([^\"]+)\","),
- keywordline = Pattern.compile("\"([^\"]+)\" ?=> ?(false|true|nil),"),
- listline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[(\"[^\"]+\"(?:,\"[^\"]+\")*)\\],"),
- datalistline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[([^,]+(?:,[^,]+)*)\\],"),
- space = Pattern.compile("[ \\t]+");
- private static final AttributesImpl emptyAtts = new AttributesImpl();
-
- private ContentHandler contentHandler;
- private DTDHandler dtdHandler;
- private EntityResolver entityResolver;
- private ErrorHandler errorHandler;
- private LexicalHandler lexicalHandler;
-
- private void parseLanguageData(BufferedReader inbr) throws SAXException, IOException {
- int parseState = TOP_LEVEL;
- Language lang = null;
-
- contentHandler.startDocument();
- contentHandler.startPrefixMapping("", LANG_NAMESPACE);
- contentHandler.startElement(LANG_NAMESPACE, "languages", "languages", emptyAtts);
-
- for (String line = inbr.readLine(); line != null; line = inbr.readLine()) {
- Matcher matcher = comment.matcher(line).useAnchoringBounds(true);
- int start = 0;
- while (start < line.length()) {
- if (matcher.usePattern(comment).lookingAt()) {
- processComment(matcher.group(), parseState == TOP_LEVEL ? null : lang);
- } else if (matcher.usePattern(space).lookingAt()) {
- // do nothing
- } else if (parseState == TOP_LEVEL && matcher.usePattern(langStart).lookingAt()) {
- parseState = IN_LANG;
- lang = new Language();
- } else if ((parseState == IN_LANG) && matcher.usePattern(langEnd).lookingAt()) {
- pushoutLanguage(lang);
- lang = null;
- parseState = TOP_LEVEL;
- } else if (parseState == IN_LANG
- && (matcher.usePattern(dataline).lookingAt()
- || matcher.usePattern(keywordline).lookingAt())) {
- String key = matcher.group(1);
- String value = matcher.group(2);
- processDataline(key, value, lang);
- } else if (parseState == IN_LANG
- && (matcher.usePattern(listline).lookingAt()
- || matcher.usePattern(datalistline).lookingAt())) {
- String key = matcher.group(1);
- String values = matcher.group(2);
- processListline(key, values, lang);
- } else {
- break;
- }
- start = matcher.end();
- matcher = matcher.region(start, line.length()).useAnchoringBounds(true);
- }
- }
-
- contentHandler.endElement(LANG_NAMESPACE, "languages", "languages");
- contentHandler.endPrefixMapping(LANG_NAMESPACE);
- contentHandler.endDocument();
- }
-
- static Collection<String> attributeKeys;
- static {
- attributeKeys = new Vector<String>();
- attributeKeys.add("code");
- attributeKeys.add("name");
- attributeKeys.add("use-old-patterns");
- attributeKeys.add("use-new-loader");
- attributeKeys.add("encoding");
- attributeKeys.add("exceptions");
- }
-
- private void processComment(String comment, Language lang) throws SAXException {
- comment = comment.replace("--", "––");
- if (!comment.endsWith(" ")) {
- comment = comment + " ";
- }
- if (lang == null) {
- char[] textchars = comment.toCharArray();
- if (lexicalHandler != null) {
- lexicalHandler.comment(textchars, 1, textchars.length - 1);
- }
- } else {
- lang.elements.add(new Element("comment", comment));
- }
-
- }
-
- private void processDataline(String key, String value, Language lang) {
- key = key.replace('_', '-');
- if (value.equals("nil")) {
- value = "";
- }
- if (attributeKeys.contains(key)) {
- lang.atts.addAttribute("", key, key, "CDATA", value);
- } else {
- lang.elements.add(new Element(key, value));
- }
- }
-
- private void processListline(String key, String valuesString, Language lang) {
- key = key.replace('_', '-');
- valuesString = valuesString.replace("\"", "");
- String[] values = valuesString.split(",[ \\t]*");
- if (attributeKeys.contains(key)) {
- StringBuilder attValue = new StringBuilder();
- for (String value : values) {
- if (!value.equals("nil")) {
- attValue.append(" " + value);
- }
- }
- lang.atts.addAttribute("", key, key, "CDATA", attValue.toString());
- } else if (key.equals("hyphenmin")) {
- key = "hyphen-min";
- AttributesImpl atts = new AttributesImpl();
- atts.addAttribute("", "before", "before", "CDATA", values[0]);
- atts.addAttribute("", "after", "after", "CDATA", values[1]);
- lang.elements.add(new Element(key, "", atts));
- } else {
- key = key.replaceAll("s$", "");
- for (String value : values) {
- if (value.equals("nil")) {
- value = "";
- }
- lang.elements.add(new Element(key, value));
- }
- }
- }
-
- private void pushoutLanguage(Language lang) throws SAXException {
- contentHandler.startElement(LANG_NAMESPACE, "language", "language", lang.atts);
- Iterator<Element> iter = lang.elements.iterator();
- while (iter.hasNext()) {
- Element elt = iter.next();
- char[] text = elt.content.toCharArray();
- if (elt.tag.equals("comment")) {
- if (lexicalHandler != null) {
- lexicalHandler.comment(text, 1, text.length - 1);
- }
- } else {
- contentHandler.startElement(LANG_NAMESPACE, elt.tag, elt.tag, elt.atts);
- contentHandler.characters(text, 0, text.length);
- contentHandler.endElement(LANG_NAMESPACE, elt.tag, elt.tag);
- }
- }
- contentHandler.endElement(LANG_NAMESPACE, "language", "language");
- }
-
- public Reader getReaderFromInputSource(InputSource input) throws IOException {
- Reader reader = input.getCharacterStream();
- String encoding = null;
- if (reader == null) {
- encoding = input.getEncoding();
- }
- if (reader == null) {
- InputStream stream = input.getByteStream();
- if (stream != null) {
- if (encoding == null) {
- reader = new InputStreamReader(stream);
- } else {
- reader = new InputStreamReader(stream, encoding);
- }
- }
- }
- if (reader == null) {
- String systemId = input.getSystemId();
- reader = getReaderFromSystemId(systemId, encoding);
- }
- return reader;
- }
-
- public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException {
- if (systemId == null) {
- throw new IOException("Cannot create a reader from a null systemID");
- }
- if (encoding.isEmpty()) {
- encoding = null;
- }
- Reader reader = null;
- URI uri = null;
- File file = null;
- try {
- uri = new URI(systemId);
- } catch (URISyntaxException e) {
- // handled below
- }
- if (uri == null || !uri.isAbsolute()) {
- file = new File(systemId);
- }
- if (file != null) {
- if (encoding == null) {
- reader = new FileReader(file);
- } else {
- InputStream stream = new FileInputStream(file);
- reader = new InputStreamReader(stream, encoding);
- }
- } else if (uri != null && uri.getScheme().equals("http")) {
- URL url = uri.toURL();
- URLConnection conn = url.openConnection();
- if (encoding == null) {
- encoding = conn.getContentEncoding();
- }
- InputStream stream = conn.getInputStream();
- reader = new InputStreamReader(stream, encoding);
- }
- return reader;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getContentHandler()
- */
- public ContentHandler getContentHandler() {
- return contentHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getDTDHandler()
- */
- public DTDHandler getDTDHandler() {
- return dtdHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getEntityResolver()
- */
- public EntityResolver getEntityResolver() {
- return entityResolver;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getErrorHandler()
- */
- public ErrorHandler getErrorHandler() {
- return errorHandler;
- }
-
-
- /**
- * @return the lexicalHandler
- */
- public LexicalHandler getLexicalHandler() {
- return lexicalHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getFeature(java.lang.String)
- */
- public boolean getFeature(String arg0)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- throw new SAXNotSupportedException();
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getProperty(java.lang.String)
- */
- public Object getProperty(String arg0)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- throw new SAXNotSupportedException();
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
- */
- public void parse(InputSource input) throws IOException, SAXException {
- Reader reader = getReaderFromInputSource(input);
- if (reader == null) {
- throw new IOException("Could not open input source " + input);
- }
- BufferedReader inbr = new BufferedReader(reader);
- parseLanguageData(inbr);
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#parse(java.lang.String)
- */
- public void parse(String systemId) throws IOException, SAXException {
- Reader reader = getReaderFromSystemId(systemId, null);
- if (reader == null) {
- throw new IOException("Could not open input systemID " + systemId);
- }
- BufferedReader inbr = new BufferedReader(reader);
- parseLanguageData(inbr);
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler)
- */
- public void setContentHandler(ContentHandler contenthandler) {
- this.contentHandler = contenthandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler)
- */
- public void setDTDHandler(DTDHandler dtdhandler) {
- this.dtdHandler = dtdhandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver)
- */
- public void setEntityResolver(EntityResolver entityresolver) {
- this.entityResolver = entityresolver;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler)
- */
- public void setErrorHandler(ErrorHandler errorHandler) {
- this.errorHandler = errorHandler;
- }
-
-
- /**
- * @param lexicalHandler the lexicalHandler to set
- */
- public void setLexicalHandler(LexicalHandler lexicalHandler) {
- this.lexicalHandler = lexicalHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean)
- */
- public void setFeature(String arg0, boolean arg1)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- throw new SAXNotSupportedException();
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object)
- */
- public void setProperty(String name, Object value)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- if (name.equals("http://xml.org/sax/properties/lexical-handler")) {
- lexicalHandler = (LexicalHandler) value;
- } else {
- throw new SAXNotSupportedException();
- }
- }
-
- private static class Element {
- String tag;
- String content;
- Attributes atts;
- Element(String tag, String content) {
- this(tag, content, LanguageDataParser.emptyAtts);
- }
- Element(String tag, String content, Attributes atts) {
- this.tag = tag;
- this.content = content;
- this.atts = atts;
- }
- }
-
- private static class Language {
- AttributesImpl atts;
- List<Element> elements;
- Language() {
- atts = new AttributesImpl();
- elements = new Vector<Element>();
- }
- }
-
-}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README
deleted file mode 100644
index aaf4e074ae6..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README
+++ /dev/null
@@ -1,38 +0,0 @@
-This utility consists of two parts.
-
-Part 1 converts languages.rb to languages.xml. It consists of
-ConvertLanguageData.java, LanguageDataParser.java,
-ConvertLanguageData.xsl, codemapping.xml. Invocation:
-
-java org.tug.texhyphen.util.ConvertLanguageData path/to/languages.rb
-
-output: languages.xml, in the same directory as languages.rb.
-
-The file codemapping contains the mapping from tex-code to
-fop-code. If this file has an entry with non-empty fop-code, it is
-inserted in languages.xml as attribute fop-code. If this file has an
-entry with empty fop-code, that language gets no attribute
-fop-code. Otherwise, that language gets attribute fop-code with the
-same value as tex-code (attribute code).
-
-The current file languages.xml was modified manually after conversion,
-to add the three languages at the bottom of the file.
-
-Part 2 converts tex pattern files to fop pattern files. It consists of
-ConvertTeXPattern.java, TeXPatternParser.java, ConvertTeXPattern.xsl,
-languages.xml. Invocation:
-
-java org.tug.texhyphen.util.ConvertTeXPattern [--uselanguagedata]
-[--texcodes code1,code2,...] infileuri1 [infileuri2 ...] outfile
-
-infileuri must be a file or http URI. With a file URI, it may also be
-a directory; in that case all pattern files in that directory are
-converted. outfile must be a path to an output filename or
-directory. If there are more input files, it must be an output
-directory. Option 'uselanguagedata' converts all input files which
-represent a language in the languages file with a fop-code. Option
-'--texcodes code1,code2,...' converts only those input files which
-represent a language with one of the given tex codes. Without those
-options all input files are converted. In any case, only files are
-converted which represent a language in the languages file with a
-fop-code.
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java
deleted file mode 100644
index 0cc5a542f78..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java
+++ /dev/null
@@ -1,325 +0,0 @@
-/*
- * Copyright Simon Pepping 2009
- *
- * The copyright owner licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/* $Id: TeXPatternParser.java 304 2009-11-26 07:26:55Z spepping $ */
-
-package org.tug.texhyphen;
-
-import java.io.BufferedReader;
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileReader;
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.InputStreamReader;
-import java.io.Reader;
-import java.net.URI;
-import java.net.URISyntaxException;
-import java.net.URL;
-import java.net.URLConnection;
-import java.util.Stack;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-
-import org.xml.sax.ContentHandler;
-import org.xml.sax.DTDHandler;
-import org.xml.sax.EntityResolver;
-import org.xml.sax.ErrorHandler;
-import org.xml.sax.InputSource;
-import org.xml.sax.SAXException;
-import org.xml.sax.SAXNotRecognizedException;
-import org.xml.sax.SAXNotSupportedException;
-import org.xml.sax.XMLReader;
-import org.xml.sax.ext.LexicalHandler;
-import org.xml.sax.helpers.AttributesImpl;
-
-/**
- * The class TeXParser parses TeX hyphenation pattern files and produces SAX events
- */
-public class TeXPatternParser implements XMLReader {
-
- public static final String TEX_NAMESPACE = "urn:org:tug:texhyphen";
- private static final int TOP_LEVEL = 3, IN_COMMAND = 4, AFTER_COMMAND = 5, IN_DATA = 6;
- private static final Pattern
- comment = Pattern.compile("%.*"),
- commandStart = Pattern.compile("\\\\"),
- command = Pattern.compile("[a-zA-Z]+"),
- space = Pattern.compile(" +"),
- argOpen = Pattern.compile("\\{"),
- argClose = Pattern.compile("\\}"),
- text = Pattern.compile("[^%\\\\\\{\\}]+");
- private static final AttributesImpl emptyAtts = new AttributesImpl();
-
- private ContentHandler contentHandler;
- private DTDHandler dtdHandler;
- private EntityResolver entityResolver;
- private ErrorHandler errorHandler;
- private LexicalHandler lexicalHandler;
-
- private void parsePatterns(BufferedReader inbr) throws SAXException, IOException {
- int parseState = TOP_LEVEL;
- Stack<String> stack = new Stack<String>();
-
- contentHandler.startDocument();
- contentHandler.startPrefixMapping("", TEX_NAMESPACE);
- contentHandler.startElement(TEX_NAMESPACE, "tex", "tex", emptyAtts);
-
- for (String line = inbr.readLine(); line != null; line = inbr.readLine()) {
- Matcher matcher = comment.matcher(line).useAnchoringBounds(true);
- int start = 0;
- char[] textchars;
- boolean inComment = false;
- while (start < line.length()) {
- if (matcher.usePattern(comment).lookingAt()) {
- String text = matcher.group().replace("--", "––");
- textchars = (text + "\n").toCharArray();
- if (lexicalHandler != null) {
- lexicalHandler.comment(textchars, 1, textchars.length - 1);
- }
- inComment = true;
- } else if (parseState == IN_DATA && matcher.usePattern(text).lookingAt()) {
- textchars = matcher.group().toCharArray();
- contentHandler.characters(textchars, 0, textchars.length);
- } else if (parseState != IN_DATA && matcher.usePattern(space).lookingAt()) {
- if (parseState == TOP_LEVEL) {
- textchars = matcher.group().toCharArray();
- contentHandler.ignorableWhitespace(textchars, 0, textchars.length);
- }
- } else if (parseState == TOP_LEVEL && matcher.usePattern(commandStart).lookingAt()) {
- parseState = IN_COMMAND;
- } else if (parseState == IN_COMMAND && matcher.usePattern(command).lookingAt()) {
- String tag = matcher.group();
- contentHandler.startElement(TEX_NAMESPACE, tag, tag, emptyAtts);
- stack.push(tag);
- parseState = AFTER_COMMAND;
- } else if (parseState == AFTER_COMMAND && matcher.usePattern(argOpen).lookingAt()) {
- parseState = IN_DATA;
- } else if (parseState == IN_DATA && matcher.usePattern(argClose).lookingAt()) {
- String tag = stack.pop();
- contentHandler.endElement(TEX_NAMESPACE, tag, tag);
- parseState = TOP_LEVEL;
- } else {
- break;
- }
- start = matcher.end();
- matcher = matcher.region(start, line.length()).useAnchoringBounds(true);
- }
- textchars = "\n".toCharArray();
- if (parseState == IN_DATA && !inComment) {
- contentHandler.characters(textchars, 0, textchars.length);
- } else if (parseState == TOP_LEVEL && !inComment) {
- contentHandler.ignorableWhitespace(textchars, 0, textchars.length);
- }
- }
-
- contentHandler.endElement(TEX_NAMESPACE, "tex", "tex");
- contentHandler.endPrefixMapping(TEX_NAMESPACE);
- contentHandler.endDocument();
- }
-
- public Reader getReaderFromInputSource(InputSource input) throws IOException {
- Reader reader = input.getCharacterStream();
- String encoding = null;
- if (reader == null) {
- encoding = input.getEncoding();
- }
- if (reader == null) {
- InputStream stream = input.getByteStream();
- if (stream != null) {
- if (encoding == null) {
- reader = new InputStreamReader(stream);
- } else {
- reader = new InputStreamReader(stream, encoding);
- }
- }
- }
- if (reader == null) {
- String systemId = input.getSystemId();
- reader = getReaderFromSystemId(systemId, encoding);
- }
- return reader;
- }
-
- public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException {
- if (systemId == null) {
- throw new IOException("Cannot create a reader from a null systemID");
- }
- if (encoding.isEmpty()) {
- encoding = null;
- }
- Reader reader = null;
- URI uri = null;
- File file = null;
- try {
- uri = new URI(systemId);
- } catch (URISyntaxException e) {
- // handled below
- }
- if (uri == null || !uri.isAbsolute()) {
- file = new File(systemId);
- }
- if (file != null) {
- if (encoding == null) {
- reader = new FileReader(file);
- } else {
- InputStream stream = new FileInputStream(file);
- reader = new InputStreamReader(stream, encoding);
- }
- } else if (uri != null && uri.getScheme().equals("http")) {
- URL url = uri.toURL();
- URLConnection conn = url.openConnection();
- if (encoding == null) {
- encoding = conn.getContentEncoding();
- }
- InputStream stream = conn.getInputStream();
- reader = new InputStreamReader(stream, encoding);
- }
- return reader;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getContentHandler()
- */
- public ContentHandler getContentHandler() {
- return contentHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getDTDHandler()
- */
- public DTDHandler getDTDHandler() {
- return dtdHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getEntityResolver()
- */
- public EntityResolver getEntityResolver() {
- return entityResolver;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getErrorHandler()
- */
- public ErrorHandler getErrorHandler() {
- return errorHandler;
- }
-
-
- /**
- * @return the lexicalHandler
- */
- public LexicalHandler getLexicalHandler() {
- return lexicalHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getFeature(java.lang.String)
- */
- public boolean getFeature(String arg0)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- throw new SAXNotSupportedException();
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#getProperty(java.lang.String)
- */
- public Object getProperty(String arg0)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- throw new SAXNotSupportedException();
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
- */
- public void parse(InputSource input) throws IOException, SAXException {
- Reader reader = getReaderFromInputSource(input);
- if (reader == null) {
- throw new IOException("Could not open input source " + input);
- }
- BufferedReader inbr = new BufferedReader(reader);
- parsePatterns(inbr);
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#parse(java.lang.String)
- */
- public void parse(String systemId) throws IOException, SAXException {
- Reader reader = getReaderFromSystemId(systemId, null);
- if (reader == null) {
- throw new IOException("Could not open input systemID " + systemId);
- }
- BufferedReader inbr = new BufferedReader(reader);
- parsePatterns(inbr);
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler)
- */
- public void setContentHandler(ContentHandler contenthandler) {
- this.contentHandler = contenthandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler)
- */
- public void setDTDHandler(DTDHandler dtdhandler) {
- this.dtdHandler = dtdhandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver)
- */
- public void setEntityResolver(EntityResolver entityresolver) {
- this.entityResolver = entityresolver;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler)
- */
- public void setErrorHandler(ErrorHandler errorHandler) {
- this.errorHandler = errorHandler;
- }
-
-
- /**
- * @param lexicalHandler the lexicalHandler to set
- */
- public void setLexicalHandler(LexicalHandler lexicalHandler) {
- this.lexicalHandler = lexicalHandler;
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean)
- */
- public void setFeature(String arg0, boolean arg1)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- throw new SAXNotSupportedException();
- }
-
- /* (non-Javadoc)
- * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object)
- */
- public void setProperty(String name, Object value)
- throws SAXNotRecognizedException, SAXNotSupportedException {
- if (name.equals("http://xml.org/sax/properties/lexical-handler")) {
- lexicalHandler = (LexicalHandler) value;
- } else {
- throw new SAXNotSupportedException();
- }
- }
-
-}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml
deleted file mode 100644
index c5b64722fa0..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml
+++ /dev/null
@@ -1,23 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<code-mappings xmlns="urn:org:tug:texhyphen:languagedata">
- <code-mapping code="grc-x-ibycus" fop-code="grc_X_ibycus"/>
- <code-mapping code="zh-latn" fop-code="zh_Latn"/>
- <code-mapping code="de-1901" fop-code="de_1901"/>
- <code-mapping code="de-1996" fop-code="de"/>
- <code-mapping code="de-ch-1901" fop-code="de_CH_1901"/>
- <code-mapping code="el-monoton" fop-code="el"/>
- <code-mapping code="el-polyton" fop-code="el_Polyton"/>
- <code-mapping code="mn-cyrl" fop-code="mn"/>
- <code-mapping code="mn-cyrl-x-2a" fop-code="mn_Cyrl_x_2a"/>
- <code-mapping code="sh-cyrl" fop-code="sh_Cyrl"/>
- <code-mapping code="sh-latn" fop-code="sh_Latn"/>
- <code-mapping code="sr-cyrl" fop-code="sr"/>
- <code-mapping code="sr-latn" fop-code="sr_Latn"/>
- <code-mapping code="en-gb" fop-code="en_GB"/>
- <code-mapping code="en-us" fop-code="en"/>
- <code-mapping code="eo" fop-code=""/>
- <code-mapping code="nn" fop-code=""/>
- <code-mapping code="nb" fop-code=""/>
- <!-- hu patterns cause a stack overflow when compiled with FOP -->
- <code-mapping code="hu" fop-code=""/>
-</code-mappings>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml
deleted file mode 100644
index 2f0fd637d3c..00000000000
--- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml
+++ /dev/null
@@ -1,434 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<languages xmlns="urn:org:tug:texhyphen:languagedata">
- <!-- –––––––––––––––––––––––––––––––––––––– -->
- <!-- languages with no hyphenation patterns -->
- <!-- –––––––––––––––––––––––––––––––––––––– -->
- <!-- arabic zerohyph.tex -->
- <language fop-code="ar" code="ar" name="arabic" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false">
- <filename-old-patterns>zerohyph.tex</filename-old-patterns>
- <!-- "hyphenmin" => [], # not needed -->
- <message/>
- </language>
- <!-- farsi zerohyph.tex -->
- <!-- =persian -->
- <language fop-code="fa" code="fa" name="farsi" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false">
- <synonym>persian</synonym>
- <filename-old-patterns>zerohyph.tex</filename-old-patterns>
- <!-- "hyphenmin" => [], # not needed -->
- <message/>
- </language>
- <!-- ––––––––––––––––––––––––––––––- -->
- <!-- special patterns, not converted -->
- <!-- ––––––––––––––––––––––––––––––- -->
- <!-- ibycus ibyhyph.tex -->
- <language fop-code="grc_X_ibycus" code="grc-x-ibycus" name="ibycus" use-new-loader="false" use-old-patterns="true" encoding="" exceptions="false">
- <filename-old-patterns>ibyhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Ancient Greek Hyphenation Patterns for Ibycus encoding (v3.0)</message>
- <author>peter_heslin</author>
- </language>
- <!-- –––––––––––––––––––––––––––– -->
- <!-- languages using old patterns -->
- <!-- –––––––––––––––––––––––––––– -->
- <!-- greek xu-grphyph4.tex -->
- <!-- =polygreek -->
- <language fop-code="el_Polyton" code="el-polyton" name="greek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true">
- <synonym>polygreek</synonym>
- <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
- <filename-old-patterns>grphyph5.tex</filename-old-patterns>
- <!-- left/right hyphen min for Greek can be as low as one (1), -->
- <!-- but for aesthetic reasons keep them at 2/2. -->
- <!-- Dimitrios Filippou -->
- <hyphen-min before="1" after="1"/>
- <!-- polyglosia -->
- <!-- "message" => "Polytonic Greek Hyphenation Patterns", -->
- <message>Hyphenation patterns for multi-accent (polytonic) Modern Greek</message>
- <author>dimitrios_filippou</author>
- </language>
- <!-- monogreek xu-grmhyph4.tex -->
- <language fop-code="el" code="el-monoton" name="monogreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true">
- <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
- <filename-old-patterns>grmhyph5.tex</filename-old-patterns>
- <hyphen-min before="1" after="1"/>
- <!-- polyglosia -->
- <!-- "message" => "Monotonic Greek Hyphenation Patterns", -->
- <message>Hyphenation patterns for uni-accent (monotonic) Modern Greek</message>
- <author>dimitrios_filippou</author>
- </language>
- <!-- ancientgreek xu-grahyph4.tex -->
- <language fop-code="grc" code="grc" name="ancientgreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false">
- <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
- <filename-old-patterns>grahyph5.tex</filename-old-patterns>
- <hyphen-min before="1" after="1"/>
- <!-- polyglosia -->
- <message>Hyphenation patterns for Ancient Greek</message>
- <author>dimitrios_filippou</author>
- </language>
- <!-- coptic xu-copthyph.tex -->
- <language fop-code="cop" code="cop" name="coptic" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false">
- <use-old-patterns-comment>TODO: automatic conversion could be done, but was too complicated; leave for later.</use-old-patterns-comment>
- <filename-old-patterns>copthyph.tex</filename-old-patterns>
- <hyphen-min before="1" after="1"/>
- <!-- polyglosia TODO: no documentation found -->
- <message>Coptic Hyphenation Patterns</message>
- <author>claudio_beccari</author>
- </language>
- <!-- german xu-dehypht.tex -->
- <language fop-code="de_1901" code="de-1901" name="german" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false">
- <use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment>
- <filename-old-patterns>dehypht.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>German Hyphenation Patterns (Traditional Orthography)</message>
- </language>
- <!-- ngerman xu-dehyphn.tex -->
- <language fop-code="de" code="de-1996" name="ngerman" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false">
- <use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment>
- <filename-old-patterns>dehyphn.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>German Hyphenation Patterns (Reformed Orthography)</message>
- </language>
- <!-- swissgerman -->
- <language fop-code="de_CH_1901" code="de-ch-1901" name="swissgerman" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <!-- TODO: how is it going to be called -->
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Swiss-German Hyphenation Patterns (Traditional Orthography)</message>
- </language>
- <!-- russian xu-ruhyphen.tex -->
- <language fop-code="ru" code="ru" name="russian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false">
- <use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment>
- <filename-old-patterns>ruhyphen.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Russian Hyphenation Patterns</message>
- </language>
- <!-- ukrainian xu-ukrhyph.tex -->
- <language fop-code="uk" code="uk" name="ukrainian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false">
- <use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment>
- <filename-old-patterns>ukrhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Ukrainian Hyphenation Patterns</message>
- </language>
- <!-- –––––––––––––––––––––––––––– -->
- <!-- languages using new patterns -->
- <!-- –––––––––––––––––––––––––––– -->
- <!-- catalan cahyph.tex -->
- <language fop-code="ca" code="ca" name="catalan" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>cahyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Catalan Hyphenation Patterns</message>
- </language>
- <!-- czech xu-czhyph.tex -->
- <language fop-code="cs" code="cs" name="czech" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>czhyph.tex</filename-old-patterns>
- <filename-old-patterns-other>czhyphen.tex</filename-old-patterns-other>
- <filename-old-patterns-other>czhyphen.ex</filename-old-patterns-other>
- <!-- Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 -->
- <!-- Typographical rules allow \righthyphenmin=2 when typesetting in a -->
- <!-- narrow column (newspapers etc.). -->
- <!-- (used to be 2,2) -->
- <hyphen-min before="2" after="3"/>
- <message>Czech Hyphenation Patterns (Pavel Sevecek, v3, 1995)</message>
- </language>
- <!-- slovak xu-skhyph.tex -->
- <language fop-code="sk" code="sk" name="slovak" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>skhyph.tex</filename-old-patterns>
- <filename-old-patterns-other>skhyphen.tex</filename-old-patterns-other>
- <filename-old-patterns-other>skhyphen.ex</filename-old-patterns-other>
- <!-- see czech -->
- <hyphen-min before="2" after="3"/>
- <message>Slovak Hyphenation Patterns (Jana Chlebikova, 1992)</message>
- </language>
- <!-- welsh cyhyph.tex -->
- <language fop-code="cy" code="cy" name="welsh" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>cyhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="3"/>
- <message>Welsh Hyphenation Patterns</message>
- </language>
- <!-- danish dkhyph.tex -->
- <language fop-code="da" code="da" name="danish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>dkhyph.tex</filename-old-patterns>
- <filename-old-patterns-other>dkcommon.tex</filename-old-patterns-other>
- <filename-old-patterns-other>dkspecial.tex</filename-old-patterns-other>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Danish Hyphenation Patterns</message>
- </language>
- <!-- esperanto xu-eohyph.tex -->
- <!-- TODO -->
- <language code="eo" name="esperanto" use-new-loader="true" use-old-patterns="false" encoding="il3" exceptions="false">
- <filename-old-patterns>eohyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- TODO -->
- <message>Esperanto Hyphenation Patterns</message>
- </language>
- <!-- spanish xu-eshyph.tex -->
- <!-- =espanol -->
- <language fop-code="es" code="es" name="spanish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <synonym>espanol</synonym>
- <filename-old-patterns>eshyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Spanish Hyphenation Patterns</message>
- </language>
- <!-- basque xu-bahyph.tex -->
- <language fop-code="eu" code="eu" name="basque" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>bahyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Basque Hyphenation Patterns</message>
- </language>
- <!-- french xu-frhyph.tex -->
- <!-- =patois -->
- <!-- =francais -->
- <language fop-code="fr" code="fr" name="french" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <synonym>patois</synonym>
- <synonym>francais</synonym>
- <filename-old-patterns>frhyph.tex</filename-old-patterns>
- <!-- "hyphenmin" => [], -->
- <message>French hyphenation patterns (V2.12, 2002/12/11)</message>
- </language>
- <!-- galician xu-glhyph.tex -->
- <language fop-code="gl" code="gl" name="galician" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>glhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Galician Hyphenation Patterns</message>
- </language>
- <!-- estonian xu-ethyph.tex -->
- <language fop-code="et" code="et" name="estonian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>ethyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="3"/>
- <!-- babel -->
- <message>Estonian Hyphenation Patterns</message>
- </language>
- <!-- finnish fihyph.tex -->
- <language fop-code="fi" code="fi" name="finnish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>fihyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Finnish Hyphenation Patterns</message>
- </language>
- <!-- croatian xu-hrhyph.tex -->
- <language fop-code="hr" code="hr" name="croatian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>hrhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Croatian Hyphenation Patterns</message>
- </language>
- <!-- hungarian xu-huhyphn.tex -->
- <language code="hu" name="hungarian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>huhyphn.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- polyglosia -->
- <message>Hungarian Hyphenation Patterns (v20031107)</message>
- </language>
- <!-- interlingua iahyphen.tex -->
- <language fop-code="ia" code="ia" name="interlingua" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
- <filename-old-patterns>iahyphen.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Hyphenation Patterns for Interlingua</message>
- </language>
- <!-- indonesian inhyph.tex -->
- <language fop-code="id" code="id" name="indonesian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
- <filename-old-patterns>inhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Indonesian Hyphenation Patterns</message>
- </language>
- <!-- icelandic icehyph.tex -->
- <language fop-code="is" code="is" name="icelandic" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>icehyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Icelandic Hyphenation Patterns</message>
- </language>
- <!-- irish gahyph.tex -->
- <language fop-code="ga" code="ga" name="irish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>gahyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="3"/>
- <!-- babel -->
- <message>Irish Hyphenation Patterns</message>
- </language>
- <!-- italian ithyph.tex -->
- <language fop-code="it" code="it" name="italian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="false">
- <filename-old-patterns>ithyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Italian Hyphenation Patterns</message>
- </language>
- <!-- kurmanji -->
- <language fop-code="kmr" code="kmr" name="kurmanji" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>kmrhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Kurmanji Hyphenation Patterns (v. 1.0 2009/06/29 JKn and MSh)</message>
- </language>
- <!-- latin xu-lahyph.tex -->
- <language fop-code="la" code="la" name="latin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <use-old-patterns-comment>Old patterns support both EC &amp; OT1 encodings at the same time.</use-old-patterns-comment>
- <filename-old-patterns>lahyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Latin Hyphenation Patterns</message>
- </language>
- <!-- lithuanian -->
- <language fop-code="lt" code="lt" name="lithuanian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false">
- <hyphen-min before="2" after="2"/>
- <message>Lithuanian Hyphenation Patterns</message>
- </language>
- <!-- latvian -->
- <language fop-code="lv" code="lv" name="latvian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false">
- <hyphen-min before="2" after="2"/>
- <message>Latvian Hyphenation Patterns</message>
- </language>
- <!-- dutch nehyph96.tex -->
- <language fop-code="nl" code="nl" name="dutch" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>nehyph96.tex</filename-old-patterns>
- <!-- quoting Hans Hagen: -->
- <!-- patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) -->
- <hyphen-min before="2" after="2"/>
- <message>Dutch Hyphenation Patterns</message>
- </language>
- <!-- norsk xu-nohyphbx.tex -->
- <!-- =norwegian -->
- <!-- nynorsk nnhyph.tex -->
- <!-- bokmal nbhyph.tex -->
- <!-- polish xu-plhyph.tex -->
- <language fop-code="pl" code="pl" name="polish" use-new-loader="true" use-old-patterns="false" encoding="qx" exceptions="true">
- <filename-old-patterns>plhyph.tex</filename-old-patterns>
- <!--{}"hyphenmin" => [1,1], -->
- <hyphen-min before="2" after="2"/>
- <message>Polish Hyphenation Patterns</message>
- </language>
- <!-- portuguese pthyph.tex -->
- <!-- =portuges -->
- <language fop-code="pt" code="pt" name="portuguese" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <synonym>portuges</synonym>
- <filename-old-patterns>pthyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="3"/>
- <!-- babel -->
- <message>Portuguese Hyphenation Patterns</message>
- </language>
- <!-- pinyin xu-pyhyph.tex -->
- <language fop-code="zh_Latn" code="zh-latn" name="pinyin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>pyhyph.tex</filename-old-patterns>
- <hyphen-min before="1" after="1"/>
- <message>Hyphenation patterns for unaccented pinyin syllables (CJK 4.8.0)</message>
- </language>
- <!-- romanian xu-rohyphen.tex -->
- <language fop-code="ro" code="ro" name="romanian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>rohyphen.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Romanian Hyphenation Patterns</message>
- <!-- : `rohyphen' 1.1 <29.10.1996> -->
- </language>
- <!-- slovenian xu-sihyph.tex -->
- <!-- =slovene -->
- <language fop-code="sl" code="sl" name="slovenian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <synonym>slovene</synonym>
- <filename-old-patterns>sihyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Slovenian Hyphenation Patterns</message>
- </language>
- <!-- uppersorbian xu-sorhyph.tex -->
- <language fop-code="hsb" code="hsb" name="uppersorbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>sorhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Upper Sorbian Hyphenation Patterns (E. Werner)</message>
- <!-- \message{Hyphenation patterns for Upper Sorbian, E. Werner} -->
- <!-- \message{Completely new revision 1997, March 22} -->
- </language>
- <!-- swedish svhyph.tex -->
- <language fop-code="sv" code="sv" name="swedish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>svhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- patters say it could be 1,2; babel says 2,2 - double check -->
- <message>Swedish hyphenation patterns (Jan Michael Rynning, 1994-03-03)</message>
- </language>
- <!-- turkish xu-tkhyph.tex -->
- <language fop-code="tr" code="tr" name="turkish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
- <filename-old-patterns>tkhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- polyglosia -->
- <message>Turkish Hyphenation Patterns</message>
- </language>
- <!-- ukenglish ukhyphen.tex -->
- <!-- TODO - should we rename it or not? -->
- <language fop-code="en_GB" code="en-gb" name="ukenglish" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
- <synonym>british</synonym>
- <synonym>UKenglish</synonym>
- <filename-old-patterns>ukhyphen.tex</filename-old-patterns>
- <hyphen-min before="2" after="3"/>
- <!-- confirmed, same as what Knuth says -->
- <message>Hyphenation Patterns for British English</message>
- </language>
- <!-- serbian xu-srhyphc.tex -->
- <language fop-code="sr_Latn" code="sr-latn" name="serbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <filename-old-patterns>shhyphl.tex</filename-old-patterns>
- <!-- It is allowed to leave one character at the end of the row. -->
- <!-- However, if you think that it is graphicaly not very -->
- <!-- pleasent these patterns will work well with \lefthyphenmin=2. -->
- <!-- \lefthyphenmin=1 \righthyphenmin=2 -->
- <hyphen-min before="2" after="2"/>
- <message>Serbian hyphenation patterns in Latin script</message>
- </language>
- <!-- serbianc -->
- <language fop-code="sr" code="sr-cyrl" name="serbianc" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="true">
- <filename-old-patterns>srhyphc.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Serbian hyphenation patterns in Cyrillic script</message>
- </language>
- <!-- mongolian xu-mnhyph.tex -->
- <language fop-code="mn" code="mn-cyrl" name="mongolian" use-new-loader="true" use-old-patterns="false" encoding="lmc" exceptions="false">
- <filename-old-patterns>mnhyph.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>Mongolian hyphenation patterns</message>
- </language>
- <!-- mongolian2a -->
- <language fop-code="mn_Cyrl_x_2a" code="mn-cyrl-x-2a" name="mongolian2a" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false">
- <filename-old-patterns>mnhyphn.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <message>(New) Mongolian Hyphenation Patterns</message>
- </language>
- <!-- bulgarian xu-bghyphen.tex -->
- <language fop-code="bg" code="bg" name="bulgarian" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false">
- <filename-old-patterns>bghyphen.tex</filename-old-patterns>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Bulgarian Hyphenation Patterns</message>
- </language>
- <!-- sanskrit -->
- <language fop-code="sa" code="sa" name="sanskrit" use-new-loader="true" use-old-patterns="false" encoding="" exceptions="false">
- <hyphen-min before="1" after="5"/>
- <!-- polyglosia -->
- <!-- no patterns for 8-bit TeX -->
- <message>Sanskrit Hyphenation Patterns (v0.2, 2008/1/3)</message>
- </language>
- <!-- norsk xu-nohyphbx.tex -->
- <language code="nb" name="bokmal" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <synonym>norwegian</synonym>
- <synonym>norsk</synonym>
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Norwegian Bokmal Hyphenation Patterns</message>
- </language>
- <!-- nynorsk nnhyph.tex -->
- <language code="nn" name="nynorsk" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
- <hyphen-min before="2" after="2"/>
- <!-- babel -->
- <message>Norwegian Nynorsk Hyphenation Patterns</message>
- </language>
- <!-- The following languages are not (yet) in languages.rb. -->
- <!-- I added them after this file was generated from languages.rb. -->
- <language fop-code="en" code="en-us" name="US english">
- <hyphen-min before="2" after="3"/>
- <filename-old-patterns>ushyphmax.tex</filename-old-patterns>
- </language>
- <language fop-code="sh_Latn" code="sh-latn" name="serbocroatian">
- <hyphen-min before="1" after="2"/>
- </language>
- <language fop-code="sh_Cyrl" code="sh-cyrl" name="serbocroatianc">
- <hyphen-min before="1" after="2"/>
- </language>
- <language fop-code="no" code="no" name="norwegian">
- <filename-old-patterns>nohyphbx.tex</filename-old-patterns>
- </language>
-</languages>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb
index bdc920181f9..db699173334 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb
@@ -1,56 +1,17 @@
#!/usr/bin/env ruby
+require 'hyph-utf8'
+
$encoding_data_dir = "data/encodings"
$encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"]
-$output_data_dir = "../../../tex/generic/hyph-utf8/conversions"
-
-class UnicodeCharacter
- def initialize(code_uni, code_enc, name)
- @code_uni = code_uni
- @code_enc = code_enc
- # TODO: might be longer or shorter
- @bytes = [code_uni].pack('U').unpack('H2H2')
- @name = name
- end
-
- attr_reader :code_uni, :code_enc, :bytes, :name
-end
-
-class UnicodeCharacters < Hash
- def add_new_character(code_uni, code_enc, name)
- first_byte = [code_uni].pack('U').unpack('H2').first
- if self[first_byte] == nil then
- self[first_byte] = Array.new
- end
- self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name))
- end
-end
+$path_root=File.expand_path("../../..")
+$output_data_dir = "#{$path_root}/tex/generic/hyph-utf8/conversions"
# 0x19; U+0131; 1; dotlessi
$encodings.each do |encoding|
- #$utf_combinations = Hash.new
- $unicode_characters = UnicodeCharacters.new
+ e = Encoding.new(encoding)
- # those that need lccode to be set
- $lowercase_characters = Array.new
-
- File.open($encoding_data_dir + "/" + encoding + ".dat").grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z\.]*)$/) do |line|
- # puts line
- code_enc = $1.hex
- code_uni = $2.hex
- if $3.length > 0
- type = $3.to_i
- else
- type = 0
- end
- name = $4
- if type == 1 then
- $unicode_characters.add_new_character(code_uni, code_enc, name)
- $lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name))
- end
- end
-
$file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w")
$file_out.puts "% conv-utf8-#{encoding}.tex"
$file_out.puts "%"
@@ -65,7 +26,7 @@ $encodings.each do |encoding|
$file_out.puts "% (But consider adapting the scripts if you need modifications.)"
$file_out.puts "%"
- $unicode_characters.sort.each do |first_byte|
+ e.unicode_characters_first_byte.sort.each do |first_byte|
# sorting all the second characters alphabetically
first_byte[1].sort!{|x,y| x.code_uni <=> y.code_uni }
# make all the possible first characters active
@@ -73,7 +34,7 @@ $encodings.each do |encoding|
$file_out.puts "\\catcode\"#{first_byte[0].upcase}=\\active"
end
$file_out.puts "%"
- $unicode_characters.sort.each do |first_byte|
+ e.unicode_characters_first_byte.sort.each do |first_byte|
$file_out.puts "\\def^^#{first_byte[0]}#1{%"
string_fi = ""
for i in 1..(first_byte[1].size)
@@ -92,7 +53,7 @@ $encodings.each do |encoding|
$file_out.puts "%"
$file_out.puts "% ensure all the chars above have valid \lccode values"
$file_out.puts "%"
- $lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc }.each do |character|
+ e.lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc }.each do |character|
code = [ character.code_enc ].pack("c").unpack("H2").first.upcase
# \lccode"FF="FF
ux_code = sprintf("U+%04X", character.code_uni)
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
index 16e32e975e9..872912f94b3 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
@@ -140,7 +140,7 @@ languages.each do |language|
file.puts('\else')
file.puts(" \\message{#{language.encoding.upcase} #{language.message}}")
# a hack for OT1 encoding in three languages
- if language.code == 'da' or language.code == 'fr' or language.code == 'la' then
+ if language.code == 'da' or language.code == 'fr' then
file.puts(" % A hack to support both EC and OT1 encoding in 8-bit engines.")
file.puts(" % Kept for backward compatibility only, though we would prefer to drop it.")
file.puts(" % OT1 encoding is close-to-useless for proper hyphenation.")
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
new file mode 100755
index 00000000000..08a77340d8b
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb
@@ -0,0 +1,108 @@
+#!/usr/bin/env ruby
+
+require 'hyph-utf8'
+
+# this file generates patterns for pTeX out of the plain ones
+
+# use 'gem install unicode' if unicode is missing on your computer
+# require 'jcode'
+# require 'rubygems'
+# require 'unicode'
+
+load 'languages.rb'
+
+$path_root=File.expand_path("../../..")
+$path_ptex="#{$path_root}/tex/generic/hyph-utf8/patterns/ptex"
+
+# load encodings
+encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"]
+encodings = Hash.new
+encodings_list.each do |encoding_name|
+ encodings[encoding_name] = Encoding.new(encoding_name)
+end
+
+$l = Languages.new
+
+# TODO: should be singleton
+languages = $l.list.sort{|a,b| a.name <=> b.name}
+
+language_codes = Hash.new
+languages.each do |language|
+ language_codes[language.code] = language.code
+end
+language_codes['mn-cyrl-x-lmc'] = nil
+# language_codes['sh-latn'] = 'sr-latn'
+language_codes['sh-cyrl'] = nil
+
+# e = Encoding.new("ec")
+# puts e.convert_string_to_escaped_characters("moja čaša")
+#
+# return
+
+languages.sort{|x,y| x.code <=> y.code }.each do |language|
+ encoding = nil
+ if language.use_new_loader then
+ if language.encoding == nil or language_codes[language.code] == nil
+ include_language = false
+ puts "(skipping #{language.code} # encoding)"
+ elsif language.encoding == 'ascii'
+ include_language = false
+ puts "(skipping #{language.code} # ascii)"
+ else
+ include_language = true
+ encoding = encodings[language.encoding]
+ end
+ else
+ include_language = false
+ puts "(skipping #{language.code} # loader)"
+ end
+
+ code = language_codes[language.code]
+
+ if include_language
+ puts ">> generating #{code} (#{language.name})"
+ file_ptex = File.open("#{$path_ptex}/phyph-#{code}.tex", "w")
+
+ patterns = language.get_patterns
+ exceptions = language.get_exceptions
+
+ if code == 'nn' or code == 'nb'
+ patterns = $l['no'].get_patterns
+ end
+
+ if language.encoding != 'ascii' then
+ patterns = encoding.convert_to_escaped_characters(patterns)
+ exceptions = encoding.convert_to_escaped_characters(exceptions)
+ end
+
+ file_ptex.puts("% pTeX-friendly hyphenation patterns")
+ file_ptex.puts("%")
+ file_ptex.puts("% language: #{language.name} (#{language.code})")
+ file_ptex.puts("% encoding: #{language.encoding}")
+ file_ptex.puts("%")
+ file_ptex.puts("% This file has been auto-generated from hyph-#{language.code}.tex")
+ file_ptex.puts("% with a script [texmf]/scripts/generic/hyph-utf8/generate-ptex-patterns.rb")
+ file_ptex.puts("% See the original file for details about author, licence etc.")
+ file_ptex.puts("%")
+
+ if patterns.length > 0 then
+ # file_ptex.puts("\\patterns{\n#{encoding.convert_to_escaped_characters(patterns.join("\n"))}\n}")
+ file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}")
+ end
+ if exceptions.length > 0 then
+ # file_ptex.puts("\\hyphenation{\n#{encoding.convert_to_escaped_characters(exceptions.join("\n"))}\n}")
+ file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}")
+ end
+
+
+ # # patterns
+ # patterns.each do |pattern|
+ # $file_pat.puts pattern.gsub(/'/,"’")
+ # end
+ # # exceptions
+ # if exceptions != ""
+ # $file_hyp.puts exceptions
+ # end
+ file_ptex.close
+ end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
index e4aff48abfe..dc1c3245a70 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
@@ -120,8 +120,9 @@ language_groups.sort.each do |language_name,language_list|
if language.use_new_loader then
file = "file=loadhyph-#{language.code}.tex"
# we skip the mongolian language
- if language.code != "mn-cyrl-x-lmc" then
-
+ if language.code == "mn-cyrl-x-lmc" then
+ file = "luaspecial=\"disabled:only for 8bit montex with lmc encoding\""
+ else
filename_pat = "hyph-#{language.code}.pat.txt"
filename_hyp = "hyph-#{language.code}.hyp.txt"
@@ -143,6 +144,12 @@ language_groups.sort.each do |language_name,language_list|
end
else
file = "file=#{language.filename_old_patterns}"
+ if language.code == 'ar' or language.code == 'fa' then
+ file = file + " \\\n\tfile_patterns="
+ elsif language.code == 'grc-x-ibycus' then
+ # TODO: fix this
+ file = file + " \\\n\tluaspecial=\"disabled:8-bit only\""
+ end
end
$file_tlpsrc.puts "execute AddHyphen \\\n\t#{name}#{synonyms} \\"
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb b/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb
new file mode 100644
index 00000000000..e0004a76630
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb
@@ -0,0 +1,109 @@
+# this is a Unicode character represented in some particular encoding
+class UnicodeCharacter
+ # unicode code
+ # code in that particular encoding
+ # character name (like 'eacute')
+ def initialize(code_uni, code_enc, name)
+ @code_uni = code_uni
+ @code_enc = code_enc
+ # TODO: might be longer or shorter
+ @bytes = [code_uni].pack('U').unpack('H2H2')
+ @name = name
+ end
+
+ attr_reader :code_uni, :code_enc, :bytes, :name
+end
+
+class UnicodeCharacters < Hash
+ # a hash based on the first character
+ def add_new_character_first_byte(code_uni, code_enc, name)
+ first_byte = [code_uni].pack('U').unpack('H2').first
+ if self[first_byte] == nil then
+ self[first_byte] = Array.new
+ end
+ self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name))
+ end
+ # a hash based on the whole unicode codepoint
+ def add_new_character(code_uni, code_enc, name)
+ self[code_uni] = UnicodeCharacter.new(code_uni, code_enc, name)
+ end
+end
+
+class Encoding
+ def initialize(encoding_name)
+ @encoding_name = encoding_name
+ @unicode_characters_first_byte = UnicodeCharacters.new
+ @unicode_characters = UnicodeCharacters.new
+ @lowercase_characters = Array.new
+
+ if encoding_name != 'ascii' then
+ read_data
+ end
+ end
+
+ def convert_to_escaped_characters(str)
+ if str.kind_of?(Array) then
+ str.each_index do |i|
+ str[i] = convert_string_to_escaped_characters(str[i])
+ end
+ elsif str.kind_of?(String) then
+ str = convert_string_to_escaped_characters(str)
+ end
+ return str
+ end
+
+ attr_reader :encoding_name, :unicode_characters, :unicode_characters_first_byte, :lowercase_characters
+
+ def convert_string_to_escaped_characters(str)
+ characters = str.unpack('U*')
+ new_string = Array.new(characters.length)
+ characters.each_index do |i|
+ c = characters[i] # character code on position i
+ # check if unicode entry with that number exists
+ uc = @unicode_characters[c]
+ if uc == nil then
+ if c < 128 then
+ new_string[i] = [c].pack('U')
+ elsif c == 8217 # ’
+ new_string[i] = "'"
+ else
+ puts "There must be an error: character #{c} in string #{str} is not ASCII."
+ end
+ # an unicode character
+ else
+ new_string[i] = sprintf("^^%x", uc.code_enc)
+ end
+ end
+ return new_string.join('')
+ end
+
+private
+ def read_data
+ # fetch the characters
+ encoding_data_dir = File.expand_path("data/encodings")
+ filename = "#{encoding_data_dir}/#{@encoding_name}.dat"
+
+ if File.exists?(filename) then
+ File.open(filename).grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z\.]*)$/) do |line|
+ # puts line
+ code_enc = $1.hex
+ code_uni = $2.hex
+ if $3.length > 0
+ type = $3.to_i
+ else
+ type = 0
+ end
+ name = $4
+ if type == 1 then
+ @unicode_characters_first_byte.add_new_character_first_byte(code_uni, code_enc, name)
+ @unicode_characters.add_new_character(code_uni, code_enc, name)
+ @lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name))
+ end
+ end
+ else
+ # TODO: throw an error
+ puts "Invalid encoding name '#{@encoding_name}'."
+ puts "File '#{filename}' doesn't exist."
+ end
+ end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
index 450013b7868..fc524fc3520 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
@@ -40,7 +40,7 @@ class Language
def get_patterns
if @patterns == nil
- filename = "../../../tex/generic/hyph-utf8/patterns/tex/hyph-#{@code}.tex";
+ filename = "../../../tex/generic/hyph-utf8/patterns/tex/hyph-#{@code}.tex"
lines = IO.readlines(filename, '.').join("")
@patterns = lines.gsub(/%.*/,'').
gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1').
@@ -49,6 +49,19 @@ class Language
gsub(/\s*$/m,'').
gsub(/'/,"’").
split("\n")
+ # Russian and Ukrainian have some extra patterns with dashes
+ # we may combine these patterns with the main file anyway
+ if @code == 'ru' or @code == 'uk' then
+ filename = "../../../tex/generic/hyph-utf8/patterns/tex-special/exhyph-#{@code}.tex"
+ lines = IO.readlines(filename, '.').join("")
+ @patterns.concat(lines.gsub(/%.*/,'').
+ gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1').
+ gsub(/\s+/m,"\n").
+ gsub(/^\s*/m,'').
+ gsub(/\s*$/m,'').
+ gsub(/'/,"’").
+ split("\n"))
+ end
if @code == 'eo' then
@patterns = lines.gsub(/%.*/,'').
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README b/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README
index 6154587b019..95e00612c7b 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README
@@ -1,25 +1,25 @@
-This directory contains seven files + README, licensed all but one under lppl 1.3:
-
-glpatter-utf8.tex:
- The source file, version 2.4. When processed under the program
- mkpatter ---by typing 'tex -ini -8bit glpatter-utf8.tex' (without the quotes)
- at the command line, provided you have the utility
- mkpattern installed--- it generates the file hyph-utf8-gl.tex.
-
-(hyph-gl.tex):
- The generated file containing the patterns in UTF-8 (belongs to TEXMF/tex).
-
-
-glhyextr.tex: ##This file is in the public domain##
- A configuration file, to include additional patterns other
- than the ones in glpatter-utf8.tex. It is not \input by glpatter-utf8.tex
- (iin contrast with the single-byte encoded version glpatter.tex)
- as modern tools allow the addition of patterns at run time.
-
-glhybiox.tex, glhymed.tex, glhyquim.tex, glhytec.tex & glhyxeog.tex:
- Files with aditional patterns drawn from serveral fields of
- the language, so that you may input them from glhyextr.tex.
-
-The files for specific patterns are very incomplete and feedback is welcome.
-
---Javier A. Mgica
+This directory contains seven files + README, licensed all but one under lppl 1.3:
+
+glpatter-utf8.tex:
+ The source file, version 2.4. When processed under the program
+ mkpatter ---by typing 'tex -ini -8bit glpatter-utf8.tex' (without the quotes)
+ at the command line, provided you have the utility
+ mkpattern installed--- it generates the file hyph-utf8-gl.tex.
+
+(hyph-gl.tex):
+ The generated file containing the patterns in UTF-8 (belongs to TEXMF/tex).
+
+
+glhyextr.tex: ##This file is in the public domain##
+ A configuration file, to include additional patterns other
+ than the ones in glpatter-utf8.tex. It is not \input by glpatter-utf8.tex
+ (iin contrast with the single-byte encoded version glpatter.tex)
+ as modern tools allow the addition of patterns at run time.
+
+glhybiox.tex, glhymed.tex, glhyquim.tex, glhytec.tex & glhyxeog.tex:
+ Files with aditional patterns drawn from serveral fields of
+ the language, so that you may input them from glhyextr.tex.
+
+The files for specific patterns are very incomplete and feedback is welcome.
+
+--Javier A. Mgica
diff --git a/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx b/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx
index 2eac29d4e5c..69e0aaaa698 100644
--- a/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx
+++ b/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx
@@ -106,7 +106,7 @@ This work is under the CC0 license.
%<*driver>
\NeedsTeXFormat{LaTeX2e}
\ProvidesFile{luatex-hyphen.drv}
- [2010/04/28 v1.3beta Hyphenation file for LuaTeX]
+ [2010/04/28 v1.4 Hyphenation file for LuaTeX]
\documentclass{ltxdoc}
\usepackage[ascii]{inputenc}
\usepackage[T1]{fontenc}
@@ -140,7 +140,7 @@ This work is under the CC0 license.
% \GetFileInfo{luatex-hyphen.drv}
%
% \title{The \texttt{hyphen.cfg} file for Lua\TeX }
-% \date{2010/04/28 v1.3beta}
+% \date{2010/04/28 v1.4}
% \author{Khaled Hosny, \'Elie Roux, and Manuel P\'egouri\'e-Gonnard\\
% \texttt{khaledhosny@eglug.org} \\
% \texttt{elie.roux@telecom-bretagne.eu} \\
@@ -205,14 +205,10 @@ This work is under the CC0 license.
% directly without being parsed by \TeX. If one of these keys is
% missing or is the empty string, it is ignored and no patterns (resp.
% exceptions) are loaded for this language. The values of
-% \texttt{*hyphenmin} are currently unused.
+% \texttt{*hyphenmin} values are currently unused.
% \item Special case are supported by a field \verb+special+. Currently,
% the following kind of values are recognized:
% \begin{description}
-% \item[\texttt{'null'}] for languages with no hyphenation patterns
-% nor exceptions. (Note that this is equivalent to both
-% \verb+hyphenation+ and \verb+patterns+ being \verb+nil+ or
-% \verb+''+, but produces a more explicit message in the log.)
% \item[\texttt{'disabled:<reason>'}] allows to disable specific
% languages: when the user tries to load this language, an error
% will be issued, with the \verb+<reason>+.
@@ -325,10 +321,7 @@ function loadlanguage(lname, id)
%
% \begin{macrocode}
if ldata.special then
- if ldata.special == 'null' then
- wlog(msg, ' (null)', cname, id)
- return
- elseif ldata.special:find('^disabled:') then
+ if ldata.special:find('^disabled:') then
err("language disabled by %s: %s (%s)", dbname, cname,
ldata.special:gsub('^disabled:', ''))
elseif ldata.special == 'language0' then
@@ -344,7 +337,7 @@ function loadlanguage(lname, id)
%
% \begin{macrocode}
wlog(msg, '', cname, id)
- for _, item in ipairs{'hyphenation', 'patterns'} do
+ for _, item in ipairs{'patterns', 'hyphenation'} do
local file = ldata[item]
if file ~= nil and file ~= '' then
local file = kpse.find_file(file) or err("file not found: %s", file)
@@ -352,6 +345,9 @@ function loadlanguage(lname, id)
local data = fh:read('*a') or err("file not readable: %s", f)
fh:close()
lang[item](lang.new(id), data)
+ else
+ if item == 'hyphenation' then item = item..' exceptions' end
+ wlog("info: no %s for this language", item)
end
end
end
@@ -429,7 +425,7 @@ end
%
% \begin{macrocode}
\ProvidesFile{hyphen.cfg}
- [2010/04/26 v3.8l-luatex-1.3beta %
+ [2010/04/26 v3.8l-luatex-1.4 %
Language switching mechanism for LuaTeX, adapted from babel v3.8l]
% \end{macrocode}
%