summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/generic/hyph-utf8
diff options
context:
space:
mode:
authorNorbert Preining <preining@logic.at>2010-03-28 00:10:19 +0000
committerNorbert Preining <preining@logic.at>2010-03-28 00:10:19 +0000
commit811acfb0da04fa1b5c9d600f949364bba6b89afb (patch)
treeebf5199441da41a8674f4b7b1046feb3cc8279b7 /Master/texmf-dist/source/generic/hyph-utf8
parent266576ec33839420ae8e2efed319ada6cea885ba (diff)
hyph-utf8 update (2010-03-21)
git-svn-id: svn://tug.org/texlive/trunk@17577 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/generic/hyph-utf8')
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/README61
-rwxr-xr-x[-rw-r--r--]Master/texmf-dist/source/generic/hyph-utf8/contributed/make-exhyph.pl (renamed from Master/texmf-dist/source/generic/hyph-utf8/make-exhyph.pl)0
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java108
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl38
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java385
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl128
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java428
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README38
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java325
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml23
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml434
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/generate-offo.rb96
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb19
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb97
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb14
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb41
-rw-r--r--Master/texmf-dist/source/generic/hyph-utf8/languages.rb247
-rwxr-xr-xMaster/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb135
18 files changed, 2569 insertions, 48 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/README b/Master/texmf-dist/source/generic/hyph-utf8/README
new file mode 100644
index 00000000000..07ba26b49a4
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/README
@@ -0,0 +1,61 @@
+generate-converters.rb
+======================
+INPUT:
+- source/generic/hyph-utf8/data/encodings/*.dat
+OUTPUT:
+- tex/generic/hyph-utf8/conversions/conv-utf8-*.dat
+
+Auto-generates conversions from UTF-8 to some particular encoding.
+
+Needs to be run only in case that a new encoding is added or an old encoding is fixed.
+
+generate-offo.rb
+================
+Temporary; another version maintaned by author in Java; this one will be improved or removed.
+Will be rewritten to take raw patterns as input with no need to parse.
+
+generate-pattern-loaders.rb
+===========================
+INPUT:
+- language.rb with data
+OUTPUT:
+- tex/generic/hyph-utf8/loadhyph/loadhyph-*.tex
+
+Auto-generates pattern loaders for languages that should be used in TeX Live or other distros.
+
+Needs to be run when a new language is added, a bug needs to be fixed or a strategy is changed.
+
+generate-tl-files.rb
+====================
+Has been used once for the very first TL script generation.
+Out-of-date and not maintaned; will probably be removed.
+
+generate-webpage.rb
+===================
+Used to generated webpage with language overview.
+
+languages.rb
+============
+Database with language information used for almost every script.
+Needs some clean-up and some updates.
+
+
+languages/eu/generate_patterns_eu.rb
+====================================
+Generator for patterns for Basque.
+Author: Juan M. Aguirregabiria, adapted by Mojca Miklavec
+
+languages/gl
+============
+Generator for patterns for Galician.
+Author: Javier A. Múgica
+
+languages/tr/generate_patterns_tr.rb
+====================================
+Generator for patterns for Turkish.
+Author: See file, modified by Mojca Miklavec
+
+
+contributed/make-exhyph.pl
+==========================
+Imported script for hack related to breaking compound words.
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/make-exhyph.pl b/Master/texmf-dist/source/generic/hyph-utf8/contributed/make-exhyph.pl
index 3fe44239d21..3fe44239d21 100644..100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/make-exhyph.pl
+++ b/Master/texmf-dist/source/generic/hyph-utf8/contributed/make-exhyph.pl
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java
new file mode 100644
index 00000000000..a4f70616a36
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Simon Pepping 2009
+ *
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: ConvertLanguageData.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+/**
+ * Convert language data in ruby format to XML format
+ */
+public final class ConvertLanguageData {
+
+ /**
+ * @param languageDataPath
+ * @throws IOException
+ * @throws TransformerException
+ * @throws SAXException
+ * @throws URISyntaxException
+ */
+ public static void convert(String languageDataPath, boolean useStylesheet)
+ throws IOException, TransformerException, SAXException, URISyntaxException {
+
+ // input
+ InputStream inis = new FileInputStream(languageDataPath);
+ InputSource input = new InputSource(inis);
+ input.setSystemId(languageDataPath);
+ input.setEncoding("utf-8");
+ XMLReader reader = new LanguageDataParser();
+
+ // output
+ String outPath = languageDataPath.replaceFirst("\\.rb$", ".xml");
+ Result result = new StreamResult(outPath);
+
+ // transformation
+ TransformerFactory tf = TransformerFactory.newInstance();
+ if (!tf.getFeature(SAXTransformerFactory.FEATURE)) {
+ throw new TransformerException("TransformerFactory is not a SAXTransformerFactory");
+ }
+ SAXTransformerFactory stf = (SAXTransformerFactory) tf;
+ TransformerHandler th;
+ if (useStylesheet) {
+ URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertLanguageData.xsl");
+ File xsltFile = new File(xsltUrl.toURI());
+ InputStream xsltStream = new FileInputStream(xsltFile);
+ Source xsltSource = new StreamSource(xsltStream);
+ xsltSource.setSystemId(xsltFile.getAbsolutePath());
+ th = stf.newTransformerHandler(xsltSource);
+ } else {
+ th = stf.newTransformerHandler();
+ }
+
+ // pipeline
+ reader.setContentHandler(th);
+ reader.setProperty("http://xml.org/sax/properties/lexical-handler", th);
+ th.setResult(result);
+ reader.parse(input);
+ }
+
+ /**
+ * @param args
+ * @throws IOException
+ * @throws TransformerException
+ * @throws SAXException
+ * @throws URISyntaxException
+ */
+ public static void main(String[] args)
+ throws IOException, TransformerException, SAXException, URISyntaxException {
+ if (args[0].endsWith("--debug")) {
+ convert(args[1], false);
+ } else {
+ convert(args[0], true);
+ }
+ }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl
new file mode 100644
index 00000000000..98aa733a0c0
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet version="1.0"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:lang="urn:org:tug:texhyphen:languagedata">
+
+ <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
+
+ <xsl:variable name="language-data" select="document('codemapping.xml')/lang:code-mappings"/>
+
+ <xsl:template match="*|@*|node()">
+ <xsl:copy>
+ <xsl:apply-templates select="*|@*|node()"/>
+ </xsl:copy>
+ </xsl:template>
+
+ <xsl:template match="lang:language">
+ <xsl:variable name="code" select="$language-data/lang:code-mapping[@code=current()/@code]"/>
+ <xsl:variable name="fop-code">
+ <xsl:choose>
+ <xsl:when test="$code">
+ <xsl:value-of select="$code/@fop-code"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:value-of select="@code"/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:variable>
+ <xsl:copy>
+ <xsl:if test="string($fop-code)">
+ <xsl:attribute name="fop-code">
+ <xsl:value-of select="$fop-code"/>
+ </xsl:attribute>
+ </xsl:if>
+ <xsl:apply-templates select="*|@*|node()"/>
+ </xsl:copy>
+ </xsl:template>
+
+</xsl:stylesheet>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java
new file mode 100644
index 00000000000..1ec9cee9474
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java
@@ -0,0 +1,385 @@
+/*
+ * Copyright Simon Pepping 2009
+ *
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: ConvertTeXPattern.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.DefaultHandler;
+
+/**
+ * Convert modern UTF8 TeX hyphenation patterns to XML format
+ */
+public final class ConvertTeXPattern {
+
+ public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet,
+ boolean useLanguagedata)
+ throws IOException, TransformerException, SAXException, URISyntaxException,
+ ParserConfigurationException, CodeMappingException {
+ checkCodeMapping();
+ Collection<String> languages = codeMapping.keySet();
+ convert(texPatterns, outfilePath, useStylesheet, languages);
+ }
+
+ public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet)
+ throws IOException, TransformerException, SAXException, URISyntaxException,
+ CodeMappingException {
+ convert(texPatterns, outfilePath, useStylesheet, null);
+ }
+
+ /**
+ * infile outfile
+ * indir outdir (file protocol only)
+ * infiles outdir
+ * file and http protocols allowed
+ *
+ * @param texPatternUri
+ * @param outfilePath
+ * @param useStylesheet
+ * @param texcodes filter of requested tex codes; is allowed to be null
+ * @throws IOException
+ * @throws TransformerException
+ * @throws SAXException
+ * @throws URISyntaxException
+ * @throws CodeMappingException
+ */
+ public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet,
+ Collection<String> texcodes)
+ throws IOException, TransformerException, SAXException, URISyntaxException,
+ CodeMappingException {
+ File outDir = new File(outfilePath);
+ boolean oneTexcode = (texcodes != null && texcodes.size() == 1);
+ boolean oneInputfile = (texPatterns.length == 1);
+ boolean oneFilteredInput = (oneTexcode || oneInputfile);
+ if (!oneFilteredInput && !outDir.isDirectory()) {
+ throw new IllegalArgumentException
+ ("with multiple input files the output path " + outfilePath + " must be a directory");
+ }
+ for (String texPattern : texPatterns) {
+ URI texPatternUri = makeTexPatternUri(texPattern);
+ URI[] texPatternUris = makeTexPatternUris(texPatternUri);
+ oneInputfile = (texPatternUris.length == 1);
+ oneFilteredInput = (oneTexcode || oneInputfile);
+ if (!oneFilteredInput && !outDir.isDirectory()) {
+ throw new IllegalArgumentException
+ ("with an input directory " + texPattern + " the output path " + outfilePath + " must be a directory");
+ }
+ for (URI t : texPatternUris) {
+ TransformationData transformationData = makeTransformationData(t, outDir, texcodes);
+ if (transformationData == null) {
+ continue;
+ }
+ doConvert(t, transformationData, useStylesheet);
+ }
+ }
+ }
+
+ /**
+ * @param texPattern
+ * @return
+ * @throws URISyntaxException
+ * @throws FileNotFoundException
+ */
+ private static URI makeTexPatternUri(String texPattern)
+ throws URISyntaxException, FileNotFoundException {
+ URI texPatternUri;
+ texPatternUri = new URI(texPattern);
+ String scheme = texPatternUri.getScheme();
+ // see if it is a relative file path
+ if (scheme == null) {
+ File f = new File(texPattern);
+ texPatternUri = new URI("file", null, f.getAbsolutePath(), null, null);
+ scheme = texPatternUri.getScheme();
+ }
+ if (scheme == null || !(scheme.equals("http") || scheme.equals("file"))) {
+ throw new FileNotFoundException
+ ("URI with file or http scheme required for hyphenation pattern file");
+ }
+ return texPatternUri;
+ }
+
+ /**
+ * @param outfilePath
+ * @param outDir
+ * @param texPatternUri
+ * @param scheme
+ * @return
+ * @throws URISyntaxException
+ */
+ private static URI[] makeTexPatternUris(URI texPatternUri) throws URISyntaxException {
+ URI[] texPatternUris;
+ texPatternUris = new URI[] {texPatternUri};
+ String scheme = texPatternUri.getScheme();
+ if (scheme.equals("file")) {
+ File dir = new File(texPatternUri);
+ if (dir.isDirectory()) {
+ ArrayList<URI> l = new ArrayList<URI>();
+ FilenameFilter filter = new FilenameFilter() {
+ public boolean accept(File dir, String name) {
+ return name.endsWith(".tex");
+ }
+ };
+ for (File f : dir.listFiles(filter)) {
+ l.add(new URI("file", null, f.getAbsolutePath(), null, null));
+ }
+ texPatternUris = l.toArray(texPatternUris);
+ }
+ }
+ return texPatternUris;
+ }
+
+ /**
+ * @param t
+ * @param outDir
+ * @param texcodes filter of requested tex codes; is allowed to be null
+ * @return
+ * @throws CodeMappingException
+ */
+ private static TransformationData makeTransformationData
+ (URI t, File outDir, Collection<String> texcodes) throws CodeMappingException {
+ File outFile;
+ String path = t.getPath();
+ String basename = path.substring(path.lastIndexOf('/') + 1);
+ String base = basename.substring(0, basename.lastIndexOf('.'));
+ // xmlCode, texCode
+ String[] codes = mapCode(base);
+ // code mapping lists no xmlCode
+ if (codes[0] == null) {
+ return null;
+ }
+ if (texcodes != null && !texcodes.contains(codes[1])) {
+ return null;
+ }
+ if (!outDir.isDirectory()) {
+ outFile = outDir;
+ } else {
+ outFile = new File(outDir, codes[0] + ".xml");
+ }
+ return new TransformationData(outFile, codes[1]);
+ }
+
+ private static class TransformationData {
+ File outFile;
+ String texCode;
+ TransformationData(File outFile, String texCode) {
+ this.outFile = outFile;
+ this.texCode = texCode;
+ }
+ }
+
+ private static class CodeMappingException extends Exception {
+ public CodeMappingException(Exception e) {
+ super(e);
+ }
+ public CodeMappingException(String m) {
+ super(m);
+ }
+ }
+
+ static Map<String, String> codeMapping;
+ static CodeMappingException codeMappingException;
+ static {
+ try {
+ codeMapping = readLanguagedata();
+ } catch (ParserConfigurationException e) {
+ codeMappingException = new CodeMappingException(e);
+ } catch (SAXException e) {
+ codeMappingException = new CodeMappingException(e);
+ } catch (IOException e) {
+ codeMappingException = new CodeMappingException(e);
+ }
+ }
+
+ private static String[] mapCode(String texCode) throws CodeMappingException {
+ checkCodeMapping();
+ String hyp = "hyph-";
+ String xmlCode = texCode;
+ if (texCode.startsWith(hyp)) {
+ texCode = texCode.substring(hyp.length());
+ xmlCode = codeMapping.get(texCode);
+ }
+ return new String[] {xmlCode, texCode};
+ }
+
+ /**
+ * @throws CodeMappingException
+ */
+ private static void checkCodeMapping() throws CodeMappingException {
+ if (codeMapping == null) {
+ if (codeMappingException != null) {
+ throw codeMappingException;
+ } else {
+ throw new CodeMappingException("Failure initializing code mapping");
+ }
+ }
+ }
+
+ public static Map<String,String> readLanguagedata()
+ throws ParserConfigurationException, SAXException, IOException {
+ SAXParserFactory spf = SAXParserFactory.newInstance();
+ spf.setNamespaceAware(true);
+ SAXParser parser = spf.newSAXParser();
+ InputStream is = ConvertTeXPattern.class.getResourceAsStream("languages.xml");
+ TexcodeReader texcodeReader = new TexcodeReader();
+ parser.parse(is, texcodeReader);
+ return texcodeReader.getTexcodes();
+ }
+
+ private static class TexcodeReader extends DefaultHandler {
+
+ private Map<String, String> texcodes = new HashMap<String, String>();
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.helpers.DefaultHandler#startElement(java.lang.String, java.lang.String, java.lang.String, org.xml.sax.Attributes)
+ */
+ @Override
+ public void startElement(String uri, String localName, String qName,
+ Attributes attributes) throws SAXException {
+ if (uri.equals(LanguageDataParser.LANG_NAMESPACE) && localName.equals("language")) {
+ String texcode = attributes.getValue("code");
+ String fopcode = attributes.getValue("fop-code");
+ if (fopcode != null) {
+ texcodes.put(texcode, fopcode);
+ }
+ }
+ }
+
+ /**
+ * @return the texcodes
+ */
+ public Map<String,String> getTexcodes() {
+ return texcodes;
+ }
+
+ }
+
+ public static void doConvert(URI texPatternUri, TransformationData outdata, boolean useStylesheet)
+ throws TransformerException, SAXException, MalformedURLException, IOException, URISyntaxException {
+
+ String scheme = texPatternUri.getScheme();
+ InputStream inis = null;
+ if (scheme.equals("file")) {
+ File in = new File(texPatternUri);
+ inis = new FileInputStream(in);
+ } else if (scheme.equals("http")) {
+ inis = texPatternUri.toURL().openStream();
+ } else {
+ throw new FileNotFoundException
+ ("URI with file or http scheme required for hyphenation pattern file");
+ }
+
+ InputSource input = new InputSource(inis);
+ input.setSystemId(texPatternUri.toString());
+ input.setEncoding("utf-8");
+ XMLReader reader = new TeXPatternParser();
+ Result result = new StreamResult(outdata.outFile);
+ TransformerFactory tf = TransformerFactory.newInstance();
+ if (!tf.getFeature(SAXTransformerFactory.FEATURE)) {
+ throw new TransformerException("TransformerFactory is not a SAXTransformerFactory");
+ }
+ SAXTransformerFactory stf = (SAXTransformerFactory) tf;
+ TransformerHandler th;
+ if (useStylesheet) {
+ URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertTeXPattern.xsl");
+ File xsltFile = new File(xsltUrl.toURI());
+ InputStream xsltStream = new FileInputStream(xsltFile);
+ Source xsltSource = new StreamSource(xsltStream);
+ xsltSource.setSystemId(xsltFile.getAbsolutePath());
+ th = stf.newTransformerHandler(xsltSource);
+ Transformer tr = th.getTransformer();
+ tr.setParameter("tex-code", outdata.texCode);
+ } else {
+ th = stf.newTransformerHandler();
+ }
+ reader.setContentHandler(th);
+ reader.setProperty("http://xml.org/sax/properties/lexical-handler", th);
+ th.setResult(result);
+ reader.parse(input);
+ }
+
+ /**
+ * @param args input URI, output file
+ * @throws URISyntaxException if the URI is not correct
+ * @throws IOException if a file is not found, or contains illegal content
+ * @throws TransformerException
+ * @throws SAXException
+ * @throws ParserConfigurationException
+ * @throws CodeMappingException
+ */
+ public static void main(String[] args)
+ throws URISyntaxException, IOException, TransformerException, SAXException,
+ ParserConfigurationException, CodeMappingException {
+ String prefix = "--";
+ int i = 0;
+ boolean useStylesheet = true;
+ boolean useLanguagedata = false;
+ Collection<String> texcodes = null;
+ while (args[i].startsWith(prefix)) {
+ String option = args[i].substring(prefix.length());
+ if (option.equals("debug")) {
+ useStylesheet = false;
+ } else if (option.equals("uselanguagedata") || option.equals("langdata")) {
+ useLanguagedata = true;
+ } else if (option.equals("texcodes")) {
+ texcodes = Arrays.asList(args[++i].split(","));
+ } else {
+ throw new IllegalArgumentException("Unknown option: " + option);
+ }
+ ++i;
+ }
+ if (texcodes != null) {
+ convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1],
+ useStylesheet, texcodes);
+ } else {
+ convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1],
+ useStylesheet, useLanguagedata);
+ }
+ }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl
new file mode 100644
index 00000000000..803962b7d5a
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl
@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0"
+ xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+ xmlns:tex="urn:org:tug:texhyphen"
+ xmlns:lang="urn:org:tug:texhyphen:languagedata"
+ exclude-result-prefixes="tex lang">
+
+ <xsl:output doctype-system="hyphenation.dtd" indent="yes"/>
+
+ <xsl:param name="comment-length" select="72"/>
+ <xsl:param name="tex-code"/>
+ <xsl:param name="hyphen-min-before-default" select="2"/>
+ <xsl:param name="hyphen-min-after-default" select="3"/>
+
+ <xsl:template match="/tex:tex">
+ <hyphenation-info>
+ <xsl:choose>
+ <xsl:when test="tex:hyphenation">
+ <!-- (comment*), (patterns), (comment*, hyphenation, comment*) => (comment*), hyphen-min, (comment*, exceptions, comment*), (patterns) -->
+ <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/>
+ <xsl:variable name="set2" select="tex:patterns"/>
+ <xsl:variable name="set3" select="node()[preceding-sibling::tex:patterns]"/>
+ <xsl:apply-templates select="$set1"/>
+ <xsl:call-template name="hyphen-min"/>
+ <xsl:apply-templates select="$set3"/>
+ <xsl:apply-templates select="$set2"/>
+ </xsl:when>
+ <xsl:otherwise>
+ <!-- (comment*), (patterns, comment*) => (comment*), hyphen-min, (patterns, comment*) -->
+ <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/>
+ <xsl:variable name="set2" select="node()[preceding-sibling::tex:patterns or self::tex:patterns]"/>
+ <xsl:apply-templates select="$set1"/>
+ <xsl:call-template name="hyphen-min"/>
+ <xsl:apply-templates select="$set2"/>
+ </xsl:otherwise>
+ </xsl:choose>
+ </hyphenation-info>
+ </xsl:template>
+
+ <xsl:template match="tex:patterns">
+ <patterns>
+ <xsl:apply-templates />
+ </patterns>
+ </xsl:template>
+
+ <xsl:template match="tex:patterns" mode="call-hyphen-min">
+ <xsl:call-template name="hyphen-min"/>
+ <patterns>
+ <xsl:apply-templates />
+ </patterns>
+ </xsl:template>
+
+ <xsl:template name="hyphen-min">
+ <xsl:variable name="hyphen-min"
+ select="document('languages.xml')/lang:languages/lang:language[@code=$tex-code]/lang:hyphen-min" />
+ <xsl:if test="count($hyphen-min)">
+ <hyphen-min before="{$hyphen-min/@before}" after="{$hyphen-min/@after}" />
+ </xsl:if>
+ </xsl:template>
+
+ <xsl:template match="tex:message"/>
+
+ <xsl:template match="tex:hyphenation">
+ <exceptions>
+ <xsl:apply-templates />
+ </exceptions>
+ </xsl:template>
+
+ <!-- Comments in TeX contain the trailing new line. -->
+ <!-- Here we keep the trailing new line if the comment is immediately -->
+ <!-- preceded or followed by a text node. -->
+ <!-- Otherwise we strip the new line and pad the comment -->
+ <!-- to the parameter comment-length. -->
+ <!-- The XSLT engine uses the same criteria to decide if the comment -->
+ <!-- should start on a new line or not. -->
+ <!-- This is not quite correct, because we risk adding spaces to -->
+ <!-- elements with mixed content. -->
+ <!-- The following test would be more appropriate: -->
+ <!-- test="preceding-sibling::text() or following-sibling::text()". -->
+ <xsl:template match="comment()">
+ <xsl:choose>
+ <xsl:when
+ test="preceding-sibling::node()[1][self::text()]
+ or following-sibling::node()[1][self::text()]">
+ <xsl:comment>
+ <xsl:value-of select="." />
+ </xsl:comment>
+ </xsl:when>
+ <xsl:otherwise>
+ <xsl:variable name="length" select="string-length(.)" />
+ <xsl:comment>
+ <xsl:value-of select="substring(.,1,$length - 1)" />
+ <xsl:text> </xsl:text>
+ <xsl:call-template name="make-spaces">
+ <xsl:with-param name="length"
+ select="$comment-length - ($length - 1)" />
+ </xsl:call-template>
+ </xsl:comment>
+ </xsl:otherwise>
+ </xsl:choose>
+ </xsl:template>
+
+ <xsl:template name="make-spaces">
+ <xsl:param name="length" select="0" />
+ <xsl:choose>
+ <xsl:when test="$length >= 10">
+ <xsl:text> </xsl:text>
+ <xsl:call-template name="make-spaces">
+ <xsl:with-param name="length" select="$length - 10" />
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:when test="$length >= 5">
+ <xsl:text> </xsl:text>
+ <xsl:call-template name="make-spaces">
+ <xsl:with-param name="length" select="$length - 5" />
+ </xsl:call-template>
+ </xsl:when>
+ <xsl:when test="$length >= 1">
+ <xsl:text> </xsl:text>
+ <xsl:call-template name="make-spaces">
+ <xsl:with-param name="length" select="$length - 1" />
+ </xsl:call-template>
+ </xsl:when>
+ </xsl:choose>
+ <xsl:text></xsl:text>
+ </xsl:template>
+
+</xsl:stylesheet> \ No newline at end of file
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java
new file mode 100644
index 00000000000..95c484b25ab
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java
@@ -0,0 +1,428 @@
+/*
+ * Copyright Simon Pepping 2009
+ *
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: LanguageDataParser.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Vector;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.DTDHandler;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.ext.LexicalHandler;
+import org.xml.sax.helpers.AttributesImpl;
+
+/**
+ * The class TeXParser parses TeX hyphenation pattern files and produces SAX events
+ */
+public class LanguageDataParser implements XMLReader {
+
+ public static final String LANG_NAMESPACE = "urn:org:tug:texhyphen:languagedata";
+ public static int lineLength = 72;
+ private static final int TOP_LEVEL = 3, IN_LANG = 4;
+ private static final Pattern
+ comment = Pattern.compile("#.*"),
+ langStart = Pattern.compile("{", Pattern.LITERAL),
+ langEnd = Pattern.compile("}", Pattern.LITERAL),
+ dataline = Pattern.compile("\"([^\"]+)\" ?=> ?\"([^\"]+)\","),
+ keywordline = Pattern.compile("\"([^\"]+)\" ?=> ?(false|true|nil),"),
+ listline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[(\"[^\"]+\"(?:,\"[^\"]+\")*)\\],"),
+ datalistline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[([^,]+(?:,[^,]+)*)\\],"),
+ space = Pattern.compile("[ \\t]+");
+ private static final AttributesImpl emptyAtts = new AttributesImpl();
+
+ private ContentHandler contentHandler;
+ private DTDHandler dtdHandler;
+ private EntityResolver entityResolver;
+ private ErrorHandler errorHandler;
+ private LexicalHandler lexicalHandler;
+
+ private void parseLanguageData(BufferedReader inbr) throws SAXException, IOException {
+ int parseState = TOP_LEVEL;
+ Language lang = null;
+
+ contentHandler.startDocument();
+ contentHandler.startPrefixMapping("", LANG_NAMESPACE);
+ contentHandler.startElement(LANG_NAMESPACE, "languages", "languages", emptyAtts);
+
+ for (String line = inbr.readLine(); line != null; line = inbr.readLine()) {
+ Matcher matcher = comment.matcher(line).useAnchoringBounds(true);
+ int start = 0;
+ while (start < line.length()) {
+ if (matcher.usePattern(comment).lookingAt()) {
+ processComment(matcher.group(), parseState == TOP_LEVEL ? null : lang);
+ } else if (matcher.usePattern(space).lookingAt()) {
+ // do nothing
+ } else if (parseState == TOP_LEVEL && matcher.usePattern(langStart).lookingAt()) {
+ parseState = IN_LANG;
+ lang = new Language();
+ } else if ((parseState == IN_LANG) && matcher.usePattern(langEnd).lookingAt()) {
+ pushoutLanguage(lang);
+ lang = null;
+ parseState = TOP_LEVEL;
+ } else if (parseState == IN_LANG
+ && (matcher.usePattern(dataline).lookingAt()
+ || matcher.usePattern(keywordline).lookingAt())) {
+ String key = matcher.group(1);
+ String value = matcher.group(2);
+ processDataline(key, value, lang);
+ } else if (parseState == IN_LANG
+ && (matcher.usePattern(listline).lookingAt()
+ || matcher.usePattern(datalistline).lookingAt())) {
+ String key = matcher.group(1);
+ String values = matcher.group(2);
+ processListline(key, values, lang);
+ } else {
+ break;
+ }
+ start = matcher.end();
+ matcher = matcher.region(start, line.length()).useAnchoringBounds(true);
+ }
+ }
+
+ contentHandler.endElement(LANG_NAMESPACE, "languages", "languages");
+ contentHandler.endPrefixMapping(LANG_NAMESPACE);
+ contentHandler.endDocument();
+ }
+
+ static Collection<String> attributeKeys;
+ static {
+ attributeKeys = new Vector<String>();
+ attributeKeys.add("code");
+ attributeKeys.add("name");
+ attributeKeys.add("use-old-patterns");
+ attributeKeys.add("use-new-loader");
+ attributeKeys.add("encoding");
+ attributeKeys.add("exceptions");
+ }
+
+ private void processComment(String comment, Language lang) throws SAXException {
+ comment = comment.replace("--", "––");
+ if (!comment.endsWith(" ")) {
+ comment = comment + " ";
+ }
+ if (lang == null) {
+ char[] textchars = comment.toCharArray();
+ if (lexicalHandler != null) {
+ lexicalHandler.comment(textchars, 1, textchars.length - 1);
+ }
+ } else {
+ lang.elements.add(new Element("comment", comment));
+ }
+
+ }
+
+ private void processDataline(String key, String value, Language lang) {
+ key = key.replace('_', '-');
+ if (value.equals("nil")) {
+ value = "";
+ }
+ if (attributeKeys.contains(key)) {
+ lang.atts.addAttribute("", key, key, "CDATA", value);
+ } else {
+ lang.elements.add(new Element(key, value));
+ }
+ }
+
+ private void processListline(String key, String valuesString, Language lang) {
+ key = key.replace('_', '-');
+ valuesString = valuesString.replace("\"", "");
+ String[] values = valuesString.split(",[ \\t]*");
+ if (attributeKeys.contains(key)) {
+ StringBuilder attValue = new StringBuilder();
+ for (String value : values) {
+ if (!value.equals("nil")) {
+ attValue.append(" " + value);
+ }
+ }
+ lang.atts.addAttribute("", key, key, "CDATA", attValue.toString());
+ } else if (key.equals("hyphenmin")) {
+ key = "hyphen-min";
+ AttributesImpl atts = new AttributesImpl();
+ atts.addAttribute("", "before", "before", "CDATA", values[0]);
+ atts.addAttribute("", "after", "after", "CDATA", values[1]);
+ lang.elements.add(new Element(key, "", atts));
+ } else {
+ key = key.replaceAll("s$", "");
+ for (String value : values) {
+ if (value.equals("nil")) {
+ value = "";
+ }
+ lang.elements.add(new Element(key, value));
+ }
+ }
+ }
+
+ private void pushoutLanguage(Language lang) throws SAXException {
+ contentHandler.startElement(LANG_NAMESPACE, "language", "language", lang.atts);
+ Iterator<Element> iter = lang.elements.iterator();
+ while (iter.hasNext()) {
+ Element elt = iter.next();
+ char[] text = elt.content.toCharArray();
+ if (elt.tag.equals("comment")) {
+ if (lexicalHandler != null) {
+ lexicalHandler.comment(text, 1, text.length - 1);
+ }
+ } else {
+ contentHandler.startElement(LANG_NAMESPACE, elt.tag, elt.tag, elt.atts);
+ contentHandler.characters(text, 0, text.length);
+ contentHandler.endElement(LANG_NAMESPACE, elt.tag, elt.tag);
+ }
+ }
+ contentHandler.endElement(LANG_NAMESPACE, "language", "language");
+ }
+
+ public Reader getReaderFromInputSource(InputSource input) throws IOException {
+ Reader reader = input.getCharacterStream();
+ String encoding = null;
+ if (reader == null) {
+ encoding = input.getEncoding();
+ }
+ if (reader == null) {
+ InputStream stream = input.getByteStream();
+ if (stream != null) {
+ if (encoding == null) {
+ reader = new InputStreamReader(stream);
+ } else {
+ reader = new InputStreamReader(stream, encoding);
+ }
+ }
+ }
+ if (reader == null) {
+ String systemId = input.getSystemId();
+ reader = getReaderFromSystemId(systemId, encoding);
+ }
+ return reader;
+ }
+
+ public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException {
+ if (systemId == null) {
+ throw new IOException("Cannot create a reader from a null systemID");
+ }
+ if (encoding.isEmpty()) {
+ encoding = null;
+ }
+ Reader reader = null;
+ URI uri = null;
+ File file = null;
+ try {
+ uri = new URI(systemId);
+ } catch (URISyntaxException e) {
+ // handled below
+ }
+ if (uri == null || !uri.isAbsolute()) {
+ file = new File(systemId);
+ }
+ if (file != null) {
+ if (encoding == null) {
+ reader = new FileReader(file);
+ } else {
+ InputStream stream = new FileInputStream(file);
+ reader = new InputStreamReader(stream, encoding);
+ }
+ } else if (uri != null && uri.getScheme().equals("http")) {
+ URL url = uri.toURL();
+ URLConnection conn = url.openConnection();
+ if (encoding == null) {
+ encoding = conn.getContentEncoding();
+ }
+ InputStream stream = conn.getInputStream();
+ reader = new InputStreamReader(stream, encoding);
+ }
+ return reader;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getContentHandler()
+ */
+ public ContentHandler getContentHandler() {
+ return contentHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getDTDHandler()
+ */
+ public DTDHandler getDTDHandler() {
+ return dtdHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getEntityResolver()
+ */
+ public EntityResolver getEntityResolver() {
+ return entityResolver;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getErrorHandler()
+ */
+ public ErrorHandler getErrorHandler() {
+ return errorHandler;
+ }
+
+
+ /**
+ * @return the lexicalHandler
+ */
+ public LexicalHandler getLexicalHandler() {
+ return lexicalHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getFeature(java.lang.String)
+ */
+ public boolean getFeature(String arg0)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ throw new SAXNotSupportedException();
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getProperty(java.lang.String)
+ */
+ public Object getProperty(String arg0)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ throw new SAXNotSupportedException();
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
+ */
+ public void parse(InputSource input) throws IOException, SAXException {
+ Reader reader = getReaderFromInputSource(input);
+ if (reader == null) {
+ throw new IOException("Could not open input source " + input);
+ }
+ BufferedReader inbr = new BufferedReader(reader);
+ parseLanguageData(inbr);
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#parse(java.lang.String)
+ */
+ public void parse(String systemId) throws IOException, SAXException {
+ Reader reader = getReaderFromSystemId(systemId, null);
+ if (reader == null) {
+ throw new IOException("Could not open input systemID " + systemId);
+ }
+ BufferedReader inbr = new BufferedReader(reader);
+ parseLanguageData(inbr);
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler)
+ */
+ public void setContentHandler(ContentHandler contenthandler) {
+ this.contentHandler = contenthandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler)
+ */
+ public void setDTDHandler(DTDHandler dtdhandler) {
+ this.dtdHandler = dtdhandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver)
+ */
+ public void setEntityResolver(EntityResolver entityresolver) {
+ this.entityResolver = entityresolver;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler)
+ */
+ public void setErrorHandler(ErrorHandler errorHandler) {
+ this.errorHandler = errorHandler;
+ }
+
+
+ /**
+ * @param lexicalHandler the lexicalHandler to set
+ */
+ public void setLexicalHandler(LexicalHandler lexicalHandler) {
+ this.lexicalHandler = lexicalHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean)
+ */
+ public void setFeature(String arg0, boolean arg1)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ throw new SAXNotSupportedException();
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object)
+ */
+ public void setProperty(String name, Object value)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ if (name.equals("http://xml.org/sax/properties/lexical-handler")) {
+ lexicalHandler = (LexicalHandler) value;
+ } else {
+ throw new SAXNotSupportedException();
+ }
+ }
+
+ private static class Element {
+ String tag;
+ String content;
+ Attributes atts;
+ Element(String tag, String content) {
+ this(tag, content, LanguageDataParser.emptyAtts);
+ }
+ Element(String tag, String content, Attributes atts) {
+ this.tag = tag;
+ this.content = content;
+ this.atts = atts;
+ }
+ }
+
+ private static class Language {
+ AttributesImpl atts;
+ List<Element> elements;
+ Language() {
+ atts = new AttributesImpl();
+ elements = new Vector<Element>();
+ }
+ }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README
new file mode 100644
index 00000000000..aaf4e074ae6
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README
@@ -0,0 +1,38 @@
+This utility consists of two parts.
+
+Part 1 converts languages.rb to languages.xml. It consists of
+ConvertLanguageData.java, LanguageDataParser.java,
+ConvertLanguageData.xsl, codemapping.xml. Invocation:
+
+java org.tug.texhyphen.util.ConvertLanguageData path/to/languages.rb
+
+output: languages.xml, in the same directory as languages.rb.
+
+The file codemapping contains the mapping from tex-code to
+fop-code. If this file has an entry with non-empty fop-code, it is
+inserted in languages.xml as attribute fop-code. If this file has an
+entry with empty fop-code, that language gets no attribute
+fop-code. Otherwise, that language gets attribute fop-code with the
+same value as tex-code (attribute code).
+
+The current file languages.xml was modified manually after conversion,
+to add the three languages at the bottom of the file.
+
+Part 2 converts tex pattern files to fop pattern files. It consists of
+ConvertTeXPattern.java, TeXPatternParser.java, ConvertTeXPattern.xsl,
+languages.xml. Invocation:
+
+java org.tug.texhyphen.util.ConvertTeXPattern [--uselanguagedata]
+[--texcodes code1,code2,...] infileuri1 [infileuri2 ...] outfile
+
+infileuri must be a file or http URI. With a file URI, it may also be
+a directory; in that case all pattern files in that directory are
+converted. outfile must be a path to an output filename or
+directory. If there are more input files, it must be an output
+directory. Option 'uselanguagedata' converts all input files which
+represent a language in the languages file with a fop-code. Option
+'--texcodes code1,code2,...' converts only those input files which
+represent a language with one of the given tex codes. Without those
+options all input files are converted. In any case, only files are
+converted which represent a language in the languages file with a
+fop-code.
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java
new file mode 100644
index 00000000000..0cc5a542f78
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java
@@ -0,0 +1,325 @@
+/*
+ * Copyright Simon Pepping 2009
+ *
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: TeXPatternParser.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.xml.sax.ContentHandler;
+import org.xml.sax.DTDHandler;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.ext.LexicalHandler;
+import org.xml.sax.helpers.AttributesImpl;
+
+/**
+ * The class TeXParser parses TeX hyphenation pattern files and produces SAX events
+ */
+public class TeXPatternParser implements XMLReader {
+
+ public static final String TEX_NAMESPACE = "urn:org:tug:texhyphen";
+ private static final int TOP_LEVEL = 3, IN_COMMAND = 4, AFTER_COMMAND = 5, IN_DATA = 6;
+ private static final Pattern
+ comment = Pattern.compile("%.*"),
+ commandStart = Pattern.compile("\\\\"),
+ command = Pattern.compile("[a-zA-Z]+"),
+ space = Pattern.compile(" +"),
+ argOpen = Pattern.compile("\\{"),
+ argClose = Pattern.compile("\\}"),
+ text = Pattern.compile("[^%\\\\\\{\\}]+");
+ private static final AttributesImpl emptyAtts = new AttributesImpl();
+
+ private ContentHandler contentHandler;
+ private DTDHandler dtdHandler;
+ private EntityResolver entityResolver;
+ private ErrorHandler errorHandler;
+ private LexicalHandler lexicalHandler;
+
+ private void parsePatterns(BufferedReader inbr) throws SAXException, IOException {
+ int parseState = TOP_LEVEL;
+ Stack<String> stack = new Stack<String>();
+
+ contentHandler.startDocument();
+ contentHandler.startPrefixMapping("", TEX_NAMESPACE);
+ contentHandler.startElement(TEX_NAMESPACE, "tex", "tex", emptyAtts);
+
+ for (String line = inbr.readLine(); line != null; line = inbr.readLine()) {
+ Matcher matcher = comment.matcher(line).useAnchoringBounds(true);
+ int start = 0;
+ char[] textchars;
+ boolean inComment = false;
+ while (start < line.length()) {
+ if (matcher.usePattern(comment).lookingAt()) {
+ String text = matcher.group().replace("--", "––");
+ textchars = (text + "\n").toCharArray();
+ if (lexicalHandler != null) {
+ lexicalHandler.comment(textchars, 1, textchars.length - 1);
+ }
+ inComment = true;
+ } else if (parseState == IN_DATA && matcher.usePattern(text).lookingAt()) {
+ textchars = matcher.group().toCharArray();
+ contentHandler.characters(textchars, 0, textchars.length);
+ } else if (parseState != IN_DATA && matcher.usePattern(space).lookingAt()) {
+ if (parseState == TOP_LEVEL) {
+ textchars = matcher.group().toCharArray();
+ contentHandler.ignorableWhitespace(textchars, 0, textchars.length);
+ }
+ } else if (parseState == TOP_LEVEL && matcher.usePattern(commandStart).lookingAt()) {
+ parseState = IN_COMMAND;
+ } else if (parseState == IN_COMMAND && matcher.usePattern(command).lookingAt()) {
+ String tag = matcher.group();
+ contentHandler.startElement(TEX_NAMESPACE, tag, tag, emptyAtts);
+ stack.push(tag);
+ parseState = AFTER_COMMAND;
+ } else if (parseState == AFTER_COMMAND && matcher.usePattern(argOpen).lookingAt()) {
+ parseState = IN_DATA;
+ } else if (parseState == IN_DATA && matcher.usePattern(argClose).lookingAt()) {
+ String tag = stack.pop();
+ contentHandler.endElement(TEX_NAMESPACE, tag, tag);
+ parseState = TOP_LEVEL;
+ } else {
+ break;
+ }
+ start = matcher.end();
+ matcher = matcher.region(start, line.length()).useAnchoringBounds(true);
+ }
+ textchars = "\n".toCharArray();
+ if (parseState == IN_DATA && !inComment) {
+ contentHandler.characters(textchars, 0, textchars.length);
+ } else if (parseState == TOP_LEVEL && !inComment) {
+ contentHandler.ignorableWhitespace(textchars, 0, textchars.length);
+ }
+ }
+
+ contentHandler.endElement(TEX_NAMESPACE, "tex", "tex");
+ contentHandler.endPrefixMapping(TEX_NAMESPACE);
+ contentHandler.endDocument();
+ }
+
+ public Reader getReaderFromInputSource(InputSource input) throws IOException {
+ Reader reader = input.getCharacterStream();
+ String encoding = null;
+ if (reader == null) {
+ encoding = input.getEncoding();
+ }
+ if (reader == null) {
+ InputStream stream = input.getByteStream();
+ if (stream != null) {
+ if (encoding == null) {
+ reader = new InputStreamReader(stream);
+ } else {
+ reader = new InputStreamReader(stream, encoding);
+ }
+ }
+ }
+ if (reader == null) {
+ String systemId = input.getSystemId();
+ reader = getReaderFromSystemId(systemId, encoding);
+ }
+ return reader;
+ }
+
+ public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException {
+ if (systemId == null) {
+ throw new IOException("Cannot create a reader from a null systemID");
+ }
+ if (encoding.isEmpty()) {
+ encoding = null;
+ }
+ Reader reader = null;
+ URI uri = null;
+ File file = null;
+ try {
+ uri = new URI(systemId);
+ } catch (URISyntaxException e) {
+ // handled below
+ }
+ if (uri == null || !uri.isAbsolute()) {
+ file = new File(systemId);
+ }
+ if (file != null) {
+ if (encoding == null) {
+ reader = new FileReader(file);
+ } else {
+ InputStream stream = new FileInputStream(file);
+ reader = new InputStreamReader(stream, encoding);
+ }
+ } else if (uri != null && uri.getScheme().equals("http")) {
+ URL url = uri.toURL();
+ URLConnection conn = url.openConnection();
+ if (encoding == null) {
+ encoding = conn.getContentEncoding();
+ }
+ InputStream stream = conn.getInputStream();
+ reader = new InputStreamReader(stream, encoding);
+ }
+ return reader;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getContentHandler()
+ */
+ public ContentHandler getContentHandler() {
+ return contentHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getDTDHandler()
+ */
+ public DTDHandler getDTDHandler() {
+ return dtdHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getEntityResolver()
+ */
+ public EntityResolver getEntityResolver() {
+ return entityResolver;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getErrorHandler()
+ */
+ public ErrorHandler getErrorHandler() {
+ return errorHandler;
+ }
+
+
+ /**
+ * @return the lexicalHandler
+ */
+ public LexicalHandler getLexicalHandler() {
+ return lexicalHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getFeature(java.lang.String)
+ */
+ public boolean getFeature(String arg0)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ throw new SAXNotSupportedException();
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#getProperty(java.lang.String)
+ */
+ public Object getProperty(String arg0)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ throw new SAXNotSupportedException();
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
+ */
+ public void parse(InputSource input) throws IOException, SAXException {
+ Reader reader = getReaderFromInputSource(input);
+ if (reader == null) {
+ throw new IOException("Could not open input source " + input);
+ }
+ BufferedReader inbr = new BufferedReader(reader);
+ parsePatterns(inbr);
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#parse(java.lang.String)
+ */
+ public void parse(String systemId) throws IOException, SAXException {
+ Reader reader = getReaderFromSystemId(systemId, null);
+ if (reader == null) {
+ throw new IOException("Could not open input systemID " + systemId);
+ }
+ BufferedReader inbr = new BufferedReader(reader);
+ parsePatterns(inbr);
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler)
+ */
+ public void setContentHandler(ContentHandler contenthandler) {
+ this.contentHandler = contenthandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler)
+ */
+ public void setDTDHandler(DTDHandler dtdhandler) {
+ this.dtdHandler = dtdhandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver)
+ */
+ public void setEntityResolver(EntityResolver entityresolver) {
+ this.entityResolver = entityresolver;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler)
+ */
+ public void setErrorHandler(ErrorHandler errorHandler) {
+ this.errorHandler = errorHandler;
+ }
+
+
+ /**
+ * @param lexicalHandler the lexicalHandler to set
+ */
+ public void setLexicalHandler(LexicalHandler lexicalHandler) {
+ this.lexicalHandler = lexicalHandler;
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean)
+ */
+ public void setFeature(String arg0, boolean arg1)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ throw new SAXNotSupportedException();
+ }
+
+ /* (non-Javadoc)
+ * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object)
+ */
+ public void setProperty(String name, Object value)
+ throws SAXNotRecognizedException, SAXNotSupportedException {
+ if (name.equals("http://xml.org/sax/properties/lexical-handler")) {
+ lexicalHandler = (LexicalHandler) value;
+ } else {
+ throw new SAXNotSupportedException();
+ }
+ }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml
new file mode 100644
index 00000000000..c5b64722fa0
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<code-mappings xmlns="urn:org:tug:texhyphen:languagedata">
+ <code-mapping code="grc-x-ibycus" fop-code="grc_X_ibycus"/>
+ <code-mapping code="zh-latn" fop-code="zh_Latn"/>
+ <code-mapping code="de-1901" fop-code="de_1901"/>
+ <code-mapping code="de-1996" fop-code="de"/>
+ <code-mapping code="de-ch-1901" fop-code="de_CH_1901"/>
+ <code-mapping code="el-monoton" fop-code="el"/>
+ <code-mapping code="el-polyton" fop-code="el_Polyton"/>
+ <code-mapping code="mn-cyrl" fop-code="mn"/>
+ <code-mapping code="mn-cyrl-x-2a" fop-code="mn_Cyrl_x_2a"/>
+ <code-mapping code="sh-cyrl" fop-code="sh_Cyrl"/>
+ <code-mapping code="sh-latn" fop-code="sh_Latn"/>
+ <code-mapping code="sr-cyrl" fop-code="sr"/>
+ <code-mapping code="sr-latn" fop-code="sr_Latn"/>
+ <code-mapping code="en-gb" fop-code="en_GB"/>
+ <code-mapping code="en-us" fop-code="en"/>
+ <code-mapping code="eo" fop-code=""/>
+ <code-mapping code="nn" fop-code=""/>
+ <code-mapping code="nb" fop-code=""/>
+ <!-- hu patterns cause a stack overflow when compiled with FOP -->
+ <code-mapping code="hu" fop-code=""/>
+</code-mappings>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml
new file mode 100644
index 00000000000..2f0fd637d3c
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml
@@ -0,0 +1,434 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<languages xmlns="urn:org:tug:texhyphen:languagedata">
+ <!-- –––––––––––––––––––––––––––––––––––––– -->
+ <!-- languages with no hyphenation patterns -->
+ <!-- –––––––––––––––––––––––––––––––––––––– -->
+ <!-- arabic zerohyph.tex -->
+ <language fop-code="ar" code="ar" name="arabic" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false">
+ <filename-old-patterns>zerohyph.tex</filename-old-patterns>
+ <!-- "hyphenmin" => [], # not needed -->
+ <message/>
+ </language>
+ <!-- farsi zerohyph.tex -->
+ <!-- =persian -->
+ <language fop-code="fa" code="fa" name="farsi" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false">
+ <synonym>persian</synonym>
+ <filename-old-patterns>zerohyph.tex</filename-old-patterns>
+ <!-- "hyphenmin" => [], # not needed -->
+ <message/>
+ </language>
+ <!-- ––––––––––––––––––––––––––––––- -->
+ <!-- special patterns, not converted -->
+ <!-- ––––––––––––––––––––––––––––––- -->
+ <!-- ibycus ibyhyph.tex -->
+ <language fop-code="grc_X_ibycus" code="grc-x-ibycus" name="ibycus" use-new-loader="false" use-old-patterns="true" encoding="" exceptions="false">
+ <filename-old-patterns>ibyhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Ancient Greek Hyphenation Patterns for Ibycus encoding (v3.0)</message>
+ <author>peter_heslin</author>
+ </language>
+ <!-- –––––––––––––––––––––––––––– -->
+ <!-- languages using old patterns -->
+ <!-- –––––––––––––––––––––––––––– -->
+ <!-- greek xu-grphyph4.tex -->
+ <!-- =polygreek -->
+ <language fop-code="el_Polyton" code="el-polyton" name="greek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true">
+ <synonym>polygreek</synonym>
+ <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
+ <filename-old-patterns>grphyph5.tex</filename-old-patterns>
+ <!-- left/right hyphen min for Greek can be as low as one (1), -->
+ <!-- but for aesthetic reasons keep them at 2/2. -->
+ <!-- Dimitrios Filippou -->
+ <hyphen-min before="1" after="1"/>
+ <!-- polyglosia -->
+ <!-- "message" => "Polytonic Greek Hyphenation Patterns", -->
+ <message>Hyphenation patterns for multi-accent (polytonic) Modern Greek</message>
+ <author>dimitrios_filippou</author>
+ </language>
+ <!-- monogreek xu-grmhyph4.tex -->
+ <language fop-code="el" code="el-monoton" name="monogreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true">
+ <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
+ <filename-old-patterns>grmhyph5.tex</filename-old-patterns>
+ <hyphen-min before="1" after="1"/>
+ <!-- polyglosia -->
+ <!-- "message" => "Monotonic Greek Hyphenation Patterns", -->
+ <message>Hyphenation patterns for uni-accent (monotonic) Modern Greek</message>
+ <author>dimitrios_filippou</author>
+ </language>
+ <!-- ancientgreek xu-grahyph4.tex -->
+ <language fop-code="grc" code="grc" name="ancientgreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false">
+ <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
+ <filename-old-patterns>grahyph5.tex</filename-old-patterns>
+ <hyphen-min before="1" after="1"/>
+ <!-- polyglosia -->
+ <message>Hyphenation patterns for Ancient Greek</message>
+ <author>dimitrios_filippou</author>
+ </language>
+ <!-- coptic xu-copthyph.tex -->
+ <language fop-code="cop" code="cop" name="coptic" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false">
+ <use-old-patterns-comment>TODO: automatic conversion could be done, but was too complicated; leave for later.</use-old-patterns-comment>
+ <filename-old-patterns>copthyph.tex</filename-old-patterns>
+ <hyphen-min before="1" after="1"/>
+ <!-- polyglosia TODO: no documentation found -->
+ <message>Coptic Hyphenation Patterns</message>
+ <author>claudio_beccari</author>
+ </language>
+ <!-- german xu-dehypht.tex -->
+ <language fop-code="de_1901" code="de-1901" name="german" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false">
+ <use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment>
+ <filename-old-patterns>dehypht.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>German Hyphenation Patterns (Traditional Orthography)</message>
+ </language>
+ <!-- ngerman xu-dehyphn.tex -->
+ <language fop-code="de" code="de-1996" name="ngerman" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false">
+ <use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment>
+ <filename-old-patterns>dehyphn.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>German Hyphenation Patterns (Reformed Orthography)</message>
+ </language>
+ <!-- swissgerman -->
+ <language fop-code="de_CH_1901" code="de-ch-1901" name="swissgerman" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <!-- TODO: how is it going to be called -->
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Swiss-German Hyphenation Patterns (Traditional Orthography)</message>
+ </language>
+ <!-- russian xu-ruhyphen.tex -->
+ <language fop-code="ru" code="ru" name="russian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false">
+ <use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment>
+ <filename-old-patterns>ruhyphen.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Russian Hyphenation Patterns</message>
+ </language>
+ <!-- ukrainian xu-ukrhyph.tex -->
+ <language fop-code="uk" code="uk" name="ukrainian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false">
+ <use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment>
+ <filename-old-patterns>ukrhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Ukrainian Hyphenation Patterns</message>
+ </language>
+ <!-- –––––––––––––––––––––––––––– -->
+ <!-- languages using new patterns -->
+ <!-- –––––––––––––––––––––––––––– -->
+ <!-- catalan cahyph.tex -->
+ <language fop-code="ca" code="ca" name="catalan" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>cahyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Catalan Hyphenation Patterns</message>
+ </language>
+ <!-- czech xu-czhyph.tex -->
+ <language fop-code="cs" code="cs" name="czech" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>czhyph.tex</filename-old-patterns>
+ <filename-old-patterns-other>czhyphen.tex</filename-old-patterns-other>
+ <filename-old-patterns-other>czhyphen.ex</filename-old-patterns-other>
+ <!-- Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 -->
+ <!-- Typographical rules allow \righthyphenmin=2 when typesetting in a -->
+ <!-- narrow column (newspapers etc.). -->
+ <!-- (used to be 2,2) -->
+ <hyphen-min before="2" after="3"/>
+ <message>Czech Hyphenation Patterns (Pavel Sevecek, v3, 1995)</message>
+ </language>
+ <!-- slovak xu-skhyph.tex -->
+ <language fop-code="sk" code="sk" name="slovak" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>skhyph.tex</filename-old-patterns>
+ <filename-old-patterns-other>skhyphen.tex</filename-old-patterns-other>
+ <filename-old-patterns-other>skhyphen.ex</filename-old-patterns-other>
+ <!-- see czech -->
+ <hyphen-min before="2" after="3"/>
+ <message>Slovak Hyphenation Patterns (Jana Chlebikova, 1992)</message>
+ </language>
+ <!-- welsh cyhyph.tex -->
+ <language fop-code="cy" code="cy" name="welsh" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>cyhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="3"/>
+ <message>Welsh Hyphenation Patterns</message>
+ </language>
+ <!-- danish dkhyph.tex -->
+ <language fop-code="da" code="da" name="danish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>dkhyph.tex</filename-old-patterns>
+ <filename-old-patterns-other>dkcommon.tex</filename-old-patterns-other>
+ <filename-old-patterns-other>dkspecial.tex</filename-old-patterns-other>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Danish Hyphenation Patterns</message>
+ </language>
+ <!-- esperanto xu-eohyph.tex -->
+ <!-- TODO -->
+ <language code="eo" name="esperanto" use-new-loader="true" use-old-patterns="false" encoding="il3" exceptions="false">
+ <filename-old-patterns>eohyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- TODO -->
+ <message>Esperanto Hyphenation Patterns</message>
+ </language>
+ <!-- spanish xu-eshyph.tex -->
+ <!-- =espanol -->
+ <language fop-code="es" code="es" name="spanish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <synonym>espanol</synonym>
+ <filename-old-patterns>eshyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Spanish Hyphenation Patterns</message>
+ </language>
+ <!-- basque xu-bahyph.tex -->
+ <language fop-code="eu" code="eu" name="basque" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>bahyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Basque Hyphenation Patterns</message>
+ </language>
+ <!-- french xu-frhyph.tex -->
+ <!-- =patois -->
+ <!-- =francais -->
+ <language fop-code="fr" code="fr" name="french" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <synonym>patois</synonym>
+ <synonym>francais</synonym>
+ <filename-old-patterns>frhyph.tex</filename-old-patterns>
+ <!-- "hyphenmin" => [], -->
+ <message>French hyphenation patterns (V2.12, 2002/12/11)</message>
+ </language>
+ <!-- galician xu-glhyph.tex -->
+ <language fop-code="gl" code="gl" name="galician" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>glhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Galician Hyphenation Patterns</message>
+ </language>
+ <!-- estonian xu-ethyph.tex -->
+ <language fop-code="et" code="et" name="estonian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>ethyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="3"/>
+ <!-- babel -->
+ <message>Estonian Hyphenation Patterns</message>
+ </language>
+ <!-- finnish fihyph.tex -->
+ <language fop-code="fi" code="fi" name="finnish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>fihyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Finnish Hyphenation Patterns</message>
+ </language>
+ <!-- croatian xu-hrhyph.tex -->
+ <language fop-code="hr" code="hr" name="croatian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>hrhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Croatian Hyphenation Patterns</message>
+ </language>
+ <!-- hungarian xu-huhyphn.tex -->
+ <language code="hu" name="hungarian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>huhyphn.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- polyglosia -->
+ <message>Hungarian Hyphenation Patterns (v20031107)</message>
+ </language>
+ <!-- interlingua iahyphen.tex -->
+ <language fop-code="ia" code="ia" name="interlingua" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
+ <filename-old-patterns>iahyphen.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Hyphenation Patterns for Interlingua</message>
+ </language>
+ <!-- indonesian inhyph.tex -->
+ <language fop-code="id" code="id" name="indonesian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
+ <filename-old-patterns>inhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Indonesian Hyphenation Patterns</message>
+ </language>
+ <!-- icelandic icehyph.tex -->
+ <language fop-code="is" code="is" name="icelandic" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>icehyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Icelandic Hyphenation Patterns</message>
+ </language>
+ <!-- irish gahyph.tex -->
+ <language fop-code="ga" code="ga" name="irish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>gahyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="3"/>
+ <!-- babel -->
+ <message>Irish Hyphenation Patterns</message>
+ </language>
+ <!-- italian ithyph.tex -->
+ <language fop-code="it" code="it" name="italian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="false">
+ <filename-old-patterns>ithyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Italian Hyphenation Patterns</message>
+ </language>
+ <!-- kurmanji -->
+ <language fop-code="kmr" code="kmr" name="kurmanji" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>kmrhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Kurmanji Hyphenation Patterns (v. 1.0 2009/06/29 JKn and MSh)</message>
+ </language>
+ <!-- latin xu-lahyph.tex -->
+ <language fop-code="la" code="la" name="latin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <use-old-patterns-comment>Old patterns support both EC &amp; OT1 encodings at the same time.</use-old-patterns-comment>
+ <filename-old-patterns>lahyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Latin Hyphenation Patterns</message>
+ </language>
+ <!-- lithuanian -->
+ <language fop-code="lt" code="lt" name="lithuanian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false">
+ <hyphen-min before="2" after="2"/>
+ <message>Lithuanian Hyphenation Patterns</message>
+ </language>
+ <!-- latvian -->
+ <language fop-code="lv" code="lv" name="latvian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false">
+ <hyphen-min before="2" after="2"/>
+ <message>Latvian Hyphenation Patterns</message>
+ </language>
+ <!-- dutch nehyph96.tex -->
+ <language fop-code="nl" code="nl" name="dutch" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>nehyph96.tex</filename-old-patterns>
+ <!-- quoting Hans Hagen: -->
+ <!-- patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) -->
+ <hyphen-min before="2" after="2"/>
+ <message>Dutch Hyphenation Patterns</message>
+ </language>
+ <!-- norsk xu-nohyphbx.tex -->
+ <!-- =norwegian -->
+ <!-- nynorsk nnhyph.tex -->
+ <!-- bokmal nbhyph.tex -->
+ <!-- polish xu-plhyph.tex -->
+ <language fop-code="pl" code="pl" name="polish" use-new-loader="true" use-old-patterns="false" encoding="qx" exceptions="true">
+ <filename-old-patterns>plhyph.tex</filename-old-patterns>
+ <!--{}"hyphenmin" => [1,1], -->
+ <hyphen-min before="2" after="2"/>
+ <message>Polish Hyphenation Patterns</message>
+ </language>
+ <!-- portuguese pthyph.tex -->
+ <!-- =portuges -->
+ <language fop-code="pt" code="pt" name="portuguese" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <synonym>portuges</synonym>
+ <filename-old-patterns>pthyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="3"/>
+ <!-- babel -->
+ <message>Portuguese Hyphenation Patterns</message>
+ </language>
+ <!-- pinyin xu-pyhyph.tex -->
+ <language fop-code="zh_Latn" code="zh-latn" name="pinyin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>pyhyph.tex</filename-old-patterns>
+ <hyphen-min before="1" after="1"/>
+ <message>Hyphenation patterns for unaccented pinyin syllables (CJK 4.8.0)</message>
+ </language>
+ <!-- romanian xu-rohyphen.tex -->
+ <language fop-code="ro" code="ro" name="romanian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>rohyphen.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Romanian Hyphenation Patterns</message>
+ <!-- : `rohyphen' 1.1 <29.10.1996> -->
+ </language>
+ <!-- slovenian xu-sihyph.tex -->
+ <!-- =slovene -->
+ <language fop-code="sl" code="sl" name="slovenian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <synonym>slovene</synonym>
+ <filename-old-patterns>sihyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Slovenian Hyphenation Patterns</message>
+ </language>
+ <!-- uppersorbian xu-sorhyph.tex -->
+ <language fop-code="hsb" code="hsb" name="uppersorbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>sorhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Upper Sorbian Hyphenation Patterns (E. Werner)</message>
+ <!-- \message{Hyphenation patterns for Upper Sorbian, E. Werner} -->
+ <!-- \message{Completely new revision 1997, March 22} -->
+ </language>
+ <!-- swedish svhyph.tex -->
+ <language fop-code="sv" code="sv" name="swedish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>svhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- patters say it could be 1,2; babel says 2,2 - double check -->
+ <message>Swedish hyphenation patterns (Jan Michael Rynning, 1994-03-03)</message>
+ </language>
+ <!-- turkish xu-tkhyph.tex -->
+ <language fop-code="tr" code="tr" name="turkish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+ <filename-old-patterns>tkhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- polyglosia -->
+ <message>Turkish Hyphenation Patterns</message>
+ </language>
+ <!-- ukenglish ukhyphen.tex -->
+ <!-- TODO - should we rename it or not? -->
+ <language fop-code="en_GB" code="en-gb" name="ukenglish" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
+ <synonym>british</synonym>
+ <synonym>UKenglish</synonym>
+ <filename-old-patterns>ukhyphen.tex</filename-old-patterns>
+ <hyphen-min before="2" after="3"/>
+ <!-- confirmed, same as what Knuth says -->
+ <message>Hyphenation Patterns for British English</message>
+ </language>
+ <!-- serbian xu-srhyphc.tex -->
+ <language fop-code="sr_Latn" code="sr-latn" name="serbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <filename-old-patterns>shhyphl.tex</filename-old-patterns>
+ <!-- It is allowed to leave one character at the end of the row. -->
+ <!-- However, if you think that it is graphicaly not very -->
+ <!-- pleasent these patterns will work well with \lefthyphenmin=2. -->
+ <!-- \lefthyphenmin=1 \righthyphenmin=2 -->
+ <hyphen-min before="2" after="2"/>
+ <message>Serbian hyphenation patterns in Latin script</message>
+ </language>
+ <!-- serbianc -->
+ <language fop-code="sr" code="sr-cyrl" name="serbianc" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="true">
+ <filename-old-patterns>srhyphc.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Serbian hyphenation patterns in Cyrillic script</message>
+ </language>
+ <!-- mongolian xu-mnhyph.tex -->
+ <language fop-code="mn" code="mn-cyrl" name="mongolian" use-new-loader="true" use-old-patterns="false" encoding="lmc" exceptions="false">
+ <filename-old-patterns>mnhyph.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>Mongolian hyphenation patterns</message>
+ </language>
+ <!-- mongolian2a -->
+ <language fop-code="mn_Cyrl_x_2a" code="mn-cyrl-x-2a" name="mongolian2a" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false">
+ <filename-old-patterns>mnhyphn.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <message>(New) Mongolian Hyphenation Patterns</message>
+ </language>
+ <!-- bulgarian xu-bghyphen.tex -->
+ <language fop-code="bg" code="bg" name="bulgarian" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false">
+ <filename-old-patterns>bghyphen.tex</filename-old-patterns>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Bulgarian Hyphenation Patterns</message>
+ </language>
+ <!-- sanskrit -->
+ <language fop-code="sa" code="sa" name="sanskrit" use-new-loader="true" use-old-patterns="false" encoding="" exceptions="false">
+ <hyphen-min before="1" after="5"/>
+ <!-- polyglosia -->
+ <!-- no patterns for 8-bit TeX -->
+ <message>Sanskrit Hyphenation Patterns (v0.2, 2008/1/3)</message>
+ </language>
+ <!-- norsk xu-nohyphbx.tex -->
+ <language code="nb" name="bokmal" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <synonym>norwegian</synonym>
+ <synonym>norsk</synonym>
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Norwegian Bokmal Hyphenation Patterns</message>
+ </language>
+ <!-- nynorsk nnhyph.tex -->
+ <language code="nn" name="nynorsk" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+ <hyphen-min before="2" after="2"/>
+ <!-- babel -->
+ <message>Norwegian Nynorsk Hyphenation Patterns</message>
+ </language>
+ <!-- The following languages are not (yet) in languages.rb. -->
+ <!-- I added them after this file was generated from languages.rb. -->
+ <language fop-code="en" code="en-us" name="US english">
+ <hyphen-min before="2" after="3"/>
+ <filename-old-patterns>ushyphmax.tex</filename-old-patterns>
+ </language>
+ <language fop-code="sh_Latn" code="sh-latn" name="serbocroatian">
+ <hyphen-min before="1" after="2"/>
+ </language>
+ <language fop-code="sh_Cyrl" code="sh-cyrl" name="serbocroatianc">
+ <hyphen-min before="1" after="2"/>
+ </language>
+ <language fop-code="no" code="no" name="norwegian">
+ <filename-old-patterns>nohyphbx.tex</filename-old-patterns>
+ </language>
+</languages>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-offo.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-offo.rb
new file mode 100755
index 00000000000..f164c421aeb
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-offo.rb
@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+# this file generates FOP XML Hyphenation Patterns
+
+# use 'gem install unicode' if unicode is missing on your computer
+# require 'jcode'
+# require 'rubygems'
+# require 'unicode'
+
+load 'languages.rb'
+
+$path_OFFO="../../../../collaboration/offo"
+
+$l = Languages.new
+# TODO: should be singleton
+languages = $l.list.sort{|a,b| a.name <=> b.name}
+
+# TODO: we should rewrite this
+# not using: eo, el
+# todo: mn, no!, sa, sh
+# codes = ['bg', 'ca', 'cs', 'cy', 'da', 'de-1901', 'de-1996', 'de-ch-1901', 'en-gb', 'en-us', 'es', 'et', 'eu', 'fi', 'fr', 'ga', 'gl', 'hr', 'hsb', 'hu', 'ia', 'id', 'is', 'it', 'kmr', 'la', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr-cyrl', 'sv', 'tr', 'uk']
+
+language_codes = Hash.new
+languages.each do |language|
+ language_codes[language.code] = language.code
+end
+language_codes['de-1901'] = 'de_1901'
+language_codes['de-1996'] = 'de'
+language_codes['de-ch-1901'] = 'de_CH'
+language_codes['en-gb'] = 'en_GB'
+language_codes['en-us'] = 'en_US'
+language_codes['zh-latn'] = 'zh_Latn'
+language_codes['el-monoton'] = 'el'
+language_codes['el-polyton'] = 'el_polyton'
+language_codes['mn-cyrl'] = 'mn_Cyrl'
+language_codes['mn-cyrl-x-2a'] = 'mn'
+language_codes['sh-latn'] = 'sr_Latn'
+language_codes['sh-cyrl'] = nil
+language_codes['sr-cyrl'] = 'sr_Cyrl'
+
+languages.each do |language|
+ include_language = language.use_new_loader
+ code = language_codes[language.code]
+ if code == nil
+ include_language = false
+ end
+ if code == 'en_US'
+ include_language = true
+ end
+
+ if include_language
+ puts "generating #{code}"
+
+ $file_offo_pattern = File.open("#{$path_OFFO}/#{code}.xml", 'w')
+
+ $file_offo_pattern.puts '<?xml version="1.0" encoding="utf-8"?>'
+ $file_offo_pattern.puts '<hyphenation-info>'
+ $file_offo_pattern.puts
+
+ # lefthyphenmin/righthyphenmin
+ if language.hyphenmin == nil or language.hyphenmin.length == 0 then
+ lmin = ''
+ rmin = ''
+ elsif language.filename_old_patterns == "zerohyph.tex" then
+ lmin = ''
+ rmin = ''
+ else
+ lmin = language.hyphenmin[0]
+ rmin = language.hyphenmin[1]
+ end
+ patterns = language.get_patterns
+ exceptions = language.get_exceptions
+
+ if code == 'nn' or code == 'nb'
+ patterns = ""
+ patterns = $l['no'].get_patterns
+ end
+
+ $file_offo_pattern.puts "<hyphen-min before=\"#{lmin}\" after=\"#{rmin}\"/>"
+ $file_offo_pattern.puts
+ $file_offo_pattern.puts '<exceptions>'
+ if exceptions != ""
+ $file_offo_pattern.puts exceptions
+ end
+ $file_offo_pattern.puts '</exceptions>'
+ $file_offo_pattern.puts
+ $file_offo_pattern.puts '<patterns>'
+ patterns.each do |pattern|
+ $file_offo_pattern.puts pattern.gsub(/'/,"’")
+ end
+ $file_offo_pattern.puts '</patterns>'
+ $file_offo_pattern.puts '</hyphenation-info>'
+
+ $file_offo_pattern.close
+ end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
index 3fcbe9e1f9d..8a1b1044532 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
@@ -59,7 +59,7 @@ languages.each do |language|
if language.code == 'it' or language.code == 'fr' or language.code == 'uk' or language.code == 'la' or language.code == 'zh-latn' then
file.puts("\\lccode`\\'=`\\'")
end
- if language.code == 'pt' then
+ if language.code == 'pt' or language.code == 'tk' then
file.puts("\\lccode`\\-=`\\-")
end
@@ -67,15 +67,28 @@ languages.each do |language|
#
# some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines
# TODO - maybe consider doing something similar for ibycus
- if language.code == 'sa' then
+ if language.code == 'sa' or
+ language.code == 'as' or
+ language.code == 'bn' or
+ language.code == 'gu' or
+ language.code == 'hi' or
+ language.code == 'kn' or
+ language.code == 'ml' or
+ language.code == 'mr' or
+ language.code == 'or' or
+ language.code == 'pa' or
+ language.code == 'ta' or
+ language.code == 'te' then
file.puts(text_if_native_utf)
file.puts(" \\message{UTF-8 #{language.message}}")
file.puts(' % Set \lccode for ZWNJ and ZWJ.')
file.puts(' \lccode"200C="200C')
file.puts(' \lccode"200D="200D')
- file.puts(' % Set \lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.')
+ if language.code == 'sa'
+ file.puts(' % Set \lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.')
file.puts(' \lccode"0CF1="0CF1')
file.puts(' \lccode"0CF2="0CF2')
+ end
file.puts(" \\input hyph-#{language.code}.tex")
file.puts('\else')
file.puts(" \\message{No #{language.message} - only available with Unicode engines}")
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
new file mode 100755
index 00000000000..59bb491f8a0
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+# this file generates plain patterns (one-per-line) out of TeX source
+
+# use 'gem install unicode' if unicode is missing on your computer
+require 'jcode'
+require 'rubygems'
+require 'unicode'
+
+load 'languages.rb'
+
+$path_plain="../../../../plain"
+
+$l = Languages.new
+# TODO: should be singleton
+languages = $l.list.sort{|a,b| a.name <=> b.name}
+
+# TODO: we should rewrite this
+# not using: eo, el
+# todo: mn, no!, sa, sh
+# codes = ['bg', 'ca', 'cs', 'cy', 'da', 'de-1901', 'de-1996', 'de-ch-1901', 'en-gb', 'en-us', 'es', 'et', 'eu', 'fi', 'fr', 'ga', 'gl', 'hr', 'hsb', 'hu', 'ia', 'id', 'is', 'it', 'kmr', 'la', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr-cyrl', 'sv', 'tr', 'uk']
+
+language_codes = Hash.new
+languages.each do |language|
+ language_codes[language.code] = language.code
+end
+# language_codes['de-1901'] = 'de-1901'
+# language_codes['de-1996'] = 'de-1996'
+language_codes['de-ch-1901'] = 'de-CH-1901'
+language_codes['en-gb'] = 'en-GB'
+language_codes['en-us'] = 'en-US'
+language_codes['zh-latn'] = 'zh-Latn'
+# language_codes['el-monoton'] = 'el-monoton'
+# language_codes['el-polyton'] = 'el-polyton'
+language_codes['mn-cyrl'] = nil
+language_codes['mn-cyrl-x-2a'] = 'mn'
+language_codes['sh-latn'] = 'sr-Latn'
+language_codes['sh-cyrl'] = nil
+language_codes['sr-cyrl'] = 'sr-Cyrl'
+
+languages.sort{|x,y| x.code <=> y.code }.each do |language|
+ if language.use_new_loader or language.code == 'en-us' then
+ include_language = true
+ else
+ include_language = false
+ puts "(skipping #{language.code})"
+ end
+
+ code = language_codes[language.code]
+ if code == nil
+ include_language = false
+ end
+ if code == 'en_US'
+ include_language = true
+ end
+
+ if include_language
+ puts "generating #{code}"
+
+ $file_pat = File.open("#{$path_plain}/#{code}.pat.txt", 'w')
+ $file_hyp = File.open("#{$path_plain}/#{code}.hyp.txt", 'w')
+ $file_let = File.open("#{$path_plain}/#{code}.chr.txt", 'w')
+ $file_inf = File.open("#{$path_plain}/#{code}.lic.txt", 'w')
+
+ patterns = language.get_patterns
+ exceptions = language.get_exceptions
+
+ if code == 'nn' or code == 'nb'
+ patterns = ""
+ patterns = $l['no'].get_patterns
+ end
+
+ characters_indexes = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq
+
+ # patterns
+ patterns.each do |pattern|
+ $file_pat.puts pattern.gsub(/'/,"’")
+ end
+ # exceptions
+ if exceptions != ""
+ $file_hyp.puts exceptions
+ end
+ # letters
+ characters_indexes.each do |c|
+ ch = [c].pack('U')
+ $file_let.puts ch + Unicode.upcase(ch)
+ end
+ # licence and readme
+ $file_inf.puts "#{language.message}\n\n(more info about the licence to be added later)\n\n"
+ $file_inf.puts language.get_comments_and_licence
+
+ $file_pat.close
+ $file_hyp.close
+ $file_let.close
+ $file_inf.close
+ end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
index 3a8365f2fa5..233fb60bbcb 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
@@ -24,9 +24,10 @@ language_grouping = {
'greek' => ['el-monoton', 'el-polyton'],
'ancientgreek' => ['grc', 'grc-x-ibycus'],
'chinese' => ['zh-latn'],
- # TODO - until someone tells what to do
+ 'indic' => ['as', 'bn', 'gu', 'hi', 'kn', 'ml', 'mr', 'or', 'pa', 'ta', 'te'],
+ # TODO - until someone tells what to do; but this is wrong anyway
#'serbian' => ['sr-latn', 'sr-cyrl'],
- 'serbian' => ['sr-latn'],
+ 'serbian' => ['sh-latn'],
}
language_used_in_group = Hash.new
@@ -41,12 +42,17 @@ language_groups = Hash.new
# single languages first
languages.each do |language|
# temporary remove cyrilic serbian until someone explains what is needed
- if language.code == 'sr-cyrl' then
- languages.delete(language)
+ if language.code == 'sr-cyrl' or language.code == 'en-us' then
+ # ignore the language
elsif language_used_in_group[language.code] == nil then
language_groups[language.name] = [language]
end
+
+ if language.code == 'sh-latn' then
+ language.code = 'sr-latn'
+ end
end
+
# then groups of languages
language_grouping.each do |name,group|
language_groups[name] = []
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb
index 0f5c202b379..aa3dec8d0f2 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb
@@ -23,7 +23,7 @@ language_grouping = {
'chinese' => ['zh-latn'],
# TODO - until someone tells what to do
#'serbian' => ['sr-latn', 'sr-cyrl'],
- 'serbian' => ['sr-latn'],
+ 'serbian' => ['sh-latn'],
}
language_used_in_group = Hash.new
@@ -40,6 +40,8 @@ languages.each do |language|
# temporary remove cyrilic serbian until someone explains what is needed
if language.code == 'sr-cyrl' then
languages.delete(language)
+ elsif language.code == 'sh-latn' then
+ language.code = 'sr-latn'
elsif language_used_in_group[language.code] == nil then
language_groups[language.name] = [language]
end
@@ -108,42 +110,5 @@ language_groups.sort.each do |language_name,language_list|
puts "\t<td>#{encoding}</td>"
puts "</tr>\n"
end
- # if language_name != "russian" and language_name != "ukrainian" then
- # language_list.each do |language|
- # if language.use_old_patterns and language.filename_old_patterns != "zerohyph.tex" then
- # $file_tlpsrc.puts "runpattern f texmf/tex/generic/hyphen/#{language.filename_old_patterns}"
- # end
- # end
- # end
- # if language_name == "greek" then
- # $file_tlpsrc.puts "docpattern d texmf/doc/generic/elhyphen"
- # elsif language_name == "hungarian" then
- # $file_tlpsrc.puts "docpattern d texmf/doc/generic/huhyphen"
- # elsif language_name == "german" then
- # $file_tlpsrc.puts "runpattern f texmf/tex/generic/hyphen/dehyphtex.tex"
- # $file_tlpsrc.puts "runpattern f texmf/tex/generic/hyphen/ghyphen.README"
- # end
- # $file_tlpsrc.close
end
-#--------------#
-# language.dat #
-#--------------#
-$file_language_dat = File.open("#{$path_language_dat}/language.dat", "w")
-language_groups.sort.each do |language_name,language_list|
- language_list.each do |language|
- if language.use_new_loader then
- $file_language_dat.puts "#{language.name}\tloadhyph-#{language.code}.tex"
- else
- $file_language_dat.puts "#{language.name}\t#{language.filename_old_patterns}"
- end
-
- # synonyms
- if language.synonyms != nil then
- language.synonyms.each do |synonym|
- $file_language_dat.puts "=#{synonym}"
- end
- end
- end
-end
-$file_language_dat.close
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
index 685255c3b0f..c9e3c5c463f 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
@@ -16,8 +16,80 @@ class Language
if @synonyms==nil then @synonyms = [] end
end
-
+
+ # TODO: simplify this (reduce duplication)
+
+ def get_exceptions
+ if @exceptions1 == nil
+ filename = "../../../tex/generic/hyph-utf8/patterns/hyph-#{@code}.tex";
+ lines = IO.readlines(filename, '.').join("")
+ exceptions = lines.gsub(/%.*/,'');
+ if (exceptions.index('\hyphenation') != nil)
+ @exceptions1 = exceptions.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').
+ gsub(/\s+/m,"\n").
+ gsub(/^\s*/m,'').
+ gsub(/\s*$/m,'').
+ split("\n")
+ else
+ @exceptions1 = ""
+ end
+ end
+
+ return @exceptions1
+ end
+
+ def get_patterns
+ if @patterns == nil
+ filename = "../../../tex/generic/hyph-utf8/patterns/hyph-#{@code}.tex";
+ lines = IO.readlines(filename, '.').join("")
+ @patterns = lines.gsub(/%.*/,'').
+ gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1').
+ gsub(/\s+/m,"\n").
+ gsub(/^\s*/m,'').
+ gsub(/\s*$/m,'').
+ gsub(/'/,"’").
+ split("\n")
+
+ if @code == 'eo' then
+ @patterns = lines.gsub(/%.*/,'').
+ gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1').
+ #
+ gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.').
+ gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.').
+ gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.').
+ #
+ gsub(/\s+/m,"\n").
+ gsub(/^\s*/m,'').
+ gsub(/\s*$/m,'').
+ split("\n")
+ end
+ end
+ return @patterns
+ end
+
+ def get_comments_and_licence
+ if @comments_and_licence == nil then
+ filename = "../../../tex/generic/hyph-utf8/patterns/hyph-#{@code}.tex";
+ lines = IO.readlines(filename, '.').join("")
+ @comments_and_licence = lines.
+ gsub(/(.*)\\patterns.*/m,'\1')
+ end
+ return @comments_and_licence
+ end
+
+ # def lc_characters
+ # if @lc_characters == nil
+ # lc_characters = Hash.new
+ # p = self.patterns
+ # p.each do |pattern|
+ # end
+ # end
+ # return @lc_characters
+ # end
+
attr_reader :use_new_loader, :use_old_patterns, :use_old_patterns_comment, :filename_old_patterns, :code, :name, :synonyms, :hyphenmin, :encoding, :exceptions, :message
+ # this hack is needed for Serbian
+ attr_writer :code
end
@@ -36,7 +108,7 @@ authors = {
},
"claudio_beccari" => {
"name" => "Claudio",
- "email" => "claudio.beccari@polito.it",
+ "email" => "claudio{dot}beccari{at}polito{dot}it",
}
}
@@ -637,6 +709,18 @@ class Languages < Hash
"exceptions" => false,
"message" => "Swedish hyphenation patterns (Jan Michael Rynning, 1994-03-03)",
},
+# turkmen
+{
+ "code" => "tk",
+ "name" => "turkmen",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "filename_old_patterns" => nil,
+ "hyphenmin" => [1,2],
+ "encoding" => "ec",
+ "exceptions" => false,
+ "message" => "Turkmen Hyphenation Patterns",
+},
# turkish xu-tkhyph.tex
{
"code" => "tr",
@@ -662,9 +746,22 @@ class Languages < Hash
"exceptions" => true,
"message" => "Hyphenation Patterns for British English",
},
+# US english
+{
+ "code" => "en-us",
+ "name" => "english",
+ "use_new_loader" => false,
+ "use_old_patterns" => false,
+ "filename_old_patterns" => "ushyphmax.tex",
+ "hyphenmin" => [2,3], # confirmed, same as what Knuth says
+ "encoding" => "ascii",
+ "exceptions" => true,
+ "message" => "Hyphenation Patterns for American English",
+},
+# TODO: FIXME!!!
# serbian xu-srhyphc.tex
{
- "code" => "sr-latn",
+ "code" => "sh-latn",
"name" => "serbian",
"use_new_loader" => true,
"use_old_patterns" => false,
@@ -737,6 +834,17 @@ class Languages < Hash
"exceptions" => false,
"message" => "Sanskrit Hyphenation Patterns (v0.2, 2008/1/3)",
},
+# norwegian nohyph.tex
+{
+ "code" => "no",
+ "name" => "norwegian", # TODO: fixme
+ "use_new_loader" => false,
+ "use_old_patterns" => false,
+ "hyphenmin" => [2,2], # babel
+ "encoding" => "ec",
+ "exceptions" => false,
+ "message" => "Norwegian Hyphenation Patterns",
+},
# norsk xu-nohyphbx.tex
{
"code" => "nb",
@@ -759,6 +867,139 @@ class Languages < Hash
"exceptions" => true,
"message" => "Norwegian Nynorsk Hyphenation Patterns",
},
+#####
+# assamese
+{
+ "code" => "as",
+ "name" => "assamese",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Assameze Hyphenation Patterns",
+},
+# bengali
+{
+ "code" => "bn",
+ "name" => "bengali",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Bengali Hyphenation Patterns",
+},
+# guajrati
+{
+ "code" => "gu",
+ "name" => "guajrati",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Guajrati Hyphenation Patterns",
+},
+# assamese
+{
+ "code" => "as",
+ "name" => "assamese",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Assameze Hyphenation Patterns",
+},
+# hindi
+{
+ "code" => "hi",
+ "name" => "hindi",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Hindi Hyphenation Patterns",
+},
+# kannada
+{
+ "code" => "kn",
+ "name" => "assamese",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Kannada Hyphenation Patterns",
+},
+# malayalam
+{
+ "code" => "ml",
+ "name" => "malayalam",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Malayalam Hyphenation Patterns",
+},
+# marathi
+{
+ "code" => "mr",
+ "name" => "marathi",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Marathi Hyphenation Patterns",
+},
+# oriya
+{
+ "code" => "or",
+ "name" => "oriya",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Oriya Hyphenation Patterns",
+},
+# panjabi
+{
+ "code" => "pa",
+ "name" => "panjabi",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Panjabi Hyphenation Patterns",
+},
+# tamil
+{
+ "code" => "ta",
+ "name" => "tamil",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Tamil Hyphenation Patterns",
+},
+# telugu
+{
+ "code" => "te",
+ "name" => "telugu",
+ "use_new_loader" => true,
+ "use_old_patterns" => false,
+ "hyphenmin" => [1,1], # TODO
+ "encoding" => nil, # no patterns for 8-bit TeX
+ "exceptions" => false,
+ "message" => "Telugu Hyphenation Patterns",
+},
]
languages.each do |l|
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb
new file mode 100755
index 00000000000..072ea34de52
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb
@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+#
+# This script generates hyphenation patterns for Turkmen
+#
+# This script has been written by Mojca Miklavec <mojca dot miklavec dot lists at gmail dot com>
+
+# open file for writing the patterns
+# $tr = File.new("hyph-tk.tex", "w")
+# in TDS
+$tr = File.new("../../../../../tex/generic/hyph-utf8/patterns/hyph-tk.tex", "w")
+
+# write comments into the file
+def add_comment(str)
+ $tr.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n")
+end
+
+# define a class of vowels and consonants
+# vowels are split into so that unnecessary permutations are not generated
+front_vowels = %w{ä e i ö ü}
+back_vowels = %w{a y o u}
+consonants = %w{b ç d f g h j k l m n p r s t w ý z ň ž ş}
+# This is to eliminate impossible combinations
+common_suffix_consonants = %w{b ç d g j k l m n p s t ý z ş}
+
+
+# start the file
+add_comment(
+"Hyphenation patterns for Turkmen (hyph-tk.tex)
+
+Author: Nazar Annagurban <nazartm at gmail.com>
+License: Public domain
+Version: 0.1
+Date: 16 March 2010
+
+----------------------------------------------------------------------
+
+The file has been auto-generated from generate_patterns_tk.rb
+that is part of hyph-utf8.
+
+For more information about UTF-8 hyphenation patterns for TeX and
+links to this file see
+ http://www.tug.org/tex-hyphen/
+")
+
+# we have the following comment for Basque:
+#
+# Some of the patterns below represent combinations that never
+# happen in Turkmen. Would they happen, they would be hyphenated
+# according to the rules.
+
+$tr.puts '\patterns{'
+add_comment("Some suffixes are added through a hyphen. When hyphenating these words, a hyphen is added before the hyphen so that the line ends with a hyphen and the new line starts with a hyphen.")
+$tr.puts "1-4"
+
+add_comment("Allow hyphen after a vowel if and only if there is a single consonant before next the vowel")
+front_vowels.each do |v1|
+ consonants.each do |c|
+ front_vowels.each do |v2|
+ $tr.puts "#{v1}1#{c}#{v2}"
+ end
+ end
+end
+
+back_vowels.each do |v1|
+ consonants.each do |c|
+ back_vowels.each do |v2|
+ $tr.puts "#{v1}1#{c}#{v2}"
+ end
+ end
+end
+
+add_comment("These combinations occur in words of foreign origin or joined words")
+consonants.each do |c|
+ $tr.puts "a1#{c}i"
+ $tr.puts "a1#{c}e"
+ $tr.puts "y1#{c}ä"
+ $tr.puts "y1#{c}i"
+ $tr.puts "y1#{c}e"
+ $tr.puts "o1#{c}i"
+ $tr.puts "o1#{c}e"
+ $tr.puts "u1#{c}i"
+ $tr.puts "u1#{c}e"
+ $tr.puts "i1#{c}a"
+ $tr.puts "i1#{c}o"
+ $tr.puts "e1#{c}a"
+ $tr.puts "e1#{c}o"
+ $tr.puts "ä1#{c}o"
+ $tr.puts "ä1#{c}a"
+ $tr.puts "ö1#{c}a"
+end
+
+add_comment("Allow hyphen between two consonants (if there is only two of them), except when they are at the begining of the word")
+consonants.each do |c1|
+ consonants.each do |c2|
+ $tr.puts "#{c1}1#{c2}"
+ $tr.puts ".#{c1}2#{c2}"
+ end
+end
+
+add_comment("Patterns for triple consonants. There may be additions to this category, as this list is not exhaustive.")
+common_suffix_consonants.each do |c|
+ $tr.puts "ý2t1#{c}"
+ $tr.puts "ý2n1#{c}"
+ $tr.puts "ý2d1#{c}"
+ $tr.puts "r2t1#{c}"
+ $tr.puts "ý2p1#{c}"
+ $tr.puts "l2p1#{c}"
+ $tr.puts "l2t1#{c}"
+ $tr.puts "g2t1#{c}"
+ $tr.puts "n2t1#{c}"
+ $tr.puts "r2k1#{c}"
+ $tr.puts "r2p1#{c}"
+ $tr.puts "k2t1#{c}"
+ $tr.puts "r2h1#{c}"
+ $tr.puts "s2t1#{c}"
+ $tr.puts "l2k1#{c}"
+ $tr.puts "w2p1#{c}"
+ $tr.puts "n2s1#{c}"
+ $tr.puts "r2s1#{c}"
+ $tr.puts "l2m1#{c}"
+end
+
+add_comment("Exceptions and single word occurence patterns for words of foreign origin i.e. Russian")
+$tr.puts "s2k1d"
+$tr.puts "l1s2k"
+$tr.puts "l1s2t"
+$tr.puts "s1t2r"
+$tr.puts "n2g1l"
+$tr.puts "n1g2r"
+$tr.puts "s2k1w"
+
+# end the file
+$tr.puts '}'
+$tr.close
+