hyph-utf8 update (2010-03-21)

git-svn-id: svn://tug.org/texlive/trunk@17577 c570f23f-e606-0410-a88d-b1316a301751

18 files changed, 2569 insertions, 48 deletions

diff --git a/Master/texmf-dist/source/generic/hyph-utf8/README b/Master/texmf-dist/source/generic/hyph-utf8/README
new file mode 100644
index 00000000000..07ba26b49a4
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/README
@@ -0,0 +1,61 @@
+generate-converters.rb
+======================
+INPUT:
+- source/generic/hyph-utf8/data/encodings/*.dat
+OUTPUT:
+- tex/generic/hyph-utf8/conversions/conv-utf8-*.dat
+
+Auto-generates conversions from UTF-8 to some particular encoding.
+
+Needs to be run only in case that a new encoding is added or an old encoding is fixed.
+
+generate-offo.rb
+================
+Temporary; another version maintaned by author in Java; this one will be improved or removed.
+Will be rewritten to take raw patterns as input with no need to parse.
+
+generate-pattern-loaders.rb
+===========================
+INPUT:
+- language.rb with data
+OUTPUT:
+- tex/generic/hyph-utf8/loadhyph/loadhyph-*.tex
+
+Auto-generates pattern loaders for languages that should be used in TeX Live or other distros.
+
+Needs to be run when a new language is added, a bug needs to be fixed or a strategy is changed.
+
+generate-tl-files.rb
+====================
+Has been used once for the very first TL script generation.
+Out-of-date and not maintaned; will probably be removed.
+
+generate-webpage.rb
+===================
+Used to generated webpage with language overview.
+
+languages.rb
+============
+Database with language information used for almost every script.
+Needs some clean-up and some updates.
+
+
+languages/eu/generate_patterns_eu.rb
+====================================
+Generator for patterns for Basque.
+Author: Juan M. Aguirregabiria, adapted by Mojca Miklavec
+
+languages/gl
+============
+Generator for patterns for Galician.
+Author: Javier A. Múgica
+
+languages/tr/generate_patterns_tr.rb
+====================================
+Generator for patterns for Turkish.
+Author: See file, modified by Mojca Miklavec
+
+
+contributed/make-exhyph.pl
+==========================
+Imported script for hack related to breaking compound words.
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/make-exhyph.pl b/Master/texmf-dist/source/generic/hyph-utf8/contributed/make-exhyph.pl
index 3fe44239d21..3fe44239d21 100644..100755
--- a/Master/texmf-dist/source/generic/hyph-utf8/make-exhyph.pl
+++ b/Master/texmf-dist/source/generic/hyph-utf8/contributed/make-exhyph.pl
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java
new file mode 100644
index 00000000000..a4f70616a36
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java
@@ -0,0 +1,108 @@
+/*
+ * Copyright Simon Pepping 2009
+ * 
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: ConvertLanguageData.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URISyntaxException;
+import java.net.URL;
+
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+
+/**
+ * Convert language data in ruby format to XML format
+ */
+public final class ConvertLanguageData {
+    
+    /**
+     * @param languageDataPath
+     * @throws IOException
+     * @throws TransformerException
+     * @throws SAXException
+     * @throws URISyntaxException 
+     */
+    public static void convert(String languageDataPath, boolean useStylesheet)
+    throws IOException, TransformerException, SAXException, URISyntaxException {
+
+        // input
+        InputStream inis = new FileInputStream(languageDataPath);
+        InputSource input = new InputSource(inis);
+        input.setSystemId(languageDataPath);
+        input.setEncoding("utf-8");
+        XMLReader reader = new LanguageDataParser();
+
+        // output
+        String outPath = languageDataPath.replaceFirst("\\.rb$", ".xml");
+        Result result = new StreamResult(outPath);
+
+        // transformation
+        TransformerFactory tf = TransformerFactory.newInstance();
+        if (!tf.getFeature(SAXTransformerFactory.FEATURE)) {
+            throw new TransformerException("TransformerFactory is not a SAXTransformerFactory");
+        }
+        SAXTransformerFactory stf = (SAXTransformerFactory) tf;
+        TransformerHandler th;
+        if (useStylesheet) {
+            URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertLanguageData.xsl");
+            File xsltFile = new File(xsltUrl.toURI()); 
+            InputStream xsltStream = new FileInputStream(xsltFile);
+            Source xsltSource = new StreamSource(xsltStream);
+            xsltSource.setSystemId(xsltFile.getAbsolutePath());
+            th = stf.newTransformerHandler(xsltSource);
+        } else {
+            th = stf.newTransformerHandler();
+        }
+
+        // pipeline
+        reader.setContentHandler(th);
+        reader.setProperty("http://xml.org/sax/properties/lexical-handler", th);
+        th.setResult(result);
+        reader.parse(input);
+    }
+
+    /**
+     * @param args
+     * @throws IOException
+     * @throws TransformerException
+     * @throws SAXException
+     * @throws URISyntaxException 
+     */
+    public static void main(String[] args)
+    throws IOException, TransformerException, SAXException, URISyntaxException {
+        if (args[0].endsWith("--debug")) {
+            convert(args[1], false);
+        } else {
+            convert(args[0], true);
+        }
+    }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl
new file mode 100644
index 00000000000..98aa733a0c0
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl
@@ -0,0 +1,38 @@
+<?xml version="1.0" encoding="utf-8"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:lang="urn:org:tug:texhyphen:languagedata">
+
+  <xsl:output method="xml" encoding="UTF-8" indent="yes"/>
+
+  <xsl:variable name="language-data" select="document('codemapping.xml')/lang:code-mappings"/>
+
+  <xsl:template match="*|@*|node()">
+	<xsl:copy>
+	  <xsl:apply-templates select="*|@*|node()"/>
+	</xsl:copy>
+  </xsl:template>
+
+  <xsl:template match="lang:language">
+	<xsl:variable name="code" select="$language-data/lang:code-mapping[@code=current()/@code]"/>
+	<xsl:variable name="fop-code">
+	  <xsl:choose>
+		<xsl:when test="$code">
+		  <xsl:value-of select="$code/@fop-code"/>
+		</xsl:when>
+		<xsl:otherwise>
+		  <xsl:value-of select="@code"/>
+		</xsl:otherwise>
+	  </xsl:choose>
+	</xsl:variable>
+	<xsl:copy>
+	  <xsl:if test="string($fop-code)">
+		<xsl:attribute name="fop-code">
+		  <xsl:value-of select="$fop-code"/>
+		</xsl:attribute>
+	  </xsl:if>
+	  <xsl:apply-templates select="*|@*|node()"/>
+	</xsl:copy>
+  </xsl:template>
+
+</xsl:stylesheet>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java
new file mode 100644
index 00000000000..1ec9cee9474
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java
@@ -0,0 +1,385 @@
+/*
+ * Copyright Simon Pepping 2009
+ * 
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: ConvertTeXPattern.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+
+import javax.xml.parsers.ParserConfigurationException;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import javax.xml.transform.Result;
+import javax.xml.transform.Source;
+import javax.xml.transform.Transformer;
+import javax.xml.transform.TransformerException;
+import javax.xml.transform.TransformerFactory;
+import javax.xml.transform.sax.SAXTransformerFactory;
+import javax.xml.transform.sax.TransformerHandler;
+import javax.xml.transform.stream.StreamResult;
+import javax.xml.transform.stream.StreamSource;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.helpers.DefaultHandler;
+
+/**
+ * Convert modern UTF8 TeX hyphenation patterns to XML format
+ */
+public final class ConvertTeXPattern {
+    
+    public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet,
+                               boolean useLanguagedata)
+    throws IOException, TransformerException, SAXException, URISyntaxException,
+    ParserConfigurationException, CodeMappingException {
+        checkCodeMapping();
+        Collection<String> languages = codeMapping.keySet();
+        convert(texPatterns, outfilePath, useStylesheet, languages);
+    }
+
+    public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet)
+    throws IOException, TransformerException, SAXException, URISyntaxException,
+    CodeMappingException {
+        convert(texPatterns, outfilePath, useStylesheet, null);
+    }
+    
+    /**
+     * infile outfile
+     * indir outdir (file protocol only)
+     * infiles outdir
+     * file and http protocols allowed
+     * 
+     * @param texPatternUri
+     * @param outfilePath
+     * @param useStylesheet
+     * @param texcodes filter of requested tex codes; is allowed to be null 
+     * @throws IOException
+     * @throws TransformerException
+     * @throws SAXException
+     * @throws URISyntaxException 
+     * @throws CodeMappingException 
+     */
+    public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet,
+                               Collection<String> texcodes)
+    throws IOException, TransformerException, SAXException, URISyntaxException,
+    CodeMappingException {
+        File outDir = new File(outfilePath);
+        boolean oneTexcode = (texcodes != null && texcodes.size() == 1);
+        boolean oneInputfile = (texPatterns.length == 1);
+        boolean oneFilteredInput = (oneTexcode || oneInputfile);
+        if (!oneFilteredInput && !outDir.isDirectory()) {
+            throw new IllegalArgumentException
+            ("with multiple input files the output path " + outfilePath + " must be a directory");
+        }
+        for (String texPattern : texPatterns) {
+            URI texPatternUri = makeTexPatternUri(texPattern);
+            URI[] texPatternUris = makeTexPatternUris(texPatternUri);
+            oneInputfile = (texPatternUris.length == 1);
+            oneFilteredInput = (oneTexcode || oneInputfile);
+            if (!oneFilteredInput && !outDir.isDirectory()) {
+                throw new IllegalArgumentException
+                ("with an input directory " + texPattern + " the output path " + outfilePath + " must be a directory");
+            }
+            for (URI t : texPatternUris) {
+                TransformationData transformationData = makeTransformationData(t, outDir, texcodes);
+                if (transformationData == null) {
+                    continue;
+                }
+                doConvert(t, transformationData, useStylesheet);
+            }
+        }
+    }
+    
+    /**
+     * @param texPattern
+     * @return
+     * @throws URISyntaxException
+     * @throws FileNotFoundException
+     */
+    private static URI makeTexPatternUri(String texPattern)
+    throws URISyntaxException, FileNotFoundException {
+        URI texPatternUri;
+        texPatternUri = new URI(texPattern);
+        String scheme = texPatternUri.getScheme();
+        // see if it is a relative file path
+        if (scheme == null) {
+            File f = new File(texPattern);
+            texPatternUri = new URI("file", null, f.getAbsolutePath(), null, null);
+            scheme = texPatternUri.getScheme();
+        }
+        if (scheme == null || !(scheme.equals("http") || scheme.equals("file"))) {
+            throw new FileNotFoundException
+            ("URI with file or http scheme required for hyphenation pattern file");
+        }
+        return texPatternUri;
+    }
+
+    /**
+     * @param outfilePath
+     * @param outDir
+     * @param texPatternUri
+     * @param scheme
+     * @return
+     * @throws URISyntaxException
+     */
+    private static URI[] makeTexPatternUris(URI texPatternUri) throws URISyntaxException {
+        URI[] texPatternUris;
+        texPatternUris = new URI[] {texPatternUri};
+        String scheme = texPatternUri.getScheme();
+        if (scheme.equals("file")) {
+            File dir = new File(texPatternUri);
+            if (dir.isDirectory()) {
+                ArrayList<URI> l = new ArrayList<URI>();
+                FilenameFilter filter = new FilenameFilter() {
+                    public boolean accept(File dir, String name) {
+                        return name.endsWith(".tex");
+                    }
+                };
+                for (File f : dir.listFiles(filter)) {
+                    l.add(new URI("file", null, f.getAbsolutePath(), null, null));
+                }
+                texPatternUris = l.toArray(texPatternUris);
+            }
+        }
+        return texPatternUris;
+    }
+
+    /**
+     * @param t
+     * @param outDir
+     * @param texcodes filter of requested tex codes; is allowed to be null 
+     * @return
+     * @throws CodeMappingException 
+     */
+    private static TransformationData makeTransformationData
+    (URI t, File outDir, Collection<String> texcodes) throws CodeMappingException {
+        File outFile;
+        String path = t.getPath();
+        String basename = path.substring(path.lastIndexOf('/') + 1);
+        String base = basename.substring(0, basename.lastIndexOf('.'));
+        // xmlCode, texCode
+        String[] codes = mapCode(base);
+        // code mapping lists no xmlCode
+        if (codes[0] == null) {
+            return null;
+        }
+        if (texcodes != null && !texcodes.contains(codes[1])) {
+            return null;
+        }
+        if (!outDir.isDirectory()) {
+            outFile = outDir;
+        } else {
+            outFile = new File(outDir, codes[0] + ".xml");
+        }
+        return new TransformationData(outFile, codes[1]);
+    }
+    
+    private static class TransformationData {
+        File outFile;
+        String texCode;
+        TransformationData(File outFile, String texCode) {
+            this.outFile = outFile;
+            this.texCode = texCode;
+        }
+    }
+    
+    private static class CodeMappingException extends Exception {
+        public CodeMappingException(Exception e) {
+            super(e);
+        }
+        public CodeMappingException(String m) {
+            super(m);
+        }
+    }
+
+    static Map<String, String> codeMapping;
+    static CodeMappingException codeMappingException;
+    static {
+        try {
+            codeMapping = readLanguagedata();
+        } catch (ParserConfigurationException e) {
+            codeMappingException = new CodeMappingException(e);
+        } catch (SAXException e) {
+            codeMappingException = new CodeMappingException(e);
+        } catch (IOException e) {
+            codeMappingException = new CodeMappingException(e);
+        }
+    }
+    
+    private static String[] mapCode(String texCode) throws CodeMappingException {
+        checkCodeMapping();
+        String hyp = "hyph-";
+        String xmlCode = texCode;
+        if (texCode.startsWith(hyp)) {
+            texCode = texCode.substring(hyp.length());
+            xmlCode = codeMapping.get(texCode);
+        }
+        return new String[] {xmlCode, texCode};
+    }
+
+    /**
+     * @throws CodeMappingException
+     */
+    private static void checkCodeMapping() throws CodeMappingException {
+        if (codeMapping == null) {
+            if (codeMappingException != null) {
+                throw codeMappingException;
+            } else {
+                throw new CodeMappingException("Failure initializing code mapping");
+            }
+        }
+    }
+    
+    public static Map<String,String> readLanguagedata()
+    throws ParserConfigurationException, SAXException, IOException {
+        SAXParserFactory spf = SAXParserFactory.newInstance();
+        spf.setNamespaceAware(true);
+        SAXParser parser = spf.newSAXParser();
+        InputStream is = ConvertTeXPattern.class.getResourceAsStream("languages.xml");
+        TexcodeReader texcodeReader = new TexcodeReader();
+        parser.parse(is, texcodeReader);
+        return texcodeReader.getTexcodes();
+    }
+    
+    private static class TexcodeReader extends DefaultHandler {
+        
+        private Map<String, String> texcodes = new HashMap<String, String>();
+
+        /* (non-Javadoc)
+         * @see org.xml.sax.helpers.DefaultHandler#startElement(java.lang.String, java.lang.String, java.lang.String, org.xml.sax.Attributes)
+         */
+        @Override
+        public void startElement(String uri, String localName, String qName,
+                                 Attributes attributes) throws SAXException {
+            if (uri.equals(LanguageDataParser.LANG_NAMESPACE) && localName.equals("language")) {
+                String texcode = attributes.getValue("code");
+                String fopcode = attributes.getValue("fop-code");
+                if (fopcode != null) {
+                    texcodes.put(texcode, fopcode);
+                }
+            }
+        }
+        
+        /**
+         * @return the texcodes
+         */
+        public Map<String,String> getTexcodes() {
+            return texcodes;
+        }
+        
+    }
+    
+    public static void doConvert(URI texPatternUri, TransformationData outdata, boolean useStylesheet)
+        throws TransformerException, SAXException, MalformedURLException, IOException, URISyntaxException {
+
+        String scheme = texPatternUri.getScheme();
+        InputStream inis = null;
+        if (scheme.equals("file")) {
+            File in = new File(texPatternUri);
+            inis = new FileInputStream(in);
+        } else if (scheme.equals("http")) {
+            inis = texPatternUri.toURL().openStream();
+        } else {
+            throw new FileNotFoundException
+            ("URI with file or http scheme required for hyphenation pattern file");
+        }
+
+        InputSource input = new InputSource(inis);
+        input.setSystemId(texPatternUri.toString());
+        input.setEncoding("utf-8");
+        XMLReader reader = new TeXPatternParser();
+        Result result = new StreamResult(outdata.outFile);
+        TransformerFactory tf = TransformerFactory.newInstance();
+        if (!tf.getFeature(SAXTransformerFactory.FEATURE)) {
+            throw new TransformerException("TransformerFactory is not a SAXTransformerFactory");
+        }
+        SAXTransformerFactory stf = (SAXTransformerFactory) tf;
+        TransformerHandler th;
+        if (useStylesheet) {
+            URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertTeXPattern.xsl");
+            File xsltFile = new File(xsltUrl.toURI()); 
+            InputStream xsltStream = new FileInputStream(xsltFile);
+            Source xsltSource = new StreamSource(xsltStream);
+            xsltSource.setSystemId(xsltFile.getAbsolutePath());
+            th = stf.newTransformerHandler(xsltSource);
+            Transformer tr = th.getTransformer();
+            tr.setParameter("tex-code", outdata.texCode);
+        } else {
+            th = stf.newTransformerHandler();
+        }
+        reader.setContentHandler(th);
+        reader.setProperty("http://xml.org/sax/properties/lexical-handler", th);
+        th.setResult(result);
+        reader.parse(input);
+    }
+
+    /**
+     * @param args input URI, output file
+     * @throws URISyntaxException if the URI is not correct
+     * @throws IOException if a file is not found, or contains illegal content
+     * @throws TransformerException 
+     * @throws SAXException 
+     * @throws ParserConfigurationException 
+     * @throws CodeMappingException 
+     */
+    public static void main(String[] args)
+    throws URISyntaxException, IOException, TransformerException, SAXException,
+    ParserConfigurationException, CodeMappingException {
+        String prefix = "--";
+        int i = 0;
+        boolean useStylesheet = true;
+        boolean useLanguagedata = false;
+        Collection<String> texcodes = null;
+        while (args[i].startsWith(prefix)) {
+            String option = args[i].substring(prefix.length());
+            if (option.equals("debug")) {
+                useStylesheet = false;
+            } else if (option.equals("uselanguagedata") || option.equals("langdata")) {
+                useLanguagedata = true;
+            } else if (option.equals("texcodes")) {
+                texcodes = Arrays.asList(args[++i].split(","));
+            } else {
+                throw new IllegalArgumentException("Unknown option: " + option);
+            }
+            ++i;
+        }
+        if (texcodes != null) {
+            convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1],
+                    useStylesheet, texcodes);
+        } else {
+            convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1],
+                    useStylesheet, useLanguagedata);
+        }
+    }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl
new file mode 100644
index 00000000000..803962b7d5a
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl
@@ -0,0 +1,128 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<xsl:stylesheet version="1.0"
+  xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
+  xmlns:tex="urn:org:tug:texhyphen"
+  xmlns:lang="urn:org:tug:texhyphen:languagedata"
+  exclude-result-prefixes="tex lang">
+
+  <xsl:output doctype-system="hyphenation.dtd" indent="yes"/>
+  
+  <xsl:param name="comment-length" select="72"/>
+  <xsl:param name="tex-code"/>
+  <xsl:param name="hyphen-min-before-default" select="2"/>
+  <xsl:param name="hyphen-min-after-default" select="3"/>
+
+  <xsl:template match="/tex:tex">
+    <hyphenation-info>
+	  <xsl:choose>
+		<xsl:when test="tex:hyphenation">
+		  <!-- (comment*), (patterns), (comment*, hyphenation, comment*) => (comment*), hyphen-min, (comment*, exceptions, comment*), (patterns) -->
+		  <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/>
+		  <xsl:variable name="set2" select="tex:patterns"/>
+		  <xsl:variable name="set3" select="node()[preceding-sibling::tex:patterns]"/>
+		  <xsl:apply-templates select="$set1"/>
+		  <xsl:call-template name="hyphen-min"/>
+		  <xsl:apply-templates select="$set3"/>
+		  <xsl:apply-templates select="$set2"/>
+		</xsl:when>
+		<xsl:otherwise>
+		  <!-- (comment*), (patterns, comment*) => (comment*), hyphen-min, (patterns, comment*) -->
+		  <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/>
+		  <xsl:variable name="set2" select="node()[preceding-sibling::tex:patterns or self::tex:patterns]"/>
+		  <xsl:apply-templates select="$set1"/>
+		  <xsl:call-template name="hyphen-min"/>
+		  <xsl:apply-templates select="$set2"/>
+		</xsl:otherwise>
+	  </xsl:choose>
+    </hyphenation-info>
+  </xsl:template>
+
+  <xsl:template match="tex:patterns">
+    <patterns>
+      <xsl:apply-templates />
+    </patterns>
+  </xsl:template>
+
+  <xsl:template match="tex:patterns" mode="call-hyphen-min">
+	<xsl:call-template name="hyphen-min"/>
+    <patterns>
+      <xsl:apply-templates />
+    </patterns>
+  </xsl:template>
+
+  <xsl:template name="hyphen-min">
+    <xsl:variable name="hyphen-min"
+      select="document('languages.xml')/lang:languages/lang:language[@code=$tex-code]/lang:hyphen-min" />
+    <xsl:if test="count($hyphen-min)">
+      <hyphen-min before="{$hyphen-min/@before}" after="{$hyphen-min/@after}" />
+    </xsl:if>
+  </xsl:template>
+
+  <xsl:template match="tex:message"/>
+
+  <xsl:template match="tex:hyphenation">
+    <exceptions>
+      <xsl:apply-templates />
+    </exceptions>
+  </xsl:template>
+
+  <!-- Comments in TeX contain the trailing new line.                     -->
+  <!-- Here we keep the trailing new line if the comment is immediately   -->
+  <!-- preceded or followed by a text node.                               -->
+  <!-- Otherwise we strip the new line and pad the comment                -->
+  <!-- to the parameter comment-length.                                   -->
+  <!-- The XSLT engine uses the same criteria to decide if the comment    -->
+  <!-- should start on a new line or not.                                 -->
+  <!-- This is not quite correct, because we risk adding spaces to        -->
+  <!-- elements with mixed content.                                       -->
+  <!-- The following test would be more appropriate:                      -->
+  <!-- test="preceding-sibling::text() or following-sibling::text()".     -->
+  <xsl:template match="comment()">
+    <xsl:choose>
+      <xsl:when
+        test="preceding-sibling::node()[1][self::text()]
+            or following-sibling::node()[1][self::text()]">
+        <xsl:comment>
+          <xsl:value-of select="." />
+        </xsl:comment>
+      </xsl:when>
+      <xsl:otherwise>
+        <xsl:variable name="length" select="string-length(.)" />
+        <xsl:comment>
+          <xsl:value-of select="substring(.,1,$length - 1)" />
+          <xsl:text> </xsl:text>
+          <xsl:call-template name="make-spaces">
+            <xsl:with-param name="length"
+              select="$comment-length - ($length - 1)" />
+          </xsl:call-template>
+        </xsl:comment>
+      </xsl:otherwise>
+    </xsl:choose>
+  </xsl:template>
+
+  <xsl:template name="make-spaces">
+    <xsl:param name="length" select="0" />
+    <xsl:choose>
+      <xsl:when test="$length >= 10">
+        <xsl:text>          </xsl:text>
+        <xsl:call-template name="make-spaces">
+          <xsl:with-param name="length" select="$length - 10" />
+        </xsl:call-template>
+      </xsl:when>
+      <xsl:when test="$length >= 5">
+        <xsl:text>     </xsl:text>
+        <xsl:call-template name="make-spaces">
+          <xsl:with-param name="length" select="$length - 5" />
+        </xsl:call-template>
+      </xsl:when>
+      <xsl:when test="$length >= 1">
+        <xsl:text> </xsl:text>
+        <xsl:call-template name="make-spaces">
+          <xsl:with-param name="length" select="$length - 1" />
+        </xsl:call-template>
+      </xsl:when>
+    </xsl:choose>
+    <xsl:text></xsl:text>
+  </xsl:template>
+
+</xsl:stylesheet>
\ No newline at end of file
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java
new file mode 100644
index 00000000000..95c484b25ab
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java
@@ -0,0 +1,428 @@
+/*
+ * Copyright Simon Pepping 2009
+ * 
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: LanguageDataParser.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Vector;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.xml.sax.Attributes;
+import org.xml.sax.ContentHandler;
+import org.xml.sax.DTDHandler;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.ext.LexicalHandler;
+import org.xml.sax.helpers.AttributesImpl;
+
+/**
+ * The class TeXParser parses TeX hyphenation pattern files and produces SAX events
+ */
+public class LanguageDataParser implements XMLReader {
+    
+    public static final String LANG_NAMESPACE = "urn:org:tug:texhyphen:languagedata";
+    public static int lineLength = 72;
+    private static final int TOP_LEVEL = 3, IN_LANG = 4;
+    private static final Pattern
+    comment = Pattern.compile("#.*"),
+    langStart = Pattern.compile("{", Pattern.LITERAL),
+    langEnd = Pattern.compile("}", Pattern.LITERAL),
+    dataline = Pattern.compile("\"([^\"]+)\" ?=> ?\"([^\"]+)\","),
+    keywordline = Pattern.compile("\"([^\"]+)\" ?=> ?(false|true|nil),"),
+    listline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[(\"[^\"]+\"(?:,\"[^\"]+\")*)\\],"),
+    datalistline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[([^,]+(?:,[^,]+)*)\\],"),
+    space = Pattern.compile("[ \\t]+");
+    private static final AttributesImpl emptyAtts = new AttributesImpl();
+
+    private ContentHandler contentHandler;
+    private DTDHandler dtdHandler;
+    private EntityResolver entityResolver;
+    private ErrorHandler errorHandler;
+    private LexicalHandler lexicalHandler;
+    
+    private void parseLanguageData(BufferedReader inbr) throws SAXException, IOException {
+        int parseState = TOP_LEVEL;
+        Language lang = null;
+        
+        contentHandler.startDocument();
+        contentHandler.startPrefixMapping("", LANG_NAMESPACE);
+        contentHandler.startElement(LANG_NAMESPACE, "languages", "languages", emptyAtts);
+        
+        for (String line = inbr.readLine(); line != null; line = inbr.readLine()) {
+            Matcher matcher = comment.matcher(line).useAnchoringBounds(true);
+            int start = 0;
+            while (start < line.length()) {
+                if (matcher.usePattern(comment).lookingAt()) {
+                    processComment(matcher.group(), parseState == TOP_LEVEL ? null : lang);
+                } else if (matcher.usePattern(space).lookingAt()) {
+                    // do nothing
+                } else if (parseState == TOP_LEVEL && matcher.usePattern(langStart).lookingAt()) {
+                    parseState = IN_LANG;
+                    lang = new Language();
+                } else if ((parseState == IN_LANG) && matcher.usePattern(langEnd).lookingAt()) {
+                    pushoutLanguage(lang);
+                    lang = null;
+                    parseState = TOP_LEVEL;
+                } else if (parseState == IN_LANG
+                        && (matcher.usePattern(dataline).lookingAt()
+                                || matcher.usePattern(keywordline).lookingAt())) {
+                    String key = matcher.group(1);
+                    String value = matcher.group(2);
+                    processDataline(key, value, lang);
+                } else if (parseState == IN_LANG
+                        && (matcher.usePattern(listline).lookingAt()
+                                || matcher.usePattern(datalistline).lookingAt())) {
+                    String key = matcher.group(1);
+                    String values = matcher.group(2);
+                    processListline(key, values, lang);
+                } else {
+                    break;
+                }
+                start = matcher.end();
+                matcher = matcher.region(start, line.length()).useAnchoringBounds(true);
+            }
+        }
+
+        contentHandler.endElement(LANG_NAMESPACE, "languages", "languages");
+        contentHandler.endPrefixMapping(LANG_NAMESPACE);
+        contentHandler.endDocument();
+    }
+    
+    static Collection<String> attributeKeys;
+    static {
+        attributeKeys = new Vector<String>();
+        attributeKeys.add("code");
+        attributeKeys.add("name");
+        attributeKeys.add("use-old-patterns");
+        attributeKeys.add("use-new-loader");
+        attributeKeys.add("encoding");
+        attributeKeys.add("exceptions");
+    }
+    
+    private void processComment(String comment, Language lang) throws SAXException {
+        comment = comment.replace("--", "––");
+        if (!comment.endsWith(" ")) {
+            comment = comment + " ";
+        }
+        if (lang == null) {
+            char[] textchars = comment.toCharArray();
+            if (lexicalHandler != null) {
+                lexicalHandler.comment(textchars, 1, textchars.length - 1);
+            }
+        } else {
+            lang.elements.add(new Element("comment", comment));
+        }
+
+    }
+    
+    private void processDataline(String key, String value, Language lang) {
+        key = key.replace('_', '-');
+        if (value.equals("nil")) {
+            value = "";
+        }
+        if (attributeKeys.contains(key)) {
+            lang.atts.addAttribute("", key, key, "CDATA", value);
+        } else {
+            lang.elements.add(new Element(key, value));
+        }
+    }
+
+    private void processListline(String key, String valuesString, Language lang) {
+        key = key.replace('_', '-');
+        valuesString = valuesString.replace("\"", "");
+        String[] values = valuesString.split(",[ \\t]*");
+        if (attributeKeys.contains(key)) {
+            StringBuilder attValue = new StringBuilder();
+            for (String value : values) {
+                if (!value.equals("nil")) {
+                    attValue.append(" " + value);
+                }
+            }
+            lang.atts.addAttribute("", key, key, "CDATA", attValue.toString());
+        } else if (key.equals("hyphenmin")) {
+            key = "hyphen-min";
+            AttributesImpl atts = new AttributesImpl();
+            atts.addAttribute("", "before", "before", "CDATA", values[0]);
+            atts.addAttribute("", "after", "after", "CDATA", values[1]);
+            lang.elements.add(new Element(key, "", atts));
+        } else {
+            key = key.replaceAll("s$", "");
+            for (String value : values) {
+            if (value.equals("nil")) {
+                    value = "";
+                }
+                lang.elements.add(new Element(key, value));
+            }
+        }
+    }
+
+    private void pushoutLanguage(Language lang) throws SAXException {
+        contentHandler.startElement(LANG_NAMESPACE, "language", "language", lang.atts);
+        Iterator<Element> iter = lang.elements.iterator();
+        while (iter.hasNext()) {
+            Element elt = iter.next();
+            char[] text = elt.content.toCharArray();
+            if (elt.tag.equals("comment")) {
+                if (lexicalHandler != null) {
+                    lexicalHandler.comment(text, 1, text.length - 1);
+                }
+            } else {
+                contentHandler.startElement(LANG_NAMESPACE, elt.tag, elt.tag, elt.atts);
+                contentHandler.characters(text, 0, text.length);
+                contentHandler.endElement(LANG_NAMESPACE, elt.tag, elt.tag);
+            }
+        }
+        contentHandler.endElement(LANG_NAMESPACE, "language", "language");
+    }
+    
+    public Reader getReaderFromInputSource(InputSource input) throws IOException {
+        Reader reader = input.getCharacterStream();
+        String encoding = null;
+        if (reader == null) {
+            encoding = input.getEncoding();
+        }
+        if (reader == null) {
+            InputStream stream = input.getByteStream();
+            if (stream != null) {
+                if (encoding == null) {
+                    reader = new InputStreamReader(stream);
+                } else {
+                    reader = new InputStreamReader(stream, encoding);
+                }
+            }
+        }
+        if (reader == null) {
+            String systemId = input.getSystemId();
+            reader = getReaderFromSystemId(systemId, encoding);
+        }
+        return reader;
+    }
+
+    public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException {
+        if (systemId == null) {
+            throw new IOException("Cannot create a reader from a null systemID");
+        }
+        if (encoding.isEmpty()) {
+            encoding = null;
+        }
+        Reader reader = null;
+        URI uri = null;
+        File file = null;
+        try {
+            uri = new URI(systemId);
+        } catch (URISyntaxException e) {
+            // handled below
+        }
+        if (uri == null || !uri.isAbsolute()) {
+            file = new File(systemId);
+        }
+        if (file != null) {
+            if (encoding == null) {
+                reader = new FileReader(file);
+            } else {
+                InputStream stream = new FileInputStream(file);
+                reader = new InputStreamReader(stream, encoding);
+            }
+        } else if (uri != null && uri.getScheme().equals("http")) {
+            URL url = uri.toURL();
+            URLConnection conn = url.openConnection();
+            if (encoding == null) {
+                encoding = conn.getContentEncoding();
+            }
+            InputStream stream = conn.getInputStream();
+            reader = new InputStreamReader(stream, encoding);
+        }
+        return reader;
+    }
+    
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getContentHandler()
+     */
+    public ContentHandler getContentHandler() {
+        return contentHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getDTDHandler()
+     */
+    public DTDHandler getDTDHandler() {
+        return dtdHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getEntityResolver()
+     */
+    public EntityResolver getEntityResolver() {
+        return entityResolver;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getErrorHandler()
+     */
+    public ErrorHandler getErrorHandler() {
+        return errorHandler;
+    }
+
+    
+    /**
+     * @return the lexicalHandler
+     */
+    public LexicalHandler getLexicalHandler() {
+        return lexicalHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getFeature(java.lang.String)
+     */
+    public boolean getFeature(String arg0)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        throw new SAXNotSupportedException();
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getProperty(java.lang.String)
+     */
+    public Object getProperty(String arg0)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        throw new SAXNotSupportedException();
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
+     */
+    public void parse(InputSource input) throws IOException, SAXException {
+        Reader reader = getReaderFromInputSource(input);
+        if (reader == null) {
+            throw new IOException("Could not open input source " + input);
+        }
+        BufferedReader inbr = new BufferedReader(reader);
+        parseLanguageData(inbr);
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#parse(java.lang.String)
+     */
+    public void parse(String systemId) throws IOException, SAXException {
+        Reader reader = getReaderFromSystemId(systemId, null);
+        if (reader == null) {
+            throw new IOException("Could not open input systemID " + systemId);
+        }
+        BufferedReader inbr = new BufferedReader(reader);
+        parseLanguageData(inbr);
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler)
+     */
+    public void setContentHandler(ContentHandler contenthandler) {
+        this.contentHandler = contenthandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler)
+     */
+    public void setDTDHandler(DTDHandler dtdhandler) {
+        this.dtdHandler = dtdhandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver)
+     */
+    public void setEntityResolver(EntityResolver entityresolver) {
+        this.entityResolver = entityresolver;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler)
+     */
+    public void setErrorHandler(ErrorHandler errorHandler) {
+        this.errorHandler = errorHandler;
+    }
+
+    
+    /**
+     * @param lexicalHandler the lexicalHandler to set
+     */
+    public void setLexicalHandler(LexicalHandler lexicalHandler) {
+        this.lexicalHandler = lexicalHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean)
+     */
+    public void setFeature(String arg0, boolean arg1)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        throw new SAXNotSupportedException();
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object)
+     */
+    public void setProperty(String name, Object value)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        if (name.equals("http://xml.org/sax/properties/lexical-handler")) {
+            lexicalHandler = (LexicalHandler) value;
+        } else {
+            throw new SAXNotSupportedException();
+        }
+    }
+    
+    private static class Element {
+        String tag;
+        String content;
+        Attributes atts;
+        Element(String tag, String content) {
+            this(tag, content, LanguageDataParser.emptyAtts);
+        }
+        Element(String tag, String content, Attributes atts) {
+            this.tag = tag;
+            this.content = content;
+            this.atts = atts;
+        }
+    }
+    
+    private static class Language {
+        AttributesImpl atts;
+        List<Element> elements;
+        Language() {
+            atts = new AttributesImpl();
+            elements = new Vector<Element>();
+        }
+    }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README
new file mode 100644
index 00000000000..aaf4e074ae6
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README
@@ -0,0 +1,38 @@
+This utility consists of two parts.
+
+Part 1 converts languages.rb to languages.xml. It consists of
+ConvertLanguageData.java, LanguageDataParser.java,
+ConvertLanguageData.xsl, codemapping.xml. Invocation:
+
+java org.tug.texhyphen.util.ConvertLanguageData path/to/languages.rb
+
+output: languages.xml, in the same directory as languages.rb.
+
+The file codemapping contains the mapping from tex-code to
+fop-code. If this file has an entry with non-empty fop-code, it is
+inserted in languages.xml as attribute fop-code. If this file has an
+entry with empty fop-code, that language gets no attribute
+fop-code. Otherwise, that language gets attribute fop-code with the
+same value as tex-code (attribute code).
+
+The current file languages.xml was modified manually after conversion,
+to add the three languages at the bottom of the file.
+
+Part 2 converts tex pattern files to fop pattern files. It consists of
+ConvertTeXPattern.java, TeXPatternParser.java, ConvertTeXPattern.xsl,
+languages.xml. Invocation:
+
+java org.tug.texhyphen.util.ConvertTeXPattern [--uselanguagedata]
+[--texcodes code1,code2,...] infileuri1 [infileuri2 ...] outfile
+
+infileuri must be a file or http URI. With a file URI, it may also be
+a directory; in that case all pattern files in that directory are
+converted. outfile must be a path to an output filename or
+directory. If there are more input files, it must be an output
+directory. Option 'uselanguagedata' converts all input files which
+represent a language in the languages file with a fop-code. Option
+'--texcodes code1,code2,...' converts only those input files which
+represent a language with one of the given tex codes. Without those
+options all input files are converted. In any case, only files are
+converted which represent a language in the languages file with a
+fop-code.
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java
new file mode 100644
index 00000000000..0cc5a542f78
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java
@@ -0,0 +1,325 @@
+/*
+ * Copyright Simon Pepping 2009
+ * 
+ * The copyright owner licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* $Id: TeXPatternParser.java 304 2009-11-26 07:26:55Z spepping $ */
+
+package org.tug.texhyphen;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.net.URL;
+import java.net.URLConnection;
+import java.util.Stack;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.xml.sax.ContentHandler;
+import org.xml.sax.DTDHandler;
+import org.xml.sax.EntityResolver;
+import org.xml.sax.ErrorHandler;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.SAXNotRecognizedException;
+import org.xml.sax.SAXNotSupportedException;
+import org.xml.sax.XMLReader;
+import org.xml.sax.ext.LexicalHandler;
+import org.xml.sax.helpers.AttributesImpl;
+
+/**
+ * The class TeXParser parses TeX hyphenation pattern files and produces SAX events
+ */
+public class TeXPatternParser implements XMLReader {
+    
+    public static final String TEX_NAMESPACE = "urn:org:tug:texhyphen";
+    private static final int TOP_LEVEL = 3, IN_COMMAND = 4, AFTER_COMMAND = 5, IN_DATA = 6;
+    private static final Pattern
+    comment = Pattern.compile("%.*"),
+    commandStart = Pattern.compile("\\\\"),
+    command = Pattern.compile("[a-zA-Z]+"),
+    space = Pattern.compile(" +"),
+    argOpen = Pattern.compile("\\{"),
+    argClose = Pattern.compile("\\}"),
+    text = Pattern.compile("[^%\\\\\\{\\}]+");
+    private static final AttributesImpl emptyAtts = new AttributesImpl();
+
+    private ContentHandler contentHandler;
+    private DTDHandler dtdHandler;
+    private EntityResolver entityResolver;
+    private ErrorHandler errorHandler;
+    private LexicalHandler lexicalHandler;
+    
+    private void parsePatterns(BufferedReader inbr) throws SAXException, IOException {
+        int parseState = TOP_LEVEL;
+        Stack<String> stack = new Stack<String>();
+        
+        contentHandler.startDocument();
+        contentHandler.startPrefixMapping("", TEX_NAMESPACE);
+        contentHandler.startElement(TEX_NAMESPACE, "tex", "tex", emptyAtts);
+        
+        for (String line = inbr.readLine(); line != null; line = inbr.readLine()) {
+            Matcher matcher = comment.matcher(line).useAnchoringBounds(true);
+            int start = 0;
+            char[] textchars;
+            boolean inComment = false;
+            while (start < line.length()) {
+                if (matcher.usePattern(comment).lookingAt()) {
+                    String text = matcher.group().replace("--", "––");
+                    textchars = (text + "\n").toCharArray();
+                    if (lexicalHandler != null) {
+                        lexicalHandler.comment(textchars, 1, textchars.length - 1);
+                    }
+                    inComment = true;
+                } else if (parseState == IN_DATA && matcher.usePattern(text).lookingAt()) {
+                    textchars = matcher.group().toCharArray();
+                    contentHandler.characters(textchars, 0, textchars.length);
+                } else if (parseState != IN_DATA && matcher.usePattern(space).lookingAt()) {
+                    if (parseState == TOP_LEVEL) {
+                        textchars = matcher.group().toCharArray();
+                        contentHandler.ignorableWhitespace(textchars, 0, textchars.length);
+                    }
+                } else if (parseState == TOP_LEVEL && matcher.usePattern(commandStart).lookingAt()) {
+                    parseState = IN_COMMAND;
+                } else if (parseState == IN_COMMAND && matcher.usePattern(command).lookingAt()) {
+                    String tag = matcher.group();
+                    contentHandler.startElement(TEX_NAMESPACE, tag, tag, emptyAtts);
+                    stack.push(tag);
+                    parseState = AFTER_COMMAND;
+                } else if (parseState == AFTER_COMMAND && matcher.usePattern(argOpen).lookingAt()) {
+                    parseState = IN_DATA;
+                } else if (parseState == IN_DATA && matcher.usePattern(argClose).lookingAt()) {
+                    String tag = stack.pop();
+                    contentHandler.endElement(TEX_NAMESPACE, tag, tag);
+                    parseState = TOP_LEVEL;
+                } else {
+                    break;
+                }
+                start = matcher.end();
+                matcher = matcher.region(start, line.length()).useAnchoringBounds(true);
+            }
+                textchars = "\n".toCharArray();
+                if (parseState == IN_DATA && !inComment) {
+                    contentHandler.characters(textchars, 0, textchars.length);
+                } else if (parseState == TOP_LEVEL && !inComment) {
+                    contentHandler.ignorableWhitespace(textchars, 0, textchars.length);
+                }
+        }
+
+        contentHandler.endElement(TEX_NAMESPACE, "tex", "tex");
+        contentHandler.endPrefixMapping(TEX_NAMESPACE);
+        contentHandler.endDocument();
+    }
+
+    public Reader getReaderFromInputSource(InputSource input) throws IOException {
+        Reader reader = input.getCharacterStream();
+        String encoding = null;
+        if (reader == null) {
+            encoding = input.getEncoding();
+        }
+        if (reader == null) {
+            InputStream stream = input.getByteStream();
+            if (stream != null) {
+                if (encoding == null) {
+                    reader = new InputStreamReader(stream);
+                } else {
+                    reader = new InputStreamReader(stream, encoding);
+                }
+            }
+        }
+        if (reader == null) {
+            String systemId = input.getSystemId();
+            reader = getReaderFromSystemId(systemId, encoding);
+        }
+        return reader;
+    }
+
+    public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException {
+        if (systemId == null) {
+            throw new IOException("Cannot create a reader from a null systemID");
+        }
+        if (encoding.isEmpty()) {
+            encoding = null;
+        }
+        Reader reader = null;
+        URI uri = null;
+        File file = null;
+        try {
+            uri = new URI(systemId);
+        } catch (URISyntaxException e) {
+            // handled below
+        }
+        if (uri == null || !uri.isAbsolute()) {
+            file = new File(systemId);
+        }
+        if (file != null) {
+            if (encoding == null) {
+                reader = new FileReader(file);
+            } else {
+                InputStream stream = new FileInputStream(file);
+                reader = new InputStreamReader(stream, encoding);
+            }
+        } else if (uri != null && uri.getScheme().equals("http")) {
+            URL url = uri.toURL();
+            URLConnection conn = url.openConnection();
+            if (encoding == null) {
+                encoding = conn.getContentEncoding();
+            }
+            InputStream stream = conn.getInputStream();
+            reader = new InputStreamReader(stream, encoding);
+        }
+        return reader;
+    }
+    
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getContentHandler()
+     */
+    public ContentHandler getContentHandler() {
+        return contentHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getDTDHandler()
+     */
+    public DTDHandler getDTDHandler() {
+        return dtdHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getEntityResolver()
+     */
+    public EntityResolver getEntityResolver() {
+        return entityResolver;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getErrorHandler()
+     */
+    public ErrorHandler getErrorHandler() {
+        return errorHandler;
+    }
+
+    
+    /**
+     * @return the lexicalHandler
+     */
+    public LexicalHandler getLexicalHandler() {
+        return lexicalHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getFeature(java.lang.String)
+     */
+    public boolean getFeature(String arg0)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        throw new SAXNotSupportedException();
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#getProperty(java.lang.String)
+     */
+    public Object getProperty(String arg0)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        throw new SAXNotSupportedException();
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
+     */
+    public void parse(InputSource input) throws IOException, SAXException {
+        Reader reader = getReaderFromInputSource(input);
+        if (reader == null) {
+            throw new IOException("Could not open input source " + input);
+        }
+        BufferedReader inbr = new BufferedReader(reader);
+        parsePatterns(inbr);
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#parse(java.lang.String)
+     */
+    public void parse(String systemId) throws IOException, SAXException {
+        Reader reader = getReaderFromSystemId(systemId, null);
+        if (reader == null) {
+            throw new IOException("Could not open input systemID " + systemId);
+        }
+        BufferedReader inbr = new BufferedReader(reader);
+        parsePatterns(inbr);
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler)
+     */
+    public void setContentHandler(ContentHandler contenthandler) {
+        this.contentHandler = contenthandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler)
+     */
+    public void setDTDHandler(DTDHandler dtdhandler) {
+        this.dtdHandler = dtdhandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver)
+     */
+    public void setEntityResolver(EntityResolver entityresolver) {
+        this.entityResolver = entityresolver;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler)
+     */
+    public void setErrorHandler(ErrorHandler errorHandler) {
+        this.errorHandler = errorHandler;
+    }
+
+    
+    /**
+     * @param lexicalHandler the lexicalHandler to set
+     */
+    public void setLexicalHandler(LexicalHandler lexicalHandler) {
+        this.lexicalHandler = lexicalHandler;
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean)
+     */
+    public void setFeature(String arg0, boolean arg1)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        throw new SAXNotSupportedException();
+    }
+
+    /* (non-Javadoc)
+     * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object)
+     */
+    public void setProperty(String name, Object value)
+    throws SAXNotRecognizedException, SAXNotSupportedException {
+        if (name.equals("http://xml.org/sax/properties/lexical-handler")) {
+            lexicalHandler = (LexicalHandler) value;
+        } else {
+            throw new SAXNotSupportedException();
+        }
+    }
+
+}
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml
new file mode 100644
index 00000000000..c5b64722fa0
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<code-mappings xmlns="urn:org:tug:texhyphen:languagedata">
+  <code-mapping code="grc-x-ibycus" fop-code="grc_X_ibycus"/>
+  <code-mapping code="zh-latn" fop-code="zh_Latn"/>
+  <code-mapping code="de-1901" fop-code="de_1901"/>
+  <code-mapping code="de-1996" fop-code="de"/>
+  <code-mapping code="de-ch-1901" fop-code="de_CH_1901"/>
+  <code-mapping code="el-monoton" fop-code="el"/>
+  <code-mapping code="el-polyton" fop-code="el_Polyton"/>
+  <code-mapping code="mn-cyrl" fop-code="mn"/>
+  <code-mapping code="mn-cyrl-x-2a" fop-code="mn_Cyrl_x_2a"/>
+  <code-mapping code="sh-cyrl" fop-code="sh_Cyrl"/>
+  <code-mapping code="sh-latn" fop-code="sh_Latn"/>
+  <code-mapping code="sr-cyrl" fop-code="sr"/>
+  <code-mapping code="sr-latn" fop-code="sr_Latn"/>
+  <code-mapping code="en-gb" fop-code="en_GB"/>
+  <code-mapping code="en-us" fop-code="en"/>
+  <code-mapping code="eo" fop-code=""/>
+  <code-mapping code="nn" fop-code=""/>
+  <code-mapping code="nb" fop-code=""/>
+  <!-- hu patterns cause a stack overflow when compiled with FOP -->
+  <code-mapping code="hu" fop-code=""/>
+</code-mappings>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml
new file mode 100644
index 00000000000..2f0fd637d3c
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml
@@ -0,0 +1,434 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<languages xmlns="urn:org:tug:texhyphen:languagedata">
+  <!-- –––––––––––––––––––––––––––––––––––––– -->
+  <!-- languages with no hyphenation patterns -->
+  <!-- –––––––––––––––––––––––––––––––––––––– -->
+  <!-- arabic		zerohyph.tex -->
+  <language fop-code="ar" code="ar" name="arabic" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false">
+	<filename-old-patterns>zerohyph.tex</filename-old-patterns>
+	<!--	"hyphenmin" => [], # not needed -->
+	<message/>
+  </language>
+  <!-- farsi		zerohyph.tex -->
+  <!-- =persian -->
+  <language fop-code="fa" code="fa" name="farsi" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false">
+	<synonym>persian</synonym>
+	<filename-old-patterns>zerohyph.tex</filename-old-patterns>
+	<!--	"hyphenmin" => [], # not needed -->
+	<message/>
+  </language>
+  <!-- ––––––––––––––––––––––––––––––- -->
+  <!-- special patterns, not converted -->
+  <!-- ––––––––––––––––––––––––––––––- -->
+  <!-- ibycus ibyhyph.tex -->
+  <language fop-code="grc_X_ibycus" code="grc-x-ibycus" name="ibycus" use-new-loader="false" use-old-patterns="true" encoding="" exceptions="false">
+	<filename-old-patterns>ibyhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Ancient Greek Hyphenation Patterns for Ibycus encoding (v3.0)</message>
+	<author>peter_heslin</author>
+  </language>
+  <!-- –––––––––––––––––––––––––––– -->
+  <!-- languages using old patterns -->
+  <!-- –––––––––––––––––––––––––––– -->
+  <!-- greek		xu-grphyph4.tex -->
+  <!-- =polygreek -->
+  <language fop-code="el_Polyton" code="el-polyton" name="greek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true">
+	<synonym>polygreek</synonym>
+	<use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
+	<filename-old-patterns>grphyph5.tex</filename-old-patterns>
+	<!-- left/right hyphen min for Greek can be as low as one (1), -->
+	<!-- but for aesthetic reasons keep them at 2/2. -->
+	<!-- Dimitrios Filippou -->
+	<hyphen-min before="1" after="1"/>
+	<!-- polyglosia -->
+	<!--	"message" => "Polytonic Greek Hyphenation Patterns", -->
+	<message>Hyphenation patterns for multi-accent (polytonic) Modern Greek</message>
+	<author>dimitrios_filippou</author>
+  </language>
+  <!-- monogreek	xu-grmhyph4.tex -->
+  <language fop-code="el" code="el-monoton" name="monogreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true">
+	<use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
+	<filename-old-patterns>grmhyph5.tex</filename-old-patterns>
+	<hyphen-min before="1" after="1"/>
+	<!-- polyglosia -->
+	<!--	"message" => "Monotonic Greek Hyphenation Patterns", -->
+	<message>Hyphenation patterns for uni-accent (monotonic) Modern Greek</message>
+	<author>dimitrios_filippou</author>
+  </language>
+  <!-- ancientgreek	xu-grahyph4.tex -->
+  <language fop-code="grc" code="grc" name="ancientgreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false">
+	<use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment>
+	<filename-old-patterns>grahyph5.tex</filename-old-patterns>
+	<hyphen-min before="1" after="1"/>
+	<!-- polyglosia -->
+	<message>Hyphenation patterns for Ancient Greek</message>
+	<author>dimitrios_filippou</author>
+  </language>
+  <!-- coptic		xu-copthyph.tex -->
+  <language fop-code="cop" code="cop" name="coptic" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false">
+	<use-old-patterns-comment>TODO: automatic conversion could be done, but was too complicated; leave for later.</use-old-patterns-comment>
+	<filename-old-patterns>copthyph.tex</filename-old-patterns>
+	<hyphen-min before="1" after="1"/>
+	<!-- polyglosia TODO: no documentation found -->
+	<message>Coptic Hyphenation Patterns</message>
+	<author>claudio_beccari</author>
+  </language>
+  <!-- german		xu-dehypht.tex -->
+  <language fop-code="de_1901" code="de-1901" name="german" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false">
+	<use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment>
+	<filename-old-patterns>dehypht.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>German Hyphenation Patterns (Traditional Orthography)</message>
+  </language>
+  <!-- ngerman		xu-dehyphn.tex -->
+  <language fop-code="de" code="de-1996" name="ngerman" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false">
+	<use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment>
+	<filename-old-patterns>dehyphn.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>German Hyphenation Patterns (Reformed Orthography)</message>
+  </language>
+  <!-- swissgerman -->
+  <language fop-code="de_CH_1901" code="de-ch-1901" name="swissgerman" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<!-- TODO: how is it going to be called -->
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Swiss-German Hyphenation Patterns (Traditional Orthography)</message>
+  </language>
+  <!-- russian	xu-ruhyphen.tex -->
+  <language fop-code="ru" code="ru" name="russian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false">
+	<use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment>
+	<filename-old-patterns>ruhyphen.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Russian Hyphenation Patterns</message>
+  </language>
+  <!-- ukrainian	xu-ukrhyph.tex -->
+  <language fop-code="uk" code="uk" name="ukrainian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false">
+	<use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment>
+	<filename-old-patterns>ukrhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Ukrainian Hyphenation Patterns</message>
+  </language>
+  <!-- –––––––––––––––––––––––––––– -->
+  <!-- languages using new patterns -->
+  <!-- –––––––––––––––––––––––––––– -->
+  <!-- catalan		cahyph.tex -->
+  <language fop-code="ca" code="ca" name="catalan" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>cahyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Catalan Hyphenation Patterns</message>
+  </language>
+  <!-- czech		xu-czhyph.tex -->
+  <language fop-code="cs" code="cs" name="czech" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>czhyph.tex</filename-old-patterns>
+	<filename-old-patterns-other>czhyphen.tex</filename-old-patterns-other>
+	<filename-old-patterns-other>czhyphen.ex</filename-old-patterns-other>
+	<!-- Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 -->
+	<!-- Typographical rules allow \righthyphenmin=2 when typesetting in a -->
+	<!-- narrow column (newspapers etc.). -->
+	<!-- (used to be 2,2) -->
+	<hyphen-min before="2" after="3"/>
+	<message>Czech Hyphenation Patterns (Pavel Sevecek, v3, 1995)</message>
+  </language>
+  <!-- slovak		xu-skhyph.tex -->
+  <language fop-code="sk" code="sk" name="slovak" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>skhyph.tex</filename-old-patterns>
+	<filename-old-patterns-other>skhyphen.tex</filename-old-patterns-other>
+	<filename-old-patterns-other>skhyphen.ex</filename-old-patterns-other>
+	<!-- see czech -->
+	<hyphen-min before="2" after="3"/>
+	<message>Slovak Hyphenation Patterns (Jana Chlebikova, 1992)</message>
+  </language>
+  <!-- welsh		cyhyph.tex -->
+  <language fop-code="cy" code="cy" name="welsh" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>cyhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="3"/>
+	<message>Welsh Hyphenation Patterns</message>
+  </language>
+  <!-- danish		dkhyph.tex -->
+  <language fop-code="da" code="da" name="danish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>dkhyph.tex</filename-old-patterns>
+	<filename-old-patterns-other>dkcommon.tex</filename-old-patterns-other>
+	<filename-old-patterns-other>dkspecial.tex</filename-old-patterns-other>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Danish Hyphenation Patterns</message>
+  </language>
+  <!-- esperanto	xu-eohyph.tex -->
+  <!-- TODO -->
+  <language code="eo" name="esperanto" use-new-loader="true" use-old-patterns="false" encoding="il3" exceptions="false">
+	<filename-old-patterns>eohyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- TODO -->
+	<message>Esperanto Hyphenation Patterns</message>
+  </language>
+  <!-- spanish		xu-eshyph.tex -->
+  <!-- =espanol -->
+  <language fop-code="es" code="es" name="spanish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<synonym>espanol</synonym>
+	<filename-old-patterns>eshyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Spanish Hyphenation Patterns</message>
+  </language>
+  <!-- basque		xu-bahyph.tex -->
+  <language fop-code="eu" code="eu" name="basque" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>bahyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Basque Hyphenation Patterns</message>
+  </language>
+  <!-- french		xu-frhyph.tex -->
+  <!-- =patois -->
+  <!-- =francais -->
+  <language fop-code="fr" code="fr" name="french" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<synonym>patois</synonym>
+	<synonym>francais</synonym>
+	<filename-old-patterns>frhyph.tex</filename-old-patterns>
+	<!--	"hyphenmin" => [], -->
+	<message>French hyphenation patterns (V2.12, 2002/12/11)</message>
+  </language>
+  <!-- galician	xu-glhyph.tex -->
+  <language fop-code="gl" code="gl" name="galician" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>glhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Galician Hyphenation Patterns</message>
+  </language>
+  <!-- estonian	xu-ethyph.tex -->
+  <language fop-code="et" code="et" name="estonian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>ethyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="3"/>
+	<!-- babel -->
+	<message>Estonian Hyphenation Patterns</message>
+  </language>
+  <!-- finnish		fihyph.tex -->
+  <language fop-code="fi" code="fi" name="finnish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>fihyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Finnish Hyphenation Patterns</message>
+  </language>
+  <!-- croatian	xu-hrhyph.tex -->
+  <language fop-code="hr" code="hr" name="croatian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>hrhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Croatian Hyphenation Patterns</message>
+  </language>
+  <!-- hungarian	xu-huhyphn.tex -->
+  <language code="hu" name="hungarian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>huhyphn.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- polyglosia -->
+	<message>Hungarian Hyphenation Patterns (v20031107)</message>
+  </language>
+  <!-- interlingua	iahyphen.tex -->
+  <language fop-code="ia" code="ia" name="interlingua" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
+	<filename-old-patterns>iahyphen.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Hyphenation Patterns for Interlingua</message>
+  </language>
+  <!-- indonesian	inhyph.tex -->
+  <language fop-code="id" code="id" name="indonesian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
+	<filename-old-patterns>inhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Indonesian Hyphenation Patterns</message>
+  </language>
+  <!-- icelandic	icehyph.tex -->
+  <language fop-code="is" code="is" name="icelandic" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>icehyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Icelandic Hyphenation Patterns</message>
+  </language>
+  <!-- irish		gahyph.tex -->
+  <language fop-code="ga" code="ga" name="irish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>gahyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="3"/>
+	<!-- babel -->
+	<message>Irish Hyphenation Patterns</message>
+  </language>
+  <!-- italian		ithyph.tex -->
+  <language fop-code="it" code="it" name="italian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="false">
+	<filename-old-patterns>ithyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Italian Hyphenation Patterns</message>
+  </language>
+  <!-- kurmanji -->
+  <language fop-code="kmr" code="kmr" name="kurmanji" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>kmrhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Kurmanji Hyphenation Patterns (v. 1.0 2009/06/29 JKn and MSh)</message>
+  </language>
+  <!-- latin		xu-lahyph.tex -->
+  <language fop-code="la" code="la" name="latin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<use-old-patterns-comment>Old patterns support both EC &amp; OT1 encodings at the same time.</use-old-patterns-comment>
+	<filename-old-patterns>lahyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Latin Hyphenation Patterns</message>
+  </language>
+  <!-- lithuanian -->
+  <language fop-code="lt" code="lt" name="lithuanian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false">
+	<hyphen-min before="2" after="2"/>
+	<message>Lithuanian Hyphenation Patterns</message>
+  </language>
+  <!-- latvian -->
+  <language fop-code="lv" code="lv" name="latvian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false">
+	<hyphen-min before="2" after="2"/>
+	<message>Latvian Hyphenation Patterns</message>
+  </language>
+  <!-- dutch		nehyph96.tex -->
+  <language fop-code="nl" code="nl" name="dutch" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>nehyph96.tex</filename-old-patterns>
+	<!-- quoting Hans Hagen: -->
+	<!-- patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) -->
+	<hyphen-min before="2" after="2"/>
+	<message>Dutch Hyphenation Patterns</message>
+  </language>
+  <!-- norsk		xu-nohyphbx.tex -->
+  <!-- =norwegian -->
+  <!-- nynorsk         nnhyph.tex -->
+  <!-- bokmal          nbhyph.tex -->
+  <!-- polish		xu-plhyph.tex -->
+  <language fop-code="pl" code="pl" name="polish" use-new-loader="true" use-old-patterns="false" encoding="qx" exceptions="true">
+	<filename-old-patterns>plhyph.tex</filename-old-patterns>
+	<!--{}"hyphenmin" => [1,1], -->
+	<hyphen-min before="2" after="2"/>
+	<message>Polish Hyphenation Patterns</message>
+  </language>
+  <!-- portuguese	pthyph.tex -->
+  <!-- =portuges -->
+  <language fop-code="pt" code="pt" name="portuguese" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<synonym>portuges</synonym>
+	<filename-old-patterns>pthyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="3"/>
+	<!-- babel -->
+	<message>Portuguese Hyphenation Patterns</message>
+  </language>
+  <!-- pinyin		xu-pyhyph.tex -->
+  <language fop-code="zh_Latn" code="zh-latn" name="pinyin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>pyhyph.tex</filename-old-patterns>
+	<hyphen-min before="1" after="1"/>
+	<message>Hyphenation patterns for unaccented pinyin syllables (CJK 4.8.0)</message>
+  </language>
+  <!-- romanian	xu-rohyphen.tex -->
+  <language fop-code="ro" code="ro" name="romanian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>rohyphen.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Romanian Hyphenation Patterns</message>
+	<!-- : `rohyphen' 1.1 <29.10.1996> -->
+  </language>
+  <!-- slovenian	xu-sihyph.tex -->
+  <!-- =slovene -->
+  <language fop-code="sl" code="sl" name="slovenian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<synonym>slovene</synonym>
+	<filename-old-patterns>sihyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Slovenian Hyphenation Patterns</message>
+  </language>
+  <!-- uppersorbian	xu-sorhyph.tex -->
+  <language fop-code="hsb" code="hsb" name="uppersorbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>sorhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Upper Sorbian Hyphenation Patterns (E. Werner)</message>
+	<!--	\message{Hyphenation patterns for Upper Sorbian, E. Werner} -->
+	<!--	\message{Completely new revision 1997, March 22} -->
+  </language>
+  <!-- swedish		svhyph.tex -->
+  <language fop-code="sv" code="sv" name="swedish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>svhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- patters say it could be 1,2; babel says 2,2 - double check -->
+	<message>Swedish hyphenation patterns (Jan Michael Rynning, 1994-03-03)</message>
+  </language>
+  <!-- turkish		xu-tkhyph.tex -->
+  <language fop-code="tr" code="tr" name="turkish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false">
+	<filename-old-patterns>tkhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- polyglosia -->
+	<message>Turkish Hyphenation Patterns</message>
+  </language>
+  <!-- ukenglish	ukhyphen.tex -->
+  <!-- TODO - should we rename it or not? -->
+  <language fop-code="en_GB" code="en-gb" name="ukenglish" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true">
+	<synonym>british</synonym>
+	<synonym>UKenglish</synonym>
+	<filename-old-patterns>ukhyphen.tex</filename-old-patterns>
+	<hyphen-min before="2" after="3"/>
+	<!-- confirmed, same as what Knuth says -->
+	<message>Hyphenation Patterns for British English</message>
+  </language>
+  <!-- serbian		xu-srhyphc.tex -->
+  <language fop-code="sr_Latn" code="sr-latn" name="serbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<filename-old-patterns>shhyphl.tex</filename-old-patterns>
+	<!-- It is allowed to leave one character at the end of the row. -->
+	<!-- However, if you think that it is graphicaly not very -->
+	<!-- pleasent these patterns will work well with \lefthyphenmin=2. -->
+	<!-- \lefthyphenmin=1 \righthyphenmin=2 -->
+	<hyphen-min before="2" after="2"/>
+	<message>Serbian hyphenation patterns in Latin script</message>
+  </language>
+  <!-- serbianc -->
+  <language fop-code="sr" code="sr-cyrl" name="serbianc" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="true">
+	<filename-old-patterns>srhyphc.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Serbian hyphenation patterns in Cyrillic script</message>
+  </language>
+  <!-- mongolian	xu-mnhyph.tex -->
+  <language fop-code="mn" code="mn-cyrl" name="mongolian" use-new-loader="true" use-old-patterns="false" encoding="lmc" exceptions="false">
+	<filename-old-patterns>mnhyph.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>Mongolian hyphenation patterns</message>
+  </language>
+  <!-- mongolian2a -->
+  <language fop-code="mn_Cyrl_x_2a" code="mn-cyrl-x-2a" name="mongolian2a" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false">
+	<filename-old-patterns>mnhyphn.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<message>(New) Mongolian Hyphenation Patterns</message>
+  </language>
+  <!-- bulgarian	xu-bghyphen.tex -->
+  <language fop-code="bg" code="bg" name="bulgarian" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false">
+	<filename-old-patterns>bghyphen.tex</filename-old-patterns>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Bulgarian Hyphenation Patterns</message>
+  </language>
+  <!-- sanskrit -->
+  <language fop-code="sa" code="sa" name="sanskrit" use-new-loader="true" use-old-patterns="false" encoding="" exceptions="false">
+	<hyphen-min before="1" after="5"/>
+	<!-- polyglosia -->
+	<!-- no patterns for 8-bit TeX -->
+	<message>Sanskrit Hyphenation Patterns (v0.2, 2008/1/3)</message>
+  </language>
+  <!-- norsk	xu-nohyphbx.tex -->
+  <language code="nb" name="bokmal" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<synonym>norwegian</synonym>
+	<synonym>norsk</synonym>
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Norwegian Bokmal Hyphenation Patterns</message>
+  </language>
+  <!-- nynorsk	nnhyph.tex -->
+  <language code="nn" name="nynorsk" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true">
+	<hyphen-min before="2" after="2"/>
+	<!-- babel -->
+	<message>Norwegian Nynorsk Hyphenation Patterns</message>
+  </language>
+  <!-- The following languages are not (yet) in languages.rb. -->
+  <!-- I added them after this file was generated from languages.rb. -->
+  <language fop-code="en" code="en-us" name="US english">
+	<hyphen-min before="2" after="3"/>
+	<filename-old-patterns>ushyphmax.tex</filename-old-patterns>
+  </language>
+  <language fop-code="sh_Latn" code="sh-latn" name="serbocroatian">
+	<hyphen-min before="1" after="2"/>
+ </language>
+  <language fop-code="sh_Cyrl" code="sh-cyrl" name="serbocroatianc">
+	<hyphen-min before="1" after="2"/>
+  </language>
+  <language fop-code="no" code="no" name="norwegian">
+	<filename-old-patterns>nohyphbx.tex</filename-old-patterns>
+  </language>
+</languages>
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-offo.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-offo.rb
new file mode 100755
index 00000000000..f164c421aeb
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-offo.rb
@@ -0,0 +1,96 @@
+#!/usr/bin/env ruby
+
+# this file generates FOP XML Hyphenation Patterns
+
+# use 'gem install unicode' if unicode is missing on your computer
+# require 'jcode'
+# require 'rubygems'
+# require 'unicode'
+
+load 'languages.rb'
+
+$path_OFFO="../../../../collaboration/offo"
+
+$l = Languages.new
+# TODO: should be singleton
+languages = $l.list.sort{|a,b| a.name <=> b.name}
+
+# TODO: we should rewrite this
+# not using: eo, el
+# todo: mn, no!, sa, sh
+# codes = ['bg', 'ca', 'cs', 'cy', 'da', 'de-1901', 'de-1996', 'de-ch-1901', 'en-gb', 'en-us', 'es', 'et', 'eu', 'fi', 'fr', 'ga', 'gl', 'hr', 'hsb', 'hu', 'ia', 'id', 'is', 'it', 'kmr', 'la', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr-cyrl', 'sv', 'tr', 'uk']
+
+language_codes = Hash.new
+languages.each do |language|
+	language_codes[language.code] = language.code
+end
+language_codes['de-1901']      = 'de_1901'
+language_codes['de-1996']      = 'de'
+language_codes['de-ch-1901']   = 'de_CH'
+language_codes['en-gb']        = 'en_GB'
+language_codes['en-us']        = 'en_US'
+language_codes['zh-latn']      = 'zh_Latn'
+language_codes['el-monoton']   = 'el'
+language_codes['el-polyton']   = 'el_polyton'
+language_codes['mn-cyrl']      = 'mn_Cyrl'
+language_codes['mn-cyrl-x-2a'] = 'mn'
+language_codes['sh-latn']      = 'sr_Latn'
+language_codes['sh-cyrl']      = nil
+language_codes['sr-cyrl']      = 'sr_Cyrl'
+
+languages.each do |language|
+	include_language = language.use_new_loader
+	code = language_codes[language.code]
+	if code == nil
+		include_language = false
+	end
+	if code == 'en_US'
+		include_language = true
+	end
+
+	if include_language
+		puts "generating #{code}"
+	
+		$file_offo_pattern = File.open("#{$path_OFFO}/#{code}.xml", 'w')
+
+		$file_offo_pattern.puts '<?xml version="1.0" encoding="utf-8"?>'
+		$file_offo_pattern.puts '<hyphenation-info>'
+		$file_offo_pattern.puts
+
+		# lefthyphenmin/righthyphenmin
+		if language.hyphenmin == nil or language.hyphenmin.length == 0 then
+			lmin = ''
+			rmin = ''
+		elsif language.filename_old_patterns == "zerohyph.tex" then
+			lmin = ''
+			rmin = ''
+		else
+			lmin = language.hyphenmin[0]
+			rmin = language.hyphenmin[1]
+		end
+		patterns   = language.get_patterns
+		exceptions = language.get_exceptions
+
+		if code == 'nn' or code == 'nb'
+			patterns = ""
+			patterns = $l['no'].get_patterns
+		end
+
+		$file_offo_pattern.puts "<hyphen-min before=\"#{lmin}\" after=\"#{rmin}\"/>"
+		$file_offo_pattern.puts
+		$file_offo_pattern.puts '<exceptions>'
+		if exceptions != ""
+			$file_offo_pattern.puts exceptions
+		end
+		$file_offo_pattern.puts '</exceptions>'
+		$file_offo_pattern.puts
+		$file_offo_pattern.puts '<patterns>'
+		patterns.each do |pattern|
+			$file_offo_pattern.puts pattern.gsub(/'/,"’")
+		end
+		$file_offo_pattern.puts '</patterns>'
+		$file_offo_pattern.puts '</hyphenation-info>'
+
+		$file_offo_pattern.close
+	end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
index 3fcbe9e1f9d..8a1b1044532 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb
@@ -59,7 +59,7 @@ languages.each do |language|
 			if language.code == 'it' or language.code == 'fr' or language.code == 'uk' or language.code == 'la' or language.code == 'zh-latn' then
 				file.puts("\\lccode`\\'=`\\'")
 			end
-			if language.code == 'pt' then
+			if language.code == 'pt' or language.code == 'tk' then
 				file.puts("\\lccode`\\-=`\\-")
 			end
 			
@@ -67,15 +67,28 @@ languages.each do |language|
 			#
 			# some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines
 			# TODO - maybe consider doing something similar for ibycus
-			if language.code == 'sa' then
+			if language.code == 'sa' or
+				 language.code == 'as' or
+				 language.code == 'bn' or
+				 language.code == 'gu' or
+				 language.code == 'hi' or
+				 language.code == 'kn' or
+				 language.code == 'ml' or
+				 language.code == 'mr' or
+				 language.code == 'or' or
+				 language.code == 'pa' or
+				 language.code == 'ta' or
+				 language.code == 'te' then
 				file.puts(text_if_native_utf)
 				file.puts("    \\message{UTF-8 #{language.message}}")
 				file.puts('    % Set \lccode for ZWNJ and ZWJ.')
 				file.puts('    \lccode"200C="200C')
 				file.puts('    \lccode"200D="200D')
-				file.puts('    % Set \lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.')
+				if language.code == 'sa'
+					file.puts('    % Set \lccode for KANNADA SIGN JIHVAMULIYA and KANNADA SIGN UPADHMANIYA.')
 				file.puts('    \lccode"0CF1="0CF1')
 				file.puts('    \lccode"0CF2="0CF2')
+			end
 				file.puts("    \\input hyph-#{language.code}.tex")
 				file.puts('\else')
 				file.puts("    \\message{No #{language.message} - only available with Unicode engines}")
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
new file mode 100755
index 00000000000..59bb491f8a0
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-plain-patterns.rb
@@ -0,0 +1,97 @@
+#!/usr/bin/env ruby
+
+# this file generates plain patterns (one-per-line) out of TeX source
+
+# use 'gem install unicode' if unicode is missing on your computer
+require 'jcode'
+require 'rubygems'
+require 'unicode'
+
+load 'languages.rb'
+
+$path_plain="../../../../plain"
+
+$l = Languages.new
+# TODO: should be singleton
+languages = $l.list.sort{|a,b| a.name <=> b.name}
+
+# TODO: we should rewrite this
+# not using: eo, el
+# todo: mn, no!, sa, sh
+# codes = ['bg', 'ca', 'cs', 'cy', 'da', 'de-1901', 'de-1996', 'de-ch-1901', 'en-gb', 'en-us', 'es', 'et', 'eu', 'fi', 'fr', 'ga', 'gl', 'hr', 'hsb', 'hu', 'ia', 'id', 'is', 'it', 'kmr', 'la', 'lt', 'lv', 'nl', 'no', 'pl', 'pt', 'ro', 'ru', 'sk', 'sl', 'sr-cyrl', 'sv', 'tr', 'uk']
+
+language_codes = Hash.new
+languages.each do |language|
+	language_codes[language.code] = language.code
+end
+# language_codes['de-1901']      = 'de-1901'
+# language_codes['de-1996']      = 'de-1996'
+language_codes['de-ch-1901']   = 'de-CH-1901'
+language_codes['en-gb']        = 'en-GB'
+language_codes['en-us']        = 'en-US'
+language_codes['zh-latn']      = 'zh-Latn'
+# language_codes['el-monoton']   = 'el-monoton'
+# language_codes['el-polyton']   = 'el-polyton'
+language_codes['mn-cyrl']      = nil
+language_codes['mn-cyrl-x-2a'] = 'mn'
+language_codes['sh-latn']      = 'sr-Latn'
+language_codes['sh-cyrl']      = nil
+language_codes['sr-cyrl']      = 'sr-Cyrl'
+
+languages.sort{|x,y| x.code <=> y.code }.each do |language|
+	if language.use_new_loader or language.code == 'en-us' then
+		include_language = true
+	else
+		include_language = false
+		puts "(skipping #{language.code})"
+	end
+
+	code = language_codes[language.code]
+	if code == nil
+		include_language = false
+	end
+	if code == 'en_US'
+		include_language = true
+	end
+
+	if include_language
+		puts "generating #{code}"
+	
+		$file_pat = File.open("#{$path_plain}/#{code}.pat.txt", 'w')
+		$file_hyp = File.open("#{$path_plain}/#{code}.hyp.txt", 'w')
+		$file_let = File.open("#{$path_plain}/#{code}.chr.txt", 'w')
+		$file_inf = File.open("#{$path_plain}/#{code}.lic.txt", 'w')
+
+		patterns   = language.get_patterns
+		exceptions = language.get_exceptions
+
+		if code == 'nn' or code == 'nb'
+			patterns = ""
+			patterns = $l['no'].get_patterns
+		end
+
+		characters_indexes = patterns.join('').gsub(/[.0-9]/,'').unpack('U*').sort.uniq
+
+		# patterns
+		patterns.each do |pattern|
+			$file_pat.puts pattern.gsub(/'/,"’")
+		end
+		# exceptions
+		if exceptions != ""
+			$file_hyp.puts exceptions
+		end
+		# letters
+		characters_indexes.each do |c|
+			ch = [c].pack('U')
+			$file_let.puts ch + Unicode.upcase(ch)
+		end
+		# licence and readme
+		$file_inf.puts "#{language.message}\n\n(more info about the licence to be added later)\n\n"
+		$file_inf.puts language.get_comments_and_licence
+
+		$file_pat.close
+		$file_hyp.close
+		$file_let.close
+		$file_inf.close
+	end
+end
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
index 3a8365f2fa5..233fb60bbcb 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb
@@ -24,9 +24,10 @@ language_grouping = {
 	'greek' => ['el-monoton', 'el-polyton'],
 	'ancientgreek' => ['grc', 'grc-x-ibycus'],
 	'chinese' => ['zh-latn'],
-	# TODO - until someone tells what to do
+	'indic' => ['as', 'bn', 'gu', 'hi', 'kn', 'ml', 'mr', 'or', 'pa', 'ta', 'te'],
+	# TODO - until someone tells what to do; but this is wrong anyway
 	#'serbian' => ['sr-latn', 'sr-cyrl'],
-	'serbian' => ['sr-latn'],
+	'serbian' => ['sh-latn'],
 }
 
 language_used_in_group = Hash.new
@@ -41,12 +42,17 @@ language_groups = Hash.new
 # single languages first
 languages.each do |language|
 	# temporary remove cyrilic serbian until someone explains what is needed
-	if language.code == 'sr-cyrl' then
-		languages.delete(language)
+	if language.code == 'sr-cyrl' or language.code == 'en-us' then
+		# ignore the language
 	elsif language_used_in_group[language.code] == nil then
 		language_groups[language.name] = [language]
 	end
+
+	if language.code == 'sh-latn' then
+		language.code = 'sr-latn'
+	end
 end
+
 # then groups of languages
 language_grouping.each do |name,group|
 	language_groups[name] = []
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb
index 0f5c202b379..aa3dec8d0f2 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb
@@ -23,7 +23,7 @@ language_grouping = {
 	'chinese' => ['zh-latn'],
 	# TODO - until someone tells what to do
 	#'serbian' => ['sr-latn', 'sr-cyrl'],
-	'serbian' => ['sr-latn'],
+	'serbian' => ['sh-latn'],
 }
 
 language_used_in_group = Hash.new
@@ -40,6 +40,8 @@ languages.each do |language|
 	# temporary remove cyrilic serbian until someone explains what is needed
 	if language.code == 'sr-cyrl' then
 		languages.delete(language)
+	elsif language.code == 'sh-latn' then
+		language.code = 'sr-latn'
 	elsif language_used_in_group[language.code] == nil then
 		language_groups[language.name] = [language]
 	end
@@ -108,42 +110,5 @@ language_groups.sort.each do |language_name,language_list|
 		puts "\t<td>#{encoding}</td>"
 		puts "</tr>\n"
 	end
-	# if language_name != "russian" and language_name != "ukrainian" then
-	# 	language_list.each do |language|
-	# 		if language.use_old_patterns and language.filename_old_patterns != "zerohyph.tex" then
-	# 			$file_tlpsrc.puts "runpattern f texmf/tex/generic/hyphen/#{language.filename_old_patterns}"
-	# 		end
-	# 	end
-	# end
-	# if language_name == "greek" then
-	# 	$file_tlpsrc.puts "docpattern d texmf/doc/generic/elhyphen"
-	# elsif language_name == "hungarian" then
-	# 	$file_tlpsrc.puts "docpattern d texmf/doc/generic/huhyphen"
-	# elsif language_name == "german" then
-	# 	$file_tlpsrc.puts "runpattern f texmf/tex/generic/hyphen/dehyphtex.tex"
-	# 	$file_tlpsrc.puts "runpattern f texmf/tex/generic/hyphen/ghyphen.README"
-	# end
-	# $file_tlpsrc.close
 end
 
-#--------------#
-# language.dat #
-#--------------#
-$file_language_dat = File.open("#{$path_language_dat}/language.dat", "w")
-language_groups.sort.each do |language_name,language_list|
-	language_list.each do |language|
-		if language.use_new_loader then
-			$file_language_dat.puts "#{language.name}\tloadhyph-#{language.code}.tex"
-		else
-			$file_language_dat.puts "#{language.name}\t#{language.filename_old_patterns}"
-		end
-
-		# synonyms
-		if language.synonyms != nil then
-			language.synonyms.each do |synonym|
-				$file_language_dat.puts "=#{synonym}"
-			end
-		end
-	end
-end
-$file_language_dat.close
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
index 685255c3b0f..c9e3c5c463f 100644
--- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb
@@ -16,8 +16,80 @@ class Language
 		
 		if @synonyms==nil then @synonyms = [] end
 	end
-	
+
+	# TODO: simplify this (reduce duplication)
+
+	def get_exceptions
+		if @exceptions1 == nil
+			filename = "../../../tex/generic/hyph-utf8/patterns/hyph-#{@code}.tex";
+			lines = IO.readlines(filename, '.').join("")
+			exceptions = lines.gsub(/%.*/,'');
+			if (exceptions.index('\hyphenation') != nil)
+				@exceptions1 = exceptions.gsub(/.*\\hyphenation\s*\{(.*?)\}.*/m,'\1').
+					gsub(/\s+/m,"\n").
+					gsub(/^\s*/m,'').
+					gsub(/\s*$/m,'').
+					split("\n")
+			else
+				@exceptions1 = ""
+			end
+		end
+
+		return @exceptions1
+	end
+
+	def get_patterns
+		if @patterns == nil
+			filename = "../../../tex/generic/hyph-utf8/patterns/hyph-#{@code}.tex";
+			lines = IO.readlines(filename, '.').join("")
+			@patterns = lines.gsub(/%.*/,'').
+				gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1').
+				gsub(/\s+/m,"\n").
+				gsub(/^\s*/m,'').
+				gsub(/\s*$/m,'').
+				gsub(/'/,"’").
+				split("\n")
+
+			if @code == 'eo' then
+				@patterns = lines.gsub(/%.*/,'').
+					gsub(/.*\\patterns\s*\{(.*)\}.*/m,'\1').
+					#
+					gsub(/\\adj\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e.').
+					gsub(/\\nom\{(.*?)\}/m,'\1a. \1aj. \1ajn. \1an. \1e. \1o. \1oj. \1ojn. \1on.').
+					gsub(/\\ver\{(.*?)\}/m,'\1as. \1i. \1is. \1os. \1u. \1us.').
+					#
+					gsub(/\s+/m,"\n").
+					gsub(/^\s*/m,'').
+					gsub(/\s*$/m,'').
+					split("\n")
+			end
+		end
+		return @patterns
+	end
+
+	def get_comments_and_licence
+		if @comments_and_licence == nil then
+			filename = "../../../tex/generic/hyph-utf8/patterns/hyph-#{@code}.tex";
+			lines = IO.readlines(filename, '.').join("")
+			@comments_and_licence = lines.
+				gsub(/(.*)\\patterns.*/m,'\1')
+		end
+		return @comments_and_licence
+	end
+
+	# def lc_characters
+	# 	if @lc_characters == nil
+	# 		lc_characters = Hash.new
+	# 		p = self.patterns
+	# 		p.each do |pattern|
+	# 		end
+	# 	end
+	# 	return @lc_characters
+	# end
+
 	attr_reader :use_new_loader, :use_old_patterns, :use_old_patterns_comment, :filename_old_patterns, :code, :name, :synonyms, :hyphenmin, :encoding, :exceptions, :message
+	# this hack is needed for Serbian
+	attr_writer :code
 end
 
 
@@ -36,7 +108,7 @@ authors = {
 	},
 	"claudio_beccari" => {
 		"name" => "Claudio",
-		"email" => "claudio.beccari@polito.it",
+		"email" => "claudio{dot}beccari{at}polito{dot}it",
 	}
 
 }
@@ -637,6 +709,18 @@ class Languages < Hash
 	"exceptions" => false,
 	"message" => "Swedish hyphenation patterns (Jan Michael Rynning, 1994-03-03)",
 },
+# turkmen
+{
+	"code" => "tk",
+	"name" => "turkmen",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"filename_old_patterns" => nil,
+	"hyphenmin" => [1,2],
+	"encoding" => "ec",
+	"exceptions" => false,
+	"message" => "Turkmen Hyphenation Patterns",
+},
 # turkish		xu-tkhyph.tex
 {
 	"code" => "tr",
@@ -662,9 +746,22 @@ class Languages < Hash
 	"exceptions" => true,
 	"message" => "Hyphenation Patterns for British English",
 },
+# US english
+{
+	"code" => "en-us",
+	"name" => "english",
+	"use_new_loader" => false,
+	"use_old_patterns" => false,
+	"filename_old_patterns" => "ushyphmax.tex",
+	"hyphenmin" => [2,3], # confirmed, same as what Knuth says
+	"encoding" => "ascii",
+	"exceptions" => true,
+	"message" => "Hyphenation Patterns for American English",
+},
+# TODO: FIXME!!!
 # serbian		xu-srhyphc.tex
 {
-	"code" => "sr-latn",
+	"code" => "sh-latn",
 	"name" => "serbian",
 	"use_new_loader" => true,
 	"use_old_patterns" => false,
@@ -737,6 +834,17 @@ class Languages < Hash
 	"exceptions" => false,
 	"message" => "Sanskrit Hyphenation Patterns (v0.2, 2008/1/3)",
 },
+# norwegian	nohyph.tex
+{
+	"code" => "no",
+	"name" => "norwegian", # TODO: fixme
+	"use_new_loader" => false,
+	"use_old_patterns" => false,
+	"hyphenmin" => [2,2], # babel
+	"encoding" => "ec",
+	"exceptions" => false,
+	"message" => "Norwegian Hyphenation Patterns",
+},
 # norsk	xu-nohyphbx.tex
 {
 	"code" => "nb",
@@ -759,6 +867,139 @@ class Languages < Hash
 	"exceptions" => true,
 	"message" => "Norwegian Nynorsk Hyphenation Patterns",
 },
+#####
+# assamese
+{
+	"code" => "as",
+	"name" => "assamese",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Assameze Hyphenation Patterns",
+},
+# bengali
+{
+	"code" => "bn",
+	"name" => "bengali",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Bengali Hyphenation Patterns",
+},
+# guajrati
+{
+	"code" => "gu",
+	"name" => "guajrati",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Guajrati Hyphenation Patterns",
+},
+# assamese
+{
+	"code" => "as",
+	"name" => "assamese",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Assameze Hyphenation Patterns",
+},
+# hindi
+{
+	"code" => "hi",
+	"name" => "hindi",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Hindi Hyphenation Patterns",
+},
+# kannada
+{
+	"code" => "kn",
+	"name" => "assamese",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Kannada Hyphenation Patterns",
+},
+# malayalam
+{
+	"code" => "ml",
+	"name" => "malayalam",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Malayalam Hyphenation Patterns",
+},
+# marathi
+{
+	"code" => "mr",
+	"name" => "marathi",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Marathi Hyphenation Patterns",
+},
+# oriya
+{
+	"code" => "or",
+	"name" => "oriya",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Oriya Hyphenation Patterns",
+},
+# panjabi
+{
+	"code" => "pa",
+	"name" => "panjabi",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Panjabi Hyphenation Patterns",
+},
+# tamil
+{
+	"code" => "ta",
+	"name" => "tamil",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Tamil Hyphenation Patterns",
+},
+# telugu
+{
+	"code" => "te",
+	"name" => "telugu",
+	"use_new_loader" => true,
+	"use_old_patterns" => false,
+	"hyphenmin" => [1,1], # TODO
+	"encoding" => nil, # no patterns for 8-bit TeX
+	"exceptions" => false,
+	"message" => "Telugu Hyphenation Patterns",
+},
 		]
 
 		languages.each do |l|
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb
new file mode 100755
index 00000000000..072ea34de52
--- /dev/null
+++ b/Master/texmf-dist/source/generic/hyph-utf8/languages/tk/generate_patterns_tk.rb
@@ -0,0 +1,135 @@
+#!/usr/bin/env ruby
+#
+# This script generates hyphenation patterns for Turkmen
+#
+# This script has been written by Mojca Miklavec <mojca dot miklavec dot lists at gmail dot com>
+
+# open file for writing the patterns
+# $tr = File.new("hyph-tk.tex", "w")
+# in TDS
+$tr = File.new("../../../../../tex/generic/hyph-utf8/patterns/hyph-tk.tex", "w")
+
+# write comments into the file
+def add_comment(str)
+	$tr.puts "% " + str.gsub(/\n/, "\n% ").gsub(/% \n/, "%\n")
+end
+
+# define a class of vowels and consonants
+# vowels are split into so that unnecessary permutations are not generated
+front_vowels = %w{ä e i ö ü}
+back_vowels = %w{a y o u}
+consonants = %w{b ç d f g h j k l m n p r s t w ý z ň ž ş}
+# This is to eliminate impossible combinations
+common_suffix_consonants = %w{b ç d g j k l m n p s t ý z ş}
+
+
+# start the file
+add_comment(
+"Hyphenation patterns for Turkmen (hyph-tk.tex)
+
+Author:  Nazar Annagurban <nazartm at gmail.com>
+License: Public domain
+Version: 0.1
+Date:    16 March 2010
+
+----------------------------------------------------------------------
+
+The file has been auto-generated from generate_patterns_tk.rb
+that is part of hyph-utf8.
+
+For more information about UTF-8 hyphenation patterns for TeX and
+links to this file see
+    http://www.tug.org/tex-hyphen/
+")
+
+# we have the following comment for Basque:
+#
+# Some of the patterns below represent combinations that never
+# happen in Turkmen. Would they happen, they would be hyphenated
+# according to the rules.
+
+$tr.puts '\patterns{'
+add_comment("Some suffixes are added through a hyphen. When hyphenating these words, a hyphen is added before the hyphen so that the line ends with a hyphen and the new line starts with a hyphen.")
+$tr.puts "1-4"
+
+add_comment("Allow hyphen after a vowel if and only if there is a single consonant before next the vowel")
+front_vowels.each do |v1|
+	consonants.each do |c|
+		front_vowels.each do |v2|
+			$tr.puts "#{v1}1#{c}#{v2}"
+		end
+	end
+end
+
+back_vowels.each do |v1|
+	consonants.each do |c|
+		back_vowels.each do |v2|
+			$tr.puts "#{v1}1#{c}#{v2}"
+		end
+	end
+end
+
+add_comment("These combinations occur in words of foreign origin or joined words")
+consonants.each do |c|
+  	$tr.puts "a1#{c}i"
+  	$tr.puts "a1#{c}e"
+	$tr.puts "y1#{c}ä"
+	$tr.puts "y1#{c}i"
+	$tr.puts "y1#{c}e"
+	$tr.puts "o1#{c}i"
+	$tr.puts "o1#{c}e"
+	$tr.puts "u1#{c}i"
+	$tr.puts "u1#{c}e"
+	$tr.puts "i1#{c}a"
+	$tr.puts "i1#{c}o"
+	$tr.puts "e1#{c}a"
+	$tr.puts "e1#{c}o"
+	$tr.puts "ä1#{c}o"
+	$tr.puts "ä1#{c}a"
+	$tr.puts "ö1#{c}a"
+end
+
+add_comment("Allow hyphen between two consonants (if there is only two of them), except when they are at the begining of the word")
+consonants.each do |c1|
+	consonants.each do |c2|
+		$tr.puts "#{c1}1#{c2}"
+		$tr.puts ".#{c1}2#{c2}"
+	end
+end
+
+add_comment("Patterns for triple consonants. There may be additions to this category, as this list is not exhaustive.")
+common_suffix_consonants.each do |c|
+	$tr.puts "ý2t1#{c}"
+	$tr.puts "ý2n1#{c}"
+	$tr.puts "ý2d1#{c}"
+	$tr.puts "r2t1#{c}"
+	$tr.puts "ý2p1#{c}"
+	$tr.puts "l2p1#{c}"
+	$tr.puts "l2t1#{c}"
+	$tr.puts "g2t1#{c}"
+	$tr.puts "n2t1#{c}"
+	$tr.puts "r2k1#{c}"
+	$tr.puts "r2p1#{c}"
+	$tr.puts "k2t1#{c}"
+	$tr.puts "r2h1#{c}"
+	$tr.puts "s2t1#{c}"
+	$tr.puts "l2k1#{c}"
+	$tr.puts "w2p1#{c}"
+	$tr.puts "n2s1#{c}"
+	$tr.puts "r2s1#{c}"
+	$tr.puts "l2m1#{c}"
+end
+
+add_comment("Exceptions and single word occurence patterns for words of foreign origin i.e. Russian")
+$tr.puts "s2k1d"
+$tr.puts "l1s2k"
+$tr.puts "l1s2t"
+$tr.puts "s1t2r"
+$tr.puts "n2g1l"
+$tr.puts "n1g2r"
+$tr.puts "s2k1w"
+
+# end the file
+$tr.puts '}'
+$tr.close
+