diff options
author | Mojca Miklavec <mojca.miklavec@gmail.com> | 2010-06-01 01:39:02 +0000 |
---|---|---|
committer | Mojca Miklavec <mojca.miklavec@gmail.com> | 2010-06-01 01:39:02 +0000 |
commit | 3a5cf8de164c5d0f1fdb01999af33d77f59d526f (patch) | |
tree | 1e1c013154793e9a74852ded1537869e3cfabbb0 /Master/texmf-dist/source | |
parent | 3205d02db781aec158019161797c2a156ceb0b34 (diff) |
update of hyph-utf8: add the forgotten ru and uk extra patterns, remove OT1 support from Latin, updates in luatex sources and docs, removed collaboration files
git-svn-id: svn://tug.org/texlive/trunk@18661 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source')
19 files changed, 284 insertions, 2205 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/README b/Master/texmf-dist/source/generic/hyph-utf8/README index 07ba26b49a4..97f87378761 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/README +++ b/Master/texmf-dist/source/generic/hyph-utf8/README @@ -9,11 +9,6 @@ Auto-generates conversions from UTF-8 to some particular encoding. Needs to be run only in case that a new encoding is added or an old encoding is fixed. -generate-offo.rb -================ -Temporary; another version maintaned by author in Java; this one will be improved or removed. -Will be rewritten to take raw patterns as input with no need to parse. - generate-pattern-loaders.rb =========================== INPUT: @@ -59,3 +54,4 @@ Author: See file, modified by Mojca Miklavec contributed/make-exhyph.pl ========================== Imported script for hack related to breaking compound words. + diff --git a/Master/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb b/Master/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb deleted file mode 100755 index 55a44f67f1b..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/collaboration/generate-js-Hyphenator.rb +++ /dev/null @@ -1,204 +0,0 @@ -#!/usr/bin/env ruby - -# This file generates patterns for Hyphenator.js -# http://code.google.com/p/hyphenator -# -# Collaboration with: -# Mathias Nater, <mathias at mnn.ch> - -load '../languages.rb' -# TODO: should be singleton -languages = Languages.new - -languages["de-CH-1901"] = languages["de-ch-1901"] -languages["en-GB"] = languages["en-gb"] -languages["en-US"] = languages["en-us"] -languages["en-US-x-knuth"] = languages["en-us"] -languages["mn"] = languages["mn-cyrl"] -languages["sr-Cyrl"] = languages["sr-cyrl"] -languages["sr-Latn"] = languages["sh-latn"] -languages["zh-Latn"] = languages["zh-latn"] - - -$path_root=File.expand_path(Dir.getwd + "/../../../..") -$path_plain="#{$path_root}/tex/generic/hyph-utf8/patterns/txt" -$path_js=File.expand_path("#{$path_root}/../collaboration/hyphenator/repo") - -# change to current folder and read all the files in it -Dir.chdir("#{$path_plain}") -files = Dir.glob("*.pat.txt") -# files = Dir.glob("*sl.pat.txt") - -# we need to escape some characters; for a complete list see -# http://www.jslint.com/lint.html -# but at the moment there are only two such characters present anyway -# -# this function encapsulates the string into single quotes and uses -def unescape_string_if_needed(str) - # unsafe characters - see above for complete list - unsafeCharacters = [0x200c, 0x200d] - # let's convert our string into array (to preserve proper unicode numbers) - str_array=str.unpack("U*") - # set this to false until the first replacement takes place - replacement_done = false - - # loop over all unsafe character and try to replace all occurencies - unsafeCharacters.each do |c| - # find the first occurence of that character - i = str_array.index(c) - while i != nil - # replaces a single character with '%uXXXX', where XXXX is hex code of character - # this only works with non-math characters, but it should not happen that any bigger number would occur - str_array[i,1] = sprintf("%%u%4X", c).unpack("U*") - i = str_array.index(c) - replacement_done = true - end - end - - # convert the array back to string - str = str_array.pack("U*") - - if replacement_done - return "unescape('#{str}')" - else - return "'#{str}'" - end -end - -class Pattern - # include Enumerable - - def initialize(pattern) - @pattern = pattern.strip - @pattern_array = @pattern.unpack("U*") - @length = @pattern_array.length - end - - def <=>(anOther) - # if @length == anOther.length - # 0.upto(@length-1) do |i| - # if @pattern_array[i] != anOther.pattern_array[i] - # return @pattern_array[i] <=> anOther.pattern_array[i] - # end - # end - # return 1 <=> 1 - # else - # @length <=> anOther.length - # end - @length <=> anOther.length - end - - def js_pattern - @pattern.gsub(/[.]/, "_") - end - - def to_s - @pattern - end - - def length_of_letters_only - return @pattern.gsub(/[0-9]/,'').unpack("U*").length - end - - # def sort_by_length - attr_reader :pattern, :length, :pattern_array -end - -# TODO: this should be an explicit array of patterns only -class Patterns < Array - def length_of_shortest_and_longest_pattern - # store the minimum and maximum length of pattern - a = [self.first.length_of_letters_only, self.first.length_of_letters_only] - # - # a = [0, 1] - self.each do |pat| - a[0] = [a[0], pat.length_of_letters_only].min - a[1] = [a[1], pat.length_of_letters_only].max - # a.first = [a.first, pat.length_of_letters_only].min - # a.last = [a.last, pat.length_of_letters_only].max - end - return a - end - # TODO: you need to make sure that patterns are sorted according to their length first - def each_length - current_length = 0 - first_pattern_with_some_size = Array.new - - self.each_index do |i| - pattern = self[i] - if pattern.length > current_length - current_length = pattern.length - first_pattern_with_some_size.push(i) - end - end - - first_pattern_with_some_size.each_index do |i| - i_first = first_pattern_with_some_size[i] - i_last = nil - if i < first_pattern_with_some_size.length-1 - i_last = first_pattern_with_some_size[i+1] - else - i_last = self.length - end - i_len = i_last-i_first - - yield self[i_first,i_len] - end - end -end - -files.each do |filename| - code_in = filename.gsub(/hyph-(.*).pat.txt/,'\1') - code_out = code_in.gsub(/-/,"_") - language = languages[code_in] # FIXME - # TODO: handle exceptions - puts - puts "Generating Hyphenator.js support for " + code_in - puts " writing to '#{$path_js}/#{code_out}.js'" - puts - patterns = Patterns.new - File.open(filename,'r') do |f_in| - f_in.each_line do |line| - if line.strip.length > 0 - patterns.push(Pattern.new(line)) - end - end - end - patterns.sort! - # puts patterns - specialChars = patterns.join('').gsub(/[.0-9a-z]/,'').unpack('U*').sort.uniq.pack('U*') - - File.open("#{$path_js}/#{code_out}.js", "w") do |f_out| - # BOM mark - # f_out.puts [239,187,191].pack("ccc") - # f_out.print ["EF","BB","BF"].pack("H2H2H2") - f_out.putc(239) - f_out.putc(187) - f_out.putc(191) - f_out.puts "Hyphenator.languages.#{code_out} = {" - f_out.puts "\tleftmin : #{language.hyphenmin[0]}," - f_out.puts "\trightmin : #{language.hyphenmin[1]}," - lengths = patterns.length_of_shortest_and_longest_pattern - f_out.puts "\tshortestPattern : #{lengths.first}," - f_out.puts "\tlongestPattern : #{lengths.last}," - # TODO: handle Ux201C, Ux201D - # if specialChars.gsub!(/.../, ...) ~= nil - # if specialChars =~ /[]/ - # if has_unsafe_characters(specialChars) - # end - unescape_string_if_needed(specialChars) - f_out.puts "\tspecialChars : #{unescape_string_if_needed(specialChars)}," - f_out.puts "\tpatterns : {" - - # current length of patterns (they are sorted according to their length) - current_length = 0 - pattern_string = "" - i_first = i_last = -1 - patterns.each_length do |pats| - f_out.puts "\t\t#{pats.first.length} : #{unescape_string_if_needed(pats.join(""))}" - end - - f_out.puts "\t}" - f_out.puts "};" - end -end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java deleted file mode 100644 index a4f70616a36..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.java +++ /dev/null @@ -1,108 +0,0 @@ -/* - * Copyright Simon Pepping 2009 - * - * The copyright owner licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* $Id: ConvertLanguageData.java 304 2009-11-26 07:26:55Z spepping $ */ - -package org.tug.texhyphen; - -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.net.URISyntaxException; -import java.net.URL; - -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.sax.SAXTransformerFactory; -import javax.xml.transform.sax.TransformerHandler; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; - -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; - -/** - * Convert language data in ruby format to XML format - */ -public final class ConvertLanguageData { - - /** - * @param languageDataPath - * @throws IOException - * @throws TransformerException - * @throws SAXException - * @throws URISyntaxException - */ - public static void convert(String languageDataPath, boolean useStylesheet) - throws IOException, TransformerException, SAXException, URISyntaxException { - - // input - InputStream inis = new FileInputStream(languageDataPath); - InputSource input = new InputSource(inis); - input.setSystemId(languageDataPath); - input.setEncoding("utf-8"); - XMLReader reader = new LanguageDataParser(); - - // output - String outPath = languageDataPath.replaceFirst("\\.rb$", ".xml"); - Result result = new StreamResult(outPath); - - // transformation - TransformerFactory tf = TransformerFactory.newInstance(); - if (!tf.getFeature(SAXTransformerFactory.FEATURE)) { - throw new TransformerException("TransformerFactory is not a SAXTransformerFactory"); - } - SAXTransformerFactory stf = (SAXTransformerFactory) tf; - TransformerHandler th; - if (useStylesheet) { - URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertLanguageData.xsl"); - File xsltFile = new File(xsltUrl.toURI()); - InputStream xsltStream = new FileInputStream(xsltFile); - Source xsltSource = new StreamSource(xsltStream); - xsltSource.setSystemId(xsltFile.getAbsolutePath()); - th = stf.newTransformerHandler(xsltSource); - } else { - th = stf.newTransformerHandler(); - } - - // pipeline - reader.setContentHandler(th); - reader.setProperty("http://xml.org/sax/properties/lexical-handler", th); - th.setResult(result); - reader.parse(input); - } - - /** - * @param args - * @throws IOException - * @throws TransformerException - * @throws SAXException - * @throws URISyntaxException - */ - public static void main(String[] args) - throws IOException, TransformerException, SAXException, URISyntaxException { - if (args[0].endsWith("--debug")) { - convert(args[1], false); - } else { - convert(args[0], true); - } - } - -} diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl deleted file mode 100644 index 98aa733a0c0..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertLanguageData.xsl +++ /dev/null @@ -1,38 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform" - xmlns:lang="urn:org:tug:texhyphen:languagedata"> - - <xsl:output method="xml" encoding="UTF-8" indent="yes"/> - - <xsl:variable name="language-data" select="document('codemapping.xml')/lang:code-mappings"/> - - <xsl:template match="*|@*|node()"> - <xsl:copy> - <xsl:apply-templates select="*|@*|node()"/> - </xsl:copy> - </xsl:template> - - <xsl:template match="lang:language"> - <xsl:variable name="code" select="$language-data/lang:code-mapping[@code=current()/@code]"/> - <xsl:variable name="fop-code"> - <xsl:choose> - <xsl:when test="$code"> - <xsl:value-of select="$code/@fop-code"/> - </xsl:when> - <xsl:otherwise> - <xsl:value-of select="@code"/> - </xsl:otherwise> - </xsl:choose> - </xsl:variable> - <xsl:copy> - <xsl:if test="string($fop-code)"> - <xsl:attribute name="fop-code"> - <xsl:value-of select="$fop-code"/> - </xsl:attribute> - </xsl:if> - <xsl:apply-templates select="*|@*|node()"/> - </xsl:copy> - </xsl:template> - -</xsl:stylesheet> diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java deleted file mode 100644 index 1ec9cee9474..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.java +++ /dev/null @@ -1,385 +0,0 @@ -/* - * Copyright Simon Pepping 2009 - * - * The copyright owner licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* $Id: ConvertTeXPattern.java 304 2009-11-26 07:26:55Z spepping $ */ - -package org.tug.texhyphen; - -import java.io.File; -import java.io.FileInputStream; -import java.io.FileNotFoundException; -import java.io.FilenameFilter; -import java.io.IOException; -import java.io.InputStream; -import java.net.MalformedURLException; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; - -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.parsers.SAXParser; -import javax.xml.parsers.SAXParserFactory; -import javax.xml.transform.Result; -import javax.xml.transform.Source; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.sax.SAXTransformerFactory; -import javax.xml.transform.sax.TransformerHandler; -import javax.xml.transform.stream.StreamResult; -import javax.xml.transform.stream.StreamSource; - -import org.xml.sax.Attributes; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.XMLReader; -import org.xml.sax.helpers.DefaultHandler; - -/** - * Convert modern UTF8 TeX hyphenation patterns to XML format - */ -public final class ConvertTeXPattern { - - public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet, - boolean useLanguagedata) - throws IOException, TransformerException, SAXException, URISyntaxException, - ParserConfigurationException, CodeMappingException { - checkCodeMapping(); - Collection<String> languages = codeMapping.keySet(); - convert(texPatterns, outfilePath, useStylesheet, languages); - } - - public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet) - throws IOException, TransformerException, SAXException, URISyntaxException, - CodeMappingException { - convert(texPatterns, outfilePath, useStylesheet, null); - } - - /** - * infile outfile - * indir outdir (file protocol only) - * infiles outdir - * file and http protocols allowed - * - * @param texPatternUri - * @param outfilePath - * @param useStylesheet - * @param texcodes filter of requested tex codes; is allowed to be null - * @throws IOException - * @throws TransformerException - * @throws SAXException - * @throws URISyntaxException - * @throws CodeMappingException - */ - public static void convert(String[] texPatterns, String outfilePath, boolean useStylesheet, - Collection<String> texcodes) - throws IOException, TransformerException, SAXException, URISyntaxException, - CodeMappingException { - File outDir = new File(outfilePath); - boolean oneTexcode = (texcodes != null && texcodes.size() == 1); - boolean oneInputfile = (texPatterns.length == 1); - boolean oneFilteredInput = (oneTexcode || oneInputfile); - if (!oneFilteredInput && !outDir.isDirectory()) { - throw new IllegalArgumentException - ("with multiple input files the output path " + outfilePath + " must be a directory"); - } - for (String texPattern : texPatterns) { - URI texPatternUri = makeTexPatternUri(texPattern); - URI[] texPatternUris = makeTexPatternUris(texPatternUri); - oneInputfile = (texPatternUris.length == 1); - oneFilteredInput = (oneTexcode || oneInputfile); - if (!oneFilteredInput && !outDir.isDirectory()) { - throw new IllegalArgumentException - ("with an input directory " + texPattern + " the output path " + outfilePath + " must be a directory"); - } - for (URI t : texPatternUris) { - TransformationData transformationData = makeTransformationData(t, outDir, texcodes); - if (transformationData == null) { - continue; - } - doConvert(t, transformationData, useStylesheet); - } - } - } - - /** - * @param texPattern - * @return - * @throws URISyntaxException - * @throws FileNotFoundException - */ - private static URI makeTexPatternUri(String texPattern) - throws URISyntaxException, FileNotFoundException { - URI texPatternUri; - texPatternUri = new URI(texPattern); - String scheme = texPatternUri.getScheme(); - // see if it is a relative file path - if (scheme == null) { - File f = new File(texPattern); - texPatternUri = new URI("file", null, f.getAbsolutePath(), null, null); - scheme = texPatternUri.getScheme(); - } - if (scheme == null || !(scheme.equals("http") || scheme.equals("file"))) { - throw new FileNotFoundException - ("URI with file or http scheme required for hyphenation pattern file"); - } - return texPatternUri; - } - - /** - * @param outfilePath - * @param outDir - * @param texPatternUri - * @param scheme - * @return - * @throws URISyntaxException - */ - private static URI[] makeTexPatternUris(URI texPatternUri) throws URISyntaxException { - URI[] texPatternUris; - texPatternUris = new URI[] {texPatternUri}; - String scheme = texPatternUri.getScheme(); - if (scheme.equals("file")) { - File dir = new File(texPatternUri); - if (dir.isDirectory()) { - ArrayList<URI> l = new ArrayList<URI>(); - FilenameFilter filter = new FilenameFilter() { - public boolean accept(File dir, String name) { - return name.endsWith(".tex"); - } - }; - for (File f : dir.listFiles(filter)) { - l.add(new URI("file", null, f.getAbsolutePath(), null, null)); - } - texPatternUris = l.toArray(texPatternUris); - } - } - return texPatternUris; - } - - /** - * @param t - * @param outDir - * @param texcodes filter of requested tex codes; is allowed to be null - * @return - * @throws CodeMappingException - */ - private static TransformationData makeTransformationData - (URI t, File outDir, Collection<String> texcodes) throws CodeMappingException { - File outFile; - String path = t.getPath(); - String basename = path.substring(path.lastIndexOf('/') + 1); - String base = basename.substring(0, basename.lastIndexOf('.')); - // xmlCode, texCode - String[] codes = mapCode(base); - // code mapping lists no xmlCode - if (codes[0] == null) { - return null; - } - if (texcodes != null && !texcodes.contains(codes[1])) { - return null; - } - if (!outDir.isDirectory()) { - outFile = outDir; - } else { - outFile = new File(outDir, codes[0] + ".xml"); - } - return new TransformationData(outFile, codes[1]); - } - - private static class TransformationData { - File outFile; - String texCode; - TransformationData(File outFile, String texCode) { - this.outFile = outFile; - this.texCode = texCode; - } - } - - private static class CodeMappingException extends Exception { - public CodeMappingException(Exception e) { - super(e); - } - public CodeMappingException(String m) { - super(m); - } - } - - static Map<String, String> codeMapping; - static CodeMappingException codeMappingException; - static { - try { - codeMapping = readLanguagedata(); - } catch (ParserConfigurationException e) { - codeMappingException = new CodeMappingException(e); - } catch (SAXException e) { - codeMappingException = new CodeMappingException(e); - } catch (IOException e) { - codeMappingException = new CodeMappingException(e); - } - } - - private static String[] mapCode(String texCode) throws CodeMappingException { - checkCodeMapping(); - String hyp = "hyph-"; - String xmlCode = texCode; - if (texCode.startsWith(hyp)) { - texCode = texCode.substring(hyp.length()); - xmlCode = codeMapping.get(texCode); - } - return new String[] {xmlCode, texCode}; - } - - /** - * @throws CodeMappingException - */ - private static void checkCodeMapping() throws CodeMappingException { - if (codeMapping == null) { - if (codeMappingException != null) { - throw codeMappingException; - } else { - throw new CodeMappingException("Failure initializing code mapping"); - } - } - } - - public static Map<String,String> readLanguagedata() - throws ParserConfigurationException, SAXException, IOException { - SAXParserFactory spf = SAXParserFactory.newInstance(); - spf.setNamespaceAware(true); - SAXParser parser = spf.newSAXParser(); - InputStream is = ConvertTeXPattern.class.getResourceAsStream("languages.xml"); - TexcodeReader texcodeReader = new TexcodeReader(); - parser.parse(is, texcodeReader); - return texcodeReader.getTexcodes(); - } - - private static class TexcodeReader extends DefaultHandler { - - private Map<String, String> texcodes = new HashMap<String, String>(); - - /* (non-Javadoc) - * @see org.xml.sax.helpers.DefaultHandler#startElement(java.lang.String, java.lang.String, java.lang.String, org.xml.sax.Attributes) - */ - @Override - public void startElement(String uri, String localName, String qName, - Attributes attributes) throws SAXException { - if (uri.equals(LanguageDataParser.LANG_NAMESPACE) && localName.equals("language")) { - String texcode = attributes.getValue("code"); - String fopcode = attributes.getValue("fop-code"); - if (fopcode != null) { - texcodes.put(texcode, fopcode); - } - } - } - - /** - * @return the texcodes - */ - public Map<String,String> getTexcodes() { - return texcodes; - } - - } - - public static void doConvert(URI texPatternUri, TransformationData outdata, boolean useStylesheet) - throws TransformerException, SAXException, MalformedURLException, IOException, URISyntaxException { - - String scheme = texPatternUri.getScheme(); - InputStream inis = null; - if (scheme.equals("file")) { - File in = new File(texPatternUri); - inis = new FileInputStream(in); - } else if (scheme.equals("http")) { - inis = texPatternUri.toURL().openStream(); - } else { - throw new FileNotFoundException - ("URI with file or http scheme required for hyphenation pattern file"); - } - - InputSource input = new InputSource(inis); - input.setSystemId(texPatternUri.toString()); - input.setEncoding("utf-8"); - XMLReader reader = new TeXPatternParser(); - Result result = new StreamResult(outdata.outFile); - TransformerFactory tf = TransformerFactory.newInstance(); - if (!tf.getFeature(SAXTransformerFactory.FEATURE)) { - throw new TransformerException("TransformerFactory is not a SAXTransformerFactory"); - } - SAXTransformerFactory stf = (SAXTransformerFactory) tf; - TransformerHandler th; - if (useStylesheet) { - URL xsltUrl = ConvertTeXPattern.class.getResource("ConvertTeXPattern.xsl"); - File xsltFile = new File(xsltUrl.toURI()); - InputStream xsltStream = new FileInputStream(xsltFile); - Source xsltSource = new StreamSource(xsltStream); - xsltSource.setSystemId(xsltFile.getAbsolutePath()); - th = stf.newTransformerHandler(xsltSource); - Transformer tr = th.getTransformer(); - tr.setParameter("tex-code", outdata.texCode); - } else { - th = stf.newTransformerHandler(); - } - reader.setContentHandler(th); - reader.setProperty("http://xml.org/sax/properties/lexical-handler", th); - th.setResult(result); - reader.parse(input); - } - - /** - * @param args input URI, output file - * @throws URISyntaxException if the URI is not correct - * @throws IOException if a file is not found, or contains illegal content - * @throws TransformerException - * @throws SAXException - * @throws ParserConfigurationException - * @throws CodeMappingException - */ - public static void main(String[] args) - throws URISyntaxException, IOException, TransformerException, SAXException, - ParserConfigurationException, CodeMappingException { - String prefix = "--"; - int i = 0; - boolean useStylesheet = true; - boolean useLanguagedata = false; - Collection<String> texcodes = null; - while (args[i].startsWith(prefix)) { - String option = args[i].substring(prefix.length()); - if (option.equals("debug")) { - useStylesheet = false; - } else if (option.equals("uselanguagedata") || option.equals("langdata")) { - useLanguagedata = true; - } else if (option.equals("texcodes")) { - texcodes = Arrays.asList(args[++i].split(",")); - } else { - throw new IllegalArgumentException("Unknown option: " + option); - } - ++i; - } - if (texcodes != null) { - convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1], - useStylesheet, texcodes); - } else { - convert(Arrays.copyOfRange(args, i, args.length - 1), args[args.length - 1], - useStylesheet, useLanguagedata); - } - } - -} diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl deleted file mode 100644 index 803962b7d5a..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/ConvertTeXPattern.xsl +++ /dev/null @@ -1,128 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<xsl:stylesheet version="1.0" - xmlns:xsl="http://www.w3.org/1999/XSL/Transform" - xmlns:tex="urn:org:tug:texhyphen" - xmlns:lang="urn:org:tug:texhyphen:languagedata" - exclude-result-prefixes="tex lang"> - - <xsl:output doctype-system="hyphenation.dtd" indent="yes"/> - - <xsl:param name="comment-length" select="72"/> - <xsl:param name="tex-code"/> - <xsl:param name="hyphen-min-before-default" select="2"/> - <xsl:param name="hyphen-min-after-default" select="3"/> - - <xsl:template match="/tex:tex"> - <hyphenation-info> - <xsl:choose> - <xsl:when test="tex:hyphenation"> - <!-- (comment*), (patterns), (comment*, hyphenation, comment*) => (comment*), hyphen-min, (comment*, exceptions, comment*), (patterns) --> - <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/> - <xsl:variable name="set2" select="tex:patterns"/> - <xsl:variable name="set3" select="node()[preceding-sibling::tex:patterns]"/> - <xsl:apply-templates select="$set1"/> - <xsl:call-template name="hyphen-min"/> - <xsl:apply-templates select="$set3"/> - <xsl:apply-templates select="$set2"/> - </xsl:when> - <xsl:otherwise> - <!-- (comment*), (patterns, comment*) => (comment*), hyphen-min, (patterns, comment*) --> - <xsl:variable name="set1" select="node()[following-sibling::tex:patterns]"/> - <xsl:variable name="set2" select="node()[preceding-sibling::tex:patterns or self::tex:patterns]"/> - <xsl:apply-templates select="$set1"/> - <xsl:call-template name="hyphen-min"/> - <xsl:apply-templates select="$set2"/> - </xsl:otherwise> - </xsl:choose> - </hyphenation-info> - </xsl:template> - - <xsl:template match="tex:patterns"> - <patterns> - <xsl:apply-templates /> - </patterns> - </xsl:template> - - <xsl:template match="tex:patterns" mode="call-hyphen-min"> - <xsl:call-template name="hyphen-min"/> - <patterns> - <xsl:apply-templates /> - </patterns> - </xsl:template> - - <xsl:template name="hyphen-min"> - <xsl:variable name="hyphen-min" - select="document('languages.xml')/lang:languages/lang:language[@code=$tex-code]/lang:hyphen-min" /> - <xsl:if test="count($hyphen-min)"> - <hyphen-min before="{$hyphen-min/@before}" after="{$hyphen-min/@after}" /> - </xsl:if> - </xsl:template> - - <xsl:template match="tex:message"/> - - <xsl:template match="tex:hyphenation"> - <exceptions> - <xsl:apply-templates /> - </exceptions> - </xsl:template> - - <!-- Comments in TeX contain the trailing new line. --> - <!-- Here we keep the trailing new line if the comment is immediately --> - <!-- preceded or followed by a text node. --> - <!-- Otherwise we strip the new line and pad the comment --> - <!-- to the parameter comment-length. --> - <!-- The XSLT engine uses the same criteria to decide if the comment --> - <!-- should start on a new line or not. --> - <!-- This is not quite correct, because we risk adding spaces to --> - <!-- elements with mixed content. --> - <!-- The following test would be more appropriate: --> - <!-- test="preceding-sibling::text() or following-sibling::text()". --> - <xsl:template match="comment()"> - <xsl:choose> - <xsl:when - test="preceding-sibling::node()[1][self::text()] - or following-sibling::node()[1][self::text()]"> - <xsl:comment> - <xsl:value-of select="." /> - </xsl:comment> - </xsl:when> - <xsl:otherwise> - <xsl:variable name="length" select="string-length(.)" /> - <xsl:comment> - <xsl:value-of select="substring(.,1,$length - 1)" /> - <xsl:text> </xsl:text> - <xsl:call-template name="make-spaces"> - <xsl:with-param name="length" - select="$comment-length - ($length - 1)" /> - </xsl:call-template> - </xsl:comment> - </xsl:otherwise> - </xsl:choose> - </xsl:template> - - <xsl:template name="make-spaces"> - <xsl:param name="length" select="0" /> - <xsl:choose> - <xsl:when test="$length >= 10"> - <xsl:text> </xsl:text> - <xsl:call-template name="make-spaces"> - <xsl:with-param name="length" select="$length - 10" /> - </xsl:call-template> - </xsl:when> - <xsl:when test="$length >= 5"> - <xsl:text> </xsl:text> - <xsl:call-template name="make-spaces"> - <xsl:with-param name="length" select="$length - 5" /> - </xsl:call-template> - </xsl:when> - <xsl:when test="$length >= 1"> - <xsl:text> </xsl:text> - <xsl:call-template name="make-spaces"> - <xsl:with-param name="length" select="$length - 1" /> - </xsl:call-template> - </xsl:when> - </xsl:choose> - <xsl:text></xsl:text> - </xsl:template> - -</xsl:stylesheet>
\ No newline at end of file diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java deleted file mode 100644 index 95c484b25ab..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/LanguageDataParser.java +++ /dev/null @@ -1,428 +0,0 @@ -/* - * Copyright Simon Pepping 2009 - * - * The copyright owner licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* $Id: LanguageDataParser.java 304 2009-11-26 07:26:55Z spepping $ */ - -package org.tug.texhyphen; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLConnection; -import java.util.Collection; -import java.util.Iterator; -import java.util.List; -import java.util.Vector; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.xml.sax.Attributes; -import org.xml.sax.ContentHandler; -import org.xml.sax.DTDHandler; -import org.xml.sax.EntityResolver; -import org.xml.sax.ErrorHandler; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.SAXNotRecognizedException; -import org.xml.sax.SAXNotSupportedException; -import org.xml.sax.XMLReader; -import org.xml.sax.ext.LexicalHandler; -import org.xml.sax.helpers.AttributesImpl; - -/** - * The class TeXParser parses TeX hyphenation pattern files and produces SAX events - */ -public class LanguageDataParser implements XMLReader { - - public static final String LANG_NAMESPACE = "urn:org:tug:texhyphen:languagedata"; - public static int lineLength = 72; - private static final int TOP_LEVEL = 3, IN_LANG = 4; - private static final Pattern - comment = Pattern.compile("#.*"), - langStart = Pattern.compile("{", Pattern.LITERAL), - langEnd = Pattern.compile("}", Pattern.LITERAL), - dataline = Pattern.compile("\"([^\"]+)\" ?=> ?\"([^\"]+)\","), - keywordline = Pattern.compile("\"([^\"]+)\" ?=> ?(false|true|nil),"), - listline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[(\"[^\"]+\"(?:,\"[^\"]+\")*)\\],"), - datalistline = Pattern.compile("\"([^\"]+)\" ?=> ?\\[([^,]+(?:,[^,]+)*)\\],"), - space = Pattern.compile("[ \\t]+"); - private static final AttributesImpl emptyAtts = new AttributesImpl(); - - private ContentHandler contentHandler; - private DTDHandler dtdHandler; - private EntityResolver entityResolver; - private ErrorHandler errorHandler; - private LexicalHandler lexicalHandler; - - private void parseLanguageData(BufferedReader inbr) throws SAXException, IOException { - int parseState = TOP_LEVEL; - Language lang = null; - - contentHandler.startDocument(); - contentHandler.startPrefixMapping("", LANG_NAMESPACE); - contentHandler.startElement(LANG_NAMESPACE, "languages", "languages", emptyAtts); - - for (String line = inbr.readLine(); line != null; line = inbr.readLine()) { - Matcher matcher = comment.matcher(line).useAnchoringBounds(true); - int start = 0; - while (start < line.length()) { - if (matcher.usePattern(comment).lookingAt()) { - processComment(matcher.group(), parseState == TOP_LEVEL ? null : lang); - } else if (matcher.usePattern(space).lookingAt()) { - // do nothing - } else if (parseState == TOP_LEVEL && matcher.usePattern(langStart).lookingAt()) { - parseState = IN_LANG; - lang = new Language(); - } else if ((parseState == IN_LANG) && matcher.usePattern(langEnd).lookingAt()) { - pushoutLanguage(lang); - lang = null; - parseState = TOP_LEVEL; - } else if (parseState == IN_LANG - && (matcher.usePattern(dataline).lookingAt() - || matcher.usePattern(keywordline).lookingAt())) { - String key = matcher.group(1); - String value = matcher.group(2); - processDataline(key, value, lang); - } else if (parseState == IN_LANG - && (matcher.usePattern(listline).lookingAt() - || matcher.usePattern(datalistline).lookingAt())) { - String key = matcher.group(1); - String values = matcher.group(2); - processListline(key, values, lang); - } else { - break; - } - start = matcher.end(); - matcher = matcher.region(start, line.length()).useAnchoringBounds(true); - } - } - - contentHandler.endElement(LANG_NAMESPACE, "languages", "languages"); - contentHandler.endPrefixMapping(LANG_NAMESPACE); - contentHandler.endDocument(); - } - - static Collection<String> attributeKeys; - static { - attributeKeys = new Vector<String>(); - attributeKeys.add("code"); - attributeKeys.add("name"); - attributeKeys.add("use-old-patterns"); - attributeKeys.add("use-new-loader"); - attributeKeys.add("encoding"); - attributeKeys.add("exceptions"); - } - - private void processComment(String comment, Language lang) throws SAXException { - comment = comment.replace("--", "––"); - if (!comment.endsWith(" ")) { - comment = comment + " "; - } - if (lang == null) { - char[] textchars = comment.toCharArray(); - if (lexicalHandler != null) { - lexicalHandler.comment(textchars, 1, textchars.length - 1); - } - } else { - lang.elements.add(new Element("comment", comment)); - } - - } - - private void processDataline(String key, String value, Language lang) { - key = key.replace('_', '-'); - if (value.equals("nil")) { - value = ""; - } - if (attributeKeys.contains(key)) { - lang.atts.addAttribute("", key, key, "CDATA", value); - } else { - lang.elements.add(new Element(key, value)); - } - } - - private void processListline(String key, String valuesString, Language lang) { - key = key.replace('_', '-'); - valuesString = valuesString.replace("\"", ""); - String[] values = valuesString.split(",[ \\t]*"); - if (attributeKeys.contains(key)) { - StringBuilder attValue = new StringBuilder(); - for (String value : values) { - if (!value.equals("nil")) { - attValue.append(" " + value); - } - } - lang.atts.addAttribute("", key, key, "CDATA", attValue.toString()); - } else if (key.equals("hyphenmin")) { - key = "hyphen-min"; - AttributesImpl atts = new AttributesImpl(); - atts.addAttribute("", "before", "before", "CDATA", values[0]); - atts.addAttribute("", "after", "after", "CDATA", values[1]); - lang.elements.add(new Element(key, "", atts)); - } else { - key = key.replaceAll("s$", ""); - for (String value : values) { - if (value.equals("nil")) { - value = ""; - } - lang.elements.add(new Element(key, value)); - } - } - } - - private void pushoutLanguage(Language lang) throws SAXException { - contentHandler.startElement(LANG_NAMESPACE, "language", "language", lang.atts); - Iterator<Element> iter = lang.elements.iterator(); - while (iter.hasNext()) { - Element elt = iter.next(); - char[] text = elt.content.toCharArray(); - if (elt.tag.equals("comment")) { - if (lexicalHandler != null) { - lexicalHandler.comment(text, 1, text.length - 1); - } - } else { - contentHandler.startElement(LANG_NAMESPACE, elt.tag, elt.tag, elt.atts); - contentHandler.characters(text, 0, text.length); - contentHandler.endElement(LANG_NAMESPACE, elt.tag, elt.tag); - } - } - contentHandler.endElement(LANG_NAMESPACE, "language", "language"); - } - - public Reader getReaderFromInputSource(InputSource input) throws IOException { - Reader reader = input.getCharacterStream(); - String encoding = null; - if (reader == null) { - encoding = input.getEncoding(); - } - if (reader == null) { - InputStream stream = input.getByteStream(); - if (stream != null) { - if (encoding == null) { - reader = new InputStreamReader(stream); - } else { - reader = new InputStreamReader(stream, encoding); - } - } - } - if (reader == null) { - String systemId = input.getSystemId(); - reader = getReaderFromSystemId(systemId, encoding); - } - return reader; - } - - public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException { - if (systemId == null) { - throw new IOException("Cannot create a reader from a null systemID"); - } - if (encoding.isEmpty()) { - encoding = null; - } - Reader reader = null; - URI uri = null; - File file = null; - try { - uri = new URI(systemId); - } catch (URISyntaxException e) { - // handled below - } - if (uri == null || !uri.isAbsolute()) { - file = new File(systemId); - } - if (file != null) { - if (encoding == null) { - reader = new FileReader(file); - } else { - InputStream stream = new FileInputStream(file); - reader = new InputStreamReader(stream, encoding); - } - } else if (uri != null && uri.getScheme().equals("http")) { - URL url = uri.toURL(); - URLConnection conn = url.openConnection(); - if (encoding == null) { - encoding = conn.getContentEncoding(); - } - InputStream stream = conn.getInputStream(); - reader = new InputStreamReader(stream, encoding); - } - return reader; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getContentHandler() - */ - public ContentHandler getContentHandler() { - return contentHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getDTDHandler() - */ - public DTDHandler getDTDHandler() { - return dtdHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getEntityResolver() - */ - public EntityResolver getEntityResolver() { - return entityResolver; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getErrorHandler() - */ - public ErrorHandler getErrorHandler() { - return errorHandler; - } - - - /** - * @return the lexicalHandler - */ - public LexicalHandler getLexicalHandler() { - return lexicalHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getFeature(java.lang.String) - */ - public boolean getFeature(String arg0) - throws SAXNotRecognizedException, SAXNotSupportedException { - throw new SAXNotSupportedException(); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getProperty(java.lang.String) - */ - public Object getProperty(String arg0) - throws SAXNotRecognizedException, SAXNotSupportedException { - throw new SAXNotSupportedException(); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource) - */ - public void parse(InputSource input) throws IOException, SAXException { - Reader reader = getReaderFromInputSource(input); - if (reader == null) { - throw new IOException("Could not open input source " + input); - } - BufferedReader inbr = new BufferedReader(reader); - parseLanguageData(inbr); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#parse(java.lang.String) - */ - public void parse(String systemId) throws IOException, SAXException { - Reader reader = getReaderFromSystemId(systemId, null); - if (reader == null) { - throw new IOException("Could not open input systemID " + systemId); - } - BufferedReader inbr = new BufferedReader(reader); - parseLanguageData(inbr); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler) - */ - public void setContentHandler(ContentHandler contenthandler) { - this.contentHandler = contenthandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler) - */ - public void setDTDHandler(DTDHandler dtdhandler) { - this.dtdHandler = dtdhandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver) - */ - public void setEntityResolver(EntityResolver entityresolver) { - this.entityResolver = entityresolver; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler) - */ - public void setErrorHandler(ErrorHandler errorHandler) { - this.errorHandler = errorHandler; - } - - - /** - * @param lexicalHandler the lexicalHandler to set - */ - public void setLexicalHandler(LexicalHandler lexicalHandler) { - this.lexicalHandler = lexicalHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean) - */ - public void setFeature(String arg0, boolean arg1) - throws SAXNotRecognizedException, SAXNotSupportedException { - throw new SAXNotSupportedException(); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object) - */ - public void setProperty(String name, Object value) - throws SAXNotRecognizedException, SAXNotSupportedException { - if (name.equals("http://xml.org/sax/properties/lexical-handler")) { - lexicalHandler = (LexicalHandler) value; - } else { - throw new SAXNotSupportedException(); - } - } - - private static class Element { - String tag; - String content; - Attributes atts; - Element(String tag, String content) { - this(tag, content, LanguageDataParser.emptyAtts); - } - Element(String tag, String content, Attributes atts) { - this.tag = tag; - this.content = content; - this.atts = atts; - } - } - - private static class Language { - AttributesImpl atts; - List<Element> elements; - Language() { - atts = new AttributesImpl(); - elements = new Vector<Element>(); - } - } - -} diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README deleted file mode 100644 index aaf4e074ae6..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/README +++ /dev/null @@ -1,38 +0,0 @@ -This utility consists of two parts. - -Part 1 converts languages.rb to languages.xml. It consists of -ConvertLanguageData.java, LanguageDataParser.java, -ConvertLanguageData.xsl, codemapping.xml. Invocation: - -java org.tug.texhyphen.util.ConvertLanguageData path/to/languages.rb - -output: languages.xml, in the same directory as languages.rb. - -The file codemapping contains the mapping from tex-code to -fop-code. If this file has an entry with non-empty fop-code, it is -inserted in languages.xml as attribute fop-code. If this file has an -entry with empty fop-code, that language gets no attribute -fop-code. Otherwise, that language gets attribute fop-code with the -same value as tex-code (attribute code). - -The current file languages.xml was modified manually after conversion, -to add the three languages at the bottom of the file. - -Part 2 converts tex pattern files to fop pattern files. It consists of -ConvertTeXPattern.java, TeXPatternParser.java, ConvertTeXPattern.xsl, -languages.xml. Invocation: - -java org.tug.texhyphen.util.ConvertTeXPattern [--uselanguagedata] -[--texcodes code1,code2,...] infileuri1 [infileuri2 ...] outfile - -infileuri must be a file or http URI. With a file URI, it may also be -a directory; in that case all pattern files in that directory are -converted. outfile must be a path to an output filename or -directory. If there are more input files, it must be an output -directory. Option 'uselanguagedata' converts all input files which -represent a language in the languages file with a fop-code. Option -'--texcodes code1,code2,...' converts only those input files which -represent a language with one of the given tex codes. Without those -options all input files are converted. In any case, only files are -converted which represent a language in the languages file with a -fop-code. diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java deleted file mode 100644 index 0cc5a542f78..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/TeXPatternParser.java +++ /dev/null @@ -1,325 +0,0 @@ -/* - * Copyright Simon Pepping 2009 - * - * The copyright owner licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* $Id: TeXPatternParser.java 304 2009-11-26 07:26:55Z spepping $ */ - -package org.tug.texhyphen; - -import java.io.BufferedReader; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileReader; -import java.io.IOException; -import java.io.InputStream; -import java.io.InputStreamReader; -import java.io.Reader; -import java.net.URI; -import java.net.URISyntaxException; -import java.net.URL; -import java.net.URLConnection; -import java.util.Stack; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import org.xml.sax.ContentHandler; -import org.xml.sax.DTDHandler; -import org.xml.sax.EntityResolver; -import org.xml.sax.ErrorHandler; -import org.xml.sax.InputSource; -import org.xml.sax.SAXException; -import org.xml.sax.SAXNotRecognizedException; -import org.xml.sax.SAXNotSupportedException; -import org.xml.sax.XMLReader; -import org.xml.sax.ext.LexicalHandler; -import org.xml.sax.helpers.AttributesImpl; - -/** - * The class TeXParser parses TeX hyphenation pattern files and produces SAX events - */ -public class TeXPatternParser implements XMLReader { - - public static final String TEX_NAMESPACE = "urn:org:tug:texhyphen"; - private static final int TOP_LEVEL = 3, IN_COMMAND = 4, AFTER_COMMAND = 5, IN_DATA = 6; - private static final Pattern - comment = Pattern.compile("%.*"), - commandStart = Pattern.compile("\\\\"), - command = Pattern.compile("[a-zA-Z]+"), - space = Pattern.compile(" +"), - argOpen = Pattern.compile("\\{"), - argClose = Pattern.compile("\\}"), - text = Pattern.compile("[^%\\\\\\{\\}]+"); - private static final AttributesImpl emptyAtts = new AttributesImpl(); - - private ContentHandler contentHandler; - private DTDHandler dtdHandler; - private EntityResolver entityResolver; - private ErrorHandler errorHandler; - private LexicalHandler lexicalHandler; - - private void parsePatterns(BufferedReader inbr) throws SAXException, IOException { - int parseState = TOP_LEVEL; - Stack<String> stack = new Stack<String>(); - - contentHandler.startDocument(); - contentHandler.startPrefixMapping("", TEX_NAMESPACE); - contentHandler.startElement(TEX_NAMESPACE, "tex", "tex", emptyAtts); - - for (String line = inbr.readLine(); line != null; line = inbr.readLine()) { - Matcher matcher = comment.matcher(line).useAnchoringBounds(true); - int start = 0; - char[] textchars; - boolean inComment = false; - while (start < line.length()) { - if (matcher.usePattern(comment).lookingAt()) { - String text = matcher.group().replace("--", "––"); - textchars = (text + "\n").toCharArray(); - if (lexicalHandler != null) { - lexicalHandler.comment(textchars, 1, textchars.length - 1); - } - inComment = true; - } else if (parseState == IN_DATA && matcher.usePattern(text).lookingAt()) { - textchars = matcher.group().toCharArray(); - contentHandler.characters(textchars, 0, textchars.length); - } else if (parseState != IN_DATA && matcher.usePattern(space).lookingAt()) { - if (parseState == TOP_LEVEL) { - textchars = matcher.group().toCharArray(); - contentHandler.ignorableWhitespace(textchars, 0, textchars.length); - } - } else if (parseState == TOP_LEVEL && matcher.usePattern(commandStart).lookingAt()) { - parseState = IN_COMMAND; - } else if (parseState == IN_COMMAND && matcher.usePattern(command).lookingAt()) { - String tag = matcher.group(); - contentHandler.startElement(TEX_NAMESPACE, tag, tag, emptyAtts); - stack.push(tag); - parseState = AFTER_COMMAND; - } else if (parseState == AFTER_COMMAND && matcher.usePattern(argOpen).lookingAt()) { - parseState = IN_DATA; - } else if (parseState == IN_DATA && matcher.usePattern(argClose).lookingAt()) { - String tag = stack.pop(); - contentHandler.endElement(TEX_NAMESPACE, tag, tag); - parseState = TOP_LEVEL; - } else { - break; - } - start = matcher.end(); - matcher = matcher.region(start, line.length()).useAnchoringBounds(true); - } - textchars = "\n".toCharArray(); - if (parseState == IN_DATA && !inComment) { - contentHandler.characters(textchars, 0, textchars.length); - } else if (parseState == TOP_LEVEL && !inComment) { - contentHandler.ignorableWhitespace(textchars, 0, textchars.length); - } - } - - contentHandler.endElement(TEX_NAMESPACE, "tex", "tex"); - contentHandler.endPrefixMapping(TEX_NAMESPACE); - contentHandler.endDocument(); - } - - public Reader getReaderFromInputSource(InputSource input) throws IOException { - Reader reader = input.getCharacterStream(); - String encoding = null; - if (reader == null) { - encoding = input.getEncoding(); - } - if (reader == null) { - InputStream stream = input.getByteStream(); - if (stream != null) { - if (encoding == null) { - reader = new InputStreamReader(stream); - } else { - reader = new InputStreamReader(stream, encoding); - } - } - } - if (reader == null) { - String systemId = input.getSystemId(); - reader = getReaderFromSystemId(systemId, encoding); - } - return reader; - } - - public Reader getReaderFromSystemId(String systemId, String encoding) throws IOException { - if (systemId == null) { - throw new IOException("Cannot create a reader from a null systemID"); - } - if (encoding.isEmpty()) { - encoding = null; - } - Reader reader = null; - URI uri = null; - File file = null; - try { - uri = new URI(systemId); - } catch (URISyntaxException e) { - // handled below - } - if (uri == null || !uri.isAbsolute()) { - file = new File(systemId); - } - if (file != null) { - if (encoding == null) { - reader = new FileReader(file); - } else { - InputStream stream = new FileInputStream(file); - reader = new InputStreamReader(stream, encoding); - } - } else if (uri != null && uri.getScheme().equals("http")) { - URL url = uri.toURL(); - URLConnection conn = url.openConnection(); - if (encoding == null) { - encoding = conn.getContentEncoding(); - } - InputStream stream = conn.getInputStream(); - reader = new InputStreamReader(stream, encoding); - } - return reader; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getContentHandler() - */ - public ContentHandler getContentHandler() { - return contentHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getDTDHandler() - */ - public DTDHandler getDTDHandler() { - return dtdHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getEntityResolver() - */ - public EntityResolver getEntityResolver() { - return entityResolver; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getErrorHandler() - */ - public ErrorHandler getErrorHandler() { - return errorHandler; - } - - - /** - * @return the lexicalHandler - */ - public LexicalHandler getLexicalHandler() { - return lexicalHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getFeature(java.lang.String) - */ - public boolean getFeature(String arg0) - throws SAXNotRecognizedException, SAXNotSupportedException { - throw new SAXNotSupportedException(); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#getProperty(java.lang.String) - */ - public Object getProperty(String arg0) - throws SAXNotRecognizedException, SAXNotSupportedException { - throw new SAXNotSupportedException(); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource) - */ - public void parse(InputSource input) throws IOException, SAXException { - Reader reader = getReaderFromInputSource(input); - if (reader == null) { - throw new IOException("Could not open input source " + input); - } - BufferedReader inbr = new BufferedReader(reader); - parsePatterns(inbr); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#parse(java.lang.String) - */ - public void parse(String systemId) throws IOException, SAXException { - Reader reader = getReaderFromSystemId(systemId, null); - if (reader == null) { - throw new IOException("Could not open input systemID " + systemId); - } - BufferedReader inbr = new BufferedReader(reader); - parsePatterns(inbr); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setContentHandler(org.xml.sax.ContentHandler) - */ - public void setContentHandler(ContentHandler contenthandler) { - this.contentHandler = contenthandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setDTDHandler(org.xml.sax.DTDHandler) - */ - public void setDTDHandler(DTDHandler dtdhandler) { - this.dtdHandler = dtdhandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setEntityResolver(org.xml.sax.EntityResolver) - */ - public void setEntityResolver(EntityResolver entityresolver) { - this.entityResolver = entityresolver; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setErrorHandler(org.xml.sax.ErrorHandler) - */ - public void setErrorHandler(ErrorHandler errorHandler) { - this.errorHandler = errorHandler; - } - - - /** - * @param lexicalHandler the lexicalHandler to set - */ - public void setLexicalHandler(LexicalHandler lexicalHandler) { - this.lexicalHandler = lexicalHandler; - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setFeature(java.lang.String, boolean) - */ - public void setFeature(String arg0, boolean arg1) - throws SAXNotRecognizedException, SAXNotSupportedException { - throw new SAXNotSupportedException(); - } - - /* (non-Javadoc) - * @see org.xml.sax.XMLReader#setProperty(java.lang.String, java.lang.Object) - */ - public void setProperty(String name, Object value) - throws SAXNotRecognizedException, SAXNotSupportedException { - if (name.equals("http://xml.org/sax/properties/lexical-handler")) { - lexicalHandler = (LexicalHandler) value; - } else { - throw new SAXNotSupportedException(); - } - } - -} diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml deleted file mode 100644 index c5b64722fa0..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/codemapping.xml +++ /dev/null @@ -1,23 +0,0 @@ -<?xml version="1.0" encoding="utf-8"?> -<code-mappings xmlns="urn:org:tug:texhyphen:languagedata"> - <code-mapping code="grc-x-ibycus" fop-code="grc_X_ibycus"/> - <code-mapping code="zh-latn" fop-code="zh_Latn"/> - <code-mapping code="de-1901" fop-code="de_1901"/> - <code-mapping code="de-1996" fop-code="de"/> - <code-mapping code="de-ch-1901" fop-code="de_CH_1901"/> - <code-mapping code="el-monoton" fop-code="el"/> - <code-mapping code="el-polyton" fop-code="el_Polyton"/> - <code-mapping code="mn-cyrl" fop-code="mn"/> - <code-mapping code="mn-cyrl-x-2a" fop-code="mn_Cyrl_x_2a"/> - <code-mapping code="sh-cyrl" fop-code="sh_Cyrl"/> - <code-mapping code="sh-latn" fop-code="sh_Latn"/> - <code-mapping code="sr-cyrl" fop-code="sr"/> - <code-mapping code="sr-latn" fop-code="sr_Latn"/> - <code-mapping code="en-gb" fop-code="en_GB"/> - <code-mapping code="en-us" fop-code="en"/> - <code-mapping code="eo" fop-code=""/> - <code-mapping code="nn" fop-code=""/> - <code-mapping code="nb" fop-code=""/> - <!-- hu patterns cause a stack overflow when compiled with FOP --> - <code-mapping code="hu" fop-code=""/> -</code-mappings> diff --git a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml b/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml deleted file mode 100644 index 2f0fd637d3c..00000000000 --- a/Master/texmf-dist/source/generic/hyph-utf8/conversion-to-xml/org/tug/texhyphen/languages.xml +++ /dev/null @@ -1,434 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<languages xmlns="urn:org:tug:texhyphen:languagedata"> - <!-- –––––––––––––––––––––––––––––––––––––– --> - <!-- languages with no hyphenation patterns --> - <!-- –––––––––––––––––––––––––––––––––––––– --> - <!-- arabic zerohyph.tex --> - <language fop-code="ar" code="ar" name="arabic" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false"> - <filename-old-patterns>zerohyph.tex</filename-old-patterns> - <!-- "hyphenmin" => [], # not needed --> - <message/> - </language> - <!-- farsi zerohyph.tex --> - <!-- =persian --> - <language fop-code="fa" code="fa" name="farsi" use-new-loader="false" use-old-patterns="false" encoding="" exceptions="false"> - <synonym>persian</synonym> - <filename-old-patterns>zerohyph.tex</filename-old-patterns> - <!-- "hyphenmin" => [], # not needed --> - <message/> - </language> - <!-- ––––––––––––––––––––––––––––––- --> - <!-- special patterns, not converted --> - <!-- ––––––––––––––––––––––––––––––- --> - <!-- ibycus ibyhyph.tex --> - <language fop-code="grc_X_ibycus" code="grc-x-ibycus" name="ibycus" use-new-loader="false" use-old-patterns="true" encoding="" exceptions="false"> - <filename-old-patterns>ibyhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Ancient Greek Hyphenation Patterns for Ibycus encoding (v3.0)</message> - <author>peter_heslin</author> - </language> - <!-- –––––––––––––––––––––––––––– --> - <!-- languages using old patterns --> - <!-- –––––––––––––––––––––––––––– --> - <!-- greek xu-grphyph4.tex --> - <!-- =polygreek --> - <language fop-code="el_Polyton" code="el-polyton" name="greek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true"> - <synonym>polygreek</synonym> - <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment> - <filename-old-patterns>grphyph5.tex</filename-old-patterns> - <!-- left/right hyphen min for Greek can be as low as one (1), --> - <!-- but for aesthetic reasons keep them at 2/2. --> - <!-- Dimitrios Filippou --> - <hyphen-min before="1" after="1"/> - <!-- polyglosia --> - <!-- "message" => "Polytonic Greek Hyphenation Patterns", --> - <message>Hyphenation patterns for multi-accent (polytonic) Modern Greek</message> - <author>dimitrios_filippou</author> - </language> - <!-- monogreek xu-grmhyph4.tex --> - <language fop-code="el" code="el-monoton" name="monogreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="true"> - <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment> - <filename-old-patterns>grmhyph5.tex</filename-old-patterns> - <hyphen-min before="1" after="1"/> - <!-- polyglosia --> - <!-- "message" => "Monotonic Greek Hyphenation Patterns", --> - <message>Hyphenation patterns for uni-accent (monotonic) Modern Greek</message> - <author>dimitrios_filippou</author> - </language> - <!-- ancientgreek xu-grahyph4.tex --> - <language fop-code="grc" code="grc" name="ancientgreek" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false"> - <use-old-patterns-comment>Old patterns work in a different way, one-to-one conversion from UTF-8 is not possible.</use-old-patterns-comment> - <filename-old-patterns>grahyph5.tex</filename-old-patterns> - <hyphen-min before="1" after="1"/> - <!-- polyglosia --> - <message>Hyphenation patterns for Ancient Greek</message> - <author>dimitrios_filippou</author> - </language> - <!-- coptic xu-copthyph.tex --> - <language fop-code="cop" code="cop" name="coptic" use-new-loader="true" use-old-patterns="true" encoding="" exceptions="false"> - <use-old-patterns-comment>TODO: automatic conversion could be done, but was too complicated; leave for later.</use-old-patterns-comment> - <filename-old-patterns>copthyph.tex</filename-old-patterns> - <hyphen-min before="1" after="1"/> - <!-- polyglosia TODO: no documentation found --> - <message>Coptic Hyphenation Patterns</message> - <author>claudio_beccari</author> - </language> - <!-- german xu-dehypht.tex --> - <language fop-code="de_1901" code="de-1901" name="german" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false"> - <use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment> - <filename-old-patterns>dehypht.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>German Hyphenation Patterns (Traditional Orthography)</message> - </language> - <!-- ngerman xu-dehyphn.tex --> - <language fop-code="de" code="de-1996" name="ngerman" use-new-loader="true" use-old-patterns="true" encoding="ec" exceptions="false"> - <use-old-patterns-comment>Kept for the sake of backward compatibility, but newer and better patterns by WL are available.</use-old-patterns-comment> - <filename-old-patterns>dehyphn.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>German Hyphenation Patterns (Reformed Orthography)</message> - </language> - <!-- swissgerman --> - <language fop-code="de_CH_1901" code="de-ch-1901" name="swissgerman" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <!-- TODO: how is it going to be called --> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Swiss-German Hyphenation Patterns (Traditional Orthography)</message> - </language> - <!-- russian xu-ruhyphen.tex --> - <language fop-code="ru" code="ru" name="russian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false"> - <use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment> - <filename-old-patterns>ruhyphen.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Russian Hyphenation Patterns</message> - </language> - <!-- ukrainian xu-ukrhyph.tex --> - <language fop-code="uk" code="uk" name="ukrainian" use-new-loader="true" use-old-patterns="true" encoding="t2a" exceptions="false"> - <use-old-patterns-comment>The old system allows choosing patterns and encodings manually. That mechanism needs to be implemented first in this package, so we still fall back on old system.</use-old-patterns-comment> - <filename-old-patterns>ukrhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Ukrainian Hyphenation Patterns</message> - </language> - <!-- –––––––––––––––––––––––––––– --> - <!-- languages using new patterns --> - <!-- –––––––––––––––––––––––––––– --> - <!-- catalan cahyph.tex --> - <language fop-code="ca" code="ca" name="catalan" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>cahyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Catalan Hyphenation Patterns</message> - </language> - <!-- czech xu-czhyph.tex --> - <language fop-code="cs" code="cs" name="czech" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>czhyph.tex</filename-old-patterns> - <filename-old-patterns-other>czhyphen.tex</filename-old-patterns-other> - <filename-old-patterns-other>czhyphen.ex</filename-old-patterns-other> - <!-- Both Czech and Slovak: \lefthyphenmin=2, \righthyphenmin=3 --> - <!-- Typographical rules allow \righthyphenmin=2 when typesetting in a --> - <!-- narrow column (newspapers etc.). --> - <!-- (used to be 2,2) --> - <hyphen-min before="2" after="3"/> - <message>Czech Hyphenation Patterns (Pavel Sevecek, v3, 1995)</message> - </language> - <!-- slovak xu-skhyph.tex --> - <language fop-code="sk" code="sk" name="slovak" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>skhyph.tex</filename-old-patterns> - <filename-old-patterns-other>skhyphen.tex</filename-old-patterns-other> - <filename-old-patterns-other>skhyphen.ex</filename-old-patterns-other> - <!-- see czech --> - <hyphen-min before="2" after="3"/> - <message>Slovak Hyphenation Patterns (Jana Chlebikova, 1992)</message> - </language> - <!-- welsh cyhyph.tex --> - <language fop-code="cy" code="cy" name="welsh" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>cyhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="3"/> - <message>Welsh Hyphenation Patterns</message> - </language> - <!-- danish dkhyph.tex --> - <language fop-code="da" code="da" name="danish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>dkhyph.tex</filename-old-patterns> - <filename-old-patterns-other>dkcommon.tex</filename-old-patterns-other> - <filename-old-patterns-other>dkspecial.tex</filename-old-patterns-other> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Danish Hyphenation Patterns</message> - </language> - <!-- esperanto xu-eohyph.tex --> - <!-- TODO --> - <language code="eo" name="esperanto" use-new-loader="true" use-old-patterns="false" encoding="il3" exceptions="false"> - <filename-old-patterns>eohyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- TODO --> - <message>Esperanto Hyphenation Patterns</message> - </language> - <!-- spanish xu-eshyph.tex --> - <!-- =espanol --> - <language fop-code="es" code="es" name="spanish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <synonym>espanol</synonym> - <filename-old-patterns>eshyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Spanish Hyphenation Patterns</message> - </language> - <!-- basque xu-bahyph.tex --> - <language fop-code="eu" code="eu" name="basque" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>bahyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Basque Hyphenation Patterns</message> - </language> - <!-- french xu-frhyph.tex --> - <!-- =patois --> - <!-- =francais --> - <language fop-code="fr" code="fr" name="french" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <synonym>patois</synonym> - <synonym>francais</synonym> - <filename-old-patterns>frhyph.tex</filename-old-patterns> - <!-- "hyphenmin" => [], --> - <message>French hyphenation patterns (V2.12, 2002/12/11)</message> - </language> - <!-- galician xu-glhyph.tex --> - <language fop-code="gl" code="gl" name="galician" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>glhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Galician Hyphenation Patterns</message> - </language> - <!-- estonian xu-ethyph.tex --> - <language fop-code="et" code="et" name="estonian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>ethyph.tex</filename-old-patterns> - <hyphen-min before="2" after="3"/> - <!-- babel --> - <message>Estonian Hyphenation Patterns</message> - </language> - <!-- finnish fihyph.tex --> - <language fop-code="fi" code="fi" name="finnish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>fihyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Finnish Hyphenation Patterns</message> - </language> - <!-- croatian xu-hrhyph.tex --> - <language fop-code="hr" code="hr" name="croatian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>hrhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Croatian Hyphenation Patterns</message> - </language> - <!-- hungarian xu-huhyphn.tex --> - <language code="hu" name="hungarian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>huhyphn.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- polyglosia --> - <message>Hungarian Hyphenation Patterns (v20031107)</message> - </language> - <!-- interlingua iahyphen.tex --> - <language fop-code="ia" code="ia" name="interlingua" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true"> - <filename-old-patterns>iahyphen.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Hyphenation Patterns for Interlingua</message> - </language> - <!-- indonesian inhyph.tex --> - <language fop-code="id" code="id" name="indonesian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true"> - <filename-old-patterns>inhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Indonesian Hyphenation Patterns</message> - </language> - <!-- icelandic icehyph.tex --> - <language fop-code="is" code="is" name="icelandic" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>icehyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Icelandic Hyphenation Patterns</message> - </language> - <!-- irish gahyph.tex --> - <language fop-code="ga" code="ga" name="irish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>gahyph.tex</filename-old-patterns> - <hyphen-min before="2" after="3"/> - <!-- babel --> - <message>Irish Hyphenation Patterns</message> - </language> - <!-- italian ithyph.tex --> - <language fop-code="it" code="it" name="italian" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="false"> - <filename-old-patterns>ithyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Italian Hyphenation Patterns</message> - </language> - <!-- kurmanji --> - <language fop-code="kmr" code="kmr" name="kurmanji" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>kmrhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Kurmanji Hyphenation Patterns (v. 1.0 2009/06/29 JKn and MSh)</message> - </language> - <!-- latin xu-lahyph.tex --> - <language fop-code="la" code="la" name="latin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <use-old-patterns-comment>Old patterns support both EC & OT1 encodings at the same time.</use-old-patterns-comment> - <filename-old-patterns>lahyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Latin Hyphenation Patterns</message> - </language> - <!-- lithuanian --> - <language fop-code="lt" code="lt" name="lithuanian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false"> - <hyphen-min before="2" after="2"/> - <message>Lithuanian Hyphenation Patterns</message> - </language> - <!-- latvian --> - <language fop-code="lv" code="lv" name="latvian" use-new-loader="true" use-old-patterns="false" encoding="l7x" exceptions="false"> - <hyphen-min before="2" after="2"/> - <message>Latvian Hyphenation Patterns</message> - </language> - <!-- dutch nehyph96.tex --> - <language fop-code="nl" code="nl" name="dutch" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>nehyph96.tex</filename-old-patterns> - <!-- quoting Hans Hagen: --> - <!-- patterns generated with 2,2 (so don't go less) but use prefered values 2,3 (educational publishers want 4,5 -) --> - <hyphen-min before="2" after="2"/> - <message>Dutch Hyphenation Patterns</message> - </language> - <!-- norsk xu-nohyphbx.tex --> - <!-- =norwegian --> - <!-- nynorsk nnhyph.tex --> - <!-- bokmal nbhyph.tex --> - <!-- polish xu-plhyph.tex --> - <language fop-code="pl" code="pl" name="polish" use-new-loader="true" use-old-patterns="false" encoding="qx" exceptions="true"> - <filename-old-patterns>plhyph.tex</filename-old-patterns> - <!--{}"hyphenmin" => [1,1], --> - <hyphen-min before="2" after="2"/> - <message>Polish Hyphenation Patterns</message> - </language> - <!-- portuguese pthyph.tex --> - <!-- =portuges --> - <language fop-code="pt" code="pt" name="portuguese" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <synonym>portuges</synonym> - <filename-old-patterns>pthyph.tex</filename-old-patterns> - <hyphen-min before="2" after="3"/> - <!-- babel --> - <message>Portuguese Hyphenation Patterns</message> - </language> - <!-- pinyin xu-pyhyph.tex --> - <language fop-code="zh_Latn" code="zh-latn" name="pinyin" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>pyhyph.tex</filename-old-patterns> - <hyphen-min before="1" after="1"/> - <message>Hyphenation patterns for unaccented pinyin syllables (CJK 4.8.0)</message> - </language> - <!-- romanian xu-rohyphen.tex --> - <language fop-code="ro" code="ro" name="romanian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>rohyphen.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Romanian Hyphenation Patterns</message> - <!-- : `rohyphen' 1.1 <29.10.1996> --> - </language> - <!-- slovenian xu-sihyph.tex --> - <!-- =slovene --> - <language fop-code="sl" code="sl" name="slovenian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <synonym>slovene</synonym> - <filename-old-patterns>sihyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Slovenian Hyphenation Patterns</message> - </language> - <!-- uppersorbian xu-sorhyph.tex --> - <language fop-code="hsb" code="hsb" name="uppersorbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>sorhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Upper Sorbian Hyphenation Patterns (E. Werner)</message> - <!-- \message{Hyphenation patterns for Upper Sorbian, E. Werner} --> - <!-- \message{Completely new revision 1997, March 22} --> - </language> - <!-- swedish svhyph.tex --> - <language fop-code="sv" code="sv" name="swedish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>svhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- patters say it could be 1,2; babel says 2,2 - double check --> - <message>Swedish hyphenation patterns (Jan Michael Rynning, 1994-03-03)</message> - </language> - <!-- turkish xu-tkhyph.tex --> - <language fop-code="tr" code="tr" name="turkish" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="false"> - <filename-old-patterns>tkhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- polyglosia --> - <message>Turkish Hyphenation Patterns</message> - </language> - <!-- ukenglish ukhyphen.tex --> - <!-- TODO - should we rename it or not? --> - <language fop-code="en_GB" code="en-gb" name="ukenglish" use-new-loader="true" use-old-patterns="false" encoding="ascii" exceptions="true"> - <synonym>british</synonym> - <synonym>UKenglish</synonym> - <filename-old-patterns>ukhyphen.tex</filename-old-patterns> - <hyphen-min before="2" after="3"/> - <!-- confirmed, same as what Knuth says --> - <message>Hyphenation Patterns for British English</message> - </language> - <!-- serbian xu-srhyphc.tex --> - <language fop-code="sr_Latn" code="sr-latn" name="serbian" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <filename-old-patterns>shhyphl.tex</filename-old-patterns> - <!-- It is allowed to leave one character at the end of the row. --> - <!-- However, if you think that it is graphicaly not very --> - <!-- pleasent these patterns will work well with \lefthyphenmin=2. --> - <!-- \lefthyphenmin=1 \righthyphenmin=2 --> - <hyphen-min before="2" after="2"/> - <message>Serbian hyphenation patterns in Latin script</message> - </language> - <!-- serbianc --> - <language fop-code="sr" code="sr-cyrl" name="serbianc" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="true"> - <filename-old-patterns>srhyphc.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Serbian hyphenation patterns in Cyrillic script</message> - </language> - <!-- mongolian xu-mnhyph.tex --> - <language fop-code="mn" code="mn-cyrl" name="mongolian" use-new-loader="true" use-old-patterns="false" encoding="lmc" exceptions="false"> - <filename-old-patterns>mnhyph.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>Mongolian hyphenation patterns</message> - </language> - <!-- mongolian2a --> - <language fop-code="mn_Cyrl_x_2a" code="mn-cyrl-x-2a" name="mongolian2a" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false"> - <filename-old-patterns>mnhyphn.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <message>(New) Mongolian Hyphenation Patterns</message> - </language> - <!-- bulgarian xu-bghyphen.tex --> - <language fop-code="bg" code="bg" name="bulgarian" use-new-loader="true" use-old-patterns="false" encoding="t2a" exceptions="false"> - <filename-old-patterns>bghyphen.tex</filename-old-patterns> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Bulgarian Hyphenation Patterns</message> - </language> - <!-- sanskrit --> - <language fop-code="sa" code="sa" name="sanskrit" use-new-loader="true" use-old-patterns="false" encoding="" exceptions="false"> - <hyphen-min before="1" after="5"/> - <!-- polyglosia --> - <!-- no patterns for 8-bit TeX --> - <message>Sanskrit Hyphenation Patterns (v0.2, 2008/1/3)</message> - </language> - <!-- norsk xu-nohyphbx.tex --> - <language code="nb" name="bokmal" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <synonym>norwegian</synonym> - <synonym>norsk</synonym> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Norwegian Bokmal Hyphenation Patterns</message> - </language> - <!-- nynorsk nnhyph.tex --> - <language code="nn" name="nynorsk" use-new-loader="true" use-old-patterns="false" encoding="ec" exceptions="true"> - <hyphen-min before="2" after="2"/> - <!-- babel --> - <message>Norwegian Nynorsk Hyphenation Patterns</message> - </language> - <!-- The following languages are not (yet) in languages.rb. --> - <!-- I added them after this file was generated from languages.rb. --> - <language fop-code="en" code="en-us" name="US english"> - <hyphen-min before="2" after="3"/> - <filename-old-patterns>ushyphmax.tex</filename-old-patterns> - </language> - <language fop-code="sh_Latn" code="sh-latn" name="serbocroatian"> - <hyphen-min before="1" after="2"/> - </language> - <language fop-code="sh_Cyrl" code="sh-cyrl" name="serbocroatianc"> - <hyphen-min before="1" after="2"/> - </language> - <language fop-code="no" code="no" name="norwegian"> - <filename-old-patterns>nohyphbx.tex</filename-old-patterns> - </language> -</languages> diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb index bdc920181f9..db699173334 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb @@ -1,56 +1,17 @@ #!/usr/bin/env ruby +require 'hyph-utf8' + $encoding_data_dir = "data/encodings" $encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"] -$output_data_dir = "../../../tex/generic/hyph-utf8/conversions" - -class UnicodeCharacter - def initialize(code_uni, code_enc, name) - @code_uni = code_uni - @code_enc = code_enc - # TODO: might be longer or shorter - @bytes = [code_uni].pack('U').unpack('H2H2') - @name = name - end - - attr_reader :code_uni, :code_enc, :bytes, :name -end - -class UnicodeCharacters < Hash - def add_new_character(code_uni, code_enc, name) - first_byte = [code_uni].pack('U').unpack('H2').first - if self[first_byte] == nil then - self[first_byte] = Array.new - end - self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name)) - end -end +$path_root=File.expand_path("../../..") +$output_data_dir = "#{$path_root}/tex/generic/hyph-utf8/conversions" # 0x19; U+0131; 1; dotlessi $encodings.each do |encoding| - #$utf_combinations = Hash.new - $unicode_characters = UnicodeCharacters.new + e = Encoding.new(encoding) - # those that need lccode to be set - $lowercase_characters = Array.new - - File.open($encoding_data_dir + "/" + encoding + ".dat").grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z\.]*)$/) do |line| - # puts line - code_enc = $1.hex - code_uni = $2.hex - if $3.length > 0 - type = $3.to_i - else - type = 0 - end - name = $4 - if type == 1 then - $unicode_characters.add_new_character(code_uni, code_enc, name) - $lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name)) - end - end - $file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w") $file_out.puts "% conv-utf8-#{encoding}.tex" $file_out.puts "%" @@ -65,7 +26,7 @@ $encodings.each do |encoding| $file_out.puts "% (But consider adapting the scripts if you need modifications.)" $file_out.puts "%" - $unicode_characters.sort.each do |first_byte| + e.unicode_characters_first_byte.sort.each do |first_byte| # sorting all the second characters alphabetically first_byte[1].sort!{|x,y| x.code_uni <=> y.code_uni } # make all the possible first characters active @@ -73,7 +34,7 @@ $encodings.each do |encoding| $file_out.puts "\\catcode\"#{first_byte[0].upcase}=\\active" end $file_out.puts "%" - $unicode_characters.sort.each do |first_byte| + e.unicode_characters_first_byte.sort.each do |first_byte| $file_out.puts "\\def^^#{first_byte[0]}#1{%" string_fi = "" for i in 1..(first_byte[1].size) @@ -92,7 +53,7 @@ $encodings.each do |encoding| $file_out.puts "%" $file_out.puts "% ensure all the chars above have valid \lccode values" $file_out.puts "%" - $lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc }.each do |character| + e.lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc }.each do |character| code = [ character.code_enc ].pack("c").unpack("H2").first.upcase # \lccode"FF="FF ux_code = sprintf("U+%04X", character.code_uni) diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb index 16e32e975e9..872912f94b3 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb @@ -140,7 +140,7 @@ languages.each do |language| file.puts('\else') file.puts(" \\message{#{language.encoding.upcase} #{language.message}}") # a hack for OT1 encoding in three languages - if language.code == 'da' or language.code == 'fr' or language.code == 'la' then + if language.code == 'da' or language.code == 'fr' then file.puts(" % A hack to support both EC and OT1 encoding in 8-bit engines.") file.puts(" % Kept for backward compatibility only, though we would prefer to drop it.") file.puts(" % OT1 encoding is close-to-useless for proper hyphenation.") diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb new file mode 100755 index 00000000000..08a77340d8b --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-ptex-patterns.rb @@ -0,0 +1,108 @@ +#!/usr/bin/env ruby + +require 'hyph-utf8' + +# this file generates patterns for pTeX out of the plain ones + +# use 'gem install unicode' if unicode is missing on your computer +# require 'jcode' +# require 'rubygems' +# require 'unicode' + +load 'languages.rb' + +$path_root=File.expand_path("../../..") +$path_ptex="#{$path_root}/tex/generic/hyph-utf8/patterns/ptex" + +# load encodings +encodings_list = ["ascii", "ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"] +encodings = Hash.new +encodings_list.each do |encoding_name| + encodings[encoding_name] = Encoding.new(encoding_name) +end + +$l = Languages.new + +# TODO: should be singleton +languages = $l.list.sort{|a,b| a.name <=> b.name} + +language_codes = Hash.new +languages.each do |language| + language_codes[language.code] = language.code +end +language_codes['mn-cyrl-x-lmc'] = nil +# language_codes['sh-latn'] = 'sr-latn' +language_codes['sh-cyrl'] = nil + +# e = Encoding.new("ec") +# puts e.convert_string_to_escaped_characters("moja čaša") +# +# return + +languages.sort{|x,y| x.code <=> y.code }.each do |language| + encoding = nil + if language.use_new_loader then + if language.encoding == nil or language_codes[language.code] == nil + include_language = false + puts "(skipping #{language.code} # encoding)" + elsif language.encoding == 'ascii' + include_language = false + puts "(skipping #{language.code} # ascii)" + else + include_language = true + encoding = encodings[language.encoding] + end + else + include_language = false + puts "(skipping #{language.code} # loader)" + end + + code = language_codes[language.code] + + if include_language + puts ">> generating #{code} (#{language.name})" + file_ptex = File.open("#{$path_ptex}/phyph-#{code}.tex", "w") + + patterns = language.get_patterns + exceptions = language.get_exceptions + + if code == 'nn' or code == 'nb' + patterns = $l['no'].get_patterns + end + + if language.encoding != 'ascii' then + patterns = encoding.convert_to_escaped_characters(patterns) + exceptions = encoding.convert_to_escaped_characters(exceptions) + end + + file_ptex.puts("% pTeX-friendly hyphenation patterns") + file_ptex.puts("%") + file_ptex.puts("% language: #{language.name} (#{language.code})") + file_ptex.puts("% encoding: #{language.encoding}") + file_ptex.puts("%") + file_ptex.puts("% This file has been auto-generated from hyph-#{language.code}.tex") + file_ptex.puts("% with a script [texmf]/scripts/generic/hyph-utf8/generate-ptex-patterns.rb") + file_ptex.puts("% See the original file for details about author, licence etc.") + file_ptex.puts("%") + + if patterns.length > 0 then + # file_ptex.puts("\\patterns{\n#{encoding.convert_to_escaped_characters(patterns.join("\n"))}\n}") + file_ptex.puts("\\patterns{\n#{patterns.join("\n")}\n}") + end + if exceptions.length > 0 then + # file_ptex.puts("\\hyphenation{\n#{encoding.convert_to_escaped_characters(exceptions.join("\n"))}\n}") + file_ptex.puts("\\hyphenation{\n#{exceptions.join("\n")}\n}") + end + + + # # patterns + # patterns.each do |pattern| + # $file_pat.puts pattern.gsub(/'/,"’") + # end + # # exceptions + # if exceptions != "" + # $file_hyp.puts exceptions + # end + file_ptex.close + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb index e4aff48abfe..dc1c3245a70 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb @@ -120,8 +120,9 @@ language_groups.sort.each do |language_name,language_list| if language.use_new_loader then file = "file=loadhyph-#{language.code}.tex" # we skip the mongolian language - if language.code != "mn-cyrl-x-lmc" then - + if language.code == "mn-cyrl-x-lmc" then + file = "luaspecial=\"disabled:only for 8bit montex with lmc encoding\"" + else filename_pat = "hyph-#{language.code}.pat.txt" filename_hyp = "hyph-#{language.code}.hyp.txt" @@ -143,6 +144,12 @@ language_groups.sort.each do |language_name,language_list| end else file = "file=#{language.filename_old_patterns}" + if language.code == 'ar' or language.code == 'fa' then + file = file + " \\\n\tfile_patterns=" + elsif language.code == 'grc-x-ibycus' then + # TODO: fix this + file = file + " \\\n\tluaspecial=\"disabled:8-bit only\"" + end end $file_tlpsrc.puts "execute AddHyphen \\\n\t#{name}#{synonyms} \\" diff --git a/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb b/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb new file mode 100644 index 00000000000..e0004a76630 --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/hyph-utf8.rb @@ -0,0 +1,109 @@ +# this is a Unicode character represented in some particular encoding +class UnicodeCharacter + # unicode code + # code in that particular encoding + # character name (like 'eacute') + def initialize(code_uni, code_enc, name) + @code_uni = code_uni + @code_enc = code_enc + # TODO: might be longer or shorter + @bytes = [code_uni].pack('U').unpack('H2H2') + @name = name + end + + attr_reader :code_uni, :code_enc, :bytes, :name +end + +class UnicodeCharacters < Hash + # a hash based on the first character + def add_new_character_first_byte(code_uni, code_enc, name) + first_byte = [code_uni].pack('U').unpack('H2').first + if self[first_byte] == nil then + self[first_byte] = Array.new + end + self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name)) + end + # a hash based on the whole unicode codepoint + def add_new_character(code_uni, code_enc, name) + self[code_uni] = UnicodeCharacter.new(code_uni, code_enc, name) + end +end + +class Encoding + def initialize(encoding_name) + @encoding_name = encoding_name + @unicode_characters_first_byte = UnicodeCharacters.new + @unicode_characters = UnicodeCharacters.new + @lowercase_characters = Array.new + + if encoding_name != 'ascii' then + read_data + end + end + + def convert_to_escaped_characters(str) + if str.kind_of?(Array) then + str.each_index do |i| + str[i] = convert_string_to_escaped_characters(str[i]) + end + elsif str.kind_of?(String) then + str = convert_string_to_escaped_characters(str) + end + return str + end + + attr_reader :encoding_name, :unicode_characters, :unicode_characters_first_byte, :lowercase_characters + + def convert_string_to_escaped_characters(str) + characters = str.unpack('U*') + new_string = Array.new(characters.length) + characters.each_index do |i| + c = characters[i] # character code on position i + # check if unicode entry with that number exists + uc = @unicode_characters[c] + if uc == nil then + if c < 128 then + new_string[i] = [c].pack('U') + elsif c == 8217 # ’ + new_string[i] = "'" + else + puts "There must be an error: character #{c} in string #{str} is not ASCII." + end + # an unicode character + else + new_string[i] = sprintf("^^%x", uc.code_enc) + end + end + return new_string.join('') + end + +private + def read_data + # fetch the characters + encoding_data_dir = File.expand_path("data/encodings") + filename = "#{encoding_data_dir}/#{@encoding_name}.dat" + + if File.exists?(filename) then + File.open(filename).grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z\.]*)$/) do |line| + # puts line + code_enc = $1.hex + code_uni = $2.hex + if $3.length > 0 + type = $3.to_i + else + type = 0 + end + name = $4 + if type == 1 then + @unicode_characters_first_byte.add_new_character_first_byte(code_uni, code_enc, name) + @unicode_characters.add_new_character(code_uni, code_enc, name) + @lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name)) + end + end + else + # TODO: throw an error + puts "Invalid encoding name '#{@encoding_name}'." + puts "File '#{filename}' doesn't exist." + end + end +end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb index 450013b7868..fc524fc3520 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb @@ -40,7 +40,7 @@ class Language def get_patterns if @patterns == nil - filename = "../../../tex/generic/hyph-utf8/patterns/tex/hyph-#{@code}.tex"; + filename = "../../../tex/generic/hyph-utf8/patterns/tex/hyph-#{@code}.tex" lines = IO.readlines(filename, '.').join("") @patterns = lines.gsub(/%.*/,''). gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1'). @@ -49,6 +49,19 @@ class Language gsub(/\s*$/m,''). gsub(/'/,"’"). split("\n") + # Russian and Ukrainian have some extra patterns with dashes + # we may combine these patterns with the main file anyway + if @code == 'ru' or @code == 'uk' then + filename = "../../../tex/generic/hyph-utf8/patterns/tex-special/exhyph-#{@code}.tex" + lines = IO.readlines(filename, '.').join("") + @patterns.concat(lines.gsub(/%.*/,''). + gsub(/.*\\patterns\s*\{(.*?)\}.*/m,'\1'). + gsub(/\s+/m,"\n"). + gsub(/^\s*/m,''). + gsub(/\s*$/m,''). + gsub(/'/,"’"). + split("\n")) + end if @code == 'eo' then @patterns = lines.gsub(/%.*/,''). diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README b/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README index 6154587b019..95e00612c7b 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README +++ b/Master/texmf-dist/source/generic/hyph-utf8/languages/gl/README @@ -1,25 +1,25 @@ -This directory contains seven files + README, licensed all but one under lppl 1.3: - -glpatter-utf8.tex: - The source file, version 2.4. When processed under the program - mkpatter ---by typing 'tex -ini -8bit glpatter-utf8.tex' (without the quotes) - at the command line, provided you have the utility - mkpattern installed--- it generates the file hyph-utf8-gl.tex. - -(hyph-gl.tex): - The generated file containing the patterns in UTF-8 (belongs to TEXMF/tex). - - -glhyextr.tex: ##This file is in the public domain## - A configuration file, to include additional patterns other - than the ones in glpatter-utf8.tex. It is not \input by glpatter-utf8.tex - (iin contrast with the single-byte encoded version glpatter.tex) - as modern tools allow the addition of patterns at run time. - -glhybiox.tex, glhymed.tex, glhyquim.tex, glhytec.tex & glhyxeog.tex: - Files with aditional patterns drawn from serveral fields of - the language, so that you may input them from glhyextr.tex. - -The files for specific patterns are very incomplete and feedback is welcome. - ---Javier A. Mgica +This directory contains seven files + README, licensed all but one under lppl 1.3:
+
+glpatter-utf8.tex:
+ The source file, version 2.4. When processed under the program
+ mkpatter ---by typing 'tex -ini -8bit glpatter-utf8.tex' (without the quotes)
+ at the command line, provided you have the utility
+ mkpattern installed--- it generates the file hyph-utf8-gl.tex.
+
+(hyph-gl.tex):
+ The generated file containing the patterns in UTF-8 (belongs to TEXMF/tex).
+
+
+glhyextr.tex: ##This file is in the public domain##
+ A configuration file, to include additional patterns other
+ than the ones in glpatter-utf8.tex. It is not \input by glpatter-utf8.tex
+ (iin contrast with the single-byte encoded version glpatter.tex)
+ as modern tools allow the addition of patterns at run time.
+
+glhybiox.tex, glhymed.tex, glhyquim.tex, glhytec.tex & glhyxeog.tex:
+ Files with aditional patterns drawn from serveral fields of
+ the language, so that you may input them from glhyextr.tex.
+
+The files for specific patterns are very incomplete and feedback is welcome.
+
+--Javier A. Mgica
diff --git a/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx b/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx index 2eac29d4e5c..69e0aaaa698 100644 --- a/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx +++ b/Master/texmf-dist/source/luatex/hyph-utf8/luatex-hyphen.dtx @@ -106,7 +106,7 @@ This work is under the CC0 license. %<*driver> \NeedsTeXFormat{LaTeX2e} \ProvidesFile{luatex-hyphen.drv} - [2010/04/28 v1.3beta Hyphenation file for LuaTeX] + [2010/04/28 v1.4 Hyphenation file for LuaTeX] \documentclass{ltxdoc} \usepackage[ascii]{inputenc} \usepackage[T1]{fontenc} @@ -140,7 +140,7 @@ This work is under the CC0 license. % \GetFileInfo{luatex-hyphen.drv} % % \title{The \texttt{hyphen.cfg} file for Lua\TeX } -% \date{2010/04/28 v1.3beta} +% \date{2010/04/28 v1.4} % \author{Khaled Hosny, \'Elie Roux, and Manuel P\'egouri\'e-Gonnard\\ % \texttt{khaledhosny@eglug.org} \\ % \texttt{elie.roux@telecom-bretagne.eu} \\ @@ -205,14 +205,10 @@ This work is under the CC0 license. % directly without being parsed by \TeX. If one of these keys is % missing or is the empty string, it is ignored and no patterns (resp. % exceptions) are loaded for this language. The values of -% \texttt{*hyphenmin} are currently unused. +% \texttt{*hyphenmin} values are currently unused. % \item Special case are supported by a field \verb+special+. Currently, % the following kind of values are recognized: % \begin{description} -% \item[\texttt{'null'}] for languages with no hyphenation patterns -% nor exceptions. (Note that this is equivalent to both -% \verb+hyphenation+ and \verb+patterns+ being \verb+nil+ or -% \verb+''+, but produces a more explicit message in the log.) % \item[\texttt{'disabled:<reason>'}] allows to disable specific % languages: when the user tries to load this language, an error % will be issued, with the \verb+<reason>+. @@ -325,10 +321,7 @@ function loadlanguage(lname, id) % % \begin{macrocode} if ldata.special then - if ldata.special == 'null' then - wlog(msg, ' (null)', cname, id) - return - elseif ldata.special:find('^disabled:') then + if ldata.special:find('^disabled:') then err("language disabled by %s: %s (%s)", dbname, cname, ldata.special:gsub('^disabled:', '')) elseif ldata.special == 'language0' then @@ -344,7 +337,7 @@ function loadlanguage(lname, id) % % \begin{macrocode} wlog(msg, '', cname, id) - for _, item in ipairs{'hyphenation', 'patterns'} do + for _, item in ipairs{'patterns', 'hyphenation'} do local file = ldata[item] if file ~= nil and file ~= '' then local file = kpse.find_file(file) or err("file not found: %s", file) @@ -352,6 +345,9 @@ function loadlanguage(lname, id) local data = fh:read('*a') or err("file not readable: %s", f) fh:close() lang[item](lang.new(id), data) + else + if item == 'hyphenation' then item = item..' exceptions' end + wlog("info: no %s for this language", item) end end end @@ -429,7 +425,7 @@ end % % \begin{macrocode} \ProvidesFile{hyphen.cfg} - [2010/04/26 v3.8l-luatex-1.3beta % + [2010/04/26 v3.8l-luatex-1.4 % Language switching mechanism for LuaTeX, adapted from babel v3.8l] % \end{macrocode} % |