diff options
Diffstat (limited to 'Master/texmf-dist/source/generic/hyph-utf8')
6 files changed, 303 insertions, 143 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb index 3c94441229c..9ea3ca061f8 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-pattern-loaders.rb @@ -16,74 +16,108 @@ languages = Languages.new.list #text_if_native_utf = "\input pattern-loader.tex\n\\ifNativeUtfEightPatterns" -text_if_native_utf = <<EOT -% Test whether we received one or two arguments -\\def\\testengine#1#2!{\\def\\secondarg{#2}} -% That's Tau (as in Taco or ΤΕΧ, Tau-Epsilon-Chi), a 2-byte UTF-8 character -\\testengine Τ!\\relax -% Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument -\\ifx\\secondarg\\empty -EOT - languages.each do |language| + +string_enc = (language.encoding == nil) ? "" : language.encoding.upcase + " " + +################ +# Header texts # +################ + +# a message about auto-generation +# TODO: write a more comprehensive one +text_header = +"% filename: loadhyph-#{language.code}.tex +% language: #{language.name} +% +% Loader for hyphenation patterns, generated by +% source/generic/hyph-utf8/generate-pattern-loaders.rb +% See also http://tug.org/tex-hyphen +% +% Copyright 2008-2011 TeX Users Group. +% You may freely use, modify and/or distribute this file. +% (But consider adapting the scripts if you need modifications.) +% +% Once it turns out that more than a simple definition is needed, +% these lines may be moved to a separate file. +%" + +text_if_native_utf = +'% Test for pTeX +\\ifx\\kanjiskip\\undefined +% Test for native UTF-8 (which gets only a single argument) +% That\'s Tau (as in Taco or ΤΕΧ, Tau-Epsilon-Chi), a 2-byte UTF-8 character +\\def\\testengine#1#2!{\\def\\secondarg{#2}}\\testengine Τ!\\relax +\\ifx\\secondarg\\empty' + +comment_engine_utf8 = "% Unicode-aware engine (such as XeTeX or LuaTeX) only sees a single (2-byte) argument" +comment_engine_8bit = "% 8-bit engine (such as TeX or pdfTeX)" +comment_engine_ptex = "% pTeX" + +text_engine_ascii = ["% ASCII patterns - no additional support is needed", + "\\message{ASCII #{language.message}}", + "\\input hyph-#{language.code}.tex"] +text_engine_utf8 = [" #{comment_engine_utf8}", + " \\message{UTF-8 #{language.message}}"] +text_engine_8bit = [" #{comment_engine_8bit}", + " \\message{#{string_enc}#{language.message}}"] +text_engine_ptex = [" #{comment_engine_ptex}", + " \\message{#{string_enc}#{language.message}}"] +text_engine_8bit_no = [" #{comment_engine_8bit}", + " \\message{No #{language.message} - only for Unicode engines}", + " %\\input zerohyph.tex"] +text_engine_ptex_no = [" #{comment_engine_ptex}", + " \\message{No #{language.message} - only for Unicode engines}", + " %\\input zerohyph.tex"] +text_patterns = " \\input hyph-#{language.code}.tex" +text_patterns_ptex = " \\input phyph-#{language.code}.tex" +text_patterns_old = " \\input #{language.filename_old_patterns}" +text_patterns_conv = " \\input conv-utf8-#{language.encoding}.tex" + +########### +# lccodes # +########### + +lccodes_common = [] +if language.code == 'af' or language.code == 'it' or language.code == 'fr' or language.code == 'uk' or language.code == 'zh-latn' then + lccodes_common.push("\\lccode`\\'=`\\'") +end +if language.code == 'af' or language.code == 'pt' or language.code == 'tk' or language.code == 'ru' or language.code == 'uk' then + lccodes_common.push("\\lccode`\\-=`\\-") +end + if language.use_new_loader then - filename = "#{$path_loadhyph}/loadhyph-#{language.code}.tex" + if language.code == 'sh-latn' then + filename = "#{$path_loadhyph}/loadhyph-sr-latn.tex" + else + filename = "#{$path_loadhyph}/loadhyph-#{language.code}.tex" + end puts "generating '#{filename}'" + File.open(filename, "w") do |file| - # a message about auto-generation - # TODO: write a more comprehensive one - file.puts "% loadhyph-#{language.code}.tex" - file.puts "%" - file.puts "% Autogenerated loader for hyphenation patterns for \"#{language.name}\"" - file.puts "% by source/generic/hyph-utf8/generate-pattern-loaders.rb" - file.puts "% See also http://tug.org/tex-hyphen" - file.puts "%" - file.puts "% Copyright 2008 TeX Users Group." - file.puts "% You may freely use, modify and/or distribute this file." - file.puts "% (But consider adapting the scripts if you need modifications.)" - file.puts "%" - file.puts "% Once it turns out that more than a simple definition is needed," - file.puts "% these lines may be moved to a separate file." - file.puts "%" - - # message (where it exists) - # if language.message != nil - # file.puts("\\message{#{language.message}}\n%") - # end - - # TODO: - # \lefthyphenmin=2 \righthyphenmin=2 - # but probably this needs to reside outside of \begingroup/endgroup - + file.puts text_header file.puts('\begingroup') - if language.code == 'af' or language.code == 'it' or language.code == 'fr' or language.code == 'uk' or language.code == 'zh-latn' then - file.puts("\\lccode`\\'=`\\'") - end - if language.code == 'pt' or language.code == 'tk' then - file.puts("\\lccode`\\-=`\\-") + + if lccodes_common.length > 0 then + file.puts lccodes_common.join("\n") end - + +######################################## +# GROUP nr. 1 - ONLY USABLE WITH UTF-8 # +######################################## # some special cases firs # # some languages (sanskrit) are useless in 8-bit engines; we only want to load them for UTF engines # TODO - maybe consider doing something similar for ibycus - if language.code == 'sa' or - language.code == 'as' or - language.code == 'bn' or - language.code == 'gu' or - language.code == 'hi' or - language.code == 'hy' or - language.code == 'kn' or - language.code == 'lo' or - language.code == 'ml' or - language.code == 'mr' or - language.code == 'or' or - language.code == 'pa' or - language.code == 'ta' or - language.code == 'te' then + if ['sa','as','bn','gu','hi','hy','kn','lo','mul-ethi','ml','mr','or','pa','ta','te'].include?(language.code) then file.puts(text_if_native_utf) - file.puts(" \\message{UTF-8 #{language.message}}") - if language.code != 'lo' then + file.puts(text_engine_utf8) + # lccodes + if language.code == 'mul-ethi' then + file.puts(' % Set \lccode for Ethiopian word space.') + file.puts(' \lccode"1361="1361') + file.puts(' \lccode"1362="1362') + elsif language.code != 'lo' then file.puts(' % Set \lccode for ZWNJ and ZWJ.') file.puts(' \lccode"200C="200C') file.puts(' \lccode"200D="200D') @@ -93,38 +127,38 @@ languages.each do |language| file.puts(' \lccode"0CF2="0CF2') end end - file.puts(" \\input hyph-#{language.code}.tex") + file.puts(text_patterns) file.puts('\else') - file.puts(" \\message{No #{language.message} - only available with Unicode engines}") - file.puts(' \input zerohyph.tex') + file.puts(text_engine_8bit_no) + file.puts('\fi\else') + file.puts(text_engine_ptex_no) file.puts('\fi') + +####################### +# GROUP nr. 2 - ASCII # +####################### # for ASCII encoding, we don't load any special support files, but simply load everything elsif language.encoding == "ascii" then - file.puts('% ASCII patterns - no additional support is needed') - # for UK English we simply load the old file without bothering about renaming it - if language.use_old_patterns then - file.puts("\\input #{language.filename_old_patterns}") - # for the rest we load the new file - else - file.puts("\\message{ASCII #{language.message}}") - file.puts("\\input hyph-#{language.code}.tex") - end + file.puts(text_engine_ascii) +#################################### +# GROUP nr. 3 - different patterns # +#################################### # when lanugage uses old patterns for 8-bit engines, load two different patterns rather than using the converter elsif language.use_old_patterns then file.puts(text_if_native_utf) - file.puts(" \\message{UTF-8 #{language.message}}") + file.puts(text_engine_utf8) # some catcodes for XeTeX if language.code == 'grc' or language.code.slice(0,2) == 'el' then file.puts(" \\lccode`'=`'\\lccode`’=`’\\lccode`ʼ=`ʼ\\lccode`᾽=`᾽\\lccode`᾿=`᾿") end - file.puts(" \\input hyph-#{language.code}.tex") + file.puts(text_patterns) # russian and ukrainian exceptions - hacks with dashes if language.code == 'ru' or language.code == 'uk' then file.puts(' % Additional patterns with hyphen/dash: a hack to prevent breaking after hyphen, but not before.') file.puts(" \\input exhyph-#{language.code}.tex") end file.puts('\else') - file.puts(" \\message{#{language.message}}") + file.puts(text_engine_8bit) # explain why we are still using the old patterns if language.use_old_patterns_comment != nil then file.puts(" % #{language.use_old_patterns_comment}") @@ -132,30 +166,35 @@ languages.each do |language| puts "Missing comment for #{language.name}" file.puts(' % we still load old patterns for 8-bit TeX') end - file.puts(" \\input #{language.filename_old_patterns}") + file.puts(text_patterns_old) + file.puts('\fi\else') + file.puts(text_engine_ptex) + # greek, coptic + if language.encoding == nil then + file.puts(text_patterns_old) + else + file.puts(text_patterns_ptex) + end file.puts('\fi') +######################### +# GROUP nr. 4 - regular # +######################### else file.puts(text_if_native_utf) - file.puts(" \\message{UTF-8 #{language.message}}") + file.puts(text_engine_utf8) + file.puts(text_patterns) file.puts('\else') - file.puts(" \\message{#{language.encoding.upcase} #{language.message}}") - # a hack for OT1 encoding in three languages - if language.code == 'da' or language.code == 'fr' then - file.puts(" % A hack to support both EC and OT1 encoding in 8-bit engines.") - file.puts(" % Kept for backward compatibility only, though we would prefer to drop it.") - file.puts(" % OT1 encoding is close-to-useless for proper hyphenation.") - file.puts(" \\input spechyph-ot1-#{language.code}.tex") - end - file.puts(" \\input conv-utf8-#{language.encoding}.tex") + file.puts(text_engine_8bit) + file.puts(text_patterns_conv) + file.puts(text_patterns) + file.puts('\fi\else') + file.puts(text_engine_ptex) + file.puts(text_patterns_ptex) file.puts('\fi') - if language.code == 'sr-latn' then - file.puts("% Load Serbo-Croatian patterns") - file.puts("\\input hyph-sh-latn.tex") - else - file.puts("\\input hyph-#{language.code}.tex") - end -# file.puts("\\loadhyphpatterns{#{language.code}}{#{language.encoding}}%") end +####### +# end # +####### file.puts('\endgroup') end end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb index 417fa8a5c4b..b738686b012 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-tl-files.rb @@ -136,7 +136,7 @@ language_groups.sort.each do |language_name,language_list| end file_patterns = "file_patterns=#{filename_pat}" - if !File::size?( "#{$path_txt}/#{filename_hyp}" ) then + if File::size?( "#{$path_txt}/#{filename_hyp}" ) != nil then file_exceptions = "file_exceptions=#{filename_hyp}" else file_exceptions = "file_exceptions=" diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb index aa3dec8d0f2..5086adf728f 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-webpage.rb @@ -57,58 +57,60 @@ end language_groups.sort.each do |language_name,language_list| first_line_printed = false language_list.each do |language| - if not first_line_printed then - puts "<tr>\n\t<td><b>#{language_name.capitalize}</b></td>" - first_line_printed = true; - else - puts "<tr>\n\t<td> </td>" + if language != nil then + if not first_line_printed then + puts "<tr>\n\t<td><b>#{language_name.capitalize}</b></td>" + first_line_printed = true; + else + puts "<tr>\n\t<td> </td>" + end + + # synonyms + if language.synonyms != nil and language.synonyms.length > 0 then + synonyms=", #{language.synonyms.join(', ')}" + else + synonyms="" + end + puts "\t<td>#{language.name}#{synonyms}</td>" + + # if language.use_old_patterns == false then + if language.use_new_loader == true then + url_patterns = "#{$ctan_url}/patterns/tex/hyph-#{language.code}.tex" + code = "<a href=\"#{url_patterns}\">#{language.code}</a>" + else + url_patterns = "" + code = language.code + end + + puts "\t<td>#{code}</td>" + + # lefthyphenmin/righthyphenmin + if language.hyphenmin == nil or language.hyphenmin.length == 0 then + lmin = '' + rmin = '' + elsif language.filename_old_patterns == "zerohyph.tex" then + lmin = '' + rmin = '' + else + lmin = language.hyphenmin[0] + rmin = language.hyphenmin[1] + end + puts "\t<td>(#{lmin},#{rmin})</td>" + # which file to use + if language.use_new_loader then + file = "loadhyph-#{language.code}.tex" + else + file = "#{language.filename_old_patterns}" + end + #puts "\t<td>#{file}</td>" + if language.encoding == nil then + encoding = "" + else + encoding = language.encoding.upcase + end + puts "\t<td>#{encoding}</td>" + puts "</tr>\n" end - - # synonyms - if language.synonyms != nil and language.synonyms.length > 0 then - synonyms=", #{language.synonyms.join(', ')}" - else - synonyms="" - end - puts "\t<td>#{language.name}#{synonyms}</td>" - -# if language.use_old_patterns == false then - if language.use_new_loader == true then - url_patterns = "#{$ctan_url}/patterns/hyph-#{language.code}.tex" - code = "<a href=\"#{url_patterns}\">#{language.code}</a>" - else - url_patterns = "" - code = language.code - end - - puts "\t<td>#{code}</td>" - - # lefthyphenmin/righthyphenmin - if language.hyphenmin == nil or language.hyphenmin.length == 0 then - lmin = '' - rmin = '' - elsif language.filename_old_patterns == "zerohyph.tex" then - lmin = '' - rmin = '' - else - lmin = language.hyphenmin[0] - rmin = language.hyphenmin[1] - end - puts "\t<td>(#{lmin},#{rmin})</td>" - # which file to use - if language.use_new_loader then - file = "loadhyph-#{language.code}.tex" - else - file = "#{language.filename_old_patterns}" - end - #puts "\t<td>#{file}</td>" - if language.encoding == nil then - encoding = "" - else - encoding = language.encoding.upcase - end - puts "\t<td>#{encoding}</td>" - puts "</tr>\n" end end diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages-txt.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages-txt.rb index 9c0981762f3..1bff59b37cd 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/languages-txt.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/languages-txt.rb @@ -333,6 +333,18 @@ class Languages < Hash # ---------------------------- # languages using new patterns # ---------------------------- +# afrikaans +{ + "code" => "af", + "name" => "afrikaans", + "use_new_loader" => true, + "use_old_patterns" => false, + "filename_old_patterns" => nil, + "hyphenmin" => [1,2], # in babel: 2,2 + "encoding" => "ec", + "exceptions" => true, + "message" => "Afrikaans Hyphenation Patterns", +}, # catalan cahyph.tex { "code" => "ca", diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb index c157e60d425..5287e25f4ed 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/languages.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/languages.rb @@ -335,7 +335,7 @@ class Languages < Hash "use_new_loader" => true, "use_old_patterns" => false, "filename_old_patterns" => nil, - "hyphenmin" => [2,2], # babel + "hyphenmin" => [1,2], # in babel: 2,2 "encoding" => "ec", "exceptions" => true, "message" => "Afrikaans Hyphenation Patterns", @@ -1050,6 +1050,17 @@ class Languages < Hash "exceptions" => false, "message" => "Lao Hyphenation Patterns", }, +# pan-Ethiopic +{ + "code" => "mul-ethi", + "name" => "ethiopic", "synonyms" => ["amharic", "geez"], + "use_new_loader" => true, + "use_old_patterns" => false, + "hyphenmin" => [1,1], + "encoding" => nil, # no patterns for 8-bit TeX + "exceptions" => false, + "message" => "Pan-Ethiopic Hyphenation Patterns", +}, # dumylang -> dumyhyph.tex # nohyphenation -> zerohyph.tex # arabic -> zerohyph.tex diff --git a/Master/texmf-dist/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua b/Master/texmf-dist/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua new file mode 100644 index 00000000000..a5fa334f22c --- /dev/null +++ b/Master/texmf-dist/source/generic/hyph-utf8/languages/mul-ethi/generate_patterns_mul-ethi.lua @@ -0,0 +1,96 @@ +#!/usr/bin/env texlua + +-- Extract Ethiopic syllables from the Unicode Character Database. +-- Arthur Reutenauer, London, 2011-01-09 & 10, for the hyph-utf8 project +-- http://tug.org/tex-hyphen +-- Copyright (c) TeX Users Group, 2011. +-- You may freely use, modify and / or redistribute this file. + +-- Use with TeXLua. + +-- Small library to convert a hexadecimal string to a number +local function hex_digit_to_num(h) + local c = string.byte(h) + if c >= 48 and c < 58 + then return c - 48 + elseif c >= 65 and c < 71 + then return c - 55 + elseif c >= 97 and c < 103 + then return c - 87 + end + return 0 +end + +local function hex_string_to_num(hex) + local n = 0 + for i = 1, string.len(hex) do + n = 16 * n + hex_digit_to_num(string.sub(hex, i, i + 1)) + end + return n +end + +local ucd_file = "UnicodeData.txt" + +local error_install_ucd = [[ +Error: the Unicode Character Database could not be found. +Please download it from http://www.unicode.org/Public/UNIDATA/UnicodeData.txt +and install it in the current directory.]] + +if not lfs.attributes(ucd_file) then + print(error_install_ucd) + return -1 +end + +local P, R, C, match = lpeg.P, lpeg.R, lpeg.C, lpeg.match + +local digit = R"09" + R"AF" +local colon = P";" +local field = (1 - colon)^0 +local eth_syll = P"ETHIOPIC SYLLABLE " * C(field) +local ucd_entry = C(digit^4) * colon * eth_syll + +local pattfile = assert(io.open("hyph-mul-ethi.tex", "w")) + +pattfile:write[[ +% Experimental pattern file for languages written using the Ethiopic script. +% Arthur Reutenauer, London, 2011-01-09 & 10, for the hyph-utf8 project. +% Copyright (c) TeX Users Group, 2011. +% You may freely use, modify and / or redistribute this file. +% +% The BCP 47 language tag for that file is "mul-ethi" to reflect the fact that +% it can be used by multiple languages (and a single script, Ethiopic). It is, +% though, not supposed to be linguistically relevant and should, for proper +% typography, be replaced by files tailored to individual languages. What we +% do for the moment is to simply allow break on either sides of Ethiopic +% syllables, and to forbid it before some punctuation marks particular to +% the Ethiopic script (which we thus make letters for this purpose). +% +% This is a generated file. If you wish to edit it, consider adapting the +% generating programme +% (svn://tug.org/texhyphen/trunk/hyph-utf8/source/generic/hyph-utf8/languages/ethiopic/make-patterns.lua. + +]] + +pattfile:write"\\patterns{%\n" +for ucd_line in io.lines(ucd_file) do + local usv, char_name = match(ucd_entry, ucd_line) + -- print(string.format("%04X", usv)) + -- if usv then print(usv, char_name) end + if usv then + -- arbitrarily excluding Ethiopic Extended-A + -- because they're not in unicode-letters.tex + local hex = hex_string_to_num(usv) + if hex < 0xAB00 then + local hex = hex_string_to_num + pattfile:write("1", unicode.utf8.char(hex_string_to_num(usv)), "1 ") + pattfile:write("% U+", usv, " ", char_name:lower(), "\n") + end + end +end + +pattfile:write("2", unicode.utf8.char(0x1361), "1 % U+1361 ETHIOPIC WORDSPACE\n") +pattfile:write("2", unicode.utf8.char(0x1362), "1 % U+1362 ETHIOPIC FULL STOP\n") + +pattfile:write"}" + +assert(io.close(pattfile)) |