diff options
Diffstat (limited to 'Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb')
-rw-r--r-- | Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb | 289 |
1 files changed, 150 insertions, 139 deletions
diff --git a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb index 7b3a0cff320..d76aa29657e 100644 --- a/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb +++ b/Master/texmf-dist/source/generic/hyph-utf8/generate-converters.rb @@ -1,166 +1,177 @@ #!/usr/bin/env ruby -require File.expand_path(File.join(File.dirname(__FILE__), 'hyph-utf8')) - -# $encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x", "t8m", "lth"] -$encodings = ["t8m", "lth"] +require_relative 'hyph-utf8' $path_root=File.expand_path("../../../..", __FILE__) $encoding_data_dir = File.expand_path("../data/encodings", __FILE__) $output_data_dir = "#{$path_root}/tex/generic/hyph-utf8/conversions" -$encodings.each do |encoding| +def output_file_name(encoding) + File.join($output_data_dir, sprintf('conv-utf8-%s.tex', encoding)) +end + +$header = <<__EOHEADER__ +%% conv-utf8-%s.tex +%% +%% Conversion from UTF-8 to %s, +%% used before loading hyphenation patterns for 8-bit TeX engines. +%% +%% This file is part of hyph-utf8 package and autogenerated. +%% See http://tug.org/tex-hyphen +%% +%% Copyright 2008-%d TeX Users Group. +%% You may freely use, modify and/or distribute this file. +%% (But consider adapting the scripts if you need modifications.) +__EOHEADER__ + +def output_copyright_notice(outfile, encoding) + outfile.printf $header, encoding, encoding.upcase, Time.new.year +end + +$uniconvmacro1 = <<__EOUNIMAC1__ +% macros adapted from ConTeXt MKII; see unic-ini.mkii +\\def\\unicodechar#1{% + \\ifcsname unichar@\\number#1\\endcsname + \\csname unichar@\\number#1\\endcsname + \\else + \\errmessage{Unicode character [#1] not in encoding.}% + \\fi} +__EOUNIMAC1__ + +$uniconvmacros = [nil, nil] + +$uniconvmacros << <<__EOTWOBYTES__ +\\def\\utftwouniglyph#1#2% + {\\expandafter\\unicodechar\\expandafter + {\\the\\numexpr64*(#1-192)+`#2-128\\relax}} +__EOTWOBYTES__ + +$uniconvmacros << <<__EOTHREEBYTES__ +\\def\\utfthreeuniglyph#1#2#3% + {\\expandafter\\unicodechar\\expandafter + {\\the\\numexpr4096*(#1-224)+64*(`#2-128)+`#3-128\\relax}} +__EOTHREEBYTES__ + +$uniconvmacros << <<__EOFOURBYTES__ +\\def\\utffouruniglyph#1#2#3#4% + {\\expandafter\\unicodechar\\expandafter + {\\the\\numexpr262144*(#1-240)+4096*(`#2-128)+64*(`#3-128)+`#4-128\\relax}} +__EOFOURBYTES__ + +$uniconvmacro2 = <<__EOUNIMAC2__ + +\\def\\addunichar #1 #2 {\\expandafter\\def\\csname unichar@\\number#1\\endcsname{#2}} + +% \\addunichar "unicode_code - ^^font_encoding_code +__EOUNIMAC2__ + +["t8m", "lth"].each do |encoding| # load encoding e = HyphEncoding.new(encoding) # open file - file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w") - - # copyright notice - file_out.puts "% conv-utf8-#{encoding}.tex" - file_out.puts "%" - file_out.puts "% Conversion from UTF-8 to #{encoding.upcase}," - file_out.puts "% used before loading hyphenation patterns for 8-bit TeX engines." - file_out.puts "%" - file_out.puts "% This file is part of hyph-utf8 package and autogenerated." - file_out.puts "% See http://tug.org/tex-hyphen" - file_out.puts "%" - file_out.puts "% Copyright 2008-#{Time.new.year} TeX Users Group." - file_out.puts "% You may freely use, modify and/or distribute this file." - file_out.puts "% (But consider adapting the scripts if you need modifications.)" - file_out.puts - - # macro to get mapping unicode -> font encoding & error message if screwed up - file_out.puts '% macros adapted from ConTeXt MKII; see unic-ini.mkii' - file_out.puts '\def\unicodechar#1{%' - file_out.puts ' \ifcsname unichar@\number#1\endcsname' - file_out.puts ' \csname unichar@\number#1\endcsname' - file_out.puts ' \else' - file_out.puts ' \errmessage{Unicode character [#1] not in encoding.}%' - file_out.puts ' \fi}' - - # minimal and maximal lenght of characters in the encoding (until now just 2 & 3) - unicode_characters_array = e.unicode_characters.sort - length_min = unicode_characters_array.first[1].bytes.size - length_max = unicode_characters_array.last[1].bytes.size - - # only output the necessary macros for transforming UTF-8 -> Unicode number - if length_min <= 2 and length_max >= 2 then - file_out.puts '\def\utftwouniglyph#1#2%' - file_out.puts ' {\expandafter\unicodechar\expandafter' - file_out.puts ' {\the\numexpr64*(#1-192)+`#2-128\relax}}' - end - if length_min <= 3 and length_max >= 3 then - file_out.puts '\def\utfthreeuniglyph#1#2#3%' - file_out.puts ' {\expandafter\unicodechar\expandafter' - file_out.puts ' {\the\numexpr4096*(#1-224)+64*(`#2-128)+`#3-128\relax}}' - end - if length_min <= 4 and length_max >= 4 then - file_out.puts '\def\utffouruniglyph#1#2#3#4%' - file_out.puts ' {\expandafter\unicodechar\expandafter' - file_out.puts ' {\the\numexpr262144*(#1-240)+4096*(`#2-128)+64*(`#3-128)+`#4-128\relax}}' - end + File.open(output_file_name(encoding), "w") do |file_out| - # macro to store mapping unicode -> font encoding - file_out.puts - file_out.puts '\def\addunichar #1 #2 {\expandafter\def\csname unichar@\number#1\endcsname{#2}}' - file_out.puts - file_out.puts '% \addunichar "unicode_code - ^^font_encoding_code' + # copyright notice + output_copyright_notice(file_out, encoding) + file_out.puts - # all unicode characters in the encoding - e.unicode_characters.sort.each do |code,c| - file_out.puts sprintf("\\addunichar \"%04X ^^%02x \\lccode\"%02X=\"%02X %% %s - %s", - c.code_uni, c.code_enc, c.code_enc, c.code_enc, [c.code_uni].pack('U'), c.name) - end - file_out.puts - - # make all the possible first characters active - # output the definition into file - e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| - byte = first_byte_code.hex - size = chars[0].bytes.size - # 2-byte: 0b11000000 <= byte < 0b11100000 - if size == 2 then - str = "two" - # 3-byte: 0b11100000 <= byte < 0b11110000 - elsif size == 3 then - str = "three" - # 4-byte: 0b11110000 <= byte < 0b11111000 - elsif size == 4 then - str = "four" + # macro to get mapping unicode -> font encoding & error message if screwed up + file_out.puts $uniconvmacro1 + + # minimal and maximal length of characters in the encoding (until now just 2 & 3) + unicode_characters_array = e.unicode_characters.sort + length_min = unicode_characters_array.first[1].bytes.size + length_max = unicode_characters_array.last[1].bytes.size + + # only output the necessary macros for transforming UTF-8 -> Unicode number + (length_min..length_max).each do |nbytes| + file_out.puts $uniconvmacros[nbytes] end - file_out.puts sprintf("\\catcode\"%02X=\\active \\def^^%02x{\\utf%suniglyph{\"%02X}}", byte, byte, str, byte) - end - file_out.close -end + # macro to store mapping unicode -> font encoding + file_out.puts $uniconvmacro2 -$encodings = ["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"] + # all unicode characters in the encoding + e.unicode_characters.sort.each do |code,c| + file_out.printf("\\addunichar \"%04X ^^%02x \\lccode\"%02X=\"%02X %% %s - %s\n", + c.code_uni, c.code_enc, c.code_enc, c.code_enc, [c.code_uni].pack('U'), c.name) + end + file_out.puts + + # make all the possible first characters active + # output the definition into file + e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| + byte = first_byte_code.hex + size = chars[0].bytes.size + # 2-byte: 0b11000000 <= byte < 0b11100000 + str = case size + when 2 then + "two" + # 3-byte: 0b11100000 <= byte < 0b11110000 + when 3 then + "three" + # 4-byte: 0b11110000 <= byte < 0b11111000 + when 4 then + "four" + end + file_out.printf("\\catcode\"%02X=\\active \\def^^%02x{\\utf%suniglyph{\"%02X}}\n", byte, byte, str, byte) + end + end +end -$encodings.each do |encoding| +["ec", "qx", "t2a", "lmc", "il2", "il3", "l7x"].each do |encoding| # load encoding e = HyphEncoding.new(encoding) # open file - file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w") - - # copyright notice - file_out.puts "% conv-utf8-#{encoding}.tex" - file_out.puts "%" - file_out.puts "% Conversion from UTF-8 to #{encoding.upcase}," - file_out.puts "% used before loading hyphenation patterns for 8-bit TeX engines." - file_out.puts "%" - file_out.puts "% This file is part of hyph-utf8 package and autogenerated." - file_out.puts "% See http://tug.org/tex-hyphen" - file_out.puts "%" - file_out.puts "% Copyright 2008-#{Time.new.year} TeX Users Group." - file_out.puts "% You may freely use, modify and/or distribute this file." - file_out.puts "% (But consider adapting the scripts if you need modifications.)" - file_out.puts "%" - - e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| - # sorting all the second characters alphabetically - chars.sort!{|x,y| x.code_uni <=> y.code_uni } - # make all the possible first characters active - # output the definition into file - file_out.puts sprintf("\\catcode\"%02X=\\active", first_byte_code.hex) - end - file_out.puts "%" - e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| - first_byte_code = first_byte_code.hex - size = chars[0].bytes.size - if size != 2 then - throw "The encoding #{encoding} uses more than two bytes to encode characters" - else - - file_out.puts sprintf("\\def^^%02x#1{%%", first_byte_code) - string_fi = "" - for i in 1..(chars.size) - uni_character = chars[i-1] - enc_byte = uni_character.code_enc - enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2') - ux_code = sprintf("U+%04X", uni_character.code_uni) - file_out.puts sprintf("\t\\ifx#1^^%02x^^%02x\\else %% %s - U+%04X - %s", uni_character.bytes[1], uni_character.code_enc, [uni_character.code_uni].pack('U'), uni_character.code_uni, uni_character.name) - string_fi = string_fi + "\\fi" + File.open(output_file_name(encoding), "w") do |file_out| + + # copyright notice + output_copyright_notice(file_out, encoding) + file_out.puts '%' + + e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| + # sorting all the second characters alphabetically + chars.sort!{|x,y| x.code_uni <=> y.code_uni } + # make all the possible first characters active + # output the definition into file + file_out.printf("\\catcode\"%02X=\\active\n", first_byte_code.hex) + end + file_out.puts "%" + e.unicode_characters_first_byte.sort.each do |first_byte_code,chars| + first_byte_code = first_byte_code.hex + size = chars[0].bytes.size + if size != 2 then + throw "The encoding #{encoding} uses more than two bytes to encode characters" + else + + file_out.printf("\\def^^%02x#1{%%\n", first_byte_code) + string_fi = "" + for i in 1..(chars.size) + uni_character = chars[i-1] + enc_byte = uni_character.code_enc + enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2') + file_out.printf("\t\\ifx#1^^%02x^^%02x\\else %% %s - U+%04X - %s\n", uni_character.bytes[1], uni_character.code_enc, [uni_character.code_uni].pack('U'), uni_character.code_uni, uni_character.name) + string_fi = string_fi + "\\fi" + end + + # at least three bytes end - - # at least three bytes + file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}" + file_out.puts string_fi + "}" + end + file_out.puts '%' + file_out.puts '% ensure all the chars above have valid \lccode values' + file_out.puts '%' + e.lowercase_characters.each do |character| + code = [ character.code_enc ].pack("c").unpack("H2").first.upcase + # \lccode"FF="FF + file_out.printf "\\lccode\"%s=\"%s %% %s - U+%04X - %s\n", code, code, [character.code_uni].pack('U'), character.code_uni, character.name end - file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}" - file_out.puts string_fi + "}" - end - file_out.puts '%' - file_out.puts '% ensure all the chars above have valid \lccode values' - file_out.puts '%' - e.lowercase_characters.each do |character| - code = [ character.code_enc ].pack("c").unpack("H2").first.upcase - # \lccode"FF="FF - ux_code = sprintf("U+%04X", character.code_uni) - file_out.puts "\\lccode\"#{code}=\"#{code} % #{[character.code_uni].pack('U')} - #{ux_code} - #{character.name}" - end - file_out.puts - file_out.close + file_out.puts + end end |