1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
|
#!/usr/bin/env ruby
$encoding_data_dir = "data/encodings"
$encodings = ["ec", "qx", "t2a", "lmc", "il3"]
$output_data_dir = "../../../tex/generic/hyph-utf8/conversions"
class UnicodeCharacter
def initialize(code_uni, code_enc, name)
@code_uni = code_uni
@code_enc = code_enc
# TODO: might be longer or shorter
@bytes = [code_uni].pack('U').unpack('H2H2')
@name = name
end
attr_reader :code_uni, :code_enc, :bytes, :name
end
class UnicodeCharacters < Hash
def add_new_character(code_uni, code_enc, name)
first_byte = [code_uni].pack('U').unpack('H2').first
if self[first_byte] == nil then
self[first_byte] = Array.new
end
self[first_byte].push(UnicodeCharacter.new(code_uni, code_enc, name))
end
end
# 0x19; U+0131; 1; dotlessi
$encodings.each do |encoding|
#$utf_combinations = Hash.new
$unicode_characters = UnicodeCharacters.new
# those that need lccode to be set
$lowercase_characters = Array.new
File.open($encoding_data_dir + "/" + encoding + ".dat").grep(/^0x(\w+)\tU\+(\w+)\t(\d*)\t([_a-zA-Z\.]*)$/) do |line|
# puts line
code_enc = $1.hex
code_uni = $2.hex
if $3.length > 0
type = $3.to_i
else
type = 0
end
name = $4
if type == 1 then
$unicode_characters.add_new_character(code_uni, code_enc, name)
$lowercase_characters.push(UnicodeCharacter.new(code_uni, code_enc, name))
end
end
$file_out = File.open("#{$output_data_dir}#{File::Separator}conv-utf8-#{encoding}.tex", "w")
$file_out.puts "% conv-utf8-#{encoding}.tex"
$file_out.puts "%"
$file_out.puts "% Conversion from UTF-8 to #{encoding.upcase},"
$file_out.puts "% used before loading hyphenation patterns for 8-bit TeX engines."
$file_out.puts "%"
$file_out.puts "% This file is part of hyph-utf8 package and autogenerated."
$file_out.puts "% See http://tug.org/tex-hyphen"
$file_out.puts "%"
$file_out.puts "% Copyright 2008 TeX Users Group."
$file_out.puts "% You may freely use, modify and/or distribute this file."
$file_out.puts "% (But consider adapting the scripts if you need modifications.)"
$file_out.puts "%"
$unicode_characters.sort.each do |first_byte|
# sorting all the second characters alphabetically
first_byte[1].sort!{|x,y| x.code_uni <=> y.code_uni }
# make all the possible first characters active
# output the definition into file
$file_out.puts "\\catcode\"#{first_byte[0].upcase}=\\active"
end
$file_out.puts "%"
$unicode_characters.sort.each do |first_byte|
$file_out.puts "\\def^^#{first_byte[0]}#1{%"
string_fi = ""
for i in 1..(first_byte[1].size)
uni_character = first_byte[1][i-1]
second_byte = uni_character.bytes[1]
enc_byte = uni_character.code_enc
enc_byte = [ uni_character.code_enc ].pack('c').unpack('H2')
ux_code = sprintf("U+%04X", uni_character.code_uni)
$file_out.puts "\t\\ifx#1^^#{second_byte}^^#{enc_byte}\\else % #{[uni_character.code_uni].pack('U')} - #{ux_code} - #{uni_character.name}"
string_fi = string_fi + "\\fi"
end
$file_out.puts "\t\\errmessage{Hyphenation pattern file corrupted or #{encoding} encoding not supported!}"
$file_out.puts string_fi+"}"
end
$file_out.puts "%"
$file_out.puts "% ensure all the chars above have valid \lccode values"
$file_out.puts "%"
$lowercase_characters.sort!{|x,y| x.code_enc <=> y.code_enc }.each do |character|
code = [ character.code_enc ].pack("c").unpack("H2").first.upcase
# \lccode"FF="FF
ux_code = sprintf("U+%04X", character.code_uni)
$file_out.puts "\\lccode\"#{code}=\"#{code} % #{[character.code_uni].pack('U')} - #{ux_code} - #{character.name}"
end
$file_out.puts
# if encoding == 'ec'
# $file_out.puts '% TODO: test if needed and the exact syntax'
# $file_out.puts '% some patterns use apostrophe and hyphen'
# # \lccode`\'=`\'
# $file_out.puts "\\lccode`\\'=`\\'"
# $file_out.puts "\\catcode`-=11"
# $file_out.puts
# end
# if encoding == 't2a'
# $file_out.puts "\\lccode`\\'=`\\'"
# end
$file_out.close
end
|
