summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2012-06-02 21:57:13 +0000
committerKarl Berry <karl@freefriends.org>2012-06-02 21:57:13 +0000
commit1be4c489af95633ffb30295d7aa468f7ba72f263 (patch)
treeb79d367ece6ca87c47acacc2fa5830f2c11c0af3
parent25e71cc945572e1b9dbb80b36384b531b5b01375 (diff)
consistency
git-svn-id: svn://tug.org/texlive/trunk@26806 c570f23f-e606-0410-a88d-b1316a301751
-rwxr-xr-xBuild/source/texk/texlive/linked_scripts/uptex/convbkmk.rb124
1 files changed, 84 insertions, 40 deletions
diff --git a/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb b/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb
index 2794329384b..1d512fb10d1 100755
--- a/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb
+++ b/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb
@@ -3,7 +3,7 @@
=begin
-convbkmk Ver.0.03
+convbkmk Ver.0.04
= License
@@ -35,12 +35,17 @@ THE SOFTWARE.
2011.05.02 0.01 Bug fix: BOM was not correct.
2012.05.08 0.02 Bug fix: for a case of dvips with -z option and Ruby1.8.
Add conversion of /Creator and /Producer .
-2012.05.09 0.03 Suppress halfwidth -> fullwidth katakana conversion
+2012.05.12 0.03 Suppress halfwidth -> fullwidth katakana conversion
and MIME decoding in Ruby1.8.
+2012.06.01 0.04 Support escape sequences: \n, \r, \t, \b, \f, \\,
+ \ddd (octal, PDFDocEncoding) and \0xUUUU (Unicode UTF-16BE).
+ Support sequences of end of line:
+ '\' or other followed by "\n", "\r\n" or "\r" .
+ Set file IO to binary mode.
=end
-Version = "0.03"
+Version = "0.04"
require "optparse"
@@ -48,8 +53,14 @@ if RUBY_VERSION >= "1.9"
$RUBY_M17N = true
class String
- def to_utf16be(enc)
- self.force_encoding(enc.current).encode('UTF-16BE')
+ def to_utf8(enc)
+ self.force_encoding(enc.current).encode('UTF-8')
+ end
+ def utf16be_to_utf8
+ self.force_encoding('UTF-16BE').encode('UTF-8')
+ end
+ def utf8_to_utf16be
+ self.force_encoding('UTF-8').encode('UTF-16BE')
end
end
else
@@ -58,18 +69,44 @@ else
require "jcode" # for method each_char
require "nkf"
class String
- def to_utf16be(enc)
- NKF.nkf('-w16 -x -m0 '+enc.kconv_enc, self)
+ def to_utf8(enc)
+ NKF.nkf('-w -x -m0 '+enc.kconv_enc, self)
+ end
+ def utf16be_to_utf8
+ NKF.nkf('-w -x -m0 --utf16-input', self)
+ end
+ def utf8_to_utf16be
+ NKF.nkf('-w16 -x -m0 --utf8-input', self)
end
def ascii_only?
return self !~ /[\x80-\xFF]/n
end
+ REPLACE_TABLE = {"\n" => "\\n", "\r" => "\\r", "\t" => "\\t",
+ "\b" => "\\b", "\f" => "\\f", "\e" => "\\e", "\"" => "\\\""}
def valid_encoding?
- tmp = "\"" + self + "\""
- tmp.gsub!("\\(","\\\\\\(")
- tmp.gsub!("\\)","\\\\\\)")
- tmp.gsub!("\n","\\n")
- tmp.gsub!("\r","\\r")
+ conv = ''
+ tmp = self.dup
+ while tmp.length>0 do
+ case tmp
+ when /\A\\\\/
+ conv += "\\\\\\\\"
+ when /\A\\[nrtbf()]/, /\A\\[0-3][0-7][0-7]/, /\A\\0x[0-9A-F]{4}/i
+ conv += "\\"+$&
+ when /\A\\/
+ conv += "\\\\"
+ when /\A[^\\]*/
+ conv += $&
+ else
+ raise 'unexpected input!'
+ end
+ tmp = $'
+ end
+ conv.gsub!(/[\n\r\t\b\f\e"]/) { |s| REPLACE_TABLE[s] }
+ conv.gsub!(/([\000-\037\177])/) { "\\%#03o" % $1.unpack("C*") }
+ tmp = "\"" + conv + "\""
+# puts 's:' + self
+# puts 't:' + tmp
+# puts 'i:' + self.inspect
return tmp == self.inspect
end
def force_encoding(enc)
@@ -206,15 +243,15 @@ def check_parentheses_balance(line, enc)
# illegal input
$stdout = STDERR
p 'parameters: '
- p enc.status
- p enc.option
- p enc.current
+ p ' status: ' + enc.status
+ p ' option: ' + enc.option
+ p ' current: ' + enc.current
p enc.is_8bit
if !$RUBY_M17N
- p tmp_rest.inspect
- p tmp_rest
+ p ' inspect: ' + tmp_rest.inspect
+ p ' rest : [' + tmp_rest + ']'
end
- p line
+ p ' [' + line + ']'
raise 'encoding is not consistent'
end
end
@@ -240,6 +277,14 @@ def check_parentheses_balance(line, enc)
return depth, count, tmp_prev, tmp_rest
end
+# PDFDocEncoding -> UTF-16BE
+PDF2UNI = Array(0..255)
+PDF2UNI[0o030..0o037] = 0x02d8, 0x02c7, 0x02c6, 0x02d9, 0x02dd, 0x02db, 0x02da, 0x02dc
+PDF2UNI[0o200..0o207] = 0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044
+PDF2UNI[0o210..0o217] = 0x2039, 0x203a, 0x2212, 0x2030, 0x201e, 0x201c, 0x201d, 0x2018
+PDF2UNI[0o220..0o227] = 0x2019, 0x201a, 0x2122, 0xfb01, 0xfb02, 0x0141, 0x0152, 0x0160
+PDF2UNI[0o230..0o237] = 0x0178, 0x017d, 0x0131, 0x0142, 0x0153, 0x0161, 0x017e, 0xfffd
+PDF2UNI[0o240 ] = 0x20ac
def conv_string_to_utf16be(line, enc)
if line !~ /(\()(.*)(\))/m
@@ -247,24 +292,30 @@ def conv_string_to_utf16be(line, enc)
end
pre, tmp, post = $`, $2, $'
- if tmp.ascii_only?
+ if tmp.ascii_only? && tmp !~ /\\0x[0-9A-F]{4}/i
return line
end
conv = ''
- conv.force_encoding(enc.current)
+ conv.force_encoding('UTF-8')
tmp.force_encoding(enc.current)
+
while tmp.length>0 do
- if tmp =~ /\A[^\\]+/
- conv += $&
- elsif tmp =~ /\\([0-3][0-7][0-7])/
- conv += $RUBY_M17N ? $&.oct.chr('ASCII-8BIT') : $&.oct.chr
- elsif tmp =~ /\\([\r\n]{1,2})/
+ case tmp
+ when /\A[^\\\n\r]+/
+ conv += $&.to_utf8(enc)
+ when /\A\\([0-3][0-7][0-7])/ # PDFDocEncoding -> UTF-8
+ conv += [PDF2UNI[$1.oct]].pack("U*")
+ when /\A\\0x(D[8-B][0-9A-F]{2})\\0x(D[C-F][0-9A-F]{2})/i # surrogate pair
+ conv += [$1.hex, $2.hex].pack("n*").utf16be_to_utf8
+ when /\A\\0x([0-9A-F]{4})/i
+ conv += [$1.hex].pack("U*")
+ when /\A\\[nrtbf\\]/
+ conv += eval(%!"#{$&}"!)
+ when /\A(\r\n|\r|\n)/
+ conv += "\n"
+ when /\A\\([\r\n]{1,2})|\\/
# ignore
- elsif tmp =~ /\\[nrtbf]/
- conv += eval($&)
- elsif tmp =~ /\\(.)/
- conv += $1
else
raise 'unexpected input!'
end
@@ -272,14 +323,8 @@ def conv_string_to_utf16be(line, enc)
end
buf = 'FEFF' # BOM for UTF-16BE
- conv.each_char { |chr|
- if chr == "\r" || chr == "\n"
- buf += chr
- else
- chr.to_utf16be(enc).each_byte {|byte|
- buf += '%02X' % byte
- }
- end
+ conv.utf8_to_utf16be.each_byte {|byte|
+ buf += '%02X' % byte
}
return pre + '<' + buf + '>' + post
end
@@ -341,12 +386,11 @@ else
raise 'input file does not seem PS file'
end
fout = fin.gsub(/\.ps$/i, "-convbkmk#{$&}")
- open(fin, 'r') {|ifile|
- open(fout, 'w') {|ofile|
+ open(fin, 'rb') {|ifile|
+ open(fout, 'wb') {|ofile|
file_treatment(ifile, ofile, enc)
}
}
}
end
-