From 1be4c489af95633ffb30295d7aa468f7ba72f263 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Sat, 2 Jun 2012 21:57:13 +0000 Subject: consistency git-svn-id: svn://tug.org/texlive/trunk@26806 c570f23f-e606-0410-a88d-b1316a301751 --- .../texk/texlive/linked_scripts/uptex/convbkmk.rb | 124 ++++++++++++++------- 1 file changed, 84 insertions(+), 40 deletions(-) (limited to 'Build') diff --git a/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb b/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb index 2794329384b..1d512fb10d1 100755 --- a/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb +++ b/Build/source/texk/texlive/linked_scripts/uptex/convbkmk.rb @@ -3,7 +3,7 @@ =begin -convbkmk Ver.0.03 +convbkmk Ver.0.04 = License @@ -35,12 +35,17 @@ THE SOFTWARE. 2011.05.02 0.01 Bug fix: BOM was not correct. 2012.05.08 0.02 Bug fix: for a case of dvips with -z option and Ruby1.8. Add conversion of /Creator and /Producer . -2012.05.09 0.03 Suppress halfwidth -> fullwidth katakana conversion +2012.05.12 0.03 Suppress halfwidth -> fullwidth katakana conversion and MIME decoding in Ruby1.8. +2012.06.01 0.04 Support escape sequences: \n, \r, \t, \b, \f, \\, + \ddd (octal, PDFDocEncoding) and \0xUUUU (Unicode UTF-16BE). + Support sequences of end of line: + '\' or other followed by "\n", "\r\n" or "\r" . + Set file IO to binary mode. =end -Version = "0.03" +Version = "0.04" require "optparse" @@ -48,8 +53,14 @@ if RUBY_VERSION >= "1.9" $RUBY_M17N = true class String - def to_utf16be(enc) - self.force_encoding(enc.current).encode('UTF-16BE') + def to_utf8(enc) + self.force_encoding(enc.current).encode('UTF-8') + end + def utf16be_to_utf8 + self.force_encoding('UTF-16BE').encode('UTF-8') + end + def utf8_to_utf16be + self.force_encoding('UTF-8').encode('UTF-16BE') end end else @@ -58,18 +69,44 @@ else require "jcode" # for method each_char require "nkf" class String - def to_utf16be(enc) - NKF.nkf('-w16 -x -m0 '+enc.kconv_enc, self) + def to_utf8(enc) + NKF.nkf('-w -x -m0 '+enc.kconv_enc, self) + end + def utf16be_to_utf8 + NKF.nkf('-w -x -m0 --utf16-input', self) + end + def utf8_to_utf16be + NKF.nkf('-w16 -x -m0 --utf8-input', self) end def ascii_only? return self !~ /[\x80-\xFF]/n end + REPLACE_TABLE = {"\n" => "\\n", "\r" => "\\r", "\t" => "\\t", + "\b" => "\\b", "\f" => "\\f", "\e" => "\\e", "\"" => "\\\""} def valid_encoding? - tmp = "\"" + self + "\"" - tmp.gsub!("\\(","\\\\\\(") - tmp.gsub!("\\)","\\\\\\)") - tmp.gsub!("\n","\\n") - tmp.gsub!("\r","\\r") + conv = '' + tmp = self.dup + while tmp.length>0 do + case tmp + when /\A\\\\/ + conv += "\\\\\\\\" + when /\A\\[nrtbf()]/, /\A\\[0-3][0-7][0-7]/, /\A\\0x[0-9A-F]{4}/i + conv += "\\"+$& + when /\A\\/ + conv += "\\\\" + when /\A[^\\]*/ + conv += $& + else + raise 'unexpected input!' + end + tmp = $' + end + conv.gsub!(/[\n\r\t\b\f\e"]/) { |s| REPLACE_TABLE[s] } + conv.gsub!(/([\000-\037\177])/) { "\\%#03o" % $1.unpack("C*") } + tmp = "\"" + conv + "\"" +# puts 's:' + self +# puts 't:' + tmp +# puts 'i:' + self.inspect return tmp == self.inspect end def force_encoding(enc) @@ -206,15 +243,15 @@ def check_parentheses_balance(line, enc) # illegal input $stdout = STDERR p 'parameters: ' - p enc.status - p enc.option - p enc.current + p ' status: ' + enc.status + p ' option: ' + enc.option + p ' current: ' + enc.current p enc.is_8bit if !$RUBY_M17N - p tmp_rest.inspect - p tmp_rest + p ' inspect: ' + tmp_rest.inspect + p ' rest : [' + tmp_rest + ']' end - p line + p ' [' + line + ']' raise 'encoding is not consistent' end end @@ -240,6 +277,14 @@ def check_parentheses_balance(line, enc) return depth, count, tmp_prev, tmp_rest end +# PDFDocEncoding -> UTF-16BE +PDF2UNI = Array(0..255) +PDF2UNI[0o030..0o037] = 0x02d8, 0x02c7, 0x02c6, 0x02d9, 0x02dd, 0x02db, 0x02da, 0x02dc +PDF2UNI[0o200..0o207] = 0x2022, 0x2020, 0x2021, 0x2026, 0x2014, 0x2013, 0x0192, 0x2044 +PDF2UNI[0o210..0o217] = 0x2039, 0x203a, 0x2212, 0x2030, 0x201e, 0x201c, 0x201d, 0x2018 +PDF2UNI[0o220..0o227] = 0x2019, 0x201a, 0x2122, 0xfb01, 0xfb02, 0x0141, 0x0152, 0x0160 +PDF2UNI[0o230..0o237] = 0x0178, 0x017d, 0x0131, 0x0142, 0x0153, 0x0161, 0x017e, 0xfffd +PDF2UNI[0o240 ] = 0x20ac def conv_string_to_utf16be(line, enc) if line !~ /(\()(.*)(\))/m @@ -247,24 +292,30 @@ def conv_string_to_utf16be(line, enc) end pre, tmp, post = $`, $2, $' - if tmp.ascii_only? + if tmp.ascii_only? && tmp !~ /\\0x[0-9A-F]{4}/i return line end conv = '' - conv.force_encoding(enc.current) + conv.force_encoding('UTF-8') tmp.force_encoding(enc.current) + while tmp.length>0 do - if tmp =~ /\A[^\\]+/ - conv += $& - elsif tmp =~ /\\([0-3][0-7][0-7])/ - conv += $RUBY_M17N ? $&.oct.chr('ASCII-8BIT') : $&.oct.chr - elsif tmp =~ /\\([\r\n]{1,2})/ + case tmp + when /\A[^\\\n\r]+/ + conv += $&.to_utf8(enc) + when /\A\\([0-3][0-7][0-7])/ # PDFDocEncoding -> UTF-8 + conv += [PDF2UNI[$1.oct]].pack("U*") + when /\A\\0x(D[8-B][0-9A-F]{2})\\0x(D[C-F][0-9A-F]{2})/i # surrogate pair + conv += [$1.hex, $2.hex].pack("n*").utf16be_to_utf8 + when /\A\\0x([0-9A-F]{4})/i + conv += [$1.hex].pack("U*") + when /\A\\[nrtbf\\]/ + conv += eval(%!"#{$&}"!) + when /\A(\r\n|\r|\n)/ + conv += "\n" + when /\A\\([\r\n]{1,2})|\\/ # ignore - elsif tmp =~ /\\[nrtbf]/ - conv += eval($&) - elsif tmp =~ /\\(.)/ - conv += $1 else raise 'unexpected input!' end @@ -272,14 +323,8 @@ def conv_string_to_utf16be(line, enc) end buf = 'FEFF' # BOM for UTF-16BE - conv.each_char { |chr| - if chr == "\r" || chr == "\n" - buf += chr - else - chr.to_utf16be(enc).each_byte {|byte| - buf += '%02X' % byte - } - end + conv.utf8_to_utf16be.each_byte {|byte| + buf += '%02X' % byte } return pre + '<' + buf + '>' + post end @@ -341,12 +386,11 @@ else raise 'input file does not seem PS file' end fout = fin.gsub(/\.ps$/i, "-convbkmk#{$&}") - open(fin, 'r') {|ifile| - open(fout, 'w') {|ofile| + open(fin, 'rb') {|ifile| + open(fout, 'wb') {|ofile| file_treatment(ifile, ofile, enc) } } } end - -- cgit v1.2.3