Update OTPs and OCPs that handle UTF-8 to work with UTF-16 properly and reject invalid input.

git-svn-id: svn://tug.org/texlive/trunk@1726 c570f23f-e606-0410-a88d-b1316a301751

4 files changed, 58 insertions, 9 deletions

diff --git a/Master/texmf-dist/omega/ocp/char2uni/inutf8.ocp b/Master/texmf-dist/omega/ocp/char2uni/inutf8.ocp
index f03b702f43e..92f9886b79f 100644
--- a/Master/texmf-dist/omega/ocp/char2uni/inutf8.ocp
+++ b/Master/texmf-dist/omega/ocp/char2uni/inutf8.ocp
diff --git a/Master/texmf-dist/omega/ocp/uni2char/oututf8.ocp b/Master/texmf-dist/omega/ocp/uni2char/oututf8.ocp
index fbbafb732ba..5a2fc2b507b 100644
--- a/Master/texmf-dist/omega/ocp/uni2char/oututf8.ocp
+++ b/Master/texmf-dist/omega/ocp/uni2char/oututf8.ocp
diff --git a/Master/texmf-dist/omega/otp/char2uni/inutf8.otp b/Master/texmf-dist/omega/otp/char2uni/inutf8.otp
index 174c465dc00..2bbab400c42 100644
--- a/Master/texmf-dist/omega/otp/char2uni/inutf8.otp
+++ b/Master/texmf-dist/omega/otp/char2uni/inutf8.otp
@@ -1,9 +1,11 @@
 % File inutf8.otp
-% Conversion to Unicode from UTF-8 
+% Conversion to UTF-16 from UTF-8 
 % Copyright (c) 1999 John Plaice and Yannis Haralambous
+% Copyright (C) 2005, 2006 Roozbeh Pournader
 % This file is part of the Omega project.
 %
-% The information was provided by Martin Duerst.
+% The information was originally provided by Martin Dürst.
+% Updated to incorporate exact definition of UTF-8 from Unicode 4.0.
 %
 
 input:  1;
@@ -11,10 +13,42 @@ output: 2;
 
 expressions:
 
+% One-byte sequences get mapped to proper ASCII
 @"00-@"7F
   => \1;
-(@"C0-@"DF)(@"80-@"BF)
+
+% Two-byte sequences:
+% U+0080..U+07FF
+(@"C2-@"DF)(@"80-@"BF)
   => #(((\1-@"C0)*@"40) + (\2-@"80));
-(@"E0-@"EF)(@"80-@"BF)(@"80-@"BF)
+
+% Three-byte sequences:
+% U+0800..U+0FFF
+@"E0(@"A0-@"BF)(@"80-@"BF)
+  => #(((\2-@"80)*@"40) + (\3-@"80));
+% U+1000..U+CFFF, U+E000..U+FFFF
+(@"E1-@"EC|@"EE-@"EF)(@"80-@"BF)(@"80-@"BF)
   => #(((\1-@"E0)*@"1000) + ((\2-@"80)*@"40) + (\3-@"80));
+% U+D000..U+D7FF
+@"ED(@"80-@"9F)(@"80-@"BF)
+  => #(@"D000 + ((\2-@"80)*@"40) + (\3-@"80));
+
+% Four-byte sequences:
+% The arithmatics is a little complex here, since the characters
+% are converted to UTF-16 directly.
+% U+10000..U+3FFFF
+@"F0(@"90-@"BF)(@"80-@"BF)(@"80-@"BF)
+  => #(@"D7C0 + ((\2-@"80)*4) + ((\3-@"80) div: @"10))
+     #(@"DC00 + ((\3 mod: 10)*@"40) + (\4 - @"80));
+% U+40000..U+FFFFF
+(@"F1-@"F3)(@"80-@"BF)(@"80-@"BF)(@"80-@"BF)
+  => #(@"D7C0 + ((\1-@"F0)*@"100) + ((\2-@"80)*4) + ((\3-@"80) div: @"10))
+     #(@"DC00 + ((\3 mod: 10)*@"40) + (\4 - @"80));
+% U+100000..U+10FFFF
+@"F4(@"80-@"8F)(@"80-@"BF)(@"80-@"BF)
+  => #(@"DBC0 + ((\2-@"80)*4) + ((\3-@"80) div: @"10))
+     #(@"DC00 + ((\3 mod: 10)*@"40) + (\4 - @"80));
+
+% Anything else gets mapped to U+FFFD. This should probably
+% get replaced by an error message, instead.
 . => @"FFFD;
diff --git a/Master/texmf-dist/omega/otp/uni2char/oututf8.otp b/Master/texmf-dist/omega/otp/uni2char/oututf8.otp
index cb4d971f533..bc1ddd617fa 100644
--- a/Master/texmf-dist/omega/otp/uni2char/oututf8.otp
+++ b/Master/texmf-dist/omega/otp/uni2char/oututf8.otp
@@ -1,7 +1,7 @@
-
 % File oututf8.otp
-% Conversion to UTF-8 from Unicode
+% Conversion to UTF-8 from UTF-16
 % Copyright (c) 1999 John Plaice and Yannis Haralambous
+% Copyright (C) 2005, 2006 Roozbeh Pournader
 % This file is part of the Omega project.
 %
 
@@ -10,13 +10,28 @@ output: 1;
 
 expressions:
 
-@"00-@"7F
+% One-byte sequences get mapped to proper ASCII
+@"0000-@"007F
   => \1;
-@"80-@"7FF
+
+% Two-byte sequences
+@"0080-@"07FF
   => #((\1 div: @"40)+@"C0)
      #((\1 mod: @"40)+@"80);
-@"800-@"FFFF
+
+% Three-byte sequences
+(@"0800-@"D7FF|@"E000-@"FFFF)
   => #((\1 div: @"1000)+@"E0)
      #(((\1 mod: @"1000) div: @"40)+@"80)
      #((\1 mod: @"40)+@"80);
+
+% Surrogate pairs, which make four-byte sequences:
+(@"D800-@"DBFF)(@"DC00-@"DFFF)
+  => #(((((\1 mod: @"400) div: 4)+@"10) div: @"40)+@"F0)
+     #(((((\1 mod: @"400) div: 4)+@"10) mod: @"40)+@"80)
+     #(((\1 mod: 4)*@"10) +((\2 mod: @"400) div: @"40)+@"80)
+     #((\2 mod: @"40)+@"80);
+
+% Anything else gets mapped to 0x00. This should probably
+% get replaced by an error message, instead.
 . => #(@"00);