summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/omega/otp
diff options
context:
space:
mode:
authorRoozbeh Pournader <roozbeh@sharif.edu>2006-06-26 12:40:23 +0000
committerRoozbeh Pournader <roozbeh@sharif.edu>2006-06-26 12:40:23 +0000
commit284d59ffe5ed6bb246286acc2aebb035c7daa2b5 (patch)
tree1e9100cce6ee6133230652da56627c302eed256c /Master/texmf-dist/omega/otp
parent808096204b5ef751309840ba3b459dd81dc90ad4 (diff)
Update OTPs and OCPs that handle UTF-8 to work with UTF-16 properly and reject invalid input.
git-svn-id: svn://tug.org/texlive/trunk@1726 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/omega/otp')
-rw-r--r--Master/texmf-dist/omega/otp/char2uni/inutf8.otp42
-rw-r--r--Master/texmf-dist/omega/otp/uni2char/oututf8.otp25
2 files changed, 58 insertions, 9 deletions
diff --git a/Master/texmf-dist/omega/otp/char2uni/inutf8.otp b/Master/texmf-dist/omega/otp/char2uni/inutf8.otp
index 174c465dc00..2bbab400c42 100644
--- a/Master/texmf-dist/omega/otp/char2uni/inutf8.otp
+++ b/Master/texmf-dist/omega/otp/char2uni/inutf8.otp
@@ -1,9 +1,11 @@
% File inutf8.otp
-% Conversion to Unicode from UTF-8
+% Conversion to UTF-16 from UTF-8
% Copyright (c) 1999 John Plaice and Yannis Haralambous
+% Copyright (C) 2005, 2006 Roozbeh Pournader
% This file is part of the Omega project.
%
-% The information was provided by Martin Duerst.
+% The information was originally provided by Martin Dürst.
+% Updated to incorporate exact definition of UTF-8 from Unicode 4.0.
%
input: 1;
@@ -11,10 +13,42 @@ output: 2;
expressions:
+% One-byte sequences get mapped to proper ASCII
@"00-@"7F
=> \1;
-(@"C0-@"DF)(@"80-@"BF)
+
+% Two-byte sequences:
+% U+0080..U+07FF
+(@"C2-@"DF)(@"80-@"BF)
=> #(((\1-@"C0)*@"40) + (\2-@"80));
-(@"E0-@"EF)(@"80-@"BF)(@"80-@"BF)
+
+% Three-byte sequences:
+% U+0800..U+0FFF
+@"E0(@"A0-@"BF)(@"80-@"BF)
+ => #(((\2-@"80)*@"40) + (\3-@"80));
+% U+1000..U+CFFF, U+E000..U+FFFF
+(@"E1-@"EC|@"EE-@"EF)(@"80-@"BF)(@"80-@"BF)
=> #(((\1-@"E0)*@"1000) + ((\2-@"80)*@"40) + (\3-@"80));
+% U+D000..U+D7FF
+@"ED(@"80-@"9F)(@"80-@"BF)
+ => #(@"D000 + ((\2-@"80)*@"40) + (\3-@"80));
+
+% Four-byte sequences:
+% The arithmatics is a little complex here, since the characters
+% are converted to UTF-16 directly.
+% U+10000..U+3FFFF
+@"F0(@"90-@"BF)(@"80-@"BF)(@"80-@"BF)
+ => #(@"D7C0 + ((\2-@"80)*4) + ((\3-@"80) div: @"10))
+ #(@"DC00 + ((\3 mod: 10)*@"40) + (\4 - @"80));
+% U+40000..U+FFFFF
+(@"F1-@"F3)(@"80-@"BF)(@"80-@"BF)(@"80-@"BF)
+ => #(@"D7C0 + ((\1-@"F0)*@"100) + ((\2-@"80)*4) + ((\3-@"80) div: @"10))
+ #(@"DC00 + ((\3 mod: 10)*@"40) + (\4 - @"80));
+% U+100000..U+10FFFF
+@"F4(@"80-@"8F)(@"80-@"BF)(@"80-@"BF)
+ => #(@"DBC0 + ((\2-@"80)*4) + ((\3-@"80) div: @"10))
+ #(@"DC00 + ((\3 mod: 10)*@"40) + (\4 - @"80));
+
+% Anything else gets mapped to U+FFFD. This should probably
+% get replaced by an error message, instead.
. => @"FFFD;
diff --git a/Master/texmf-dist/omega/otp/uni2char/oututf8.otp b/Master/texmf-dist/omega/otp/uni2char/oututf8.otp
index cb4d971f533..bc1ddd617fa 100644
--- a/Master/texmf-dist/omega/otp/uni2char/oututf8.otp
+++ b/Master/texmf-dist/omega/otp/uni2char/oututf8.otp
@@ -1,7 +1,7 @@
-
% File oututf8.otp
-% Conversion to UTF-8 from Unicode
+% Conversion to UTF-8 from UTF-16
% Copyright (c) 1999 John Plaice and Yannis Haralambous
+% Copyright (C) 2005, 2006 Roozbeh Pournader
% This file is part of the Omega project.
%
@@ -10,13 +10,28 @@ output: 1;
expressions:
-@"00-@"7F
+% One-byte sequences get mapped to proper ASCII
+@"0000-@"007F
=> \1;
-@"80-@"7FF
+
+% Two-byte sequences
+@"0080-@"07FF
=> #((\1 div: @"40)+@"C0)
#((\1 mod: @"40)+@"80);
-@"800-@"FFFF
+
+% Three-byte sequences
+(@"0800-@"D7FF|@"E000-@"FFFF)
=> #((\1 div: @"1000)+@"E0)
#(((\1 mod: @"1000) div: @"40)+@"80)
#((\1 mod: @"40)+@"80);
+
+% Surrogate pairs, which make four-byte sequences:
+(@"D800-@"DBFF)(@"DC00-@"DFFF)
+ => #(((((\1 mod: @"400) div: 4)+@"10) div: @"40)+@"F0)
+ #(((((\1 mod: @"400) div: 4)+@"10) mod: @"40)+@"80)
+ #(((\1 mod: 4)*@"10) +((\2 mod: @"400) div: @"40)+@"80)
+ #((\2 mod: @"40)+@"80);
+
+% Anything else gets mapped to 0x00. This should probably
+% get replaced by an error message, instead.
. => #(@"00);