%% %% This is file `l3str-enc-utf16.def', %% generated with the docstrip utility. %% %% The original source files were: %% %% l3str-convert.dtx (with options: `utf16') %% %% EXPERIMENTAL CODE %% %% Do not distribute this file without also distributing the %% source files specified above. %% %% Do not distribute a modified version of this file. %% %% File: l3str-convert.dtx Copyright (C) 2013 The LaTeX3 Project %% %% It may be distributed and/or modified under the conditions of the %% LaTeX Project Public License (LPPL), either version 1.3c of this %% license or (at your option) any later version. The latest version %% of this license is in the file %% %% http://www.latex-project.org/lppl.txt %% %% This file is part of the "l3experimental bundle" (The Work in LPPL) %% and all files in that bundle must be distributed together. %% %% The released version of this bundle is available from CTAN. %% %% ----------------------------------------------------------------------- %% %% The development version of the bundle can be found at %% %% http://www.latex-project.org/svnroot/experimental/trunk/ %% %% for those people who are interested. %% %%%%%%%%%%% %% NOTE: %% %%%%%%%%%%% %% %% Snapshots taken from the repository represent work in progress and may %% not work or may contain conflicting material! We therefore ask %% people _not_ to put them into distributions, archives, etc. without %% prior consultation with the LaTeX3 Project. %% %% ----------------------------------------------------------------------- \group_begin: \char_set_catcode_other:N \^^fe \char_set_catcode_other:N \^^ff \cs_new_protected_nopar:cpn { __str_convert_encode_utf16: } { \__str_encode_utf_xvi_aux:N \__str_output_byte_pair_be:n \tl_gput_left:Nx \g__str_result_tl { ^^fe ^^ff } } \cs_new_protected_nopar:cpn { __str_convert_encode_utf16be: } { \__str_encode_utf_xvi_aux:N \__str_output_byte_pair_be:n } \cs_new_protected_nopar:cpn { __str_convert_encode_utf16le: } { \__str_encode_utf_xvi_aux:N \__str_output_byte_pair_le:n } \cs_new_protected:Npn \__str_encode_utf_xvi_aux:N #1 { \flag_clear:n { str_error } \cs_set_eq:NN \__str_tmp:w #1 \__str_convert_gmap_internal:N \__str_encode_utf_xvi_char:n \__str_if_flag_error:nnx { str_error } { utf16-encode } { } } \cs_new:Npn \__str_encode_utf_xvi_char:n #1 { \if_int_compare:w #1 < "D800 \exp_stop_f: \__str_tmp:w {#1} \else: \if_int_compare:w #1 < "10000 \exp_stop_f: \if_int_compare:w #1 < "E000 \exp_stop_f: \flag_raise:n { str_error } \__str_tmp:w { \c__str_replacement_char_int } \else: \__str_tmp:w {#1} \fi: \else: \exp_args:Nf \__str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D7C0 } \exp_args:Nf \__str_tmp:w { \int_mod:nn {#1} {"400} + "DC00 } \fi: \fi: } \flag_clear_new:n { str_missing } \flag_clear_new:n { str_extra } \flag_clear_new:n { str_end } \__msg_kernel_new:nnnn { str } { utf16-encode } { Unicode~string~cannot~be~expressed~in~UTF-16:~surrogate. } { Surrogate~code~points~(in~the~range~[U+D800,~U+DFFF])~ can~be~expressed~in~the~UTF-8~and~UTF-32~encodings,~ but~not~in~the~UTF-16~encoding. } \__msg_kernel_new:nnnn { str } { utf16-decode } { Invalid~UTF-16~string: \exp_last_unbraced:Nf \use_none:n \__str_if_flag_times:nT { str_missing } { ,~missing~trail~surrogate } \__str_if_flag_times:nT { str_extra } { ,~extra~trail~surrogate } \__str_if_flag_times:nT { str_end } { ,~odd~number~of~bytes } . } { In~the~UTF-16~encoding,~each~Unicode~character~is~encoded~as~ 2~or~4~bytes: \\ \iow_indent:n { Code~point~in~[U+0000,~U+D7FF]:~two~bytes \\ Code~point~in~[U+D800,~U+DFFF]:~illegal \\ Code~point~in~[U+E000,~U+FFFF]:~two~bytes \\ Code~point~in~[U+10000,~U+10FFFF]:~ a~lead~surrogate~and~a~trail~surrogate \\ } Lead~surrogates~are~pairs~of~bytes~in~the~range~[0xD800,~0xDBFF],~ and~trail~surrogates~are~in~the~range~[0xDC00,~0xDFFF]. \flag_if_raised:nT { str_missing } { \\\\ A~lead~surrogate~was~not~followed~by~a~trail~surrogate. } \flag_if_raised:nT { str_extra } { \\\\ LaTeX~came~across~a~trail~surrogate~when~it~was~not~expected. } \flag_if_raised:nT { str_end } { \\\\ The~string~contained~an~odd~number~of~bytes.~This~is~invalid:~ the~basic~code~unit~for~UTF-16~is~16~bits~(2~bytes). } } \cs_new_protected_nopar:cpn { __str_convert_decode_utf16be: } { \__str_decode_utf_xvi:Nw 1 \g__str_result_tl \s__stop } \cs_new_protected_nopar:cpn { __str_convert_decode_utf16le: } { \__str_decode_utf_xvi:Nw 2 \g__str_result_tl \s__stop } \cs_new_protected_nopar:cpn { __str_convert_decode_utf16: } { \exp_after:wN \__str_decode_utf_xvi_bom:NN \g__str_result_tl \s__stop \s__stop \s__stop } \cs_new_protected:Npn \__str_decode_utf_xvi_bom:NN #1#2 { \str_if_eq_x:nnTF { #1#2 } { ^^ff ^^fe } { \__str_decode_utf_xvi:Nw 2 } { \str_if_eq_x:nnTF { #1#2 } { ^^fe ^^ff } { \__str_decode_utf_xvi:Nw 1 } { \__str_decode_utf_xvi:Nw 1 #1#2 } } } \cs_new_protected:Npn \__str_decode_utf_xvi:Nw #1#2 \s__stop { \flag_clear:n { str_error } \flag_clear:n { str_missing } \flag_clear:n { str_extra } \flag_clear:n { str_end } \cs_set:Npn \__str_tmp:w ##1 ##2 { ` ## #1 } \tl_gset:Nx \g__str_result_tl { \exp_after:wN \__str_decode_utf_xvi_pair:NN #2 \q_nil \q_nil \__prg_break_point: } \__str_if_flag_error:nnx { str_error } { utf16-decode } { } } \cs_new:Npn \__str_decode_utf_xvi_pair:NN #1#2 { \if_meaning:w \q_nil #2 \__str_decode_utf_xvi_pair_end:Nw #1 \fi: \if_case:w \__int_eval:w ( \__str_tmp:w #1#2 - "D6 ) / \c_four \__int_eval_end: \or: \exp_after:wN \__str_decode_utf_xvi_quad:NNwNN \or: \exp_after:wN \__str_decode_utf_xvi_extra:NNw \fi: #1#2 \s__tl \int_eval:n { "100 * \__str_tmp:w #1#2 + \__str_tmp:w #2#1 } \s__tl \__str_decode_utf_xvi_pair:NN } \cs_new:Npn \__str_decode_utf_xvi_quad:NNwNN #1#2 #3 \__str_decode_utf_xvi_pair:NN #4#5 { \if_meaning:w \q_nil #5 \__str_decode_utf_xvi_error:nNN { missing } #1#2 \__str_decode_utf_xvi_pair_end:Nw #4 \fi: \if_int_compare:w \if_int_compare:w \__str_tmp:w #4#5 < "DC \exp_stop_f: \c_zero = \c_one \else: \__str_tmp:w #4#5 < "E0 \exp_stop_f: \fi: #1 #2 #4 #5 \s__tl \int_eval:n { ( "100 * \__str_tmp:w #1#2 + \__str_tmp:w #2#1 - "D7F7 ) * "400 + "100 * \__str_tmp:w #4#5 + \__str_tmp:w #5#4 } \s__tl \exp_after:wN \use_i:nnn \else: \__str_decode_utf_xvi_error:nNN { missing } #1#2 \fi: \__str_decode_utf_xvi_pair:NN #4#5 } \cs_new:Npn \__str_decode_utf_xvi_pair_end:Nw #1 \fi: { \fi: \if_meaning:w \q_nil #1 \else: \__str_decode_utf_xvi_error:nNN { end } #1 \prg_do_nothing: \fi: \__prg_break: } \cs_new:Npn \__str_decode_utf_xvi_extra:NNw #1#2 \s__tl #3 \s__tl { \__str_decode_utf_xvi_error:nNN { extra } #1#2 } \cs_new:Npn \__str_decode_utf_xvi_error:nNN #1#2#3 { \flag_raise:n { str_error } \flag_raise:n { str_#1 } #2 #3 \s__tl \int_use:N \c__str_replacement_char_int \s__tl } \group_end: %% %% %% End of file `l3str-enc-utf16.def'.