diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx | 3692 |
1 files changed, 1956 insertions, 1736 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx index cddd7349adc..3d73ed10b8f 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx @@ -35,13 +35,14 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 3159 2012-01-09 00:51:58Z bruno $ +\GetIdInfo$Id: l3str.dtx 3257 2012-01-29 13:37:03Z bruno $ {L3 Experimental Strings} %</driver|package> %<*driver> \documentclass[full]{l3doc} \usepackage{amsmath} \begin{document} + \tableofcontents \DocInput{\jobname.dtx} \end{document} %</driver> @@ -75,25 +76,23 @@ % characters. % % String variables are simply specialised token lists, but by convention -% should be named with the suffix \texttt{\ldots str}. Such variables should -% contain characters with category code $12$ (other), except spaces, which -% have category code $10$ (blank space). All the functions in -% this module first convert their argument to a string for internal processing, -% and will not treat a token list or the corresponding string representation -% differently. +% should be named with the suffix \texttt{\ldots str}. Such variables +% should contain characters with category code $12$ (other), except +% spaces, which have category code $10$ (blank space). All the functions +% in this module first convert their argument to a string for internal +% processing, and will not treat a token list or the corresponding +% string representation differently. % % Most functions in this module come in three flavours: % \begin{itemize} % \item \cs{str_...:N...}, which expect a token list or string % variable as their argument; -% \item \cs{str_...:n...}, taking any token list (or string) as an argument; -% \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered -% during the operation: these functions are faster than those -% which take care of escaping spaces appropriately; +% \item \cs{str_...:n...}, taking any token list (or string) as an +% argument; +% \item \cs{str_..._ignore_spaces:n...}, which ignores any space +% encountered during the operation: these functions are faster than +% those which take care of escaping spaces appropriately; % \end{itemize} -% When performance is critical, the internal \cs{str_..._unsafe:n...} -% functions, which expect an \enquote{other string} in which spaces -% have category code $12$ instead of $10$, might be useful. % % \section{Building strings} % @@ -106,8 +105,9 @@ % \c_tilde_str, % \c_percent_str % } -% Constant strings, containing a single character, with category code $12$. -% Any character can be accessed as \cs{iow_char:N} |\|\meta{character}. +% Constant strings, containing a single character, with category code +% $12$. Any character can be accessed as \cs{iow_char:N} +% |\|\meta{character}. % \end{variable} % % \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n} @@ -115,28 +115,36 @@ % \cs{tl_to_str:N} \meta{tl var} % \cs{tl_to_str:n} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, leaving the resulting -% tokens in the input stream. +% Converts the \meta{token list} to a \meta{string}, leaving the +% resulting tokens in the input stream. % \begin{texnote} -% The string representation of a token list may depend -% on the category codes in effect when it is measured, -% and the value of the \tn{escapechar}: for instance -% |\tl_to_str:n {\a}| may produce $1$, $2$ or $3$ -% characters depending on the escape character, and -% the category code of \texttt{a}. This impacts almost -% all functions in the module, which use \cs{tl_to_str:n} -% internally. +% The string representation of a token list may depend on the +% category codes in effect when it is evaluated, and the value of +% the \tn{escapechar}: for instance |\tl_to_str:n {\a}| normally +% produces the three character \enquote{backslash}, +% \enquote{lower-case a}, \enquote{space}, but it may also produce +% $1$ or $2$ characters depending on the escape character, and the +% category code of \texttt{a}. This impacts almost all functions in +% the module, which use \cs{tl_to_str:n} internally. % \end{texnote} % \end{function} % +% \begin{function}{\str_new:N, \str_new:c} +% \begin{syntax} +% \cs{str_new:N} \meta{str~var} +% \end{syntax} +% Creates a new \meta{str~var} or raises an error if the name is +% already taken. The declaration is global. The \meta{str~var} will +% initially be empty. +% \end{function} +% % \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx} % \begin{syntax} % \cs{str_const:Nn} \meta{str~var} \Arg{token list} % \end{syntax} -% Creates a new constant \meta{str~var} or raises an error -% if the name is already taken. The value of the -% \meta{str~var} will be set globally to the -% \meta{token list}, converted to a string. +% Creates a new constant \meta{str~var} or raises an error if the name +% is already taken. The value of the \meta{str~var} will be set +% globally to the \meta{token list}, converted to a string. % \end{function} % % \begin{function} @@ -147,8 +155,8 @@ % \begin{syntax} % \cs{str_set:Nn} \meta{str var} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and stores the result in \meta{str var}. +% Converts the \meta{token list} to a \meta{string}, and stores the +% result in \meta{str var}. % \end{function} % % \begin{function} @@ -161,10 +169,9 @@ % \begin{syntax} % \cs{str_put_left:Nn} \meta{str var} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and prepends the result to \meta{str var}. -% The current contents of the \meta{str var} are not -% automatically converted to a string. +% Converts the \meta{token list} to a \meta{string}, and prepends the +% result to \meta{str var}. The current contents of the \meta{str +% var} are not automatically converted to a string. % \end{function} % % \begin{function} @@ -177,50 +184,9 @@ % \begin{syntax} % \cs{str_put_right:Nn} \meta{str var} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and appends the result to \meta{str var}. -% The current contents of the \meta{str var} are not -% automatically converted to a string. -% \end{function} -% -% \begin{function}{\str_input:Nn, \str_ginput:Nn} -% \begin{syntax} -% \cs{str_input:Nn} \meta{str var} \Arg{token list} -% \end{syntax} -% Converts the \meta{token list} into a \meta{string}, and stores -% it in the \meta{str var}. -% Special characters can be input by -% escaping them with a backslash. -% \begin{itemize} -% \item Spaces are ignored unless escaped with a backslash. -% \item |\xhh| produces the character with code \texttt{hh} -% in hexadecimal: when |\x| is encountered, up to two hexadecimal digits -% (\texttt{0}--\texttt{9}, \texttt{a}--\texttt{f}, -% \texttt{A}--\texttt{F}) -% are read to give a number between $0$ and $255$. -% \item |\x{hh...}| produces the character with code \texttt{hh...} -% (an arbitrary number of hexadecimal digits are read): -% this is mostly useful for Unicode-aware engines. -% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters: -% \begin{center} -% \begin{tabular}{cccc} -% |\a| & |\^^G| & alarm & hex \texttt{07} \\ -% |\e| & |\^^[| & escape & hex \texttt{1B} \\ -% |\f| & |\^^L| & form feed & hex \texttt{0C} \\ -% |\n| & |\^^J| & new line & hex \texttt{0A} \\ -% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\ -% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\ -% \end{tabular} -% \end{center} -% \end{itemize} -% For instance, -% \begin{quote} -% \cs{tl_new:N} \cs{l_my_str} \\ -% \cs{str_input:Nn} \cs{l_my_str} |{\x3C \\ \# abc\ def\^\n}| -% \end{quote} -% results in \cs{l_my_str} containing the characters |<\#abc def^|, -% followed by a newline character (hex \texttt{0A}) -% since |<| has \textsc{ascii} code \texttt{3C} (in hexadecimal). +% Converts the \meta{token list} to a \meta{string}, and appends the +% result to \meta{str var}. The current contents of the \meta{str +% var} are not automatically converted to a string. % \end{function} % % \section{Characters given by their position} @@ -231,63 +197,67 @@ % \cs{str_length:n} \Arg{token list} % \end{syntax} % Leaves the length of the string representation of \meta{token list} -% in the input stream as an integer denotation. The functions differ in -% their treatment of spaces. -% In the case of \cs{str_length:N} and \cs{str_length:n}, -% all characters including spaces are counted. -% The \cs{str_length_ignore_spaces:n} leaves the number of non-space +% in the input stream as an integer denotation. The functions differ +% in their treatment of spaces. In the case of \cs{str_length:N} and +% \cs{str_length:n}, all characters including spaces are counted. The +% \cs{str_length_ignore_spaces:n} leaves the number of non-space % characters in the input stream. % \end{function} % +% \begin{function}[EXP]{\str_count_spaces:N, \str_count_spaces:n} +% \begin{syntax} +% \cs{str_count_spaces:n} \Arg{token list} +% \end{syntax} +% Leaves in the input stream the number of space characters in the +% string representation of \meta{token list}, as an integer +% denotation. Of course, this function has no \texttt{_ignore_spaces} +% variant. +% \end{function} +% % \begin{function}[EXP]{\str_head:N, \str_head:n, \str_head_ignore_spaces:n} % \begin{syntax} % \cs{str_head:n} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} into a \meta{string}. -% The first character in the \meta{string} is then -% left in the input stream, with category code \enquote{other}. -% The functions differ in their treatment of spaces. -% In the case of \cs{str_head:N} and \cs{str_head:n}, -% a leading space is returned with category code $10$ (blank space). -% The \cs{str_head_ignore_spaces:n} function leaves the first -% non-space character in the input stream. -% If the \meta{token list} is empty (or blank in the case of the -% \texttt{_ignore_spaces} variant), then nothing is left on the -% input stream. +% Converts the \meta{token list} into a \meta{string}. The first +% character in the \meta{string} is then left in the input stream, +% with category code \enquote{other}. The functions differ in their +% treatment of spaces. In the case of \cs{str_head:N} and +% \cs{str_head:n}, a leading space is returned with category code $10$ +% (blank space). The \cs{str_head_ignore_spaces:n} function leaves +% the first non-space character in the input stream. If the +% \meta{token list} is empty (or blank in the case of the +% \texttt{_ignore_spaces} variant), then nothing is left on the input +% stream. % \end{function} % % \begin{function}[EXP]{\str_tail:N, \str_tail:n, \str_tail_ignore_spaces:n} % \begin{syntax} % \cs{str_tail:n} \Arg{token list} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% removes the first character, and leaves -% the remaining characters (if any) in the input stream, -% with category codes $12$ and $10$ (for spaces). -% The functions differ in the case where the first character -% is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim -% only that space, while \cs{str_tail_ignore_spaces:n} removes -% the first non-space character and any space before it. -% If the \meta{token list} is empty (or blank in the case of the -% \texttt{_ignore_spaces} variant), then nothing is left on the -% input stream. +% Converts the \meta{token list} to a \meta{string}, removes the first +% character, and leaves the remaining characters (if any) in the input +% stream, with category codes $12$ and $10$ (for spaces). The +% functions differ in the case where the first character is a space: +% \cs{str_tail:N} and \cs{str_tail:n} will trim only that space, while +% \cs{str_tail_ignore_spaces:n} removes the first non-space character +% and any space before it. If the \meta{token list} is empty (or +% blank in the case of the \texttt{_ignore_spaces} variant), then +% nothing is left on the input stream. % \end{function} % % \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn} % \begin{syntax} % \cs{str_item:nn} \Arg{token list} \Arg{integer expression} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, and -% leaves in the input stream the character in position -% \meta{integer expression} of the \meta{string}. -% In the case of \cs{str_item:Nn} and \cs{str_item:nn}, -% all characters including spaces are taken into account. -% The \cs{str_item_ignore_spaces:nn} function skips spaces -% in its argument. -% If the \meta{integer expression} is negative, characters -% are counted from the end of the \meta{string}. Hence, $-1$ is -% the right-most character, \emph{etc.}, while $0$ is the first -% (left-most) character. +% Converts the \meta{token list} to a \meta{string}, and leaves in the +% input stream the character in position \meta{integer expression} of +% the \meta{string}. In the case of \cs{str_item:Nn} and +% \cs{str_item:nn}, all characters including spaces are taken into +% account. The \cs{str_item_ignore_spaces:nn} function skips spaces +% in its argument. If the \meta{integer expression} is negative, +% characters are counted from the end of the \meta{string}. Hence, +% $-1$ is the right-most character, \emph{etc.}, while $0$ is the +% first (left-most) character. % \end{function} % % \begin{function}[EXP] @@ -295,16 +265,16 @@ % \begin{syntax} % \cs{str_substr:nnn} \Arg{token list} \Arg{start index} \Arg{end index} % \end{syntax} -% Converts the \meta{token list} to a \meta{string}, -% and leaves in the input stream the characters -% between \meta{start index} (inclusive) and \meta{end index} (exclusive). +% Converts the \meta{token list} to a \meta{string}, and leaves in the +% input stream the characters from the \meta{start index} inclusive to +% the \meta{end index} exclusive. Note that the length of the +% substring is equal to the difference between the two \meta{indices}. % If either of \meta{start index} or \meta{end index} is negative, -% then it is incremented by the length of the list. -% If either of \meta{start index} or \meta{end index} is empty, -% it is replaced by the corresponding end-point of the string. -% Both \meta{start index} and \meta{end index} count from $0$ -% for the first (left most) character. -% For instance, +% then it is incremented by the length of the list. If either of +% \meta{start index} or \meta{end index} is empty, it is replaced by +% the corresponding end-point of the string. Both \meta{start index} +% and \meta{end index} count from $0$ for the first (left most) +% character. For instance, % \begin{verbatim} % \iow_term:x { \str_substr:nnn { abcdef } { 1 } { 4 } } % \iow_term:x { \str_substr:nnn { abcdef } { -4 } { } } @@ -330,8 +300,8 @@ % \begin{verbatim} % \str_if_eq_p:xx { abc } { \tl_to_str:n { abc } } % \end{verbatim} -% is logically \texttt{true}. All versions of these functions are fully -% expandable (including those involving an \texttt{x}-type +% is logically \texttt{true}. All versions of these functions are +% fully expandable (including those involving an \texttt{x}-type % expansion). % \end{function} % @@ -340,29 +310,31 @@ % Traditionally, string encodings only specify how strings of characters % should be stored as bytes. However, the resulting lists of bytes are % often to be used in contexts where only a restricted subset of bytes -% are permitted (\textsc{pdf} string objects, \textsc{url}s). -% Hence, storing strings of characters has two components. +% are permitted (\emph{e.g.}, \textsc{pdf} string objects, +% \textsc{url}s). Hence, storing a string of characters is done in two +% steps. % \begin{itemize} -% \item The code points (\enquote{character codes}) -% are expressed as bytes following a given -% \enquote{encoding}. This can be \textsc{utf-16}, -% \textsc{iso 8859-1}, \emph{etc.} -% See Table~\ref{tab:encodings} for a list -% of encodings supported.\footnote{Encodings and escapings -% will be added as they are requested.} -% \item Bytes are translated to \TeX{} tokens through -% a given \enquote{escaping}. Those are defined for -% the most part by the \texttt{pdf} file format. -% See Table~\ref{tab:escapings} for a list -% of escaping methods supported.\footnotemark +% \item The code points (\enquote{character codes}) are expressed as +% bytes following a given \enquote{encoding}. This can be +% \textsc{utf-16}, \textsc{iso 8859-1}, \emph{etc.} See +% Table~\ref{tab:encodings} for a list of supported +% encodings.\footnote{Encodings and escapings will be added as they +% are requested.} +% \item Bytes are translated to \TeX{} tokens through a given +% \enquote{escaping}. Those are defined for the most part by the +% \texttt{pdf} file format. See Table~\ref{tab:escapings} for a +% list of escaping methods supported.\footnotemark % \end{itemize} % % \begin{table}\centering % \caption{\label{tab:encodings}Supported encodings. % Non-alphanumeric characters are ignored, -% and capital letters are lower-cased.} +% and capital letters are lower-cased +% before searching for the encoding in this list.} % \begin{tabular}{cc} % \toprule +% \meta{Encoding} & description \\ +% \midrule % \texttt{utf8} & \textsc{utf-8} \\ % \texttt{utf16} & \textsc{utf-16}, with byte-order mark \\ % \texttt{utf16be} & \textsc{utf-16}, big-endian \\ @@ -386,8 +358,7 @@ % \texttt{iso885915}, \texttt{latin9} & \textsc{iso 8859-15} \\ % \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\ % \midrule -% \texttt{native} & Native Unicode string. \\ -% \texttt{internal} & For curious programmers. \\ +% Empty & Native (Unicode) string. \\ % \bottomrule % \end{tabular} % \end{table} @@ -395,9 +366,12 @@ % \begin{table}\centering % \caption{\label{tab:escapings}Supported escapings. % Non-alphanumeric characters are ignored, -% and capital letters are lower-cased.} +% and capital letters are lower-cased +% before searching for the escaping in this list.} % \begin{tabular}{cc} % \toprule +% \meta{Escaping} & description \\ +% \midrule % \texttt{bytes}, or empty % & arbitrary bytes \\ % \texttt{hex}, \texttt{hexadecimal} @@ -412,86 +386,91 @@ % \end{tabular} % \end{table} % -% \begin{function}{\str_set_convert:Nnnn} +% \begin{function}{\str_set_convert:Nnnn,\str_gset_convert:Nnnn} % \begin{syntax} -% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string} -% ~~\Arg{name 1} \Arg{name 2} +% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string} \Arg{name~1} \Arg{name~2} % \end{syntax} -% This function converts the \meta{string} from the encoding -% given by \meta{name 1} to the encoding given by \meta{name 2}, -% and stores the result in the \meta{str~var}. -% Each \meta{name} can have the form \meta{encoding} or -% \meta{encoding}\texttt{/}\meta{escaping}, where the possible values -% of \meta{encoding} and \meta{escaping} are given in -% Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively. -% The default escaping is to input and output bytes directly. -% The special encodings \texttt{native} and \texttt{internal} -% do not support a \meta{escaping}, since those formats do not -% correspond to strings of bytes. +% This function converts the \meta{string} from the encoding given by +% \meta{name~1} to the encoding given by \meta{name~2}, and stores the +% result in the \meta{str~var}. Each \meta{name} can have the form +% \meta{encoding} or \meta{encoding}\texttt{/}\meta{escaping}, where +% the possible values of \meta{encoding} and \meta{escaping} are given +% in Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively. +% The default escaping is to input and output bytes directly. The +% special case of an empty \meta{name} indicates the use of +% \enquote{native} strings, 8-bit for pdf\TeX{}, and Unicode strings +% for the other two engines. % % For example, % \begin{verbatim} -% \str_set_convert:Nnnn \l_foo_str { Hello! } -% { native } { utf16/hex } +% \str_set_convert:Nnnn \l_foo_str { Hello! } { } { utf16/hex } % \end{verbatim} % results in the variable \cs{l_foo_str} holding the string -% \texttt{FEFF00480065006C006C006F0021}. This is obtained -% by converting each character in the string \texttt{Hello!} -% to the \textsc{utf-16} encoding, and expressing each byte -% as a pair of hexadecimal digits. Note the presence of a -% (big-endian) byte order mark \hexnum{FEFF}, which can be -% avoided by specifying the encoding \texttt{utf16be/hex}. -% -% Parts of the \meta{string} which cannot be converted to bytes -% using the \meta{escaping 1} are silently ignored (including non-byte -% characters which could be present in the string). -% Sequences of bytes which are not valid in the \meta{encoding 1} -% are converted to the replacement character \hexnum{FFFD}, and raise -% an error. -% Characters which cannot be encoded in the \meta{encoding 2} -% are silently ignored. +% \texttt{FEFF00480065006C006C006F0021}. This is obtained by +% converting each character in the (native) string \texttt{Hello!} to +% the \textsc{utf-16} encoding, and expressing each byte as a pair of +% hexadecimal digits. Note the presence of a (big-endian) byte order +% mark \hexnum{FEFF}, which can be avoided by specifying the encoding +% \texttt{utf16be/hex}. +% +% An error is raised if the \meta{string} is not valid according to +% the \meta{escaping~1} and \meta{encoding~1}, or if it cannot be +% reencoded in the \meta{encoding~2} and \meta{escaping~2} (for +% instance, if a character does not exist in the \meta{encoding~2}). +% Erroneous input is replaced by the Unicode replacement character +% \hexnum{FFFD}, and characters which cannot be reencoded are replaced +% by either the replacement character \hexnum{FFFD} if it exists in +% the \meta{encoding~2}, or an encoding-specific replacement +% character, or the question mark character. % \end{function} % -% \section{Internal string functions} -% -% \begin{function}{\str_escape_use:NNNn} +% \begin{function}[TF]{\str_set_convert:Nnnn,\str_gset_convert:Nnnn} % \begin{syntax} -% \cs{str_escape_use:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list} +% \cs{str_set_convert:NnnnTF} \meta{str~var} \Arg{string} \Arg{name~1} \Arg{name~2} \Arg{true code} \Arg{false code} % \end{syntax} -% The \meta{token list} is converted to a string, then read from -% left to right, interpreting backslashes as escaping the next character. -% Unescaped characters are fed to the function \meta{fn1}, -% and escaped characters are fed to the function \meta{fn2} -% within an \texttt{x}-expansion context (typically those functions -% perform some tests on their argument to decide how to output them). -% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x| -% are recognized as described for \cs{str_input:Nn}, and those -% are replaced by the corresponding character, then fed to -% \meta{fn3}. The result is then left in the input stream. +% As \cs{str_set_convert:Nnnn}, converts the \meta{string} from the +% encoding given by \meta{name~1} to the encoding given by +% \meta{name~2}, and assigns the result to \meta{str~var}. Contrarily +% to \cs{str_set_convert:Nnnn}, the conditional variant does not raise +% errors in case the \meta{string} is not valid according to the +% \meta{name~1} encoding, or cannot be expressed in the \meta{name~2} +% encoding. Instead, the \meta{false code} is performed. % \end{function} % -% \begin{function}[rEXP]{\str_aux_toks_range:nn} -% \begin{syntax} -% \cs{str_aux_toks_range:nn} \meta{start} \meta{end} -% \end{syntax} -% Expands to the contents of the \tn{toks} registers numbered -% from \meta{start} (inclusive) to \meta{end} (exclusive). -% This function is used in \pkg{l3regex}. -% \end{function} +% \section{Internal string functions} +% +% \begin{variable}{\c_max_char_int} +% The maximum valid character code, $255$ for pdf\TeX{}, and $1114111$ +% for the two other engines. +% \end{variable} % % \begin{function}{\str_aux_hexadecimal_use:NTF} % \begin{syntax} -% \cs{str_aux_hexadecimal_use:NTF} \meta{token} -% ~~\Arg{true code} \Arg{false code} +% \cs{str_aux_hexadecimal_use:NTF} \meta{token} \Arg{true code} \Arg{false code} % \end{syntax} -% If the \meta{token} is a hexdecimal digit (upper case or lower case), -% its upper-case version is left in the input stream, \emph{followed} -% by the \meta{true code}. Otherwise, the \meta{false code} is left -% in the input stream. +% If the \meta{token} is a hexdecimal digit (upper case or lower +% case), its upper-case version is left in the input stream, +% \emph{followed} by the \meta{true code}. Otherwise, the \meta{false +% code} is left in the input stream. % \begin{texnote} % This function will fail if the escape character is a hexadecimal % digit, or has a character code in the range $[0,5]$. We are thus -% careful to set the escape character to a known value before using it. +% careful to set the escape character to a known value before using +% it. +% \end{texnote} +% \end{function} +% +% \begin{function}{\str_aux_octal_use:NTF} +% \begin{syntax} +% \cs{str_aux_octal_use:NTF} \meta{token} \Arg{true code} \Arg{false code} +% \end{syntax} +% If the \meta{token} is an octal digit, it is left in the input +% stream, \emph{followed} by the \meta{true code}. Otherwise, the +% \meta{false code} is left in the input stream. +% \begin{texnote} +% This function will fail if the escape character is an octal +% digit. We are thus careful to set the escape character to a known +% value before using it. % \end{texnote} % \end{function} % @@ -502,9 +481,10 @@ % Converts the \meta{token list} to a \meta{other string}, where % spaces have category code \enquote{other}. % \begin{texnote} -% These functions can be \texttt{f}-expanded without fear of losing -% a leading space, since spaces do not have category code $10$ in -% their result. +% This function can be \texttt{f}-expanded without fear of losing a +% leading space, since spaces do not have category code $10$ in its +% result. This function takes a time quadratic in the length of the +% string; \cs{str_gset_other:Nn} is faster but not expandable. % \end{texnote} % \end{function} % @@ -513,8 +493,8 @@ % \cs{str_gset_other:Nn} \meta{tl~var} \Arg{token list} % \end{syntax} % Converts the \meta{token list} to an \meta{other string}, where -% spaces have category code \enquote{other}, and assigns the result -% to the \meta{tl~var}, globally. +% spaces have category code \enquote{other}, and assigns the result to +% the \meta{tl~var}, globally. % \end{function} % % \begin{function}[EXP] @@ -526,24 +506,56 @@ % \begin{syntax} % \cs{str_if_contains_char:nNTF} \Arg{token list} \meta{char} % \end{syntax} -% Converts the \meta{token list} to a \meta{string} -% and tests whether the \meta{char} is present in the \meta{string}. -% Spaces are ignored. +% Converts the \meta{token list} to a \meta{string} and tests whether +% the \meta{char} is present in the \meta{string}. Spaces are +% ignored. +% \end{function} +% +% \begin{function}{\str_declare_eight_bit_encoding:nnn} +% \begin{syntax} +% \cs{str_declare_eight_bit_encoding:nnn} \Arg{name} \Arg{mapping} \Arg{missing} +% \end{syntax} +% This declares the encoding \meta{name} to map bytes to Unicode +% characters according to the \meta{mapping}, and map those bytes +% which are not mentionned in the \meta{mapping} either to the +% replacement character (if they appear in \meta{missing}), or to +% themselves. % \end{function} % % \section{Possibilities, and things to do} % +% Encoding/escaping-related tasks. +% \begin{itemize} +% \item Describe the internal format in the code comments. Refuse code +% points in $[\hexnum{D800}, \hexnum{DFFF}]$ in the internal +% representation? +% \item Add documentation about each encoding and escaping method, and +% add examples. +% \item The \texttt{hex} unescaping should raise an error for +% odd-length strings. +% \item Decide what bytes should be escaped in the \texttt{url} +% escaping. Perhaps |!'()*-./0123456789_| are safe, and all other +% characters should be escaped? +% \item Automate generation of 8-bit mapping files. +% \item Change the framework for 8-bit encodings: for decoding from +% 8-bit to Unicode, use $256$ integer registers; for encoding, use a +% tree-box. +% \item More encodings (see Heiko's \pkg{stringenc}). CESU? +% \item More escapings: shell escapes, lua escapes, etc? +% \end{itemize} +% +% Other string tasks. % \begin{itemize} -% \item Use flags rather than expandable errors. -% \item Add documentation about each encoding and escaping method, -% and add examples. -% \item Perhaps use the pdf\TeX{} primitives for escapings when available. -% But this changes error detection between engines. -% \item More encodings (see Heiko's \pkg{stringenc}). CESU. -% \item More escapings: shell escapes, lua escapes, etc? -% \item \cs{str_if_head_eq:nN} -% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}? -% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}? +% \item \cs{str_if_head_eq:nN} +% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}? +% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}? +% \item Should \cs{str_substr:Nnn} be \cs{str_range:Nnn}? +% \item Introduce \cs{str_slice:Nnnn} with a third \enquote{step} +% argument? Or should we simply have \cs{str_slice:Nn} +% \meta{string} \Arg{clist}, where the \meta{clist}'s items are +% either one integer expression, two integer expressions separated +% by |:|, or three integer expressions separated by |:|, \emph{cf.} +% Python's extended slice syntax? % \end{itemize} % % \end{documentation} @@ -559,13 +571,14 @@ % \begin{macrocode} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3tl-analysis} +\RequirePackage{l3tl-analysis,l3flag} % \end{macrocode} % -% Those string-related functions are defined in \pkg{l3kernel}. +% The following string-related functions are currently defined in +% \pkg{l3kernel}. % \begin{itemize} % \item \cs{str_if_eq:nn}[pTF] and variants, -% \item \cs{str_if_eq_return:on}, +% \item \cs{str_if_eq_return:on}, \cs{str_if_eq_return:xx} % \item \cs{tl_to_str:n}, \cs{tl_to_str:N}, \cs{tl_to_str:c}, % \item \cs{token_to_str:N}, \cs{cs_to_str:N} % \item \cs{str_head:n}, \cs{str_head_aux:w}, (copied here) @@ -576,6 +589,16 @@ % % \subsection{Helpers} % +% \subsubsection{A function unrelated to strings} +% +% \begin{macro}[EXP,aux]{\use_ii_i:nn} +% A function used to swap its arguments. +% \begin{macrocode} +\cs_if_exist:NF \use_ii_i:nn + { \cs_new:Npn \use_ii_i:nn #1#2 { #2 #1 } } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Assigning strings} % % \begin{macro} @@ -597,65 +620,58 @@ % } % Simply convert the token list inputs to \meta{strings}. % \begin{macrocode} -\group_begin: - \cs_set_protected_nopar:Npn \str_tmp:w #1 - { - \cs_new_protected:cpx { str_ #1 :Nn } ##1##2 - { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } } - \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx } - } - \str_tmp:w { set } - \str_tmp:w { gset } - \str_tmp:w { const } - \str_tmp:w { put_left } - \str_tmp:w { gput_left } - \str_tmp:w { put_right } - \str_tmp:w { gput_right } -\group_end: +\tl_map_inline:nn + { + { set } + { gset } + { const } + { put_left } + { gput_left } + { put_right } + { gput_right } + } + { + \cs_new_protected:cpx { str_ #1 :Nn } ##1##2 + { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } } + \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx } + } % \end{macrocode} % \end{macro} % % \subsubsection{Variables and constants} % % \begin{macro}{\str_tmp:w} -% \begin{variable}{\l_str_tmpa_tl} +% \begin{variable}{\l_str_internal_int} +% \begin{variable}{\l_str_internal_tl} % Internal scratch space for some functions. % \begin{macrocode} \cs_new_protected_nopar:Npn \str_tmp:w { } -\tl_new:N \l_str_tmpa_tl -\int_new:N \l_str_tmpa_int +\tl_new:N \l_str_internal_tl +\int_new:N \l_str_internal_int % \end{macrocode} % \end{variable} +% \end{variable} % \end{macro} % % \begin{variable}{\g_str_result_tl} -% The \cs{g_str_result_tl} variable is used to hold the result -% of various internal string operations which are typically -% performed in a group. The variable is global so that it remains -% defined outside the group, to be assigned to a user provided variable. +% The \cs{g_str_result_tl} variable is used to hold the result of +% various internal string operations (mostly conversions) which are +% typically performed in a group. The variable is global so that it +% remains defined outside the group, to be assigned to a user-provided +% variable. % \begin{macrocode} \tl_new:N \g_str_result_tl % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_str_char_int} -% When converting from various forms to characters, -% \cs{l_str_char_int} is the character code of the -% character which should be created. -% \begin{macrocode} -\int_new:N \l_str_char_int -% \end{macrocode} -% \end{variable} -% % \begin{variable} % { % \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one, % \c_ninety_seven, \c_one_hundred_twenty_three, % \c_one_hundred_twenty_seven % } -% We declare here some integer values which delimit ranges of -% ASCII characters of various types. This is mostly used in -% \pkg{l3regex}. +% We declare here some integer values which delimit ranges of ASCII +% characters of various types. This is mostly used in \pkg{l3regex}. % \begin{macrocode} \int_const:Nn \c_forty_eight { 48 } \int_const:Nn \c_fifty_eight { 58 } @@ -668,10 +684,11 @@ % \end{variable} % % \begin{variable}{\c_max_char_int} -% The maximum character code depends on the engine. +% The maximum valid character code is $255$ for pdf\TeX{}, and +% $1114111$ for other engines. % \begin{macrocode} \int_const:Nn \c_max_char_int - { \pdftex_if_engine:TF { 255 } { 1114111 } } + { \pdftex_if_engine:TF { "FF } { "10FFFF } } % \end{macrocode} % \end{variable} % @@ -692,8 +709,8 @@ % \c_tilde_str, % \c_percent_str % } -% For all of those strings, \cs{cs_to_str:N} produce characters -% with the correct category code. +% For all of those strings, \cs{cs_to_str:N} produces characters with +% the correct category code. % \begin{macrocode} \tl_const:Nx \c_backslash_str { \cs_to_str:N \\ } \tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ } @@ -704,22 +721,46 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_str_aliases_prop} -% To avoid needing one file per encoding/escaping alias, -% we keep track of those in a property list. +% \begin{variable}{\g_str_file_prop} +% To avoid needing one file per encoding/escaping alias, we keep track +% of those in a property list. +% \begin{macrocode} +\prop_new:N \g_str_file_prop +\prop_gput:Nnn \g_str_file_prop { latin1 } { iso88591 } +\prop_gput:Nnn \g_str_file_prop { latin2 } { iso88592 } +\prop_gput:Nnn \g_str_file_prop { latin3 } { iso88593 } +\prop_gput:Nnn \g_str_file_prop { latin4 } { iso88594 } +\prop_gput:Nnn \g_str_file_prop { latin5 } { iso88599 } +\prop_gput:Nnn \g_str_file_prop { latin6 } { iso885910 } +\prop_gput:Nnn \g_str_file_prop { latin7 } { iso885913 } +\prop_gput:Nnn \g_str_file_prop { latin8 } { iso885914 } +\prop_gput:Nnn \g_str_file_prop { latin9 } { iso885915 } +\prop_gput:Nnn \g_str_file_prop { latin10 } { iso885916 } +\prop_gput:Nnn \g_str_file_prop { utf16le } { utf16 } +\prop_gput:Nnn \g_str_file_prop { utf16be } { utf16 } +\prop_gput:Nnn \g_str_file_prop { utf32le } { utf32 } +\prop_gput:Nnn \g_str_file_prop { utf32be } { utf32 } +\prop_gput:Nnn \g_str_file_prop { hexadecimal } { hex } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_str_error_bool} +% In conversion functions with a built-in conditional, errors are not +% reported directly to the user, but the information is collected in +% this boolean, used at the end to decide on which branch of the +% conditional to take. +% \begin{macrocode} +\bool_new:N \g_str_error_bool +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{str_byte, str_error} +% Conversions from one \meta{encoding}/\meta{escaping} pair to another +% are done within \texttt{x}-expanding assignments. Errors are +% signalled by raising the relevant flag. % \begin{macrocode} -\prop_new:N \g_str_aliases_prop -\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 } -\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 } -\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 } -\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 } -\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 } -\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 } -\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 } -\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 } -\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 } -\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 } -\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex } +\flag_new:n { str_byte } +\flag_new:n { str_error } % \end{macrocode} % \end{variable} % @@ -727,10 +768,10 @@ % % \begin{macro}[EXP]{\tl_to_other_str:n} % \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w} -% Replaces all spaces by \enquote{other} spaces, after converting -% the token list to a string via \cs{tl_to_str:n}. -% This function is \texttt{f}-expandable, storing the part of the string -% with escaped spaces between the \cs{q_mark} and \cs{q_stop} markers. +% Replaces all spaces by \enquote{other} spaces, after converting the +% token list to a string via \cs{tl_to_str:n}. This function is +% \texttt{f}-expandable, storing the part of the string with escaped +% spaces between the \cs{q_mark} and \cs{q_stop} markers. % \begin{macrocode} \group_begin: \char_set_lccode:nn { `\* } { `\ } @@ -743,7 +784,7 @@ \exp_after:wN \tl_to_other_str_loop:w \tl_to_str:n {#1} ~ % A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop } - \cs_new_nopar:Npn \tl_to_other_str_loop:w + \cs_new:Npn \tl_to_other_str_loop:w #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 \q_stop { \if_meaning:w A #8 @@ -752,7 +793,7 @@ \tl_to_other_str_loop:w #9 #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * \q_stop } - \cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop + \cs_new:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop { \fi: #2 } } % \end{macrocode} @@ -762,12 +803,11 @@ % \begin{macro}[int]{\str_gset_other:Nn} % \begin{macro}[aux,EXP]{\str_gset_other_loop:w} % \begin{macro}[aux,EXP]{\str_gset_other_end:w} -% This function could be done by using \cs{tl_to_other_str:n} -% within an \texttt{x}-expansion, but that would take a time -% quadratic in the size of the string. Instead, we can -% \enquote{leave the result behind us} in the input stream, -% to be captured into the expanding assignment. This gives us -% a linear time. +% This function could be done by using \cs{tl_to_other_str:n} within +% an \texttt{x}-expansion, but that would take a time quadratic in the +% size of the string. Instead, we can \enquote{leave the result behind +% us} in the input stream, to be captured into the expanding +% assignment. This gives us a linear time. % \begin{macrocode} \group_begin: \char_set_lccode:nn { `\* } { `\ } @@ -780,10 +820,10 @@ \tl_gset:Nx #1 { \exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ % - A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop + A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_stop } } - \cs_new_nopar:Npn \str_gset_other_loop:w + \cs_new:Npn \str_gset_other_loop:w #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~ { \if_meaning:w A #9 @@ -792,7 +832,7 @@ #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9 \str_gset_other_loop:w * } - \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop + \cs_new:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop { \fi: #1 } } % \end{macrocode} @@ -800,66 +840,70 @@ % \end{macro} % \end{macro} % -% \subsubsection{Functions unrelated to strings} -% -% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn} -% A function which may already be defined elsewhere. -% \begin{macrocode} -\cs_if_exist:NF \use_i:nnnnnnnn - { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} } -% \end{macrocode} -% \end{macro} +% \subsection{Characters given by their position} % -% \begin{macro}[int, rEXP]{\str_aux_toks_range:nn} -% \begin{macro}[aux, rEXP]{\str_aux_toks_range:ww} -% Some non-expandable functions (in this module, -% \cs{str_aux_escape:NNNn}, which is used by \cs{str_(g)input:Nn}) -% store pieces of their result in \tn{toks} registers. -% Those pieces are concatenated using \cs{str_aux_toks_range:nn}, -% which expects two integer registers, \meta{start} and \meta{end}, -% and expands to the contents of the \tn{toks} registers from -% \meta{start} (inclusive) to \meta{end} (exclusive). -% This function is also used in the \pkg{l3regex} module. +% \begin{macro}[EXP]{\str_count_spaces:N} +% \begin{macro}[EXP]{\str_count_spaces:n} +% \begin{macro}[EXP,aux]{\str_count_spaces_loop:wwwwwwwww} +% To speed up this function, we grab $9$ spaces in each step. The +% loop stops when the last argument is one of the trailing +% |X|\meta{number}, and that \meta{number} is added to the sum of $9$ +% that preceeds, to adjust the result. % \begin{macrocode} -\cs_new:Npn \str_aux_toks_range:nn #1#2 +\cs_new:Npn \str_count_spaces:N + { \exp_args:No \str_count_spaces:n } +\cs_new:Npn \str_count_spaces:n #1 { - \exp_after:wN \str_aux_toks_range:ww - \int_use:N \int_eval:w #1 \exp_after:wN ; - \int_use:N \int_eval:w #2 ; - \prg_break_point:n { } + \int_eval:n + { + \exp_after:wN \str_count_spaces_loop:wwwwwwwww + \tl_to_str:n {#1} ~ + X 7 ~ X 6 ~ X 5 ~ X 4 ~ X 3 ~ X 2 ~ X 1 ~ X 0 ~ X -1 ~ + \q_stop + } } -\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ; +\cs_new:Npn \str_count_spaces_loop:wwwwwwwww #1~#2~#3~#4~#5~#6~#7~#8~#9~ { - \if_num:w #1 = #2 \exp_stop_f: - \exp_after:wN \prg_map_break: + \if_meaning:w X #9 + \exp_after:wN \use_none_delimit_by_q_stop:w \fi: - \tex_the:D \tex_toks:D #1 \exp_stop_f: - \exp_after:wN \str_aux_toks_range:ww - \int_use:N \int_eval:w #1 + \c_one ; #2 ; + \c_nine + \str_count_spaces_loop:wwwwwwwww } % \end{macrocode} % \end{macro} % \end{macro} -% -% \subsection{Characters given by their position} +% \end{macro} % % \begin{macro}[EXP]{\str_length:N} % \begin{macro}[EXP]{\str_length:n} -% \begin{macro}[EXP]{\str_length_ignore_spaces:n} % \begin{macro}[EXP]{\str_length_unsafe:n} -% \begin{macro}[EXP,aux]{\str_length_aux:n,\str_length_loop:NNNNNNNNN} -% The length of a string is found by first changing all spaces -% to other spaces using \cs{tl_to_other_str:n}, then counting -% characters $9$ at a time. When the end is reached, |#9| -% has the form |X|\meta{digit}, the catcode test is true, -% the digit gets added to the sum, and the loop is terminated -% by \cs{use_none_delimit_by_q_stop:w}. +% \begin{macro}[EXP]{\str_length_ignore_spaces:n} +% \begin{macro}[EXP,aux]{\str_length_aux:n, \str_length_loop:NNNNNNNNN} +% To measure the length of a string we could first escape all spaces +% using \cs{tl_to_other_str:N}, then measure the length of this token +% list. However, this would be quadratic in the length of the string, +% and we can do better. Namely, add the number of spaces (counted +% using the functions defined above) to the length ignoring spaces. +% To measure the length ignoring spaces we use the same technique as +% for counting spaces: loop, grabbing $9$ characters at each step, and +% end as soon as we reach one of the $9$ trailing items. The +% \texttt{_unsafe} variant expects a token list consisting entirely of +% category code $12$ characters. % \begin{macrocode} \cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n } \cs_new:Npn \str_length:n #1 - { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } } -\cs_new_nopar:Npn \str_length_unsafe:n #1 - { \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } } + { + \str_length_aux:n + { + \str_count_spaces:n {#1} + + \exp_after:wN \str_length_loop:NNNNNNNNN \tl_to_str:n {#1} + } + } +\cs_new:Npn \str_length_unsafe:n #1 + { + \str_length_aux:n + { \str_length_loop:NNNNNNNNN #1 } + } \cs_new:Npn \str_length_ignore_spaces:n #1 { \str_length_aux:n @@ -878,7 +922,7 @@ } \cs_set:Npn \str_length_loop:NNNNNNNNN #1#2#3#4#5#6#7#8#9 { - \if_catcode:w X #9 + \if_meaning:w X #9 \exp_after:wN \use_none_delimit_by_q_stop:w \fi: \c_nine + \str_length_loop:NNNNNNNNN @@ -893,21 +937,18 @@ % \begin{macro}[EXP]{\str_head:N} % \begin{macro}[EXP]{\str_head:n} % \begin{macro}[EXP]{\str_head_ignore_spaces:n} -% \begin{macro}[EXP]{\str_head_unsafe:n} % \begin{macro}[EXP,aux]{\str_head_aux:w} -% The cases of \cs{str_head_ignore_spaces:n} and -% \cs{str_head_unsafe:n} are mostly identical to -% \cs{tl_head:n}. -% As usual, \cs{str_head:N} expands its argument and hands it to -% \cs{str_head:n}. To circumvent the fact that \TeX{} skips spaces -% when grabbing undelimited macro parameters, \cs{str_head_aux:w} -% takes an argument delimited by a space. If |#1| starts with a -% non-space character, \cs{use_i_delimit_by_q_stop:nw} leaves that -% in the input stream. On the other hand, if |#1| starts with a -% space, the \cs{str_head_aux:w} takes an empty argument, and +% The \texttt{_ignore_spaces} variant is almost identical to +% \cs{tl_head:n}. As usual, \cs{str_head:N} expands its argument and +% hands it to \cs{str_head:n}. To circumvent the fact that \TeX{} +% skips spaces when grabbing undelimited macro parameters, +% \cs{str_head_aux:w} takes an argument delimited by a space. If |#1| +% starts with a non-space character, \cs{use_i_delimit_by_q_stop:nw} +% leaves that in the input stream. On the other hand, if |#1| starts +% with a space, the \cs{str_head_aux:w} takes an empty argument, and % the single (braced) space in the definition of \cs{str_head_aux:w} -% makes its way to the output. Finally, for an empty argument, -% the (braced) empty brace group in the definition of \cs{str_head:n} +% makes its way to the output. Finally, for an empty argument, the +% (braced) empty brace group in the definition of \cs{str_head:n} % gives an empty result after passing through % \cs{use_i_delimit_by_q_stop:nw}. % \begin{macrocode} @@ -918,36 +959,33 @@ \tl_to_str:n {#1} { { } } ~ \q_stop } -\cs_set_nopar:Npn \str_head_aux:w #1 ~ % +\cs_set:Npn \str_head_aux:w #1 ~ % { \use_i_delimit_by_q_stop:nw #1 { ~ } } \cs_new:Npn \str_head_ignore_spaces:n #1 - { \exp_after:wN \use_i_delimit_by_q_stop:nw \tl_to_str:n {#1} { } \q_stop } -\cs_new_nopar:Npn \str_head_unsafe:n #1 - { \use_i_delimit_by_q_stop:nw #1 { } \q_stop } + { + \exp_after:wN \use_i_delimit_by_q_stop:nw + \tl_to_str:n {#1} { } \q_stop + } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % % \begin{macro}[EXP]{\str_tail:N} % \begin{macro}[EXP]{\str_tail:n} % \begin{macro}[EXP]{\str_tail_ignore_spaces:n} -% \begin{macro}[EXP]{\str_tail_unsafe:n} % \begin{macro}[EXP,aux]{\str_tail_aux:w} % \begin{macro}[EXP,aux]{\str_tail_aux_ii:w} -% As when fetching the head of a string, the cases -% \enquote{\texttt{ignore_spaces:n}} and \enquote{\texttt{unsafe:n}} -% are similar to \cs{tl_tail:n}. -% The more commonly used \cs{str_tail:n} function is a little bit -% more convoluted: hitting the front of the string with -% \cs{reverse_if:N} \cs{if_charcode:w} \cs{scan_stop:} -% removes the first character (which necessarily makes the test true, -% since it cannot match \cs{scan_stop:}). The auxiliary function inserts -% the required \cs{fi:} to close the conditional, and leaves the tail -% of the string in the input string. The details are such that an empty -% string has an empty tail. +% As when fetching the head of a string, the \texttt{_ignore_spaces} +% variant is similar to \cs{tl_tail:n}. The more commonly used +% \cs{str_tail:n} function is a little bit more convoluted: hitting +% the front of the string with \cs{reverse_if:N} \cs{if_charcode:w} +% \cs{scan_stop:} removes the first character (which necessarily makes +% the test true, since it cannot match \cs{scan_stop:}). The auxiliary +% function inserts the required \cs{fi:} to close the conditional, and +% leaves the tail of the string in the input string. The details are +% such that an empty string has an empty tail. % \begin{macrocode} \cs_new_nopar:Npn \str_tail:N { \exp_args:No \str_tail:n } \cs_set:Npn \str_tail:n #1 @@ -956,113 +994,103 @@ \reverse_if:N \if_charcode:w \scan_stop: \tl_to_str:n {#1} X X \q_stop } -\cs_set_nopar:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 } +\cs_set:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 } \cs_new:Npn \str_tail_ignore_spaces:n #1 { \exp_after:wN \str_tail_aux_ii:w - \tl_to_str:n {#1} X { } X \q_stop + \tl_to_str:n {#1} X X \q_stop } -\cs_new_nopar:Npn \str_tail_unsafe:n #1 - { \str_tail_aux_ii:w #1 X { } X \q_stop } -\cs_new_nopar:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 } +\cs_new:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[EXP,int]{\str_skip_do:nn} -% \begin{macro}[EXP,aux]{\str_skip_aux:nnnnnnnnn} -% \begin{macro}[EXP,aux]{\str_skip_end:nn} -% \begin{macro}[EXP,aux]{\str_skip_end_ii:nwn} -% Removes |max(#1,0)| characters then leaves |#2| in the input stream. -% We remove characters $7$ at a time. When the number of characters -% to remove is not a multiple of $7$, we need to remove less than -% $7$ characters in the last step. This is done by inserting a number -% of \texttt{X}, which are discarded as if they were part of the string. +% \begin{macro}[EXP, int]{\str_skip_c_zero:w} +% \begin{macro}[EXP, aux]{\str_skip_aux:wNNNNNNNN} +% \begin{macro}[EXP, aux]{\str_skip_end:w, \str_skip_end_ii:NNNNNNNN} +% Removes |max(#1,0)| characters from the input stream, and then +% leaves \cs{c_zero}. This should be expanded using +% \cs{tex_romannumeral:D}. We remove characters $8$ at a time until +% there are at most $8$ to remove. Then we do a dirty trick: the +% \cs{if_case:w} construction leaves between $0$ and $8$ times the +% \cs{or:} control sequence, and those \cs{or:} become arguments of +% \cs{str_skip_end_ii:NNNNNNNN}. If the number of characters to remove +% is $6$, say, then there are two \cs{or:} left, and the $8$ arguments +% of \cs{str_skip_end_ii:NNNNNNNN} are the two \cs{or:}, and $6$ +% characters from the input stream, exactly what we wanted to +% remove. Then close the \cs{if_case:w} conditional with \cs{fi:}, and +% stop the initial expansion with \cs{c_zero} (see places where +% \cs{str_skip_c_zero:w} is called). % \begin{macrocode} -\cs_new:Npn \str_skip_do:nn #1 +\cs_new:Npn \str_skip_c_zero:w #1; { - \if_num:w \int_eval:w #1 > \c_seven - \exp_after:wN \str_skip_aux:nnnnnnnnn + \if_num:w \int_eval:w #1 > \c_eight + \exp_after:wN \str_skip_aux:wNNNNNNNN \else: - \exp_after:wN \str_skip_end:n + \exp_after:wN \str_skip_end:w + \int_use:N \int_eval:w \fi: - {#1} + #1 ; } -\cs_new:Npn \str_skip_aux:nnnnnnnnn #1#2#3#4#5#6#7#8#9 - { \exp_args:Nf \str_skip_do:nn { \int_eval:n { #1 - \c_seven } } {#2} } -\cs_new:Npn \str_skip_end:n #1 +\cs_new:Npn \str_skip_aux:wNNNNNNNN #1; #2#3#4#5#6#7#8#9 + { \exp_after:wN \str_skip_c_zero:w \int_use:N \int_eval:w #1 - \c_eight ; } +\cs_new:Npn \str_skip_end:w #1 ; { - \if_case:w \int_eval:w #1 \int_eval_end: - \str_skip_end_ii:nwn { XXXXXXX } - \or: \str_skip_end_ii:nwn { XXXXXX } - \or: \str_skip_end_ii:nwn { XXXXX } - \or: \str_skip_end_ii:nwn { XXXX } - \or: \str_skip_end_ii:nwn { XXX } - \or: \str_skip_end_ii:nwn { XX } - \or: \str_skip_end_ii:nwn { X } - \or: \str_skip_end_ii:nwn { } - \else: \str_skip_end_ii:nwn { XXXXXXX } - \fi: + \exp_after:wN \str_skip_end_ii:NNNNNNNN + \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f: + \or: \or: \or: \or: \or: \or: \or: \or: } -\cs_new:Npn \str_skip_end_ii:nwn #1#2 \fi: #3 - { \fi: \use_i:nnnnnnnn {#3} #1 } +\cs_new:Npn \str_skip_end_ii:NNNNNNNN #1#2#3#4#5#6#7#8 { \fi: \c_zero } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[EXP,int]{\str_collect_do:nn} -% \begin{macro}[EXP,aux]{\str_collect_aux:n,\str_collect_aux:nnNNNNNNN} +% \begin{macro}[EXP,int]{\str_collect_delimit_by_q_stop:w} +% \begin{macro}[EXP,aux]{\str_collect_aux:wn, \str_collect_aux:wnNNNNNNN} % \begin{macro}[EXP,aux] -% { -% \str_collect_end:nn, -% \str_collect_end_ii:nwn, -% \str_collect_end_iii:nwNNNNNNN -% } -% Collects |max(#1,0)| characters, and feeds them as an argument to |#2|. -% Again, we grab $7$ characters at a time. Instead of inserting -% a string of \texttt{X} to fill in to a multiple of $7$, we insert -% empty groups, so that they disappear in this context where arguments -% are accumulated. +% {\str_collect_end:wn, \str_collect_end_ii:nnnnnnnnw} Collects +% |max(#1,0)| characters, and removes everything else until +% \cs{q_stop}. This is somewhat similar to \cs{str_skip_c_zero:w}, but +% this time we can only grab $7$ characters at a time. At the end, we +% use an \cs{if_case:w} trick again, so that the $8$ first arguments +% of \cs{str_collect_end_ii:nnnnnnnnw} are some \cs{or:}, followed by +% an \cs{if:}, followed by |#1| characters from the input +% stream. Simply leaving this in the input stream will close the +% conditional properly and the \cs{or:} disappear. % \begin{macrocode} -\cs_new:Npn \str_collect_do:nn #1#2 - { \str_collect_aux:n {#1} { \str_collect_end_iii:nwNNNNNNN {#2} } } -\cs_new:Npn \str_collect_aux:n #1 +\cs_new:Npn \str_collect_delimit_by_q_stop:w #1; { - \int_compare:nNnTF {#1} > \c_seven - { \str_collect_aux:nnNNNNNNN } - { \str_collect_end:n } - {#1} + \exp_after:wN \str_collect_aux:wn + \int_use:N \int_eval:w #1 ; + { } } -\cs_new:Npn \str_collect_aux:nnNNNNNNN #1#2 #3#4#5#6#7#8#9 +\cs_new:Npn \str_collect_aux:wn #1 ; { - \exp_args:Nf \str_collect_aux:n - { \int_eval:n { #1 - \c_seven } } - { #2 #3#4#5#6#7#8#9 } + \if_num:w #1 > \c_seven + \exp_after:wN \str_collect_aux:wnNNNNNNN + \else: + \exp_after:wN \str_collect_end:wn + \fi: + #1 ; } -\cs_new:Npn \str_collect_end:n #1 +\cs_new:Npn \str_collect_aux:wnNNNNNNN #1; #2 #3#4#5#6#7#8#9 { - \if_case:w \int_eval:w #1 \int_eval_end: - \str_collect_end_ii:nwn { { } { } { } { } { } { } { } } - \or: \str_collect_end_ii:nwn { { } { } { } { } { } { } } - \or: \str_collect_end_ii:nwn { { } { } { } { } { } } - \or: \str_collect_end_ii:nwn { { } { } { } { } } - \or: \str_collect_end_ii:nwn { { } { } { } } - \or: \str_collect_end_ii:nwn { { } { } } - \or: \str_collect_end_ii:nwn { { } } - \or: \str_collect_end_ii:nwn { } - \else: \str_collect_end_ii:nwn { { } { } { } { } { } { } { } } - \fi: + \exp_after:wN \str_collect_aux:wn + \int_use:N \int_eval:w #1 - \c_seven ; + { #2 #3#4#5#6#7#8#9 } } -\cs_new:Npn \str_collect_end_ii:nwn #1#2 \fi: #3 - { \fi: #3 \q_stop #1 } -\cs_new:Npn \str_collect_end_iii:nwNNNNNNN #1 #2 \q_stop #3#4#5#6#7#8#9 - { #1 {#2#3#4#5#6#7#8#9} } +\cs_new:Npn \str_collect_end:wn #1 ; + { + \exp_after:wN \str_collect_end_ii:nnnnnnnnw + \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f: + \or: \or: \or: \or: \or: \or: \fi: + } +\cs_new:Npn \str_collect_end_ii:nnnnnnnnw #1#2#3#4#5#6#7#8 #9 \q_stop + { #1#2#3#4#5#6#7#8 } % \end{macrocode} % \end{macro} % \end{macro} @@ -1072,48 +1100,47 @@ % \begin{macro}[EXP]{\str_item:nn} % \begin{macro}[EXP]{\str_item_ignore_spaces:nn} % \begin{macro}[EXP]{\str_item_unsafe:nn} -% \begin{macro}[EXP,aux]{\str_item_aux:nn} -% This is mostly shuffling arguments around to avoid measuring -% the length of the string more than once, and make sure that -% the parameters given to \cs{str_skip_do:nn} are necessarily -% within the bounds of the length of the string. -% The \cs{str_item_ignore_spaces:nn} function cheats a little bit -% in that it doesn't hand to \cs{str_item_unsafe:nn} an -% \enquote{other string}. This is alright, as everything else -% is done with undelimited arguments. +% \begin{macro}[EXP,aux]{\str_item_aux:ww} +% This is mostly shuffling arguments around to avoid measuring the +% length of the string more than once, and make sure that the +% parameters given to \cs{str_skip_c_zero:w} are necessarily within +% the bounds of the length of the string. The \texttt{_ignore_spaces} +% function cheats a little bit in that it doesn't hand to +% \cs{str_item_unsafe:nn} an \enquote{other string}. This is alright, +% as everything else is done with undelimited arguments. % \begin{macrocode} \cs_new_nopar:Npn \str_item:Nn { \exp_args:No \str_item:nn } -\cs_new:Npn \str_item:nn #1#2 +\cs_new:Npn \str_item:nn #1 { - \exp_args:Nf \tl_to_str:n - { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} } + \exp_last_unbraced:Nf \token_to_str:N + \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } } \cs_new:Npn \str_item_ignore_spaces:nn #1 { \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } } -\cs_new_nopar:Npn \str_item_unsafe:nn #1#2 +\cs_new:Npn \str_item_unsafe:nn #1#2 { - \exp_args:Nff \str_item_aux:nn - { \int_eval:n {#2} } - { \str_length_unsafe:n {#1} } + \exp_after:wN \str_item_aux:ww + \int_use:N \int_eval:w #2 \exp_after:wN ; + \int_value:w \str_length_unsafe:n {#1} ; #1 \q_stop } -\cs_new_nopar:Npn \str_item_aux:nn #1#2 +\cs_new:Npn \str_item_aux:ww #1; #2; { \int_compare:nNnTF {#1} < \c_zero { \int_compare:nNnTF {#1} < {-#2} { \use_none_delimit_by_q_stop:w } { - \str_skip_do:nn { #1 + #2 } - { \use_i_delimit_by_q_stop:nw } + \exp_after:wN \use_i_delimit_by_q_stop:nw + \tex_romannumeral:D \str_skip_c_zero:w #1 + #2 ; } } { \int_compare:nNnTF {#1} < {#2} { - \str_skip_do:nn {#1} - { \use_i_delimit_by_q_stop:nw } + \exp_after:wN \use_i_delimit_by_q_stop:nw + \tex_romannumeral:D \str_skip_c_zero:w #1 ; } { \use_none_delimit_by_q_stop:w } } @@ -1128,24 +1155,27 @@ % \begin{macro}[EXP]{\str_substr:Nnn} % \begin{macro}[EXP]{\str_substr:nnn} % \begin{macro}[EXP]{\str_substr_ignore_spaces:nnn} -% \begin{macro}[EXP]{\str_substr_unsafe:nnn} -% \begin{macro}[EXP,aux]{\str_substr_aux:www} -% \begin{macro}[EXP,aux]{\str_substr_aux:nnw} -% \begin{macro}[EXP,aux]{\str_aux_normalize_range:nn} -% Sanitize the string. Then evaluate the arguments, replacing -% them by \cs{c_zero} or \cs{c_max_int} if they are empty. -% This is done by using the construction +% \begin{macro}[EXP,aux] +% { +% \str_substr_unsafe:nnn, +% \str_substr_aux:nN, +% \str_substr_aux:www, +% \str_substr_aux:nnw, +% \str_substr_normalize_range:nn +% } +% Sanitize the string. Then evaluate the arguments, replacing them by +% \cs{c_zero} or \cs{c_max_int} if they are empty. This is done by +% using the construction % \begin{verbatim} % \int_use:N \int_eval:w #2 \c_zero \c_zero ; % \end{verbatim} % which expands to the value of |#2|, followed by |\c_zero \c_zero ;| % if |#2| is an expression, and expands to |0\c_zero ;| otherwise. -% The same is done to the end-point of the range. -% Then limit the range to be at most the length of the string -% (this avoids needing to check for the end of the string when -% grabbing characters). -% Afterwards, skip characters, then keep some more, and finally -% drop the end of the string. +% The same is done to the end-point of the range. Then limit the +% range to be at most the length of the string (this avoids needing to +% check for the end of the string when grabbing characters). +% Afterwards, skip characters, then keep some more, and finally drop +% the end of the string. % \begin{macrocode} \cs_new_nopar:Npn \str_substr:Nnn { \exp_args:No \str_substr:nnn } \cs_new:Npn \str_substr:nnn #1#2#3 @@ -1163,28 +1193,26 @@ \exp_after:wN \str_substr_aux:www \int_value:w \str_length_unsafe:n {#1} \exp_after:wN ; \int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ; - \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ; + \int_use:N \int_eval:w + \exp_args:Nf \str_substr_aux:nN {#3} \c_max_int ; #1 \q_stop } -\cs_new:Npn \str_substr_unsafe:nN #1 #2 +\cs_new:Npn \str_substr_aux:nN #1 #2 { \tl_if_empty:nTF {#1} {#2} {#1} } \cs_new:Npn \str_substr_aux:www #1; #2; #3; { \exp_args:Nf \str_substr_aux:nnw - { \str_aux_normalize_range:nn {#2} {#1} } - { \str_aux_normalize_range:nn {#3} {#1} } + { \str_substr_normalize_range:nn {#2} {#1} } + { \str_substr_normalize_range:nn {#3} {#1} } } \cs_new:Npn \str_substr_aux:nnw #1#2 { - \str_skip_do:nn {#1} - { - \exp_args:Nf \str_collect_do:nn - { \int_eval:n { #2 - #1 } } - { \use_i_delimit_by_q_stop:nw } - } + \exp_after:wN \str_collect_delimit_by_q_stop:w + \int_use:N \int_eval:w #2 - #1 \exp_after:wN ; + \tex_romannumeral:D \str_skip_c_zero:w #1 ; } -\cs_new:Npn \str_aux_normalize_range:nn #1#2 +\cs_new:Npn \str_substr_normalize_range:nn #1#2 { \int_eval:n { @@ -1208,18 +1236,14 @@ % \end{macro} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % % \subsection{String conditionals} % % \begin{macro}[EXP,pTF]{\str_if_eq:NN} % \begin{macro}[EXP,pTF]{\str_if_eq:nn,\str_if_eq:xx} -% The \texttt{nn} and \texttt{xx} variants are already -% defined in \pkg{l3basics}. Note that \cs{str_if_eq:NN} -% is different from \cs{tl_if_eq:NN} because it needs to -% ignore category codes. +% The \texttt{nn} and \texttt{xx} variants are already defined in +% \pkg{l3basics}. Note that \cs{str_if_eq:NN} is different from +% \cs{tl_if_eq:NN} because it needs to ignore category codes. % \begin{macrocode} \prg_new_conditional:Npnn \str_if_eq:NN #1#2 { p , TF , T , F } { @@ -1233,17 +1257,15 @@ % \begin{macro}[EXP]{\str_if_contains_char:NNT, \str_if_contains_char:NNTF} % \begin{macro}[EXP]{\str_if_contains_char:nNTF} % \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN} -% \begin{macro}[EXP,aux]{\str_if_contains_char_end:} +% \begin{macro}[EXP,aux]{\str_if_contains_char_true:} % Loop over the characters of the string, comparing character codes. -% We allow |#2| to be a single-character control sequence, hence the -% use of \cs{int_compare:nNnT} rather than \cs{token_if_eq_charcode:NNT}. % The loop is broken if character codes match. Otherwise we return % \enquote{false}. % \begin{macrocode} \prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF } { \exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2 - #1 { \prg_map_break:n { ? \fi: } } + \tl_to_str:n \exp_after:wN {#1} { \prg_map_break:n { ? \fi: } } \prg_break_point:n { } \prg_return_false: } @@ -1254,14 +1276,14 @@ \prg_break_point:n { } \prg_return_false: } -\cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2 +\cs_new:Npn \str_if_contains_char_aux:NN #1#2 { \if_charcode:w #1 #2 - \exp_after:wN \str_if_contains_char_end: + \exp_after:wN \str_if_contains_char_true: \fi: \str_if_contains_char_aux:NN #1 } -\cs_new_nopar:Npn \str_if_contains_char_end: +\cs_new_nopar:Npn \str_if_contains_char_true: { \prg_map_break:n { \prg_return_true: \use_none:n } } % \end{macrocode} % \end{macro} @@ -1270,11 +1292,10 @@ % \end{macro} % % \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF} -% \TeX{} dutifully detects octal digits for us: if |#1| -% is an octal digit, then the right-hand side of the -% comparison is |'1#1|, greater than $1$. Otherwise, -% the right-hand side stops as |'1|, and the conditional -% takes the \texttt{false} branch. +% \TeX{} dutifully detects octal digits for us: if |#1| is an octal +% digit, then the right-hand side of the comparison is |'1#1|, greater +% than $1$. Otherwise, the right-hand side stops as |'1|, and the +% conditional takes the \texttt{false} branch. % \begin{macrocode} \prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF } { @@ -1288,10 +1309,9 @@ % \end{macro} % % \begin{macro}[aux, rEXP]{\str_aux_hexadecimal_use:NTF} -% \TeX{} detects uppercase hexadecimal digits for us -% (see \cs{str_aux_octal_use:NTF}, but not the -% lowercase letters, which we need to detect and replace -% by their uppercase counterpart. +% \TeX{} detects uppercase hexadecimal digits for us (see +% \cs{str_aux_octal_use:NTF}), but not the lowercase letters, which we +% need to detect and replace by their uppercase counterpart. % \begin{macrocode} \prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF } { @@ -1321,25 +1341,24 @@ % % \subsubsection{Producing one byte or character} % -% \begin{variable}{\c_str_byte_0_tl} +% \begin{variable}{\c_str_byte_0_tl, \c_str_byte_1_tl, \c_str_byte_255_tl} % \begin{variable}{\c_str_byte_-1_tl} % \begin{variable}{\c_str_positive_bytes_tl} -% For each integer $N$ in the range $[0,255]$, we create -% a constant token list which holds three character tokens -% with category code other: the character with character -% code $N$, followed by the representation of $N$ as -% two hexadecimal digits. -% The value $-1$ is given a default token list which ensures -% that later functions give an empty result for the input $-1$. -% Simultaneously, we build a list of all bytes (from which we -% remove the null byte) in \cs{c_str_positive_bytes_tl}. +% For each integer $N$ in the range $[0,255]$, we create a constant +% token list which holds three character tokens with category code +% other: the character with character code $N$, followed by the +% representation of $N$ as two hexadecimal digits. The value $-1$ is +% given a default token list which ensures that later functions give +% an empty result for the input $-1$. Simultaneously, we build a list +% of all bytes (from which we remove the null byte) in +% \cs{c_str_positive_bytes_tl}. % \begin{macrocode} \group_begin: - \tl_clear:N \l_str_tmpa_tl + \tl_clear:N \l_str_internal_tl \char_set_catcode_other:n { \c_zero } \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } } - \tl_map_inline:Nn \g_str_result_tl - { \char_set_lccode:nn {`#1} {`#1} } + \exp_args:No \tl_map_inline:nn { \g_str_result_tl " } + { \char_set_lccode:nn {`#1} { \c_zero } } \tl_map_inline:Nn \g_str_result_tl { \tl_map_inline:Nn \g_str_result_tl @@ -1350,12 +1369,12 @@ \tl_const:cx { c_str_byte_ \int_eval:n {"#1##1} _tl } { ^^@ #1 ##1 } - \tl_put_right:Nn \l_str_tmpa_tl { ^^@ } + \tl_put_right:Nn \l_str_internal_tl { ^^@ } } } } \tl_const:Nx \c_str_positive_bytes_tl - { \exp_after:wN \use_none:n \l_str_tmpa_tl } + { \exp_after:wN \use_none:n \l_str_internal_tl } \group_end: \tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } } % \end{macrocode} @@ -1364,15 +1383,17 @@ % \end{variable} % % \begin{macro}[int, EXP]{\str_output_byte:n} -% \begin{macro}[aux, EXP]{\str_output_byte:w} +% \begin{macro}[int, EXP]{\str_output_byte:w} % \begin{macro}[int, EXP]{\str_output_hexadecimal:n} -% \begin{macro}[aux, EXP]{\str_output_hexadecimal:w} -% \begin{macro}[aux, EXP]{\str_output_end:} -% For now, no error detection, we simply pick either -% the byte or the hexadecimal representation of it -% from the three-character token list corresponding -% to the argument. -% The value $-1$ produces an empty result. +% \begin{macro}[int, EXP]{\str_output_hexadecimal:w} +% \begin{macro}[int, EXP]{\str_output_end:} +% Those functions must be used carefully: feeding them a value outside +% the range $[-1,255]$ will attempt to use the undefined token list +% variable \cs{c_str_byte_\meta{number}_tl}. Assuming that the +% argument is in the right range, we expand the corresponding token +% list, and pick either the byte (first token) or the hexadecimal +% representations (second and third tokens). The value $-1$ produces +% an empty result in both cases. % \begin{macrocode} \cs_new:Npn \str_output_byte:n #1 { \str_output_byte:w #1 \str_output_end: } @@ -1399,445 +1420,346 @@ % \end{macro} % \end{macro} % -% \subsubsection{Mapping functions for encoding conversion} +% \begin{macro}[int, rEXP]{\str_output_byte_pair_be:n} +% \begin{macro}[int, rEXP]{\str_output_byte_pair_le:n} +% \begin{macro}[aux, rEXP]{\str_output_byte_pair_aux:nnN} +% Convert a number in the range $[0,65535]$ to a pair of bytes, either +% big-endian or little-endian. +% \begin{macrocode} +\cs_new:Npn \str_output_byte_pair_be:n #1 + { + \exp_args:Nf \str_output_byte_pair_aux:nnN + { \int_div_truncate:nn { #1 } { "100 } } {#1} \use:nn + } +\cs_new:Npn \str_output_byte_pair_le:n #1 + { + \exp_args:Nf \str_output_byte_pair_aux:nnN + { \int_div_truncate:nn { #1 } { "100 } } {#1} \use_ii_i:nn + } +\cs_new:Npn \str_output_byte_pair_aux:nnN #1#2#3 + { + #3 + { \str_output_byte:n { #1 } } + { \str_output_byte:n { #2 - #1 * "100 } } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsubsection{Mapping functions for conversions} % -% \begin{macro}{\str_aux_gmap_result:N} -% \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN} -% This maps the function |#1| over all characters -% in \cs{g_str_result_tl}, which should be a byte -% string in most cases, sometimes a native string. +% \begin{macro}{\str_convert_gmap:N} +% \begin{macro}[aux, rEXP]{\str_convert_gmap_loop:NN} +% This maps the function |#1| over all characters in +% \cs{g_str_result_tl}, which should be a byte string in most cases, +% sometimes a native string. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1 +\cs_new_protected:Npn \str_convert_gmap:N #1 { \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_gmap_result_loop:NN + \exp_after:wN \str_convert_gmap_loop:NN \exp_after:wN #1 \g_str_result_tl { ? \prg_map_break: } \prg_break_point:n { } } } -\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2 +\cs_new:Npn \str_convert_gmap_loop:NN #1#2 { \use_none:n #2 #1#2 - \str_aux_gmap_result_loop:NN #1 + \str_convert_gmap_loop:NN #1 } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}{\str_aux_gmap_internal_result:N} -% \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw} -% This maps the function |#1| over all character codes -% in \cs{g_str_result_tl}, which must be in the internal -% representation. +% \begin{macro}{\str_convert_gmap_internal:N} +% \begin{macro}[aux, rEXP]{\str_convert_gmap_internal_loop:Nw} +% This maps the function |#1| over all character codes in +% \cs{g_str_result_tl}, which must be in the internal representation. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1 +\cs_new_protected:Npn \str_convert_gmap_internal:N #1 { \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_gmap_internal_result_loop:Nww + \exp_after:wN \str_convert_gmap_internal_loop:Nww \exp_after:wN #1 \g_str_result_tl \s_tl \q_stop \prg_map_break: \s_tl \prg_break_point:n { } } } -\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1 #2 \s_tl #3 \s_tl +\cs_new:Npn \str_convert_gmap_internal_loop:Nww #1 #2 \s_tl #3 \s_tl { \use_none_delimit_by_q_stop:w #3 \q_stop #1 {#3} - \str_aux_gmap_internal_result_loop:Nww #1 + \str_convert_gmap_internal_loop:Nww #1 } % \end{macrocode} % \end{macro} % \end{macro} % -% \subsubsection{Unescape user input} -% -% The code of this section is used both here for \cs{str_(g)input:Nn}, -% and in the regular expression module to go through the regular expression -% once before actually parsing it. The goal in that case is to turn any -% character with a meaning in regular expressions (\texttt{*}, \texttt{?}, -% \texttt{\{}, etc.) into a marker indicating that this was a special -% character, -% and replace any escaped character by the corresponding unescaped character, -% so that the \pkg{l3regex} code can avoid caring about escaping issues -% (those can become quite complex to handle in combination with ranges -% in character classes). -% -% The idea is to feed unescaped characters to one function, -% escaped characters to another, and feed |\a|, |\e|, |\f|, -% |\n|, |\r|, |\t| and |\x| converted to the appropriate -% character to a third function. Spaces are ignored unless -% escaped. -% For the \cs{str_(g)input:Nn} application, all the functions are simply -% \cs{token_to_str:N} (this ensures that spaces correctly get assigned -% category code $10$). -% For the \pkg{l3regex} applications, the functions do some further -% tests on the character they receive. -% -% \begin{macro}{\str_input:Nn,\str_ginput:Nn} -% Simple wrappers around the internal \cs{str_aux_escape:NNNn}. -% \begin{macrocode} -\cs_new_protected:Npn \str_input:Nn #1#2 - { - \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2} - \tl_set_eq:NN #1 \g_str_result_tl - } -\cs_new_protected:Npn \str_ginput:Nn #1#2 - { - \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2} - \tl_gset_eq:NN #1 \g_str_result_tl - } -% \end{macrocode} -% \end{macro} +% \subsubsection{Error-reporting during conversion} % -% \begin{macro}[int]{\str_escape_use:NNNn} -% Use the internal \cs{str_aux_escape:NNNn}, then leave the result -% in the input stream. This is used in \pkg{l3regex}. +% \begin{macro}[int]{\str_if_flag_error:nnx} +% \begin{macro}[aux]{\str_if_flag_no_error:nnx} +% When converting using the function \cs{str_set_convert:Nnnn}, errors +% should be reported to the user after each step in the +% conversion. Errors are signalled by raising some flag (typically +% \texttt{str_error}), so here we test that flag: if it is raised, +% give the user an error, otherwise remove the arguments. On the other +% hand, in the conditional functions \cs{str_set_convert:NnnnTF}, +% errors should be suppressed. This is done by changing +% \cs{str_if_flag_error:nnx} into \cs{str_if_flag_no_error:nnx} +% locally. % \begin{macrocode} -\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4 +\cs_new_protected:Npn \str_if_flag_error:nnx #1 { - \str_aux_escape:NNNn #1#2#3 {#4} - \g_str_result_tl + \flag_if_raised:nTF {#1} + { \msg_kernel_error:nnx { str } } + { \use_none:nn } } +\cs_new_protected:Npn \str_if_flag_no_error:nnx #1#2#3 + { \flag_if_raised:nT {#1} { \bool_gset_true:N \g_str_error_bool } } % \end{macrocode} % \end{macro} -% -% \begin{macro}[int]{\str_aux_escape:NNNn} -% \begin{macro}[aux]{\str_aux_escape_loop:N} -% \begin{macro}[aux]+\str_aux_escape_\:w+ -% Most of the work is done within an \texttt{x}-expanding assignment, -% but the |\x| escape sequence cannot be done in that way. Therefore, -% we interrupt the assignment at each |\x| escape sequence, store the -% partial result in a \tn{toks} register, and at the end unpack all -% of the \tn{toks} registers to the left of the last chunk of -% \cs{g_str_result_tl}. -% \begin{macrocode} -\cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4 - { - \group_begin: - \cs_set_nopar:Npn \str_aux_escape_unescaped:N { #1 } - \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 } - \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 } - \int_set:Nn \tex_escapechar:D { 92 } - \str_gset_other:Nn \g_str_result_tl {#4} - \int_zero:N \l_str_tmpa_int - \tl_gset:Nx \g_str_result_tl - { - \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl - \q_recursion_tail \q_recursion_stop - } - \tl_gput_left:Nx \g_str_result_tl - { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int } - \group_end: - } -\cs_new_nopar:Npn \str_aux_escape_loop:N #1 - { - \cs_if_exist_use:cF { str_aux_escape_\token_to_str:N #1:w } - { \str_aux_escape_unescaped:N #1 } - \str_aux_escape_loop:N - } -\cs_new_nopar:cpn { str_aux_escape_ \c_backslash_str :w } - \str_aux_escape_loop:N #1 - { - \cs_if_exist_use:cF { str_aux_escape_/\token_to_str:N #1:w } - { \str_aux_escape_escaped:N #1 } - \str_aux_escape_loop:N - } -% \end{macrocode} -% \end{macro} -% \end{macro} % \end{macro} % -% \begin{macro}[aux]{\str_aux_escape_unescaped:N} -% \begin{macro}[aux]{\str_aux_escape_escaped:N} -% \begin{macro}[aux]{\str_aux_escape_raw:N} -% Those functions are never called before being given a new meaning, -% so their definitions here don't matter. +% \begin{macro}[int]{\str_if_flag_times:nT} +% At the end of each conversion step, we raise all relevant errors as +% one error message, built on the fly. The height of each flag +% indicates how many times a given error was encountered. This +% function prints |#2| followed by the number of occurrences of an +% error if it occurred, nothing otherwise. % \begin{macrocode} -\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing: -\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing: -\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing: +\cs_new_protected:Npn \str_if_flag_times:nT #1#2 + { \flag_if_raised:nT {#1} { #2~(x \flag_height:n {#1} ) } } % \end{macrocode} % \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[aux]+\str_aux_escape_\q_recursion_tail:w+ -% \begin{macro}[aux]+\str_aux_escape_/\q_recursion_tail:w+ -% \begin{macro}[aux]+\str_aux_escape_ :w+ -% \begin{macro}[aux] -% { -% \str_aux_escape_/a:w, \str_aux_escape_/e:w, \str_aux_escape_/f:w, -% \str_aux_escape_/n:w, \str_aux_escape_/r:w, \str_aux_escape_/t:w -% } -% The loop is ended upon seeing \cs{q_recursion_tail}. -% Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, |\t| take -% their meaning here. -% \begin{macrocode} -\cs_new_eq:cN - { str_aux_escape_ \c_backslash_str q_recursion_tail :w } - \use_none_delimit_by_q_recursion_stop:w -\cs_new_eq:cN - { str_aux_escape_ / \c_backslash_str q_recursion_tail :w } - \use_none_delimit_by_q_recursion_stop:w -\cs_new_nopar:cpn { str_aux_escape_~:w } { } -\cs_new_nopar:cpx { str_aux_escape_/a:w } - { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^G } -\cs_new_nopar:cpx { str_aux_escape_/t:w } - { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^I } -\cs_new_nopar:cpx { str_aux_escape_/n:w } - { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^J } -\cs_new_nopar:cpx { str_aux_escape_/f:w } - { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^L } -\cs_new_nopar:cpx { str_aux_escape_/r:w } - { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^M } -\cs_new_nopar:cpx { str_aux_escape_/e:w } - { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^[ } -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} +% \subsubsection{Framework for conversions} +% +% Most functions in this module expect to be working with +% \enquote{native} strings. Strings can also be stored as bytes, in one +% of many encodings, for instance \textsc{utf8}. The bytes themselves +% can be expressed in various ways in terms of \TeX{} tokens, for +% instance as pairs of hexadecimal digits. The questions of going from +% arbitrary Unicode code points to bytes, and from bytes to tokens are +% mostly independent. % -% \begin{macro}[aux]{\str_aux_escape_/x:w} -% \begin{macro}[aux]{\str_aux_escape_x_test:N} -% \begin{macro}[aux]{\str_aux_escape_x_unbraced_i:N} -% \begin{macro}[aux]{\str_aux_escape_x_unbraced_ii:N} -% \begin{macro}[aux]{\str_aux_escape_x_braced_loop:N} -% \begin{macro}[aux]{\str_aux_escape_x_braced_end:N} -% \begin{macro}[aux]{\str_aux_escape_x_end:} -% When |\x| is encountered, interrupt the assignment, -% and distinguish the cases of a braced or unbraced syntax. -% In the braced case, collect arbitrarily many hexadecimal digits, -% building the number in \cs{l_str_char_int} (using -% \cs{str_aux_hexadecimal_use:NTF}), and check that -% the run of digits was stopped by a closing brace. -% In the unbraced case, collect up to two hexadecimal digits, -% possibly less, building the character number in \cs{l_str_char_int}. -% In both cases, once all digits have been collected, use -% the \TeX{} primitive \tn{lowercase} to produce that character, -% and use an \cs{if_false:} trick to restart the assignment. +% Conversions are done in four steps: +% \begin{itemize} +% \item \enquote{unescape} produces a string of bytes; +% \item \enquote{decode} takes in a string of bytes, and converts it +% to a list of Unicode characters in an internal representation, +% with items of the form +% \begin{quote} +% \meta{bytes} \cs{s_tl} \meta{Unicode code point} \cs{s_tl} +% \end{quote} +% where we have collected the \meta{bytes} which combined to form +% this particular Unicode character, and the \meta{Unicode code +% point} is in the range $[0,\hexnum{10FFFF}]$. +% \item \enquote{encode} encodes the internal list of code points as a +% byte string in the new encoding; +% \item \enquote{escape} escapes bytes as requested. +% \end{itemize} +% The process is modified in case one of the encoding is empty (or the +% conversion function has been set equal to the empty encoding because +% it was not found): then the unescape or escape step is ignored, and +% the decode or encode steps work on tokens instead of bytes. Otherwise, +% each step must ensure that it passes a correct byte string or internal +% string to the next step. +% +% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn} +% \begin{macro}[TF]{\str_set_convert:Nnnn, \str_gset_convert:Nnnn} +% \begin{macro}[aux]{\str_convert_aux_i:nNNnnn} +% The input string is stored in \cs{g_str_result_tl}, then we: +% unescape and decode; encode and escape; exit the group and store the +% result in the user's variable. The various conversion functions all +% act on \cs{g_str_result_tl}. Errors are silenced for the conditional +% functions by redefining \cs{str_if_flag_error:nnx} locally. % \begin{macrocode} -\cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N - { - \if_false: { \fi: } - \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl } - \int_incr:N \l_str_tmpa_int - \int_zero:N \l_str_char_int - \str_aux_escape_x_test:N - } -\cs_new_protected_nopar:Npx \str_aux_escape_x_test:N #1 - { - \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1 - { \exp_not:N \str_aux_escape_x_test:N } - { - \l_str_char_int = "0 - \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1 - { \exp_not:N \str_aux_escape_x_braced_loop:N } - { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 } - } - } -\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1 +\cs_new_protected_nopar:Npn \str_set_convert:Nnnn + { \str_convert_aux_i:nNNnnn { } \tl_set_eq:NN } +\cs_new_protected_nopar:Npn \str_gset_convert:Nnnn + { \str_convert_aux_i:nNNnnn { } \tl_gset_eq:NN } +\prg_new_protected_conditional:Npnn + \str_set_convert:Nnnn #1#2#3#4 { T , F , TF } { - \str_aux_hexadecimal_use:NTF #1 - { \str_aux_escape_x_unbraced_ii:N } - { \exp_stop_f: \str_aux_escape_x_end: #1 } + \bool_gset_false:N \g_str_error_bool + \str_convert_aux_i:nNNnnn + { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx } + \tl_set_eq:NN #1 {#2} {#3} {#4} + \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true: } -\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1 +\prg_new_protected_conditional:Npnn + \str_gset_convert:Nnnn #1#2#3#4 { T , F , TF } { - \token_if_eq_charcode:NNTF \c_space_token #1 - { \str_aux_escape_x_unbraced_ii:N } - { - \str_aux_hexadecimal_use:NTF #1 - { \exp_stop_f: \str_aux_escape_x_end: } - { \exp_stop_f: \str_aux_escape_x_end: #1 } - } - } -\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1 - { - \token_if_eq_charcode:NNTF \c_space_token #1 - { \str_aux_escape_x_braced_loop:N } - { - \str_aux_hexadecimal_use:NTF #1 - { \str_aux_escape_x_braced_loop:N } - { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 } - } + \bool_gset_false:N \g_str_error_bool + \str_convert_aux_i:nNNnnn + { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx } + \tl_gset_eq:NN #1 {#2} {#3} {#4} + \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true: } -\cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1 +\cs_new_protected:Npn \str_convert_aux_i:nNNnnn #1#2#3#4#5#6 { - \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1 - { \exp_not:N \str_aux_escape_x_end: } - { - \msg_kernel_error:nn { str } { x-missing-brace } - \exp_not:N \str_aux_escape_x_end: #1 - } + \group_begin: + #1 + \str_gset_other:Nn \g_str_result_tl {#4} + \exp_after:wN \str_convert_aux_ii:wwwnn + \tl_to_str:n {#5} /// \q_stop + { decode } { unescape } + \prg_do_nothing: + \str_convert_decode_: + \exp_after:wN \str_convert_aux_ii:wwwnn + \tl_to_str:n {#6} /// \q_stop + { encode } { escape } + \use_ii_i:nn + \str_convert_encode_: + \group_end: + #2 #3 \g_str_result_tl } -\group_begin: - \char_set_catcode_other:N \^^@ - \cs_new_protected_nopar:Npn \str_aux_escape_x_end: - { - \tex_lccode:D \c_zero \l_str_char_int - \tl_to_lowercase:n - { - \tl_gset:Nx \g_str_result_tl - { \if_false: } \fi: - \str_aux_escape_raw:N ^^@ - \str_aux_escape_loop:N - } - } -\group_end: % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsubsection{Framework for encoding conversions} % -% Most functions in this module expect to be working with -% \enquote{native} strings. Strings can also be stored as -% bytes, in one of many encodings, for instance \textsc{utf8}. -% The bytes themselves can be expressed as character -% tokens with the relevant character code, or as pairs -% of hexadecimal digits, \emph{etc.} The questions of going -% from arbitrary characters to bytes, and from bytes to \TeX{} -% tokens are mostly independent. -% -% Conversions are done in four steps: -% \begin{itemize} -% \item \enquote{input} produces a string of bytes; -% \item \enquote{from} interprets the byte string in the old encoding, -% and converts it to a comma-list, each character becoming -% \enquote{\ttfamily \meta{character code},}; -% \item \enquote{to} encodes the internal comma list as a byte string -% in the new encoding; -% \item \enquote{output} escapes bytes as requested. -% \end{itemize} -% The process is modified in case one of the encoding is -% \texttt{internal} or \texttt{native}: then the input or -% output step is ignored. -% -% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn} -% \begin{macro}[aux]{\str_convert_aux_i:NNnnn} % \begin{macro}[aux]{\str_convert_aux_ii:wwwnn} -% \begin{macro}[aux]{\str_convert_aux_iii:nnF} -% \begin{macro}[aux]{\str_convert_aux_iv:nnnF} -% \begin{macro}[aux]{\str_convert_aux_v:NNnNN} -% The input string is stored in \cs{g_str_result_tl}, -% then we call the various conversion functions, which all -% act on \cs{g_str_result_tl}. The arguments |#3| and |#4| -% of the \texttt{aux_i} function are split into their -% \meta{encoding} and \meta{escaping} parts by \texttt{aux_ii}. -% The conversion function names are built by \texttt{aux_iii}, -% which first tries to use the name as supplied by the user, -% then filters it with \cs{str_aux_lowercase_alphanum:n}, -% and calls \texttt{aux_iv} to load the relevant definition file. -% In all cases, \texttt{aux_iii} ensures that the two -% required conversion functions are defined, with -% a fall-back to the native encoding with no escaping. -% Once this is done, \texttt{aux_v} does the main work: +% \begin{macro}[aux]{\str_convert_aux_iii:NNnNN} +% The task of \cs{str_convert_aux_ii:wwwnn} is to split +% \meta{encoding}/\meta{escaping} pairs into their components, |#1| +% and |#2|. Calls to \cs{str_convert_aux_iv:nnn} ensure that the +% corresponding conversion functions are defined. The third auxiliary +% does the main work. % \begin{itemize} -% \item |#1| is the encoding conversion function; -% \item |#2| is the escaping function; -% \item |#3| is the escaping name; -% \item |#4| is the \enquote{native} encoding function -% (either \enquote{from} or \enquote{to}); -% \item |#5| is \cs{use_i:nn} or \cs{use_ii:nn}. +% \item |#1| is the encoding conversion function; +% \item |#2| is the escaping function; +% \item |#3| is the escaping name for use in an error message; +% \item |#4| is \cs{prg_do_nothing:} for unescaping/decoding, and +% \cs{use_ii_i:nn} for encoding/escaping; +% \item |#5| is the default encoding function (either +% \enquote{decode} or \enquote{encode}), for which there should be +% no escaping. % \end{itemize} -% When converting to the internal format, the escaping must be done -% first, then the encoding (this is controlled by |#5|). -% If the encoding is native, then the escaping is ignored and -% must be empty. Otherwise, the escaping |#2| is performed. +% Let us ignore the native encoding for a second. In the +% unescaping/decoding phase, we want to do |#2#1| in this order, and +% in the encoding/escaping phase, the order should be reversed: +% |#4#2#1| does exactly that. If one of the encodings is the default +% (native), then the escaping should be ignored, with an error if any +% was given, and only the encoding, |#1|, should be performed. % \begin{macrocode} -\cs_new_protected:Npn \str_set_convert:Nnnn - { \str_convert_aux_i:NNnnn \tl_set_eq:NN } -\cs_new_protected:Npn \str_gset_convert:Nnnn - { \str_convert_aux_i:NNnnn \tl_gset_eq:NN } -\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5 +\cs_new_protected:Npn \str_convert_aux_ii:wwwnn + #1 / #2 // #3 \q_stop #4#5 { - \group_begin: - \str_gset_other:Nn \g_str_result_tl {#5} - - \exp_after:wN \str_convert_aux_ii:wwwnn - \tl_to_str:n {#3} /// \q_stop - { from } { input } \str_convert_from_native: \use_i:nn - - \exp_after:wN \str_convert_aux_ii:wwwnn - \tl_to_str:n {#4} /// \q_stop - { to } { output } \str_convert_to_native: \use_ii:nn - - \group_end: - #1 #2 \g_str_result_tl - } -\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5 - { - \str_convert_aux_iii:nnF {#4} {#1} {native} - \str_convert_aux_iii:nnF {#5} {#2} { } - \exp_args:Ncc \str_convert_aux_v:NNnNN + \str_convert_aux_iv:nnn {enc} {#4} {#1} + \str_convert_aux_iv:nnn {esc} {#5} {#2} + \exp_args:Ncc \str_convert_aux_iii:NNnNN { str_convert_#4_#1: } { str_convert_#5_#2: } {#2} } -\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3 +\cs_new_protected:Npn \str_convert_aux_iii:NNnNN #1#2#3#4#5 { - \cs_if_exist:cF { str_convert_#1_#2: } - { - \exp_args:Nx \str_convert_aux_iv:nnnF - { \str_aux_lowercase_alphanum:n {#2} } - {#1} {#2} {#3} - } + \if_meaning:w #1 #5 + \tl_if_empty:nF {#3} + { \msg_kernel_error:nnx { str } { native-escaping } {#3} } + #1 + \else: + #4 #2 #1 + \fi: } -\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4 +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux]{\str_convert_aux_iv:nnn} +% \begin{macro}[aux]{\str_convert_aux_v:nnnn} +% The arguments of \cs{str_convert_aux_iv:nnn} are: \texttt{enc} or +% \texttt{esc}, used to build filenames, the type of the conversion +% (unescape, decode, encode, escape), and the encoding or escaping +% name. If the function is already defined, no need to do anything. +% Otherwise, filter out all non-alphanumerics in the name, and +% lowercase it. Feed that, and the same three arguments, to +% \cs{str_convert_aux_v:nnnn}. The task is then to make sure that the +% conversion function |#3_#1| corresponding to the type |#3| and +% filtered name |#1| is defined, then set our initial conversion +% function |#3_#4| equal to that. +% +% How do we get the |#3_#1| conversion to be defined if it isn't? +% Two main cases. +% +% First, if |#1| is a key in \cs{g_str_file_prop}, then the value +% \cs{l_str_internal_tl} tells us what file to load. Loading is +% skipped if the file was already read, \emph{i.e.}, if the conversion +% command based on \cs{l_str_internal_tl} already exists. Otherwise, +% try to load the file; if that fails, there is an error, use the +% default empty name instead. +% +% Second, |#1| may be absent from the property list. The +% \cs{cs_if_exist:cF} test is automatically false, and we search for a +% file defining the encoding or escaping |#1| (this should allow +% third-party \texttt{.def} files). If the file is not found, there is +% an error, use the default empty name instead. +% +% In all cases, the conversion based on \cs{l_str_internal_tl} is +% defined, so we can set the |#3_#1| function equal to that. In some +% cases (\emph{e.g.}, \texttt{utf16be}), the |#3_#1| function is +% actually defined within the file we just loaded, and it is different +% from the \cs{l_str_internal_tl}-based function: we mustn't clobber +% that different definition. +% \begin{macrocode} +\cs_new_protected:Npn \str_convert_aux_iv:nnn #1#2#3 { - \cs_if_exist:cF { str_convert_#2_#1: } - { - \group_begin: - \str_load_catcodes: - \str_load_one:n {#1} - \group_end: - } - \cs_if_exist:cTF { str_convert_#2_#1: } - { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } } + \cs_if_exist:cF { str_convert_#2_#3: } { - \msg_kernel_error:nnx { str } { unknown-#2 } {#3} - \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: } + \exp_args:Nx \str_convert_aux_v:nnnn + { \str_convert_lowercase_alphanum:n {#3} } + {#1} {#2} {#3} } } -\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5 +\cs_new_protected:Npn \str_convert_aux_v:nnnn #1#2#3#4 { - #5 { } {#1} - \cs_if_eq:NNTF #1 #4 + \cs_if_exist:cF { str_convert_#3_#1: } { - \tl_if_empty:nF {#3} - { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} } + \prop_get:NnNF \g_str_file_prop {#1} \l_str_internal_tl + { \tl_set:Nn \l_str_internal_tl {#1} } + \cs_if_exist:cF { str_convert_#3_ \l_str_internal_tl : } + { + \file_if_exist:nTF { l3str-#2- \l_str_internal_tl .def } + { + \group_begin: + \str_load_catcodes: + \file_input:n { l3str-#2- \l_str_internal_tl .def } + \group_end: + } + { + \tl_clear:N \l_str_internal_tl + \msg_kernel_error:nnxx { str } { unknown-#2 } {#4} {#1} + } + } + \cs_if_exist:cF { str_convert_#3_#1: } + { + \cs_gset_eq:cc { str_convert_#3_#1: } + { str_convert_#3_ \l_str_internal_tl : } + } } - { #2 } - #5 {#1} { } + \cs_gset_eq:cc { str_convert_#3_#4: } { str_convert_#3_#1: } } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % -% \begin{macro}[int, rEXP]{\str_aux_lowercase_alphanum:n} -% \begin{macro}[aux, rEXP]{\str_aux_lowercase_alphanum_loop:N} -% This function keeps only letters and digits, with upper case -% letters converted to lower case. +% \begin{macro}[int, rEXP]{\str_convert_lowercase_alphanum:n} +% \begin{macro}[aux, rEXP]{\str_convert_lowercase_alphanum_loop:N} +% This function keeps only letters and digits, with upper case letters +% converted to lower case. % \begin{macrocode} -\cs_new:Npn \str_aux_lowercase_alphanum:n #1 +\cs_new:Npn \str_convert_lowercase_alphanum:n #1 { - \exp_after:wN \str_aux_lowercase_alphanum_loop:N + \exp_after:wN \str_convert_lowercase_alphanum_loop:N \tl_to_str:n {#1} { ? \prg_map_break: } \prg_break_point:n { } } -\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1 +\cs_new:Npn \str_convert_lowercase_alphanum_loop:N #1 { \use_none:n #1 \if_num:w `#1 < \c_ninety_one @@ -1856,149 +1778,331 @@ \fi: \fi: \fi: - \str_aux_lowercase_alphanum_loop:N + \str_convert_lowercase_alphanum_loop:N } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_convert_from_internal:} -% \begin{macro}[int]{\str_convert_to_internal:} -% Converting from the internal format to itself is -% of course trivial. +% \begin{macro}[int]{\str_load_catcodes:} +% Since encoding files may be loaded at arbitrary places in a \TeX{} +% document, including within verbatim mode, we set the catcodes of all +% characters appearing in any encoding definition file. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_convert_from_internal: { } -\cs_new_protected_nopar:Npn \str_convert_to_internal: { } +\cs_new_protected:Npn \str_load_catcodes: + { + \char_set_catcode_escape:N \\ + \char_set_catcode_group_begin:N \{ + \char_set_catcode_group_end:N \} + \char_set_catcode_math_toggle:N \$ + \char_set_catcode_alignment:N \& + \char_set_catcode_parameter:N \# + \char_set_catcode_math_superscript:N \^ + \char_set_catcode_ignore:N \ % + \char_set_catcode_space:N \~ + \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFILNPSTUX } + \char_set_catcode_letter:N + \tl_map_function:nN { 0123456789"'?*+-.(),`!/<>[];= } + \char_set_catcode_other:N + \char_set_catcode_comment:N \% + \int_set:Nn \tex_endlinechar:D {32} + } % \end{macrocode} % \end{macro} -% \end{macro} % -% \subsubsection{Loading encoding files} +% \subsubsection{Byte unescape and escape} % -% \begin{macro}{\str_load:n} -% \begin{macro}[int]{\str_load_one:n} -% \begin{macro}[aux]{\str_load_alias:n, \str_load_alias_aux:nnn} -% Loading encoding or escaping files can be done in bulk by -% specifying a comma-list of encodings and escapings. Each -% name is normalized: only alphanumeric characters are kept, -% lowercased. The relevant file is loaded if it was not loaded -% already. +% Strings of bytes may need to be stored in auxiliary files in safe +% \enquote{escaping} formats. Each such escaping is only loaded as +% needed. By default, on input any non-byte is filtered out, while the +% output simply consists in letting bytes through. +% +% \begin{macro}[int, rEXP]{\str_filter_bytes:n} +% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N} +% In the case of pdf\TeX{}, every character is a byte. For +% Unicode-aware engines, test the character code; non-bytes cause us +% to raise the flag \texttt{str_byte}. Spaces have already been given +% the correct category code when this function is called. % \begin{macrocode} -\cs_new_protected:Npn \str_load:n #1 - { - \group_begin: - \str_load_catcodes: - \clist_map_inline:nn {#1} - { - \exp_args:Nx \str_load_one:n - { \str_aux_lowercase_alphanum:n {#1} } - } - \group_end: - } -\cs_new_protected:Npn \str_load_one:n #1 +\pdftex_if_engine:TF + { \cs_new_eq:NN \str_filter_bytes:n \use:n } { - \cs_if_exist:cF { str_convert_from_ #1 : } + \cs_new:Npn \str_filter_bytes:n #1 { - \cs_if_exist:cF { str_convert_input_#1: } - { - \file_if_exist:nTF { l3str- #1 .def } - { \file_input:n { l3str- #1 .def } } - { \str_load_alias:n {#1} } - } + \str_filter_bytes_aux:N #1 + { ? \prg_map_break: } + \prg_break_point:n { } + } + \cs_new:Npn \str_filter_bytes_aux:N #1 + { + \use_none:n #1 + \if_num:w `#1 < 256 \exp_stop_f: + #1 + \else: + \flag_raise:n { str_byte } + \fi: + \str_filter_bytes_aux:N } } -\cs_new_protected:Npn \str_load_alias:n #1 +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_unescape_:} +% \begin{macro}[int]{\str_convert_unescape_bytes:} +% The simplest unescaping method removes non-bytes from +% \cs{g_str_result_tl}. +% \begin{macrocode} +\pdftex_if_engine:TF + { \cs_new_protected_nopar:Npn \str_convert_unescape_: { } } { - \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl + \cs_new_protected_nopar:Npn \str_convert_unescape_: { - \str_load_one:n { \l_str_tmpa_tl } - \cs_if_exist:cTF - { str_convert_from_ \l_str_tmpa_tl : } - { - \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from } - \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to } - } - { - \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input } - \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output } - } + \flag_clear:n { str_byte } + \tl_gset:Nx \g_str_result_tl + { \exp_args:No \str_filter_bytes:n \g_str_result_tl } + \str_if_flag_error:nnx { str_byte } { non-byte } { bytes } } - { \msg_kernel_error:nnx { str } { file-not-found } {#1} } } -\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3 - { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } } +\cs_new_eq:NN \str_convert_unescape_bytes: \str_convert_unescape_: % \end{macrocode} % \end{macro} % \end{macro} +% +% \begin{macro}[int]{\str_convert_escape_:} +% \begin{macro}[int]{\str_convert_escape_bytes:} +% The simplest form of escape leaves the bytes from the previous step +% of the conversion unchanged. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \str_convert_escape_: { } +\cs_new_eq:NN \str_convert_escape_bytes: \str_convert_escape_: +% \end{macrocode} +% \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_load_catcodes:} -% Since encoding files may be loaded at arbitrary places -% in a \TeX{} document, including within verbatim mode, -% we set the catcodes of all characters appearing in any -% encoding definition file. +% \subsubsection{Native strings} +% +% \begin{macro}[int]{\str_convert_decode_:} +% \begin{macro}[aux, rEXP]{\str_decode_native_char:N} +% Convert each character to its character code, one at a time. % \begin{macrocode} -\cs_new_protected:Npn \str_load_catcodes: +\cs_new_protected_nopar:Npn \str_convert_decode_: + { \str_convert_gmap:N \str_decode_native_char:N } +\cs_new:Npn \str_decode_native_char:N #1 + { #1 \s_tl \int_value:w `#1 \s_tl } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_encode_:} +% The conversion from an internal string to native character tokens is +% very different in pdf\TeX{} and in other engines. For Unicode-aware +% engines, we need the definitions to be read when the null byte has +% category code $12$, so we set that inside a group. +% \begin{macrocode} +\group_begin: + \char_set_catcode_other:n { 0 } + \pdftex_if_engine:TF +% \end{macrocode} +% \begin{macro}[aux, EXP]{\str_encode_native_char:n} +% Since pdf\TeX{} only supports 8-bit characters, and we have a table +% of all bytes, the conversion can be done in linear time within an +% \texttt{x}-expanding assignment. Look out for character codes larger +% than $255$, those characters are replaced by |?|, and raise a flag, +% which then triggers a pdf\TeX{}-specific error. +% \begin{macrocode} + { + \cs_new_protected_nopar:Npn \str_convert_encode_: + { + \flag_clear:n { str_error } + \str_convert_gmap_internal:N \str_encode_native_char:n + \str_if_flag_error:nnx { str_error } + { pdfTeX-native-overflow } { } + } + \cs_new:Npn \str_encode_native_char:n #1 + { + \if_num:w #1 < \c_two_hundred_fifty_six + \str_output_byte:n {#1} + \else: + \flag_raise:n { str_error } + ? + \fi: + } + \msg_kernel_new:nnnn { str } { pdfTeX-native-overflow } + { Character~code~too~large~for~pdfTeX. } + { + The~pdfTeX~engine~only~supports~8-bit~characters:~ + valid~character~codes~are~in~the~range~[0,255].~ + To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX. + } + } +% \end{macrocode} +% \end{macro} +% \begin{macro}[aux]{\str_encode_native_loop:w} +% \begin{macro}[aux]{\str_encode_native_flush:} +% \begin{macro}[aux, rEXP]{\str_encode_native_filter:N} +% In Unicode-aware engines, since building particular characters +% cannot be done expandably in \TeX{}, we cannot hope to get a +% linear-time function. However, we get quite close by building the +% result one block of $255$ characters at a time. Set the lowercase +% code of the \textsc{ascii} character $1$ to the first character code +% in the block, \emph{etc.}, then lowercase the contents of +% \cs{c_str_positive_bytes_tl}. Unfortunately, this is complicated by +% the fact that \TeX{} won't lowercase a character to the null +% character, thus the need for filtering: whenever the lowercase code +% of a given character is zero, it is replaced \enquote{by hand} by +% the null character. Unicode-aware engines will never incur an +% overflow because the internal string is guaranteed to only contain +% code points in $[0,\hexnum{10FFFF}]$. +% \begin{macrocode} + { + \cs_new_protected_nopar:Npn \str_convert_encode_: + { + \tex_lccode:D \c_zero \c_zero + \int_zero:N \l_str_internal_int + \tl_set_eq:NN \l_str_internal_tl \g_str_result_tl + \tl_gclear:N \g_str_result_tl + \exp_after:wN \str_encode_native_loop:w + \l_str_internal_tl \s_tl { \q_stop \prg_map_break: } \s_tl + \prg_break_point:n { \str_encode_native_flush: } + } + \cs_new_protected:Npn \str_encode_native_loop:w #1 \s_tl #2 \s_tl + { + \use_none_delimit_by_q_stop:w #2 \q_stop + \if_num:w \l_str_internal_int = \c_two_hundred_fifty_five + \str_encode_native_flush: + \l_str_internal_int \c_zero + \fi: + \tex_advance:D \l_str_internal_int \c_one + \tex_lccode:D \l_str_internal_int \int_eval:w #2 \int_eval_end: + \str_encode_native_loop:w + } + \cs_new_protected_nopar:Npn \str_encode_native_flush: + { + \use:x + { + \tl_to_lowercase:n + { + \tl_gput_right:Nx \exp_not:N \g_str_result_tl + { + \exp_after:wN \str_encode_native_filter:N + \c_str_positive_bytes_tl + { ? = \c_zero \fi: \prg_map_break: } + \prg_break_point:n { } + } + } + } + } + \cs_new:Npn \str_encode_native_filter:N #1 + { + \if_num:w `#1 > \l_str_internal_int + \exp_after:wN \prg_map_break: + \fi: + \if_num:w \tex_lccode:D `#1 = \c_zero + ^^@ + \else: + #1 + \fi: + \str_encode_native_filter:N + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% End the group to restore the catcode of the null byte. +% \begin{macrocode} +\group_end: +% \end{macrocode} +% \end{macro} +% +% \subsubsection{8-bit encodings} +% +% This section will be entirely rewritten: it is not yet clear in what +% situations 8-bit encodings are used, hence I don't know what exactly +% should be optimized. The current approach is reasonably efficient to +% convert long strings, and it scales well when using many different +% encodings. An approach based on csnames would have a smaller constant +% load time for each individual conversion, but has a large hash table +% cost. Using a range of \tn{count} registers works for decoding, but +% not for encoding: one possibility there would be to use a binary tree +% for the mapping of Unicode characters to bytes, stored as a box, one +% per encoding. +% +% Since the section is going to be rewritten, documentation lacks. +% +% All the 8-bit encodings which \pkg{l3str} supports rely on the same +% internal functions. +% +% \begin{macro}[int]{\str_declare_eight_bit_encoding:nnn} +% All the 8-bit encoding definition file start with +% \cs{str_declare_eight_bit_encoding:nnn} \Arg{encoding name} +% \Arg{mapping} \Arg{missing bytes}. The \meta{mapping} argument is a +% token list of pairs \Arg{byte} \Arg{Unicode} expressed in uppercase +% hexadecimal notation. The \meta{missing} argument is a token list +% of \Arg{byte}. Every \meta{byte} which does not appear in the +% \meta{mapping} nor the \meta{missing} lists maps to the same code +% point in Unicode. +% \begin{macrocode} +\cs_new_protected:Npn \str_declare_eight_bit_encoding:nnn #1#2#3 { - \char_set_catcode_escape:N \\ - \char_set_catcode_group_begin:N \{ - \char_set_catcode_group_end:N \} - \char_set_catcode_math_superscript:N \^ - \char_set_catcode_ignore:N \ % - \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN } - \char_set_catcode_letter:N - \tl_map_function:nN { 0123456789"'?*+ } - \char_set_catcode_other:N - \char_set_catcode_comment:N \% - \int_set:Nn \tex_endlinechar:D {32} + \tl_set:Nn \l_str_internal_tl {#1} + \cs_new_protected_nopar:cpn { str_convert_decode_#1: } + { \str_convert_decode_eight_bit:n {#1} } + \cs_new_protected_nopar:cpn { str_convert_encode_#1: } + { \str_convert_encode_eight_bit:n {#1} } + \tl_const:cn { c_str_encoding_#1_tl } {#2} + \tl_const:cn { c_str_encoding_#1_missing_tl } {#3} } % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\str_convert_from_eight_bit:n} -% \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn} -% \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n} -% \begin{macro}[aux, EXP]{\str_convert_from_eight_bit_aux:N} +% \begin{macro}[int]{\str_convert_decode_eight_bit:n} +% \begin{macro}[aux]{\str_decode_eight_bit_load:nn} +% \begin{macro}[aux]{\str_decode_eight_bit_load_missing:n} +% \begin{macro}[aux, EXP]{\str_decode_eight_bit_char:N} %^^A todo: document % \begin{macrocode} -\cs_new_protected:Npn \str_convert_from_eight_bit:n #1 +\cs_new_protected:Npn \str_convert_decode_eight_bit:n #1 { \group_begin: - \int_zero:N \l_str_tmpa_int - \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn + \int_zero:N \l_str_internal_int + \exp_last_unbraced:Nx \str_decode_eight_bit_load:nn { \tl_use:c { c_str_encoding_#1_tl } } { \q_stop \prg_map_break: } { } \prg_break_point:n { } - \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n + \exp_last_unbraced:Nx \str_decode_eight_bit_load_missing:n { \tl_use:c { c_str_encoding_#1_missing_tl } } { \q_stop \prg_map_break: } \prg_break_point:n { } - \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N + \flag_clear:n { str_error } + \str_convert_gmap:N \str_decode_eight_bit_char:N + \str_if_flag_error:nnx { str_error } { decode-8-bit } {#1} \group_end: } -\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2 +\cs_new_protected:Npn \str_decode_eight_bit_load:nn #1#2 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: - \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 } - \tex_advance:D \l_str_tmpa_int \c_one - \str_convert_from_eight_bit_load:nn + \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop: + \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_internal_int \exp_after:wN { \int_value:w "#2 } + \tex_advance:D \l_str_internal_int \c_one + \str_decode_eight_bit_load:nn } -\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1 +\cs_new_protected:Npn \str_decode_eight_bit_load_missing:n #1 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop: - \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop: - \tex_toks:D \l_str_tmpa_int \exp_after:wN + \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop: + \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_internal_int \exp_after:wN { \int_use:N \c_str_replacement_char_int } - \tex_advance:D \l_str_tmpa_int \c_one - \str_convert_from_eight_bit_load_missing:n + \tex_advance:D \l_str_internal_int \c_one + \str_decode_eight_bit_load_missing:n } -\cs_new:Npn \str_convert_from_eight_bit_aux:N #1 +\cs_new:Npn \str_decode_eight_bit_char:N #1 { #1 \s_tl - \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int + \if_num:w \tex_dimen:D `#1 < \l_str_internal_int \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D \fi: @@ -2011,54 +2115,55 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_convert_to_eight_bit:n} -% \begin{macro}[aux]{\str_convert_to_eight_bit_load:nn} -% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux:n} -% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux_ii:n} +% \begin{macro}[int]{\str_convert_encode_eight_bit:n} +% \begin{macro}[aux]{\str_encode_eight_bit_load:nn} +% \begin{macro}[aux, rEXP]{\str_encode_eight_bit_char:n} +% \begin{macro}[aux, rEXP]{\str_encode_eight_bit_char_aux:n} %^^A todo: document % \begin{macrocode} -\cs_new_protected:Npn \str_convert_to_eight_bit:n #1 +\cs_new_protected:Npn \str_convert_encode_eight_bit:n #1 { \group_begin: - \int_zero:N \l_str_tmpa_int - \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn + \int_zero:N \l_str_internal_int + \exp_last_unbraced:Nx \str_encode_eight_bit_load:nn { \tl_use:c { c_str_encoding_#1_tl } } { \q_stop \prg_map_break: } { } \prg_break_point:n { } - \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n + \flag_clear:n { str_error } + \str_convert_gmap_internal:N \str_encode_eight_bit_char:n + \str_if_flag_error:nnx { str_error } { encode-8-bit } {#1} \group_end: } -\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2 +\cs_new_protected:Npn \str_encode_eight_bit_load:nn #1#2 { \use_none_delimit_by_q_stop:w #1 \q_stop - \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop: - \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop: - \exp_args:NNf \tex_toks:D \l_str_tmpa_int + \tex_dimen:D "#2 = \l_str_internal_int sp \scan_stop: + \tex_skip:D \l_str_internal_int = "#2 sp \scan_stop: + \exp_args:NNf \tex_toks:D \l_str_internal_int { \str_output_byte:n { "#1 } } - \tex_advance:D \l_str_tmpa_int \c_one - \str_convert_to_eight_bit_load:nn + \tex_advance:D \l_str_internal_int \c_one + \str_encode_eight_bit_load:nn } -\cs_new:Npn \str_convert_to_eight_bit_aux:n #1 +\cs_new:Npn \str_encode_eight_bit_char:n #1 { \if_num:w #1 > \c_max_register_int - \msg_expandable_kernel_error:nnn - { str } { eight-bit-to-byte-overflow } {#1} + \flag_raise:n { str_error } \else: - \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int + \if_num:w \tex_dimen:D #1 < \l_str_internal_int \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f: \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f: \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn \fi: \fi: - \str_convert_to_eight_bit_aux_ii:n {#1} + \str_encode_eight_bit_char_aux:n {#1} \fi: } -\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1 +\cs_new:Npn \str_encode_eight_bit_char_aux:n #1 { \if_num:w #1 < \c_two_hundred_fifty_six \str_output_byte:n {#1} \else: - \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte } + \flag_raise:n { str_error } \fi: } % \end{macrocode} @@ -2067,255 +2172,72 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_encoding_eight_bit:n} -%^^A todo: document -% \begin{macrocode} -\cs_new_protected:Npn \str_encoding_eight_bit:n #1 - { - \cs_new_protected:cpn { str_convert_from_#1: } - { \str_convert_from_eight_bit:n {#1} } - \cs_new_protected:cpn { str_convert_to_#1: } - { \str_convert_to_eight_bit:n {#1} } - } -% \end{macrocode} -% \end{macro} -% -% \subsubsection{Byte input and output} -% -% Strings of bytes may need to be stored in auxiliary files -% in safe \enquote{escaping} formats. Each such escaping -% is only loaded as needed. By default, on input any non-byte -% is filtered out, while the output simply consists in letting -% bytes through. -% -% \begin{macro}[int, rEXP]{\str_filter_bytes:n} -% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N} -% In the case of pdf\TeX{}, every character is a byte. -% For Unicode-aware engines, test the character code. -% Spaces have already been given the correct category -% code when this function is called. -% \begin{macrocode} -\pdftex_if_engine:TF - { \cs_new_eq:NN \str_filter_bytes:n \use:n } - { - \cs_new_nopar:Npn \str_filter_bytes:n #1 - { - \str_filter_bytes_aux:N #1 - { ? \prg_map_break: } - \prg_break_point:n { } - } - \cs_new_nopar:Npn \str_filter_bytes_aux:N #1 - { - \use_none:n #1 - \if_num:w `#1 < 256 \exp_stop_f: #1 \fi: - \str_filter_bytes_aux:N - } - } -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \begin{macro}[int]{\str_convert_input_:} -% \begin{macro}[int]{\str_convert_input_bytes:} -% The simplest input method removes non-bytes -% from \cs{g_str_result_tl}. -% \begin{macrocode} -\pdftex_if_engine:TF - { \cs_new_protected_nopar:Npn \str_convert_input_: { } } - { - \cs_new_protected_nopar:Npn \str_convert_input_: - { - \tl_gset:Nx \g_str_result_tl - { \exp_args:No \str_filter_bytes:n \g_str_result_tl } - } - } -\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_: -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \begin{macro}[int]{\str_convert_output_:} -% \begin{macro}[int]{\str_convert_output_bytes:} -% The simplest form of output leaves the bytes from the previous -% step of the conversion unchanged. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \str_convert_output_: { } -\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_: -% \end{macrocode} -% \end{macro} -% \end{macro} -% -% \subsubsection{Convert to and from native strings} -% -% \begin{macro}[int]{\str_convert_from_native:} -% \begin{macro}[aux, rEXP]{\str_convert_from_native_aux:N} -% Convert each character to its character code, one at a time. -% \begin{macrocode} -\cs_new_protected_nopar:Npn \str_convert_from_native: - { \str_aux_gmap_result:N \str_convert_from_native_aux:N } -\cs_new:Npn \str_convert_from_native_aux:N #1 - { #1 \s_tl \int_value:w `#1 \s_tl } -% \end{macrocode} -% \end{macro} -% \end{macro} +% \subsection{Messages} % -% \begin{macro}[int]{\str_convert_to_native:} -% \begin{macro}[aux]{\str_convert_to_native_aux:w} -% This function is based on the internal functions -% \cs{str_aux_convert_to_native:n} and -% \cs{str_aux_convert_to_native_step:n}. -% Loop through the comma-list, and call -% \cs{str_aux_convert_to_native_step:n} -% with the relevant integer argument. -% We need to store \cs{g_str_result_tl} in a temporary -% token list, because that global \enquote{result} is -% cleared silently by \cs{str_aux_convert_to_native:n}. -%^^A todo: update +% General messages, and messages for the encodings and escapings loaded +% by default (\enquote{native}, and \enquote{bytes}). % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_convert_to_native: - { - \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl - \str_aux_convert_to_native:n - { - \exp_after:wN \str_convert_to_native_aux:w - \l_str_tmpa_tl \s_tl { \q_stop \prg_map_break: } \s_tl - } - } -\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 \s_tl #2 \s_tl +\msg_kernel_new:nnn { str } { unknown-esc } + { Escaping~scheme~'#1'~(filtered:~'#2')~unknown. } +\msg_kernel_new:nnn { str } { unknown-enc } + { Encoding~scheme~'#1'~(filtered:~'#2')~unknown. } +\msg_kernel_new:nnnn { str } { native-escaping } + { The~'native'~encoding~scheme~does~not~support~any~escaping. } { - \use_none_delimit_by_q_stop:w #2 \q_stop - \str_aux_convert_to_native_step:n {#2} - \str_convert_to_native_aux:w + Since~native~strings~do~not~consist~in~bytes,~ + none~of~the~escaping~methods~make~sense.~ + The~specified~escaping,~'#1',~will be ignored. } +\msg_kernel_new:nnn { str } { file-not-found } + { File~'l3str-#1.def'~not~found. } % \end{macrocode} -% \end{macro} -% \end{macro} % -% \begin{macro}[int]{\str_aux_convert_to_native:n} -% \begin{macro}[int]{\str_aux_convert_to_native_step:n} -% \begin{macro}[aux]{\str_aux_convert_to_native_flush:} -% \begin{macro}[aux, rEXP]{\str_aux_convert_to_native_filter:N} -%^^A todo: document. +% Message used when the \enquote{bytes} unescaping fails because the +% string given to \cs{str_set_convert:Nnnn} contains a non-byte. This +% cannot happen for the pdf\TeX{} engine, since that engine only +% supports 8-bit characters. Messages used for other escapings and +% encodings are defined in each definition file. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1 - { - \tex_lccode:D \c_zero \c_zero - \int_zero:N \l_str_tmpa_int - \tl_gclear:N \g_str_result_tl - #1 - \prg_break_point:n { \str_aux_convert_to_native_flush: } - } -\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1 - { - \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five - \str_aux_convert_to_native_flush: - \l_str_tmpa_int \c_zero - \fi: - \tex_advance:D \l_str_tmpa_int \c_one - \if_int_compare:w #1 > \c_max_char_int - \msg_kernel_error:nnx { str } { overflow } {#1} - \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int - \else: - \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end: - \fi: - } -\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush: +\pdftex_if_engine:F { - \use:x + \msg_kernel_new:nnnn { str } { non-byte } + { String~invalid~in~escaping~'#1':~it~may~only~contain~bytes. } { - \tl_to_lowercase:n + Some~characters~in~the~string~you~asked~to~convert~are~not~ + 8-bit~characters.~Perhaps~the~string~is~a~'native'~Unicode~string?~ + If~it~is,~try~using\\ + \\ + \iow_indent:n { - \tl_gput_right:Nx \exp_not:N \g_str_result_tl - { - \exp_after:wN \str_aux_convert_to_native_filter:N - \c_str_positive_bytes_tl - { ? = \c_zero \fi: \prg_map_break: } - \prg_break_point:n { } - } + \iow_char:N\\str_set_convert:Nnnn \\ + \ \ <str~var>~\{~<string>~\}~\{~native~\}~\{~<target~encoding>~\} } } } -\group_begin: - \char_set_catcode_other:n { 0 } - \cs_new:Npn \str_aux_convert_to_native_filter:N #1 - { - \if_num:w `#1 > \l_str_tmpa_int - \exp_after:wN \prg_map_break: - \fi: - \if_num:w \tex_lccode:D `#1 = \c_zero - ^^@ - \else: - #1 - \fi: - \str_aux_convert_to_native_filter:N - } -\group_end: % \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% -% \subsection{Messages} % +% Those messages are used when converting to and from 8-bit encodings. % \begin{macrocode} -\msg_kernel_new:nnnn { str } { x-missing-brace } - { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. } +\msg_kernel_new:nnnn { str } { decode-8-bit } + { Invalid~string~in~encoding~'#1'. } { - You~wrote~something~like~ - `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~ - The~closing~brace~is~missing. + LaTeX~came~across~a~byte~which~is~not~defined~to~represent~ + any~character~in~the~encoding~'#1'. } -\msg_kernel_new:nnnn { str } { overflow } - { Character~code~#1~too~big. } +\msg_kernel_new:nnnn { str } { encode-8-bit } + { Unicode~string~cannot~be~converted~to~encoding~'#1'. } { - \int_compare:nNnTF {#1} > { 1114111 } - { The~Unicode~standard~limits~code~points~to~1114111. } - { - The~pdfTeX~engine~only~supports~8-bit~characters:~ - valid~character~codes~are~in~the~range~[0,255].~ - To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX. - } + The~encoding~'#1'~only~contains~a~subset~of~all~Unicode~characters.~ + LaTeX~was~asked~to~convert~a~string~to~that~encoding,~but~that~ + string~contains~a~character~that~'#1'~does~not~support. } -\msg_kernel_new:nnn { str } { convert-input } - { Input~scheme~`#1'~unknown. } -\msg_kernel_new:nnn { str } { convert-from } - { Encoding~`#1'~unknown. } -\msg_kernel_new:nnn { str } { convert-to } - { Encoding~`#1'~unknown.~Using~UTF-8. } -\msg_kernel_new:nnn { str } { convert-output } - { Output~scheme~`#1~unknown. } -\msg_kernel_new:nnn { str } { to-native } - { Use~\str_set_convert:Nnn to~convert~to~native~strings. } -\msg_kernel_new:nnn { str } { unicode-surrogate } - { Code~point~#1~is~an~unpaired~surrogate. } -\msg_kernel_new:nnn { str } { utf16-surrogate } - { UTF-16~unpaired~surrogate~#1. } -\msg_kernel_new:nnn { str } { utf16-odd } - { UTF-16~dangling~byte~#1. } -\msg_kernel_new:nnn { str } { utf8-extra-conti } - { UTF-8~extra~continuation~byte~#1. } -\msg_kernel_new:nnn { str } { utf8-missing-conti } - { UTF-8~missing~continuation~byte~#1. } -\msg_kernel_new:nnn { str } { utf8-invalid-byte } - { Byte~#1~cannot~appear~in~UTF-8. } -\msg_kernel_new:nnn { str } { utf8-overlong } - { Overlong~UTF-8~byte~sequence~for~code~point~#1. } -\msg_kernel_new:nnn { str } { utf8-premature-end } - { Incomplete~last~UTF-8~character. } -\msg_kernel_new:nnn { str } { utf8-overflow } - { Code~point~#1~too~large~(UTF-8). } -\msg_kernel_new:nnn { str } { utf32-overflow } - { Code~point~too~large~(UTF-32~`#1'). } -\msg_kernel_new:nnn { str } { utf32-truncated } - { Truncated~UTF-32~string~`...#1'. } % \end{macrocode} % % \subsection{Deprecated string functions} % % \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n} -% The naming scheme is a little bit more consistent -% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}. +% The naming scheme is a little bit more consistent with +% \enquote{ignore_spaces} instead of \enquote{skip_spaces}. % \begin{macrocode} \cs_set:Npn \str_length_skip_spaces:N { \exp_args:No \str_length_skip_spaces:n } @@ -2329,102 +2251,116 @@ % % \subsection{Escaping definition files} % -% Several of those encodings are defined by the pdf file format. -% The following byte storage methods are defined: +% Several of those encodings are defined by the pdf file format. The +% following byte storage methods are defined: % \begin{itemize} -% \item \texttt{bytes} (default), non-bytes are filtered out, -% and bytes are left untouched; -% \item \texttt{hex} or \texttt{hexadecimal}, -% as per the pdf\TeX{} primitive \tn{pdfescapehex} -% \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename} -% \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring} -% \item \texttt{url}, as per the percent encoding of urls. +% \item \texttt{bytes} (default), non-bytes are filtered out, and +% bytes are left untouched (this is defined by default); +% \item \texttt{hex} or \texttt{hexadecimal}, as per the pdf\TeX{} +% primitive \tn{pdfescapehex} +% \item \texttt{name}, as per the pdf\TeX{} primitive +% \tn{pdfescapename} +% \item \texttt{string}, as per the pdf\TeX{} primitive +% \tn{pdfescapestring} +% \item \texttt{url}, as per the percent encoding of urls. % \end{itemize} % -% \subsubsection{Input methods} -% -% \begin{macro}[int]{\str_convert_input_hex:} -% \begin{macro}[int]{\str_convert_input_hexadecimal:} -% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux:N} -% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux_ii:N} -% Take chars two by two, and interpret each pair as -% the hexadecimal code for a byte. Anything else than -% hexadecimal digits is ignored. -% A string which contains an odd number of hexadecimal -% digits gets |0| appended to it: this is equivalent -% to appending a |0| in all cases, and dropping it if -% it is alone. +% \subsubsection{Unescape methods} +% +% \begin{macro}[int]{\str_convert_unescape_hex:} +% \begin{macro}[aux, rEXP]{\str_unescape_hex_i:N} +% \begin{macro}[aux, rEXP]{\str_unescape_hex_ii:N} +% Take chars two by two, and interpret each pair as the hexadecimal +% code for a byte. Anything else than hexadecimal digits is ignored, +% raising the flag. A string which contains an odd number of +% hexadecimal digits gets |0| appended to it: this is equivalent to +% appending a |0| in all cases, and dropping it if it is alone. % \begin{macrocode} %<*hex> -\cs_new_protected_nopar:Npn \str_convert_input_hex: +\cs_new_protected_nopar:Npn \str_convert_unescape_hex: { \group_begin: + \flag_clear:n { str_error } \int_set:Nn \tex_escapechar:D { 92 } \tl_gset:Nx \g_str_result_tl { \str_output_byte:w " - \exp_after:wN \str_convert_input_hex_aux:N - \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } + \exp_last_unbraced:Nf \str_unescape_hex_i:N + { \tl_to_str:N \g_str_result_tl } + 0 { ? 0 - \c_one \prg_map_break: } \prg_break_point:n { \str_output_end: } } + \str_if_flag_error:nnx { str_error } { unescape-hex } { } \group_end: } -\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1 +\cs_new:Npn \str_unescape_hex_i:N #1 { \use_none:n #1 \str_aux_hexadecimal_use:NTF #1 - \str_convert_input_hex_aux_ii:N - \str_convert_input_hex_aux:N + { \str_unescape_hex_ii:N } + { + \flag_raise:n { str_error } + \str_unescape_hex_i:N + } } -\cs_new_nopar:Npn \str_convert_input_hex_aux_ii:N #1 +\cs_new:Npn \str_unescape_hex_ii:N #1 { \use_none:n #1 \str_aux_hexadecimal_use:NTF #1 { \str_output_end: - \str_output_byte:w " \str_convert_input_hex_aux:N + \str_output_byte:w " \str_unescape_hex_i:N + } + { + \flag_raise:n { str_error } + \str_unescape_hex_ii:N } - \str_convert_input_hex_aux_ii:N + } +\msg_kernel_new:nnnn { str } { unescape-hex } + { String~invalid~in~escaping~'hex':~only~hexadecimal~digits~allowed. } + { + Some~characters~in~the~string~you~asked~to~convert~are~not~ + hexadecimal~digits~(0-9,~A-F,~a-f)~nor~spaces. } %</hex> % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[int]{\str_convert_input_name:} -% \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN} -% \begin{macro}[int]{\str_convert_input_url:} -% \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN} -% The \cs{str_convert_input_name:} function replaces each -% occurrence of |#| followed by two hexadecimal digits -% in \cs{g_str_result_tl} by the corresponding byte. -% The \texttt{url} function is identical, with escape -% character |%| instead of |#|. Thus we define the two -% together. The arguments of \cs{str_tmp:w} are the character -% code of |#| or |%| in hexadecimal, the name of the main -% function to define, and the name of the auxiliary which +% \begin{macro}[int]{\str_convert_unescape_name:} +% \begin{macro}[aux, rEXP]{\str_unescape_name_loop:wNN} +% \begin{macro}[int]{\str_convert_unescape_url:} +% \begin{macro}[aux, rEXP]{\str_unescape_url_loop:wNN} +% The \cs{str_convert_unescape_name:} function replaces each +% occurrence of |#| followed by two hexadecimal digits in +% \cs{g_str_result_tl} by the corresponding byte. The \texttt{url} +% function is identical, with escape character |%| instead of |#|. +% Thus we define the two together. The arguments of \cs{str_tmp:w} are +% the character code of |#| or |%| in hexadecimal, the name of the +% main function to define, and the name of the auxiliary which % performs the loop. % -% The looping auxiliary |#3| finds the next escape character, -% reads the following two characters, and tests them. The test -% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit -% in the input stream, hence we surround the test with -% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}. -% If both characters are hexadecimal digits, they should be -% removed before looping: this is done by \cs{use_i:nnn}. -% If one of the characters is not a hexadecimal digit, -% then feed |"#1| to \cs{str_output_byte:w} to produce the -% escape character, and call the looping function followed -% by the two characters (remove \cs{use_i:nnn}). +% The looping auxiliary |#3| finds the next escape character, reads +% the following two characters, and tests them. The test +% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit in the +% input stream, hence we surround the test with +% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}. If both +% characters are hexadecimal digits, they should be removed before +% looping: this is done by \cs{use_i:nnn}. If one of the characters +% is not a hexadecimal digit, then feed |"#1| to +% \cs{str_output_byte:w} to produce the escape character, raise the +% flag, and call the looping function followed by the two characters +% (remove \cs{use_i:nnn}). % \begin{macrocode} %<*name|url> \cs_set_protected:Npn \str_tmp:w #1#2#3 { - \cs_new_protected_nopar:Npn #2 + \cs_new_protected:cpn { str_convert_unescape_#2: } { \group_begin: + \flag_clear:n { str_byte } + \flag_clear:n { str_error } \int_set:Nn \tex_escapechar:D { 92 } \tl_gset:Nx \g_str_result_tl { @@ -2432,9 +2368,11 @@ #1 ? { ? \prg_map_break: } \prg_break_point:n { } } + \str_if_flag_error:nnx { str_byte } { non-byte } { #2 } + \str_if_flag_error:nnx { str_error } { unescape-#2 } { } \group_end: } - \cs_new_nopar:Npn #3 ##1#1##2##3 + \cs_new:Npn #3 ##1#1##2##3 { \str_filter_bytes:n {##1} \use_none:n ##3 @@ -2443,50 +2381,63 @@ { \str_aux_hexadecimal_use:NTF ##3 { } - { * \c_zero + `#1 \use_i:nn } + { + \flag_raise:n { str_error } + * \c_zero + `#1 \use_i:nn + } + } + { + \flag_raise:n { str_error } + 0 + `#1 \use_i:nn } - { 0 + `#1 \use_i:nn } \str_output_end: \use_i:nnn #3 ##2##3 } + \msg_kernel_new:nnnn { str } { unescape-#2 } + { String~invalid~in~escaping~'#2'. } + { + LaTeX~came~across~the~escape~character~'#1'~not~followed~by~ + two~hexadecimal~digits.~This~is~invalid~in~the~escaping~'#2'. + } } %</name|url> -%<name>\exp_after:wN \str_tmp:w \c_hash_str -%<name> \str_convert_input_name: \str_convert_input_name_aux:wNN -%<url>\exp_after:wN \str_tmp:w \c_percent_str -%<url> \str_convert_input_url: \str_convert_input_url_aux:wNN +%<name>\exp_after:wN \str_tmp:w \c_hash_str { name } +%<name> \str_unescape_name_loop:wNN +%<url>\exp_after:wN \str_tmp:w \c_percent_str { url } +%<url> \str_unescape_url_loop:wNN % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_convert_input_string:} -% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN} -% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wNNN} -% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:NNNNNN} -% The \texttt{string} escaping is somewhat similar to -% the \texttt{name} and \texttt{url} escapings, with -% escape character |\|. -% The first step is to convert all three line endings, -% |^^J|, |^^M|, and |^^M^^J| to the common |^^J|, as per -% the \textsc{pdf} specification. +% \begin{macro}[int]{\str_convert_unescape_string:} +% \begin{macro}[aux, rEXP]{\str_unescape_string_newlines:wN} +% \begin{macro}[aux, rEXP]{\str_unescape_string_loop:wNNN} +% \begin{macro}[aux, rEXP]{\str_unescape_string_repeat:NNNNNN} +% The \texttt{string} escaping is somewhat similar to the +% \texttt{name} and \texttt{url} escapings, with escape character |\|. +% The first step is to convert all three line endings, |^^J|, |^^M|, +% and |^^M^^J| to the common |^^J|, as per the \textsc{pdf} +% specification. This step cannot raise the flag. +% % Then the following escape sequences are decoded. % \begin{itemize}\def\makelabel#1{\hss\llap{\ttfamily\string#1}} -% \item[\n] Line feed ($10$) -% \item[\r] Carriage return ($13$) -% \item[\t] Horizontal tab ($9$) -% \item[\b] Backspace ($8$) -% \item[\f] Form feed ($12$) -% \item[\(] Left parenthesis -% \item[\)] Right parenthesis -% \item[\\] Backslash -% \item[\ddd] (backslash followed by $1$ to $3$ octal digits) -% Byte \texttt{ddd} (octal), subtracting $256$ in case of overflow. +% \item[\n] Line feed ($10$) +% \item[\r] Carriage return ($13$) +% \item[\t] Horizontal tab ($9$) +% \item[\b] Backspace ($8$) +% \item[\f] Form feed ($12$) +% \item[\(] Left parenthesis +% \item[\)] Right parenthesis +% \item[\\] Backslash +% \item[\ddd] (backslash followed by $1$ to $3$ octal digits) Byte +% \texttt{ddd} (octal), subtracting $256$ in case of overflow. % \end{itemize} -% If followed by an end-of-line character, the backslash and -% the end-of-line are ignored. If followed by anything else, -% the backslash is ignored. +% If followed by an end-of-line character, the backslash and the +% end-of-line are ignored. If followed by anything else, the backslash +% is ignored, raising the error flag. +%^^A Be paranoid: \tl_to_lowercase:n is unsafe. % \begin{macrocode} %<*string> \group_begin: @@ -2495,25 +2446,29 @@ \char_set_catcode_other:N \^^M \tl_to_lowercase:n { - \cs_new_protected_nopar:Npn \str_convert_input_string: + \cs_new_protected_nopar:Npn \str_convert_unescape_string: { \group_begin: + \flag_clear:n { str_byte } + \flag_clear:n { str_error } \int_set:Nn \tex_escapechar:D { 92 } \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_convert_input_string_aux:wN + \exp_after:wN \str_unescape_string_newlines:wN \g_str_result_tl \prg_map_break: ^^M ? \prg_break_point:n { } } \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_convert_input_string_aux:wNNN + \exp_after:wN \str_unescape_string_loop:wNNN \g_str_result_tl * ?? { ? \prg_map_break: } \prg_break_point:n { } } + \str_if_flag_error:nnx { str_byte } { non-byte } { string } + \str_if_flag_error:nnx { str_error } { unescape-string } { } \group_end: } - \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4 + \cs_new:Npn \str_unescape_string_loop:wNNN #1 *#2#3#4 } { \str_filter_bytes:n {#1} @@ -2528,11 +2483,11 @@ \if_int_compare:w #2 > \c_three - 256 \fi: - \str_convert_input_string_aux:NNNNNN + \str_unescape_string_repeat:NNNNNN } - { \str_convert_input_string_aux:NNNNNN ? } + { \str_unescape_string_repeat:NNNNNN ? } } - { \str_convert_input_string_aux:NNNNNN ?? } + { \str_unescape_string_repeat:NNNNNN ?? } } { \prg_case_str:xxn {#2} @@ -2547,20 +2502,29 @@ { b } { 10 } { ^^J } { 0 - \c_one } } - { 0 - \c_one \use_i:nn } + { + \flag_raise:n { str_error } + 0 - \c_one \use_i:nn + } } \str_output_end: - \use_i:nn \str_convert_input_string_aux:wNNN #2#3#4 + \use_i:nn \str_unescape_string_loop:wNNN #2#3#4 } - \cs_new_nopar:Npn \str_convert_input_string_aux:NNNNNN #1#2#3#4#5#6 - { \str_output_end: \str_convert_input_string_aux:wNNN } - \cs_new_nopar:Npn \str_convert_input_string_aux:wN #1 ^^M #2 + \cs_new:Npn \str_unescape_string_repeat:NNNNNN #1#2#3#4#5#6 + { \str_output_end: \str_unescape_string_loop:wNNN } + \cs_new:Npn \str_unescape_string_newlines:wN #1 ^^M #2 { - #1 ^^J - \if_charcode:w ^^J #2 - \exp_after:wN \use_i:nn - \fi: - \str_convert_input_string_aux:wN #2 + #1 + \if_charcode:w ^^J #2 \else: ^^J \fi: + \str_unescape_string_newlines:wN #2 + } + \msg_kernel_new:nnnn { str } { unescape-string } + { String~invalid~in~escaping~'string'. } + { + LaTeX~came~across~an~escape~character~'\c_backslash_str'~ + not~followed~by~any~of:~'n',~'r',~'t',~'b',~'f',~'(',~')',~ + '\c_backslash_str',~one~to~three~octal~digits,~or~the~end~ + of~a~line. } \group_end: %</string> @@ -2570,57 +2534,57 @@ % \end{macro} % \end{macro} % -% \subsubsection{Output methods} +% \subsubsection{Escape methods} % -% \begin{macro}[int]{\str_convert_output_hex:} -% \begin{macro}[int]{\str_convert_output_hexadecimal:} -% \begin{macro}[aux, rEXP]{\str_convert_output_hex_aux:N} +% Currently, none of the escape methods can lead to errors, assuming +% that their input is made out of bytes. +% +% \begin{macro}[int]{\str_convert_escape_hex:} +% \begin{macro}[aux, rEXP]{\str_escape_hex_char:N} % Loop and convert each byte to hexadecimal. % \begin{macrocode} %<*hex> -\cs_new_protected_nopar:Npn \str_convert_output_hex: - { \str_aux_gmap_result:N \str_convert_output_hex_aux:N } -\cs_new_nopar:Npn \str_convert_output_hex_aux:N #1 +\cs_new_protected_nopar:Npn \str_convert_escape_hex: + { \str_convert_gmap:N \str_escape_hex_char:N } +\cs_new:Npn \str_escape_hex_char:N #1 { \str_output_hexadecimal:n { `#1 } } %</hex> % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[int]{\str_convert_output_name:} -% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:N} -% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:NTF} -% \begin{variable}{\c_str_convert_output_name_str} -% \begin{variable}{\c_str_convert_output_name_not_str} -% For each byte, test whether it should -% be output as is, or be \enquote{hash-encoded}. -% Roughly, bytes outside the range $[\hexnum{2A},\hexnum{7E}]$ -% are hash-encoded. We keep two lists of exceptions: -% characters in \cs{c_str_convert_output_name_not_str} -% are not hash-encoded, and characters in the -% \cs{c_str_convert_output_name_str} are encoded. +% \begin{macro}[int]{\str_convert_escape_name:} +% \begin{macro}[aux, rEXP]{\str_escape_name_char:N} +% \begin{macro}[aux, rEXP]{\str_if_escape_name:NTF} +% \begin{variable}{\c_str_escape_name_str} +% \begin{variable}{\c_str_escape_name_not_str} +% For each byte, test whether it should be output as is, or be +% \enquote{hash-encoded}. Roughly, bytes outside the range +% $[\hexnum{2A},\hexnum{7E}]$ are hash-encoded. We keep two lists of +% exceptions: characters in \cs{c_str_escape_name_not_str} are not +% hash-encoded, and characters in the \cs{c_str_escape_name_str} are +% encoded. % \begin{macrocode} %<*name> -\str_const:Nn \c_str_convert_output_name_not_str { ! " $ & ' } %$ -\str_const:Nn \c_str_convert_output_name_str { {}/<>[] } -\cs_new_protected_nopar:Npn \str_convert_output_name: - { \str_aux_gmap_result:N \str_convert_output_name_aux:N } -\cs_new_nopar:Npn \str_convert_output_name_aux:N #1 +\str_const:Nn \c_str_escape_name_not_str { ! " $ & ' } %$ +\str_const:Nn \c_str_escape_name_str { {}/<>[] } +\cs_new_protected_nopar:Npn \str_convert_escape_name: + { \str_convert_gmap:N \str_escape_name_char:N } +\cs_new:Npn \str_escape_name_char:N #1 { - \str_convert_output_name_aux:NTF #1 {#1} + \str_if_escape_name:NTF #1 {#1} { \c_hash_str \str_output_hexadecimal:n {`#1} } } -\prg_new_conditional:Npnn \str_convert_output_name_aux:N #1 { TF } +\prg_new_conditional:Npnn \str_if_escape_name:N #1 { TF } { \if_num:w `#1 < "2A \exp_stop_f: - \str_if_contains_char:NNTF \c_str_convert_output_name_not_str #1 + \str_if_contains_char:NNTF \c_str_escape_name_not_str #1 \prg_return_true: \prg_return_false: \else: \if_num:w `#1 > "7E \exp_stop_f: \prg_return_false: \else: - \str_if_contains_char:NNTF \c_str_convert_output_name_str #1 + \str_if_contains_char:NNTF \c_str_escape_name_str #1 \prg_return_false: \prg_return_true: \fi: \fi: @@ -2633,25 +2597,25 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_convert_output_string:} -% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:N} -% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:NTF} -% \begin{variable}{\c_str_convert_output_string_str} -% Any character below (and including) space, and any character -% above (and including) \texttt{del}, are converted to octal. -% One backslash is added before each parenthesis and backslash. +% \begin{macro}[int]{\str_convert_escape_string:} +% \begin{macro}[aux, rEXP]{\str_escape_string_char:N} +% \begin{macro}[aux, rEXP]{\str_if_escape_string:NTF} +% \begin{variable}{\c_str_escape_string_str} +% Any character below (and including) space, and any character above +% (and including) \texttt{del}, are converted to octal. One backslash +% is added before each parenthesis and backslash. % \begin{macrocode} %<*string> -\str_const:Nx \c_str_convert_output_string_str +\str_const:Nx \c_str_escape_string_str { \c_backslash_str ( ) } -\cs_new_protected_nopar:Npn \str_convert_output_string: - { \str_aux_gmap_result:N \str_convert_output_string_aux:N } -\cs_new_nopar:Npn \str_convert_output_string_aux:N #1 +\cs_new_protected_nopar:Npn \str_convert_escape_string: + { \str_convert_gmap:N \str_escape_string_char:N } +\cs_new:Npn \str_escape_string_char:N #1 { - \str_convert_output_string_aux:NTF #1 + \str_if_escape_string:NTF #1 { \str_if_contains_char:NNT - \c_str_convert_output_string_str #1 + \c_str_escape_string_str #1 { \c_backslash_str } #1 } @@ -2662,7 +2626,7 @@ \int_mod:nn {`#1} \c_eight } } -\prg_new_conditional:Npnn \str_convert_output_string_aux:N #1 { TF } +\prg_new_conditional:Npnn \str_if_escape_string:N #1 { TF } { \if_num:w `#1 < "21 \exp_stop_f: \prg_return_false: @@ -2681,23 +2645,21 @@ % \end{macro} % \end{macro} % -% \begin{macro}[int]{\str_convert_output_url:} -% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N} -% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF} -%^^A todo: Figure out what bytes should be percent encoded! Safe could be -%^^A !'()*-./0123456789_ -% This function is similar to \cs{str_convert_output_name:}, -% escaping different characters. +% \begin{macro}[int]{\str_convert_escape_url:} +% \begin{macro}[aux, rEXP]{\str_escape_url_char:N} +% \begin{macro}[aux, rEXP]{\str_if_escape_url:NTF} +% This function is similar to \cs{str_convert_escape_name:}, escaping +% different characters. % \begin{macrocode} %<*url> -\cs_new_protected_nopar:Npn \str_convert_output_url: - { \str_aux_gmap_result:N \str_convert_output_url_aux:N } -\cs_new_nopar:Npn \str_convert_output_url_aux:N #1 +\cs_new_protected_nopar:Npn \str_convert_escape_url: + { \str_convert_gmap:N \str_escape_url_char:N } +\cs_new:Npn \str_escape_url_char:N #1 { - \str_convert_output_url_aux:NTF #1 {#1} + \str_if_escape_url:NTF #1 {#1} { \c_percent_str \str_output_hexadecimal:n { `#1 } } } -\prg_new_conditional:Npnn \str_convert_output_url_aux:N #1 { TF } +\prg_new_conditional:Npnn \str_if_escape_url:N #1 { TF } { \if_num:w `#1 < "41 \exp_stop_f: \str_if_contains_char:nNTF { "-.<> } #1 @@ -2719,15 +2681,14 @@ % % \subsection{Encoding definition files} % -% The \texttt{native} and \texttt{internal} encodings are automatically -% defined. Other encodings are loaded as needed. The following encodings -% are supported: +% The \texttt{native} encoding is automatically defined. Other encodings +% are loaded as needed. The following encodings are supported: % \begin{itemize} -% \item \textsc{utf-8}; -% \item \textsc{utf-16}, big-endian, little-endian, or with byte order mark; -% \item \textsc{utf-32}, big-endian, little-endian, or with byte order mark; -% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$, -% skipping the inexistent \textsc{iso 8859-12}. +% \item \textsc{utf-8}; +% \item \textsc{utf-16}, big-, little-endian, or with byte order mark; +% \item \textsc{utf-32}, big-, little-endian, or with byte order mark; +% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$, +% skipping the inexistent \textsc{iso 8859-12}. % \end{itemize} % % \subsubsection{\textsc{utf-8} support} @@ -2736,74 +2697,235 @@ %<*utf8> % \end{macrocode} % -% \begin{macro}[int]{\str_convert_to_utf8:} -% \begin{macro}[aux, rEXP] -% { -% \str_aux_to_utf_viii:n, -% \str_aux_to_utf_viii:nnnnw -% } -% Loop through the internal string, and convert each character -% to the \textsc{utf-8} representation. The details are messy. -%^^A todo: document. +% \begin{macro}[int]{\str_convert_encode_utf8:} +% \begin{macro}[aux, rEXP]{\str_encode_utf_viii_char:n} +% \begin{macro}[aux, rEXP]{\str_encode_utf_viii_loop:wwnnw} +% Loop through the internal string, and convert each character to its +% \textsc{utf-8} representation. The representation is built from the +% right-most (least significant) byte to the left-most (most +% significant) byte. Continuation bytes are in the range $[128,191]$, +% taking $64$ different values, hence we roughly want to express the +% character code in base $64$, shifting the first digit in the +% representation by some number depending on how many continuation +% bytes there are. In the range $[0,127]$, output the corresponding +% byte directly. In the range $[128,2047]$, output the remainder +% modulo $64$, plus $128$ as a continuation byte, then output the +% quotient (which is in the range $[0,31]$), shifted by $192$. In the +% next range, $[2048,65535]$, split the character code into residue +% and quotient modulo $64$, output the residue as a first continuation +% byte, then repeat; this leaves us with a quotient in the range +% $[0,15]$, which we output shifted by $224$. The last range, +% $[65536,1114111]$, follows the same pattern: once we realize that +% dividing twice by $64$ leaves us with a number larger than $15$, we +% repeat, producing a last continuation byte, and offset the quotient +% by $240$ for the leading byte. +% +% How is that implemented? \cs{str_encode_utf_vii_loop:wwnnw} takes +% successive quotients as its first argument, the quotient from the +% previous step as its second argument (except in step~$1$), the bound +% for quotients that trigger one more step or not, and finally the +% offset used if this step should produce the leading byte. Leading +% bytes can be in the ranges $[0,127]$, $[192,223]$, $[224,239]$, and +% $[240,247]$ (really, that last limit should be $244$ because Unicode +% stops at the code point $1114111$). At each step, if the quotient +% |#1| is less than the limit |#3| for that range, output the leading +% byte (|#1| shifted by |#4|) and stop. Otherwise, we need one more +% step: use the quotient of |#1| by $64$, and |#1| as arguments for +% the looping auxiliary, and output the continuation byte +% corresponding to the remainder $|#2|-64|#1|+128$. The bizarre +% construction |\c_minus_one + \c_zero *| removes the spurious initial +% continuation byte (better methods welcome). % \begin{macrocode} -\cs_new_protected_nopar:cpn { str_convert_to_utf8: } - { \str_aux_gmap_internal_result:N \str_aux_to_utf_viii:n } -\cs_new:Npn \str_aux_to_utf_viii:n #1 +\cs_new_protected_nopar:cpn { str_convert_encode_utf8: } + { \str_convert_gmap_internal:N \str_encode_utf_viii_char:n } +\cs_new:Npn \str_encode_utf_viii_char:n #1 { - \str_aux_to_utf_viii:nnnnw - {#1} { \c_minus_one + \c_zero * \use_none:n } + \str_encode_utf_viii_loop:wwnnw #1 ; \c_minus_one + \c_zero * ; { 128 } { \c_zero } { 32 } { 192 } { 16 } { 224 } { 8 } { 240 } \q_stop } -\cs_new_nopar:Npn \str_aux_to_utf_viii:nnnnw #1#2#3#4 #5 \q_stop +\cs_new:Npn \str_encode_utf_viii_loop:wwnnw #1; #2; #3#4 #5 \q_stop { \if_num:w #1 < #3 \exp_stop_f: \str_output_byte:n { #1 + #4 } - \else: - \exp_args:Nf \str_aux_to_utf_viii:nnnnw - { \int_div_truncate:nn {#1} {64} } - {#1} - #5 \q_stop + \exp_after:wN \use_none_delimit_by_q_stop:w \fi: + \exp_after:wN \str_encode_utf_viii_loop:wwnnw + \int_value:w \int_div_truncate:nn {#1} {64} ; #1 ; + #5 \q_stop \str_output_byte:n { #2 - 64 * ( #1 - \c_two ) } } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} % -% \begin{macro}[int]{\str_convert_from_utf8:} -% \begin{macro}[aux, rEXP] +% \begin{variable} % { -% \str_aux_from_utf_viii:N, -% \str_aux_from_utf_viii:wwN, -% \str_aux_from_utf_viii:wNnnwN +% \l_str_missing_flag , +% \l_str_extra_flag , +% \l_str_overlong_flag , +% \l_str_overflow_flag , % } +% When decoding a string that is purportedly in the \textsc{utf-8} +% encoding, four different errors can occur, signalled by a specific +% flag for each (we define those flags using \cs{flag_clear_new:n} +% rather than \cs{flag_new:n}, because they are shared with other +% encoding definition files). +% \begin{itemize} +% \item \enquote{Missing continuation byte}: a leading byte is not +% followed by the right number of continuation bytes. +% \item \enquote{Extra continuation byte}: a continuation byte +% appears where it was not expected, \emph{i.e.}, not after an +% appropriate leading byte. +% \item \enquote{Overlong}: a Unicode character is expressed using +% more bytes than necessary, for instance, \hexnum{C0}\hexnum{80} +% for the code point $0$, instead of a single null byte. +% \item \enquote{Overflow}: this occurs when decoding produces +% Unicode code points greater than $1114111$. +% \end{itemize} +% We only raise one \LaTeX3 error message, combining all the errors +% which occurred. In the short message, the leading comma must be +% removed to get a grammatically correct sentence. In the long text, +% first remind the user what a correct \textsc{utf-8} string should +% look like, then add error-specific information. +% \begin{macrocode} +\flag_clear_new:n { str_missing } +\flag_clear_new:n { str_extra } +\flag_clear_new:n { str_overlong } +\flag_clear_new:n { str_overflow } +\msg_kernel_new:nnnn { str } { utf8-decode } + { + Invalid~UTF-8~string: \exp_last_unbraced:Nf \use_none:n + \str_if_flag_times:nT { str_missing } { ,~missing~continuation~byte } + \str_if_flag_times:nT { str_extra } { ,~extra~continuation~byte } + \str_if_flag_times:nT { str_overlong } { ,~overlong~form } + \str_if_flag_times:nT { str_overflow } { ,~code~point~too~large } + . + } + { + In~the~UTF-8~encoding,~each~Unicode~character~consists~in~ + 1~to~4~bytes,~with~the~following~bit~pattern: \\ + \iow_indent:n + { + Code~point~\ \ \ \ <~128:~0xxxxxxx \\ + Code~point~\ \ \ <~2048:~110xxxxx~10xxxxxx \\ + Code~point~\ \ <~65536:~1110xxxx~10xxxxxx~10xxxxxx \\ + Code~point~ <~1114112:~11110xxx~10xxxxxx~10xxxxxx~10xxxxxx \\ + } + Bytes~of~the~form~10xxxxxx~are~called~continuation~bytes. + \flag_if_raised:nT { str_missing } + { + \\\\ + A~leading~byte~(in~the~range~[192,255])~was~not~followed~by~ + the~appropriate~number~of~continuation~bytes. + } + \flag_if_raised:nT { str_extra } + { + \\\\ + LaTeX~came~across~a~continuation~byte~when~it~was~not~expected. + } + \flag_if_raised:nT { str_overlong } + { + \\\\ + Every~Unicode~code~point~must~be~expressed~in~the~shortest~ + possible~form.~For~instance,~'0xC0'~'0x83'~is~not~a~valid~ + representation~for~the~code~point~3. + } + \flag_if_raised:nT { str_overflow } + { + \\\\ + Unicode~limits~code~points~to~the~range~[0,1114111]. + } + } +% \end{macrocode} +% \end{variable} +% +% \begin{macro}[int]{\str_convert_decode_utf8:} % \begin{macro}[aux, rEXP] % { -% \str_aux_from_utf_viii_overflow:w , -% \str_aux_from_utf_viii_error: +% \str_decode_utf_viii_start:N, +% \str_decode_utf_viii_continuation:wwN, +% \str_decode_utf_viii_aux:wNnnwN % } -% In the \textsc{utf-8} encoding, bytes in the range -% $[0,\hexnum{7F}]$ stand for themselves, bytes in the range -% $[\hexnum{80},\hexnum{BF}]$ are continuation bytes, -% and larger bytes must be followed by a number of continuation bytes. -% Documentation to come. -% ^^A todo: refuse "D800 -- "DFFF? -% ^^A todo: document +% \begin{macro}[aux, rEXP] +% {\str_decode_utf_viii_overflow:w, \str_decode_utf_viii_end:} +% Decoding is significantly harder than encoding. As before, lower +% some flags, which are tested at the end (in bulk, to trigger at most +% one \LaTeX3 error, as explained above). We expect successive +% multi-byte sequences of the form \meta{start byte} +% \meta{continuation bytes}. The \texttt{_start} auxiliary tests the +% first byte: +% \begin{itemize} +% \item $[0,\hexnum{7F}]$: the byte stands alone, and is converted +% to its own character code; +% \item $[\hexnum{80}, \hexnum{BF}]$: unexpected continuation byte, +% raise the appropriate flag, and convert that byte to the +% replacement character \hexnum{FFFD}; +% \item $[\hexnum{C0}, \hexnum{FF}]$: this byte should be followed +% by some continuation byte(s). +% \end{itemize} +% In the first two cases, \cs{use_none_delimit_by_q_stop:w} removes +% data that only the third case requires, namely the limits of ranges +% of Unicode characters which can be expressed with $1$, $2$, $3$, or +% $4$ bytes. +% +% We can now concentrate on the multi-byte case and the +% \texttt{_continuation} auxiliary. We expect |#3| to be in the range +% $[\hexnum{80}, \hexnum{BF}]$. The test for this goes as follows: if +% the character code is less than \hexnum{80}, we compare it to +% $-\hexnum{C0}$, yielding \texttt{false}; otherwise to \hexnum{C0}, +% yielding \texttt{true} in the range $[\hexnum{80}, \hexnum{BF}]$ and +% \texttt{false} otherwise. If we find that the byte is not a +% continuation range, stop the current slew of bytes, output the +% replacement character, and continue parsing with the \texttt{_start} +% auxiliary, starting at the byte we just tested. Once we know that +% the byte is a continuation byte, leave it behind us in the input +% stream, compute what code point the bytes read so far would produce, +% and feed that number to the \texttt{_aux} function. +% +% The \texttt{_aux} function tests whether we should look for more +% continuation bytes or not. If the number it receives as |#1| is less +% than the maximum |#4| for the current range, then we are done: check +% for an overlong representation by comparing |#1| with the maximum +% |#3| for the previous range. Otherwise, we call the +% \texttt{_continuation} auxiliary again, after shifting the +% \enquote{current code point} by |#4| (maximum from the range we just +% checkedd). +% +% Two additional tests are needed: if we reach the end of the list of +% range maxima and we are still not done, then we are faced with an +% overflow. Clean up, and again insert the code point \hexnum{FFFD} +% for the replacement character. Also, every time we read a byte, we +% need to check whether we reached the end of the string. In a correct +% \textsc{utf-8} string, this happens automatically when the +% \texttt{_start} auxiliary leaves its first argument in the input +% stream: the end-marker begins with \cs{prg_map_break:}, which ends +% the loop. On the other hand, if the end is reached when looking for +% a continuation byte, the \cs{use_none:n} |#3| construction removes +% the first token from the end-marker, and leaves the \texttt{_end} +% auxiliary, which raises the appropriate error flag before ending the +% mapping. % \begin{macrocode} -\cs_new_protected_nopar:cpn { str_convert_from_utf8: } +\cs_new_protected_nopar:cpn { str_convert_decode_utf8: } { + \flag_clear:n { str_error } + \flag_clear:n { str_missing } + \flag_clear:n { str_extra } + \flag_clear:n { str_overlong } + \flag_clear:n { str_overflow } \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl - { \prg_map_break: \str_aux_from_utf_viii_error: } + \exp_after:wN \str_decode_utf_viii_start:N \g_str_result_tl + { \prg_map_break: \str_decode_utf_viii_end: } \prg_break_point:n { } } + \str_if_flag_error:nnx { str_error } { utf8-decode } { } } -\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1 +\cs_new:Npn \str_decode_utf_viii_start:N #1 { #1 \if_num:w `#1 < "C0 \exp_stop_f: @@ -2811,78 +2933,73 @@ \if_num:w `#1 < "80 \exp_stop_f: \int_value:w `#1 \else: - \msg_expandable_kernel_error:nnn - { str } { utf8-extra-conti } {#1} + \flag_raise:n { str_extra } + \flag_raise:n { str_error } \int_use:N \c_str_replacement_char_int \fi: \else: - \if_num:w `#1 < "F5 \exp_stop_f: - \exp_after:wN \str_aux_from_utf_viii:wwN - \int_value:w \int_eval:n { `#1 - "C0 } - \else: - \s_tl - \msg_expandable_kernel_error:nnn - { str } { utf8-invalid-byte } {#1} - \int_use:N \c_str_replacement_char_int - \fi: + \exp_after:wN \str_decode_utf_viii_continuation:wwN + \int_use:N \int_eval:w `#1 - "C0 \exp_after:wN \int_eval_end: \fi: \s_tl \use_none_delimit_by_q_stop:w {"80} {"800} {"10000} {"110000} \q_stop - \str_aux_from_utf_viii:N + \str_decode_utf_viii_start:N } -\cs_new_nopar:Npn \str_aux_from_utf_viii:wwN - #1 \s_tl #2 \str_aux_from_utf_viii:N #3 +\cs_new:Npn \str_decode_utf_viii_continuation:wwN + #1 \s_tl #2 \str_decode_utf_viii_start:N #3 { \use_none:n #3 - \if_num:w \int_eval:w ( `#3 + "20 ) / "40 = \c_three + \if_num:w `#3 < \if_num:w `#3 < "80 \exp_stop_f: - \fi: "C0 \exp_stop_f: #3 - \exp_after:wN \str_aux_from_utf_viii:wNnnwN - \int_value:w \int_eval:n { #1 * "40 + `#3 - "80 } + \exp_after:wN \str_decode_utf_viii_aux:wNnnwN + \int_use:N \int_eval:w + #1 * "40 + `#3 - "80 + \exp_after:wN \int_eval_end: \else: \s_tl - \msg_expandable_kernel_error:nnn - { str } { utf8-missing-conti } {#3} + \flag_raise:n { str_missing } + \flag_raise:n { str_error } \int_use:N \c_str_replacement_char_int \fi: \s_tl #2 - \str_aux_from_utf_viii:N #3 + \str_decode_utf_viii_start:N #3 } -\cs_new_nopar:Npn \str_aux_from_utf_viii:wNnnwN - #1 \s_tl #2#3#4 #5 \str_aux_from_utf_viii:N #6 +\cs_new:Npn \str_decode_utf_viii_aux:wNnnwN + #1 \s_tl #2#3#4 #5 \str_decode_utf_viii_start:N #6 { - \if_num:w #1 < #4 + \if_num:w #1 < #4 \exp_stop_f: \s_tl \if_num:w #1 < #3 \exp_stop_f: - \msg_expandable_kernel_error:nnn - { str } { utf8-overlong } {#1} + \flag_raise:n { str_overlong } + \flag_raise:n { str_error } \int_use:N \c_str_replacement_char_int \else: #1 \fi: \else: \if_meaning:w \q_stop #5 - \str_aux_from_utf_viii_overflow:w #1 + \str_decode_utf_viii_overflow:w #1 \fi: - \exp_after:wN \str_aux_from_utf_viii:wwN - \int_value:w \int_eval:n { #1 - #4 } + \exp_after:wN \str_decode_utf_viii_continuation:wwN + \int_use:N \int_eval:w #1 - #4 \exp_after:wN \int_eval_end: \fi: \s_tl #2 {#4} #5 - \str_aux_from_utf_viii:N + \str_decode_utf_viii_start:N } -\cs_new_nopar:Npn \str_aux_from_utf_viii_overflow:w #1 \fi: #2 \fi: +\cs_new:Npn \str_decode_utf_viii_overflow:w #1 \fi: #2 \fi: { \fi: \fi: - \msg_expandable_kernel_error:nnn - { str } { utf8-overflow } {#1} + \flag_raise:n { str_overflow } + \flag_raise:n { str_error } \int_use:N \c_str_replacement_char_int } -\cs_new_nopar:Npn \str_aux_from_utf_viii_error: +\cs_new_nopar:Npn \str_decode_utf_viii_end: { \s_tl - \msg_expandable_kernel_error:nn - { str } { utf8-premature-end } + \flag_raise:n { str_missing } + \flag_raise:n { str_error } \int_use:N \c_str_replacement_char_int \s_tl \prg_map_break: } @@ -2897,9 +3014,8 @@ % % \subsubsection{\textsc{utf-16} support} % -% The definitions are done in a category code regime where -% the bytes $254$ and $255$ used by the byte order mark -% have catcode \enquote{other}. +% The definitions are done in a category code regime where the bytes +% $254$ and $255$ used by the byte order mark have catcode~$12$. % \begin{macrocode} %<*utf16> \group_begin: @@ -2907,53 +3023,61 @@ \char_set_catcode_other:N \^^ff % \end{macrocode} % -% \begin{macro}[int]{\str_convert_to_utf16:} -% \begin{macro}[int]{\str_convert_to_utf16be:} -% \begin{macro}[int]{\str_convert_to_utf16le:} +% \begin{macro}[int] +% { +% \str_convert_encode_utf16: , +% \str_convert_encode_utf16be: , +% \str_convert_encode_utf16le: , +% } % \begin{macro}[aux, rEXP] % { -% \str_aux_to_utf_xvi:n, -% \str_aux_to_utf_xvi_be:n, -% \str_aux_to_utf_xvi_le:n +% \str_encode_utf_xvi_aux:N , +% \str_encode_utf_xvi_char:n , % } -% Convert characters one by one in a loop, with different -% behaviours depending on the character code: +% When the endianness is not specified, it is big-endian by default, +% and we add a byte-order mark. Convert characters one by one in a +% loop, with different behaviours depending on the character code. % \begin{itemize} -% \item $[0, 55295]$ converted to two bytes; -% \item $[55296, 57343]$ cannot be converted, replaced by -% the replacement character; -% \item $[57344, 65535]$ converted to two bytes; -% \item $[65536, 1114111]$ converted to a pair of surrogates, -% each two bytes. +% \item $[0, \hexnum{D7FF}]$: converted to two bytes; +% \item $[\hexnum{D800}, \hexnum{DFFF}]$ are used as surrogates: +% they cannot be converted and are replaced by the replacement +% character; +% \item $[\hexnum{E000}, \hexnum{FFFF}]$: converted to two bytes; +% \item $[\hexnum{10000}, \hexnum{10FFFF}]$: converted to a pair of +% surrogates, each two bytes. % \end{itemize} -% For the duration of this operation, \cs{str_tmp:w} is defined -% as a function to convert a number in the range $[0, 65536]$ -% to a pair of bytes (either big endian or little endian). +% For the duration of this operation, \cs{str_tmp:w} is defined as a +% function to convert a number in the range $[0, \hexnum{FFFF}]$ to a +% pair of bytes (either big endian or little endian), by feeding the +% quotient of the division of |#1| by \hexnum{100}, followed by |#1| +% to \cs{str_encode_utf_xvi_be:nn} or its \texttt{le} analog: those +% compute the remainder, and output two bytes for the quotient and +% remainder. % \begin{macrocode} - \cs_new_protected_nopar:cpx { str_convert_to_utf16: } + \cs_new_protected_nopar:cpn { str_convert_encode_utf16: } { - \exp_not:c { str_convert_to_utf16be: } - \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } } + \str_encode_utf_xvi_aux:N \str_output_byte_pair_be:n + \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } } - \cs_new_protected_nopar:cpn { str_convert_to_utf16be: } + \cs_new_protected_nopar:cpn { str_convert_encode_utf16be: } + { \str_encode_utf_xvi_aux:N \str_output_byte_pair_be:n } + \cs_new_protected_nopar:cpn { str_convert_encode_utf16le: } + { \str_encode_utf_xvi_aux:N \str_output_byte_pair_le:n } + \cs_new_protected:Npn \str_encode_utf_xvi_aux:N #1 { - \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_be:n - \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n + \flag_clear:n { str_error } + \cs_set_eq:NN \str_tmp:w #1 + \str_convert_gmap_internal:N \str_encode_utf_xvi_char:n + \str_if_flag_error:nnx { str_error } { utf16-encode } { } } - \cs_new_protected_nopar:cpn { str_convert_to_utf16le: } - { - \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_le:n - \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n - } - \cs_new_nopar:Npn \str_aux_to_utf_xvi:n #1 + \cs_new:Npn \str_encode_utf_xvi_char:n #1 { \if_int_compare:w #1 < "D800 \exp_stop_f: \str_tmp:w {#1} \else: \if_int_compare:w #1 < "10000 \exp_stop_f: \if_int_compare:w #1 < "E000 \exp_stop_f: - \msg_expandable_kernel_error:nnn - { str } { unicode-surrogate } {#1} + \flag_raise:n { str_error } \str_tmp:w { \c_str_replacement_char_int } \else: \str_tmp:w {#1} @@ -2964,120 +3088,208 @@ \fi: \fi: } - \cs_new_nopar:Npn \str_aux_to_utf_xvi_be:n #1 +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{variable} +% { +% \l_str_missing_flag , +% \l_str_extra_flag , +% \l_str_end_flag , +% } +% When encoding a Unicode string to \textsc{utf-16}, only one error +% can occur: code points in the range $[\hexnum{D800}, +% \hexnum{DFFF}]$, corresponding to surrogates, cannot be encoded. We +% use the all-purpose flag \texttt{str_error} to signal that error. +% +% When decoding a Unicode string which is purportedly in +% \textsc{utf-16}, three errors can occur: a missing trail surrogate, +% an unexpected trail surrogate, and a string containing an odd number +% of bytes. +% \begin{macrocode} + \flag_clear_new:n { str_missing } + \flag_clear_new:n { str_extra } + \flag_clear_new:n { str_end } + \msg_kernel_new:nnnn { str } { utf16-encode } + { Unicode~string~cannot~be~expressed~in~UTF-16:~surrogate. } + { + Surrogate~code~points~(in~the~range~[U+D800,~U+DFFF])~ + can~be~expressed~in~the~UTF-8~and~UTF-32~encodings,~ + but~not~in~the~UTF-16~encoding. + } + \msg_kernel_new:nnnn { str } { utf16-decode } { - \str_output_byte:n { \int_div_truncate:nn {#1} {"100} } - \str_output_byte:n { \int_mod:nn {#1} {"100} } + Invalid~UTF-16~string: \exp_last_unbraced:Nf \use_none:n + \str_if_flag_times:nT { str_missing } { ,~missing~trail~surrogate } + \str_if_flag_times:nT { str_extra } { ,~extra~trail~surrogate } + \str_if_flag_times:nT { str_end } { ,~odd~number~of~bytes } + . } - \cs_new_nopar:Npn \str_aux_to_utf_xvi_le:n #1 { - \str_output_byte:n { \int_mod:nn {#1} {"100} } - \str_output_byte:n { \int_div_truncate:nn {#1} {"100} } + In~the~UTF-16~encoding,~each~Unicode~character~is~encoded~as~ + 2~or~4~bytes: \\ + \iow_indent:n + { + Code~point~in~[U+0000,~U+D7FF]:~two~bytes \\ + Code~point~in~[U+D800,~U+DFFF]:~illegal \\ + Code~point~in~[U+E000,~U+FFFF]:~two~bytes \\ + Code~point~in~[U+10000,~U+10FFFF]:~ + a~lead~surrogate~and~a~trail~surrogate \\ + } + Lead~surrogates~are~pairs~of~bytes~in~the~range~[0xD800,~0xDBFF],~ + and~trail~surrogates~are~in~the~range~[0xDC00,~0xDFFF]. + \flag_if_raised:nT { str_missing } + { + \\\\ + A~lead~surrogate~was~not~followed~by~a~trail~surrogate. + } + \flag_if_raised:nT { str_extra } + { + \\\\ + LaTeX~came~across~a~trail~surrogate~when~it~was~not~expected. + } + \flag_if_raised:nT { str_end } + { + \\\\ + The~string~contained~an~odd~number~of~bytes.~This~is~invalid:~ + the~basic~code~unit~for~UTF-16~is~16~bits~(2~bytes). + } } % \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} +% \end{variable} % % \begin{macro}[int] % { -% \str_convert_from_utf16:, -% \str_convert_from_utf16be:, -% \str_convert_from_utf16le: +% \str_convert_decode_utf16: , +% \str_convert_decode_utf16be: , +% \str_convert_decode_utf16le: , % } -% \begin{macro}[aux]{\str_aux_from_utf_xvi_bom:NNw} -% \begin{macro}[aux]{\str_aux_from_utf_xvi:No} -% Define \cs{str_tmp:w} to take two arguments and return the -% character code of the first one if the string is big-endian, -% and the second one if the string is little-endian. -% If the endianness is not known, look for a byte order mark -% to decide. +% \begin{macro}[aux]{\str_decode_utf_xvi_bom:NN, \str_decode_utf_xvi:Nw} +% As for \textsc{utf-8}, decoding \textsc{utf-16} is harder than +% encoding it. If the endianness is unknown, check the first two +% bytes: if those are \hexnum{FE} and \hexnum{FF} in either order, +% remove them and use the corresponding endianness, otherwise assume +% big-endianness. The three endianness cases are based on a common +% auxiliary whose first argument is $1$ for big-endian and $2$ for +% little-endian, and whose second argument, delimited by the scan mark +% \cs{s_stop}, is expanded once (the string may be long; passing +% \cs{g_str_result_tl} as an argument before expansion is cheaper). +% +% The \cs{strr_decode_utf_xvi:Nw} function defines \cs{str_tmp:w} to +% take two arguments and return the character code of the first one if +% the string is big-endian, and the second one if the string is +% little-endian, then loops over the string using +% \cs{str_decode_utf_xvi_pair:NN} described below. % \begin{macrocode} - \cs_new_protected_nopar:cpn { str_convert_from_utf16be: } - { \str_aux_from_utf_xvi:No 1 { \g_str_result_tl } } - \cs_new_protected_nopar:cpn { str_convert_from_utf16le: } - { \str_aux_from_utf_xvi:No 2 { \g_str_result_tl } } - \cs_new_protected_nopar:cpn { str_convert_from_utf16: } + \cs_new_protected_nopar:cpn { str_convert_decode_utf16be: } + { \str_decode_utf_xvi:Nw 1 \g_str_result_tl \s_stop } + \cs_new_protected_nopar:cpn { str_convert_decode_utf16le: } + { \str_decode_utf_xvi:Nw 2 \g_str_result_tl \s_stop } + \cs_new_protected_nopar:cpn { str_convert_decode_utf16: } { - \exp_after:wN \str_aux_from_utf_xvi_bom:NNw - \g_str_result_tl \scan_stop: \scan_stop: \scan_stop: + \exp_after:wN \str_decode_utf_xvi_bom:NN + \g_str_result_tl \s_stop \s_stop \s_stop } - \cs_new_protected_nopar:Npn - \str_aux_from_utf_xvi_bom:NNw #1#2#3\scan_stop: + \cs_new_protected:Npn \str_decode_utf_xvi_bom:NN #1#2 { - \str_if_eq:nnTF { #1#2 } { ^^ff ^^fe } - { \str_aux_from_utf_xvi:No 2 {#3} } + \str_if_eq:xxTF { #1#2 } { ^^ff ^^fe } + { \str_decode_utf_xvi:Nw 2 } { - \str_if_eq:nnTF { #1#2 } { ^^fe ^^ff } - { \str_aux_from_utf_xvi:No 1 {#3} } - { \str_aux_from_utf_xvi:No 1 {#1#2#3} } + \str_if_eq:xxTF { #1#2 } { ^^fe ^^ff } + { \str_decode_utf_xvi:Nw 1 } + { \str_decode_utf_xvi:Nw 1 #1#2 } } } - \cs_new_protected_nopar:Npn \str_aux_from_utf_xvi:No #1#2 + \cs_new_protected:Npn \str_decode_utf_xvi:Nw #1#2 \s_stop { - \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 } + \flag_clear:n { str_error } + \flag_clear:n { str_missing } + \flag_clear:n { str_extra } + \flag_clear:n { str_end } + \cs_set:Npn \str_tmp:w ##1 ##2 { ` ## #1 } \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_from_utf_xvi:NN - #2 \scan_stop: \scan_stop: + \exp_after:wN \str_decode_utf_xvi_pair:NN + #2 \q_nil \q_nil \prg_break_point:n { } } + \str_if_flag_error:nnx { str_error } { utf16-decode } { } } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} % -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NNwNNN} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNw} -% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNN} -% When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w} -% is such that |\str_tmp:w #1#2| expands to |`#1| if the string -% is big-endian, and |`#2| if the string is little-endian. -% If the code unit (a pair of characters) does not represent -% a surrogate, the \cs{if_case:w} construction expands to nothing, -% and we leave in the input stream \meta{characters} \cs{s_tl} -% \meta{char code} \cs{s_tl}, and move on to the next code unit. -% When the end is found, we check for the case of an odd-length -% string (an error) and end the mapping. -% -% Surrogates are characterized by a most significant byte in the -% ranges $[\hexnum{D8},\hexnum{DB}]$ and $[\hexnum{DC},\hexnum{DF}]$, -% detected by dividing the character code by $4$ (\eTeX{} rounds to -% nearest, ties away from zero). Here, a trail surrogate is invalid, -% replaced by the replacement character. Lead surrogates are treated by -% \cs{str_aux_from_utf_xvi:NNwNNN}, which grabs the next two -% characters, checks for a premature end, and checks whether -% the code unit is a trailing surrogate. If it is, leave -% \meta{characters} \cs{s_tl} \meta{char code} \cs{s_tl} in the -% input stream and move on to the next code unit. Otherwise, raise -% an error, and continue parsing, replacing only the first code unit -% (the leading surrogate) by the replacement character. +% \begin{macro}[aux, rEXP] +% { +% \str_decode_utf_xvi_pair:NN , +% \str_decode_utf_xvi_quad:NNwNN , +% \str_decode_utf_xvi_pair_end:Nw , +% } +% \begin{macro}[aux, rEXP] +% { +% \str_decode_utf_xvi_error:nNN , +% \str_decode_utf_xvi_extra:NNw , +% } +% Bytes are read two at a time. At this stage, |\str_tmp:w #1#2| +% expands to the character code of the most significant byte, and we +% distinguish cases depending on which range it lies in: +% \begin{itemize} +% \item $[\hexnum{D8}, \hexnum{DB}]$ signals a lead surrogate, and +% the integer expression yields $1$ (\eTeX{} rounds ties away from +% zero); +% \item $[\hexnum{DC}, \hexnum{DF}]$ signals a trail surrogate, +% unexpected here, and the integer expression yields $2$; +% \item any other value signals a code point in the Basic +% Multilingual Plane, which stands for itself, and the +% \cs{if_case:w} construction expands to nothing (cases other than +% $1$ or $2$), leaving the relevant material in the input stream, +% followed by another call to the \texttt{_pair} auxiliary. +% \end{itemize} +% The case of a lead surrogate is treated by the \texttt{_quad} +% auxiliary, whose arguments |#1|, |#2|, |#4| and |#5| are the four +% bytes. We expect the most significant byte of |#4#5| to be in the +% range $[\hexnum{DC}, \hexnum{DF}]$ (trail surrogate). The test is +% similar to the test used for continuation bytes in the +% \textsc{utf-8} decoding functions. In the case where |#4#5| is +% indeed a trail surrogate, leave |#1#2#4#5| \cs{s_tl} +% \meta{code~point} \cs{s_tl}, and remove the pair |#4#5| before +% looping with \cs{str_decode_utf_xvi_pair:NN}. Otherwise, of course, +% complain about the missing surrogate. +% +% Every time we read a pair of bytes, we test for the end-marker +% \cs{q_nil}. When reaching the end, we additionally check that the +% string had an even length. Also, if the end is reached when +% expecting a trail surrogate, we treat that as a missing surrogate. % \begin{macrocode} - \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2 + \cs_new:Npn \str_decode_utf_xvi_pair:NN #1#2 { - \if_meaning:w \scan_stop: #2 - \str_aux_from_utf_xvi_end:Nw #1 + \if_meaning:w \q_nil #2 + \str_decode_utf_xvi_pair_end:Nw #1 \fi: \if_case:w \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: - \or: \exp_after:wN \str_aux_from_utf_xvi:NNwNNN - \or: \exp_after:wN \str_aux_from_utf_xvi_error:NNw + \or: \exp_after:wN \str_decode_utf_xvi_quad:NNwNN + \or: \exp_after:wN \str_decode_utf_xvi_extra:NNw \fi: #1#2 \s_tl \int_eval:n { "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 } \s_tl - \str_aux_from_utf_xvi:NN + \str_decode_utf_xvi_pair:NN } - \cs_new:Npn \str_aux_from_utf_xvi:NNwNNN #1#2 \s_tl #3 \s_tl #4#5#6 + \cs_new:Npn \str_decode_utf_xvi_quad:NNwNN + #1#2 #3 \str_decode_utf_xvi_pair:NN #4#5 { - \if_meaning:w \scan_stop: #6 - \str_aux_from_utf_xvi_error:NNN #1#2#5 + \if_meaning:w \q_nil #5 + \str_decode_utf_xvi_error:nNN { missing } #1#2 + \str_decode_utf_xvi_pair_end:Nw #4 \fi: - \if_num:w \int_eval:w ( \str_tmp:w #5#6 - "DA ) / \c_four = \c_one - #1 #2 #5 #6 \s_tl + \if_num:w + \if_num:w \str_tmp:w #4#5 < "DC \exp_stop_f: + \c_zero = \c_one + \else: + \str_tmp:w #4#5 < "E0 \exp_stop_f: + \fi: + #1 #2 #4 #5 \s_tl \int_eval:n { ( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400 @@ -3086,49 +3298,31 @@ \s_tl \exp_after:wN \use_i:nnn \else: - \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl \s_tl + \str_decode_utf_xvi_error:nNN { missing } #1#2 \fi: - #4#5#6 + \str_decode_utf_xvi_pair:NN #4#5 } - \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi: + \cs_new:Npn \str_decode_utf_xvi_pair_end:Nw #1 \fi: { \fi: - \if_meaning:w \scan_stop: #1 + \if_meaning:w \q_nil #1 \else: - #1 \s_tl - \msg_expandable_kernel_error:nnn - { str } { utf16-odd } {#1} - \int_use:N \c_str_replacement_char_int \s_tl + \str_decode_utf_xvi_error:nNN { end } #1 \prg_do_nothing: \fi: \prg_map_break: } - \cs_new:Npn \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl #3 \s_tl + \cs_new:Npn \str_decode_utf_xvi_extra:NNw #1#2 \s_tl #3 \s_tl + { \str_decode_utf_xvi_error:nNN { extra } #1#2 } + \cs_new:Npn \str_decode_utf_xvi_error:nNN #1#2#3 { - #1 #2 \s_tl - \msg_expandable_kernel_error:nnn - { str } { utf16-surrogate } { #1#2 } + \flag_raise:n { str_error } + \flag_raise:n { str_#1 } + #2 #3 \s_tl \int_use:N \c_str_replacement_char_int \s_tl } - \cs_new:Npn \str_aux_from_utf_xvi_error:NNN #1#2#3 - { - \if_meaning:w \scan_stop: #3 - #1#2 \s_tl - \msg_expandable_kernel_error:nnn - { str } { utf16-surrogate } { #1#2 } - \else: - #1#2#3 \s_tl - \msg_expandable_kernel_error:nnn - { str } { utf16-odd } { #1#2#3 } - \fi: - \int_use:N \c_str_replacement_char_int \s_tl - \prg_map_break: - } % \end{macrocode} % \end{macro} % \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} % % Restore the original catcodes of bytes $254$ and $255$. % \begin{macrocode} @@ -3138,9 +3332,9 @@ % % \subsubsection{\textsc{utf-32} support} % -% The definitions are done in a category code regime where -% the bytes $0$, $254$ and $255$ used by the byte order mark -% have catcode \enquote{other}. +% The definitions are done in a category code regime where the bytes +% $0$, $254$ and $255$ used by the byte order mark have catcode +% \enquote{other}. % \begin{macrocode} %<*utf32> \group_begin: @@ -3151,130 +3345,188 @@ % % \begin{macro}[int] % { -% \str_convert_to_utf32:, -% \str_convert_to_utf32be:, -% \str_convert_to_utf32le: +% \str_convert_encode_utf32: , +% \str_convert_encode_utf32be: , +% \str_convert_encode_utf32le: , % } % \begin{macro}[aux, rEXP] % { -% \str_aux_to_utf_xxxii_be:n, -% \str_aux_to_utf_xxxii_le:n +% \str_encode_utf_xxxii_be:n , +% \str_encode_utf_xxxii_be_aux:nn , +% \str_encode_utf_xxxii_le:n , +% \str_encode_utf_xxxii_le_aux:nn , % } -% Convert each integer in the comma-list \cs{g_str_result_tl} -% to a sequence of four bytes. The functions for big-endian -% and little-endian encodings are very similar, but the +% Convert each integer in the comma-list \cs{g_str_result_tl} to a +% sequence of four bytes. The functions for big-endian and +% little-endian encodings are very similar, but the % \cs{str_output_byte:n} instructions are reversed. % \begin{macrocode} - \cs_new_protected_nopar:cpx { str_convert_to_utf32: } + \cs_new_protected_nopar:cpn { str_convert_encode_utf32: } { - \exp_not:c { str_convert_to_utf32be: } - \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } } + \str_convert_gmap_internal:N \str_encode_utf_xxxii_be:n + \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } } - \cs_new_protected_nopar:cpn { str_convert_to_utf32be: } - { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_be:n } - \cs_new_protected_nopar:cpn { str_convert_to_utf32le: } - { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_le:n } - \cs_new:Npn \str_aux_to_utf_xxxii_be:n #1 + \cs_new_protected_nopar:cpn { str_convert_encode_utf32be: } + { \str_convert_gmap_internal:N \str_encode_utf_xxxii_be:n } + \cs_new_protected_nopar:cpn { str_convert_encode_utf32le: } + { \str_convert_gmap_internal:N \str_encode_utf_xxxii_le:n } + \cs_new:Npn \str_encode_utf_xxxii_be:n #1 + { + \exp_args:Nf \str_encode_utf_xxxii_be_aux:nn + { \int_div_truncate:nn {#1} { "100 } } {#1} + } + \cs_new:Npn \str_encode_utf_xxxii_be_aux:nn #1#2 { ^^00 - \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } } - \str_output_byte:n - { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} } - \str_output_byte:n { \int_mod:nn {#1} {"100} } + \str_output_byte_pair_be:n {#1} + \str_output_byte:n { #2 - #1 * "100 } } - \cs_new:Npn \str_aux_to_utf_xxxii_le:n #1 + \cs_new:Npn \str_encode_utf_xxxii_le:n #1 { - \str_output_byte:n { \int_mod:nn {#1} {"100} } - \str_output_byte:n - { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} } - \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } } + \exp_args:Nf \str_encode_utf_xxxii_le_aux:nn + { \int_div_truncate:nn {#1} { "100 } } {#1} + } + \cs_new:Npn \str_encode_utf_xxxii_le_aux:nn #1#2 + { + \str_output_byte:n { #2 - #1 * "100 } + \str_output_byte_pair_le:n {#1} ^^00 } % \end{macrocode} % \end{macro} % \end{macro} % +% \begin{variable}{str_overflow, str_end} +% There can be no error when encoding in \textsc{utf-32}. When +% decoding, the string may not have length $4n$, or it may contain +% code points larger than \hexnum{10FFFF}. The latter case often +% happens if the encoding was in fact not \textsc{utf-32}, because +% most arbitrary strings are not valid in \textsc{utf-32}. +% \begin{macrocode} + \flag_clear_new:n { str_overflow } + \flag_clear_new:n { str_end } + \msg_kernel_new:nnnn { str } { utf32-decode } + { + Invalid~UTF-32~string: \exp_last_unbraced:Nf \use_none:n + \str_if_flag_times:nT { str_overflow } { ,~code~point~too~large } + \str_if_flag_times:nT { str_end } { ,~truncated~string } + . + } + { + In~the~UTF-32~encoding,~every~Unicode~character~ + (in~the~range~[U+0000,~U+10FFFF])~is~encoded~as~4~bytes. + \flag_if_raised:nT { str_overflow } + { + \\\\ + LaTeX~came~across~a~code~point~larger~than~1114111,~ + the~maximum~code~point~defined~by~Unicode.~ + Perhaps~the~string~was~not~encoded~in~the~UTF-32~encoding? + } + \flag_if_raised:nT { str_end } + { + \\\\ + The~length~of~the~string~is~not~a~multiple~of~4.~ + Perhaps~the~string~was~truncated? + } + } +% \end{macrocode} +% \end{variable} +% % \begin{macro}[int] % { -% \str_convert_from_utf32:, -% \str_convert_from_utf32be:, -% \str_convert_from_utf32le: +% \str_convert_decode_utf32: , +% \str_convert_decode_utf32be: , +% \str_convert_decode_utf32le: , % } % \begin{macro}[aux] -% { -% \str_aux_from_utf_xxxii_bom:NNNNw, -% \str_aux_from_utf_xxxii:No -% } +% {\str_decode_utf_xxxii_bom:NNNN, \str_decode_utf_xxxii:Nw} % \begin{macro}[aux, rEXP] -% { -% \str_aux_from_utf_xxxii:NNNN, -% \str_aux_from_utf_xxxii_end:w -% } -% See the conversion functions from \textsc{utf-16} encodings. -% The \cs{str_aux_from_utf_xxxii:No} auxiliary defines -% \cs{str_tmp:w} to take two arguments and give the character -% code of either the first argument (big-endian) or the second -% argument (little-endian). Then we loop over the string, -% four bytes at a time, check for overflow, and otherwise -% produce the number corresponding to the four bytes with -% the given endianness. +% {\str_decode_utf_xxxii_loop:NNNN, \str_decode_utf_xxxii_end:w} +% +% The structure is similar to \textsc{utf-16} decoding functions. If +% the endianness is not given, test the first $4$ bytes of the string +% (possibly \cs{s_stop} if the string is too short) for the presence +% of a byte-order mark. If there is a byte-order mark, use that +% endianness, and remove the $4$ bytes, otherwise default to +% big-endian, and leave the $4$ bytes in place. The +% \cs{str_decode_utf_xxxii:Nw} auxiliary recieves $1$ or $2$ as its +% first argument indicating endianness, and the string to convert as +% its second argument (expanded or not). It sets \cs{str_tmp:w} to +% expand to the character code of either of its two arguments +% depending on endianness, then triggers the \texttt{_loop} auxiliary +% inside an \texttt{x}-expanding assignment to \cs{g_str_result_tl}. +% +% The \texttt{_loop} auxiliary first checks for the end-of-string +% marker \cs{s_stop}, calling the \texttt{_end} auxiliary if +% appropriate. Otherwise, leave the \meta{4~bytes} \cs{s_tl} behind, +% then check that the code point is not overflowing: the leading byte +% must be $0$, and the following byte at most $16$. +% +% In the ending code, we check that there remains no byte: there +% should be nothing left until the first \cs{s_stop}. Break the map. % \begin{macrocode} - \cs_new_protected_nopar:cpn { str_convert_from_utf32be: } - { \str_aux_from_utf_xxxii:No 1 { \g_str_result_tl } } - \cs_new_protected_nopar:cpn { str_convert_from_utf32le: } - { \str_aux_from_utf_xxxii:No 2 { \g_str_result_tl } } - \cs_new_protected_nopar:cpn { str_convert_from_utf32: } + \cs_new_protected_nopar:cpn { str_convert_decode_utf32be: } + { \str_decode_utf_xxxii:Nw 1 \g_str_result_tl \s_stop } + \cs_new_protected_nopar:cpn { str_convert_decode_utf32le: } + { \str_decode_utf_xxxii:Nw 2 \g_str_result_tl \s_stop } + \cs_new_protected_nopar:cpn { str_convert_decode_utf32: } { - \exp_after:wN \str_aux_from_utf_xxxii_bom:NNNNw \g_str_result_tl - \scan_stop: \scan_stop: \scan_stop: \scan_stop: \scan_stop: + \exp_after:wN \str_decode_utf_xxxii_bom:NNNN \g_str_result_tl + \s_stop \s_stop \s_stop \s_stop \s_stop } - \cs_new_protected_nopar:Npn - \str_aux_from_utf_xxxii_bom:NNNNw #1#2#3#4#5\scan_stop: + \cs_new_protected:Npn \str_decode_utf_xxxii_bom:NNNN #1#2#3#4 { - \str_if_eq:nnTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 } - { \str_aux_from_utf_xxxii:No 2 {#5} } + \str_if_eq:xxTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 } + { \str_decode_utf_xxxii:Nw 2 } { - \str_if_eq:nnTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff } - { \str_aux_from_utf_xxxii:No 1 {#5} } - { \str_aux_from_utf_xxxii:No 1 {#1#2#3#4#5} } + \str_if_eq:xxTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff } + { \str_decode_utf_xxxii:Nw 1 } + { \str_decode_utf_xxxii:Nw 1 #1#2#3#4 } } } - \cs_new_protected_nopar:Npn \str_aux_from_utf_xxxii:No #1#2 + \cs_new_protected:Npn \str_decode_utf_xxxii:Nw #1#2 \s_stop { - \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 } + \flag_clear:n { str_overflow } + \flag_clear:n { str_end } + \flag_clear:n { str_error } + \cs_set:Npn \str_tmp:w ##1 ##2 { ` ## #1 } \tl_gset:Nx \g_str_result_tl { - \exp_after:wN \str_aux_from_utf_xxxii:NNNN - #2 \scan_stop: \scan_stop: \scan_stop: \scan_stop: + \exp_after:wN \str_decode_utf_xxxii_loop:NNNN + #2 \s_stop \s_stop \s_stop \s_stop \prg_break_point:n { } } + \str_if_flag_error:nnx { str_error } { utf32-decode } { } } - \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4 + \cs_new:Npn \str_decode_utf_xxxii_loop:NNNN #1#2#3#4 { - \if_meaning:w \scan_stop: #4 - \exp_after:wN \str_aux_from_utf_xxxii_end:w + \if_meaning:w \s_stop #4 + \exp_after:wN \str_decode_utf_xxxii_end:w \fi: #1#2#3#4 \s_tl - \if_num:w \int_eval:w - \str_tmp:w #1#4 * \c_two_hundred_fifty_six - + \str_tmp:w #2#3 - > \c_sixteen - \msg_expandable_kernel_error:nnn - { str } { utf32-overflow } { #1#2#3#4 } + \if_num:w \str_tmp:w #1#4 > \c_zero + \flag_raise:n { str_overflow } + \flag_raise:n { str_error } \int_use:N \c_str_replacement_char_int \else: - \int_eval:n - { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 } + \if_num:w \str_tmp:w #2#3 > \c_sixteen + \flag_raise:n { str_overflow } + \flag_raise:n { str_error } + \int_use:N \c_str_replacement_char_int + \else: + \int_eval:n + { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 } + \fi: \fi: \s_tl - \str_aux_from_utf_xxxii:NNNN + \str_decode_utf_xxxii_loop:NNNN } - \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop: + \cs_new:Npn \str_decode_utf_xxxii_end:w #1 \s_stop { \tl_if_empty:nF {#1} { - \msg_expandable_kernel_error:nnn - { str } { utf32-truncated } { #1 } + \flag_raise:n { str_end } + \flag_raise:n { str_error } #1 \s_tl \int_use:N \c_str_replacement_char_int \s_tl } @@ -3292,19 +3544,15 @@ % \end{macrocode} % % \subsubsection{\textsc{iso 8859} support} -%^^A todo: document % -% The \textsc{iso-8859-1} encoding exactly matches with the $256$ -% first Unicode characters. For other 8-bit encodings -% of the \textsc{iso-8859} family, we keep track only of -% differences, and of unassigned bytes. +% The \textsc{iso-8859-1} encoding exactly matches with the $256$ first +% Unicode characters. For other 8-bit encodings of the \textsc{iso-8859} +% family, we keep track only of differences, and of unassigned bytes. % \begin{macrocode} %<*iso88591> -\str_encoding_eight_bit:n { iso88591 } -\tl_const:cn { c_str_encoding_iso88591_tl } +\str_declare_eight_bit_encoding:nnn { iso88591 } { } -\tl_const:cn { c_str_encoding_iso88591_missing_tl } { } %</iso88591> @@ -3312,8 +3560,7 @@ % % \begin{macrocode} %<*iso88592> -\str_encoding_eight_bit:n { iso88592 } -\tl_const:cn { c_str_encoding_iso88592_tl } +\str_declare_eight_bit_encoding:nnn { iso88592 } { { A1 } { 0104 } { A2 } { 02D8 } @@ -3373,7 +3620,6 @@ { FE } { 0163 } { FF } { 02D9 } } -\tl_const:cn { c_str_encoding_iso88592_missing_tl } { } %</iso88592> @@ -3381,8 +3627,7 @@ % % \begin{macrocode} %<*iso88593> -\str_encoding_eight_bit:n { iso88593 } -\tl_const:cn { c_str_encoding_iso88593_tl } +\str_declare_eight_bit_encoding:nnn { iso88593 } { { A1 } { 0126 } { A2 } { 02D8 } @@ -3413,7 +3658,6 @@ { FE } { 015D } { FF } { 02D9 } } -\tl_const:cn { c_str_encoding_iso88593_missing_tl } { { A5 } { AE } @@ -3428,8 +3672,7 @@ % % \begin{macrocode} %<*iso88594> -\str_encoding_eight_bit:n { iso88594 } -\tl_const:cn { c_str_encoding_iso88594_tl } +\str_declare_eight_bit_encoding:nnn { iso88594 } { { A1 } { 0104 } { A2 } { 0138 } @@ -3482,7 +3725,6 @@ { FE } { 016B } { FF } { 02D9 } } -\tl_const:cn { c_str_encoding_iso88594_missing_tl } { } %</iso88594> @@ -3490,8 +3732,7 @@ % % \begin{macrocode} %<*iso88595> -\str_encoding_eight_bit:n { iso88595 } -\tl_const:cn { c_str_encoding_iso88595_tl } +\str_declare_eight_bit_encoding:nnn { iso88595 } { { A1 } { 0401 } { A2 } { 0402 } @@ -3588,7 +3829,6 @@ { FE } { 045E } { FF } { 045F } } -\tl_const:cn { c_str_encoding_iso88595_missing_tl } { } %</iso88595> @@ -3596,8 +3836,7 @@ % % \begin{macrocode} %<*iso88596> -\str_encoding_eight_bit:n { iso88596 } -\tl_const:cn { c_str_encoding_iso88596_tl } +\str_declare_eight_bit_encoding:nnn { iso88596 } { { AC } { 060C } { BB } { 061B } @@ -3648,7 +3887,6 @@ { F1 } { 0651 } { F2 } { 0652 } } -\tl_const:cn { c_str_encoding_iso88596_missing_tl } { { A1 } { A2 } @@ -3688,8 +3926,7 @@ % % \begin{macrocode} %<*iso88597> -\str_encoding_eight_bit:n { iso88597 } -\tl_const:cn { c_str_encoding_iso88597_tl } +\str_declare_eight_bit_encoding:nnn { iso88597 } { { A1 } { 2018 } { A2 } { 2019 } @@ -3769,7 +4006,6 @@ { FD } { 03CD } { FE } { 03CE } } -\tl_const:cn { c_str_encoding_iso88597_missing_tl } { { AE } { D2 } @@ -3779,8 +4015,7 @@ % % \begin{macrocode} %<*iso88598> -\str_encoding_eight_bit:n { iso88598 } -\tl_const:cn { c_str_encoding_iso88598_tl } +\str_declare_eight_bit_encoding:nnn { iso88598 } { { AA } { 00D7 } { BA } { 00F7 } @@ -3815,7 +4050,6 @@ { FD } { 200E } { FE } { 200F } } -\tl_const:cn { c_str_encoding_iso88598_missing_tl } { { A1 } { BF } @@ -3858,8 +4092,7 @@ % % \begin{macrocode} %<*iso88599> -\str_encoding_eight_bit:n { iso88599 } -\tl_const:cn { c_str_encoding_iso88599_tl } +\str_declare_eight_bit_encoding:nnn { iso88599 } { { D0 } { 011E } { DD } { 0130 } @@ -3868,7 +4101,6 @@ { FD } { 0131 } { FE } { 015F } } -\tl_const:cn { c_str_encoding_iso88599_missing_tl } { } %</iso88599> @@ -3876,8 +4108,7 @@ % % \begin{macrocode} %<*iso885910> -\str_encoding_eight_bit:n { iso885910 } -\tl_const:cn { c_str_encoding_iso885910_tl } +\str_declare_eight_bit_encoding:nnn { iso885910 } { { A1 } { 0104 } { A2 } { 0112 } @@ -3926,7 +4157,6 @@ { F9 } { 0173 } { FF } { 0138 } } -\tl_const:cn { c_str_encoding_iso885910_missing_tl } { } %</iso885910> @@ -3934,8 +4164,7 @@ % % \begin{macrocode} %<*iso885911> -\str_encoding_eight_bit:n { iso885911 } -\tl_const:cn { c_str_encoding_iso885911_tl } +\str_declare_eight_bit_encoding:nnn { iso885911 } { { A1 } { 0E01 } { A2 } { 0E02 } @@ -4025,7 +4254,6 @@ { FA } { 0E5A } { FB } { 0E5B } } -\tl_const:cn { c_str_encoding_iso885911_missing_tl } { { DB } { DC } @@ -4037,8 +4265,7 @@ % % \begin{macrocode} %<*iso885913> -\str_encoding_eight_bit:n { iso885913 } -\tl_const:cn { c_str_encoding_iso885913_tl } +\str_declare_eight_bit_encoding:nnn { iso885913 } { { A1 } { 201D } { A5 } { 201E } @@ -4097,7 +4324,6 @@ { FE } { 017E } { FF } { 2019 } } -\tl_const:cn { c_str_encoding_iso885913_missing_tl } { } %</iso885913> @@ -4105,8 +4331,7 @@ % % \begin{macrocode} %<*iso885914> -\str_encoding_eight_bit:n { iso885914 } -\tl_const:cn { c_str_encoding_iso885914_tl } +\str_declare_eight_bit_encoding:nnn { iso885914 } { { A1 } { 1E02 } { A2 } { 1E03 } @@ -4140,7 +4365,6 @@ { F7 } { 1E6B } { FE } { 0177 } } -\tl_const:cn { c_str_encoding_iso885914_missing_tl } { } %</iso885914> @@ -4148,8 +4372,7 @@ % % \begin{macrocode} %<*iso885915> -\str_encoding_eight_bit:n { iso885915 } -\tl_const:cn { c_str_encoding_iso885915_tl } +\str_declare_eight_bit_encoding:nnn { iso885915 } { { A4 } { 20AC } { A6 } { 0160 } @@ -4160,7 +4383,6 @@ { BD } { 0153 } { BE } { 0178 } } -\tl_const:cn { c_str_encoding_iso885915_missing_tl } { } %</iso885915> @@ -4168,8 +4390,7 @@ % % \begin{macrocode} %<*iso885916> -\str_encoding_eight_bit:n { iso885916 } -\tl_const:cn { c_str_encoding_iso885916_tl } +\str_declare_eight_bit_encoding:nnn { iso885916 } { { A1 } { 0104 } { A2 } { 0105 } @@ -4212,7 +4433,6 @@ { FD } { 0119 } { FE } { 021B } } -\tl_const:cn { c_str_encoding_iso885916_missing_tl } { } %</iso885916> |