summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx')
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx3692
1 files changed, 1956 insertions, 1736 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
index cddd7349adc..3d73ed10b8f 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
@@ -35,13 +35,14 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3159 2012-01-09 00:51:58Z bruno $
+\GetIdInfo$Id: l3str.dtx 3257 2012-01-29 13:37:03Z bruno $
{L3 Experimental Strings}
%</driver|package>
%<*driver>
\documentclass[full]{l3doc}
\usepackage{amsmath}
\begin{document}
+ \tableofcontents
\DocInput{\jobname.dtx}
\end{document}
%</driver>
@@ -75,25 +76,23 @@
% characters.
%
% String variables are simply specialised token lists, but by convention
-% should be named with the suffix \texttt{\ldots str}. Such variables should
-% contain characters with category code $12$ (other), except spaces, which
-% have category code $10$ (blank space). All the functions in
-% this module first convert their argument to a string for internal processing,
-% and will not treat a token list or the corresponding string representation
-% differently.
+% should be named with the suffix \texttt{\ldots str}. Such variables
+% should contain characters with category code $12$ (other), except
+% spaces, which have category code $10$ (blank space). All the functions
+% in this module first convert their argument to a string for internal
+% processing, and will not treat a token list or the corresponding
+% string representation differently.
%
% Most functions in this module come in three flavours:
% \begin{itemize}
% \item \cs{str_...:N...}, which expect a token list or string
% variable as their argument;
-% \item \cs{str_...:n...}, taking any token list (or string) as an argument;
-% \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered
-% during the operation: these functions are faster than those
-% which take care of escaping spaces appropriately;
+% \item \cs{str_...:n...}, taking any token list (or string) as an
+% argument;
+% \item \cs{str_..._ignore_spaces:n...}, which ignores any space
+% encountered during the operation: these functions are faster than
+% those which take care of escaping spaces appropriately;
% \end{itemize}
-% When performance is critical, the internal \cs{str_..._unsafe:n...}
-% functions, which expect an \enquote{other string} in which spaces
-% have category code $12$ instead of $10$, might be useful.
%
% \section{Building strings}
%
@@ -106,8 +105,9 @@
% \c_tilde_str,
% \c_percent_str
% }
-% Constant strings, containing a single character, with category code $12$.
-% Any character can be accessed as \cs{iow_char:N} |\|\meta{character}.
+% Constant strings, containing a single character, with category code
+% $12$. Any character can be accessed as \cs{iow_char:N}
+% |\|\meta{character}.
% \end{variable}
%
% \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n}
@@ -115,28 +115,36 @@
% \cs{tl_to_str:N} \meta{tl var}
% \cs{tl_to_str:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}, leaving the resulting
-% tokens in the input stream.
+% Converts the \meta{token list} to a \meta{string}, leaving the
+% resulting tokens in the input stream.
% \begin{texnote}
-% The string representation of a token list may depend
-% on the category codes in effect when it is measured,
-% and the value of the \tn{escapechar}: for instance
-% |\tl_to_str:n {\a}| may produce $1$, $2$ or $3$
-% characters depending on the escape character, and
-% the category code of \texttt{a}. This impacts almost
-% all functions in the module, which use \cs{tl_to_str:n}
-% internally.
+% The string representation of a token list may depend on the
+% category codes in effect when it is evaluated, and the value of
+% the \tn{escapechar}: for instance |\tl_to_str:n {\a}| normally
+% produces the three character \enquote{backslash},
+% \enquote{lower-case a}, \enquote{space}, but it may also produce
+% $1$ or $2$ characters depending on the escape character, and the
+% category code of \texttt{a}. This impacts almost all functions in
+% the module, which use \cs{tl_to_str:n} internally.
% \end{texnote}
% \end{function}
%
+% \begin{function}{\str_new:N, \str_new:c}
+% \begin{syntax}
+% \cs{str_new:N} \meta{str~var}
+% \end{syntax}
+% Creates a new \meta{str~var} or raises an error if the name is
+% already taken. The declaration is global. The \meta{str~var} will
+% initially be empty.
+% \end{function}
+%
% \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx}
% \begin{syntax}
% \cs{str_const:Nn} \meta{str~var} \Arg{token list}
% \end{syntax}
-% Creates a new constant \meta{str~var} or raises an error
-% if the name is already taken. The value of the
-% \meta{str~var} will be set globally to the
-% \meta{token list}, converted to a string.
+% Creates a new constant \meta{str~var} or raises an error if the name
+% is already taken. The value of the \meta{str~var} will be set
+% globally to the \meta{token list}, converted to a string.
% \end{function}
%
% \begin{function}
@@ -147,8 +155,8 @@
% \begin{syntax}
% \cs{str_set:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and stores the result in \meta{str var}.
+% Converts the \meta{token list} to a \meta{string}, and stores the
+% result in \meta{str var}.
% \end{function}
%
% \begin{function}
@@ -161,10 +169,9 @@
% \begin{syntax}
% \cs{str_put_left:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and prepends the result to \meta{str var}.
-% The current contents of the \meta{str var} are not
-% automatically converted to a string.
+% Converts the \meta{token list} to a \meta{string}, and prepends the
+% result to \meta{str var}. The current contents of the \meta{str
+% var} are not automatically converted to a string.
% \end{function}
%
% \begin{function}
@@ -177,50 +184,9 @@
% \begin{syntax}
% \cs{str_put_right:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and appends the result to \meta{str var}.
-% The current contents of the \meta{str var} are not
-% automatically converted to a string.
-% \end{function}
-%
-% \begin{function}{\str_input:Nn, \str_ginput:Nn}
-% \begin{syntax}
-% \cs{str_input:Nn} \meta{str var} \Arg{token list}
-% \end{syntax}
-% Converts the \meta{token list} into a \meta{string}, and stores
-% it in the \meta{str var}.
-% Special characters can be input by
-% escaping them with a backslash.
-% \begin{itemize}
-% \item Spaces are ignored unless escaped with a backslash.
-% \item |\xhh| produces the character with code \texttt{hh}
-% in hexadecimal: when |\x| is encountered, up to two hexadecimal digits
-% (\texttt{0}--\texttt{9}, \texttt{a}--\texttt{f},
-% \texttt{A}--\texttt{F})
-% are read to give a number between $0$ and $255$.
-% \item |\x{hh...}| produces the character with code \texttt{hh...}
-% (an arbitrary number of hexadecimal digits are read):
-% this is mostly useful for Unicode-aware engines.
-% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters:
-% \begin{center}
-% \begin{tabular}{cccc}
-% |\a| & |\^^G| & alarm & hex \texttt{07} \\
-% |\e| & |\^^[| & escape & hex \texttt{1B} \\
-% |\f| & |\^^L| & form feed & hex \texttt{0C} \\
-% |\n| & |\^^J| & new line & hex \texttt{0A} \\
-% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\
-% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\
-% \end{tabular}
-% \end{center}
-% \end{itemize}
-% For instance,
-% \begin{quote}
-% \cs{tl_new:N} \cs{l_my_str} \\
-% \cs{str_input:Nn} \cs{l_my_str} |{\x3C \\ \# abc\ def\^\n}|
-% \end{quote}
-% results in \cs{l_my_str} containing the characters |<\#abc def^|,
-% followed by a newline character (hex \texttt{0A})
-% since |<| has \textsc{ascii} code \texttt{3C} (in hexadecimal).
+% Converts the \meta{token list} to a \meta{string}, and appends the
+% result to \meta{str var}. The current contents of the \meta{str
+% var} are not automatically converted to a string.
% \end{function}
%
% \section{Characters given by their position}
@@ -231,63 +197,67 @@
% \cs{str_length:n} \Arg{token list}
% \end{syntax}
% Leaves the length of the string representation of \meta{token list}
-% in the input stream as an integer denotation. The functions differ in
-% their treatment of spaces.
-% In the case of \cs{str_length:N} and \cs{str_length:n},
-% all characters including spaces are counted.
-% The \cs{str_length_ignore_spaces:n} leaves the number of non-space
+% in the input stream as an integer denotation. The functions differ
+% in their treatment of spaces. In the case of \cs{str_length:N} and
+% \cs{str_length:n}, all characters including spaces are counted. The
+% \cs{str_length_ignore_spaces:n} leaves the number of non-space
% characters in the input stream.
% \end{function}
%
+% \begin{function}[EXP]{\str_count_spaces:N, \str_count_spaces:n}
+% \begin{syntax}
+% \cs{str_count_spaces:n} \Arg{token list}
+% \end{syntax}
+% Leaves in the input stream the number of space characters in the
+% string representation of \meta{token list}, as an integer
+% denotation. Of course, this function has no \texttt{_ignore_spaces}
+% variant.
+% \end{function}
+%
% \begin{function}[EXP]{\str_head:N, \str_head:n, \str_head_ignore_spaces:n}
% \begin{syntax}
% \cs{str_head:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} into a \meta{string}.
-% The first character in the \meta{string} is then
-% left in the input stream, with category code \enquote{other}.
-% The functions differ in their treatment of spaces.
-% In the case of \cs{str_head:N} and \cs{str_head:n},
-% a leading space is returned with category code $10$ (blank space).
-% The \cs{str_head_ignore_spaces:n} function leaves the first
-% non-space character in the input stream.
-% If the \meta{token list} is empty (or blank in the case of the
-% \texttt{_ignore_spaces} variant), then nothing is left on the
-% input stream.
+% Converts the \meta{token list} into a \meta{string}. The first
+% character in the \meta{string} is then left in the input stream,
+% with category code \enquote{other}. The functions differ in their
+% treatment of spaces. In the case of \cs{str_head:N} and
+% \cs{str_head:n}, a leading space is returned with category code $10$
+% (blank space). The \cs{str_head_ignore_spaces:n} function leaves
+% the first non-space character in the input stream. If the
+% \meta{token list} is empty (or blank in the case of the
+% \texttt{_ignore_spaces} variant), then nothing is left on the input
+% stream.
% \end{function}
%
% \begin{function}[EXP]{\str_tail:N, \str_tail:n, \str_tail_ignore_spaces:n}
% \begin{syntax}
% \cs{str_tail:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% removes the first character, and leaves
-% the remaining characters (if any) in the input stream,
-% with category codes $12$ and $10$ (for spaces).
-% The functions differ in the case where the first character
-% is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim
-% only that space, while \cs{str_tail_ignore_spaces:n} removes
-% the first non-space character and any space before it.
-% If the \meta{token list} is empty (or blank in the case of the
-% \texttt{_ignore_spaces} variant), then nothing is left on the
-% input stream.
+% Converts the \meta{token list} to a \meta{string}, removes the first
+% character, and leaves the remaining characters (if any) in the input
+% stream, with category codes $12$ and $10$ (for spaces). The
+% functions differ in the case where the first character is a space:
+% \cs{str_tail:N} and \cs{str_tail:n} will trim only that space, while
+% \cs{str_tail_ignore_spaces:n} removes the first non-space character
+% and any space before it. If the \meta{token list} is empty (or
+% blank in the case of the \texttt{_ignore_spaces} variant), then
+% nothing is left on the input stream.
% \end{function}
%
% \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn}
% \begin{syntax}
% \cs{str_item:nn} \Arg{token list} \Arg{integer expression}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}, and
-% leaves in the input stream the character in position
-% \meta{integer expression} of the \meta{string}.
-% In the case of \cs{str_item:Nn} and \cs{str_item:nn},
-% all characters including spaces are taken into account.
-% The \cs{str_item_ignore_spaces:nn} function skips spaces
-% in its argument.
-% If the \meta{integer expression} is negative, characters
-% are counted from the end of the \meta{string}. Hence, $-1$ is
-% the right-most character, \emph{etc.}, while $0$ is the first
-% (left-most) character.
+% Converts the \meta{token list} to a \meta{string}, and leaves in the
+% input stream the character in position \meta{integer expression} of
+% the \meta{string}. In the case of \cs{str_item:Nn} and
+% \cs{str_item:nn}, all characters including spaces are taken into
+% account. The \cs{str_item_ignore_spaces:nn} function skips spaces
+% in its argument. If the \meta{integer expression} is negative,
+% characters are counted from the end of the \meta{string}. Hence,
+% $-1$ is the right-most character, \emph{etc.}, while $0$ is the
+% first (left-most) character.
% \end{function}
%
% \begin{function}[EXP]
@@ -295,16 +265,16 @@
% \begin{syntax}
% \cs{str_substr:nnn} \Arg{token list} \Arg{start index} \Arg{end index}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and leaves in the input stream the characters
-% between \meta{start index} (inclusive) and \meta{end index} (exclusive).
+% Converts the \meta{token list} to a \meta{string}, and leaves in the
+% input stream the characters from the \meta{start index} inclusive to
+% the \meta{end index} exclusive. Note that the length of the
+% substring is equal to the difference between the two \meta{indices}.
% If either of \meta{start index} or \meta{end index} is negative,
-% then it is incremented by the length of the list.
-% If either of \meta{start index} or \meta{end index} is empty,
-% it is replaced by the corresponding end-point of the string.
-% Both \meta{start index} and \meta{end index} count from $0$
-% for the first (left most) character.
-% For instance,
+% then it is incremented by the length of the list. If either of
+% \meta{start index} or \meta{end index} is empty, it is replaced by
+% the corresponding end-point of the string. Both \meta{start index}
+% and \meta{end index} count from $0$ for the first (left most)
+% character. For instance,
% \begin{verbatim}
% \iow_term:x { \str_substr:nnn { abcdef } { 1 } { 4 } }
% \iow_term:x { \str_substr:nnn { abcdef } { -4 } { } }
@@ -330,8 +300,8 @@
% \begin{verbatim}
% \str_if_eq_p:xx { abc } { \tl_to_str:n { abc } }
% \end{verbatim}
-% is logically \texttt{true}. All versions of these functions are fully
-% expandable (including those involving an \texttt{x}-type
+% is logically \texttt{true}. All versions of these functions are
+% fully expandable (including those involving an \texttt{x}-type
% expansion).
% \end{function}
%
@@ -340,29 +310,31 @@
% Traditionally, string encodings only specify how strings of characters
% should be stored as bytes. However, the resulting lists of bytes are
% often to be used in contexts where only a restricted subset of bytes
-% are permitted (\textsc{pdf} string objects, \textsc{url}s).
-% Hence, storing strings of characters has two components.
+% are permitted (\emph{e.g.}, \textsc{pdf} string objects,
+% \textsc{url}s). Hence, storing a string of characters is done in two
+% steps.
% \begin{itemize}
-% \item The code points (\enquote{character codes})
-% are expressed as bytes following a given
-% \enquote{encoding}. This can be \textsc{utf-16},
-% \textsc{iso 8859-1}, \emph{etc.}
-% See Table~\ref{tab:encodings} for a list
-% of encodings supported.\footnote{Encodings and escapings
-% will be added as they are requested.}
-% \item Bytes are translated to \TeX{} tokens through
-% a given \enquote{escaping}. Those are defined for
-% the most part by the \texttt{pdf} file format.
-% See Table~\ref{tab:escapings} for a list
-% of escaping methods supported.\footnotemark
+% \item The code points (\enquote{character codes}) are expressed as
+% bytes following a given \enquote{encoding}. This can be
+% \textsc{utf-16}, \textsc{iso 8859-1}, \emph{etc.} See
+% Table~\ref{tab:encodings} for a list of supported
+% encodings.\footnote{Encodings and escapings will be added as they
+% are requested.}
+% \item Bytes are translated to \TeX{} tokens through a given
+% \enquote{escaping}. Those are defined for the most part by the
+% \texttt{pdf} file format. See Table~\ref{tab:escapings} for a
+% list of escaping methods supported.\footnotemark
% \end{itemize}
%
% \begin{table}\centering
% \caption{\label{tab:encodings}Supported encodings.
% Non-alphanumeric characters are ignored,
-% and capital letters are lower-cased.}
+% and capital letters are lower-cased
+% before searching for the encoding in this list.}
% \begin{tabular}{cc}
% \toprule
+% \meta{Encoding} & description \\
+% \midrule
% \texttt{utf8} & \textsc{utf-8} \\
% \texttt{utf16} & \textsc{utf-16}, with byte-order mark \\
% \texttt{utf16be} & \textsc{utf-16}, big-endian \\
@@ -386,8 +358,7 @@
% \texttt{iso885915}, \texttt{latin9} & \textsc{iso 8859-15} \\
% \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\
% \midrule
-% \texttt{native} & Native Unicode string. \\
-% \texttt{internal} & For curious programmers. \\
+% Empty & Native (Unicode) string. \\
% \bottomrule
% \end{tabular}
% \end{table}
@@ -395,9 +366,12 @@
% \begin{table}\centering
% \caption{\label{tab:escapings}Supported escapings.
% Non-alphanumeric characters are ignored,
-% and capital letters are lower-cased.}
+% and capital letters are lower-cased
+% before searching for the escaping in this list.}
% \begin{tabular}{cc}
% \toprule
+% \meta{Escaping} & description \\
+% \midrule
% \texttt{bytes}, or empty
% & arbitrary bytes \\
% \texttt{hex}, \texttt{hexadecimal}
@@ -412,86 +386,91 @@
% \end{tabular}
% \end{table}
%
-% \begin{function}{\str_set_convert:Nnnn}
+% \begin{function}{\str_set_convert:Nnnn,\str_gset_convert:Nnnn}
% \begin{syntax}
-% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string}
-% ~~\Arg{name 1} \Arg{name 2}
+% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string} \Arg{name~1} \Arg{name~2}
% \end{syntax}
-% This function converts the \meta{string} from the encoding
-% given by \meta{name 1} to the encoding given by \meta{name 2},
-% and stores the result in the \meta{str~var}.
-% Each \meta{name} can have the form \meta{encoding} or
-% \meta{encoding}\texttt{/}\meta{escaping}, where the possible values
-% of \meta{encoding} and \meta{escaping} are given in
-% Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively.
-% The default escaping is to input and output bytes directly.
-% The special encodings \texttt{native} and \texttt{internal}
-% do not support a \meta{escaping}, since those formats do not
-% correspond to strings of bytes.
+% This function converts the \meta{string} from the encoding given by
+% \meta{name~1} to the encoding given by \meta{name~2}, and stores the
+% result in the \meta{str~var}. Each \meta{name} can have the form
+% \meta{encoding} or \meta{encoding}\texttt{/}\meta{escaping}, where
+% the possible values of \meta{encoding} and \meta{escaping} are given
+% in Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively.
+% The default escaping is to input and output bytes directly. The
+% special case of an empty \meta{name} indicates the use of
+% \enquote{native} strings, 8-bit for pdf\TeX{}, and Unicode strings
+% for the other two engines.
%
% For example,
% \begin{verbatim}
-% \str_set_convert:Nnnn \l_foo_str { Hello! }
-% { native } { utf16/hex }
+% \str_set_convert:Nnnn \l_foo_str { Hello! } { } { utf16/hex }
% \end{verbatim}
% results in the variable \cs{l_foo_str} holding the string
-% \texttt{FEFF00480065006C006C006F0021}. This is obtained
-% by converting each character in the string \texttt{Hello!}
-% to the \textsc{utf-16} encoding, and expressing each byte
-% as a pair of hexadecimal digits. Note the presence of a
-% (big-endian) byte order mark \hexnum{FEFF}, which can be
-% avoided by specifying the encoding \texttt{utf16be/hex}.
-%
-% Parts of the \meta{string} which cannot be converted to bytes
-% using the \meta{escaping 1} are silently ignored (including non-byte
-% characters which could be present in the string).
-% Sequences of bytes which are not valid in the \meta{encoding 1}
-% are converted to the replacement character \hexnum{FFFD}, and raise
-% an error.
-% Characters which cannot be encoded in the \meta{encoding 2}
-% are silently ignored.
+% \texttt{FEFF00480065006C006C006F0021}. This is obtained by
+% converting each character in the (native) string \texttt{Hello!} to
+% the \textsc{utf-16} encoding, and expressing each byte as a pair of
+% hexadecimal digits. Note the presence of a (big-endian) byte order
+% mark \hexnum{FEFF}, which can be avoided by specifying the encoding
+% \texttt{utf16be/hex}.
+%
+% An error is raised if the \meta{string} is not valid according to
+% the \meta{escaping~1} and \meta{encoding~1}, or if it cannot be
+% reencoded in the \meta{encoding~2} and \meta{escaping~2} (for
+% instance, if a character does not exist in the \meta{encoding~2}).
+% Erroneous input is replaced by the Unicode replacement character
+% \hexnum{FFFD}, and characters which cannot be reencoded are replaced
+% by either the replacement character \hexnum{FFFD} if it exists in
+% the \meta{encoding~2}, or an encoding-specific replacement
+% character, or the question mark character.
% \end{function}
%
-% \section{Internal string functions}
-%
-% \begin{function}{\str_escape_use:NNNn}
+% \begin{function}[TF]{\str_set_convert:Nnnn,\str_gset_convert:Nnnn}
% \begin{syntax}
-% \cs{str_escape_use:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list}
+% \cs{str_set_convert:NnnnTF} \meta{str~var} \Arg{string} \Arg{name~1} \Arg{name~2} \Arg{true code} \Arg{false code}
% \end{syntax}
-% The \meta{token list} is converted to a string, then read from
-% left to right, interpreting backslashes as escaping the next character.
-% Unescaped characters are fed to the function \meta{fn1},
-% and escaped characters are fed to the function \meta{fn2}
-% within an \texttt{x}-expansion context (typically those functions
-% perform some tests on their argument to decide how to output them).
-% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x|
-% are recognized as described for \cs{str_input:Nn}, and those
-% are replaced by the corresponding character, then fed to
-% \meta{fn3}. The result is then left in the input stream.
+% As \cs{str_set_convert:Nnnn}, converts the \meta{string} from the
+% encoding given by \meta{name~1} to the encoding given by
+% \meta{name~2}, and assigns the result to \meta{str~var}. Contrarily
+% to \cs{str_set_convert:Nnnn}, the conditional variant does not raise
+% errors in case the \meta{string} is not valid according to the
+% \meta{name~1} encoding, or cannot be expressed in the \meta{name~2}
+% encoding. Instead, the \meta{false code} is performed.
% \end{function}
%
-% \begin{function}[rEXP]{\str_aux_toks_range:nn}
-% \begin{syntax}
-% \cs{str_aux_toks_range:nn} \meta{start} \meta{end}
-% \end{syntax}
-% Expands to the contents of the \tn{toks} registers numbered
-% from \meta{start} (inclusive) to \meta{end} (exclusive).
-% This function is used in \pkg{l3regex}.
-% \end{function}
+% \section{Internal string functions}
+%
+% \begin{variable}{\c_max_char_int}
+% The maximum valid character code, $255$ for pdf\TeX{}, and $1114111$
+% for the two other engines.
+% \end{variable}
%
% \begin{function}{\str_aux_hexadecimal_use:NTF}
% \begin{syntax}
-% \cs{str_aux_hexadecimal_use:NTF} \meta{token}
-% ~~\Arg{true code} \Arg{false code}
+% \cs{str_aux_hexadecimal_use:NTF} \meta{token} \Arg{true code} \Arg{false code}
% \end{syntax}
-% If the \meta{token} is a hexdecimal digit (upper case or lower case),
-% its upper-case version is left in the input stream, \emph{followed}
-% by the \meta{true code}. Otherwise, the \meta{false code} is left
-% in the input stream.
+% If the \meta{token} is a hexdecimal digit (upper case or lower
+% case), its upper-case version is left in the input stream,
+% \emph{followed} by the \meta{true code}. Otherwise, the \meta{false
+% code} is left in the input stream.
% \begin{texnote}
% This function will fail if the escape character is a hexadecimal
% digit, or has a character code in the range $[0,5]$. We are thus
-% careful to set the escape character to a known value before using it.
+% careful to set the escape character to a known value before using
+% it.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}{\str_aux_octal_use:NTF}
+% \begin{syntax}
+% \cs{str_aux_octal_use:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% If the \meta{token} is an octal digit, it is left in the input
+% stream, \emph{followed} by the \meta{true code}. Otherwise, the
+% \meta{false code} is left in the input stream.
+% \begin{texnote}
+% This function will fail if the escape character is an octal
+% digit. We are thus careful to set the escape character to a known
+% value before using it.
% \end{texnote}
% \end{function}
%
@@ -502,9 +481,10 @@
% Converts the \meta{token list} to a \meta{other string}, where
% spaces have category code \enquote{other}.
% \begin{texnote}
-% These functions can be \texttt{f}-expanded without fear of losing
-% a leading space, since spaces do not have category code $10$ in
-% their result.
+% This function can be \texttt{f}-expanded without fear of losing a
+% leading space, since spaces do not have category code $10$ in its
+% result. This function takes a time quadratic in the length of the
+% string; \cs{str_gset_other:Nn} is faster but not expandable.
% \end{texnote}
% \end{function}
%
@@ -513,8 +493,8 @@
% \cs{str_gset_other:Nn} \meta{tl~var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} to an \meta{other string}, where
-% spaces have category code \enquote{other}, and assigns the result
-% to the \meta{tl~var}, globally.
+% spaces have category code \enquote{other}, and assigns the result to
+% the \meta{tl~var}, globally.
% \end{function}
%
% \begin{function}[EXP]
@@ -526,24 +506,56 @@
% \begin{syntax}
% \cs{str_if_contains_char:nNTF} \Arg{token list} \meta{char}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}
-% and tests whether the \meta{char} is present in the \meta{string}.
-% Spaces are ignored.
+% Converts the \meta{token list} to a \meta{string} and tests whether
+% the \meta{char} is present in the \meta{string}. Spaces are
+% ignored.
+% \end{function}
+%
+% \begin{function}{\str_declare_eight_bit_encoding:nnn}
+% \begin{syntax}
+% \cs{str_declare_eight_bit_encoding:nnn} \Arg{name} \Arg{mapping} \Arg{missing}
+% \end{syntax}
+% This declares the encoding \meta{name} to map bytes to Unicode
+% characters according to the \meta{mapping}, and map those bytes
+% which are not mentionned in the \meta{mapping} either to the
+% replacement character (if they appear in \meta{missing}), or to
+% themselves.
% \end{function}
%
% \section{Possibilities, and things to do}
%
+% Encoding/escaping-related tasks.
+% \begin{itemize}
+% \item Describe the internal format in the code comments. Refuse code
+% points in $[\hexnum{D800}, \hexnum{DFFF}]$ in the internal
+% representation?
+% \item Add documentation about each encoding and escaping method, and
+% add examples.
+% \item The \texttt{hex} unescaping should raise an error for
+% odd-length strings.
+% \item Decide what bytes should be escaped in the \texttt{url}
+% escaping. Perhaps |!'()*-./0123456789_| are safe, and all other
+% characters should be escaped?
+% \item Automate generation of 8-bit mapping files.
+% \item Change the framework for 8-bit encodings: for decoding from
+% 8-bit to Unicode, use $256$ integer registers; for encoding, use a
+% tree-box.
+% \item More encodings (see Heiko's \pkg{stringenc}). CESU?
+% \item More escapings: shell escapes, lua escapes, etc?
+% \end{itemize}
+%
+% Other string tasks.
% \begin{itemize}
-% \item Use flags rather than expandable errors.
-% \item Add documentation about each encoding and escaping method,
-% and add examples.
-% \item Perhaps use the pdf\TeX{} primitives for escapings when available.
-% But this changes error detection between engines.
-% \item More encodings (see Heiko's \pkg{stringenc}). CESU.
-% \item More escapings: shell escapes, lua escapes, etc?
-% \item \cs{str_if_head_eq:nN}
-% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}?
-% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}?
+% \item \cs{str_if_head_eq:nN}
+% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}?
+% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}?
+% \item Should \cs{str_substr:Nnn} be \cs{str_range:Nnn}?
+% \item Introduce \cs{str_slice:Nnnn} with a third \enquote{step}
+% argument? Or should we simply have \cs{str_slice:Nn}
+% \meta{string} \Arg{clist}, where the \meta{clist}'s items are
+% either one integer expression, two integer expressions separated
+% by |:|, or three integer expressions separated by |:|, \emph{cf.}
+% Python's extended slice syntax?
% \end{itemize}
%
% \end{documentation}
@@ -559,13 +571,14 @@
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3tl-analysis}
+\RequirePackage{l3tl-analysis,l3flag}
% \end{macrocode}
%
-% Those string-related functions are defined in \pkg{l3kernel}.
+% The following string-related functions are currently defined in
+% \pkg{l3kernel}.
% \begin{itemize}
% \item \cs{str_if_eq:nn}[pTF] and variants,
-% \item \cs{str_if_eq_return:on},
+% \item \cs{str_if_eq_return:on}, \cs{str_if_eq_return:xx}
% \item \cs{tl_to_str:n}, \cs{tl_to_str:N}, \cs{tl_to_str:c},
% \item \cs{token_to_str:N}, \cs{cs_to_str:N}
% \item \cs{str_head:n}, \cs{str_head_aux:w}, (copied here)
@@ -576,6 +589,16 @@
%
% \subsection{Helpers}
%
+% \subsubsection{A function unrelated to strings}
+%
+% \begin{macro}[EXP,aux]{\use_ii_i:nn}
+% A function used to swap its arguments.
+% \begin{macrocode}
+\cs_if_exist:NF \use_ii_i:nn
+ { \cs_new:Npn \use_ii_i:nn #1#2 { #2 #1 } }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Assigning strings}
%
% \begin{macro}
@@ -597,65 +620,58 @@
% }
% Simply convert the token list inputs to \meta{strings}.
% \begin{macrocode}
-\group_begin:
- \cs_set_protected_nopar:Npn \str_tmp:w #1
- {
- \cs_new_protected:cpx { str_ #1 :Nn } ##1##2
- { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } }
- \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx }
- }
- \str_tmp:w { set }
- \str_tmp:w { gset }
- \str_tmp:w { const }
- \str_tmp:w { put_left }
- \str_tmp:w { gput_left }
- \str_tmp:w { put_right }
- \str_tmp:w { gput_right }
-\group_end:
+\tl_map_inline:nn
+ {
+ { set }
+ { gset }
+ { const }
+ { put_left }
+ { gput_left }
+ { put_right }
+ { gput_right }
+ }
+ {
+ \cs_new_protected:cpx { str_ #1 :Nn } ##1##2
+ { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } }
+ \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx }
+ }
% \end{macrocode}
% \end{macro}
%
% \subsubsection{Variables and constants}
%
% \begin{macro}{\str_tmp:w}
-% \begin{variable}{\l_str_tmpa_tl}
+% \begin{variable}{\l_str_internal_int}
+% \begin{variable}{\l_str_internal_tl}
% Internal scratch space for some functions.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_tmp:w { }
-\tl_new:N \l_str_tmpa_tl
-\int_new:N \l_str_tmpa_int
+\tl_new:N \l_str_internal_tl
+\int_new:N \l_str_internal_int
% \end{macrocode}
% \end{variable}
+% \end{variable}
% \end{macro}
%
% \begin{variable}{\g_str_result_tl}
-% The \cs{g_str_result_tl} variable is used to hold the result
-% of various internal string operations which are typically
-% performed in a group. The variable is global so that it remains
-% defined outside the group, to be assigned to a user provided variable.
+% The \cs{g_str_result_tl} variable is used to hold the result of
+% various internal string operations (mostly conversions) which are
+% typically performed in a group. The variable is global so that it
+% remains defined outside the group, to be assigned to a user-provided
+% variable.
% \begin{macrocode}
\tl_new:N \g_str_result_tl
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_str_char_int}
-% When converting from various forms to characters,
-% \cs{l_str_char_int} is the character code of the
-% character which should be created.
-% \begin{macrocode}
-\int_new:N \l_str_char_int
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}
% {
% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one,
% \c_ninety_seven, \c_one_hundred_twenty_three,
% \c_one_hundred_twenty_seven
% }
-% We declare here some integer values which delimit ranges of
-% ASCII characters of various types. This is mostly used in
-% \pkg{l3regex}.
+% We declare here some integer values which delimit ranges of ASCII
+% characters of various types. This is mostly used in \pkg{l3regex}.
% \begin{macrocode}
\int_const:Nn \c_forty_eight { 48 }
\int_const:Nn \c_fifty_eight { 58 }
@@ -668,10 +684,11 @@
% \end{variable}
%
% \begin{variable}{\c_max_char_int}
-% The maximum character code depends on the engine.
+% The maximum valid character code is $255$ for pdf\TeX{}, and
+% $1114111$ for other engines.
% \begin{macrocode}
\int_const:Nn \c_max_char_int
- { \pdftex_if_engine:TF { 255 } { 1114111 } }
+ { \pdftex_if_engine:TF { "FF } { "10FFFF } }
% \end{macrocode}
% \end{variable}
%
@@ -692,8 +709,8 @@
% \c_tilde_str,
% \c_percent_str
% }
-% For all of those strings, \cs{cs_to_str:N} produce characters
-% with the correct category code.
+% For all of those strings, \cs{cs_to_str:N} produces characters with
+% the correct category code.
% \begin{macrocode}
\tl_const:Nx \c_backslash_str { \cs_to_str:N \\ }
\tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ }
@@ -704,22 +721,46 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_str_aliases_prop}
-% To avoid needing one file per encoding/escaping alias,
-% we keep track of those in a property list.
+% \begin{variable}{\g_str_file_prop}
+% To avoid needing one file per encoding/escaping alias, we keep track
+% of those in a property list.
+% \begin{macrocode}
+\prop_new:N \g_str_file_prop
+\prop_gput:Nnn \g_str_file_prop { latin1 } { iso88591 }
+\prop_gput:Nnn \g_str_file_prop { latin2 } { iso88592 }
+\prop_gput:Nnn \g_str_file_prop { latin3 } { iso88593 }
+\prop_gput:Nnn \g_str_file_prop { latin4 } { iso88594 }
+\prop_gput:Nnn \g_str_file_prop { latin5 } { iso88599 }
+\prop_gput:Nnn \g_str_file_prop { latin6 } { iso885910 }
+\prop_gput:Nnn \g_str_file_prop { latin7 } { iso885913 }
+\prop_gput:Nnn \g_str_file_prop { latin8 } { iso885914 }
+\prop_gput:Nnn \g_str_file_prop { latin9 } { iso885915 }
+\prop_gput:Nnn \g_str_file_prop { latin10 } { iso885916 }
+\prop_gput:Nnn \g_str_file_prop { utf16le } { utf16 }
+\prop_gput:Nnn \g_str_file_prop { utf16be } { utf16 }
+\prop_gput:Nnn \g_str_file_prop { utf32le } { utf32 }
+\prop_gput:Nnn \g_str_file_prop { utf32be } { utf32 }
+\prop_gput:Nnn \g_str_file_prop { hexadecimal } { hex }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_str_error_bool}
+% In conversion functions with a built-in conditional, errors are not
+% reported directly to the user, but the information is collected in
+% this boolean, used at the end to decide on which branch of the
+% conditional to take.
+% \begin{macrocode}
+\bool_new:N \g_str_error_bool
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{str_byte, str_error}
+% Conversions from one \meta{encoding}/\meta{escaping} pair to another
+% are done within \texttt{x}-expanding assignments. Errors are
+% signalled by raising the relevant flag.
% \begin{macrocode}
-\prop_new:N \g_str_aliases_prop
-\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 }
-\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 }
-\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 }
-\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 }
-\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 }
-\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 }
-\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 }
-\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 }
-\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 }
-\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 }
-\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex }
+\flag_new:n { str_byte }
+\flag_new:n { str_error }
% \end{macrocode}
% \end{variable}
%
@@ -727,10 +768,10 @@
%
% \begin{macro}[EXP]{\tl_to_other_str:n}
% \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w}
-% Replaces all spaces by \enquote{other} spaces, after converting
-% the token list to a string via \cs{tl_to_str:n}.
-% This function is \texttt{f}-expandable, storing the part of the string
-% with escaped spaces between the \cs{q_mark} and \cs{q_stop} markers.
+% Replaces all spaces by \enquote{other} spaces, after converting the
+% token list to a string via \cs{tl_to_str:n}. This function is
+% \texttt{f}-expandable, storing the part of the string with escaped
+% spaces between the \cs{q_mark} and \cs{q_stop} markers.
% \begin{macrocode}
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
@@ -743,7 +784,7 @@
\exp_after:wN \tl_to_other_str_loop:w \tl_to_str:n {#1} ~ %
A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
}
- \cs_new_nopar:Npn \tl_to_other_str_loop:w
+ \cs_new:Npn \tl_to_other_str_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 \q_stop
{
\if_meaning:w A #8
@@ -752,7 +793,7 @@
\tl_to_other_str_loop:w
#9 #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * \q_stop
}
- \cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
+ \cs_new:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
{ \fi: #2 }
}
% \end{macrocode}
@@ -762,12 +803,11 @@
% \begin{macro}[int]{\str_gset_other:Nn}
% \begin{macro}[aux,EXP]{\str_gset_other_loop:w}
% \begin{macro}[aux,EXP]{\str_gset_other_end:w}
-% This function could be done by using \cs{tl_to_other_str:n}
-% within an \texttt{x}-expansion, but that would take a time
-% quadratic in the size of the string. Instead, we can
-% \enquote{leave the result behind us} in the input stream,
-% to be captured into the expanding assignment. This gives us
-% a linear time.
+% This function could be done by using \cs{tl_to_other_str:n} within
+% an \texttt{x}-expansion, but that would take a time quadratic in the
+% size of the string. Instead, we can \enquote{leave the result behind
+% us} in the input stream, to be captured into the expanding
+% assignment. This gives us a linear time.
% \begin{macrocode}
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
@@ -780,10 +820,10 @@
\tl_gset:Nx #1
{
\exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ %
- A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
+ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_stop
}
}
- \cs_new_nopar:Npn \str_gset_other_loop:w
+ \cs_new:Npn \str_gset_other_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~
{
\if_meaning:w A #9
@@ -792,7 +832,7 @@
#1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9
\str_gset_other_loop:w *
}
- \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
+ \cs_new:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
{ \fi: #1 }
}
% \end{macrocode}
@@ -800,66 +840,70 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Functions unrelated to strings}
-%
-% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn}
-% A function which may already be defined elsewhere.
-% \begin{macrocode}
-\cs_if_exist:NF \use_i:nnnnnnnn
- { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
-% \end{macrocode}
-% \end{macro}
+% \subsection{Characters given by their position}
%
-% \begin{macro}[int, rEXP]{\str_aux_toks_range:nn}
-% \begin{macro}[aux, rEXP]{\str_aux_toks_range:ww}
-% Some non-expandable functions (in this module,
-% \cs{str_aux_escape:NNNn}, which is used by \cs{str_(g)input:Nn})
-% store pieces of their result in \tn{toks} registers.
-% Those pieces are concatenated using \cs{str_aux_toks_range:nn},
-% which expects two integer registers, \meta{start} and \meta{end},
-% and expands to the contents of the \tn{toks} registers from
-% \meta{start} (inclusive) to \meta{end} (exclusive).
-% This function is also used in the \pkg{l3regex} module.
+% \begin{macro}[EXP]{\str_count_spaces:N}
+% \begin{macro}[EXP]{\str_count_spaces:n}
+% \begin{macro}[EXP,aux]{\str_count_spaces_loop:wwwwwwwww}
+% To speed up this function, we grab $9$ spaces in each step. The
+% loop stops when the last argument is one of the trailing
+% |X|\meta{number}, and that \meta{number} is added to the sum of $9$
+% that preceeds, to adjust the result.
% \begin{macrocode}
-\cs_new:Npn \str_aux_toks_range:nn #1#2
+\cs_new:Npn \str_count_spaces:N
+ { \exp_args:No \str_count_spaces:n }
+\cs_new:Npn \str_count_spaces:n #1
{
- \exp_after:wN \str_aux_toks_range:ww
- \int_use:N \int_eval:w #1 \exp_after:wN ;
- \int_use:N \int_eval:w #2 ;
- \prg_break_point:n { }
+ \int_eval:n
+ {
+ \exp_after:wN \str_count_spaces_loop:wwwwwwwww
+ \tl_to_str:n {#1} ~
+ X 7 ~ X 6 ~ X 5 ~ X 4 ~ X 3 ~ X 2 ~ X 1 ~ X 0 ~ X -1 ~
+ \q_stop
+ }
}
-\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ;
+\cs_new:Npn \str_count_spaces_loop:wwwwwwwww #1~#2~#3~#4~#5~#6~#7~#8~#9~
{
- \if_num:w #1 = #2 \exp_stop_f:
- \exp_after:wN \prg_map_break:
+ \if_meaning:w X #9
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
- \tex_the:D \tex_toks:D #1 \exp_stop_f:
- \exp_after:wN \str_aux_toks_range:ww
- \int_use:N \int_eval:w #1 + \c_one ; #2 ;
+ \c_nine + \str_count_spaces_loop:wwwwwwwww
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-%
-% \subsection{Characters given by their position}
+% \end{macro}
%
% \begin{macro}[EXP]{\str_length:N}
% \begin{macro}[EXP]{\str_length:n}
-% \begin{macro}[EXP]{\str_length_ignore_spaces:n}
% \begin{macro}[EXP]{\str_length_unsafe:n}
-% \begin{macro}[EXP,aux]{\str_length_aux:n,\str_length_loop:NNNNNNNNN}
-% The length of a string is found by first changing all spaces
-% to other spaces using \cs{tl_to_other_str:n}, then counting
-% characters $9$ at a time. When the end is reached, |#9|
-% has the form |X|\meta{digit}, the catcode test is true,
-% the digit gets added to the sum, and the loop is terminated
-% by \cs{use_none_delimit_by_q_stop:w}.
+% \begin{macro}[EXP]{\str_length_ignore_spaces:n}
+% \begin{macro}[EXP,aux]{\str_length_aux:n, \str_length_loop:NNNNNNNNN}
+% To measure the length of a string we could first escape all spaces
+% using \cs{tl_to_other_str:N}, then measure the length of this token
+% list. However, this would be quadratic in the length of the string,
+% and we can do better. Namely, add the number of spaces (counted
+% using the functions defined above) to the length ignoring spaces.
+% To measure the length ignoring spaces we use the same technique as
+% for counting spaces: loop, grabbing $9$ characters at each step, and
+% end as soon as we reach one of the $9$ trailing items. The
+% \texttt{_unsafe} variant expects a token list consisting entirely of
+% category code $12$ characters.
% \begin{macrocode}
\cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n }
\cs_new:Npn \str_length:n #1
- { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } }
-\cs_new_nopar:Npn \str_length_unsafe:n #1
- { \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } }
+ {
+ \str_length_aux:n
+ {
+ \str_count_spaces:n {#1}
+ + \exp_after:wN \str_length_loop:NNNNNNNNN \tl_to_str:n {#1}
+ }
+ }
+\cs_new:Npn \str_length_unsafe:n #1
+ {
+ \str_length_aux:n
+ { \str_length_loop:NNNNNNNNN #1 }
+ }
\cs_new:Npn \str_length_ignore_spaces:n #1
{
\str_length_aux:n
@@ -878,7 +922,7 @@
}
\cs_set:Npn \str_length_loop:NNNNNNNNN #1#2#3#4#5#6#7#8#9
{
- \if_catcode:w X #9
+ \if_meaning:w X #9
\exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
\c_nine + \str_length_loop:NNNNNNNNN
@@ -893,21 +937,18 @@
% \begin{macro}[EXP]{\str_head:N}
% \begin{macro}[EXP]{\str_head:n}
% \begin{macro}[EXP]{\str_head_ignore_spaces:n}
-% \begin{macro}[EXP]{\str_head_unsafe:n}
% \begin{macro}[EXP,aux]{\str_head_aux:w}
-% The cases of \cs{str_head_ignore_spaces:n} and
-% \cs{str_head_unsafe:n} are mostly identical to
-% \cs{tl_head:n}.
-% As usual, \cs{str_head:N} expands its argument and hands it to
-% \cs{str_head:n}. To circumvent the fact that \TeX{} skips spaces
-% when grabbing undelimited macro parameters, \cs{str_head_aux:w}
-% takes an argument delimited by a space. If |#1| starts with a
-% non-space character, \cs{use_i_delimit_by_q_stop:nw} leaves that
-% in the input stream. On the other hand, if |#1| starts with a
-% space, the \cs{str_head_aux:w} takes an empty argument, and
+% The \texttt{_ignore_spaces} variant is almost identical to
+% \cs{tl_head:n}. As usual, \cs{str_head:N} expands its argument and
+% hands it to \cs{str_head:n}. To circumvent the fact that \TeX{}
+% skips spaces when grabbing undelimited macro parameters,
+% \cs{str_head_aux:w} takes an argument delimited by a space. If |#1|
+% starts with a non-space character, \cs{use_i_delimit_by_q_stop:nw}
+% leaves that in the input stream. On the other hand, if |#1| starts
+% with a space, the \cs{str_head_aux:w} takes an empty argument, and
% the single (braced) space in the definition of \cs{str_head_aux:w}
-% makes its way to the output. Finally, for an empty argument,
-% the (braced) empty brace group in the definition of \cs{str_head:n}
+% makes its way to the output. Finally, for an empty argument, the
+% (braced) empty brace group in the definition of \cs{str_head:n}
% gives an empty result after passing through
% \cs{use_i_delimit_by_q_stop:nw}.
% \begin{macrocode}
@@ -918,36 +959,33 @@
\tl_to_str:n {#1}
{ { } } ~ \q_stop
}
-\cs_set_nopar:Npn \str_head_aux:w #1 ~ %
+\cs_set:Npn \str_head_aux:w #1 ~ %
{ \use_i_delimit_by_q_stop:nw #1 { ~ } }
\cs_new:Npn \str_head_ignore_spaces:n #1
- { \exp_after:wN \use_i_delimit_by_q_stop:nw \tl_to_str:n {#1} { } \q_stop }
-\cs_new_nopar:Npn \str_head_unsafe:n #1
- { \use_i_delimit_by_q_stop:nw #1 { } \q_stop }
+ {
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tl_to_str:n {#1} { } \q_stop
+ }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \begin{macro}[EXP]{\str_tail:N}
% \begin{macro}[EXP]{\str_tail:n}
% \begin{macro}[EXP]{\str_tail_ignore_spaces:n}
-% \begin{macro}[EXP]{\str_tail_unsafe:n}
% \begin{macro}[EXP,aux]{\str_tail_aux:w}
% \begin{macro}[EXP,aux]{\str_tail_aux_ii:w}
-% As when fetching the head of a string, the cases
-% \enquote{\texttt{ignore_spaces:n}} and \enquote{\texttt{unsafe:n}}
-% are similar to \cs{tl_tail:n}.
-% The more commonly used \cs{str_tail:n} function is a little bit
-% more convoluted: hitting the front of the string with
-% \cs{reverse_if:N} \cs{if_charcode:w} \cs{scan_stop:}
-% removes the first character (which necessarily makes the test true,
-% since it cannot match \cs{scan_stop:}). The auxiliary function inserts
-% the required \cs{fi:} to close the conditional, and leaves the tail
-% of the string in the input string. The details are such that an empty
-% string has an empty tail.
+% As when fetching the head of a string, the \texttt{_ignore_spaces}
+% variant is similar to \cs{tl_tail:n}. The more commonly used
+% \cs{str_tail:n} function is a little bit more convoluted: hitting
+% the front of the string with \cs{reverse_if:N} \cs{if_charcode:w}
+% \cs{scan_stop:} removes the first character (which necessarily makes
+% the test true, since it cannot match \cs{scan_stop:}). The auxiliary
+% function inserts the required \cs{fi:} to close the conditional, and
+% leaves the tail of the string in the input string. The details are
+% such that an empty string has an empty tail.
% \begin{macrocode}
\cs_new_nopar:Npn \str_tail:N { \exp_args:No \str_tail:n }
\cs_set:Npn \str_tail:n #1
@@ -956,113 +994,103 @@
\reverse_if:N \if_charcode:w
\scan_stop: \tl_to_str:n {#1} X X \q_stop
}
-\cs_set_nopar:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 }
+\cs_set:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 }
\cs_new:Npn \str_tail_ignore_spaces:n #1
{
\exp_after:wN \str_tail_aux_ii:w
- \tl_to_str:n {#1} X { } X \q_stop
+ \tl_to_str:n {#1} X X \q_stop
}
-\cs_new_nopar:Npn \str_tail_unsafe:n #1
- { \str_tail_aux_ii:w #1 X { } X \q_stop }
-\cs_new_nopar:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 }
+\cs_new:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[EXP,int]{\str_skip_do:nn}
-% \begin{macro}[EXP,aux]{\str_skip_aux:nnnnnnnnn}
-% \begin{macro}[EXP,aux]{\str_skip_end:nn}
-% \begin{macro}[EXP,aux]{\str_skip_end_ii:nwn}
-% Removes |max(#1,0)| characters then leaves |#2| in the input stream.
-% We remove characters $7$ at a time. When the number of characters
-% to remove is not a multiple of $7$, we need to remove less than
-% $7$ characters in the last step. This is done by inserting a number
-% of \texttt{X}, which are discarded as if they were part of the string.
+% \begin{macro}[EXP, int]{\str_skip_c_zero:w}
+% \begin{macro}[EXP, aux]{\str_skip_aux:wNNNNNNNN}
+% \begin{macro}[EXP, aux]{\str_skip_end:w, \str_skip_end_ii:NNNNNNNN}
+% Removes |max(#1,0)| characters from the input stream, and then
+% leaves \cs{c_zero}. This should be expanded using
+% \cs{tex_romannumeral:D}. We remove characters $8$ at a time until
+% there are at most $8$ to remove. Then we do a dirty trick: the
+% \cs{if_case:w} construction leaves between $0$ and $8$ times the
+% \cs{or:} control sequence, and those \cs{or:} become arguments of
+% \cs{str_skip_end_ii:NNNNNNNN}. If the number of characters to remove
+% is $6$, say, then there are two \cs{or:} left, and the $8$ arguments
+% of \cs{str_skip_end_ii:NNNNNNNN} are the two \cs{or:}, and $6$
+% characters from the input stream, exactly what we wanted to
+% remove. Then close the \cs{if_case:w} conditional with \cs{fi:}, and
+% stop the initial expansion with \cs{c_zero} (see places where
+% \cs{str_skip_c_zero:w} is called).
% \begin{macrocode}
-\cs_new:Npn \str_skip_do:nn #1
+\cs_new:Npn \str_skip_c_zero:w #1;
{
- \if_num:w \int_eval:w #1 > \c_seven
- \exp_after:wN \str_skip_aux:nnnnnnnnn
+ \if_num:w \int_eval:w #1 > \c_eight
+ \exp_after:wN \str_skip_aux:wNNNNNNNN
\else:
- \exp_after:wN \str_skip_end:n
+ \exp_after:wN \str_skip_end:w
+ \int_use:N \int_eval:w
\fi:
- {#1}
+ #1 ;
}
-\cs_new:Npn \str_skip_aux:nnnnnnnnn #1#2#3#4#5#6#7#8#9
- { \exp_args:Nf \str_skip_do:nn { \int_eval:n { #1 - \c_seven } } {#2} }
-\cs_new:Npn \str_skip_end:n #1
+\cs_new:Npn \str_skip_aux:wNNNNNNNN #1; #2#3#4#5#6#7#8#9
+ { \exp_after:wN \str_skip_c_zero:w \int_use:N \int_eval:w #1 - \c_eight ; }
+\cs_new:Npn \str_skip_end:w #1 ;
{
- \if_case:w \int_eval:w #1 \int_eval_end:
- \str_skip_end_ii:nwn { XXXXXXX }
- \or: \str_skip_end_ii:nwn { XXXXXX }
- \or: \str_skip_end_ii:nwn { XXXXX }
- \or: \str_skip_end_ii:nwn { XXXX }
- \or: \str_skip_end_ii:nwn { XXX }
- \or: \str_skip_end_ii:nwn { XX }
- \or: \str_skip_end_ii:nwn { X }
- \or: \str_skip_end_ii:nwn { }
- \else: \str_skip_end_ii:nwn { XXXXXXX }
- \fi:
+ \exp_after:wN \str_skip_end_ii:NNNNNNNN
+ \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f:
+ \or: \or: \or: \or: \or: \or: \or: \or:
}
-\cs_new:Npn \str_skip_end_ii:nwn #1#2 \fi: #3
- { \fi: \use_i:nnnnnnnn {#3} #1 }
+\cs_new:Npn \str_skip_end_ii:NNNNNNNN #1#2#3#4#5#6#7#8 { \fi: \c_zero }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[EXP,int]{\str_collect_do:nn}
-% \begin{macro}[EXP,aux]{\str_collect_aux:n,\str_collect_aux:nnNNNNNNN}
+% \begin{macro}[EXP,int]{\str_collect_delimit_by_q_stop:w}
+% \begin{macro}[EXP,aux]{\str_collect_aux:wn, \str_collect_aux:wnNNNNNNN}
% \begin{macro}[EXP,aux]
-% {
-% \str_collect_end:nn,
-% \str_collect_end_ii:nwn,
-% \str_collect_end_iii:nwNNNNNNN
-% }
-% Collects |max(#1,0)| characters, and feeds them as an argument to |#2|.
-% Again, we grab $7$ characters at a time. Instead of inserting
-% a string of \texttt{X} to fill in to a multiple of $7$, we insert
-% empty groups, so that they disappear in this context where arguments
-% are accumulated.
+% {\str_collect_end:wn, \str_collect_end_ii:nnnnnnnnw} Collects
+% |max(#1,0)| characters, and removes everything else until
+% \cs{q_stop}. This is somewhat similar to \cs{str_skip_c_zero:w}, but
+% this time we can only grab $7$ characters at a time. At the end, we
+% use an \cs{if_case:w} trick again, so that the $8$ first arguments
+% of \cs{str_collect_end_ii:nnnnnnnnw} are some \cs{or:}, followed by
+% an \cs{if:}, followed by |#1| characters from the input
+% stream. Simply leaving this in the input stream will close the
+% conditional properly and the \cs{or:} disappear.
% \begin{macrocode}
-\cs_new:Npn \str_collect_do:nn #1#2
- { \str_collect_aux:n {#1} { \str_collect_end_iii:nwNNNNNNN {#2} } }
-\cs_new:Npn \str_collect_aux:n #1
+\cs_new:Npn \str_collect_delimit_by_q_stop:w #1;
{
- \int_compare:nNnTF {#1} > \c_seven
- { \str_collect_aux:nnNNNNNNN }
- { \str_collect_end:n }
- {#1}
+ \exp_after:wN \str_collect_aux:wn
+ \int_use:N \int_eval:w #1 ;
+ { }
}
-\cs_new:Npn \str_collect_aux:nnNNNNNNN #1#2 #3#4#5#6#7#8#9
+\cs_new:Npn \str_collect_aux:wn #1 ;
{
- \exp_args:Nf \str_collect_aux:n
- { \int_eval:n { #1 - \c_seven } }
- { #2 #3#4#5#6#7#8#9 }
+ \if_num:w #1 > \c_seven
+ \exp_after:wN \str_collect_aux:wnNNNNNNN
+ \else:
+ \exp_after:wN \str_collect_end:wn
+ \fi:
+ #1 ;
}
-\cs_new:Npn \str_collect_end:n #1
+\cs_new:Npn \str_collect_aux:wnNNNNNNN #1; #2 #3#4#5#6#7#8#9
{
- \if_case:w \int_eval:w #1 \int_eval_end:
- \str_collect_end_ii:nwn { { } { } { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } }
- \or: \str_collect_end_ii:nwn { { } }
- \or: \str_collect_end_ii:nwn { }
- \else: \str_collect_end_ii:nwn { { } { } { } { } { } { } { } }
- \fi:
+ \exp_after:wN \str_collect_aux:wn
+ \int_use:N \int_eval:w #1 - \c_seven ;
+ { #2 #3#4#5#6#7#8#9 }
}
-\cs_new:Npn \str_collect_end_ii:nwn #1#2 \fi: #3
- { \fi: #3 \q_stop #1 }
-\cs_new:Npn \str_collect_end_iii:nwNNNNNNN #1 #2 \q_stop #3#4#5#6#7#8#9
- { #1 {#2#3#4#5#6#7#8#9} }
+\cs_new:Npn \str_collect_end:wn #1 ;
+ {
+ \exp_after:wN \str_collect_end_ii:nnnnnnnnw
+ \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f:
+ \or: \or: \or: \or: \or: \or: \fi:
+ }
+\cs_new:Npn \str_collect_end_ii:nnnnnnnnw #1#2#3#4#5#6#7#8 #9 \q_stop
+ { #1#2#3#4#5#6#7#8 }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -1072,48 +1100,47 @@
% \begin{macro}[EXP]{\str_item:nn}
% \begin{macro}[EXP]{\str_item_ignore_spaces:nn}
% \begin{macro}[EXP]{\str_item_unsafe:nn}
-% \begin{macro}[EXP,aux]{\str_item_aux:nn}
-% This is mostly shuffling arguments around to avoid measuring
-% the length of the string more than once, and make sure that
-% the parameters given to \cs{str_skip_do:nn} are necessarily
-% within the bounds of the length of the string.
-% The \cs{str_item_ignore_spaces:nn} function cheats a little bit
-% in that it doesn't hand to \cs{str_item_unsafe:nn} an
-% \enquote{other string}. This is alright, as everything else
-% is done with undelimited arguments.
+% \begin{macro}[EXP,aux]{\str_item_aux:ww}
+% This is mostly shuffling arguments around to avoid measuring the
+% length of the string more than once, and make sure that the
+% parameters given to \cs{str_skip_c_zero:w} are necessarily within
+% the bounds of the length of the string. The \texttt{_ignore_spaces}
+% function cheats a little bit in that it doesn't hand to
+% \cs{str_item_unsafe:nn} an \enquote{other string}. This is alright,
+% as everything else is done with undelimited arguments.
% \begin{macrocode}
\cs_new_nopar:Npn \str_item:Nn { \exp_args:No \str_item:nn }
-\cs_new:Npn \str_item:nn #1#2
+\cs_new:Npn \str_item:nn #1
{
- \exp_args:Nf \tl_to_str:n
- { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} }
+ \exp_last_unbraced:Nf \token_to_str:N
+ \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} }
}
\cs_new:Npn \str_item_ignore_spaces:nn #1
{ \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } }
-\cs_new_nopar:Npn \str_item_unsafe:nn #1#2
+\cs_new:Npn \str_item_unsafe:nn #1#2
{
- \exp_args:Nff \str_item_aux:nn
- { \int_eval:n {#2} }
- { \str_length_unsafe:n {#1} }
+ \exp_after:wN \str_item_aux:ww
+ \int_use:N \int_eval:w #2 \exp_after:wN ;
+ \int_value:w \str_length_unsafe:n {#1} ;
#1
\q_stop
}
-\cs_new_nopar:Npn \str_item_aux:nn #1#2
+\cs_new:Npn \str_item_aux:ww #1; #2;
{
\int_compare:nNnTF {#1} < \c_zero
{
\int_compare:nNnTF {#1} < {-#2}
{ \use_none_delimit_by_q_stop:w }
{
- \str_skip_do:nn { #1 + #2 }
- { \use_i_delimit_by_q_stop:nw }
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tex_romannumeral:D \str_skip_c_zero:w #1 + #2 ;
}
}
{
\int_compare:nNnTF {#1} < {#2}
{
- \str_skip_do:nn {#1}
- { \use_i_delimit_by_q_stop:nw }
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tex_romannumeral:D \str_skip_c_zero:w #1 ;
}
{ \use_none_delimit_by_q_stop:w }
}
@@ -1128,24 +1155,27 @@
% \begin{macro}[EXP]{\str_substr:Nnn}
% \begin{macro}[EXP]{\str_substr:nnn}
% \begin{macro}[EXP]{\str_substr_ignore_spaces:nnn}
-% \begin{macro}[EXP]{\str_substr_unsafe:nnn}
-% \begin{macro}[EXP,aux]{\str_substr_aux:www}
-% \begin{macro}[EXP,aux]{\str_substr_aux:nnw}
-% \begin{macro}[EXP,aux]{\str_aux_normalize_range:nn}
-% Sanitize the string. Then evaluate the arguments, replacing
-% them by \cs{c_zero} or \cs{c_max_int} if they are empty.
-% This is done by using the construction
+% \begin{macro}[EXP,aux]
+% {
+% \str_substr_unsafe:nnn,
+% \str_substr_aux:nN,
+% \str_substr_aux:www,
+% \str_substr_aux:nnw,
+% \str_substr_normalize_range:nn
+% }
+% Sanitize the string. Then evaluate the arguments, replacing them by
+% \cs{c_zero} or \cs{c_max_int} if they are empty. This is done by
+% using the construction
% \begin{verbatim}
% \int_use:N \int_eval:w #2 \c_zero \c_zero ;
% \end{verbatim}
% which expands to the value of |#2|, followed by |\c_zero \c_zero ;|
% if |#2| is an expression, and expands to |0\c_zero ;| otherwise.
-% The same is done to the end-point of the range.
-% Then limit the range to be at most the length of the string
-% (this avoids needing to check for the end of the string when
-% grabbing characters).
-% Afterwards, skip characters, then keep some more, and finally
-% drop the end of the string.
+% The same is done to the end-point of the range. Then limit the
+% range to be at most the length of the string (this avoids needing to
+% check for the end of the string when grabbing characters).
+% Afterwards, skip characters, then keep some more, and finally drop
+% the end of the string.
% \begin{macrocode}
\cs_new_nopar:Npn \str_substr:Nnn { \exp_args:No \str_substr:nnn }
\cs_new:Npn \str_substr:nnn #1#2#3
@@ -1163,28 +1193,26 @@
\exp_after:wN \str_substr_aux:www
\int_value:w \str_length_unsafe:n {#1} \exp_after:wN ;
\int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ;
- \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ;
+ \int_use:N \int_eval:w
+ \exp_args:Nf \str_substr_aux:nN {#3} \c_max_int ;
#1
\q_stop
}
-\cs_new:Npn \str_substr_unsafe:nN #1 #2
+\cs_new:Npn \str_substr_aux:nN #1 #2
{ \tl_if_empty:nTF {#1} {#2} {#1} }
\cs_new:Npn \str_substr_aux:www #1; #2; #3;
{
\exp_args:Nf \str_substr_aux:nnw
- { \str_aux_normalize_range:nn {#2} {#1} }
- { \str_aux_normalize_range:nn {#3} {#1} }
+ { \str_substr_normalize_range:nn {#2} {#1} }
+ { \str_substr_normalize_range:nn {#3} {#1} }
}
\cs_new:Npn \str_substr_aux:nnw #1#2
{
- \str_skip_do:nn {#1}
- {
- \exp_args:Nf \str_collect_do:nn
- { \int_eval:n { #2 - #1 } }
- { \use_i_delimit_by_q_stop:nw }
- }
+ \exp_after:wN \str_collect_delimit_by_q_stop:w
+ \int_use:N \int_eval:w #2 - #1 \exp_after:wN ;
+ \tex_romannumeral:D \str_skip_c_zero:w #1 ;
}
-\cs_new:Npn \str_aux_normalize_range:nn #1#2
+\cs_new:Npn \str_substr_normalize_range:nn #1#2
{
\int_eval:n
{
@@ -1208,18 +1236,14 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \subsection{String conditionals}
%
% \begin{macro}[EXP,pTF]{\str_if_eq:NN}
% \begin{macro}[EXP,pTF]{\str_if_eq:nn,\str_if_eq:xx}
-% The \texttt{nn} and \texttt{xx} variants are already
-% defined in \pkg{l3basics}. Note that \cs{str_if_eq:NN}
-% is different from \cs{tl_if_eq:NN} because it needs to
-% ignore category codes.
+% The \texttt{nn} and \texttt{xx} variants are already defined in
+% \pkg{l3basics}. Note that \cs{str_if_eq:NN} is different from
+% \cs{tl_if_eq:NN} because it needs to ignore category codes.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_if_eq:NN #1#2 { p , TF , T , F }
{
@@ -1233,17 +1257,15 @@
% \begin{macro}[EXP]{\str_if_contains_char:NNT, \str_if_contains_char:NNTF}
% \begin{macro}[EXP]{\str_if_contains_char:nNTF}
% \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN}
-% \begin{macro}[EXP,aux]{\str_if_contains_char_end:}
+% \begin{macro}[EXP,aux]{\str_if_contains_char_true:}
% Loop over the characters of the string, comparing character codes.
-% We allow |#2| to be a single-character control sequence, hence the
-% use of \cs{int_compare:nNnT} rather than \cs{token_if_eq_charcode:NNT}.
% The loop is broken if character codes match. Otherwise we return
% \enquote{false}.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF }
{
\exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
- #1 { \prg_map_break:n { ? \fi: } }
+ \tl_to_str:n \exp_after:wN {#1} { \prg_map_break:n { ? \fi: } }
\prg_break_point:n { }
\prg_return_false:
}
@@ -1254,14 +1276,14 @@
\prg_break_point:n { }
\prg_return_false:
}
-\cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2
+\cs_new:Npn \str_if_contains_char_aux:NN #1#2
{
\if_charcode:w #1 #2
- \exp_after:wN \str_if_contains_char_end:
+ \exp_after:wN \str_if_contains_char_true:
\fi:
\str_if_contains_char_aux:NN #1
}
-\cs_new_nopar:Npn \str_if_contains_char_end:
+\cs_new_nopar:Npn \str_if_contains_char_true:
{ \prg_map_break:n { \prg_return_true: \use_none:n } }
% \end{macrocode}
% \end{macro}
@@ -1270,11 +1292,10 @@
% \end{macro}
%
% \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF}
-% \TeX{} dutifully detects octal digits for us: if |#1|
-% is an octal digit, then the right-hand side of the
-% comparison is |'1#1|, greater than $1$. Otherwise,
-% the right-hand side stops as |'1|, and the conditional
-% takes the \texttt{false} branch.
+% \TeX{} dutifully detects octal digits for us: if |#1| is an octal
+% digit, then the right-hand side of the comparison is |'1#1|, greater
+% than $1$. Otherwise, the right-hand side stops as |'1|, and the
+% conditional takes the \texttt{false} branch.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF }
{
@@ -1288,10 +1309,9 @@
% \end{macro}
%
% \begin{macro}[aux, rEXP]{\str_aux_hexadecimal_use:NTF}
-% \TeX{} detects uppercase hexadecimal digits for us
-% (see \cs{str_aux_octal_use:NTF}, but not the
-% lowercase letters, which we need to detect and replace
-% by their uppercase counterpart.
+% \TeX{} detects uppercase hexadecimal digits for us (see
+% \cs{str_aux_octal_use:NTF}), but not the lowercase letters, which we
+% need to detect and replace by their uppercase counterpart.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF }
{
@@ -1321,25 +1341,24 @@
%
% \subsubsection{Producing one byte or character}
%
-% \begin{variable}{\c_str_byte_0_tl}
+% \begin{variable}{\c_str_byte_0_tl, \c_str_byte_1_tl, \c_str_byte_255_tl}
% \begin{variable}{\c_str_byte_-1_tl}
% \begin{variable}{\c_str_positive_bytes_tl}
-% For each integer $N$ in the range $[0,255]$, we create
-% a constant token list which holds three character tokens
-% with category code other: the character with character
-% code $N$, followed by the representation of $N$ as
-% two hexadecimal digits.
-% The value $-1$ is given a default token list which ensures
-% that later functions give an empty result for the input $-1$.
-% Simultaneously, we build a list of all bytes (from which we
-% remove the null byte) in \cs{c_str_positive_bytes_tl}.
+% For each integer $N$ in the range $[0,255]$, we create a constant
+% token list which holds three character tokens with category code
+% other: the character with character code $N$, followed by the
+% representation of $N$ as two hexadecimal digits. The value $-1$ is
+% given a default token list which ensures that later functions give
+% an empty result for the input $-1$. Simultaneously, we build a list
+% of all bytes (from which we remove the null byte) in
+% \cs{c_str_positive_bytes_tl}.
% \begin{macrocode}
\group_begin:
- \tl_clear:N \l_str_tmpa_tl
+ \tl_clear:N \l_str_internal_tl
\char_set_catcode_other:n { \c_zero }
\tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
- \tl_map_inline:Nn \g_str_result_tl
- { \char_set_lccode:nn {`#1} {`#1} }
+ \exp_args:No \tl_map_inline:nn { \g_str_result_tl " }
+ { \char_set_lccode:nn {`#1} { \c_zero } }
\tl_map_inline:Nn \g_str_result_tl
{
\tl_map_inline:Nn \g_str_result_tl
@@ -1350,12 +1369,12 @@
\tl_const:cx
{ c_str_byte_ \int_eval:n {"#1##1} _tl }
{ ^^@ #1 ##1 }
- \tl_put_right:Nn \l_str_tmpa_tl { ^^@ }
+ \tl_put_right:Nn \l_str_internal_tl { ^^@ }
}
}
}
\tl_const:Nx \c_str_positive_bytes_tl
- { \exp_after:wN \use_none:n \l_str_tmpa_tl }
+ { \exp_after:wN \use_none:n \l_str_internal_tl }
\group_end:
\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
% \end{macrocode}
@@ -1364,15 +1383,17 @@
% \end{variable}
%
% \begin{macro}[int, EXP]{\str_output_byte:n}
-% \begin{macro}[aux, EXP]{\str_output_byte:w}
+% \begin{macro}[int, EXP]{\str_output_byte:w}
% \begin{macro}[int, EXP]{\str_output_hexadecimal:n}
-% \begin{macro}[aux, EXP]{\str_output_hexadecimal:w}
-% \begin{macro}[aux, EXP]{\str_output_end:}
-% For now, no error detection, we simply pick either
-% the byte or the hexadecimal representation of it
-% from the three-character token list corresponding
-% to the argument.
-% The value $-1$ produces an empty result.
+% \begin{macro}[int, EXP]{\str_output_hexadecimal:w}
+% \begin{macro}[int, EXP]{\str_output_end:}
+% Those functions must be used carefully: feeding them a value outside
+% the range $[-1,255]$ will attempt to use the undefined token list
+% variable \cs{c_str_byte_\meta{number}_tl}. Assuming that the
+% argument is in the right range, we expand the corresponding token
+% list, and pick either the byte (first token) or the hexadecimal
+% representations (second and third tokens). The value $-1$ produces
+% an empty result in both cases.
% \begin{macrocode}
\cs_new:Npn \str_output_byte:n #1
{ \str_output_byte:w #1 \str_output_end: }
@@ -1399,445 +1420,346 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Mapping functions for encoding conversion}
+% \begin{macro}[int, rEXP]{\str_output_byte_pair_be:n}
+% \begin{macro}[int, rEXP]{\str_output_byte_pair_le:n}
+% \begin{macro}[aux, rEXP]{\str_output_byte_pair_aux:nnN}
+% Convert a number in the range $[0,65535]$ to a pair of bytes, either
+% big-endian or little-endian.
+% \begin{macrocode}
+\cs_new:Npn \str_output_byte_pair_be:n #1
+ {
+ \exp_args:Nf \str_output_byte_pair_aux:nnN
+ { \int_div_truncate:nn { #1 } { "100 } } {#1} \use:nn
+ }
+\cs_new:Npn \str_output_byte_pair_le:n #1
+ {
+ \exp_args:Nf \str_output_byte_pair_aux:nnN
+ { \int_div_truncate:nn { #1 } { "100 } } {#1} \use_ii_i:nn
+ }
+\cs_new:Npn \str_output_byte_pair_aux:nnN #1#2#3
+ {
+ #3
+ { \str_output_byte:n { #1 } }
+ { \str_output_byte:n { #2 - #1 * "100 } }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Mapping functions for conversions}
%
-% \begin{macro}{\str_aux_gmap_result:N}
-% \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN}
-% This maps the function |#1| over all characters
-% in \cs{g_str_result_tl}, which should be a byte
-% string in most cases, sometimes a native string.
+% \begin{macro}{\str_convert_gmap:N}
+% \begin{macro}[aux, rEXP]{\str_convert_gmap_loop:NN}
+% This maps the function |#1| over all characters in
+% \cs{g_str_result_tl}, which should be a byte string in most cases,
+% sometimes a native string.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1
+\cs_new_protected:Npn \str_convert_gmap:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_result_loop:NN
+ \exp_after:wN \str_convert_gmap_loop:NN
\exp_after:wN #1
\g_str_result_tl { ? \prg_map_break: }
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2
+\cs_new:Npn \str_convert_gmap_loop:NN #1#2
{
\use_none:n #2
#1#2
- \str_aux_gmap_result_loop:NN #1
+ \str_convert_gmap_loop:NN #1
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\str_aux_gmap_internal_result:N}
-% \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw}
-% This maps the function |#1| over all character codes
-% in \cs{g_str_result_tl}, which must be in the internal
-% representation.
+% \begin{macro}{\str_convert_gmap_internal:N}
+% \begin{macro}[aux, rEXP]{\str_convert_gmap_internal_loop:Nw}
+% This maps the function |#1| over all character codes in
+% \cs{g_str_result_tl}, which must be in the internal representation.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1
+\cs_new_protected:Npn \str_convert_gmap_internal:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_internal_result_loop:Nww
+ \exp_after:wN \str_convert_gmap_internal_loop:Nww
\exp_after:wN #1
\g_str_result_tl \s_tl \q_stop \prg_map_break: \s_tl
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1 #2 \s_tl #3 \s_tl
+\cs_new:Npn \str_convert_gmap_internal_loop:Nww #1 #2 \s_tl #3 \s_tl
{
\use_none_delimit_by_q_stop:w #3 \q_stop
#1 {#3}
- \str_aux_gmap_internal_result_loop:Nww #1
+ \str_convert_gmap_internal_loop:Nww #1
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \subsubsection{Unescape user input}
-%
-% The code of this section is used both here for \cs{str_(g)input:Nn},
-% and in the regular expression module to go through the regular expression
-% once before actually parsing it. The goal in that case is to turn any
-% character with a meaning in regular expressions (\texttt{*}, \texttt{?},
-% \texttt{\{}, etc.) into a marker indicating that this was a special
-% character,
-% and replace any escaped character by the corresponding unescaped character,
-% so that the \pkg{l3regex} code can avoid caring about escaping issues
-% (those can become quite complex to handle in combination with ranges
-% in character classes).
-%
-% The idea is to feed unescaped characters to one function,
-% escaped characters to another, and feed |\a|, |\e|, |\f|,
-% |\n|, |\r|, |\t| and |\x| converted to the appropriate
-% character to a third function. Spaces are ignored unless
-% escaped.
-% For the \cs{str_(g)input:Nn} application, all the functions are simply
-% \cs{token_to_str:N} (this ensures that spaces correctly get assigned
-% category code $10$).
-% For the \pkg{l3regex} applications, the functions do some further
-% tests on the character they receive.
-%
-% \begin{macro}{\str_input:Nn,\str_ginput:Nn}
-% Simple wrappers around the internal \cs{str_aux_escape:NNNn}.
-% \begin{macrocode}
-\cs_new_protected:Npn \str_input:Nn #1#2
- {
- \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
- \tl_set_eq:NN #1 \g_str_result_tl
- }
-\cs_new_protected:Npn \str_ginput:Nn #1#2
- {
- \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
- \tl_gset_eq:NN #1 \g_str_result_tl
- }
-% \end{macrocode}
-% \end{macro}
+% \subsubsection{Error-reporting during conversion}
%
-% \begin{macro}[int]{\str_escape_use:NNNn}
-% Use the internal \cs{str_aux_escape:NNNn}, then leave the result
-% in the input stream. This is used in \pkg{l3regex}.
+% \begin{macro}[int]{\str_if_flag_error:nnx}
+% \begin{macro}[aux]{\str_if_flag_no_error:nnx}
+% When converting using the function \cs{str_set_convert:Nnnn}, errors
+% should be reported to the user after each step in the
+% conversion. Errors are signalled by raising some flag (typically
+% \texttt{str_error}), so here we test that flag: if it is raised,
+% give the user an error, otherwise remove the arguments. On the other
+% hand, in the conditional functions \cs{str_set_convert:NnnnTF},
+% errors should be suppressed. This is done by changing
+% \cs{str_if_flag_error:nnx} into \cs{str_if_flag_no_error:nnx}
+% locally.
% \begin{macrocode}
-\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4
+\cs_new_protected:Npn \str_if_flag_error:nnx #1
{
- \str_aux_escape:NNNn #1#2#3 {#4}
- \g_str_result_tl
+ \flag_if_raised:nTF {#1}
+ { \msg_kernel_error:nnx { str } }
+ { \use_none:nn }
}
+\cs_new_protected:Npn \str_if_flag_no_error:nnx #1#2#3
+ { \flag_if_raised:nT {#1} { \bool_gset_true:N \g_str_error_bool } }
% \end{macrocode}
% \end{macro}
-%
-% \begin{macro}[int]{\str_aux_escape:NNNn}
-% \begin{macro}[aux]{\str_aux_escape_loop:N}
-% \begin{macro}[aux]+\str_aux_escape_\:w+
-% Most of the work is done within an \texttt{x}-expanding assignment,
-% but the |\x| escape sequence cannot be done in that way. Therefore,
-% we interrupt the assignment at each |\x| escape sequence, store the
-% partial result in a \tn{toks} register, and at the end unpack all
-% of the \tn{toks} registers to the left of the last chunk of
-% \cs{g_str_result_tl}.
-% \begin{macrocode}
-\cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4
- {
- \group_begin:
- \cs_set_nopar:Npn \str_aux_escape_unescaped:N { #1 }
- \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 }
- \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 }
- \int_set:Nn \tex_escapechar:D { 92 }
- \str_gset_other:Nn \g_str_result_tl {#4}
- \int_zero:N \l_str_tmpa_int
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl
- \q_recursion_tail \q_recursion_stop
- }
- \tl_gput_left:Nx \g_str_result_tl
- { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int }
- \group_end:
- }
-\cs_new_nopar:Npn \str_aux_escape_loop:N #1
- {
- \cs_if_exist_use:cF { str_aux_escape_\token_to_str:N #1:w }
- { \str_aux_escape_unescaped:N #1 }
- \str_aux_escape_loop:N
- }
-\cs_new_nopar:cpn { str_aux_escape_ \c_backslash_str :w }
- \str_aux_escape_loop:N #1
- {
- \cs_if_exist_use:cF { str_aux_escape_/\token_to_str:N #1:w }
- { \str_aux_escape_escaped:N #1 }
- \str_aux_escape_loop:N
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\str_aux_escape_unescaped:N}
-% \begin{macro}[aux]{\str_aux_escape_escaped:N}
-% \begin{macro}[aux]{\str_aux_escape_raw:N}
-% Those functions are never called before being given a new meaning,
-% so their definitions here don't matter.
+% \begin{macro}[int]{\str_if_flag_times:nT}
+% At the end of each conversion step, we raise all relevant errors as
+% one error message, built on the fly. The height of each flag
+% indicates how many times a given error was encountered. This
+% function prints |#2| followed by the number of occurrences of an
+% error if it occurred, nothing otherwise.
% \begin{macrocode}
-\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing:
-\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing:
-\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing:
+\cs_new_protected:Npn \str_if_flag_times:nT #1#2
+ { \flag_if_raised:nT {#1} { #2~(x \flag_height:n {#1} ) } }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]+\str_aux_escape_\q_recursion_tail:w+
-% \begin{macro}[aux]+\str_aux_escape_/\q_recursion_tail:w+
-% \begin{macro}[aux]+\str_aux_escape_ :w+
-% \begin{macro}[aux]
-% {
-% \str_aux_escape_/a:w, \str_aux_escape_/e:w, \str_aux_escape_/f:w,
-% \str_aux_escape_/n:w, \str_aux_escape_/r:w, \str_aux_escape_/t:w
-% }
-% The loop is ended upon seeing \cs{q_recursion_tail}.
-% Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, |\t| take
-% their meaning here.
-% \begin{macrocode}
-\cs_new_eq:cN
- { str_aux_escape_ \c_backslash_str q_recursion_tail :w }
- \use_none_delimit_by_q_recursion_stop:w
-\cs_new_eq:cN
- { str_aux_escape_ / \c_backslash_str q_recursion_tail :w }
- \use_none_delimit_by_q_recursion_stop:w
-\cs_new_nopar:cpn { str_aux_escape_~:w } { }
-\cs_new_nopar:cpx { str_aux_escape_/a:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^G }
-\cs_new_nopar:cpx { str_aux_escape_/t:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^I }
-\cs_new_nopar:cpx { str_aux_escape_/n:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^J }
-\cs_new_nopar:cpx { str_aux_escape_/f:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^L }
-\cs_new_nopar:cpx { str_aux_escape_/r:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^M }
-\cs_new_nopar:cpx { str_aux_escape_/e:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^[ }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
+% \subsubsection{Framework for conversions}
+%
+% Most functions in this module expect to be working with
+% \enquote{native} strings. Strings can also be stored as bytes, in one
+% of many encodings, for instance \textsc{utf8}. The bytes themselves
+% can be expressed in various ways in terms of \TeX{} tokens, for
+% instance as pairs of hexadecimal digits. The questions of going from
+% arbitrary Unicode code points to bytes, and from bytes to tokens are
+% mostly independent.
%
-% \begin{macro}[aux]{\str_aux_escape_/x:w}
-% \begin{macro}[aux]{\str_aux_escape_x_test:N}
-% \begin{macro}[aux]{\str_aux_escape_x_unbraced_i:N}
-% \begin{macro}[aux]{\str_aux_escape_x_unbraced_ii:N}
-% \begin{macro}[aux]{\str_aux_escape_x_braced_loop:N}
-% \begin{macro}[aux]{\str_aux_escape_x_braced_end:N}
-% \begin{macro}[aux]{\str_aux_escape_x_end:}
-% When |\x| is encountered, interrupt the assignment,
-% and distinguish the cases of a braced or unbraced syntax.
-% In the braced case, collect arbitrarily many hexadecimal digits,
-% building the number in \cs{l_str_char_int} (using
-% \cs{str_aux_hexadecimal_use:NTF}), and check that
-% the run of digits was stopped by a closing brace.
-% In the unbraced case, collect up to two hexadecimal digits,
-% possibly less, building the character number in \cs{l_str_char_int}.
-% In both cases, once all digits have been collected, use
-% the \TeX{} primitive \tn{lowercase} to produce that character,
-% and use an \cs{if_false:} trick to restart the assignment.
+% Conversions are done in four steps:
+% \begin{itemize}
+% \item \enquote{unescape} produces a string of bytes;
+% \item \enquote{decode} takes in a string of bytes, and converts it
+% to a list of Unicode characters in an internal representation,
+% with items of the form
+% \begin{quote}
+% \meta{bytes} \cs{s_tl} \meta{Unicode code point} \cs{s_tl}
+% \end{quote}
+% where we have collected the \meta{bytes} which combined to form
+% this particular Unicode character, and the \meta{Unicode code
+% point} is in the range $[0,\hexnum{10FFFF}]$.
+% \item \enquote{encode} encodes the internal list of code points as a
+% byte string in the new encoding;
+% \item \enquote{escape} escapes bytes as requested.
+% \end{itemize}
+% The process is modified in case one of the encoding is empty (or the
+% conversion function has been set equal to the empty encoding because
+% it was not found): then the unescape or escape step is ignored, and
+% the decode or encode steps work on tokens instead of bytes. Otherwise,
+% each step must ensure that it passes a correct byte string or internal
+% string to the next step.
+%
+% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
+% \begin{macro}[TF]{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
+% \begin{macro}[aux]{\str_convert_aux_i:nNNnnn}
+% The input string is stored in \cs{g_str_result_tl}, then we:
+% unescape and decode; encode and escape; exit the group and store the
+% result in the user's variable. The various conversion functions all
+% act on \cs{g_str_result_tl}. Errors are silenced for the conditional
+% functions by redefining \cs{str_if_flag_error:nnx} locally.
% \begin{macrocode}
-\cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N
- {
- \if_false: { \fi: }
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl }
- \int_incr:N \l_str_tmpa_int
- \int_zero:N \l_str_char_int
- \str_aux_escape_x_test:N
- }
-\cs_new_protected_nopar:Npx \str_aux_escape_x_test:N #1
- {
- \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
- { \exp_not:N \str_aux_escape_x_test:N }
- {
- \l_str_char_int = "0
- \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
- { \exp_not:N \str_aux_escape_x_braced_loop:N }
- { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
- }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
+\cs_new_protected_nopar:Npn \str_set_convert:Nnnn
+ { \str_convert_aux_i:nNNnnn { } \tl_set_eq:NN }
+\cs_new_protected_nopar:Npn \str_gset_convert:Nnnn
+ { \str_convert_aux_i:nNNnnn { } \tl_gset_eq:NN }
+\prg_new_protected_conditional:Npnn
+ \str_set_convert:Nnnn #1#2#3#4 { T , F , TF }
{
- \str_aux_hexadecimal_use:NTF #1
- { \str_aux_escape_x_unbraced_ii:N }
- { \exp_stop_f: \str_aux_escape_x_end: #1 }
+ \bool_gset_false:N \g_str_error_bool
+ \str_convert_aux_i:nNNnnn
+ { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx }
+ \tl_set_eq:NN #1 {#2} {#3} {#4}
+ \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true:
}
-\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
+\prg_new_protected_conditional:Npnn
+ \str_gset_convert:Nnnn #1#2#3#4 { T , F , TF }
{
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \str_aux_escape_x_unbraced_ii:N }
- {
- \str_aux_hexadecimal_use:NTF #1
- { \exp_stop_f: \str_aux_escape_x_end: }
- { \exp_stop_f: \str_aux_escape_x_end: #1 }
- }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1
- {
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \str_aux_escape_x_braced_loop:N }
- {
- \str_aux_hexadecimal_use:NTF #1
- { \str_aux_escape_x_braced_loop:N }
- { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 }
- }
+ \bool_gset_false:N \g_str_error_bool
+ \str_convert_aux_i:nNNnnn
+ { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx }
+ \tl_gset_eq:NN #1 {#2} {#3} {#4}
+ \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true:
}
-\cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1
+\cs_new_protected:Npn \str_convert_aux_i:nNNnnn #1#2#3#4#5#6
{
- \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
- { \exp_not:N \str_aux_escape_x_end: }
- {
- \msg_kernel_error:nn { str } { x-missing-brace }
- \exp_not:N \str_aux_escape_x_end: #1
- }
+ \group_begin:
+ #1
+ \str_gset_other:Nn \g_str_result_tl {#4}
+ \exp_after:wN \str_convert_aux_ii:wwwnn
+ \tl_to_str:n {#5} /// \q_stop
+ { decode } { unescape }
+ \prg_do_nothing:
+ \str_convert_decode_:
+ \exp_after:wN \str_convert_aux_ii:wwwnn
+ \tl_to_str:n {#6} /// \q_stop
+ { encode } { escape }
+ \use_ii_i:nn
+ \str_convert_encode_:
+ \group_end:
+ #2 #3 \g_str_result_tl
}
-\group_begin:
- \char_set_catcode_other:N \^^@
- \cs_new_protected_nopar:Npn \str_aux_escape_x_end:
- {
- \tex_lccode:D \c_zero \l_str_char_int
- \tl_to_lowercase:n
- {
- \tl_gset:Nx \g_str_result_tl
- { \if_false: } \fi:
- \str_aux_escape_raw:N ^^@
- \str_aux_escape_loop:N
- }
- }
-\group_end:
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Framework for encoding conversions}
%
-% Most functions in this module expect to be working with
-% \enquote{native} strings. Strings can also be stored as
-% bytes, in one of many encodings, for instance \textsc{utf8}.
-% The bytes themselves can be expressed as character
-% tokens with the relevant character code, or as pairs
-% of hexadecimal digits, \emph{etc.} The questions of going
-% from arbitrary characters to bytes, and from bytes to \TeX{}
-% tokens are mostly independent.
-%
-% Conversions are done in four steps:
-% \begin{itemize}
-% \item \enquote{input} produces a string of bytes;
-% \item \enquote{from} interprets the byte string in the old encoding,
-% and converts it to a comma-list, each character becoming
-% \enquote{\ttfamily \meta{character code},};
-% \item \enquote{to} encodes the internal comma list as a byte string
-% in the new encoding;
-% \item \enquote{output} escapes bytes as requested.
-% \end{itemize}
-% The process is modified in case one of the encoding is
-% \texttt{internal} or \texttt{native}: then the input or
-% output step is ignored.
-%
-% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
-% \begin{macro}[aux]{\str_convert_aux_i:NNnnn}
% \begin{macro}[aux]{\str_convert_aux_ii:wwwnn}
-% \begin{macro}[aux]{\str_convert_aux_iii:nnF}
-% \begin{macro}[aux]{\str_convert_aux_iv:nnnF}
-% \begin{macro}[aux]{\str_convert_aux_v:NNnNN}
-% The input string is stored in \cs{g_str_result_tl},
-% then we call the various conversion functions, which all
-% act on \cs{g_str_result_tl}. The arguments |#3| and |#4|
-% of the \texttt{aux_i} function are split into their
-% \meta{encoding} and \meta{escaping} parts by \texttt{aux_ii}.
-% The conversion function names are built by \texttt{aux_iii},
-% which first tries to use the name as supplied by the user,
-% then filters it with \cs{str_aux_lowercase_alphanum:n},
-% and calls \texttt{aux_iv} to load the relevant definition file.
-% In all cases, \texttt{aux_iii} ensures that the two
-% required conversion functions are defined, with
-% a fall-back to the native encoding with no escaping.
-% Once this is done, \texttt{aux_v} does the main work:
+% \begin{macro}[aux]{\str_convert_aux_iii:NNnNN}
+% The task of \cs{str_convert_aux_ii:wwwnn} is to split
+% \meta{encoding}/\meta{escaping} pairs into their components, |#1|
+% and |#2|. Calls to \cs{str_convert_aux_iv:nnn} ensure that the
+% corresponding conversion functions are defined. The third auxiliary
+% does the main work.
% \begin{itemize}
-% \item |#1| is the encoding conversion function;
-% \item |#2| is the escaping function;
-% \item |#3| is the escaping name;
-% \item |#4| is the \enquote{native} encoding function
-% (either \enquote{from} or \enquote{to});
-% \item |#5| is \cs{use_i:nn} or \cs{use_ii:nn}.
+% \item |#1| is the encoding conversion function;
+% \item |#2| is the escaping function;
+% \item |#3| is the escaping name for use in an error message;
+% \item |#4| is \cs{prg_do_nothing:} for unescaping/decoding, and
+% \cs{use_ii_i:nn} for encoding/escaping;
+% \item |#5| is the default encoding function (either
+% \enquote{decode} or \enquote{encode}), for which there should be
+% no escaping.
% \end{itemize}
-% When converting to the internal format, the escaping must be done
-% first, then the encoding (this is controlled by |#5|).
-% If the encoding is native, then the escaping is ignored and
-% must be empty. Otherwise, the escaping |#2| is performed.
+% Let us ignore the native encoding for a second. In the
+% unescaping/decoding phase, we want to do |#2#1| in this order, and
+% in the encoding/escaping phase, the order should be reversed:
+% |#4#2#1| does exactly that. If one of the encodings is the default
+% (native), then the escaping should be ignored, with an error if any
+% was given, and only the encoding, |#1|, should be performed.
% \begin{macrocode}
-\cs_new_protected:Npn \str_set_convert:Nnnn
- { \str_convert_aux_i:NNnnn \tl_set_eq:NN }
-\cs_new_protected:Npn \str_gset_convert:Nnnn
- { \str_convert_aux_i:NNnnn \tl_gset_eq:NN }
-\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5
+\cs_new_protected:Npn \str_convert_aux_ii:wwwnn
+ #1 / #2 // #3 \q_stop #4#5
{
- \group_begin:
- \str_gset_other:Nn \g_str_result_tl {#5}
-
- \exp_after:wN \str_convert_aux_ii:wwwnn
- \tl_to_str:n {#3} /// \q_stop
- { from } { input } \str_convert_from_native: \use_i:nn
-
- \exp_after:wN \str_convert_aux_ii:wwwnn
- \tl_to_str:n {#4} /// \q_stop
- { to } { output } \str_convert_to_native: \use_ii:nn
-
- \group_end:
- #1 #2 \g_str_result_tl
- }
-\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5
- {
- \str_convert_aux_iii:nnF {#4} {#1} {native}
- \str_convert_aux_iii:nnF {#5} {#2} { }
- \exp_args:Ncc \str_convert_aux_v:NNnNN
+ \str_convert_aux_iv:nnn {enc} {#4} {#1}
+ \str_convert_aux_iv:nnn {esc} {#5} {#2}
+ \exp_args:Ncc \str_convert_aux_iii:NNnNN
{ str_convert_#4_#1: } { str_convert_#5_#2: } {#2}
}
-\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3
+\cs_new_protected:Npn \str_convert_aux_iii:NNnNN #1#2#3#4#5
{
- \cs_if_exist:cF { str_convert_#1_#2: }
- {
- \exp_args:Nx \str_convert_aux_iv:nnnF
- { \str_aux_lowercase_alphanum:n {#2} }
- {#1} {#2} {#3}
- }
+ \if_meaning:w #1 #5
+ \tl_if_empty:nF {#3}
+ { \msg_kernel_error:nnx { str } { native-escaping } {#3} }
+ #1
+ \else:
+ #4 #2 #1
+ \fi:
}
-\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\str_convert_aux_iv:nnn}
+% \begin{macro}[aux]{\str_convert_aux_v:nnnn}
+% The arguments of \cs{str_convert_aux_iv:nnn} are: \texttt{enc} or
+% \texttt{esc}, used to build filenames, the type of the conversion
+% (unescape, decode, encode, escape), and the encoding or escaping
+% name. If the function is already defined, no need to do anything.
+% Otherwise, filter out all non-alphanumerics in the name, and
+% lowercase it. Feed that, and the same three arguments, to
+% \cs{str_convert_aux_v:nnnn}. The task is then to make sure that the
+% conversion function |#3_#1| corresponding to the type |#3| and
+% filtered name |#1| is defined, then set our initial conversion
+% function |#3_#4| equal to that.
+%
+% How do we get the |#3_#1| conversion to be defined if it isn't?
+% Two main cases.
+%
+% First, if |#1| is a key in \cs{g_str_file_prop}, then the value
+% \cs{l_str_internal_tl} tells us what file to load. Loading is
+% skipped if the file was already read, \emph{i.e.}, if the conversion
+% command based on \cs{l_str_internal_tl} already exists. Otherwise,
+% try to load the file; if that fails, there is an error, use the
+% default empty name instead.
+%
+% Second, |#1| may be absent from the property list. The
+% \cs{cs_if_exist:cF} test is automatically false, and we search for a
+% file defining the encoding or escaping |#1| (this should allow
+% third-party \texttt{.def} files). If the file is not found, there is
+% an error, use the default empty name instead.
+%
+% In all cases, the conversion based on \cs{l_str_internal_tl} is
+% defined, so we can set the |#3_#1| function equal to that. In some
+% cases (\emph{e.g.}, \texttt{utf16be}), the |#3_#1| function is
+% actually defined within the file we just loaded, and it is different
+% from the \cs{l_str_internal_tl}-based function: we mustn't clobber
+% that different definition.
+% \begin{macrocode}
+\cs_new_protected:Npn \str_convert_aux_iv:nnn #1#2#3
{
- \cs_if_exist:cF { str_convert_#2_#1: }
- {
- \group_begin:
- \str_load_catcodes:
- \str_load_one:n {#1}
- \group_end:
- }
- \cs_if_exist:cTF { str_convert_#2_#1: }
- { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } }
+ \cs_if_exist:cF { str_convert_#2_#3: }
{
- \msg_kernel_error:nnx { str } { unknown-#2 } {#3}
- \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: }
+ \exp_args:Nx \str_convert_aux_v:nnnn
+ { \str_convert_lowercase_alphanum:n {#3} }
+ {#1} {#2} {#3}
}
}
-\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5
+\cs_new_protected:Npn \str_convert_aux_v:nnnn #1#2#3#4
{
- #5 { } {#1}
- \cs_if_eq:NNTF #1 #4
+ \cs_if_exist:cF { str_convert_#3_#1: }
{
- \tl_if_empty:nF {#3}
- { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} }
+ \prop_get:NnNF \g_str_file_prop {#1} \l_str_internal_tl
+ { \tl_set:Nn \l_str_internal_tl {#1} }
+ \cs_if_exist:cF { str_convert_#3_ \l_str_internal_tl : }
+ {
+ \file_if_exist:nTF { l3str-#2- \l_str_internal_tl .def }
+ {
+ \group_begin:
+ \str_load_catcodes:
+ \file_input:n { l3str-#2- \l_str_internal_tl .def }
+ \group_end:
+ }
+ {
+ \tl_clear:N \l_str_internal_tl
+ \msg_kernel_error:nnxx { str } { unknown-#2 } {#4} {#1}
+ }
+ }
+ \cs_if_exist:cF { str_convert_#3_#1: }
+ {
+ \cs_gset_eq:cc { str_convert_#3_#1: }
+ { str_convert_#3_ \l_str_internal_tl : }
+ }
}
- { #2 }
- #5 {#1} { }
+ \cs_gset_eq:cc { str_convert_#3_#4: } { str_convert_#3_#1: }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int, rEXP]{\str_aux_lowercase_alphanum:n}
-% \begin{macro}[aux, rEXP]{\str_aux_lowercase_alphanum_loop:N}
-% This function keeps only letters and digits, with upper case
-% letters converted to lower case.
+% \begin{macro}[int, rEXP]{\str_convert_lowercase_alphanum:n}
+% \begin{macro}[aux, rEXP]{\str_convert_lowercase_alphanum_loop:N}
+% This function keeps only letters and digits, with upper case letters
+% converted to lower case.
% \begin{macrocode}
-\cs_new:Npn \str_aux_lowercase_alphanum:n #1
+\cs_new:Npn \str_convert_lowercase_alphanum:n #1
{
- \exp_after:wN \str_aux_lowercase_alphanum_loop:N
+ \exp_after:wN \str_convert_lowercase_alphanum_loop:N
\tl_to_str:n {#1} { ? \prg_map_break: }
\prg_break_point:n { }
}
-\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1
+\cs_new:Npn \str_convert_lowercase_alphanum_loop:N #1
{
\use_none:n #1
\if_num:w `#1 < \c_ninety_one
@@ -1856,149 +1778,331 @@
\fi:
\fi:
\fi:
- \str_aux_lowercase_alphanum_loop:N
+ \str_convert_lowercase_alphanum_loop:N
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_from_internal:}
-% \begin{macro}[int]{\str_convert_to_internal:}
-% Converting from the internal format to itself is
-% of course trivial.
+% \begin{macro}[int]{\str_load_catcodes:}
+% Since encoding files may be loaded at arbitrary places in a \TeX{}
+% document, including within verbatim mode, we set the catcodes of all
+% characters appearing in any encoding definition file.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_from_internal: { }
-\cs_new_protected_nopar:Npn \str_convert_to_internal: { }
+\cs_new_protected:Npn \str_load_catcodes:
+ {
+ \char_set_catcode_escape:N \\
+ \char_set_catcode_group_begin:N \{
+ \char_set_catcode_group_end:N \}
+ \char_set_catcode_math_toggle:N \$
+ \char_set_catcode_alignment:N \&
+ \char_set_catcode_parameter:N \#
+ \char_set_catcode_math_superscript:N \^
+ \char_set_catcode_ignore:N \ %
+ \char_set_catcode_space:N \~
+ \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFILNPSTUX }
+ \char_set_catcode_letter:N
+ \tl_map_function:nN { 0123456789"'?*+-.(),`!/<>[];= }
+ \char_set_catcode_other:N
+ \char_set_catcode_comment:N \%
+ \int_set:Nn \tex_endlinechar:D {32}
+ }
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \subsubsection{Loading encoding files}
+% \subsubsection{Byte unescape and escape}
%
-% \begin{macro}{\str_load:n}
-% \begin{macro}[int]{\str_load_one:n}
-% \begin{macro}[aux]{\str_load_alias:n, \str_load_alias_aux:nnn}
-% Loading encoding or escaping files can be done in bulk by
-% specifying a comma-list of encodings and escapings. Each
-% name is normalized: only alphanumeric characters are kept,
-% lowercased. The relevant file is loaded if it was not loaded
-% already.
+% Strings of bytes may need to be stored in auxiliary files in safe
+% \enquote{escaping} formats. Each such escaping is only loaded as
+% needed. By default, on input any non-byte is filtered out, while the
+% output simply consists in letting bytes through.
+%
+% \begin{macro}[int, rEXP]{\str_filter_bytes:n}
+% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N}
+% In the case of pdf\TeX{}, every character is a byte. For
+% Unicode-aware engines, test the character code; non-bytes cause us
+% to raise the flag \texttt{str_byte}. Spaces have already been given
+% the correct category code when this function is called.
% \begin{macrocode}
-\cs_new_protected:Npn \str_load:n #1
- {
- \group_begin:
- \str_load_catcodes:
- \clist_map_inline:nn {#1}
- {
- \exp_args:Nx \str_load_one:n
- { \str_aux_lowercase_alphanum:n {#1} }
- }
- \group_end:
- }
-\cs_new_protected:Npn \str_load_one:n #1
+\pdftex_if_engine:TF
+ { \cs_new_eq:NN \str_filter_bytes:n \use:n }
{
- \cs_if_exist:cF { str_convert_from_ #1 : }
+ \cs_new:Npn \str_filter_bytes:n #1
{
- \cs_if_exist:cF { str_convert_input_#1: }
- {
- \file_if_exist:nTF { l3str- #1 .def }
- { \file_input:n { l3str- #1 .def } }
- { \str_load_alias:n {#1} }
- }
+ \str_filter_bytes_aux:N #1
+ { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \cs_new:Npn \str_filter_bytes_aux:N #1
+ {
+ \use_none:n #1
+ \if_num:w `#1 < 256 \exp_stop_f:
+ #1
+ \else:
+ \flag_raise:n { str_byte }
+ \fi:
+ \str_filter_bytes_aux:N
}
}
-\cs_new_protected:Npn \str_load_alias:n #1
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_unescape_:}
+% \begin{macro}[int]{\str_convert_unescape_bytes:}
+% The simplest unescaping method removes non-bytes from
+% \cs{g_str_result_tl}.
+% \begin{macrocode}
+\pdftex_if_engine:TF
+ { \cs_new_protected_nopar:Npn \str_convert_unescape_: { } }
{
- \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl
+ \cs_new_protected_nopar:Npn \str_convert_unescape_:
{
- \str_load_one:n { \l_str_tmpa_tl }
- \cs_if_exist:cTF
- { str_convert_from_ \l_str_tmpa_tl : }
- {
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from }
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to }
- }
- {
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input }
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output }
- }
+ \flag_clear:n { str_byte }
+ \tl_gset:Nx \g_str_result_tl
+ { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
+ \str_if_flag_error:nnx { str_byte } { non-byte } { bytes }
}
- { \msg_kernel_error:nnx { str } { file-not-found } {#1} }
}
-\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3
- { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } }
+\cs_new_eq:NN \str_convert_unescape_bytes: \str_convert_unescape_:
% \end{macrocode}
% \end{macro}
% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_escape_:}
+% \begin{macro}[int]{\str_convert_escape_bytes:}
+% The simplest form of escape leaves the bytes from the previous step
+% of the conversion unchanged.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_convert_escape_: { }
+\cs_new_eq:NN \str_convert_escape_bytes: \str_convert_escape_:
+% \end{macrocode}
+% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_load_catcodes:}
-% Since encoding files may be loaded at arbitrary places
-% in a \TeX{} document, including within verbatim mode,
-% we set the catcodes of all characters appearing in any
-% encoding definition file.
+% \subsubsection{Native strings}
+%
+% \begin{macro}[int]{\str_convert_decode_:}
+% \begin{macro}[aux, rEXP]{\str_decode_native_char:N}
+% Convert each character to its character code, one at a time.
% \begin{macrocode}
-\cs_new_protected:Npn \str_load_catcodes:
+\cs_new_protected_nopar:Npn \str_convert_decode_:
+ { \str_convert_gmap:N \str_decode_native_char:N }
+\cs_new:Npn \str_decode_native_char:N #1
+ { #1 \s_tl \int_value:w `#1 \s_tl }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_encode_:}
+% The conversion from an internal string to native character tokens is
+% very different in pdf\TeX{} and in other engines. For Unicode-aware
+% engines, we need the definitions to be read when the null byte has
+% category code $12$, so we set that inside a group.
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_other:n { 0 }
+ \pdftex_if_engine:TF
+% \end{macrocode}
+% \begin{macro}[aux, EXP]{\str_encode_native_char:n}
+% Since pdf\TeX{} only supports 8-bit characters, and we have a table
+% of all bytes, the conversion can be done in linear time within an
+% \texttt{x}-expanding assignment. Look out for character codes larger
+% than $255$, those characters are replaced by |?|, and raise a flag,
+% which then triggers a pdf\TeX{}-specific error.
+% \begin{macrocode}
+ {
+ \cs_new_protected_nopar:Npn \str_convert_encode_:
+ {
+ \flag_clear:n { str_error }
+ \str_convert_gmap_internal:N \str_encode_native_char:n
+ \str_if_flag_error:nnx { str_error }
+ { pdfTeX-native-overflow } { }
+ }
+ \cs_new:Npn \str_encode_native_char:n #1
+ {
+ \if_num:w #1 < \c_two_hundred_fifty_six
+ \str_output_byte:n {#1}
+ \else:
+ \flag_raise:n { str_error }
+ ?
+ \fi:
+ }
+ \msg_kernel_new:nnnn { str } { pdfTeX-native-overflow }
+ { Character~code~too~large~for~pdfTeX. }
+ {
+ The~pdfTeX~engine~only~supports~8-bit~characters:~
+ valid~character~codes~are~in~the~range~[0,255].~
+ To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \begin{macro}[aux]{\str_encode_native_loop:w}
+% \begin{macro}[aux]{\str_encode_native_flush:}
+% \begin{macro}[aux, rEXP]{\str_encode_native_filter:N}
+% In Unicode-aware engines, since building particular characters
+% cannot be done expandably in \TeX{}, we cannot hope to get a
+% linear-time function. However, we get quite close by building the
+% result one block of $255$ characters at a time. Set the lowercase
+% code of the \textsc{ascii} character $1$ to the first character code
+% in the block, \emph{etc.}, then lowercase the contents of
+% \cs{c_str_positive_bytes_tl}. Unfortunately, this is complicated by
+% the fact that \TeX{} won't lowercase a character to the null
+% character, thus the need for filtering: whenever the lowercase code
+% of a given character is zero, it is replaced \enquote{by hand} by
+% the null character. Unicode-aware engines will never incur an
+% overflow because the internal string is guaranteed to only contain
+% code points in $[0,\hexnum{10FFFF}]$.
+% \begin{macrocode}
+ {
+ \cs_new_protected_nopar:Npn \str_convert_encode_:
+ {
+ \tex_lccode:D \c_zero \c_zero
+ \int_zero:N \l_str_internal_int
+ \tl_set_eq:NN \l_str_internal_tl \g_str_result_tl
+ \tl_gclear:N \g_str_result_tl
+ \exp_after:wN \str_encode_native_loop:w
+ \l_str_internal_tl \s_tl { \q_stop \prg_map_break: } \s_tl
+ \prg_break_point:n { \str_encode_native_flush: }
+ }
+ \cs_new_protected:Npn \str_encode_native_loop:w #1 \s_tl #2 \s_tl
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \if_num:w \l_str_internal_int = \c_two_hundred_fifty_five
+ \str_encode_native_flush:
+ \l_str_internal_int \c_zero
+ \fi:
+ \tex_advance:D \l_str_internal_int \c_one
+ \tex_lccode:D \l_str_internal_int \int_eval:w #2 \int_eval_end:
+ \str_encode_native_loop:w
+ }
+ \cs_new_protected_nopar:Npn \str_encode_native_flush:
+ {
+ \use:x
+ {
+ \tl_to_lowercase:n
+ {
+ \tl_gput_right:Nx \exp_not:N \g_str_result_tl
+ {
+ \exp_after:wN \str_encode_native_filter:N
+ \c_str_positive_bytes_tl
+ { ? = \c_zero \fi: \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+ }
+ }
+ \cs_new:Npn \str_encode_native_filter:N #1
+ {
+ \if_num:w `#1 > \l_str_internal_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \if_num:w \tex_lccode:D `#1 = \c_zero
+ ^^@
+ \else:
+ #1
+ \fi:
+ \str_encode_native_filter:N
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% End the group to restore the catcode of the null byte.
+% \begin{macrocode}
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{8-bit encodings}
+%
+% This section will be entirely rewritten: it is not yet clear in what
+% situations 8-bit encodings are used, hence I don't know what exactly
+% should be optimized. The current approach is reasonably efficient to
+% convert long strings, and it scales well when using many different
+% encodings. An approach based on csnames would have a smaller constant
+% load time for each individual conversion, but has a large hash table
+% cost. Using a range of \tn{count} registers works for decoding, but
+% not for encoding: one possibility there would be to use a binary tree
+% for the mapping of Unicode characters to bytes, stored as a box, one
+% per encoding.
+%
+% Since the section is going to be rewritten, documentation lacks.
+%
+% All the 8-bit encodings which \pkg{l3str} supports rely on the same
+% internal functions.
+%
+% \begin{macro}[int]{\str_declare_eight_bit_encoding:nnn}
+% All the 8-bit encoding definition file start with
+% \cs{str_declare_eight_bit_encoding:nnn} \Arg{encoding name}
+% \Arg{mapping} \Arg{missing bytes}. The \meta{mapping} argument is a
+% token list of pairs \Arg{byte} \Arg{Unicode} expressed in uppercase
+% hexadecimal notation. The \meta{missing} argument is a token list
+% of \Arg{byte}. Every \meta{byte} which does not appear in the
+% \meta{mapping} nor the \meta{missing} lists maps to the same code
+% point in Unicode.
+% \begin{macrocode}
+\cs_new_protected:Npn \str_declare_eight_bit_encoding:nnn #1#2#3
{
- \char_set_catcode_escape:N \\
- \char_set_catcode_group_begin:N \{
- \char_set_catcode_group_end:N \}
- \char_set_catcode_math_superscript:N \^
- \char_set_catcode_ignore:N \ %
- \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN }
- \char_set_catcode_letter:N
- \tl_map_function:nN { 0123456789"'?*+ }
- \char_set_catcode_other:N
- \char_set_catcode_comment:N \%
- \int_set:Nn \tex_endlinechar:D {32}
+ \tl_set:Nn \l_str_internal_tl {#1}
+ \cs_new_protected_nopar:cpn { str_convert_decode_#1: }
+ { \str_convert_decode_eight_bit:n {#1} }
+ \cs_new_protected_nopar:cpn { str_convert_encode_#1: }
+ { \str_convert_encode_eight_bit:n {#1} }
+ \tl_const:cn { c_str_encoding_#1_tl } {#2}
+ \tl_const:cn { c_str_encoding_#1_missing_tl } {#3}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_from_eight_bit:n}
-% \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn}
-% \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n}
-% \begin{macro}[aux, EXP]{\str_convert_from_eight_bit_aux:N}
+% \begin{macro}[int]{\str_convert_decode_eight_bit:n}
+% \begin{macro}[aux]{\str_decode_eight_bit_load:nn}
+% \begin{macro}[aux]{\str_decode_eight_bit_load_missing:n}
+% \begin{macro}[aux, EXP]{\str_decode_eight_bit_char:N}
%^^A todo: document
% \begin{macrocode}
-\cs_new_protected:Npn \str_convert_from_eight_bit:n #1
+\cs_new_protected:Npn \str_convert_decode_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_tmpa_int
- \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn
+ \int_zero:N \l_str_internal_int
+ \exp_last_unbraced:Nx \str_decode_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
\prg_break_point:n { }
- \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n
+ \exp_last_unbraced:Nx \str_decode_eight_bit_load_missing:n
{ \tl_use:c { c_str_encoding_#1_missing_tl } }
{ \q_stop \prg_map_break: }
\prg_break_point:n { }
- \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N
+ \flag_clear:n { str_error }
+ \str_convert_gmap:N \str_decode_eight_bit_char:N
+ \str_if_flag_error:nnx { str_error } { decode-8-bit } {#1}
\group_end:
}
-\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2
+\cs_new_protected:Npn \str_decode_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_from_eight_bit_load:nn
+ \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_internal_int \exp_after:wN { \int_value:w "#2 }
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_decode_eight_bit_load:nn
}
-\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1
+\cs_new_protected:Npn \str_decode_eight_bit_load_missing:n #1
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN
+ \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_internal_int \exp_after:wN
{ \int_use:N \c_str_replacement_char_int }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_from_eight_bit_load_missing:n
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_decode_eight_bit_load_missing:n
}
-\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
+\cs_new:Npn \str_decode_eight_bit_char:N #1
{
#1 \s_tl
- \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int
+ \if_num:w \tex_dimen:D `#1 < \l_str_internal_int
\if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D
\fi:
@@ -2011,54 +2115,55 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_to_eight_bit:n}
-% \begin{macro}[aux]{\str_convert_to_eight_bit_load:nn}
-% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux:n}
-% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux_ii:n}
+% \begin{macro}[int]{\str_convert_encode_eight_bit:n}
+% \begin{macro}[aux]{\str_encode_eight_bit_load:nn}
+% \begin{macro}[aux, rEXP]{\str_encode_eight_bit_char:n}
+% \begin{macro}[aux, rEXP]{\str_encode_eight_bit_char_aux:n}
%^^A todo: document
% \begin{macrocode}
-\cs_new_protected:Npn \str_convert_to_eight_bit:n #1
+\cs_new_protected:Npn \str_convert_encode_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_tmpa_int
- \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn
+ \int_zero:N \l_str_internal_int
+ \exp_last_unbraced:Nx \str_encode_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
\prg_break_point:n { }
- \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n
+ \flag_clear:n { str_error }
+ \str_convert_gmap_internal:N \str_encode_eight_bit_char:n
+ \str_if_flag_error:nnx { str_error } { encode-8-bit } {#1}
\group_end:
}
-\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2
+\cs_new_protected:Npn \str_encode_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop:
- \exp_args:NNf \tex_toks:D \l_str_tmpa_int
+ \tex_dimen:D "#2 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#2 sp \scan_stop:
+ \exp_args:NNf \tex_toks:D \l_str_internal_int
{ \str_output_byte:n { "#1 } }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_to_eight_bit_load:nn
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_encode_eight_bit_load:nn
}
-\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
+\cs_new:Npn \str_encode_eight_bit_char:n #1
{
\if_num:w #1 > \c_max_register_int
- \msg_expandable_kernel_error:nnn
- { str } { eight-bit-to-byte-overflow } {#1}
+ \flag_raise:n { str_error }
\else:
- \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int
+ \if_num:w \tex_dimen:D #1 < \l_str_internal_int
\if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
\fi:
\fi:
- \str_convert_to_eight_bit_aux_ii:n {#1}
+ \str_encode_eight_bit_char_aux:n {#1}
\fi:
}
-\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
+\cs_new:Npn \str_encode_eight_bit_char_aux:n #1
{
\if_num:w #1 < \c_two_hundred_fifty_six
\str_output_byte:n {#1}
\else:
- \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte }
+ \flag_raise:n { str_error }
\fi:
}
% \end{macrocode}
@@ -2067,255 +2172,72 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_encoding_eight_bit:n}
-%^^A todo: document
-% \begin{macrocode}
-\cs_new_protected:Npn \str_encoding_eight_bit:n #1
- {
- \cs_new_protected:cpn { str_convert_from_#1: }
- { \str_convert_from_eight_bit:n {#1} }
- \cs_new_protected:cpn { str_convert_to_#1: }
- { \str_convert_to_eight_bit:n {#1} }
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \subsubsection{Byte input and output}
-%
-% Strings of bytes may need to be stored in auxiliary files
-% in safe \enquote{escaping} formats. Each such escaping
-% is only loaded as needed. By default, on input any non-byte
-% is filtered out, while the output simply consists in letting
-% bytes through.
-%
-% \begin{macro}[int, rEXP]{\str_filter_bytes:n}
-% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N}
-% In the case of pdf\TeX{}, every character is a byte.
-% For Unicode-aware engines, test the character code.
-% Spaces have already been given the correct category
-% code when this function is called.
-% \begin{macrocode}
-\pdftex_if_engine:TF
- { \cs_new_eq:NN \str_filter_bytes:n \use:n }
- {
- \cs_new_nopar:Npn \str_filter_bytes:n #1
- {
- \str_filter_bytes_aux:N #1
- { ? \prg_map_break: }
- \prg_break_point:n { }
- }
- \cs_new_nopar:Npn \str_filter_bytes_aux:N #1
- {
- \use_none:n #1
- \if_num:w `#1 < 256 \exp_stop_f: #1 \fi:
- \str_filter_bytes_aux:N
- }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[int]{\str_convert_input_:}
-% \begin{macro}[int]{\str_convert_input_bytes:}
-% The simplest input method removes non-bytes
-% from \cs{g_str_result_tl}.
-% \begin{macrocode}
-\pdftex_if_engine:TF
- { \cs_new_protected_nopar:Npn \str_convert_input_: { } }
- {
- \cs_new_protected_nopar:Npn \str_convert_input_:
- {
- \tl_gset:Nx \g_str_result_tl
- { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
- }
- }
-\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_:
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[int]{\str_convert_output_:}
-% \begin{macro}[int]{\str_convert_output_bytes:}
-% The simplest form of output leaves the bytes from the previous
-% step of the conversion unchanged.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_output_: { }
-\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_:
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Convert to and from native strings}
-%
-% \begin{macro}[int]{\str_convert_from_native:}
-% \begin{macro}[aux, rEXP]{\str_convert_from_native_aux:N}
-% Convert each character to its character code, one at a time.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_from_native:
- { \str_aux_gmap_result:N \str_convert_from_native_aux:N }
-\cs_new:Npn \str_convert_from_native_aux:N #1
- { #1 \s_tl \int_value:w `#1 \s_tl }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
+% \subsection{Messages}
%
-% \begin{macro}[int]{\str_convert_to_native:}
-% \begin{macro}[aux]{\str_convert_to_native_aux:w}
-% This function is based on the internal functions
-% \cs{str_aux_convert_to_native:n} and
-% \cs{str_aux_convert_to_native_step:n}.
-% Loop through the comma-list, and call
-% \cs{str_aux_convert_to_native_step:n}
-% with the relevant integer argument.
-% We need to store \cs{g_str_result_tl} in a temporary
-% token list, because that global \enquote{result} is
-% cleared silently by \cs{str_aux_convert_to_native:n}.
-%^^A todo: update
+% General messages, and messages for the encodings and escapings loaded
+% by default (\enquote{native}, and \enquote{bytes}).
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_to_native:
- {
- \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl
- \str_aux_convert_to_native:n
- {
- \exp_after:wN \str_convert_to_native_aux:w
- \l_str_tmpa_tl \s_tl { \q_stop \prg_map_break: } \s_tl
- }
- }
-\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 \s_tl #2 \s_tl
+\msg_kernel_new:nnn { str } { unknown-esc }
+ { Escaping~scheme~'#1'~(filtered:~'#2')~unknown. }
+\msg_kernel_new:nnn { str } { unknown-enc }
+ { Encoding~scheme~'#1'~(filtered:~'#2')~unknown. }
+\msg_kernel_new:nnnn { str } { native-escaping }
+ { The~'native'~encoding~scheme~does~not~support~any~escaping. }
{
- \use_none_delimit_by_q_stop:w #2 \q_stop
- \str_aux_convert_to_native_step:n {#2}
- \str_convert_to_native_aux:w
+ Since~native~strings~do~not~consist~in~bytes,~
+ none~of~the~escaping~methods~make~sense.~
+ The~specified~escaping,~'#1',~will be ignored.
}
+\msg_kernel_new:nnn { str } { file-not-found }
+ { File~'l3str-#1.def'~not~found. }
% \end{macrocode}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\str_aux_convert_to_native:n}
-% \begin{macro}[int]{\str_aux_convert_to_native_step:n}
-% \begin{macro}[aux]{\str_aux_convert_to_native_flush:}
-% \begin{macro}[aux, rEXP]{\str_aux_convert_to_native_filter:N}
-%^^A todo: document.
+% Message used when the \enquote{bytes} unescaping fails because the
+% string given to \cs{str_set_convert:Nnnn} contains a non-byte. This
+% cannot happen for the pdf\TeX{} engine, since that engine only
+% supports 8-bit characters. Messages used for other escapings and
+% encodings are defined in each definition file.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1
- {
- \tex_lccode:D \c_zero \c_zero
- \int_zero:N \l_str_tmpa_int
- \tl_gclear:N \g_str_result_tl
- #1
- \prg_break_point:n { \str_aux_convert_to_native_flush: }
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1
- {
- \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five
- \str_aux_convert_to_native_flush:
- \l_str_tmpa_int \c_zero
- \fi:
- \tex_advance:D \l_str_tmpa_int \c_one
- \if_int_compare:w #1 > \c_max_char_int
- \msg_kernel_error:nnx { str } { overflow } {#1}
- \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int
- \else:
- \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end:
- \fi:
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush:
+\pdftex_if_engine:F
{
- \use:x
+ \msg_kernel_new:nnnn { str } { non-byte }
+ { String~invalid~in~escaping~'#1':~it~may~only~contain~bytes. }
{
- \tl_to_lowercase:n
+ Some~characters~in~the~string~you~asked~to~convert~are~not~
+ 8-bit~characters.~Perhaps~the~string~is~a~'native'~Unicode~string?~
+ If~it~is,~try~using\\
+ \\
+ \iow_indent:n
{
- \tl_gput_right:Nx \exp_not:N \g_str_result_tl
- {
- \exp_after:wN \str_aux_convert_to_native_filter:N
- \c_str_positive_bytes_tl
- { ? = \c_zero \fi: \prg_map_break: }
- \prg_break_point:n { }
- }
+ \iow_char:N\\str_set_convert:Nnnn \\
+ \ \ <str~var>~\{~<string>~\}~\{~native~\}~\{~<target~encoding>~\}
}
}
}
-\group_begin:
- \char_set_catcode_other:n { 0 }
- \cs_new:Npn \str_aux_convert_to_native_filter:N #1
- {
- \if_num:w `#1 > \l_str_tmpa_int
- \exp_after:wN \prg_map_break:
- \fi:
- \if_num:w \tex_lccode:D `#1 = \c_zero
- ^^@
- \else:
- #1
- \fi:
- \str_aux_convert_to_native_filter:N
- }
-\group_end:
% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsection{Messages}
%
+% Those messages are used when converting to and from 8-bit encodings.
% \begin{macrocode}
-\msg_kernel_new:nnnn { str } { x-missing-brace }
- { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. }
+\msg_kernel_new:nnnn { str } { decode-8-bit }
+ { Invalid~string~in~encoding~'#1'. }
{
- You~wrote~something~like~
- `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~
- The~closing~brace~is~missing.
+ LaTeX~came~across~a~byte~which~is~not~defined~to~represent~
+ any~character~in~the~encoding~'#1'.
}
-\msg_kernel_new:nnnn { str } { overflow }
- { Character~code~#1~too~big. }
+\msg_kernel_new:nnnn { str } { encode-8-bit }
+ { Unicode~string~cannot~be~converted~to~encoding~'#1'. }
{
- \int_compare:nNnTF {#1} > { 1114111 }
- { The~Unicode~standard~limits~code~points~to~1114111. }
- {
- The~pdfTeX~engine~only~supports~8-bit~characters:~
- valid~character~codes~are~in~the~range~[0,255].~
- To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
- }
+ The~encoding~'#1'~only~contains~a~subset~of~all~Unicode~characters.~
+ LaTeX~was~asked~to~convert~a~string~to~that~encoding,~but~that~
+ string~contains~a~character~that~'#1'~does~not~support.
}
-\msg_kernel_new:nnn { str } { convert-input }
- { Input~scheme~`#1'~unknown. }
-\msg_kernel_new:nnn { str } { convert-from }
- { Encoding~`#1'~unknown. }
-\msg_kernel_new:nnn { str } { convert-to }
- { Encoding~`#1'~unknown.~Using~UTF-8. }
-\msg_kernel_new:nnn { str } { convert-output }
- { Output~scheme~`#1~unknown. }
-\msg_kernel_new:nnn { str } { to-native }
- { Use~\str_set_convert:Nnn to~convert~to~native~strings. }
-\msg_kernel_new:nnn { str } { unicode-surrogate }
- { Code~point~#1~is~an~unpaired~surrogate. }
-\msg_kernel_new:nnn { str } { utf16-surrogate }
- { UTF-16~unpaired~surrogate~#1. }
-\msg_kernel_new:nnn { str } { utf16-odd }
- { UTF-16~dangling~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-extra-conti }
- { UTF-8~extra~continuation~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-missing-conti }
- { UTF-8~missing~continuation~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-invalid-byte }
- { Byte~#1~cannot~appear~in~UTF-8. }
-\msg_kernel_new:nnn { str } { utf8-overlong }
- { Overlong~UTF-8~byte~sequence~for~code~point~#1. }
-\msg_kernel_new:nnn { str } { utf8-premature-end }
- { Incomplete~last~UTF-8~character. }
-\msg_kernel_new:nnn { str } { utf8-overflow }
- { Code~point~#1~too~large~(UTF-8). }
-\msg_kernel_new:nnn { str } { utf32-overflow }
- { Code~point~too~large~(UTF-32~`#1'). }
-\msg_kernel_new:nnn { str } { utf32-truncated }
- { Truncated~UTF-32~string~`...#1'. }
% \end{macrocode}
%
% \subsection{Deprecated string functions}
%
% \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n}
-% The naming scheme is a little bit more consistent
-% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}.
+% The naming scheme is a little bit more consistent with
+% \enquote{ignore_spaces} instead of \enquote{skip_spaces}.
% \begin{macrocode}
\cs_set:Npn \str_length_skip_spaces:N
{ \exp_args:No \str_length_skip_spaces:n }
@@ -2329,102 +2251,116 @@
%
% \subsection{Escaping definition files}
%
-% Several of those encodings are defined by the pdf file format.
-% The following byte storage methods are defined:
+% Several of those encodings are defined by the pdf file format. The
+% following byte storage methods are defined:
% \begin{itemize}
-% \item \texttt{bytes} (default), non-bytes are filtered out,
-% and bytes are left untouched;
-% \item \texttt{hex} or \texttt{hexadecimal},
-% as per the pdf\TeX{} primitive \tn{pdfescapehex}
-% \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename}
-% \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring}
-% \item \texttt{url}, as per the percent encoding of urls.
+% \item \texttt{bytes} (default), non-bytes are filtered out, and
+% bytes are left untouched (this is defined by default);
+% \item \texttt{hex} or \texttt{hexadecimal}, as per the pdf\TeX{}
+% primitive \tn{pdfescapehex}
+% \item \texttt{name}, as per the pdf\TeX{} primitive
+% \tn{pdfescapename}
+% \item \texttt{string}, as per the pdf\TeX{} primitive
+% \tn{pdfescapestring}
+% \item \texttt{url}, as per the percent encoding of urls.
% \end{itemize}
%
-% \subsubsection{Input methods}
-%
-% \begin{macro}[int]{\str_convert_input_hex:}
-% \begin{macro}[int]{\str_convert_input_hexadecimal:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux_ii:N}
-% Take chars two by two, and interpret each pair as
-% the hexadecimal code for a byte. Anything else than
-% hexadecimal digits is ignored.
-% A string which contains an odd number of hexadecimal
-% digits gets |0| appended to it: this is equivalent
-% to appending a |0| in all cases, and dropping it if
-% it is alone.
+% \subsubsection{Unescape methods}
+%
+% \begin{macro}[int]{\str_convert_unescape_hex:}
+% \begin{macro}[aux, rEXP]{\str_unescape_hex_i:N}
+% \begin{macro}[aux, rEXP]{\str_unescape_hex_ii:N}
+% Take chars two by two, and interpret each pair as the hexadecimal
+% code for a byte. Anything else than hexadecimal digits is ignored,
+% raising the flag. A string which contains an odd number of
+% hexadecimal digits gets |0| appended to it: this is equivalent to
+% appending a |0| in all cases, and dropping it if it is alone.
% \begin{macrocode}
%<*hex>
-\cs_new_protected_nopar:Npn \str_convert_input_hex:
+\cs_new_protected_nopar:Npn \str_convert_unescape_hex:
{
\group_begin:
+ \flag_clear:n { str_error }
\int_set:Nn \tex_escapechar:D { 92 }
\tl_gset:Nx \g_str_result_tl
{
\str_output_byte:w "
- \exp_after:wN \str_convert_input_hex_aux:N
- \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
+ \exp_last_unbraced:Nf \str_unescape_hex_i:N
+ { \tl_to_str:N \g_str_result_tl }
+ 0 { ? 0 - \c_one \prg_map_break: }
\prg_break_point:n { \str_output_end: }
}
+ \str_if_flag_error:nnx { str_error } { unescape-hex } { }
\group_end:
}
-\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1
+\cs_new:Npn \str_unescape_hex_i:N #1
{
\use_none:n #1
\str_aux_hexadecimal_use:NTF #1
- \str_convert_input_hex_aux_ii:N
- \str_convert_input_hex_aux:N
+ { \str_unescape_hex_ii:N }
+ {
+ \flag_raise:n { str_error }
+ \str_unescape_hex_i:N
+ }
}
-\cs_new_nopar:Npn \str_convert_input_hex_aux_ii:N #1
+\cs_new:Npn \str_unescape_hex_ii:N #1
{
\use_none:n #1
\str_aux_hexadecimal_use:NTF #1
{
\str_output_end:
- \str_output_byte:w " \str_convert_input_hex_aux:N
+ \str_output_byte:w " \str_unescape_hex_i:N
+ }
+ {
+ \flag_raise:n { str_error }
+ \str_unescape_hex_ii:N
}
- \str_convert_input_hex_aux_ii:N
+ }
+\msg_kernel_new:nnnn { str } { unescape-hex }
+ { String~invalid~in~escaping~'hex':~only~hexadecimal~digits~allowed. }
+ {
+ Some~characters~in~the~string~you~asked~to~convert~are~not~
+ hexadecimal~digits~(0-9,~A-F,~a-f)~nor~spaces.
}
%</hex>
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\str_convert_input_name:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN}
-% \begin{macro}[int]{\str_convert_input_url:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN}
-% The \cs{str_convert_input_name:} function replaces each
-% occurrence of |#| followed by two hexadecimal digits
-% in \cs{g_str_result_tl} by the corresponding byte.
-% The \texttt{url} function is identical, with escape
-% character |%| instead of |#|. Thus we define the two
-% together. The arguments of \cs{str_tmp:w} are the character
-% code of |#| or |%| in hexadecimal, the name of the main
-% function to define, and the name of the auxiliary which
+% \begin{macro}[int]{\str_convert_unescape_name:}
+% \begin{macro}[aux, rEXP]{\str_unescape_name_loop:wNN}
+% \begin{macro}[int]{\str_convert_unescape_url:}
+% \begin{macro}[aux, rEXP]{\str_unescape_url_loop:wNN}
+% The \cs{str_convert_unescape_name:} function replaces each
+% occurrence of |#| followed by two hexadecimal digits in
+% \cs{g_str_result_tl} by the corresponding byte. The \texttt{url}
+% function is identical, with escape character |%| instead of |#|.
+% Thus we define the two together. The arguments of \cs{str_tmp:w} are
+% the character code of |#| or |%| in hexadecimal, the name of the
+% main function to define, and the name of the auxiliary which
% performs the loop.
%
-% The looping auxiliary |#3| finds the next escape character,
-% reads the following two characters, and tests them. The test
-% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit
-% in the input stream, hence we surround the test with
-% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}.
-% If both characters are hexadecimal digits, they should be
-% removed before looping: this is done by \cs{use_i:nnn}.
-% If one of the characters is not a hexadecimal digit,
-% then feed |"#1| to \cs{str_output_byte:w} to produce the
-% escape character, and call the looping function followed
-% by the two characters (remove \cs{use_i:nnn}).
+% The looping auxiliary |#3| finds the next escape character, reads
+% the following two characters, and tests them. The test
+% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit in the
+% input stream, hence we surround the test with
+% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}. If both
+% characters are hexadecimal digits, they should be removed before
+% looping: this is done by \cs{use_i:nnn}. If one of the characters
+% is not a hexadecimal digit, then feed |"#1| to
+% \cs{str_output_byte:w} to produce the escape character, raise the
+% flag, and call the looping function followed by the two characters
+% (remove \cs{use_i:nnn}).
% \begin{macrocode}
%<*name|url>
\cs_set_protected:Npn \str_tmp:w #1#2#3
{
- \cs_new_protected_nopar:Npn #2
+ \cs_new_protected:cpn { str_convert_unescape_#2: }
{
\group_begin:
+ \flag_clear:n { str_byte }
+ \flag_clear:n { str_error }
\int_set:Nn \tex_escapechar:D { 92 }
\tl_gset:Nx \g_str_result_tl
{
@@ -2432,9 +2368,11 @@
#1 ? { ? \prg_map_break: }
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_byte } { non-byte } { #2 }
+ \str_if_flag_error:nnx { str_error } { unescape-#2 } { }
\group_end:
}
- \cs_new_nopar:Npn #3 ##1#1##2##3
+ \cs_new:Npn #3 ##1#1##2##3
{
\str_filter_bytes:n {##1}
\use_none:n ##3
@@ -2443,50 +2381,63 @@
{
\str_aux_hexadecimal_use:NTF ##3
{ }
- { * \c_zero + `#1 \use_i:nn }
+ {
+ \flag_raise:n { str_error }
+ * \c_zero + `#1 \use_i:nn
+ }
+ }
+ {
+ \flag_raise:n { str_error }
+ 0 + `#1 \use_i:nn
}
- { 0 + `#1 \use_i:nn }
\str_output_end:
\use_i:nnn #3 ##2##3
}
+ \msg_kernel_new:nnnn { str } { unescape-#2 }
+ { String~invalid~in~escaping~'#2'. }
+ {
+ LaTeX~came~across~the~escape~character~'#1'~not~followed~by~
+ two~hexadecimal~digits.~This~is~invalid~in~the~escaping~'#2'.
+ }
}
%</name|url>
-%<name>\exp_after:wN \str_tmp:w \c_hash_str
-%<name> \str_convert_input_name: \str_convert_input_name_aux:wNN
-%<url>\exp_after:wN \str_tmp:w \c_percent_str
-%<url> \str_convert_input_url: \str_convert_input_url_aux:wNN
+%<name>\exp_after:wN \str_tmp:w \c_hash_str { name }
+%<name> \str_unescape_name_loop:wNN
+%<url>\exp_after:wN \str_tmp:w \c_percent_str { url }
+%<url> \str_unescape_url_loop:wNN
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_input_string:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN}
-% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wNNN}
-% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:NNNNNN}
-% The \texttt{string} escaping is somewhat similar to
-% the \texttt{name} and \texttt{url} escapings, with
-% escape character |\|.
-% The first step is to convert all three line endings,
-% |^^J|, |^^M|, and |^^M^^J| to the common |^^J|, as per
-% the \textsc{pdf} specification.
+% \begin{macro}[int]{\str_convert_unescape_string:}
+% \begin{macro}[aux, rEXP]{\str_unescape_string_newlines:wN}
+% \begin{macro}[aux, rEXP]{\str_unescape_string_loop:wNNN}
+% \begin{macro}[aux, rEXP]{\str_unescape_string_repeat:NNNNNN}
+% The \texttt{string} escaping is somewhat similar to the
+% \texttt{name} and \texttt{url} escapings, with escape character |\|.
+% The first step is to convert all three line endings, |^^J|, |^^M|,
+% and |^^M^^J| to the common |^^J|, as per the \textsc{pdf}
+% specification. This step cannot raise the flag.
+%
% Then the following escape sequences are decoded.
% \begin{itemize}\def\makelabel#1{\hss\llap{\ttfamily\string#1}}
-% \item[\n] Line feed ($10$)
-% \item[\r] Carriage return ($13$)
-% \item[\t] Horizontal tab ($9$)
-% \item[\b] Backspace ($8$)
-% \item[\f] Form feed ($12$)
-% \item[\(] Left parenthesis
-% \item[\)] Right parenthesis
-% \item[\\] Backslash
-% \item[\ddd] (backslash followed by $1$ to $3$ octal digits)
-% Byte \texttt{ddd} (octal), subtracting $256$ in case of overflow.
+% \item[\n] Line feed ($10$)
+% \item[\r] Carriage return ($13$)
+% \item[\t] Horizontal tab ($9$)
+% \item[\b] Backspace ($8$)
+% \item[\f] Form feed ($12$)
+% \item[\(] Left parenthesis
+% \item[\)] Right parenthesis
+% \item[\\] Backslash
+% \item[\ddd] (backslash followed by $1$ to $3$ octal digits) Byte
+% \texttt{ddd} (octal), subtracting $256$ in case of overflow.
% \end{itemize}
-% If followed by an end-of-line character, the backslash and
-% the end-of-line are ignored. If followed by anything else,
-% the backslash is ignored.
+% If followed by an end-of-line character, the backslash and the
+% end-of-line are ignored. If followed by anything else, the backslash
+% is ignored, raising the error flag.
+%^^A Be paranoid: \tl_to_lowercase:n is unsafe.
% \begin{macrocode}
%<*string>
\group_begin:
@@ -2495,25 +2446,29 @@
\char_set_catcode_other:N \^^M
\tl_to_lowercase:n
{
- \cs_new_protected_nopar:Npn \str_convert_input_string:
+ \cs_new_protected_nopar:Npn \str_convert_unescape_string:
{
\group_begin:
+ \flag_clear:n { str_byte }
+ \flag_clear:n { str_error }
\int_set:Nn \tex_escapechar:D { 92 }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_convert_input_string_aux:wN
+ \exp_after:wN \str_unescape_string_newlines:wN
\g_str_result_tl \prg_map_break: ^^M ?
\prg_break_point:n { }
}
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_convert_input_string_aux:wNNN
+ \exp_after:wN \str_unescape_string_loop:wNNN
\g_str_result_tl * ?? { ? \prg_map_break: }
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_byte } { non-byte } { string }
+ \str_if_flag_error:nnx { str_error } { unescape-string } { }
\group_end:
}
- \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4
+ \cs_new:Npn \str_unescape_string_loop:wNNN #1 *#2#3#4
}
{
\str_filter_bytes:n {#1}
@@ -2528,11 +2483,11 @@
\if_int_compare:w #2 > \c_three
- 256
\fi:
- \str_convert_input_string_aux:NNNNNN
+ \str_unescape_string_repeat:NNNNNN
}
- { \str_convert_input_string_aux:NNNNNN ? }
+ { \str_unescape_string_repeat:NNNNNN ? }
}
- { \str_convert_input_string_aux:NNNNNN ?? }
+ { \str_unescape_string_repeat:NNNNNN ?? }
}
{
\prg_case_str:xxn {#2}
@@ -2547,20 +2502,29 @@
{ b } { 10 }
{ ^^J } { 0 - \c_one }
}
- { 0 - \c_one \use_i:nn }
+ {
+ \flag_raise:n { str_error }
+ 0 - \c_one \use_i:nn
+ }
}
\str_output_end:
- \use_i:nn \str_convert_input_string_aux:wNNN #2#3#4
+ \use_i:nn \str_unescape_string_loop:wNNN #2#3#4
}
- \cs_new_nopar:Npn \str_convert_input_string_aux:NNNNNN #1#2#3#4#5#6
- { \str_output_end: \str_convert_input_string_aux:wNNN }
- \cs_new_nopar:Npn \str_convert_input_string_aux:wN #1 ^^M #2
+ \cs_new:Npn \str_unescape_string_repeat:NNNNNN #1#2#3#4#5#6
+ { \str_output_end: \str_unescape_string_loop:wNNN }
+ \cs_new:Npn \str_unescape_string_newlines:wN #1 ^^M #2
{
- #1 ^^J
- \if_charcode:w ^^J #2
- \exp_after:wN \use_i:nn
- \fi:
- \str_convert_input_string_aux:wN #2
+ #1
+ \if_charcode:w ^^J #2 \else: ^^J \fi:
+ \str_unescape_string_newlines:wN #2
+ }
+ \msg_kernel_new:nnnn { str } { unescape-string }
+ { String~invalid~in~escaping~'string'. }
+ {
+ LaTeX~came~across~an~escape~character~'\c_backslash_str'~
+ not~followed~by~any~of:~'n',~'r',~'t',~'b',~'f',~'(',~')',~
+ '\c_backslash_str',~one~to~three~octal~digits,~or~the~end~
+ of~a~line.
}
\group_end:
%</string>
@@ -2570,57 +2534,57 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Output methods}
+% \subsubsection{Escape methods}
%
-% \begin{macro}[int]{\str_convert_output_hex:}
-% \begin{macro}[int]{\str_convert_output_hexadecimal:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_hex_aux:N}
+% Currently, none of the escape methods can lead to errors, assuming
+% that their input is made out of bytes.
+%
+% \begin{macro}[int]{\str_convert_escape_hex:}
+% \begin{macro}[aux, rEXP]{\str_escape_hex_char:N}
% Loop and convert each byte to hexadecimal.
% \begin{macrocode}
%<*hex>
-\cs_new_protected_nopar:Npn \str_convert_output_hex:
- { \str_aux_gmap_result:N \str_convert_output_hex_aux:N }
-\cs_new_nopar:Npn \str_convert_output_hex_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_escape_hex:
+ { \str_convert_gmap:N \str_escape_hex_char:N }
+\cs_new:Npn \str_escape_hex_char:N #1
{ \str_output_hexadecimal:n { `#1 } }
%</hex>
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\str_convert_output_name:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:NTF}
-% \begin{variable}{\c_str_convert_output_name_str}
-% \begin{variable}{\c_str_convert_output_name_not_str}
-% For each byte, test whether it should
-% be output as is, or be \enquote{hash-encoded}.
-% Roughly, bytes outside the range $[\hexnum{2A},\hexnum{7E}]$
-% are hash-encoded. We keep two lists of exceptions:
-% characters in \cs{c_str_convert_output_name_not_str}
-% are not hash-encoded, and characters in the
-% \cs{c_str_convert_output_name_str} are encoded.
+% \begin{macro}[int]{\str_convert_escape_name:}
+% \begin{macro}[aux, rEXP]{\str_escape_name_char:N}
+% \begin{macro}[aux, rEXP]{\str_if_escape_name:NTF}
+% \begin{variable}{\c_str_escape_name_str}
+% \begin{variable}{\c_str_escape_name_not_str}
+% For each byte, test whether it should be output as is, or be
+% \enquote{hash-encoded}. Roughly, bytes outside the range
+% $[\hexnum{2A},\hexnum{7E}]$ are hash-encoded. We keep two lists of
+% exceptions: characters in \cs{c_str_escape_name_not_str} are not
+% hash-encoded, and characters in the \cs{c_str_escape_name_str} are
+% encoded.
% \begin{macrocode}
%<*name>
-\str_const:Nn \c_str_convert_output_name_not_str { ! " $ & ' } %$
-\str_const:Nn \c_str_convert_output_name_str { {}/<>[] }
-\cs_new_protected_nopar:Npn \str_convert_output_name:
- { \str_aux_gmap_result:N \str_convert_output_name_aux:N }
-\cs_new_nopar:Npn \str_convert_output_name_aux:N #1
+\str_const:Nn \c_str_escape_name_not_str { ! " $ & ' } %$
+\str_const:Nn \c_str_escape_name_str { {}/<>[] }
+\cs_new_protected_nopar:Npn \str_convert_escape_name:
+ { \str_convert_gmap:N \str_escape_name_char:N }
+\cs_new:Npn \str_escape_name_char:N #1
{
- \str_convert_output_name_aux:NTF #1 {#1}
+ \str_if_escape_name:NTF #1 {#1}
{ \c_hash_str \str_output_hexadecimal:n {`#1} }
}
-\prg_new_conditional:Npnn \str_convert_output_name_aux:N #1 { TF }
+\prg_new_conditional:Npnn \str_if_escape_name:N #1 { TF }
{
\if_num:w `#1 < "2A \exp_stop_f:
- \str_if_contains_char:NNTF \c_str_convert_output_name_not_str #1
+ \str_if_contains_char:NNTF \c_str_escape_name_not_str #1
\prg_return_true: \prg_return_false:
\else:
\if_num:w `#1 > "7E \exp_stop_f:
\prg_return_false:
\else:
- \str_if_contains_char:NNTF \c_str_convert_output_name_str #1
+ \str_if_contains_char:NNTF \c_str_escape_name_str #1
\prg_return_false: \prg_return_true:
\fi:
\fi:
@@ -2633,25 +2597,25 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_output_string:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:NTF}
-% \begin{variable}{\c_str_convert_output_string_str}
-% Any character below (and including) space, and any character
-% above (and including) \texttt{del}, are converted to octal.
-% One backslash is added before each parenthesis and backslash.
+% \begin{macro}[int]{\str_convert_escape_string:}
+% \begin{macro}[aux, rEXP]{\str_escape_string_char:N}
+% \begin{macro}[aux, rEXP]{\str_if_escape_string:NTF}
+% \begin{variable}{\c_str_escape_string_str}
+% Any character below (and including) space, and any character above
+% (and including) \texttt{del}, are converted to octal. One backslash
+% is added before each parenthesis and backslash.
% \begin{macrocode}
%<*string>
-\str_const:Nx \c_str_convert_output_string_str
+\str_const:Nx \c_str_escape_string_str
{ \c_backslash_str ( ) }
-\cs_new_protected_nopar:Npn \str_convert_output_string:
- { \str_aux_gmap_result:N \str_convert_output_string_aux:N }
-\cs_new_nopar:Npn \str_convert_output_string_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_escape_string:
+ { \str_convert_gmap:N \str_escape_string_char:N }
+\cs_new:Npn \str_escape_string_char:N #1
{
- \str_convert_output_string_aux:NTF #1
+ \str_if_escape_string:NTF #1
{
\str_if_contains_char:NNT
- \c_str_convert_output_string_str #1
+ \c_str_escape_string_str #1
{ \c_backslash_str }
#1
}
@@ -2662,7 +2626,7 @@
\int_mod:nn {`#1} \c_eight
}
}
-\prg_new_conditional:Npnn \str_convert_output_string_aux:N #1 { TF }
+\prg_new_conditional:Npnn \str_if_escape_string:N #1 { TF }
{
\if_num:w `#1 < "21 \exp_stop_f:
\prg_return_false:
@@ -2681,23 +2645,21 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_output_url:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF}
-%^^A todo: Figure out what bytes should be percent encoded! Safe could be
-%^^A !'()*-./0123456789_
-% This function is similar to \cs{str_convert_output_name:},
-% escaping different characters.
+% \begin{macro}[int]{\str_convert_escape_url:}
+% \begin{macro}[aux, rEXP]{\str_escape_url_char:N}
+% \begin{macro}[aux, rEXP]{\str_if_escape_url:NTF}
+% This function is similar to \cs{str_convert_escape_name:}, escaping
+% different characters.
% \begin{macrocode}
%<*url>
-\cs_new_protected_nopar:Npn \str_convert_output_url:
- { \str_aux_gmap_result:N \str_convert_output_url_aux:N }
-\cs_new_nopar:Npn \str_convert_output_url_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_escape_url:
+ { \str_convert_gmap:N \str_escape_url_char:N }
+\cs_new:Npn \str_escape_url_char:N #1
{
- \str_convert_output_url_aux:NTF #1 {#1}
+ \str_if_escape_url:NTF #1 {#1}
{ \c_percent_str \str_output_hexadecimal:n { `#1 } }
}
-\prg_new_conditional:Npnn \str_convert_output_url_aux:N #1 { TF }
+\prg_new_conditional:Npnn \str_if_escape_url:N #1 { TF }
{
\if_num:w `#1 < "41 \exp_stop_f:
\str_if_contains_char:nNTF { "-.<> } #1
@@ -2719,15 +2681,14 @@
%
% \subsection{Encoding definition files}
%
-% The \texttt{native} and \texttt{internal} encodings are automatically
-% defined. Other encodings are loaded as needed. The following encodings
-% are supported:
+% The \texttt{native} encoding is automatically defined. Other encodings
+% are loaded as needed. The following encodings are supported:
% \begin{itemize}
-% \item \textsc{utf-8};
-% \item \textsc{utf-16}, big-endian, little-endian, or with byte order mark;
-% \item \textsc{utf-32}, big-endian, little-endian, or with byte order mark;
-% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$,
-% skipping the inexistent \textsc{iso 8859-12}.
+% \item \textsc{utf-8};
+% \item \textsc{utf-16}, big-, little-endian, or with byte order mark;
+% \item \textsc{utf-32}, big-, little-endian, or with byte order mark;
+% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$,
+% skipping the inexistent \textsc{iso 8859-12}.
% \end{itemize}
%
% \subsubsection{\textsc{utf-8} support}
@@ -2736,74 +2697,235 @@
%<*utf8>
% \end{macrocode}
%
-% \begin{macro}[int]{\str_convert_to_utf8:}
-% \begin{macro}[aux, rEXP]
-% {
-% \str_aux_to_utf_viii:n,
-% \str_aux_to_utf_viii:nnnnw
-% }
-% Loop through the internal string, and convert each character
-% to the \textsc{utf-8} representation. The details are messy.
-%^^A todo: document.
+% \begin{macro}[int]{\str_convert_encode_utf8:}
+% \begin{macro}[aux, rEXP]{\str_encode_utf_viii_char:n}
+% \begin{macro}[aux, rEXP]{\str_encode_utf_viii_loop:wwnnw}
+% Loop through the internal string, and convert each character to its
+% \textsc{utf-8} representation. The representation is built from the
+% right-most (least significant) byte to the left-most (most
+% significant) byte. Continuation bytes are in the range $[128,191]$,
+% taking $64$ different values, hence we roughly want to express the
+% character code in base $64$, shifting the first digit in the
+% representation by some number depending on how many continuation
+% bytes there are. In the range $[0,127]$, output the corresponding
+% byte directly. In the range $[128,2047]$, output the remainder
+% modulo $64$, plus $128$ as a continuation byte, then output the
+% quotient (which is in the range $[0,31]$), shifted by $192$. In the
+% next range, $[2048,65535]$, split the character code into residue
+% and quotient modulo $64$, output the residue as a first continuation
+% byte, then repeat; this leaves us with a quotient in the range
+% $[0,15]$, which we output shifted by $224$. The last range,
+% $[65536,1114111]$, follows the same pattern: once we realize that
+% dividing twice by $64$ leaves us with a number larger than $15$, we
+% repeat, producing a last continuation byte, and offset the quotient
+% by $240$ for the leading byte.
+%
+% How is that implemented? \cs{str_encode_utf_vii_loop:wwnnw} takes
+% successive quotients as its first argument, the quotient from the
+% previous step as its second argument (except in step~$1$), the bound
+% for quotients that trigger one more step or not, and finally the
+% offset used if this step should produce the leading byte. Leading
+% bytes can be in the ranges $[0,127]$, $[192,223]$, $[224,239]$, and
+% $[240,247]$ (really, that last limit should be $244$ because Unicode
+% stops at the code point $1114111$). At each step, if the quotient
+% |#1| is less than the limit |#3| for that range, output the leading
+% byte (|#1| shifted by |#4|) and stop. Otherwise, we need one more
+% step: use the quotient of |#1| by $64$, and |#1| as arguments for
+% the looping auxiliary, and output the continuation byte
+% corresponding to the remainder $|#2|-64|#1|+128$. The bizarre
+% construction |\c_minus_one + \c_zero *| removes the spurious initial
+% continuation byte (better methods welcome).
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { str_convert_to_utf8: }
- { \str_aux_gmap_internal_result:N \str_aux_to_utf_viii:n }
-\cs_new:Npn \str_aux_to_utf_viii:n #1
+\cs_new_protected_nopar:cpn { str_convert_encode_utf8: }
+ { \str_convert_gmap_internal:N \str_encode_utf_viii_char:n }
+\cs_new:Npn \str_encode_utf_viii_char:n #1
{
- \str_aux_to_utf_viii:nnnnw
- {#1} { \c_minus_one + \c_zero * \use_none:n }
+ \str_encode_utf_viii_loop:wwnnw #1 ; \c_minus_one + \c_zero * ;
{ 128 } { \c_zero }
{ 32 } { 192 }
{ 16 } { 224 }
{ 8 } { 240 }
\q_stop
}
-\cs_new_nopar:Npn \str_aux_to_utf_viii:nnnnw #1#2#3#4 #5 \q_stop
+\cs_new:Npn \str_encode_utf_viii_loop:wwnnw #1; #2; #3#4 #5 \q_stop
{
\if_num:w #1 < #3 \exp_stop_f:
\str_output_byte:n { #1 + #4 }
- \else:
- \exp_args:Nf \str_aux_to_utf_viii:nnnnw
- { \int_div_truncate:nn {#1} {64} }
- {#1}
- #5 \q_stop
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
+ \exp_after:wN \str_encode_utf_viii_loop:wwnnw
+ \int_value:w \int_div_truncate:nn {#1} {64} ; #1 ;
+ #5 \q_stop
\str_output_byte:n { #2 - 64 * ( #1 - \c_two ) }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}[int]{\str_convert_from_utf8:}
-% \begin{macro}[aux, rEXP]
+% \begin{variable}
% {
-% \str_aux_from_utf_viii:N,
-% \str_aux_from_utf_viii:wwN,
-% \str_aux_from_utf_viii:wNnnwN
+% \l_str_missing_flag ,
+% \l_str_extra_flag ,
+% \l_str_overlong_flag ,
+% \l_str_overflow_flag ,
% }
+% When decoding a string that is purportedly in the \textsc{utf-8}
+% encoding, four different errors can occur, signalled by a specific
+% flag for each (we define those flags using \cs{flag_clear_new:n}
+% rather than \cs{flag_new:n}, because they are shared with other
+% encoding definition files).
+% \begin{itemize}
+% \item \enquote{Missing continuation byte}: a leading byte is not
+% followed by the right number of continuation bytes.
+% \item \enquote{Extra continuation byte}: a continuation byte
+% appears where it was not expected, \emph{i.e.}, not after an
+% appropriate leading byte.
+% \item \enquote{Overlong}: a Unicode character is expressed using
+% more bytes than necessary, for instance, \hexnum{C0}\hexnum{80}
+% for the code point $0$, instead of a single null byte.
+% \item \enquote{Overflow}: this occurs when decoding produces
+% Unicode code points greater than $1114111$.
+% \end{itemize}
+% We only raise one \LaTeX3 error message, combining all the errors
+% which occurred. In the short message, the leading comma must be
+% removed to get a grammatically correct sentence. In the long text,
+% first remind the user what a correct \textsc{utf-8} string should
+% look like, then add error-specific information.
+% \begin{macrocode}
+\flag_clear_new:n { str_missing }
+\flag_clear_new:n { str_extra }
+\flag_clear_new:n { str_overlong }
+\flag_clear_new:n { str_overflow }
+\msg_kernel_new:nnnn { str } { utf8-decode }
+ {
+ Invalid~UTF-8~string: \exp_last_unbraced:Nf \use_none:n
+ \str_if_flag_times:nT { str_missing } { ,~missing~continuation~byte }
+ \str_if_flag_times:nT { str_extra } { ,~extra~continuation~byte }
+ \str_if_flag_times:nT { str_overlong } { ,~overlong~form }
+ \str_if_flag_times:nT { str_overflow } { ,~code~point~too~large }
+ .
+ }
+ {
+ In~the~UTF-8~encoding,~each~Unicode~character~consists~in~
+ 1~to~4~bytes,~with~the~following~bit~pattern: \\
+ \iow_indent:n
+ {
+ Code~point~\ \ \ \ <~128:~0xxxxxxx \\
+ Code~point~\ \ \ <~2048:~110xxxxx~10xxxxxx \\
+ Code~point~\ \ <~65536:~1110xxxx~10xxxxxx~10xxxxxx \\
+ Code~point~ <~1114112:~11110xxx~10xxxxxx~10xxxxxx~10xxxxxx \\
+ }
+ Bytes~of~the~form~10xxxxxx~are~called~continuation~bytes.
+ \flag_if_raised:nT { str_missing }
+ {
+ \\\\
+ A~leading~byte~(in~the~range~[192,255])~was~not~followed~by~
+ the~appropriate~number~of~continuation~bytes.
+ }
+ \flag_if_raised:nT { str_extra }
+ {
+ \\\\
+ LaTeX~came~across~a~continuation~byte~when~it~was~not~expected.
+ }
+ \flag_if_raised:nT { str_overlong }
+ {
+ \\\\
+ Every~Unicode~code~point~must~be~expressed~in~the~shortest~
+ possible~form.~For~instance,~'0xC0'~'0x83'~is~not~a~valid~
+ representation~for~the~code~point~3.
+ }
+ \flag_if_raised:nT { str_overflow }
+ {
+ \\\\
+ Unicode~limits~code~points~to~the~range~[0,1114111].
+ }
+ }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}[int]{\str_convert_decode_utf8:}
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_from_utf_viii_overflow:w ,
-% \str_aux_from_utf_viii_error:
+% \str_decode_utf_viii_start:N,
+% \str_decode_utf_viii_continuation:wwN,
+% \str_decode_utf_viii_aux:wNnnwN
% }
-% In the \textsc{utf-8} encoding, bytes in the range
-% $[0,\hexnum{7F}]$ stand for themselves, bytes in the range
-% $[\hexnum{80},\hexnum{BF}]$ are continuation bytes,
-% and larger bytes must be followed by a number of continuation bytes.
-% Documentation to come.
-% ^^A todo: refuse "D800 -- "DFFF?
-% ^^A todo: document
+% \begin{macro}[aux, rEXP]
+% {\str_decode_utf_viii_overflow:w, \str_decode_utf_viii_end:}
+% Decoding is significantly harder than encoding. As before, lower
+% some flags, which are tested at the end (in bulk, to trigger at most
+% one \LaTeX3 error, as explained above). We expect successive
+% multi-byte sequences of the form \meta{start byte}
+% \meta{continuation bytes}. The \texttt{_start} auxiliary tests the
+% first byte:
+% \begin{itemize}
+% \item $[0,\hexnum{7F}]$: the byte stands alone, and is converted
+% to its own character code;
+% \item $[\hexnum{80}, \hexnum{BF}]$: unexpected continuation byte,
+% raise the appropriate flag, and convert that byte to the
+% replacement character \hexnum{FFFD};
+% \item $[\hexnum{C0}, \hexnum{FF}]$: this byte should be followed
+% by some continuation byte(s).
+% \end{itemize}
+% In the first two cases, \cs{use_none_delimit_by_q_stop:w} removes
+% data that only the third case requires, namely the limits of ranges
+% of Unicode characters which can be expressed with $1$, $2$, $3$, or
+% $4$ bytes.
+%
+% We can now concentrate on the multi-byte case and the
+% \texttt{_continuation} auxiliary. We expect |#3| to be in the range
+% $[\hexnum{80}, \hexnum{BF}]$. The test for this goes as follows: if
+% the character code is less than \hexnum{80}, we compare it to
+% $-\hexnum{C0}$, yielding \texttt{false}; otherwise to \hexnum{C0},
+% yielding \texttt{true} in the range $[\hexnum{80}, \hexnum{BF}]$ and
+% \texttt{false} otherwise. If we find that the byte is not a
+% continuation range, stop the current slew of bytes, output the
+% replacement character, and continue parsing with the \texttt{_start}
+% auxiliary, starting at the byte we just tested. Once we know that
+% the byte is a continuation byte, leave it behind us in the input
+% stream, compute what code point the bytes read so far would produce,
+% and feed that number to the \texttt{_aux} function.
+%
+% The \texttt{_aux} function tests whether we should look for more
+% continuation bytes or not. If the number it receives as |#1| is less
+% than the maximum |#4| for the current range, then we are done: check
+% for an overlong representation by comparing |#1| with the maximum
+% |#3| for the previous range. Otherwise, we call the
+% \texttt{_continuation} auxiliary again, after shifting the
+% \enquote{current code point} by |#4| (maximum from the range we just
+% checkedd).
+%
+% Two additional tests are needed: if we reach the end of the list of
+% range maxima and we are still not done, then we are faced with an
+% overflow. Clean up, and again insert the code point \hexnum{FFFD}
+% for the replacement character. Also, every time we read a byte, we
+% need to check whether we reached the end of the string. In a correct
+% \textsc{utf-8} string, this happens automatically when the
+% \texttt{_start} auxiliary leaves its first argument in the input
+% stream: the end-marker begins with \cs{prg_map_break:}, which ends
+% the loop. On the other hand, if the end is reached when looking for
+% a continuation byte, the \cs{use_none:n} |#3| construction removes
+% the first token from the end-marker, and leaves the \texttt{_end}
+% auxiliary, which raises the appropriate error flag before ending the
+% mapping.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { str_convert_from_utf8: }
+\cs_new_protected_nopar:cpn { str_convert_decode_utf8: }
{
+ \flag_clear:n { str_error }
+ \flag_clear:n { str_missing }
+ \flag_clear:n { str_extra }
+ \flag_clear:n { str_overlong }
+ \flag_clear:n { str_overflow }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl
- { \prg_map_break: \str_aux_from_utf_viii_error: }
+ \exp_after:wN \str_decode_utf_viii_start:N \g_str_result_tl
+ { \prg_map_break: \str_decode_utf_viii_end: }
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_error } { utf8-decode } { }
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1
+\cs_new:Npn \str_decode_utf_viii_start:N #1
{
#1
\if_num:w `#1 < "C0 \exp_stop_f:
@@ -2811,78 +2933,73 @@
\if_num:w `#1 < "80 \exp_stop_f:
\int_value:w `#1
\else:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-extra-conti } {#1}
+ \flag_raise:n { str_extra }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\fi:
\else:
- \if_num:w `#1 < "F5 \exp_stop_f:
- \exp_after:wN \str_aux_from_utf_viii:wwN
- \int_value:w \int_eval:n { `#1 - "C0 }
- \else:
- \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf8-invalid-byte } {#1}
- \int_use:N \c_str_replacement_char_int
- \fi:
+ \exp_after:wN \str_decode_utf_viii_continuation:wwN
+ \int_use:N \int_eval:w `#1 - "C0 \exp_after:wN \int_eval_end:
\fi:
\s_tl
\use_none_delimit_by_q_stop:w {"80} {"800} {"10000} {"110000} \q_stop
- \str_aux_from_utf_viii:N
+ \str_decode_utf_viii_start:N
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:wwN
- #1 \s_tl #2 \str_aux_from_utf_viii:N #3
+\cs_new:Npn \str_decode_utf_viii_continuation:wwN
+ #1 \s_tl #2 \str_decode_utf_viii_start:N #3
{
\use_none:n #3
- \if_num:w \int_eval:w ( `#3 + "20 ) / "40 = \c_three
+ \if_num:w `#3 < \if_num:w `#3 < "80 \exp_stop_f: - \fi: "C0 \exp_stop_f:
#3
- \exp_after:wN \str_aux_from_utf_viii:wNnnwN
- \int_value:w \int_eval:n { #1 * "40 + `#3 - "80 }
+ \exp_after:wN \str_decode_utf_viii_aux:wNnnwN
+ \int_use:N \int_eval:w
+ #1 * "40 + `#3 - "80
+ \exp_after:wN \int_eval_end:
\else:
\s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf8-missing-conti } {#3}
+ \flag_raise:n { str_missing }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\fi:
\s_tl
#2
- \str_aux_from_utf_viii:N #3
+ \str_decode_utf_viii_start:N #3
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:wNnnwN
- #1 \s_tl #2#3#4 #5 \str_aux_from_utf_viii:N #6
+\cs_new:Npn \str_decode_utf_viii_aux:wNnnwN
+ #1 \s_tl #2#3#4 #5 \str_decode_utf_viii_start:N #6
{
- \if_num:w #1 < #4
+ \if_num:w #1 < #4 \exp_stop_f:
\s_tl
\if_num:w #1 < #3 \exp_stop_f:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-overlong } {#1}
+ \flag_raise:n { str_overlong }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\else:
#1
\fi:
\else:
\if_meaning:w \q_stop #5
- \str_aux_from_utf_viii_overflow:w #1
+ \str_decode_utf_viii_overflow:w #1
\fi:
- \exp_after:wN \str_aux_from_utf_viii:wwN
- \int_value:w \int_eval:n { #1 - #4 }
+ \exp_after:wN \str_decode_utf_viii_continuation:wwN
+ \int_use:N \int_eval:w #1 - #4 \exp_after:wN \int_eval_end:
\fi:
\s_tl
#2 {#4} #5
- \str_aux_from_utf_viii:N
+ \str_decode_utf_viii_start:N
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii_overflow:w #1 \fi: #2 \fi:
+\cs_new:Npn \str_decode_utf_viii_overflow:w #1 \fi: #2 \fi:
{
\fi: \fi:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-overflow } {#1}
+ \flag_raise:n { str_overflow }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii_error:
+\cs_new_nopar:Npn \str_decode_utf_viii_end:
{
\s_tl
- \msg_expandable_kernel_error:nn
- { str } { utf8-premature-end }
+ \flag_raise:n { str_missing }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int \s_tl
\prg_map_break:
}
@@ -2897,9 +3014,8 @@
%
% \subsubsection{\textsc{utf-16} support}
%
-% The definitions are done in a category code regime where
-% the bytes $254$ and $255$ used by the byte order mark
-% have catcode \enquote{other}.
+% The definitions are done in a category code regime where the bytes
+% $254$ and $255$ used by the byte order mark have catcode~$12$.
% \begin{macrocode}
%<*utf16>
\group_begin:
@@ -2907,53 +3023,61 @@
\char_set_catcode_other:N \^^ff
% \end{macrocode}
%
-% \begin{macro}[int]{\str_convert_to_utf16:}
-% \begin{macro}[int]{\str_convert_to_utf16be:}
-% \begin{macro}[int]{\str_convert_to_utf16le:}
+% \begin{macro}[int]
+% {
+% \str_convert_encode_utf16: ,
+% \str_convert_encode_utf16be: ,
+% \str_convert_encode_utf16le: ,
+% }
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_to_utf_xvi:n,
-% \str_aux_to_utf_xvi_be:n,
-% \str_aux_to_utf_xvi_le:n
+% \str_encode_utf_xvi_aux:N ,
+% \str_encode_utf_xvi_char:n ,
% }
-% Convert characters one by one in a loop, with different
-% behaviours depending on the character code:
+% When the endianness is not specified, it is big-endian by default,
+% and we add a byte-order mark. Convert characters one by one in a
+% loop, with different behaviours depending on the character code.
% \begin{itemize}
-% \item $[0, 55295]$ converted to two bytes;
-% \item $[55296, 57343]$ cannot be converted, replaced by
-% the replacement character;
-% \item $[57344, 65535]$ converted to two bytes;
-% \item $[65536, 1114111]$ converted to a pair of surrogates,
-% each two bytes.
+% \item $[0, \hexnum{D7FF}]$: converted to two bytes;
+% \item $[\hexnum{D800}, \hexnum{DFFF}]$ are used as surrogates:
+% they cannot be converted and are replaced by the replacement
+% character;
+% \item $[\hexnum{E000}, \hexnum{FFFF}]$: converted to two bytes;
+% \item $[\hexnum{10000}, \hexnum{10FFFF}]$: converted to a pair of
+% surrogates, each two bytes.
% \end{itemize}
-% For the duration of this operation, \cs{str_tmp:w} is defined
-% as a function to convert a number in the range $[0, 65536]$
-% to a pair of bytes (either big endian or little endian).
+% For the duration of this operation, \cs{str_tmp:w} is defined as a
+% function to convert a number in the range $[0, \hexnum{FFFF}]$ to a
+% pair of bytes (either big endian or little endian), by feeding the
+% quotient of the division of |#1| by \hexnum{100}, followed by |#1|
+% to \cs{str_encode_utf_xvi_be:nn} or its \texttt{le} analog: those
+% compute the remainder, and output two bytes for the quotient and
+% remainder.
% \begin{macrocode}
- \cs_new_protected_nopar:cpx { str_convert_to_utf16: }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf16: }
{
- \exp_not:c { str_convert_to_utf16be: }
- \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } }
+ \str_encode_utf_xvi_aux:N \str_output_byte_pair_be:n
+ \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff }
}
- \cs_new_protected_nopar:cpn { str_convert_to_utf16be: }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf16be: }
+ { \str_encode_utf_xvi_aux:N \str_output_byte_pair_be:n }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf16le: }
+ { \str_encode_utf_xvi_aux:N \str_output_byte_pair_le:n }
+ \cs_new_protected:Npn \str_encode_utf_xvi_aux:N #1
{
- \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_be:n
- \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n
+ \flag_clear:n { str_error }
+ \cs_set_eq:NN \str_tmp:w #1
+ \str_convert_gmap_internal:N \str_encode_utf_xvi_char:n
+ \str_if_flag_error:nnx { str_error } { utf16-encode } { }
}
- \cs_new_protected_nopar:cpn { str_convert_to_utf16le: }
- {
- \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_le:n
- \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n
- }
- \cs_new_nopar:Npn \str_aux_to_utf_xvi:n #1
+ \cs_new:Npn \str_encode_utf_xvi_char:n #1
{
\if_int_compare:w #1 < "D800 \exp_stop_f:
\str_tmp:w {#1}
\else:
\if_int_compare:w #1 < "10000 \exp_stop_f:
\if_int_compare:w #1 < "E000 \exp_stop_f:
- \msg_expandable_kernel_error:nnn
- { str } { unicode-surrogate } {#1}
+ \flag_raise:n { str_error }
\str_tmp:w { \c_str_replacement_char_int }
\else:
\str_tmp:w {#1}
@@ -2964,120 +3088,208 @@
\fi:
\fi:
}
- \cs_new_nopar:Npn \str_aux_to_utf_xvi_be:n #1
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{variable}
+% {
+% \l_str_missing_flag ,
+% \l_str_extra_flag ,
+% \l_str_end_flag ,
+% }
+% When encoding a Unicode string to \textsc{utf-16}, only one error
+% can occur: code points in the range $[\hexnum{D800},
+% \hexnum{DFFF}]$, corresponding to surrogates, cannot be encoded. We
+% use the all-purpose flag \texttt{str_error} to signal that error.
+%
+% When decoding a Unicode string which is purportedly in
+% \textsc{utf-16}, three errors can occur: a missing trail surrogate,
+% an unexpected trail surrogate, and a string containing an odd number
+% of bytes.
+% \begin{macrocode}
+ \flag_clear_new:n { str_missing }
+ \flag_clear_new:n { str_extra }
+ \flag_clear_new:n { str_end }
+ \msg_kernel_new:nnnn { str } { utf16-encode }
+ { Unicode~string~cannot~be~expressed~in~UTF-16:~surrogate. }
+ {
+ Surrogate~code~points~(in~the~range~[U+D800,~U+DFFF])~
+ can~be~expressed~in~the~UTF-8~and~UTF-32~encodings,~
+ but~not~in~the~UTF-16~encoding.
+ }
+ \msg_kernel_new:nnnn { str } { utf16-decode }
{
- \str_output_byte:n { \int_div_truncate:nn {#1} {"100} }
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ Invalid~UTF-16~string: \exp_last_unbraced:Nf \use_none:n
+ \str_if_flag_times:nT { str_missing } { ,~missing~trail~surrogate }
+ \str_if_flag_times:nT { str_extra } { ,~extra~trail~surrogate }
+ \str_if_flag_times:nT { str_end } { ,~odd~number~of~bytes }
+ .
}
- \cs_new_nopar:Npn \str_aux_to_utf_xvi_le:n #1
{
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
- \str_output_byte:n { \int_div_truncate:nn {#1} {"100} }
+ In~the~UTF-16~encoding,~each~Unicode~character~is~encoded~as~
+ 2~or~4~bytes: \\
+ \iow_indent:n
+ {
+ Code~point~in~[U+0000,~U+D7FF]:~two~bytes \\
+ Code~point~in~[U+D800,~U+DFFF]:~illegal \\
+ Code~point~in~[U+E000,~U+FFFF]:~two~bytes \\
+ Code~point~in~[U+10000,~U+10FFFF]:~
+ a~lead~surrogate~and~a~trail~surrogate \\
+ }
+ Lead~surrogates~are~pairs~of~bytes~in~the~range~[0xD800,~0xDBFF],~
+ and~trail~surrogates~are~in~the~range~[0xDC00,~0xDFFF].
+ \flag_if_raised:nT { str_missing }
+ {
+ \\\\
+ A~lead~surrogate~was~not~followed~by~a~trail~surrogate.
+ }
+ \flag_if_raised:nT { str_extra }
+ {
+ \\\\
+ LaTeX~came~across~a~trail~surrogate~when~it~was~not~expected.
+ }
+ \flag_if_raised:nT { str_end }
+ {
+ \\\\
+ The~string~contained~an~odd~number~of~bytes.~This~is~invalid:~
+ the~basic~code~unit~for~UTF-16~is~16~bits~(2~bytes).
+ }
}
% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
+% \end{variable}
%
% \begin{macro}[int]
% {
-% \str_convert_from_utf16:,
-% \str_convert_from_utf16be:,
-% \str_convert_from_utf16le:
+% \str_convert_decode_utf16: ,
+% \str_convert_decode_utf16be: ,
+% \str_convert_decode_utf16le: ,
% }
-% \begin{macro}[aux]{\str_aux_from_utf_xvi_bom:NNw}
-% \begin{macro}[aux]{\str_aux_from_utf_xvi:No}
-% Define \cs{str_tmp:w} to take two arguments and return the
-% character code of the first one if the string is big-endian,
-% and the second one if the string is little-endian.
-% If the endianness is not known, look for a byte order mark
-% to decide.
+% \begin{macro}[aux]{\str_decode_utf_xvi_bom:NN, \str_decode_utf_xvi:Nw}
+% As for \textsc{utf-8}, decoding \textsc{utf-16} is harder than
+% encoding it. If the endianness is unknown, check the first two
+% bytes: if those are \hexnum{FE} and \hexnum{FF} in either order,
+% remove them and use the corresponding endianness, otherwise assume
+% big-endianness. The three endianness cases are based on a common
+% auxiliary whose first argument is $1$ for big-endian and $2$ for
+% little-endian, and whose second argument, delimited by the scan mark
+% \cs{s_stop}, is expanded once (the string may be long; passing
+% \cs{g_str_result_tl} as an argument before expansion is cheaper).
+%
+% The \cs{strr_decode_utf_xvi:Nw} function defines \cs{str_tmp:w} to
+% take two arguments and return the character code of the first one if
+% the string is big-endian, and the second one if the string is
+% little-endian, then loops over the string using
+% \cs{str_decode_utf_xvi_pair:NN} described below.
% \begin{macrocode}
- \cs_new_protected_nopar:cpn { str_convert_from_utf16be: }
- { \str_aux_from_utf_xvi:No 1 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf16le: }
- { \str_aux_from_utf_xvi:No 2 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf16: }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf16be: }
+ { \str_decode_utf_xvi:Nw 1 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf16le: }
+ { \str_decode_utf_xvi:Nw 2 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf16: }
{
- \exp_after:wN \str_aux_from_utf_xvi_bom:NNw
- \g_str_result_tl \scan_stop: \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xvi_bom:NN
+ \g_str_result_tl \s_stop \s_stop \s_stop
}
- \cs_new_protected_nopar:Npn
- \str_aux_from_utf_xvi_bom:NNw #1#2#3\scan_stop:
+ \cs_new_protected:Npn \str_decode_utf_xvi_bom:NN #1#2
{
- \str_if_eq:nnTF { #1#2 } { ^^ff ^^fe }
- { \str_aux_from_utf_xvi:No 2 {#3} }
+ \str_if_eq:xxTF { #1#2 } { ^^ff ^^fe }
+ { \str_decode_utf_xvi:Nw 2 }
{
- \str_if_eq:nnTF { #1#2 } { ^^fe ^^ff }
- { \str_aux_from_utf_xvi:No 1 {#3} }
- { \str_aux_from_utf_xvi:No 1 {#1#2#3} }
+ \str_if_eq:xxTF { #1#2 } { ^^fe ^^ff }
+ { \str_decode_utf_xvi:Nw 1 }
+ { \str_decode_utf_xvi:Nw 1 #1#2 }
}
}
- \cs_new_protected_nopar:Npn \str_aux_from_utf_xvi:No #1#2
+ \cs_new_protected:Npn \str_decode_utf_xvi:Nw #1#2 \s_stop
{
- \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
+ \flag_clear:n { str_error }
+ \flag_clear:n { str_missing }
+ \flag_clear:n { str_extra }
+ \flag_clear:n { str_end }
+ \cs_set:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_from_utf_xvi:NN
- #2 \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xvi_pair:NN
+ #2 \q_nil \q_nil
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_error } { utf16-decode } { }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NNwNNN}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNw}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNN}
-% When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w}
-% is such that |\str_tmp:w #1#2| expands to |`#1| if the string
-% is big-endian, and |`#2| if the string is little-endian.
-% If the code unit (a pair of characters) does not represent
-% a surrogate, the \cs{if_case:w} construction expands to nothing,
-% and we leave in the input stream \meta{characters} \cs{s_tl}
-% \meta{char code} \cs{s_tl}, and move on to the next code unit.
-% When the end is found, we check for the case of an odd-length
-% string (an error) and end the mapping.
-%
-% Surrogates are characterized by a most significant byte in the
-% ranges $[\hexnum{D8},\hexnum{DB}]$ and $[\hexnum{DC},\hexnum{DF}]$,
-% detected by dividing the character code by $4$ (\eTeX{} rounds to
-% nearest, ties away from zero). Here, a trail surrogate is invalid,
-% replaced by the replacement character. Lead surrogates are treated by
-% \cs{str_aux_from_utf_xvi:NNwNNN}, which grabs the next two
-% characters, checks for a premature end, and checks whether
-% the code unit is a trailing surrogate. If it is, leave
-% \meta{characters} \cs{s_tl} \meta{char code} \cs{s_tl} in the
-% input stream and move on to the next code unit. Otherwise, raise
-% an error, and continue parsing, replacing only the first code unit
-% (the leading surrogate) by the replacement character.
+% \begin{macro}[aux, rEXP]
+% {
+% \str_decode_utf_xvi_pair:NN ,
+% \str_decode_utf_xvi_quad:NNwNN ,
+% \str_decode_utf_xvi_pair_end:Nw ,
+% }
+% \begin{macro}[aux, rEXP]
+% {
+% \str_decode_utf_xvi_error:nNN ,
+% \str_decode_utf_xvi_extra:NNw ,
+% }
+% Bytes are read two at a time. At this stage, |\str_tmp:w #1#2|
+% expands to the character code of the most significant byte, and we
+% distinguish cases depending on which range it lies in:
+% \begin{itemize}
+% \item $[\hexnum{D8}, \hexnum{DB}]$ signals a lead surrogate, and
+% the integer expression yields $1$ (\eTeX{} rounds ties away from
+% zero);
+% \item $[\hexnum{DC}, \hexnum{DF}]$ signals a trail surrogate,
+% unexpected here, and the integer expression yields $2$;
+% \item any other value signals a code point in the Basic
+% Multilingual Plane, which stands for itself, and the
+% \cs{if_case:w} construction expands to nothing (cases other than
+% $1$ or $2$), leaving the relevant material in the input stream,
+% followed by another call to the \texttt{_pair} auxiliary.
+% \end{itemize}
+% The case of a lead surrogate is treated by the \texttt{_quad}
+% auxiliary, whose arguments |#1|, |#2|, |#4| and |#5| are the four
+% bytes. We expect the most significant byte of |#4#5| to be in the
+% range $[\hexnum{DC}, \hexnum{DF}]$ (trail surrogate). The test is
+% similar to the test used for continuation bytes in the
+% \textsc{utf-8} decoding functions. In the case where |#4#5| is
+% indeed a trail surrogate, leave |#1#2#4#5| \cs{s_tl}
+% \meta{code~point} \cs{s_tl}, and remove the pair |#4#5| before
+% looping with \cs{str_decode_utf_xvi_pair:NN}. Otherwise, of course,
+% complain about the missing surrogate.
+%
+% Every time we read a pair of bytes, we test for the end-marker
+% \cs{q_nil}. When reaching the end, we additionally check that the
+% string had an even length. Also, if the end is reached when
+% expecting a trail surrogate, we treat that as a missing surrogate.
% \begin{macrocode}
- \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2
+ \cs_new:Npn \str_decode_utf_xvi_pair:NN #1#2
{
- \if_meaning:w \scan_stop: #2
- \str_aux_from_utf_xvi_end:Nw #1
+ \if_meaning:w \q_nil #2
+ \str_decode_utf_xvi_pair_end:Nw #1
\fi:
\if_case:w
\int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
- \or: \exp_after:wN \str_aux_from_utf_xvi:NNwNNN
- \or: \exp_after:wN \str_aux_from_utf_xvi_error:NNw
+ \or: \exp_after:wN \str_decode_utf_xvi_quad:NNwNN
+ \or: \exp_after:wN \str_decode_utf_xvi_extra:NNw
\fi:
#1#2 \s_tl
\int_eval:n { "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 } \s_tl
- \str_aux_from_utf_xvi:NN
+ \str_decode_utf_xvi_pair:NN
}
- \cs_new:Npn \str_aux_from_utf_xvi:NNwNNN #1#2 \s_tl #3 \s_tl #4#5#6
+ \cs_new:Npn \str_decode_utf_xvi_quad:NNwNN
+ #1#2 #3 \str_decode_utf_xvi_pair:NN #4#5
{
- \if_meaning:w \scan_stop: #6
- \str_aux_from_utf_xvi_error:NNN #1#2#5
+ \if_meaning:w \q_nil #5
+ \str_decode_utf_xvi_error:nNN { missing } #1#2
+ \str_decode_utf_xvi_pair_end:Nw #4
\fi:
- \if_num:w \int_eval:w ( \str_tmp:w #5#6 - "DA ) / \c_four = \c_one
- #1 #2 #5 #6 \s_tl
+ \if_num:w
+ \if_num:w \str_tmp:w #4#5 < "DC \exp_stop_f:
+ \c_zero = \c_one
+ \else:
+ \str_tmp:w #4#5 < "E0 \exp_stop_f:
+ \fi:
+ #1 #2 #4 #5 \s_tl
\int_eval:n
{
( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400
@@ -3086,49 +3298,31 @@
\s_tl
\exp_after:wN \use_i:nnn
\else:
- \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl \s_tl
+ \str_decode_utf_xvi_error:nNN { missing } #1#2
\fi:
- #4#5#6
+ \str_decode_utf_xvi_pair:NN #4#5
}
- \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi:
+ \cs_new:Npn \str_decode_utf_xvi_pair_end:Nw #1 \fi:
{
\fi:
- \if_meaning:w \scan_stop: #1
+ \if_meaning:w \q_nil #1
\else:
- #1 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-odd } {#1}
- \int_use:N \c_str_replacement_char_int \s_tl
+ \str_decode_utf_xvi_error:nNN { end } #1 \prg_do_nothing:
\fi:
\prg_map_break:
}
- \cs_new:Npn \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl #3 \s_tl
+ \cs_new:Npn \str_decode_utf_xvi_extra:NNw #1#2 \s_tl #3 \s_tl
+ { \str_decode_utf_xvi_error:nNN { extra } #1#2 }
+ \cs_new:Npn \str_decode_utf_xvi_error:nNN #1#2#3
{
- #1 #2 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-surrogate } { #1#2 }
+ \flag_raise:n { str_error }
+ \flag_raise:n { str_#1 }
+ #2 #3 \s_tl
\int_use:N \c_str_replacement_char_int \s_tl
}
- \cs_new:Npn \str_aux_from_utf_xvi_error:NNN #1#2#3
- {
- \if_meaning:w \scan_stop: #3
- #1#2 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-surrogate } { #1#2 }
- \else:
- #1#2#3 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-odd } { #1#2#3 }
- \fi:
- \int_use:N \c_str_replacement_char_int \s_tl
- \prg_map_break:
- }
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% Restore the original catcodes of bytes $254$ and $255$.
% \begin{macrocode}
@@ -3138,9 +3332,9 @@
%
% \subsubsection{\textsc{utf-32} support}
%
-% The definitions are done in a category code regime where
-% the bytes $0$, $254$ and $255$ used by the byte order mark
-% have catcode \enquote{other}.
+% The definitions are done in a category code regime where the bytes
+% $0$, $254$ and $255$ used by the byte order mark have catcode
+% \enquote{other}.
% \begin{macrocode}
%<*utf32>
\group_begin:
@@ -3151,130 +3345,188 @@
%
% \begin{macro}[int]
% {
-% \str_convert_to_utf32:,
-% \str_convert_to_utf32be:,
-% \str_convert_to_utf32le:
+% \str_convert_encode_utf32: ,
+% \str_convert_encode_utf32be: ,
+% \str_convert_encode_utf32le: ,
% }
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_to_utf_xxxii_be:n,
-% \str_aux_to_utf_xxxii_le:n
+% \str_encode_utf_xxxii_be:n ,
+% \str_encode_utf_xxxii_be_aux:nn ,
+% \str_encode_utf_xxxii_le:n ,
+% \str_encode_utf_xxxii_le_aux:nn ,
% }
-% Convert each integer in the comma-list \cs{g_str_result_tl}
-% to a sequence of four bytes. The functions for big-endian
-% and little-endian encodings are very similar, but the
+% Convert each integer in the comma-list \cs{g_str_result_tl} to a
+% sequence of four bytes. The functions for big-endian and
+% little-endian encodings are very similar, but the
% \cs{str_output_byte:n} instructions are reversed.
% \begin{macrocode}
- \cs_new_protected_nopar:cpx { str_convert_to_utf32: }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf32: }
{
- \exp_not:c { str_convert_to_utf32be: }
- \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } }
+ \str_convert_gmap_internal:N \str_encode_utf_xxxii_be:n
+ \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff }
}
- \cs_new_protected_nopar:cpn { str_convert_to_utf32be: }
- { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_be:n }
- \cs_new_protected_nopar:cpn { str_convert_to_utf32le: }
- { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_le:n }
- \cs_new:Npn \str_aux_to_utf_xxxii_be:n #1
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf32be: }
+ { \str_convert_gmap_internal:N \str_encode_utf_xxxii_be:n }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf32le: }
+ { \str_convert_gmap_internal:N \str_encode_utf_xxxii_le:n }
+ \cs_new:Npn \str_encode_utf_xxxii_be:n #1
+ {
+ \exp_args:Nf \str_encode_utf_xxxii_be_aux:nn
+ { \int_div_truncate:nn {#1} { "100 } } {#1}
+ }
+ \cs_new:Npn \str_encode_utf_xxxii_be_aux:nn #1#2
{
^^00
- \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } }
- \str_output_byte:n
- { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} }
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ \str_output_byte_pair_be:n {#1}
+ \str_output_byte:n { #2 - #1 * "100 }
}
- \cs_new:Npn \str_aux_to_utf_xxxii_le:n #1
+ \cs_new:Npn \str_encode_utf_xxxii_le:n #1
{
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
- \str_output_byte:n
- { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} }
- \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } }
+ \exp_args:Nf \str_encode_utf_xxxii_le_aux:nn
+ { \int_div_truncate:nn {#1} { "100 } } {#1}
+ }
+ \cs_new:Npn \str_encode_utf_xxxii_le_aux:nn #1#2
+ {
+ \str_output_byte:n { #2 - #1 * "100 }
+ \str_output_byte_pair_le:n {#1}
^^00
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
+% \begin{variable}{str_overflow, str_end}
+% There can be no error when encoding in \textsc{utf-32}. When
+% decoding, the string may not have length $4n$, or it may contain
+% code points larger than \hexnum{10FFFF}. The latter case often
+% happens if the encoding was in fact not \textsc{utf-32}, because
+% most arbitrary strings are not valid in \textsc{utf-32}.
+% \begin{macrocode}
+ \flag_clear_new:n { str_overflow }
+ \flag_clear_new:n { str_end }
+ \msg_kernel_new:nnnn { str } { utf32-decode }
+ {
+ Invalid~UTF-32~string: \exp_last_unbraced:Nf \use_none:n
+ \str_if_flag_times:nT { str_overflow } { ,~code~point~too~large }
+ \str_if_flag_times:nT { str_end } { ,~truncated~string }
+ .
+ }
+ {
+ In~the~UTF-32~encoding,~every~Unicode~character~
+ (in~the~range~[U+0000,~U+10FFFF])~is~encoded~as~4~bytes.
+ \flag_if_raised:nT { str_overflow }
+ {
+ \\\\
+ LaTeX~came~across~a~code~point~larger~than~1114111,~
+ the~maximum~code~point~defined~by~Unicode.~
+ Perhaps~the~string~was~not~encoded~in~the~UTF-32~encoding?
+ }
+ \flag_if_raised:nT { str_end }
+ {
+ \\\\
+ The~length~of~the~string~is~not~a~multiple~of~4.~
+ Perhaps~the~string~was~truncated?
+ }
+ }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}[int]
% {
-% \str_convert_from_utf32:,
-% \str_convert_from_utf32be:,
-% \str_convert_from_utf32le:
+% \str_convert_decode_utf32: ,
+% \str_convert_decode_utf32be: ,
+% \str_convert_decode_utf32le: ,
% }
% \begin{macro}[aux]
-% {
-% \str_aux_from_utf_xxxii_bom:NNNNw,
-% \str_aux_from_utf_xxxii:No
-% }
+% {\str_decode_utf_xxxii_bom:NNNN, \str_decode_utf_xxxii:Nw}
% \begin{macro}[aux, rEXP]
-% {
-% \str_aux_from_utf_xxxii:NNNN,
-% \str_aux_from_utf_xxxii_end:w
-% }
-% See the conversion functions from \textsc{utf-16} encodings.
-% The \cs{str_aux_from_utf_xxxii:No} auxiliary defines
-% \cs{str_tmp:w} to take two arguments and give the character
-% code of either the first argument (big-endian) or the second
-% argument (little-endian). Then we loop over the string,
-% four bytes at a time, check for overflow, and otherwise
-% produce the number corresponding to the four bytes with
-% the given endianness.
+% {\str_decode_utf_xxxii_loop:NNNN, \str_decode_utf_xxxii_end:w}
+%
+% The structure is similar to \textsc{utf-16} decoding functions. If
+% the endianness is not given, test the first $4$ bytes of the string
+% (possibly \cs{s_stop} if the string is too short) for the presence
+% of a byte-order mark. If there is a byte-order mark, use that
+% endianness, and remove the $4$ bytes, otherwise default to
+% big-endian, and leave the $4$ bytes in place. The
+% \cs{str_decode_utf_xxxii:Nw} auxiliary recieves $1$ or $2$ as its
+% first argument indicating endianness, and the string to convert as
+% its second argument (expanded or not). It sets \cs{str_tmp:w} to
+% expand to the character code of either of its two arguments
+% depending on endianness, then triggers the \texttt{_loop} auxiliary
+% inside an \texttt{x}-expanding assignment to \cs{g_str_result_tl}.
+%
+% The \texttt{_loop} auxiliary first checks for the end-of-string
+% marker \cs{s_stop}, calling the \texttt{_end} auxiliary if
+% appropriate. Otherwise, leave the \meta{4~bytes} \cs{s_tl} behind,
+% then check that the code point is not overflowing: the leading byte
+% must be $0$, and the following byte at most $16$.
+%
+% In the ending code, we check that there remains no byte: there
+% should be nothing left until the first \cs{s_stop}. Break the map.
% \begin{macrocode}
- \cs_new_protected_nopar:cpn { str_convert_from_utf32be: }
- { \str_aux_from_utf_xxxii:No 1 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf32le: }
- { \str_aux_from_utf_xxxii:No 2 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf32: }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf32be: }
+ { \str_decode_utf_xxxii:Nw 1 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf32le: }
+ { \str_decode_utf_xxxii:Nw 2 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf32: }
{
- \exp_after:wN \str_aux_from_utf_xxxii_bom:NNNNw \g_str_result_tl
- \scan_stop: \scan_stop: \scan_stop: \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xxxii_bom:NNNN \g_str_result_tl
+ \s_stop \s_stop \s_stop \s_stop \s_stop
}
- \cs_new_protected_nopar:Npn
- \str_aux_from_utf_xxxii_bom:NNNNw #1#2#3#4#5\scan_stop:
+ \cs_new_protected:Npn \str_decode_utf_xxxii_bom:NNNN #1#2#3#4
{
- \str_if_eq:nnTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 }
- { \str_aux_from_utf_xxxii:No 2 {#5} }
+ \str_if_eq:xxTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 }
+ { \str_decode_utf_xxxii:Nw 2 }
{
- \str_if_eq:nnTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff }
- { \str_aux_from_utf_xxxii:No 1 {#5} }
- { \str_aux_from_utf_xxxii:No 1 {#1#2#3#4#5} }
+ \str_if_eq:xxTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff }
+ { \str_decode_utf_xxxii:Nw 1 }
+ { \str_decode_utf_xxxii:Nw 1 #1#2#3#4 }
}
}
- \cs_new_protected_nopar:Npn \str_aux_from_utf_xxxii:No #1#2
+ \cs_new_protected:Npn \str_decode_utf_xxxii:Nw #1#2 \s_stop
{
- \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
+ \flag_clear:n { str_overflow }
+ \flag_clear:n { str_end }
+ \flag_clear:n { str_error }
+ \cs_set:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_from_utf_xxxii:NNNN
- #2 \scan_stop: \scan_stop: \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xxxii_loop:NNNN
+ #2 \s_stop \s_stop \s_stop \s_stop
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_error } { utf32-decode } { }
}
- \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4
+ \cs_new:Npn \str_decode_utf_xxxii_loop:NNNN #1#2#3#4
{
- \if_meaning:w \scan_stop: #4
- \exp_after:wN \str_aux_from_utf_xxxii_end:w
+ \if_meaning:w \s_stop #4
+ \exp_after:wN \str_decode_utf_xxxii_end:w
\fi:
#1#2#3#4 \s_tl
- \if_num:w \int_eval:w
- \str_tmp:w #1#4 * \c_two_hundred_fifty_six
- + \str_tmp:w #2#3
- > \c_sixteen
- \msg_expandable_kernel_error:nnn
- { str } { utf32-overflow } { #1#2#3#4 }
+ \if_num:w \str_tmp:w #1#4 > \c_zero
+ \flag_raise:n { str_overflow }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\else:
- \int_eval:n
- { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 }
+ \if_num:w \str_tmp:w #2#3 > \c_sixteen
+ \flag_raise:n { str_overflow }
+ \flag_raise:n { str_error }
+ \int_use:N \c_str_replacement_char_int
+ \else:
+ \int_eval:n
+ { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 }
+ \fi:
\fi:
\s_tl
- \str_aux_from_utf_xxxii:NNNN
+ \str_decode_utf_xxxii_loop:NNNN
}
- \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop:
+ \cs_new:Npn \str_decode_utf_xxxii_end:w #1 \s_stop
{
\tl_if_empty:nF {#1}
{
- \msg_expandable_kernel_error:nnn
- { str } { utf32-truncated } { #1 }
+ \flag_raise:n { str_end }
+ \flag_raise:n { str_error }
#1 \s_tl
\int_use:N \c_str_replacement_char_int \s_tl
}
@@ -3292,19 +3544,15 @@
% \end{macrocode}
%
% \subsubsection{\textsc{iso 8859} support}
-%^^A todo: document
%
-% The \textsc{iso-8859-1} encoding exactly matches with the $256$
-% first Unicode characters. For other 8-bit encodings
-% of the \textsc{iso-8859} family, we keep track only of
-% differences, and of unassigned bytes.
+% The \textsc{iso-8859-1} encoding exactly matches with the $256$ first
+% Unicode characters. For other 8-bit encodings of the \textsc{iso-8859}
+% family, we keep track only of differences, and of unassigned bytes.
% \begin{macrocode}
%<*iso88591>
-\str_encoding_eight_bit:n { iso88591 }
-\tl_const:cn { c_str_encoding_iso88591_tl }
+\str_declare_eight_bit_encoding:nnn { iso88591 }
{
}
-\tl_const:cn { c_str_encoding_iso88591_missing_tl }
{
}
%</iso88591>
@@ -3312,8 +3560,7 @@
%
% \begin{macrocode}
%<*iso88592>
-\str_encoding_eight_bit:n { iso88592 }
-\tl_const:cn { c_str_encoding_iso88592_tl }
+\str_declare_eight_bit_encoding:nnn { iso88592 }
{
{ A1 } { 0104 }
{ A2 } { 02D8 }
@@ -3373,7 +3620,6 @@
{ FE } { 0163 }
{ FF } { 02D9 }
}
-\tl_const:cn { c_str_encoding_iso88592_missing_tl }
{
}
%</iso88592>
@@ -3381,8 +3627,7 @@
%
% \begin{macrocode}
%<*iso88593>
-\str_encoding_eight_bit:n { iso88593 }
-\tl_const:cn { c_str_encoding_iso88593_tl }
+\str_declare_eight_bit_encoding:nnn { iso88593 }
{
{ A1 } { 0126 }
{ A2 } { 02D8 }
@@ -3413,7 +3658,6 @@
{ FE } { 015D }
{ FF } { 02D9 }
}
-\tl_const:cn { c_str_encoding_iso88593_missing_tl }
{
{ A5 }
{ AE }
@@ -3428,8 +3672,7 @@
%
% \begin{macrocode}
%<*iso88594>
-\str_encoding_eight_bit:n { iso88594 }
-\tl_const:cn { c_str_encoding_iso88594_tl }
+\str_declare_eight_bit_encoding:nnn { iso88594 }
{
{ A1 } { 0104 }
{ A2 } { 0138 }
@@ -3482,7 +3725,6 @@
{ FE } { 016B }
{ FF } { 02D9 }
}
-\tl_const:cn { c_str_encoding_iso88594_missing_tl }
{
}
%</iso88594>
@@ -3490,8 +3732,7 @@
%
% \begin{macrocode}
%<*iso88595>
-\str_encoding_eight_bit:n { iso88595 }
-\tl_const:cn { c_str_encoding_iso88595_tl }
+\str_declare_eight_bit_encoding:nnn { iso88595 }
{
{ A1 } { 0401 }
{ A2 } { 0402 }
@@ -3588,7 +3829,6 @@
{ FE } { 045E }
{ FF } { 045F }
}
-\tl_const:cn { c_str_encoding_iso88595_missing_tl }
{
}
%</iso88595>
@@ -3596,8 +3836,7 @@
%
% \begin{macrocode}
%<*iso88596>
-\str_encoding_eight_bit:n { iso88596 }
-\tl_const:cn { c_str_encoding_iso88596_tl }
+\str_declare_eight_bit_encoding:nnn { iso88596 }
{
{ AC } { 060C }
{ BB } { 061B }
@@ -3648,7 +3887,6 @@
{ F1 } { 0651 }
{ F2 } { 0652 }
}
-\tl_const:cn { c_str_encoding_iso88596_missing_tl }
{
{ A1 }
{ A2 }
@@ -3688,8 +3926,7 @@
%
% \begin{macrocode}
%<*iso88597>
-\str_encoding_eight_bit:n { iso88597 }
-\tl_const:cn { c_str_encoding_iso88597_tl }
+\str_declare_eight_bit_encoding:nnn { iso88597 }
{
{ A1 } { 2018 }
{ A2 } { 2019 }
@@ -3769,7 +4006,6 @@
{ FD } { 03CD }
{ FE } { 03CE }
}
-\tl_const:cn { c_str_encoding_iso88597_missing_tl }
{
{ AE }
{ D2 }
@@ -3779,8 +4015,7 @@
%
% \begin{macrocode}
%<*iso88598>
-\str_encoding_eight_bit:n { iso88598 }
-\tl_const:cn { c_str_encoding_iso88598_tl }
+\str_declare_eight_bit_encoding:nnn { iso88598 }
{
{ AA } { 00D7 }
{ BA } { 00F7 }
@@ -3815,7 +4050,6 @@
{ FD } { 200E }
{ FE } { 200F }
}
-\tl_const:cn { c_str_encoding_iso88598_missing_tl }
{
{ A1 }
{ BF }
@@ -3858,8 +4092,7 @@
%
% \begin{macrocode}
%<*iso88599>
-\str_encoding_eight_bit:n { iso88599 }
-\tl_const:cn { c_str_encoding_iso88599_tl }
+\str_declare_eight_bit_encoding:nnn { iso88599 }
{
{ D0 } { 011E }
{ DD } { 0130 }
@@ -3868,7 +4101,6 @@
{ FD } { 0131 }
{ FE } { 015F }
}
-\tl_const:cn { c_str_encoding_iso88599_missing_tl }
{
}
%</iso88599>
@@ -3876,8 +4108,7 @@
%
% \begin{macrocode}
%<*iso885910>
-\str_encoding_eight_bit:n { iso885910 }
-\tl_const:cn { c_str_encoding_iso885910_tl }
+\str_declare_eight_bit_encoding:nnn { iso885910 }
{
{ A1 } { 0104 }
{ A2 } { 0112 }
@@ -3926,7 +4157,6 @@
{ F9 } { 0173 }
{ FF } { 0138 }
}
-\tl_const:cn { c_str_encoding_iso885910_missing_tl }
{
}
%</iso885910>
@@ -3934,8 +4164,7 @@
%
% \begin{macrocode}
%<*iso885911>
-\str_encoding_eight_bit:n { iso885911 }
-\tl_const:cn { c_str_encoding_iso885911_tl }
+\str_declare_eight_bit_encoding:nnn { iso885911 }
{
{ A1 } { 0E01 }
{ A2 } { 0E02 }
@@ -4025,7 +4254,6 @@
{ FA } { 0E5A }
{ FB } { 0E5B }
}
-\tl_const:cn { c_str_encoding_iso885911_missing_tl }
{
{ DB }
{ DC }
@@ -4037,8 +4265,7 @@
%
% \begin{macrocode}
%<*iso885913>
-\str_encoding_eight_bit:n { iso885913 }
-\tl_const:cn { c_str_encoding_iso885913_tl }
+\str_declare_eight_bit_encoding:nnn { iso885913 }
{
{ A1 } { 201D }
{ A5 } { 201E }
@@ -4097,7 +4324,6 @@
{ FE } { 017E }
{ FF } { 2019 }
}
-\tl_const:cn { c_str_encoding_iso885913_missing_tl }
{
}
%</iso885913>
@@ -4105,8 +4331,7 @@
%
% \begin{macrocode}
%<*iso885914>
-\str_encoding_eight_bit:n { iso885914 }
-\tl_const:cn { c_str_encoding_iso885914_tl }
+\str_declare_eight_bit_encoding:nnn { iso885914 }
{
{ A1 } { 1E02 }
{ A2 } { 1E03 }
@@ -4140,7 +4365,6 @@
{ F7 } { 1E6B }
{ FE } { 0177 }
}
-\tl_const:cn { c_str_encoding_iso885914_missing_tl }
{
}
%</iso885914>
@@ -4148,8 +4372,7 @@
%
% \begin{macrocode}
%<*iso885915>
-\str_encoding_eight_bit:n { iso885915 }
-\tl_const:cn { c_str_encoding_iso885915_tl }
+\str_declare_eight_bit_encoding:nnn { iso885915 }
{
{ A4 } { 20AC }
{ A6 } { 0160 }
@@ -4160,7 +4383,6 @@
{ BD } { 0153 }
{ BE } { 0178 }
}
-\tl_const:cn { c_str_encoding_iso885915_missing_tl }
{
}
%</iso885915>
@@ -4168,8 +4390,7 @@
%
% \begin{macrocode}
%<*iso885916>
-\str_encoding_eight_bit:n { iso885916 }
-\tl_const:cn { c_str_encoding_iso885916_tl }
+\str_declare_eight_bit_encoding:nnn { iso885916 }
{
{ A1 } { 0104 }
{ A2 } { 0105 }
@@ -4212,7 +4433,6 @@
{ FD } { 0119 }
{ FE } { 021B }
}
-\tl_const:cn { c_str_encoding_iso885916_missing_tl }
{
}
%</iso885916>