summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/l3kernel/l3token.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3token.dtx')
-rw-r--r--macros/latex/contrib/l3kernel/l3token.dtx229
1 files changed, 223 insertions, 6 deletions
diff --git a/macros/latex/contrib/l3kernel/l3token.dtx b/macros/latex/contrib/l3kernel/l3token.dtx
index 676c55747a..6105f3aab0 100644
--- a/macros/latex/contrib/l3kernel/l3token.dtx
+++ b/macros/latex/contrib/l3kernel/l3token.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -754,14 +754,42 @@
% this includes primitive-like commands defined using |{token.set_lua}|.
% \end{function}
%
+% \begin{function}[added = 2020-12-03, EXP, noTF]
+% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn}
+% \begin{syntax}
+% \cs{token_case_meaning:NnTF} \meta{test token} \\
+% ~~"{" \\
+% ~~~~\meta{token case_1} \Arg{code case_1} \\
+% ~~~~\meta{token case_2} \Arg{code case_2} \\
+% ~~~~\ldots \\
+% ~~~~\meta{token case_n} \Arg{code case_n} \\
+% ~~"}" \\
+% ~~\Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% This function compares the \meta{test token} in turn with each of
+% the \meta{token cases}. If the two are equal (as described for
+% \cs{token_if_eq_catcode:NNTF}, \cs{token_if_eq_charcode:NNTF} and
+% \cs{token_if_eq_meaning:NNTF}, respectively) then the associated
+% \meta{code} is left in the input stream and other cases are
+% discarded. If any of the cases are matched, the \meta{true code} is
+% also inserted into the input stream (after the code for the
+% appropriate case), while if none match then the \meta{false code} is
+% inserted. The functions \cs{token_case_catcode:Nn},
+% \cs{token_case_charcode:Nn}, and \cs{token_case_meaning:Nn}, which
+% do nothing if there is no match, are also available.
+% \end{function}
+%
% \section{Peeking ahead at the next token}
%
% There is often a need to look ahead at the next token in the input
% stream while leaving it in place. This is handled using the
-% \enquote{peek} functions. The generic \cs{peek_after:Nw} is
-% provided along with a family of predefined tests for common cases.
-% As peeking ahead does \emph{not} skip spaces the predefined tests
-% include both a space-respecting and space-skipping version.
+% \enquote{peek} functions. The generic \cs{peek_after:Nw} is provided
+% along with a family of predefined tests for common cases. As peeking
+% ahead does \emph{not} skip spaces the predefined tests include both a
+% space-respecting and space-skipping version. In addition, using
+% \cs{peek_analysis_map_inline:n}, one can map through the following
+% tokens in the input stream and repeatedly perform some tests.
%
% \begin{function}{\peek_after:Nw}
% \begin{syntax}
@@ -980,6 +1008,128 @@
% (as appropriate to the result of the test).
% \end{function}
%
+% \begin{function}[added = 2020-12-03]{\peek_analysis_map_inline:n}
+% \begin{syntax}
+% \cs{peek_analysis_map_inline:n} \Arg{inline function}
+% \end{syntax}
+% Repeatedly removes one \meta{token} from the input stream and
+% applies the \meta{inline function} to it, until
+% \cs{peek_analysis_map_break:} is called. The \meta{inline function}
+% receives three arguments for each \meta{token} in the input stream:
+% \begin{itemize}
+% \item \meta{tokens}, which both \texttt{o}-expand and
+% \texttt{x}-expand to the \meta{token}. The detailed form of
+% \meta{tokens} may change in later releases.
+% \item \meta{char code}, a decimal representation of the character
+% code of the \meta{token}, $-1$ if it is a control sequence.
+% \item \meta{catcode}, a capital hexadecimal digit which denotes the
+% category code of the \meta{token} (0:~control sequence,
+% 1:~begin-group, 2:~end-group, 3:~math shift, 4:~alignment tab,
+% 6:~parameter, 7:~superscript, 8:~subscript, A:~space, B:~letter,
+% C:~other, D:~active). This can be converted to an integer by
+% writing |"|\meta{catcode}.
+% \end{itemize}
+% These arguments are the same as for \cs{tl_analysis_map_inline:nn}
+% defined in \pkg{l3tl-analysis}. The \meta{char code} and
+% \meta{catcode} do not take the meaning of a control sequence or
+% active character into account: for instance, upon encountering the
+% token \cs{c_group_begin_token} in the input stream,
+% \cs{peek_analysis_map_inline:n} calls the \meta{inline function}
+% with |#1| being \cs{exp_not:n} |{| \cs{c_group_begin_token} |}|
+% (with the current implementation),
+% |#2|~being~$-1$, and
+% |#3|~being~$0$, as for any other control sequence. In contrast,
+% upon encountering an explicit begin-group token~|{|, % ^^A |}|
+% the \meta{inline function} is called with arguments
+% \cs{exp_after:wN} |{| \cs{if_false:} |}| \cs{fi:}, $123$ and~$1$.
+%
+% The mapping is done at the current group level, \emph{i.e.}~any
+% local assignments made by the \meta{inline function} remain in
+% effect after the loop. Within the code, \cs{l_peek_token} is set
+% equal (as a token, not a token list) to the token under
+% consideration.
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03]
+% {\peek_analysis_map_break:, \peek_analysis_map_break:n}
+% \begin{syntax}
+% \cs{peek_analysis_map_inline:n}
+% |{| \dots{} \cs{peek_analysis_map_break:n} \Arg{code} |}|
+% \end{syntax}
+% Stops the \cs{peek_analysis_map_inline:n} loop from seeking more
+% tokens, and inserts \meta{code} in the input stream (empty for
+% \cs{peek_analysis_map_break:}).
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03, TF]{\peek_regex:n, \peek_regex:N}
+% \begin{syntax}
+% \cs{peek_regex:nTF} \Arg{regex} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{tokens} that follow in the input stream match the
+% \meta{regular expression}. Any \meta{tokens} that have been read
+% are left in the input stream after the \meta{true code} or
+% \meta{false code} (as appropriate to the result of the test). See
+% \pkg{l3regex} for documentation of the syntax of regular
+% expressions. The \meta{regular expression} is implicitly anchored
+% at the start, so for instance \cs{peek_regex:nTF}~|{|~|a|~|}| is
+% essentially equivalent to \cs{peek_charcode:NTF}~|a|.
+% \begin{texnote}
+% Implicit character tokens are correctly considered by
+% \cs{peek_regex:nTF} as control sequences, while functions that
+% inspect individual tokens (for instance \cs{peek_charcode:NTF})
+% only take into account their meaning.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03, TF]
+% {\peek_regex_remove_once:n, \peek_regex_remove_once:N}
+% \begin{syntax}
+% \cs{peek_regex_remove_once:nTF} \Arg{regex} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{tokens} that follow in the input stream match the
+% \meta{regex}. If the test is true, the \meta{tokens} are removed
+% from the input stream and the \meta{true code} is inserted, while if
+% the test is false, the \meta{false code} is inserted followed by the
+% \meta{tokens} that were originally in the input stream.
+% See \pkg{l3regex} for documentation of the syntax of
+% regular expressions. The \meta{regular expression} is implicitly
+% anchored at the start, so for instance
+% \cs{peek_regex_remove_once:nTF}~|{|~|a|~|}| is essentially equivalent to
+% \cs{peek_charcode_remove:NTF}~|a|.
+% \begin{texnote}
+% Implicit character tokens are correctly considered by
+% \cs{peek_regex_remove_once:nTF} as control sequences, while functions
+% that inspect individual tokens (for instance
+% \cs{peek_charcode:NTF}) only take into account their meaning.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03, noTF]
+% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn}
+% \begin{syntax}
+% \cs{peek_regex_replace_once:nnTF} \Arg{regex} \Arg{replacement} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% If the \meta{tokens} that follow in the input stream match the
+% \meta{regex}, replaces them according to the \meta{replacement} as
+% for \cs{regex_replace_once:nnN}, and leaves the result in the input
+% stream, after the \meta{true code}. Otherwise, leaves \meta{false
+% code} followed by the \meta{tokens} that were originally in the
+% input stream, with no modifications. See \pkg{l3regex} for
+% documentation of the syntax of regular expressions and of the
+% \meta{replacement}: for instance |\0| in the \meta{replacement} is
+% replaced by the tokens that were matched in the input stream. The
+% \meta{regular expression} is implicitly anchored at the start. In
+% contrast to \cs{regex_replace_once:nnN}, no error arises if the
+% \meta{replacement} leads to an unbalanced token list: the tokens are
+% inserted into the input stream without issue.
+% \begin{texnote}
+% Implicit character tokens are correctly considered by
+% \cs{peek_regex_replace_once:nnTF} as control sequences, while
+% functions that inspect individual tokens (for instance
+% \cs{peek_charcode:NTF}) only take into account their meaning.
+% \end{texnote}
+% \end{function}
+%
% \section{Description of all possible tokens}
% \label{sec:l3token:all-tokens}
%
@@ -1897,9 +2047,10 @@
%<@@=token>
% \end{macrocode}
%
-% \begin{variable}{\s_@@_stop}
+% \begin{variable}{\s_@@_mark, \s_@@_stop}
% Internal scan marks.
% \begin{macrocode}
+\scan_new:N \s_@@_mark
\scan_new:N \s_@@_stop
% \end{macrocode}
% \end{variable}
@@ -2566,6 +2717,72 @@ end
% \end{macro}
% \end{macro}
%
+% \begin{macro}[EXP, noTF]
+% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn}
+% \begin{macro}[EXP]{\@@_case:NNnTF, \@@_case:NNw, \@@_case_end:nw}
+% The aim here is to allow the case statement to be evaluated
+% using a known number of expansion steps (two), and without
+% needing to use an explicit \enquote{end of recursion} marker.
+% That is achieved by using the test input as the final case,
+% as this is always true. The trick is then to tidy up
+% the output such that the appropriate case code plus either
+% the \texttt{true} or \texttt{false} branch code is inserted.
+% \begin{macrocode}
+\cs_new:Npn \token_case_catcode:Nn #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } { } }
+\cs_new:Npn \token_case_catcode:NnT #1#2#3
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} {#3} { } }
+\cs_new:Npn \token_case_catcode:NnF #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } }
+\cs_new:Npn \token_case_catcode:NnTF
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF }
+\cs_new:Npn \token_case_charcode:Nn #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } { } }
+\cs_new:Npn \token_case_charcode:NnT #1#2#3
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} {#3} { } }
+\cs_new:Npn \token_case_charcode:NnF #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } }
+\cs_new:Npn \token_case_charcode:NnTF
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF }
+\cs_new:Npn \token_case_meaning:Nn #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } { } }
+\cs_new:Npn \token_case_meaning:NnT #1#2#3
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} {#3} { } }
+\cs_new:Npn \token_case_meaning:NnF #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } }
+\cs_new:Npn \token_case_meaning:NnTF
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF }
+\cs_new:Npn \@@_case:NNnTF #1#2#3#4#5
+ {
+ \@@_case:NNw #1 #2 #3 #2 { }
+ \s_@@_mark {#4}
+ \s_@@_mark {#5}
+ \s_@@_stop
+ }
+\cs_new:Npn \@@_case:NNw #1#2#3#4
+ {
+ #1 #2 #3
+ { \@@_case_end:nw {#4} }
+ { \@@_case:NNw #1 #2 }
+ }
+% \end{macrocode}
+% To tidy up the recursion, there are two outcomes. If there was a hit to
+% one of the cases searched for, then |#1| is the code to insert,
+% |#2| is the \emph{next} case to check on and |#3| is all of
+% the rest of the cases code. That means that |#4| is the \texttt{true}
+% branch code, and |#5| tidies up the spare \cs{s_@@_mark} and the
+% \texttt{false} branch. On the other hand, if none of the cases matched
+% then we arrive here using the \enquote{termination} case of comparing
+% the search with itself. That means that |#1| is empty, |#2| is
+% the first \cs{s_@@_mark} and so |#4| is the \texttt{false} code (the
+% \texttt{true} code is mopped up by |#3|).
+% \begin{macrocode}
+\cs_new:Npn \@@_case_end:nw #1#2#3 \s_@@_mark #4#5 \s_@@_stop
+ { \exp_end: #1 #4 }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
% \subsection{Peeking ahead at the next token}
%
% \begin{macrocode}