diff options
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3token.dtx')
-rw-r--r-- | macros/latex/contrib/l3kernel/l3token.dtx | 229 |
1 files changed, 223 insertions, 6 deletions
diff --git a/macros/latex/contrib/l3kernel/l3token.dtx b/macros/latex/contrib/l3kernel/l3token.dtx index 676c55747a..6105f3aab0 100644 --- a/macros/latex/contrib/l3kernel/l3token.dtx +++ b/macros/latex/contrib/l3kernel/l3token.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -754,14 +754,42 @@ % this includes primitive-like commands defined using |{token.set_lua}|. % \end{function} % +% \begin{function}[added = 2020-12-03, EXP, noTF] +% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn} +% \begin{syntax} +% \cs{token_case_meaning:NnTF} \meta{test token} \\ +% ~~"{" \\ +% ~~~~\meta{token case_1} \Arg{code case_1} \\ +% ~~~~\meta{token case_2} \Arg{code case_2} \\ +% ~~~~\ldots \\ +% ~~~~\meta{token case_n} \Arg{code case_n} \\ +% ~~"}" \\ +% ~~\Arg{true code} +% ~~\Arg{false code} +% \end{syntax} +% This function compares the \meta{test token} in turn with each of +% the \meta{token cases}. If the two are equal (as described for +% \cs{token_if_eq_catcode:NNTF}, \cs{token_if_eq_charcode:NNTF} and +% \cs{token_if_eq_meaning:NNTF}, respectively) then the associated +% \meta{code} is left in the input stream and other cases are +% discarded. If any of the cases are matched, the \meta{true code} is +% also inserted into the input stream (after the code for the +% appropriate case), while if none match then the \meta{false code} is +% inserted. The functions \cs{token_case_catcode:Nn}, +% \cs{token_case_charcode:Nn}, and \cs{token_case_meaning:Nn}, which +% do nothing if there is no match, are also available. +% \end{function} +% % \section{Peeking ahead at the next token} % % There is often a need to look ahead at the next token in the input % stream while leaving it in place. This is handled using the -% \enquote{peek} functions. The generic \cs{peek_after:Nw} is -% provided along with a family of predefined tests for common cases. -% As peeking ahead does \emph{not} skip spaces the predefined tests -% include both a space-respecting and space-skipping version. +% \enquote{peek} functions. The generic \cs{peek_after:Nw} is provided +% along with a family of predefined tests for common cases. As peeking +% ahead does \emph{not} skip spaces the predefined tests include both a +% space-respecting and space-skipping version. In addition, using +% \cs{peek_analysis_map_inline:n}, one can map through the following +% tokens in the input stream and repeatedly perform some tests. % % \begin{function}{\peek_after:Nw} % \begin{syntax} @@ -980,6 +1008,128 @@ % (as appropriate to the result of the test). % \end{function} % +% \begin{function}[added = 2020-12-03]{\peek_analysis_map_inline:n} +% \begin{syntax} +% \cs{peek_analysis_map_inline:n} \Arg{inline function} +% \end{syntax} +% Repeatedly removes one \meta{token} from the input stream and +% applies the \meta{inline function} to it, until +% \cs{peek_analysis_map_break:} is called. The \meta{inline function} +% receives three arguments for each \meta{token} in the input stream: +% \begin{itemize} +% \item \meta{tokens}, which both \texttt{o}-expand and +% \texttt{x}-expand to the \meta{token}. The detailed form of +% \meta{tokens} may change in later releases. +% \item \meta{char code}, a decimal representation of the character +% code of the \meta{token}, $-1$ if it is a control sequence. +% \item \meta{catcode}, a capital hexadecimal digit which denotes the +% category code of the \meta{token} (0:~control sequence, +% 1:~begin-group, 2:~end-group, 3:~math shift, 4:~alignment tab, +% 6:~parameter, 7:~superscript, 8:~subscript, A:~space, B:~letter, +% C:~other, D:~active). This can be converted to an integer by +% writing |"|\meta{catcode}. +% \end{itemize} +% These arguments are the same as for \cs{tl_analysis_map_inline:nn} +% defined in \pkg{l3tl-analysis}. The \meta{char code} and +% \meta{catcode} do not take the meaning of a control sequence or +% active character into account: for instance, upon encountering the +% token \cs{c_group_begin_token} in the input stream, +% \cs{peek_analysis_map_inline:n} calls the \meta{inline function} +% with |#1| being \cs{exp_not:n} |{| \cs{c_group_begin_token} |}| +% (with the current implementation), +% |#2|~being~$-1$, and +% |#3|~being~$0$, as for any other control sequence. In contrast, +% upon encountering an explicit begin-group token~|{|, % ^^A |}| +% the \meta{inline function} is called with arguments +% \cs{exp_after:wN} |{| \cs{if_false:} |}| \cs{fi:}, $123$ and~$1$. +% +% The mapping is done at the current group level, \emph{i.e.}~any +% local assignments made by the \meta{inline function} remain in +% effect after the loop. Within the code, \cs{l_peek_token} is set +% equal (as a token, not a token list) to the token under +% consideration. +% \end{function} +% +% \begin{function}[added = 2020-12-03] +% {\peek_analysis_map_break:, \peek_analysis_map_break:n} +% \begin{syntax} +% \cs{peek_analysis_map_inline:n} +% |{| \dots{} \cs{peek_analysis_map_break:n} \Arg{code} |}| +% \end{syntax} +% Stops the \cs{peek_analysis_map_inline:n} loop from seeking more +% tokens, and inserts \meta{code} in the input stream (empty for +% \cs{peek_analysis_map_break:}). +% \end{function} +% +% \begin{function}[added = 2020-12-03, TF]{\peek_regex:n, \peek_regex:N} +% \begin{syntax} +% \cs{peek_regex:nTF} \Arg{regex} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{tokens} that follow in the input stream match the +% \meta{regular expression}. Any \meta{tokens} that have been read +% are left in the input stream after the \meta{true code} or +% \meta{false code} (as appropriate to the result of the test). See +% \pkg{l3regex} for documentation of the syntax of regular +% expressions. The \meta{regular expression} is implicitly anchored +% at the start, so for instance \cs{peek_regex:nTF}~|{|~|a|~|}| is +% essentially equivalent to \cs{peek_charcode:NTF}~|a|. +% \begin{texnote} +% Implicit character tokens are correctly considered by +% \cs{peek_regex:nTF} as control sequences, while functions that +% inspect individual tokens (for instance \cs{peek_charcode:NTF}) +% only take into account their meaning. +% \end{texnote} +% \end{function} +% +% \begin{function}[added = 2020-12-03, TF] +% {\peek_regex_remove_once:n, \peek_regex_remove_once:N} +% \begin{syntax} +% \cs{peek_regex_remove_once:nTF} \Arg{regex} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{tokens} that follow in the input stream match the +% \meta{regex}. If the test is true, the \meta{tokens} are removed +% from the input stream and the \meta{true code} is inserted, while if +% the test is false, the \meta{false code} is inserted followed by the +% \meta{tokens} that were originally in the input stream. +% See \pkg{l3regex} for documentation of the syntax of +% regular expressions. The \meta{regular expression} is implicitly +% anchored at the start, so for instance +% \cs{peek_regex_remove_once:nTF}~|{|~|a|~|}| is essentially equivalent to +% \cs{peek_charcode_remove:NTF}~|a|. +% \begin{texnote} +% Implicit character tokens are correctly considered by +% \cs{peek_regex_remove_once:nTF} as control sequences, while functions +% that inspect individual tokens (for instance +% \cs{peek_charcode:NTF}) only take into account their meaning. +% \end{texnote} +% \end{function} +% +% \begin{function}[added = 2020-12-03, noTF] +% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn} +% \begin{syntax} +% \cs{peek_regex_replace_once:nnTF} \Arg{regex} \Arg{replacement} \Arg{true code} \Arg{false code} +% \end{syntax} +% If the \meta{tokens} that follow in the input stream match the +% \meta{regex}, replaces them according to the \meta{replacement} as +% for \cs{regex_replace_once:nnN}, and leaves the result in the input +% stream, after the \meta{true code}. Otherwise, leaves \meta{false +% code} followed by the \meta{tokens} that were originally in the +% input stream, with no modifications. See \pkg{l3regex} for +% documentation of the syntax of regular expressions and of the +% \meta{replacement}: for instance |\0| in the \meta{replacement} is +% replaced by the tokens that were matched in the input stream. The +% \meta{regular expression} is implicitly anchored at the start. In +% contrast to \cs{regex_replace_once:nnN}, no error arises if the +% \meta{replacement} leads to an unbalanced token list: the tokens are +% inserted into the input stream without issue. +% \begin{texnote} +% Implicit character tokens are correctly considered by +% \cs{peek_regex_replace_once:nnTF} as control sequences, while +% functions that inspect individual tokens (for instance +% \cs{peek_charcode:NTF}) only take into account their meaning. +% \end{texnote} +% \end{function} +% % \section{Description of all possible tokens} % \label{sec:l3token:all-tokens} % @@ -1897,9 +2047,10 @@ %<@@=token> % \end{macrocode} % -% \begin{variable}{\s_@@_stop} +% \begin{variable}{\s_@@_mark, \s_@@_stop} % Internal scan marks. % \begin{macrocode} +\scan_new:N \s_@@_mark \scan_new:N \s_@@_stop % \end{macrocode} % \end{variable} @@ -2566,6 +2717,72 @@ end % \end{macro} % \end{macro} % +% \begin{macro}[EXP, noTF] +% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn} +% \begin{macro}[EXP]{\@@_case:NNnTF, \@@_case:NNw, \@@_case_end:nw} +% The aim here is to allow the case statement to be evaluated +% using a known number of expansion steps (two), and without +% needing to use an explicit \enquote{end of recursion} marker. +% That is achieved by using the test input as the final case, +% as this is always true. The trick is then to tidy up +% the output such that the appropriate case code plus either +% the \texttt{true} or \texttt{false} branch code is inserted. +% \begin{macrocode} +\cs_new:Npn \token_case_catcode:Nn #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } { } } +\cs_new:Npn \token_case_catcode:NnT #1#2#3 + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} {#3} { } } +\cs_new:Npn \token_case_catcode:NnF #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } } +\cs_new:Npn \token_case_catcode:NnTF + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF } +\cs_new:Npn \token_case_charcode:Nn #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } { } } +\cs_new:Npn \token_case_charcode:NnT #1#2#3 + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} {#3} { } } +\cs_new:Npn \token_case_charcode:NnF #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } } +\cs_new:Npn \token_case_charcode:NnTF + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF } +\cs_new:Npn \token_case_meaning:Nn #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } { } } +\cs_new:Npn \token_case_meaning:NnT #1#2#3 + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} {#3} { } } +\cs_new:Npn \token_case_meaning:NnF #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } } +\cs_new:Npn \token_case_meaning:NnTF + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF } +\cs_new:Npn \@@_case:NNnTF #1#2#3#4#5 + { + \@@_case:NNw #1 #2 #3 #2 { } + \s_@@_mark {#4} + \s_@@_mark {#5} + \s_@@_stop + } +\cs_new:Npn \@@_case:NNw #1#2#3#4 + { + #1 #2 #3 + { \@@_case_end:nw {#4} } + { \@@_case:NNw #1 #2 } + } +% \end{macrocode} +% To tidy up the recursion, there are two outcomes. If there was a hit to +% one of the cases searched for, then |#1| is the code to insert, +% |#2| is the \emph{next} case to check on and |#3| is all of +% the rest of the cases code. That means that |#4| is the \texttt{true} +% branch code, and |#5| tidies up the spare \cs{s_@@_mark} and the +% \texttt{false} branch. On the other hand, if none of the cases matched +% then we arrive here using the \enquote{termination} case of comparing +% the search with itself. That means that |#1| is empty, |#2| is +% the first \cs{s_@@_mark} and so |#4| is the \texttt{false} code (the +% \texttt{true} code is mopped up by |#3|). +% \begin{macrocode} +\cs_new:Npn \@@_case_end:nw #1#2#3 \s_@@_mark #4#5 \s_@@_stop + { \exp_end: #1 #4 } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \subsection{Peeking ahead at the next token} % % \begin{macrocode} |