diff options
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx')
-rw-r--r-- | Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx | 1109 |
1 files changed, 1109 insertions, 0 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx new file mode 100644 index 00000000000..d427b756056 --- /dev/null +++ b/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx @@ -0,0 +1,1109 @@ +% \iffalse meta-comment +% +%% File: l3tl-analysis.dtx Copyright (C) 2011-2012,2015-2017 The LaTeX3 Project% +% +% It may be distributed and/or modified under the conditions of the +% LaTeX Project Public License (LPPL), either version 1.3c of this +% license or (at your option) any later version. The latest version +% of this license is in the file +% +% http://www.latex-project.org/lppl.txt +% +% This file is part of the "l3kernel bundle" (The Work in LPPL) +% and all files in that bundle must be distributed together. +% +% ----------------------------------------------------------------------- +% +% The development version of the bundle can be found at +% +% https://github.com/latex3/latex3 +% +% for those people who are interested. +% +%<*driver> +\documentclass[full]{l3doc} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% +% +% \title{^^A +% The \textsf{l3tl-analysis} package: analysing token lists^^A +% } +% +% \author{^^A +% The \LaTeX3 Project\thanks +% {^^A +% E-mail: +% \href{mailto:latex-team@latex-project.org} +% {latex-team@latex-project.org}^^A +% }^^A +% } +% +% \date{Released 2017/05/29} +% +% \maketitle +% +% \begin{documentation} +% +% \section{\pkg{l3tl-analysis} documentation} +% +% This module mostly provides internal functions for use in the +% \pkg{l3regex} module. However, it provides as a side-effect a user +% debugging function, very similar to the \cs{ShowTokens} macro from the +% \pkg{ted} package. +% +% \begin{function}[added = 2017-05-26]{\tl_show_analysis:N, \tl_show_analysis:n} +% \begin{syntax} +% \cs{tl_show_analysis:n} \Arg{token list} +% \end{syntax} +% Displays to the terminal the detailed decomposition of the +% \meta{token list} into tokens, showing the category code of each +% character token, the meaning of control sequences and active +% characters, and the value of registers. +% \end{function} +% +% \end{documentation} +% +% \begin{implementation} +% +% \section{\pkg{l3tl-analysis} implementation} +% +% \subsection{Internal functions} +% +% \begin{variable}{\s__tl} +% The format used to store token lists internally uses the scan mark +% \cs{s__tl} as a delimiter. +% \end{variable} +% +% \begin{function}{\__tl_analysis_map_inline:nn} +% \begin{syntax} +% \cs{__tl_analysis_map_inline:nn} \Arg{token list} \Arg{inline function} +% \end{syntax} +% Applies the \meta{inline function} to each individual \meta{token} +% in the \meta{token list}. The \meta{inline function} receives three +% arguments: +% \begin{itemize} +% \item \meta{tokens}, which both \texttt{o}-expand and +% \texttt{x}-expand to the \meta{token}. The detailed form of +% \meta{token} may change in later releases. +% \item \meta{catcode}, a capital hexadecimal digit which denotes +% the category code of the \meta{token} (0: control sequence, 1: +% begin-group, 2: end-group, 3: math shift, 4: alignment tab, 6: +% parameter, 7: superscript, 8: subscript, A: space, B: letter, +% C:other, D:active). +% \item \meta{char code}, a decimal representation of the character +% code of the token, $-1$ if it is a control sequence (with +% \meta{catcode} $0$). +% \end{itemize} +% \end{function} +% +% For optimizations in \pkg{l3regex} (when matching control sequences), +% it may be useful to provide a \cs{__tl_analysis_from_str_map_inline:nn} +% function, perhaps named \cs{__str_analysis_map_inline:nn}. +% +% \subsection{Internal format} +% +% The task of the \pkg{l3tl-analysis} module is to convert token lists +% to an internal format which allows us to extract all the relevant +% information about individual tokens (category code, character code), +% as well as reconstruct the token list quickly. This internal format is +% used in \pkg{l3regex} where we need to support arbitrary tokens, and +% it is used in conversion functions in \pkg{l3str-convert}, where we wish to +% support clusters of characters instead of single tokens. +% +% We thus need a way to encode any \meta{token} (even begin-group and +% end-group character tokens) in a way amenable to manipulating tokens +% individually. The best we can do is to find \meta{tokens} which both +% \texttt{o}-expand and \texttt{x}-expand to the given +% \meta{token}. Collecting more information about the category code and +% character code is also useful for regular expressions, since most +% regexes are catcode-agnostic. The internal format thus takes the form +% of a succession of items of the form +% \begin{quote} +% \meta{tokens} \cs{s__tl} \meta{catcode} \meta{char code} \cs{s__tl} +% \end{quote} +% The \meta{tokens} \texttt{o}- \emph{and} \texttt{x}-expand to the +% original token in the token list or to the cluster of tokens +% corresponding to one Unicode character in the given encoding (for +% \pkg{l3str-convert}). The \meta{catcode} is given as a single hexadecimal +% digit, $0$ for control sequences. The \meta{char code} is given as a +% decimal number, $-1$ for control sequences. +% +% Using delimited arguments lets us build the \meta{tokens} +% progressively when doing an encoding conversion in \pkg{l3str-convert}. On the +% other hand, the delimiter \cs{s__tl} may not appear unbraced in +% \meta{tokens}. This is not a problem because we are careful to wrap +% control sequences in braces (as an argument to \cs{exp_not:n}) when +% converting from a general token list to the internal format. +% +% The current rule for converting a \meta{token} to a balanced set of +% \meta{tokens} which both \texttt{o}-expands and \texttt{x}-expands to +% it is the following. +% \begin{itemize} +% \item A control sequence |\cs| becomes |\exp_not:n { \cs }| +% \cs{s__tl} $0$ $-1$ \cs{s__tl}. +% \item A begin-group character |{| becomes \cs{exp_after:wN} |{| +% \cs{if_false:} |}| \cs{fi:} \cs{s__tl} $1$ \meta{char code} +% \cs{s__tl}. +% \item An end-group character |}| becomes \cs{if_false:} |{| \cs{fi:} +% |}| \cs{s__tl} $2$ \meta{char code} \cs{s__tl}. +% \item A character with any other category code becomes +% \cs{exp_not:n} \Arg{character} \cs{s__tl} \meta{hex catcode} +% \meta{char code} \cs{s__tl}. +% \end{itemize} +% +% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char> +% ^^A which does not add the <active char> in memory. +% +% \begin{macrocode} +%<*initex|package> +% \end{macrocode} +% +% \begin{macrocode} +%<@@=tl_analysis> +% \end{macrocode} +% +% \subsection{Variables and helper functions} +% +% \begin{variable}{\s__tl} +% The scan mark \cs{s__tl} is used as a delimiter in the internal +% format. This is more practical than using a quark, because we would +% then need to control expansion much more carefully: compare +% \cs{__int_value:w} |`#1| \cs{s__tl} with \cs{__int_value:w} |`#1| +% \cs{exp_stop_f:} \cs{exp_not:N} \cs{q_mark} to extract a character +% code followed by the delimiter in an \texttt{x}-expansion. +% \begin{macrocode} +\__scan_new:N \s__tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_internal_tl} +% This token list variable is used to hand the argument of +% \cs{tl_show_analysis:n} to \cs{tl_show_analysis:N}. +% \begin{macrocode} +\tl_new:N \l_@@_internal_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_token} +% \begin{variable}{\l_@@_char_token} +% The tokens in the token list are probed with the \TeX{} primitive +% \tn{futurelet}. We use \cs{l_@@_token} in that +% construction. In some cases, we convert the following token to a +% string before probing it: then the token variable used is +% \cs{l_@@_char_token}. +% \begin{macrocode} +\cs_new_eq:NN \l_@@_token ? +\cs_new_eq:NN \l_@@_char_token ? +% \end{macrocode} +% \end{variable} +% \end{variable} +% +% \begin{variable}{\l_@@_normal_int} +% The number of normal (\texttt{N}-type argument) tokens since the +% last special token. +% \begin{macrocode} +\int_new:N \l_@@_normal_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_index_int} +% During the first pass, this is the index in the array being built. +% During the second pass, it is equal to the maximum index in the +% array from the first pass. +% \begin{macrocode} +\int_new:N \l_@@_index_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_nesting_int} +% Nesting depth of explicit begin-group and end-group characters +% during the first pass. This lets us detect the end of the token list +% without a reserved end-marker. +% \begin{macrocode} +\int_new:N \l_@@_nesting_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_type_int} +% When encountering special characters, we record their \enquote{type} +% in this integer. +% \begin{macrocode} +\int_new:N \l_@@_type_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_@@_result_tl} +% The result of the conversion is stored in this token list, with a +% succession of items of the form +% \begin{quote} +% \meta{tokens} \cs{s__tl} \meta{catcode} \meta{char code} \cs{s__tl} +% \end{quote} +% \begin{macrocode} +\tl_new:N \g_@@_result_tl +% \end{macrocode} +% \end{variable} +% +% \begin{macro}[int, EXP]{\@@_extract_charcode:} +% \begin{macro}[aux, EXP]{\@@_extract_charcode_aux:w} +% Extracting the character code from the meaning of +% \cs{l_@@_token}. This has no error checking, and should +% only be assumed to work for begin-group and end-group character +% tokens. It produces a number in the form |`|\meta{char}. +% \begin{macrocode} +\cs_new:Npn \@@_extract_charcode: + { + \exp_after:wN \@@_extract_charcode_aux:w + \token_to_meaning:N \l_@@_token + } +\cs_new:Npn \@@_extract_charcode_aux:w #1 ~ #2 ~ { ` } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int, EXP]{\@@_cs_space_count:NN} +% \begin{macro}[aux, EXP]{\@@_cs_space_count:w} +% \begin{macro}[aux, EXP]{\@@_cs_space_count_end:w} +% Counts the number of spaces in the string representation of its +% second argument, as well as the number of characters following the +% last space in that representation, and feeds the two numbers as +% semicolon-delimited arguments to the first argument. When this +% function is used, the escape character is printable and non-space. +% \begin{macrocode} +\cs_new:Npn \@@_cs_space_count:NN #1 #2 + { + \exp_after:wN #1 + \__int_value:w \__int_eval:w 0 + \exp_after:wN \@@_cs_space_count:w + \token_to_str:N #2 + \fi: \@@_cs_space_count_end:w ; ~ ! + } +\cs_new:Npn \@@_cs_space_count:w #1 ~ + { + \if_false: #1 #1 \fi: + + 1 + \@@_cs_space_count:w + } +\cs_new:Npn \@@_cs_space_count_end:w ; #1 \fi: #2 ! + { \exp_after:wN ; \__int_value:w \str_count_ignore_spaces:n {#1} ; } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Plan of attack} +% +% Our goal is to produce a token list of the form roughly +% \begin{quote} +% \meta{token 1} \cs{s__tl} \meta{catcode 1} \meta{char code 1} \cs{s__tl} \\ +% \meta{token 2} \cs{s__tl} \meta{catcode 2} \meta{char code 2} \cs{s__tl} \\ +% \ldots{} +% \meta{token N} \cs{s__tl} \meta{catcode N} \meta{char code N} \cs{s__tl} +% \end{quote} +% Most but not all tokens can be grabbed as an undelimited +% (\texttt{N}-type) argument by \TeX{}. The plan is to have a two pass +% system. In the first pass, locate special tokens, and store them in +% various \tn{toks} registers. In the second pass, which is done within +% an \texttt{x}-expanding assignment, normal tokens are taken in as +% \texttt{N}-type arguments, and special tokens are retrieved from the +% \tn{toks} registers, and removed from the input stream by some means. +% The whole process takes linear time, because we avoid building the +% result one item at a time. +% +% To ease the difficult first pass, we first do some setup with +% \cs{@@_setup:n}. Active characters set equal to non-active +% characters cause trouble, so we disable all active characters by +% setting them equal to \texttt{undefined} locally. We also set there +% the escape character to be printable (backslash, but this later +% oscillates between slash and backslash): this makes it possible to +% distinguish characters from control sequences. +% +% A token has two characteristics: its \tn{meaning}, and what it looks +% like for \TeX{} when it is in scanning mode (\emph{e.g.}, when +% capturing parameters for a macro). For our purposes, we distinguish +% the following meanings: +% \begin{itemize} +% \item begin-group token (category code $1$), either space (character +% code $32$), or non-space; +% \item end-group token (category code $2$), either space (character +% code $32$), or non-space; +% \item space token (category code $10$, character code $32$); +% \item anything else (then the token is always an \texttt{N}-type +% argument). +% \end{itemize} +% The token itself can \enquote{look like} one of the following +% \begin{itemize} +% \item a non-active character, in which case its meaning is +% automatically that associated to its character code and category +% code, we call it \enquote{true} character; +% \item an active character (we eliminate those in the setup step); +% \item a control sequence. +% \end{itemize} +% The only tokens which are not valid \texttt{N}-type arguments are true +% begin-group characters, true end-group characters, and true spaces. +% We will detect those characters by scanning ahead with \tn{futurelet}, +% then distinguishing true characters from control sequences set equal +% to them using the \tn{string} representation. +% +% The second pass is a simple exercise in expandable loops. +% +% \begin{macro}[int]{\@@:n} +% Everything is done within a group, and all definitions will be +% local. We use \cs{group_align_safe_begin/end:} to avoid problems in +% case \cs{@@:n} is used within an alignment and its argument +% contains alignment tab tokens. +% \begin{macrocode} +\cs_new_protected:Npn \@@:n #1 + { + \group_begin: + \group_align_safe_begin: + \@@_setup:n {#1} + \@@_a:n {#1} + \@@_b:n {#1} + \group_align_safe_end: + \group_end: + } +% \end{macrocode} +% \end{macro} +% +% \subsection{Setup} +% +% \begin{macro}[int]{\@@_setup:n} +% \begin{macro}[aux]{\@@_disable_loop:N} +% Active characters can cause problems later on in the processing, +% so the first step is to disable them, by setting them to +% \texttt{undefined}. Since Unicode contains too many characters +% to loop over all of them, we instead loop over the input token +% list as a string: any active character in the token list +% must appear in its string representation. The string is shortened +% a little by making the escape character unprintable. The active +% space must be disabled separately (the loop skips over it otherwise), +% and we end the loop by feeding an odd non-\texttt{N}-type argument +% to the looping macro. For \pTeX{} and \upTeX{} we skip characters +% beyond $[0,255]$ because \tn{lccode} only allows those values. +% \begin{macrocode} +\cs_new_protected:Npn \@@_setup:n #1 + { + \int_set:Nn \tex_escapechar:D { -1 } + \exp_after:wN \@@_disable_loop:N + \tl_to_str:n {#1} { ~ } { ? \__prg_break: } + \__prg_break_point: + \scan_stop: + } +\group_begin: + \char_set_catcode_active:N \^^@ + \cs_new_protected:Npn \@@_disable_loop:N #1 + { + \tex_lccode:D 0 = `#1 ~ + \tex_lowercase:D { \tex_let:D ^^@ } \tex_undefined:D + \@@_disable_loop:N + } + \cs_if_exist:NT \ptex_kanjiskip:D + { + \cs_gset_protected:Npn \@@_disable_loop:N #1 + { + \use_none:n #1 \scan_stop: + \if_int_compare:w 256 > `#1 \exp_stop_f: + \tex_lccode:D 0 = `#1 ~ + \tex_lowercase:D { \tex_let:D ^^@ } \tex_undefined:D + \fi: + \@@_disable_loop:N + } + } +\group_end: +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{First pass} +% +% The goal of this pass is to detect special (non-\texttt{N}-type) tokens, +% and count how many \texttt{N}-type tokens lie between special tokens. +% Also, we wish to store some representation of each special token +% in a \tn{toks} register. +% +% After the setup step, we have $11$ types of tokens: +% \begin{itemize} +% \item[1.] a true non-space begin-group character; +% \item[2.] a true space begin-group character; +% \item[3.] a true non-space end-group character; +% \item[4.] a true space end-group character; +% \item[5.] a true space blank space character; +% \item[6.] an undefined active character; +% \item[7.] any other true character; +% \item[8.] a control sequence equal to a begin-group token (category code $1$); +% \item[9.] a control sequence equal to an end-group token (category code $2$); +% \item[10.] a control sequence equal to a space token +% (character code $32$, category code $10$); +% \item[11.] any other control sequence. +% \end{itemize} +% Our first tool is \tn{futurelet}. This cannot distinguish +% case $8$ from $1$ or $2$, nor case $9$ from $3$ or $4$, +% nor case $10$ from case $5$. Those cases will be distinguished +% by applying the \tn{string} primitive to the following token, +% after possibly changing the escape character to ensure that +% a control sequence's string representation cannot be mistaken +% for the true character. +% +% In cases $6$, $7$, and $11$, the following token is a valid +% \texttt{N}-type argument, so we grab it and distinguish the case +% of a character from a control sequence: in the latter case, +% \cs{str_tail:n} \Arg{token} is non-empty, because the +% escape character is printable. +% +% \begin{macro}[int]{\@@_a:n} +% We read tokens one by one using \tn{futurelet}. +% While performing the loop, we keep track of the number of +% true begin-group characters minus the number of +% true end-group characters in \cs{l_@@_nesting_int}. +% This reaches $-1$ when we read the closing brace. +% \begin{macrocode} +\cs_new_protected:Npn \@@_a:n #1 + { + \int_set:Nn \tex_escapechar:D { 92 } + \int_zero:N \l_@@_normal_int + \int_zero:N \l_@@_index_int + \int_zero:N \l_@@_nesting_int + \if_false: { \fi: \@@_a_loop:w #1 } + \int_decr:N \l_@@_index_int + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_a_loop:w} +% Read one character and check its type. +% \begin{macrocode} +\cs_new_protected:Npn \@@_a_loop:w + { \tex_futurelet:D \l_@@_token \@@_a_type:w } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_a_type:w} +% At this point, \cs{l_@@_token} holds the meaning +% of the following token. We store in \cs{l_@@_type_int} +% the meaning of the token ahead: +% \begin{itemize} +% \item 0 space token; +% \item 1 begin-group token; +% \item -1 end-group token; +% \item 2 other. +% \end{itemize} +% The values $0$, $1$, $-1$ correspond to how much a true such +% character changes the nesting level ($2$ is used only here, +% and is irrelevant later). Then call the auxiliary for each case. +% Note that nesting conditionals here is safe because we only skip +% over \cs{l_@@_token} if it matches with one of the +% character tokens (hence is not a primitive conditional). +% \begin{macrocode} +\cs_new_protected:Npn \@@_a_type:w + { + \l_@@_type_int = + \if_meaning:w \l_@@_token \c_space_token + 0 + \else: + \if_catcode:w \exp_not:N \l_@@_token \c_group_begin_token + 1 + \else: + \if_catcode:w \exp_not:N \l_@@_token \c_group_end_token + - 1 + \else: + 2 + \fi: + \fi: + \fi: + \exp_stop_f: + \if_case:w \l_@@_type_int + \exp_after:wN \@@_a_space:w + \or: \exp_after:wN \@@_a_bgroup:w + \or: \exp_after:wN \@@_a_safe:N + \else: \exp_after:wN \@@_a_egroup:w + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_a_space:w} +% \begin{macro}[aux]{\@@_a_space_test:w} +% In this branch, the following token's meaning is a blank space. +% Apply \tn{string} to that token: if it is a control sequence +% the result starts with the escape character; otherwise it is +% a true blank space, whose string representation is also a blank space. +% We test for that in \cs{@@_a_space_test:w}, +% after grabbing as \cs{l_@@_char_token} the first character +% of the string representation. +% Also, since \cs{@@_a_store:} expects the special token to be +% stored in the relevant \tn{toks} register, we do that. The extra +% \cs{exp_not:n} is unnecessary of course, but it makes the treatment +% of all tokens more homogeneous. +% If we discover that the next token was actually a control sequence +% instead of a true space, then we step the counter of normal tokens. +% We now have in front of us the whole string representation of +% the control sequence, including potential spaces; those will appear +% to be true spaces later in this pass. Hence, all other branches of +% the code in this first pass need to consider the string representation, +% so that the second pass does not need to test the meaning of tokens, +% only strings. +% \begin{macrocode} +\cs_new_protected:Npn \@@_a_space:w + { + \tex_afterassignment:D \@@_a_space_test:w + \exp_after:wN \cs_set_eq:NN + \exp_after:wN \l_@@_char_token + \token_to_str:N + } +\cs_new_protected:Npn \@@_a_space_test:w + { + \if_meaning:w \l_@@_char_token \c_space_token + \tex_toks:D \l_@@_index_int { \exp_not:n { ~ } } + \@@_a_store: + \else: + \int_incr:N \l_@@_normal_int + \fi: + \@@_a_loop:w + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\@@_a_bgroup:w, \@@_a_egroup:w} +% \begin{macro}[aux]{\@@_a_group:nw} +% \begin{macro}[aux]{\@@_a_group_test:w} +% The token might be either a true character token with +% catcode $1$ or $2$, or it could be a control sequence. +% The only tricky case is if the character code happens +% to be equal to the escape character: then we change +% the escape character from backslash to solidus or back, +% so that the string representation of the true character +% and of a control sequence set equal to it start differently. +% Then probe what the first character of that string +% representation is: this is the place where we need +% \cs{l_@@_char_token} to be a separate control +% sequence from \cs{l_@@_token}, to compare them. +% \begin{macrocode} +\group_begin: + \char_set_catcode_group_begin:N \^^@ + \char_set_catcode_group_end:N \^^E + \cs_new_protected:Npn \@@_a_bgroup:w + { \@@_a_group:nw { \exp_after:wN ^^@ \if_false: ^^E \fi: } } + \char_set_catcode_group_begin:N \^^B + \char_set_catcode_group_end:N \^^@ + \cs_new_protected:Npn \@@_a_egroup:w + { \@@_a_group:nw { \if_false: ^^B \fi: ^^@ } } +\group_end: +\cs_new_protected:Npn \@@_a_group:nw #1 + { + \tex_lccode:D 0 = \@@_extract_charcode: \scan_stop: + \tex_lowercase:D { \tex_toks:D \l_@@_index_int {#1} } + \if_int_compare:w \tex_lccode:D 0 = \tex_escapechar:D + \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D } + \fi: + \tex_afterassignment:D \@@_a_group_test:w + \exp_after:wN \cs_set_eq:NN + \exp_after:wN \l_@@_char_token + \token_to_str:N + } +\cs_new_protected:Npn \@@_a_group_test:w + { + \if_charcode:w \l_@@_token \l_@@_char_token + \@@_a_store: + \else: + \int_incr:N \l_@@_normal_int + \fi: + \@@_a_loop:w + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\@@_a_store:} +% This function is called each time we meet a special token; +% at this point, the \tn{toks} register \cs{l_@@_index_int} +% holds a token list which expands to the given special token. +% Also, the value of \cs{l_@@_type_int} indicates which case +% we are in: +% \begin{itemize} +% \item -1 end-group character; +% \item 0 space character; +% \item 1 begin-group character. +% \end{itemize} +% We need to distinguish further the case of a space character +% (code $32$) from other character codes, because those will +% behave differently in the second pass. Namely, after testing +% the \tn{lccode} of $0$ (which holds the present character code) +% we change the cases above to +% \begin{itemize} +% \item -2 space end-group character; +% \item -1 non-space end-group character; +% \item 0 space blank space character; +% \item 1 non-space begin-group character; +% \item 2 space begin-group character. +% \end{itemize} +% This has the property that non-space characters correspond to odd +% values of \cs{l_@@_type_int}. +% The number of normal tokens, and the type of special token, +% are packed into a \tn{skip} register. +% Finally, we check whether we reached the last closing brace, in which +% case we stop by disabling the looping function (locally). +% \begin{macrocode} +\cs_new_protected:Npn \@@_a_store: + { + \tex_advance:D \l_@@_nesting_int \l_@@_type_int + \if_int_compare:w \tex_lccode:D 0 = `\ \exp_stop_f: + \tex_multiply:D \l_@@_type_int 2 \exp_stop_f: + \fi: + \tex_skip:D \l_@@_index_int + = \l_@@_normal_int sp plus \l_@@_type_int sp \scan_stop: + \int_incr:N \l_@@_index_int + \int_zero:N \l_@@_normal_int + \if_int_compare:w \l_@@_nesting_int = -1 \exp_stop_f: + \cs_set_eq:NN \@@_a_loop:w \scan_stop: + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int]{\@@_a_safe:N} +% \begin{macro}[aux]{\@@_a_cs:ww} +% This should be the simplest case: since the upcoming token is safe, +% we can simply grab it in a second pass. However, other branches of +% the code must pass their tokens through \tn{string}, hence we do it +% here as well, with some optimizations. If the token is a single +% character (including space), the \cs{if_charcode:w} test yields +% true, and we simply count one \enquote{normal} token. On the other +% hand, if the token is a control sequence, we should replace it by +% its string representation for compatibility with other code +% branches. Instead of slowly looping through the characters with +% the main code, we use the knowledge of how the second pass works: +% if the control sequence name contains no space, count that token +% as a number of normal tokens equal to its string length. If the +% control sequence contains spaces, they should be registered as +% special characters by increasing \cs{l_@@_index_int} +% (no need to carefully count character between each space), and +% all characters after the last space should be counted in the +% following sequence of \enquote{normal} tokens. +% \begin{macrocode} +\cs_new_protected:Npn \@@_a_safe:N #1 + { + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing: + \scan_stop: + \int_incr:N \l_@@_normal_int + \else: + \@@_cs_space_count:NN \@@_a_cs:ww #1 + \fi: + \@@_a_loop:w + } +\cs_new_protected:Npn \@@_a_cs:ww #1; #2; + { + \if_int_compare:w #1 > 0 \exp_stop_f: + \tex_skip:D \l_@@_index_int + = \__int_eval:w \l_@@_normal_int + 1 sp \scan_stop: + \tex_advance:D \l_@@_index_int #1 \exp_stop_f: + \l_@@_normal_int #2 \exp_stop_f: + \else: + \tex_advance:D \l_@@_normal_int #2 \exp_stop_f: + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Second pass} +% +% The second pass is an exercise in expandable loops. +% All the necessary information is stored in \tn{skip} +% and \tn{toks} registers. +% +% \begin{macro}[int]{\@@_b:n} +% \begin{macro}[int, EXP]{\@@_b_loop:w} +% Start the loop with the index $0$. No need for an end-marker: +% the loop will stop by itself when the last index is read. +% We will repeatedly oscillate between reading long stretches +% of normal tokens, and reading special tokens. +% \begin{macrocode} +\cs_new_protected:Npn \@@_b:n #1 + { + \tl_gset:Nx \g_@@_result_tl + { + \@@_b_loop:w 0; #1 + \__prg_break_point: + } + } +\cs_new:Npn \@@_b_loop:w #1; + { + \exp_after:wN \@@_b_normals:ww + \__int_value:w \tex_skip:D #1 ; #1 ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int, EXP]{\@@_b_normals:ww} +% \begin{macro}[aux, EXP]{\@@_b_normal:wwN} +% The first argument is the number of normal tokens which remain +% to be read, and the second argument is the index in the array +% produced in the first step. +% A character's string representation is always one character long, +% while a control sequence is always longer (we have set the escape +% character to a printable value). In both cases, we leave +% \cs{exp_not:n} \Arg{token} \cs{s__tl} in the input stream +% (after \texttt{x}-expansion). Here, \cs{exp_not:n} is used +% rather than \cs{exp_not:N} because |#3| could be \cs{s__tl}, +% hence must be hidden behind braces in the result. +% \begin{macrocode} +\cs_new:Npn \@@_b_normals:ww #1; + { + \if_int_compare:w #1 = 0 \exp_stop_f: + \@@_b_special:w + \fi: + \@@_b_normal:wwN #1; + } +\cs_new:Npn \@@_b_normal:wwN #1; #2; #3 + { + \exp_not:n { \exp_not:n { #3 } } \s__tl + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing: + \scan_stop: + \exp_after:wN \@@_b_char:Nww + \else: + \exp_after:wN \@@_b_cs:Nww + \fi: + #3 #1; #2; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int, EXP]{\@@_b_char:Nww} +% If the normal token we grab is a character, leave +% \meta{catcode} \meta{charcode} followed by \cs{s__tl} +% in the input stream, and call \cs{@@_b_normals:ww} +% with its first argument decremented. +% \begin{macrocode} +\cs_new:Npx \@@_b_char:Nww #1 + { + \exp_not:N \if_meaning:w #1 \exp_not:N \tex_undefined:D + \token_to_str:N D \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_catcode_other_token + \token_to_str:N C \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_catcode_letter_token + \token_to_str:N B \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_math_toggle_token 3 \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_alignment_token 4 \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_math_superscript_token 7 \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_math_subscript_token 8 \exp_not:N \else: + \exp_not:N \if_catcode:w #1 \c_space_token + \token_to_str:N A \exp_not:N \else: + 6 + \exp_not:n { \fi: \fi: \fi: \fi: \fi: \fi: \fi: \fi: } + \exp_not:N \__int_value:w `#1 \s__tl + \exp_not:N \exp_after:wN \exp_not:N \@@_b_normals:ww + \exp_not:N \__int_value:w \exp_not:N \__int_eval:w - 1 + + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[int, EXP]{\@@_b_cs:Nww} +% \begin{macro}[aux, EXP]{\@@_b_cs_test:ww} +% If the token we grab is a control sequence, leave +% |0 -1| (as category code and character code) in the input stream, +% followed by \cs{s__tl}, +% and call \cs{@@_b_normals:ww} with updated arguments. +% \begin{macrocode} +\cs_new:Npn \@@_b_cs:Nww #1 + { + 0 -1 \s__tl + \@@_cs_space_count:NN \@@_b_cs_test:ww #1 + } +\cs_new:Npn \@@_b_cs_test:ww #1 ; #2 ; #3 ; #4 ; + { + \exp_after:wN \@@_b_normals:ww + \__int_value:w \__int_eval:w + \if_int_compare:w #1 = 0 \exp_stop_f: + #3 + \else: + \tex_skip:D \__int_eval:w #4 + #1 \__int_eval_end: + \fi: + - #2 + \exp_after:wN ; + \__int_value:w \__int_eval:w #4 + #1 ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int, EXP]{\@@_b_special:w} +% \begin{macro}[aux, EXP]{\@@_b_special_char:wN} +% \begin{macro}[aux, EXP]{\@@_b_special_space:w} +% Here, |#1| is the current index in the array built in the first pass. +% Check now whether we reached the end (we shouldn't keep the trailing +% end-group character that marked the end of the token list in the +% first pass). +% Unpack the \tn{toks} register: when \texttt{x}-expanding again, +% we will get the special token. +% Then leave the category code in the input stream, followed by +% the character code, and call \cs{@@_b_loop:w} with the next index. +% \begin{macrocode} +\group_begin: + \char_set_catcode_other:N A + \cs_new:Npn \@@_b_special:w + \fi: \@@_b_normal:wwN 0 ; #1 ; + { + \fi: + \if_int_compare:w #1 = \l_@@_index_int + \exp_after:wN \__prg_break: + \fi: + \tex_the:D \tex_toks:D #1 \s__tl + \if_case:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + A + \or: 1 + \or: 1 + \else: 2 + \fi: + \if_int_odd:w \etex_gluestretch:D \tex_skip:D #1 \exp_stop_f: + \exp_after:wN \@@_b_special_char:wN \__int_value:w + \else: + \exp_after:wN \@@_b_special_space:w \__int_value:w + \fi: + \__int_eval:w 1 + #1 \exp_after:wN ; + \token_to_str:N + } +\group_end: +\cs_new:Npn \@@_b_special_char:wN #1 ; #2 + { + \__int_value:w `#2 \s__tl + \@@_b_loop:w #1 ; + } +\cs_new:Npn \@@_b_special_space:w #1 ; ~ + { + 32 \s__tl + \@@_b_loop:w #1 ; + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Mapping through the analysis} +% +% \begin{macro}[int]{\@@_map_inline:nn} +% \begin{macro}[aux]{\@@_map_inline_aux:Nn} +% First obtain the analysis of the token list into +% \cs{g_@@_result_tl}. To allow nested mappings, increase the +% nesting depth \cs{g__prg_map_int} (shared between all modules), then +% define the looping macro, which has a name specific to that nesting +% depth. That looping grabs the \meta{tokens}, \meta{catcode} and +% \meta{char code}; it checks for the end of the loop with +% \cs{use_none:n} |##2|, normally empty, but which becomes +% \cs{tl_map_break:} at the end; it then performs the user's code +% |#2|, and loops by calling itself. When the loop ends, remember to +% decrease the nesting depth. +% \begin{macrocode} +\cs_new_protected:Npn \@@_map_inline:nn #1 + { + \@@:n {#1} + \int_gincr:N \g__prg_map_int + \exp_args:Nc \@@_map_inline_aux:Nn + { @@_map_inline_ \int_use:N \g__prg_map_int :wNw } + } +\cs_new_protected:Npn \@@_map_inline_aux:Nn #1#2 + { + \cs_gset_protected:Npn #1 ##1 \s__tl ##2 ##3 \s__tl + { + \use_none:n ##2 + #2 + #1 + } + \exp_after:wN #1 + \g_@@_result_tl + \s__tl { ? \tl_map_break: } \s__tl + \__prg_break_point:Nn \tl_map_break: { \int_gdecr:N \g__prg_map_int } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Showing the results} +% +% \begin{macro}{\tl_show_analysis:N, \tl_show_analysis:n} +% \begin{macro}[int]{\@@_show:} +% Add to \cs{@@:n} a third pass to display tokens to the terminal. +% If the token list variable is not defined, throw the same error +% as \cs{tl_show:N} by simply calling that function. +% \begin{macrocode} +\cs_new_protected:Npn \tl_show_analysis:N #1 + { + \tl_if_exist:NTF #1 + { + \exp_args:No \@@:n {#1} + \__msg_show_pre:nnxxxx { LaTeX / kernel } { show-tl-analysis } + { \token_to_str:N #1 } { \tl_if_empty:NTF #1 { } { ? } } { } { } + \@@_show: + } + { \tl_show:N #1 } + } +\cs_new_protected:Npn \tl_show_analysis:n #1 + { + \@@:n {#1} + \__msg_show_pre:nnxxxx { LaTeX / kernel } { show-tl-analysis } + { } { \tl_if_empty:nTF {#1} { } { ? } } { } { } + \@@_show: + } +\cs_new_protected:Npn \@@_show: + { + \group_begin: + \exp_args:NNx + \group_end: + \__msg_show_wrap:n + { + \exp_after:wN \@@_show_loop:wNw \g_@@_result_tl + \s__tl { ? \__prg_break: } \s__tl + \__prg_break_point: + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[aux, rEXP]{\@@_show_loop:wNw} +% Here, |#1| \texttt{o}- and \texttt{x}-expands to the token; +% |#2| is the category code (one uppercase hexadecimal digit), +% $0$ for control sequences; +% |#3| is the character code, which we ignore. +% In the cases of control sequences and active characters, +% the meaning may overflow one line, and we want to truncate +% it. Those cases are thus separated out. +% \begin{macrocode} +\cs_new:Npn \@@_show_loop:wNw #1 \s__tl #2 #3 \s__tl + { + \use_none:n #2 + \exp_not:n { \\ > \ \ } + \if_int_compare:w "#2 = 0 \exp_stop_f: + \exp_after:wN \@@_show_cs:n + \else: + \if_int_compare:w "#2 = 13 \exp_stop_f: + \exp_after:wN \exp_after:wN + \exp_after:wN \@@_show_active:n + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \@@_show_normal:n + \fi: + \fi: + {#1} + \@@_show_loop:wNw + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux, rEXP]{\@@_show_normal:n} +% Non-active characters are a simple matter of printing +% the character, and its meaning. Our test suite checks that +% begin-group and end-group characters do not mess up +% \TeX{}'s alignment status. +% \begin{macrocode} +\cs_new:Npn \@@_show_normal:n #1 + { + \exp_after:wN \token_to_str:N #1 ~ + ( \exp_after:wN \token_to_meaning:N #1 ) + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[EXP]{\@@_show_value:N} +% This expands to the value of |#1| if it has any. +% \begin{macrocode} +\cs_new:Npn \@@_show_value:N #1 + { + \token_if_expandable:NF #1 + { + \token_if_chardef:NTF #1 \__prg_break: { } + \token_if_mathchardef:NTF #1 \__prg_break: { } + \token_if_dim_register:NTF #1 \__prg_break: { } + \token_if_int_register:NTF #1 \__prg_break: { } + \token_if_skip_register:NTF #1 \__prg_break: { } + \token_if_toks_register:NTF #1 \__prg_break: { } + \use_none:nnn + \__prg_break_point: + \use:n { \exp_after:wN = \tex_the:D #1 } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[aux, rEXP]{\@@_show_cs:n} +% \begin{macro}[aux, rEXP]{\@@_show_active:n} +% \begin{macro}[aux, rEXP]{\@@_show_long:nn} +% \begin{macro}[aux, rEXP]{\@@_show_long_aux:nnnn} +% Control sequences and active characters are printed in the same way, +% making sure not to go beyond the \cs{l_iow_line_count_int}. In case +% of an overflow, we replace the last characters by +% \cs{c_@@_show_etc_str}. +% \begin{macrocode} +\cs_new:Npn \@@_show_cs:n #1 + { \exp_args:No \@@_show_long:nn {#1} { control~sequence= } } +\cs_new:Npn \@@_show_active:n #1 + { \exp_args:No \@@_show_long:nn {#1} { active~character= } } +\cs_new:Npn \@@_show_long:nn #1 + { + \@@_show_long_aux:oofn + { \token_to_str:N #1 } + { \token_to_meaning:N #1 } + { \@@_show_value:N #1 } + } +\cs_new:Npn \@@_show_long_aux:nnnn #1#2#3#4 + { + \int_compare:nNnTF + { \str_count:n { #1 ~ ( #4 #2 #3 ) } } + > { \l_iow_line_count_int - 3 } + { + \str_range:nnn { #1 ~ ( #4 #2 #3 ) } { 1 } + { + \l_iow_line_count_int - 3 + - \str_count:N \c_@@_show_etc_str + } + \c_@@_show_etc_str + } + { #1 ~ ( #4 #2 #3 ) } + } +\cs_generate_variant:Nn \@@_show_long_aux:nnnn { oof } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Messages} +% +% \begin{variable}{\c_@@_show_etc_str} +% When a control sequence (or active character) +% and its meaning are too long to fit in one line +% of the terminal, the end is replaced by this token list. +% \begin{macrocode} +\tl_const:Nx \c_@@_show_etc_str % ( + { \token_to_str:N \ETC.) } +% \end{macrocode} +% \end{variable} +% +% \begin{macrocode} +\__msg_kernel_new:nnn { kernel } { show-tl-analysis } + { + The~token~list~ \tl_if_empty:nF {#1} { #1 ~ } + \tl_if_empty:nTF {#2} + { is~empty } + { contains~the~tokens: } + } +% \end{macrocode} +% +% \begin{macrocode} +%</initex|package> +% \end{macrocode} +% +% \end{implementation} +% +% \PrintIndex |