% \iffalse meta-comment
%
%% File: l3tl-analysis.dtx
%
% Copyright (C) 2011-2019 The LaTeX3 Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version.  The latest version
% of this license is in the file
%
%    https://www.latex-project.org/lppl.txt
%
% This file is part of the "l3kernel bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
%    https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver>
\documentclass[full,kernel]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
%
% \title{^^A
%   The \textsf{l3tl-analysis} package: Analysing token lists^^A
% }
%
% \author{^^A
%  The \LaTeX3 Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released 2019-05-28}
%
% \maketitle
%
% \begin{documentation}
%
% \section{\pkg{l3tl-analysis} documentation}
%
% This module mostly provides internal functions for use in the
% \pkg{l3regex} module. However, it provides as a side-effect a user
% debugging function, very similar to the \cs{ShowTokens} macro from the
% \pkg{ted} package.
%
% \begin{function}[added = 2018-04-09]{\tl_analysis_show:N, \tl_analysis_show:n}
%   \begin{syntax}
%     \cs{tl_analysis_show:n} \Arg{token list}
%   \end{syntax}
%   Displays to the terminal the detailed decomposition of the
%   \meta{token list} into tokens, showing the category code of each
%   character token, the meaning of control sequences and active
%   characters, and the value of registers.
% \end{function}
%
% \begin{function}[added = 2018-04-09]
%   {\tl_analysis_map_inline:nn, \tl_analysis_map_inline:Nn}
%   \begin{syntax}
%     \cs{tl_analysis_map_inline:nn} \Arg{token list} \Arg{inline function}
%   \end{syntax}
%   Applies the \meta{inline function} to each individual \meta{token}
%   in the \meta{token list}. The \meta{inline function} receives three
%   arguments:
%   \begin{itemize}
%     \item \meta{tokens}, which both \texttt{o}-expand and
%       \texttt{x}-expand to the \meta{token}. The detailed form of
%       \meta{token} may change in later releases.
%     \item \meta{char code}, a decimal representation of the character
%       code of the token, $-1$ if it is a control sequence (with
%       \meta{catcode} $0$).
%     \item \meta{catcode}, a capital hexadecimal digit which denotes
%       the category code of the \meta{token} (0: control sequence, 1:
%       begin-group, 2: end-group, 3: math shift, 4: alignment tab, 6:
%       parameter, 7: superscript, 8: subscript, A: space, B: letter,
%       C:other, D:active).
%   \end{itemize}
% \end{function}
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3tl-analysis} implementation}
%
%    \begin{macrocode}
%<@@=tl>
%    \end{macrocode}
%
% \subsection{Internal functions}
%
% \begin{variable}{\s_@@}
%   The format used to store token lists internally uses the scan mark
%   \cs{s_@@} as a delimiter.
% \end{variable}
%
% \subsection{Internal format}
%
% The task of the \pkg{l3tl-analysis} module is to convert token lists
% to an internal format which allows us to extract all the relevant
% information about individual tokens (category code, character code),
% as well as reconstruct the token list quickly. This internal format is
% used in \pkg{l3regex} where we need to support arbitrary tokens, and
% it is used in conversion functions in \pkg{l3str-convert}, where we wish to
% support clusters of characters instead of single tokens.
%
% We thus need a way to encode any \meta{token} (even begin-group and
% end-group character tokens) in a way amenable to manipulating tokens
% individually. The best we can do is to find \meta{tokens} which both
% \texttt{o}-expand and \texttt{x}-expand to the given
% \meta{token}. Collecting more information about the category code and
% character code is also useful for regular expressions, since most
% regexes are catcode-agnostic. The internal format thus takes the form
% of a succession of items of the form
% \begin{quote}
%   \meta{tokens} \cs{s_@@} \meta{catcode} \meta{char code} \cs{s_@@}
% \end{quote}
% The \meta{tokens} \texttt{o}- \emph{and} \texttt{x}-expand to the
% original token in the token list or to the cluster of tokens
% corresponding to one Unicode character in the given encoding (for
% \pkg{l3str-convert}). The \meta{catcode} is given as a single hexadecimal
% digit, $0$ for control sequences. The \meta{char code} is given as a
% decimal number, $-1$ for control sequences.
%
% Using delimited arguments lets us build the \meta{tokens}
% progressively when doing an encoding conversion in \pkg{l3str-convert}. On the
% other hand, the delimiter \cs{s_@@} may not appear unbraced in
% \meta{tokens}. This is not a problem because we are careful to wrap
% control sequences in braces (as an argument to \cs{exp_not:n}) when
% converting from a general token list to the internal format.
%
% The current rule for converting a \meta{token} to a balanced set of
% \meta{tokens} which both \texttt{o}-expands and \texttt{x}-expands to
% it is the following.
% \begin{itemize}
%   \item A control sequence |\cs| becomes |\exp_not:n { \cs }|
%     \cs{s_@@} $0$ $-1$ \cs{s_@@}.
%   \item A begin-group character |{| becomes \cs{exp_after:wN} |{|
%     \cs{if_false:} |}| \cs{fi:} \cs{s_@@} $1$ \meta{char code}
%     \cs{s_@@}.
%   \item An end-group character |}| becomes \cs{if_false:} |{| \cs{fi:}
%     |}| \cs{s_@@} $2$ \meta{char code} \cs{s_@@}.
%   \item A character with any other category code becomes
%     \cs{exp_not:n} \Arg{character} \cs{s_@@} \meta{hex catcode}
%     \meta{char code} \cs{s_@@}.
% \end{itemize}
%
% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char>
% ^^A which does not add the <active char> in memory.
%
%    \begin{macrocode}
%<*initex|package>
%    \end{macrocode}
%
% \subsection{Variables and helper functions}
%
% \begin{variable}{\s_@@}
%   The scan mark \cs{s_@@} is used as a delimiter in the internal
%   format. This is more practical than using a quark, because we would
%   then need to control expansion much more carefully: compare
%   \cs{int_value:w} |`#1| \cs{s_@@} with \cs{int_value:w} |`#1|
%   \cs{exp_stop_f:} \cs{exp_not:N} \cs{q_mark} to extract a character
%   code followed by the delimiter in an \texttt{x}-expansion.
%    \begin{macrocode}
\scan_new:N \s_@@
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_@@_analysis_token}
% \begin{variable}{\l_@@_analysis_char_token}
%   The tokens in the token list are probed with the \TeX{} primitive
%   \tn{futurelet}. We use \cs{l_@@_analysis_token} in that
%   construction. In some cases, we convert the following token to a
%   string before probing it: then the token variable used is
%   \cs{l_@@_analysis_char_token}.
%    \begin{macrocode}
\cs_new_eq:NN \l_@@_analysis_token ?
\cs_new_eq:NN \l_@@_analysis_char_token ?
%    \end{macrocode}
% \end{variable}
% \end{variable}
%
% \begin{variable}{\l_@@_analysis_normal_int}
%   The number of normal (\texttt{N}-type argument) tokens since the
%   last special token.
%    \begin{macrocode}
\int_new:N \l_@@_analysis_normal_int
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_@@_analysis_index_int}
%   During the first pass, this is the index in the array being built.
%   During the second pass, it is equal to the maximum index in the
%   array from the first pass.
%    \begin{macrocode}
\int_new:N \l_@@_analysis_index_int
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_@@_analysis_nesting_int}
%   Nesting depth of explicit begin-group and end-group characters
%   during the first pass. This lets us detect the end of the token list
%   without a reserved end-marker.
%    \begin{macrocode}
\int_new:N \l_@@_analysis_nesting_int
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_@@_analysis_type_int}
%   When encountering special characters, we record their \enquote{type}
%   in this integer.
%    \begin{macrocode}
\int_new:N \l_@@_analysis_type_int
%    \end{macrocode}
% \end{variable}
%
% \begin{variable}{\g_@@_analysis_result_tl}
%   The result of the conversion is stored in this token list, with a
%   succession of items of the form
%   \begin{quote}
%     \meta{tokens} \cs{s_@@} \meta{catcode} \meta{char code} \cs{s_@@}
%   \end{quote}
%    \begin{macrocode}
\tl_new:N \g_@@_analysis_result_tl
%    \end{macrocode}
% \end{variable}
%
% \begin{macro}[EXP]{\@@_analysis_extract_charcode:}
% \begin{macro}[EXP]{\@@_analysis_extract_charcode_aux:w}
%   Extracting the character code from the meaning of
%   \cs{l_@@_analysis_token}. This has no error checking, and should
%   only be assumed to work for begin-group and end-group character
%   tokens. It produces a number in the form |`|\meta{char}.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_extract_charcode:
  {
    \exp_after:wN \@@_analysis_extract_charcode_aux:w
      \token_to_meaning:N \l_@@_analysis_token
  }
\cs_new:Npn \@@_analysis_extract_charcode_aux:w #1 ~ #2 ~ { ` }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_analysis_cs_space_count:NN}
% \begin{macro}[EXP]{\@@_analysis_cs_space_count:w}
% \begin{macro}[EXP]{\@@_analysis_cs_space_count_end:w}
%   Counts the number of spaces in the string representation of its
%   second argument, as well as the number of characters following the
%   last space in that representation, and feeds the two numbers as
%   semicolon-delimited arguments to the first argument. When this
%   function is used, the escape character is printable and non-space.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_cs_space_count:NN #1 #2
  {
    \exp_after:wN #1
    \int_value:w \int_eval:w 0
      \exp_after:wN \@@_analysis_cs_space_count:w
        \token_to_str:N #2
        \fi: \@@_analysis_cs_space_count_end:w ; ~ !
  }
\cs_new:Npn \@@_analysis_cs_space_count:w #1 ~
  {
    \if_false: #1 #1 \fi:
    + 1
    \@@_analysis_cs_space_count:w
  }
\cs_new:Npn \@@_analysis_cs_space_count_end:w ; #1 \fi: #2 !
  { \exp_after:wN ; \int_value:w \str_count_ignore_spaces:n {#1} ; }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Plan of attack}
%
% Our goal is to produce a token list of the form roughly
% \begin{quote}
%   \meta{token 1} \cs{s@__} \meta{catcode 1} \meta{char code 1} \cs{s@__} \\
%   \meta{token 2} \cs{s_@@} \meta{catcode 2} \meta{char code 2} \cs{s_@@} \\
%   \ldots{}
%   \meta{token N} \cs{s_@@} \meta{catcode N} \meta{char code N} \cs{s_@@}
% \end{quote}
% Most but not all tokens can be grabbed as an undelimited
% (\texttt{N}-type) argument by \TeX{}. The plan is to have a two pass
% system. In the first pass, locate special tokens, and store them in
% various \tn{toks} registers. In the second pass, which is done within
% an \texttt{x}-expanding assignment, normal tokens are taken in as
% \texttt{N}-type arguments, and special tokens are retrieved from the
% \tn{toks} registers, and removed from the input stream by some means.
% The whole process takes linear time, because we avoid building the
% result one item at a time.
%
% We make the escape character printable (backslash, but this later
% oscillates between slash and backslash): this allows us to
% distinguish characters from control sequences.
%
% A token has two characteristics: its \tn{meaning}, and what it looks
% like for \TeX{} when it is in scanning mode (\emph{e.g.}, when
% capturing parameters for a macro). For our purposes, we distinguish
% the following meanings:
% \begin{itemize}
%   \item begin-group token (category code $1$), either space (character
%     code $32$), or non-space;
%   \item end-group token (category code $2$), either space (character
%     code $32$), or non-space;
%   \item space token (category code $10$, character code $32$);
%   \item anything else (then the token is always an \texttt{N}-type
%     argument).
% \end{itemize}
% The token itself can \enquote{look like} one of the following
% \begin{itemize}
%   \item a non-active character, in which case its meaning is
%     automatically that associated to its character code and category
%     code, we call it \enquote{true} character;
%   \item an active character;
%   \item a control sequence.
% \end{itemize}
% The only tokens which are not valid \texttt{N}-type arguments are true
% begin-group characters, true end-group characters, and true spaces.
% We detect those characters by scanning ahead with \tn{futurelet},
% then distinguishing true characters from control sequences set equal
% to them using the \tn{string} representation.
%
% The second pass is a simple exercise in expandable loops.
%
% \begin{macro}{\@@_analysis:n}
%   Everything is done within a group, and all definitions are
%   local. We use \cs{group_align_safe_begin/end:} to avoid problems in
%   case \cs{@@_analysis:n} is used within an alignment and its argument
%   contains alignment tab tokens.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis:n #1
  {
    \group_begin:
      \group_align_safe_begin:
        \@@_analysis_a:n {#1}
        \@@_analysis_b:n {#1}
      \group_align_safe_end:
    \group_end:
  }
%    \end{macrocode}
% \end{macro}
%
% \subsection{Disabling active characters}
%
% \begin{macro}{\@@_analysis_disable:n}
%   Active characters can cause problems later on in the processing, so
%   we provide a way to disable them, by setting them to
%   \texttt{undefined}. Since Unicode contains too many characters to
%   loop over all of them, we instead do this whenever we encounter a
%   character.  For \pTeX{} and \upTeX{} we skip characters beyond
%   $[0,255]$ because \tn{lccode} only allows those values.
%    \begin{macrocode}
\group_begin:
  \char_set_catcode_active:N \^^@
  \cs_new_protected:Npn \@@_analysis_disable:n #1
    {
      \tex_lccode:D 0 = #1 \exp_stop_f:
      \tex_lowercase:D { \tex_let:D ^^@ } \tex_undefined:D
    }
  \bool_lazy_or:nnT
    { \sys_if_engine_ptex_p: }
    { \sys_if_engine_uptex_p: }
    {
      \cs_gset_protected:Npn \@@_analysis_disable:n #1
        {
          \if_int_compare:w 256 > #1 \exp_stop_f:
            \tex_lccode:D 0 = #1 \exp_stop_f:
            \tex_lowercase:D { \tex_let:D ^^@ } \tex_undefined:D
          \fi:
        }
    }
\group_end:
%    \end{macrocode}
% \end{macro}
%
% \subsection{First pass}
%
% The goal of this pass is to detect special (non-\texttt{N}-type) tokens,
% and count how many \texttt{N}-type tokens lie between special tokens.
% Also, we wish to store some representation of each special token
% in a \tn{toks} register.
%
% We have $11$ types of tokens:
% \begin{itemize}
% \item[1.] a true non-space begin-group character;
% \item[2.] a true space begin-group character;
% \item[3.] a true non-space end-group character;
% \item[4.] a true space end-group character;
% \item[5.] a true space blank space character;
% \item[6.] an active character;
% \item[7.] any other true character;
% \item[8.] a control sequence equal to a begin-group token (category code $1$);
% \item[9.] a control sequence equal to an end-group token (category code $2$);
% \item[10.] a control sequence equal to a space token
%   (character code $32$, category code $10$);
% \item[11.] any other control sequence.
% \end{itemize}
% Our first tool is \tn{futurelet}. This cannot distinguish
% case $8$ from $1$ or $2$, nor case $9$ from $3$ or $4$,
% nor case $10$ from case $5$. Those cases are later distinguished
% by applying the \tn{string} primitive to the following token,
% after possibly changing the escape character to ensure that
% a control sequence's string representation cannot be mistaken
% for the true character.
%
% In cases $6$, $7$, and $11$, the following token is a valid
% \texttt{N}-type argument, so we grab it and distinguish the case
% of a character from a control sequence: in the latter case,
% \cs{str_tail:n} \Arg{token} is non-empty, because the
% escape character is printable.
%
% \begin{macro}{\@@_analysis_a:n}
%   We read tokens one by one using \tn{futurelet}.
%   While performing the loop, we keep track of the number of
%   true begin-group characters minus the number of
%   true end-group characters in \cs{l_@@_analysis_nesting_int}.
%   This reaches $-1$ when we read the closing brace.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_a:n #1
  {
    \@@_analysis_disable:n { 32 }
    \int_set:Nn \tex_escapechar:D { 92 }
    \int_zero:N \l_@@_analysis_normal_int
    \int_zero:N \l_@@_analysis_index_int
    \int_zero:N \l_@@_analysis_nesting_int
    \if_false: { \fi: \@@_analysis_a_loop:w #1 }
    \int_decr:N \l_@@_analysis_index_int
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\@@_analysis_a_loop:w}
%   Read one character and check its type.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_a_loop:w
  { \tex_futurelet:D \l_@@_analysis_token \@@_analysis_a_type:w }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\@@_analysis_a_type:w}
%   At this point, \cs{l_@@_analysis_token} holds the meaning
%   of the following token. We store in \cs{l_@@_analysis_type_int}
%   information about the meaning of the token ahead:
%   \begin{itemize}
%   \item 0 space token;
%   \item 1 begin-group token;
%   \item -1 end-group token;
%   \item 2 other.
%   \end{itemize}
%   The values $0$, $1$, $-1$ correspond to how much a true such
%   character changes the nesting level ($2$ is used only here,
%   and is irrelevant later). Then call the auxiliary for each case.
%   Note that nesting conditionals here is safe because we only skip
%   over \cs{l_@@_analysis_token} if it matches with one of the
%   character tokens (hence is not a primitive conditional).
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_a_type:w
  {
    \l_@@_analysis_type_int =
      \if_meaning:w \l_@@_analysis_token \c_space_token
        0
      \else:
        \if_catcode:w \exp_not:N \l_@@_analysis_token \c_group_begin_token
          1
        \else:
          \if_catcode:w \exp_not:N \l_@@_analysis_token \c_group_end_token
            - 1
          \else:
            2
          \fi:
        \fi:
      \fi:
      \exp_stop_f:
    \if_case:w \l_@@_analysis_type_int
         \exp_after:wN \@@_analysis_a_space:w
    \or: \exp_after:wN \@@_analysis_a_bgroup:w
    \or: \exp_after:wN \@@_analysis_a_safe:N
    \else: \exp_after:wN \@@_analysis_a_egroup:w
    \fi:
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\@@_analysis_a_space:w}
% \begin{macro}{\@@_analysis_a_space_test:w}
%   In this branch, the following token's meaning is a blank space.
%   Apply \tn{string} to that token: a true blank space gives a space, a
%   control sequence gives a result starting with the escape character,
%   an active character gives something else than a space since we
%   disabled the space.  We grab as \cs{l_@@_analysis_char_token} the first
%   character of the string representation then test it in
%   \cs{@@_analysis_a_space_test:w}.
%   Also, since \cs{@@_analysis_a_store:} expects the special token to be
%   stored in the relevant \tn{toks} register, we do that. The extra
%   \cs{exp_not:n} is unnecessary of course, but it makes the treatment
%   of all tokens more homogeneous.
%   If we discover that the next token was actually a control sequence
%   or an active character
%   instead of a true space, then we step the counter of normal tokens.
%   We now have in front of us the whole string representation of
%   the control sequence, including potential spaces; those will appear
%   to be true spaces later in this pass. Hence, all other branches of
%   the code in this first pass need to consider the string representation,
%   so that the second pass does not need to test the meaning of tokens,
%   only strings.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_a_space:w
  {
    \tex_afterassignment:D \@@_analysis_a_space_test:w
    \exp_after:wN \cs_set_eq:NN
    \exp_after:wN \l_@@_analysis_char_token
    \token_to_str:N
  }
\cs_new_protected:Npn \@@_analysis_a_space_test:w
  {
    \if_meaning:w \l_@@_analysis_char_token \c_space_token
      \tex_toks:D \l_@@_analysis_index_int { \exp_not:n { ~ } }
      \@@_analysis_a_store:
    \else:
      \int_incr:N \l_@@_analysis_normal_int
    \fi:
    \@@_analysis_a_loop:w
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\@@_analysis_a_bgroup:w, \@@_analysis_a_egroup:w}
% \begin{macro}
%   {\@@_analysis_a_group:nw, \@@_analysis_a_group_aux:w, \@@_analysis_a_group_auxii:w, \@@_analysis_a_group_test:w}
%   The token is most likely a true character token with catcode $1$ or
%   $2$, but it might be a control sequence, or an active character.
%   Optimizing for the first case, we store in a toks register some code
%   that expands to that token.  Since we will turn what follows into
%   a string, we make sure the escape character is different from the
%   current character code (by switching between solidus and backslash).
%   To detect the special case of an active character let to the catcode
%   $1$ or~$2$ character with the same character code, we disable the
%   active character with that character code and re-test: if the
%   following token has become undefined we can in fact safely grab it.
%   We are finally ready to turn what follows to a string and test it.
%   This is one place where we need \cs{l_@@_analysis_char_token} to be a
%   separate control sequence from \cs{l_@@_analysis_token}, to compare them.
%    \begin{macrocode}
\group_begin:
  \char_set_catcode_group_begin:N \^^@ % {
  \cs_new_protected:Npn \@@_analysis_a_bgroup:w
    { \@@_analysis_a_group:nw { \exp_after:wN ^^@ \if_false: } \fi: } }
  \char_set_catcode_group_end:N \^^@
  \cs_new_protected:Npn \@@_analysis_a_egroup:w
    { \@@_analysis_a_group:nw { \if_false: { \fi: ^^@ } } % }
\group_end:
\cs_new_protected:Npn \@@_analysis_a_group:nw #1
  {
    \tex_lccode:D 0 = \@@_analysis_extract_charcode: \scan_stop:
    \tex_lowercase:D { \tex_toks:D \l_@@_analysis_index_int {#1} }
    \if_int_compare:w \tex_lccode:D 0 = \tex_escapechar:D
      \int_set:Nn \tex_escapechar:D { 139 - \tex_escapechar:D }
    \fi:
    \@@_analysis_disable:n { \tex_lccode:D 0 }
    \tex_futurelet:D \l_@@_analysis_token \@@_analysis_a_group_aux:w
  }
\cs_new_protected:Npn \@@_analysis_a_group_aux:w
  {
    \if_meaning:w \l_@@_analysis_token \tex_undefined:D
      \exp_after:wN \@@_analysis_a_safe:N
    \else:
      \exp_after:wN \@@_analysis_a_group_auxii:w
    \fi:
  }
\cs_new_protected:Npn \@@_analysis_a_group_auxii:w
  {
    \tex_afterassignment:D \@@_analysis_a_group_test:w
    \exp_after:wN \cs_set_eq:NN
    \exp_after:wN \l_@@_analysis_char_token
    \token_to_str:N
  }
\cs_new_protected:Npn \@@_analysis_a_group_test:w
  {
    \if_charcode:w \l_@@_analysis_token \l_@@_analysis_char_token
      \@@_analysis_a_store:
    \else:
      \int_incr:N \l_@@_analysis_normal_int
    \fi:
    \@@_analysis_a_loop:w
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\@@_analysis_a_store:}
%   This function is called each time we meet a special token;
%   at this point, the \tn{toks} register \cs{l_@@_analysis_index_int}
%   holds a token list which expands to the given special token.
%   Also, the value of \cs{l_@@_analysis_type_int} indicates which case
%   we are in:
%   \begin{itemize}
%   \item -1 end-group character;
%   \item 0 space character;
%   \item 1 begin-group character.
%   \end{itemize}
%   We need to distinguish further the case of a space character
%   (code $32$) from other character codes, because those
%   behave differently in the second pass. Namely, after testing
%   the \tn{lccode} of $0$ (which holds the present character code)
%   we change the cases above to
%   \begin{itemize}
%   \item -2 space end-group character;
%   \item -1 non-space end-group character;
%   \item 0 space blank space character;
%   \item 1 non-space begin-group character;
%   \item 2 space begin-group character.
%   \end{itemize}
%   This has the property that non-space characters correspond to odd
%   values of \cs{l_@@_analysis_type_int}.  The number of normal tokens until
%   here and the type of special token are packed into a \tn{skip}
%   register.  Finally, we check whether we reached the last closing
%   brace, in which case we stop by disabling the looping function
%   (locally).
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_a_store:
  {
    \tex_advance:D \l_@@_analysis_nesting_int \l_@@_analysis_type_int
    \if_int_compare:w \tex_lccode:D 0 = `\ \exp_stop_f:
      \tex_advance:D \l_@@_analysis_type_int \l_@@_analysis_type_int
    \fi:
    \tex_skip:D \l_@@_analysis_index_int
      = \l_@@_analysis_normal_int sp
         plus \l_@@_analysis_type_int sp \scan_stop:
    \int_incr:N \l_@@_analysis_index_int
    \int_zero:N \l_@@_analysis_normal_int
    \if_int_compare:w \l_@@_analysis_nesting_int = -1 \exp_stop_f:
      \cs_set_eq:NN \@@_analysis_a_loop:w \scan_stop:
    \fi:
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\@@_analysis_a_safe:N}
% \begin{macro}{\@@_analysis_a_cs:ww}
%   This should be the simplest case: since the upcoming token is safe,
%   we can simply grab it in a second pass. If the token is a single
%   character (including space), the \cs{if_charcode:w} test yields
%   true; we disable a potentially active character (that could
%   otherwise masquerade as the true character in the next pass) and we
%   count one \enquote{normal} token. On the other
%   hand, if the token is a control sequence, we should replace it by
%   its string representation for compatibility with other code
%   branches. Instead of slowly looping through the characters with
%   the main code, we use the knowledge of how the second pass works:
%   if the control sequence name contains no space, count that token
%   as a number of normal tokens equal to its string length. If the
%   control sequence contains spaces, they should be registered as
%   special characters by increasing \cs{l_@@_analysis_index_int}
%   (no need to carefully count character between each space), and
%   all characters after the last space should be counted in the
%   following sequence of \enquote{normal} tokens.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_a_safe:N #1
  {
    \if_charcode:w
        \scan_stop:
        \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing:
        \scan_stop:
      \exp_after:wN \use_i:nn
    \else:
      \exp_after:wN \use_ii:nn
    \fi:
      {
        \@@_analysis_disable:n { `#1 }
        \int_incr:N \l_@@_analysis_normal_int
      }
      { \@@_analysis_cs_space_count:NN \@@_analysis_a_cs:ww #1 }
    \@@_analysis_a_loop:w
  }
\cs_new_protected:Npn \@@_analysis_a_cs:ww #1; #2;
  {
    \if_int_compare:w #1 > 0 \exp_stop_f:
      \tex_skip:D \l_@@_analysis_index_int
        = \int_eval:n { \l_@@_analysis_normal_int + 1 } sp \exp_stop_f:
      \tex_advance:D \l_@@_analysis_index_int #1 \exp_stop_f:
    \else:
      \tex_advance:D
    \fi:
    \l_@@_analysis_normal_int #2 \exp_stop_f:
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \subsection{Second pass}
%
% The second pass is an exercise in expandable loops.
% All the necessary information is stored in \tn{skip}
% and \tn{toks} registers.
%
% \begin{macro}{\@@_analysis_b:n}
% \begin{macro}[EXP]{\@@_analysis_b_loop:w}
%   Start the loop with the index $0$. No need for an end-marker:
%   the loop stops by itself when the last index is read.
%   We repeatedly oscillate between reading long stretches
%   of normal tokens, and reading special tokens.
%    \begin{macrocode}
\cs_new_protected:Npn \@@_analysis_b:n #1
  {
    \tl_gset:Nx \g_@@_analysis_result_tl
      {
        \@@_analysis_b_loop:w 0; #1
        \prg_break_point:
      }
  }
\cs_new:Npn \@@_analysis_b_loop:w #1;
  {
    \exp_after:wN \@@_analysis_b_normals:ww
      \int_value:w \tex_skip:D #1 ; #1 ;
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_analysis_b_normals:ww}
% \begin{macro}[EXP]{\@@_analysis_b_normal:wwN}
%   The first argument is the number of normal tokens which remain
%   to be read, and the second argument is the index in the array
%   produced in the first step.
%   A character's string representation is always one character long,
%   while a control sequence is always longer (we have set the escape
%   character to a printable value). In both cases, we leave
%   \cs{exp_not:n} \Arg{token} \cs{s_@@} in the input stream
%   (after \texttt{x}-expansion). Here, \cs{exp_not:n} is used
%   rather than \cs{exp_not:N} because |#3| could be
%   a macro parameter character or could be \cs{s_@@}
%   (which must be hidden behind braces in the result).
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_b_normals:ww #1;
  {
    \if_int_compare:w #1 = 0 \exp_stop_f:
      \@@_analysis_b_special:w
    \fi:
    \@@_analysis_b_normal:wwN #1;
  }
\cs_new:Npn \@@_analysis_b_normal:wwN #1; #2; #3
  {
    \exp_not:n { \exp_not:n { #3 } } \s_@@
    \if_charcode:w
        \scan_stop:
        \exp_after:wN \use_none:n \token_to_str:N #3 \prg_do_nothing:
        \scan_stop:
      \exp_after:wN \@@_analysis_b_char:Nww
    \else:
      \exp_after:wN \@@_analysis_b_cs:Nww
    \fi:
    #3 #1; #2;
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_analysis_b_char:Nww}
%   If the normal token we grab is a character, leave
%   \meta{catcode} \meta{charcode} followed by \cs{s_@@}
%   in the input stream, and call \cs{@@_analysis_b_normals:ww}
%   with its first argument decremented.
%    \begin{macrocode}
\cs_new:Npx \@@_analysis_b_char:Nww #1
  {
    \exp_not:N \if_meaning:w #1 \exp_not:N \tex_undefined:D
      \token_to_str:N D \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_catcode_other_token
      \token_to_str:N C \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_catcode_letter_token
      \token_to_str:N B \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_math_toggle_token      3
      \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_alignment_token        4
      \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_math_superscript_token 7
      \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_math_subscript_token   8
      \exp_not:N \else:
    \exp_not:N \if_catcode:w #1 \c_space_token
      \token_to_str:N A \exp_not:N \else:
      6
    \exp_not:n { \fi: \fi: \fi: \fi: \fi: \fi: \fi: \fi: }
    \exp_not:N \int_value:w `#1 \s_@@
   \exp_not:N \exp_after:wN \exp_not:N \@@_analysis_b_normals:ww
     \exp_not:N \int_value:w \exp_not:N \int_eval:w - 1 +
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_analysis_b_cs:Nww}
% \begin{macro}[EXP]{\@@_analysis_b_cs_test:ww}
%   If the token we grab is a control sequence, leave
%   |0 -1| (as category code and character code) in the input stream,
%   followed by \cs{s_@@},
%   and call \cs{@@_analysis_b_normals:ww} with updated arguments.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_b_cs:Nww #1
  {
    0 -1 \s_@@
    \@@_analysis_cs_space_count:NN \@@_analysis_b_cs_test:ww #1
  }
\cs_new:Npn \@@_analysis_b_cs_test:ww #1 ; #2 ; #3 ; #4 ;
  {
    \exp_after:wN \@@_analysis_b_normals:ww
    \int_value:w \int_eval:w
    \if_int_compare:w #1 = 0 \exp_stop_f:
      #3
    \else:
      \tex_skip:D \int_eval:n { #4 + #1 } \exp_stop_f:
    \fi:
    - #2
    \exp_after:wN ;
    \int_value:w \int_eval:n { #4 + #1 } ;
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_analysis_b_special:w}
% \begin{macro}[EXP]{\@@_analysis_b_special_char:wN}
% \begin{macro}[EXP]{\@@_analysis_b_special_space:w}
%   Here, |#1| is the current index in the array built in the first pass.
%   Check now whether we reached the end (we shouldn't keep the trailing
%   end-group character that marked the end of the token list in the
%   first pass).
%   Unpack the \tn{toks} register: when \texttt{x}-expanding again,
%   we will get the special token.
%   Then leave the category code in the input stream, followed by
%   the character code, and call \cs{@@_analysis_b_loop:w} with the next index.
%    \begin{macrocode}
\group_begin:
  \char_set_catcode_other:N A
  \cs_new:Npn \@@_analysis_b_special:w
      \fi: \@@_analysis_b_normal:wwN 0 ; #1 ;
    {
      \fi:
      \if_int_compare:w #1 = \l_@@_analysis_index_int
        \exp_after:wN \prg_break:
      \fi:
      \tex_the:D \tex_toks:D #1 \s_@@
      \if_case:w \tex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
             \token_to_str:N A
      \or:   1
      \or:   1
      \else: 2
      \fi:
      \if_int_odd:w \tex_gluestretch:D \tex_skip:D #1 \exp_stop_f:
        \exp_after:wN \@@_analysis_b_special_char:wN \int_value:w
      \else:
        \exp_after:wN \@@_analysis_b_special_space:w \int_value:w
      \fi:
      \int_eval:n { 1 + #1 } \exp_after:wN ;
      \token_to_str:N
    }
\group_end:
\cs_new:Npn \@@_analysis_b_special_char:wN #1 ; #2
  {
    \int_value:w `#2 \s_@@
    \@@_analysis_b_loop:w #1 ;
  }
\cs_new:Npn \@@_analysis_b_special_space:w #1 ; ~
  {
    32 \s_@@
    \@@_analysis_b_loop:w #1 ;
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Mapping through the analysis}
%
% \begin{macro}{\tl_analysis_map_inline:nn, \tl_analysis_map_inline:Nn}
% \begin{macro}{\@@_analysis_map_inline_aux:Nn}
% \begin{macro}{\@@_analysis_map_inline_aux:nnn}
%   First obtain the analysis of the token list into
%   \cs{g_@@_analysis_result_tl}. To allow nested mappings, increase the
%   nesting depth \cs{g__kernel_prg_map_int} (shared between all modules), then
%   define the looping macro, which has a name specific to that nesting
%   depth. That looping grabs the \meta{tokens}, \meta{catcode} and
%   \meta{char code}; it checks for the end of the loop with
%   \cs{use_none:n} |##2|, normally empty, but which becomes
%   \cs{tl_map_break:} at the end; it then performs the user's code
%   |#2|, and loops by calling itself. When the loop ends, remember to
%   decrease the nesting depth.
%    \begin{macrocode}
\cs_new_protected:Npn \tl_analysis_map_inline:nn #1
  {
    \@@_analysis:n {#1}
    \int_gincr:N \g__kernel_prg_map_int
    \exp_args:Nc \@@_analysis_map_inline_aux:Nn
      { @@_analysis_map_inline_ \int_use:N \g__kernel_prg_map_int :wNw }
  }
\cs_new_protected:Npn  \tl_analysis_map_inline:Nn #1
  { \exp_args:No \tl_analysis_map_inline:nn #1 }
\cs_new_protected:Npn \@@_analysis_map_inline_aux:Nn #1#2
  {
    \cs_gset_protected:Npn #1 ##1 \s_@@ ##2 ##3 \s_@@
      {
        \use_none:n ##2
        \@@_analysis_map_inline_aux:nnn {##1} {##3} {##2}
      }
    \cs_gset_protected:Npn \@@_analysis_map_inline_aux:nnn ##1##2##3
      {
        #2
        #1
      }
    \exp_after:wN #1
      \g_@@_analysis_result_tl
      \s_@@ { ? \tl_map_break: } \s_@@
    \prg_break_point:Nn \tl_map_break:
      { \int_gdecr:N \g__kernel_prg_map_int }
  }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Showing the results}
%
% \begin{macro}{\tl_analysis_show:N, \tl_analysis_show:n}
%   Add to \cs{@@_analysis:n} a third pass to display tokens to the terminal.
%   If the token list variable is not defined, throw the same error
%   as \cs{tl_show:N} by simply calling that function.
%    \begin{macrocode}
\cs_new_protected:Npn \tl_analysis_show:N #1
  {
    \tl_if_exist:NTF #1
      {
        \exp_args:No \@@_analysis:n {#1}
        \msg_show:nnxxxx { LaTeX / kernel } { show-tl-analysis }
          { \token_to_str:N #1 } { \@@_analysis_show: } { } { }
      }
      { \tl_show:N #1 }
  }
\cs_new_protected:Npn \tl_analysis_show:n #1
  {
    \@@_analysis:n {#1}
    \msg_show:nnxxxx { LaTeX / kernel } { show-tl-analysis }
      { } { \@@_analysis_show: } { } { }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[rEXP]{\@@_analysis_show:, \@@_analysis_show_loop:wNw}
%   Here, |#1| \texttt{o}- and \texttt{x}-expands to the token;
%   |#2| is the category code (one uppercase hexadecimal digit),
%   $0$ for control sequences;
%   |#3| is the character code, which we ignore.
%   In the cases of control sequences and active characters,
%   the meaning may overflow one line, and we want to truncate
%   it. Those cases are thus separated out.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_show:
  {
    \exp_after:wN \@@_analysis_show_loop:wNw \g_@@_analysis_result_tl
    \s_@@ { ? \prg_break: } \s_@@
    \prg_break_point:
  }
\cs_new:Npn \@@_analysis_show_loop:wNw #1 \s_@@ #2 #3 \s_@@
  {
    \use_none:n #2
    \iow_newline: > \use:nn { ~ } { ~ }
    \if_int_compare:w "#2 = 0 \exp_stop_f:
      \exp_after:wN \@@_analysis_show_cs:n
    \else:
      \if_int_compare:w "#2 = 13 \exp_stop_f:
        \exp_after:wN \exp_after:wN
        \exp_after:wN \@@_analysis_show_active:n
      \else:
        \exp_after:wN \exp_after:wN
        \exp_after:wN \@@_analysis_show_normal:n
      \fi:
    \fi:
    {#1}
    \@@_analysis_show_loop:wNw
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[rEXP]{\@@_analysis_show_normal:n}
%   Non-active characters are a simple matter of printing
%   the character, and its meaning. Our test suite checks that
%   begin-group and end-group characters do not mess up
%   \TeX{}'s alignment status.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_show_normal:n #1
  {
    \exp_after:wN \token_to_str:N #1 ~
    ( \exp_after:wN \token_to_meaning:N #1 )
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[EXP]{\@@_analysis_show_value:N}
%   This expands to the value of |#1| if it has any.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_show_value:N #1
  {
    \token_if_expandable:NF #1
      {
        \token_if_chardef:NTF       #1 \prg_break: { }
        \token_if_mathchardef:NTF   #1 \prg_break: { }
        \token_if_dim_register:NTF  #1 \prg_break: { }
        \token_if_int_register:NTF  #1 \prg_break: { }
        \token_if_skip_register:NTF #1 \prg_break: { }
        \token_if_toks_register:NTF #1 \prg_break: { }
        \use_none:nnn
        \prg_break_point:
        \use:n { \exp_after:wN = \tex_the:D #1 }
      }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}[rEXP]{\@@_analysis_show_cs:n}
% \begin{macro}[rEXP]{\@@_analysis_show_active:n}
% \begin{macro}[rEXP]{\@@_analysis_show_long:nn}
% \begin{macro}[rEXP]{\@@_analysis_show_long_aux:nnnn}
%   Control sequences and active characters are printed in the same way,
%   making sure not to go beyond the \cs{l_iow_line_count_int}. In case
%   of an overflow, we replace the last characters by
%   \cs{c_@@_analysis_show_etc_str}.
%    \begin{macrocode}
\cs_new:Npn \@@_analysis_show_cs:n #1
  { \exp_args:No \@@_analysis_show_long:nn {#1} { control~sequence= } }
\cs_new:Npn \@@_analysis_show_active:n #1
  { \exp_args:No \@@_analysis_show_long:nn {#1} { active~character= } }
\cs_new:Npn \@@_analysis_show_long:nn #1
  {
    \@@_analysis_show_long_aux:oofn
      { \token_to_str:N #1 }
      { \token_to_meaning:N #1 }
      { \@@_analysis_show_value:N #1 }
  }
\cs_new:Npn \@@_analysis_show_long_aux:nnnn #1#2#3#4
  {
    \int_compare:nNnTF
      { \str_count:n { #1 ~ ( #4 #2 #3 ) } }
      > { \l_iow_line_count_int - 3 }
      {
        \str_range:nnn { #1 ~ ( #4 #2 #3 ) } { 1 }
          {
            \l_iow_line_count_int - 3
            - \str_count:N \c_@@_analysis_show_etc_str
          }
        \c_@@_analysis_show_etc_str
      }
      { #1 ~ ( #4 #2 #3 ) }
  }
\cs_generate_variant:Nn \@@_analysis_show_long_aux:nnnn { oof }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Messages}
%
% \begin{variable}{\c_@@_analysis_show_etc_str}
%   When a control sequence (or active character)
%   and its meaning are too long to fit in one line
%   of the terminal, the end is replaced by this token list.
%    \begin{macrocode}
\tl_const:Nx \c_@@_analysis_show_etc_str % (
  { \token_to_str:N \ETC.) }
%    \end{macrocode}
% \end{variable}
%
%    \begin{macrocode}
\__kernel_msg_new:nnn { kernel } { show-tl-analysis }
  {
    The~token~list~ \tl_if_empty:nF {#1} { #1 ~ }
    \tl_if_empty:nTF {#2}
      { is~empty }
      { contains~the~tokens: #2 }
  }
%    \end{macrocode}
%
%    \begin{macrocode}
%</initex|package>
%    \end{macrocode}
%
% \end{implementation}
%
% \PrintIndex