summaryrefslogtreecommitdiff
path: root/macros/latex/contrib/l3kernel/l3tl-analysis.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex/contrib/l3kernel/l3tl-analysis.dtx')
-rw-r--r--macros/latex/contrib/l3kernel/l3tl-analysis.dtx103
1 files changed, 41 insertions, 62 deletions
diff --git a/macros/latex/contrib/l3kernel/l3tl-analysis.dtx b/macros/latex/contrib/l3kernel/l3tl-analysis.dtx
index 4c85ed51d9..9ff2ec04e7 100644
--- a/macros/latex/contrib/l3kernel/l3tl-analysis.dtx
+++ b/macros/latex/contrib/l3kernel/l3tl-analysis.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2024-01-22}
+% \date{Released 2024-02-13}
%
% \maketitle
%
@@ -196,18 +196,15 @@
% \end{variable}
%
% \begin{variable}
-% {\l_@@_analysis_token, \l_@@_analysis_char_token, \l_@@_analysis_next_token}
+% {\l_@@_analysis_token, \l_@@_analysis_char_token}
% The tokens in the token list are probed with the \TeX{} primitive
% \tn{futurelet}. We use \cs{l_@@_analysis_token} in that
% construction. In some cases, we convert the following token to a
% string before probing it: then the token variable used is
-% \cs{l_@@_analysis_char_token}. When getting tokens from the input
-% stream we may need to look two tokens ahead, for which we use
-% \cs{l_@@_analysis_next_token}.
+% \cs{l_@@_analysis_char_token}.
% \begin{macrocode}
\cs_new_eq:NN \l_@@_analysis_token ?
\cs_new_eq:NN \l_@@_analysis_char_token ?
-\cs_new_eq:NN \l_@@_analysis_next_token ?
% \end{macrocode}
% \end{variable}
%
@@ -1223,13 +1220,12 @@
% \@@_peek_analysis_nonexp:N, \@@_peek_analysis_cs:N,
% \@@_peek_analysis_char:N, \@@_peek_analysis_char:w,
% \@@_peek_analysis_special:, \@@_peek_analysis_retest:,
-% \@@_peek_analysis_next:, \@@_peek_analysis_nextii:,
% \@@_peek_analysis_str:,
% \@@_peek_analysis_str:w, \@@_peek_analysis_str:n,
% \@@_peek_analysis_active_str:n, \@@_peek_analysis_explicit:n,
% \@@_peek_analysis_escape:, \@@_peek_analysis_collect:w,
% \@@_peek_analysis_collect:n, \@@_peek_analysis_collect_loop:,
-% \@@_peek_analysis_collect_test:, \@@_peek_analysis_collect_end:NNN
+% \@@_peek_analysis_collect_test:, \@@_peek_analysis_collect_end:NNNN
% }
% Save the user's code in a control sequence that is suitable for
% nested maps. We may wish to pass to this function an \tn{outer}
@@ -1254,7 +1250,10 @@
#1
\@@_peek_analysis_loop:NNn
\prg_break_point:Nn \peek_analysis_map_break:
- { \group_align_safe_end: }
+ {
+ \int_gdecr:N \g__kernel_prg_map_int
+ \group_align_safe_end:
+ }
}
\@@_peek_analysis_loop:NNn ? ? ?
}
@@ -1460,7 +1459,7 @@
\if_meaning:w \l_@@_analysis_token \scan_stop:
\exp_after:wN \@@_peek_analysis_normal:N
\else:
- \exp_after:wN \@@_peek_analysis_next:
+ \exp_after:wN \@@_peek_analysis_str:
\fi:
}
% \end{macrocode}
@@ -1469,41 +1468,22 @@
% begin-group or end-group token (catcode $1$ or~$2$), and we excluded
% a few cases that would be difficult later (empty control sequence,
% active character with the same character code as its meaning or as
-% the escape character). Now look at the \meta{next token} following
-% it using a combination of \tn{afterassignment} and \tn{futurelet}.
-% (In fact look twice to reset an internal \TeX{} flag in case the
-% \meta{next token} had been hit with \cs{exp_not:N}.)
-% The syntax of this primitive is \tn{futurelet} \meta{peek token}
-% \meta{first token} \meta{next token}, and it sets \meta{peek token}
-% equal to \meta{next token}. Traditionally, one takes \meta{first
-% token} to be some macro that regains control of the code and, e.g.,
-% analyses \meta{peek token}. Here, both \meta{first token} and
-% \meta{next token} are mostly unknown tokens in the input stream (but
-% we know the \meta{first token} has catcode $1$, $2$ or $10$), where
-% \meta{first token} was already stored as \cs{l_peek_token}, and we
-% regain control using \tn{afterassignment}, which inserts its
-% argument after the assignment, hence after \meta{peek token} but
-% before \meta{first token}.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_peek_analysis_next:
- {
- \tl_if_empty:oT { \tex_the:D \tex_everyeof:D }
- { \tex_everyeof:D { \scan_stop: } }
- \tex_afterassignment:D \@@_peek_analysis_nextii:
- \tex_futurelet:D \l_@@_analysis_next_token
- }
-\cs_new_protected:Npn \@@_peek_analysis_nextii:
- {
- \tex_afterassignment:D \@@_peek_analysis_str:
- \tex_futurelet:D \l_@@_analysis_next_token
- }
-% \end{macrocode}
-% We then hit the \meta{first token} with \cs{token_to_str:N} and grab
-% characters until finding \cs{l_@@_analysis_next_token}. More
+% the escape character). The idea is to apply \cs{token_to_str:N} to
+% the \meta{token} then grab characters (of category code~$12$ except
+% for spaces that have category code~$10$) to reconstruct it. In
+% earlier versions of the code we would peek at the \meta{next token}
+% that lies after \meta{token} in the input stream, which would help
+% us be more accurate in reconstructing the \meta{token} case in edge
+% cases (mentioned below), but this had the side-effect of tokenizing
+% the input stream (turning characters into tokens) farther ahead than
+% needed.
+%
+% We hit the \meta{token} with \cs{token_to_str:N} and start grabbing
+% characters. More
% precisely, by looking at the first character in the string
-% representation of the \meta{first token} we distinguish three cases:
+% representation of the \meta{token} we distinguish three cases:
% a stringified control sequence starts with the escape character; for
-% an explicit character we find that same character; for an explicit
+% an explicit character we find that same character; for an active
% character we find anything else (we made sure to exclude the case of
% an active character whose string representation coincides with the
% other two cases).
@@ -1591,14 +1571,11 @@
% know that until we had run all the various tests including
% stringifying the token. We are thus left with the hard work of
% picking up one by one the characters in the csname (being careful
-% about spaces), until finding a token that matches the \meta{next
-% token} picked up earlier (which was not stringified), such that the
-% control sequence that we found so far indeed has the expected
-% meaning \cs{l_peek_token}. This comparison with \cs{l_peek_token}
-% catches a reasonably common case like \cs{c_group_begin_token} |_|
-% in which the trailing |_| has category code other: without
-% comparison of the constructed csname with \cs{l_peek_token}
-% collection would stop at \cs[no-index]{c}, which is wrong.
+% about spaces), until the constructed csname has the expected
+% meaning. This fails if someone defines a token like
+% \cs[no-index]{bgroup@my} whose string representation starts the same
+% as another token with the same meaning being an implicit character
+% token of category code $1$, $2$, or $10$.
% \begin{macrocode}
\cs_new_protected:Npn \@@_peek_analysis_escape:
{
@@ -1615,25 +1592,27 @@
}
\cs_new_protected:Npn \@@_peek_analysis_collect_loop:
{
- \tex_futurelet:D \l_@@_analysis_token
- \@@_peek_analysis_collect_test:
- }
-\cs_new_protected:Npn \@@_peek_analysis_collect_test:
- {
- \if_meaning:w \l_@@_analysis_token \l_@@_analysis_next_token
- \exp_after:wN \if_meaning:w \cs:w \l_@@_internal_a_tl \cs_end: \l_peek_token
- \@@_peek_analysis_collect_end:NNN
+ \exp_after:wN \if_meaning:w
+ \cs:w
+ \if_cs_exist:w \l_@@_internal_a_tl \cs_end:
+ \l_@@_internal_a_tl
+ \else:
+ c_one % anything short
\fi:
+ \cs_end:
+ \l_peek_token
+ \@@_peek_analysis_collect_end:NNNN
\fi:
- \@@_peek_analysis_collect:w
+ \tex_futurelet:D \l_@@_analysis_token
+ \@@_peek_analysis_collect:w
}
% \end{macrocode}
% End by calling the user code with suitable arguments (here |#1|,
% |#2| are \cs{fi:}), which closes the group begun early on.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_peek_analysis_collect_end:NNN #1#2#3
+\cs_new_protected:Npn \@@_peek_analysis_collect_end:NNNN #1#2#3#4
{
- #1 #2
+ #1
\tl_put_right:Ne \l_@@_peek_code_tl
{
{ \exp_not:N \exp_not:n { \exp_not:c { \l_@@_internal_a_tl } } }