diff options
| author | Karl Berry <karl@freefriends.org> | 2020-12-04 22:23:08 +0000 |
|---|---|---|
| committer | Karl Berry <karl@freefriends.org> | 2020-12-04 22:23:08 +0000 |
| commit | 308fffa9446e3b5d28b6e48b89be81575b565c70 (patch) | |
| tree | b33f5e1cf009bf510dd974451299d08dc0ffab2a /Master/texmf-dist/source/latex/l3kernel | |
| parent | c4c23071af2870bf72f421f1aae5c9544ff3a87d (diff) | |
l3kernel (4dec20)
git-svn-id: svn://tug.org/texlive/trunk@57066 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel')
55 files changed, 1610 insertions, 506 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/expl3.dtx b/Master/texmf-dist/source/latex/l3kernel/expl3.dtx index 59b8cad3a78..fc49c0aa90a 100644 --- a/Master/texmf-dist/source/latex/l3kernel/expl3.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/expl3.dtx @@ -24,7 +24,7 @@ % %<*driver|generic|package|2ekernel> %</driver|generic|package|2ekernel> -\def\ExplFileDate{2020-10-27}% +\def\ExplFileDate{2020-12-03}% %<*driver> \documentclass[full]{l3doc} \usepackage{graphicx} @@ -51,7 +51,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -526,9 +526,20 @@ % \end{arg-description} % There are two other specifiers with more general meanings: % \begin{arg-description} -% \item[D] This means: \textbf{Do not use}. This special case is used -% for \TeX{} primitives. Programmers outside the kernel team should -% not use these functions! +% \item[D] Stands for \textbf{Do not use}. This special case is used +% for \TeX{} primitives. These functions have no standardized +% syntax, they are engine dependent and their name can change +% without warning, thus their use is \emph{strongly discouraged} in +% package code: programmers should instead use the interfaces +% documented in \href{interface3.pdf}{interface3.pdf}%^^A +% \footnote{If a primitive offers a functionality not yet in the +% kernel, programmers and users are encouraged to write to the +% \texttt{LaTeX-L} mailing list +% (\url{mailto:LATEX-L@listserv.uni-heidelberg.de}) describing +% their use-case and intended behaviour, so that a possible +% interface can be discussed. Temporarily, while an interface is +% not provided, programmers may use the procedure described in the +% \href{l3styleguide.pdf}{l3styleguide.pdf}.}. % \item[w] This means that the argument syntax is \enquote{weird} in that it % does not follow any standard rule. It is used for functions with % arguments that take non standard forms: examples are \TeX{}-level @@ -1072,13 +1083,10 @@ % \begin{macrocode} %<*2ekernel|generic> \begingroup - \catcode`\>=12 - \def\aux#1>{} - \def\auxi{c__kernel_expl_date_tl}% - \edef\auxi{\expandafter\aux\meaning\auxi}% + \catcode`\_=11 \expandafter - \ifx\csname\auxi\endcsname\relax - \global\expandafter\let\csname\auxi\endcsname\ExplFileDate + \ifx\csname c__kernel_expl_date_tl\endcsname\relax + \global\let\c__kernel_expl_date_tl\ExplFileDate \fi \endgroup %</2ekernel|generic> diff --git a/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx b/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx index e7f2c4fd634..025f016eb83 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx b/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx index eba03dca568..2139dfae9fd 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3box.dtx b/Master/texmf-dist/source/latex/l3kernel/l3box.dtx index a866d6324b2..3bd2f1de85a 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3box.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3box.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx b/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx index 17f7f33891d..f376f0e0a41 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx b/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx index 6e57d4a1bf0..2918f8cae3d 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx b/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx index 779062a3cff..f45ff58a92b 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx b/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx index 0c0c3c776c0..f4d6810d9a5 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx b/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx index 1d88e3d6ff4..fb04d8eee0c 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx b/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx index 30f0f78b5eb..d05991e37d2 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx b/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx index 6d13ae1b38c..e60766e876f 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx b/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx index 25ad20f7ddf..3eff299b538 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx @@ -79,7 +79,7 @@ and all files in that bundle must be distributed together. % % \title{The \cls{l3doc} class} % \author{\Team} -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % \maketitle % \tableofcontents % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx b/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx index 2e455845db1..7dc8dbe3554 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx @@ -63,7 +63,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx b/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx index 49fafac2964..544887627c4 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3file.dtx b/Master/texmf-dist/source/latex/l3kernel/l3file.dtx index fc6f7260a0b..132741ccdaa 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3file.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3file.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx b/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx index 2475241f91c..f2df290cb5a 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx index 918c7b9941b..b629576584e 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % \maketitle % % \begin{documentation} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx index 60662e35601..ece95374b5d 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx index 4fa89b63556..634c5f6f118 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx index 1b28754a82d..ccfaba64751 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx index b30fd883bd1..77bc35d84de 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx index 2ff5305b4c8..337875215c5 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx index 28ed6bba76e..c2422c1a5e2 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx index 894cf439d97..cc8e6520d37 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx index e8682c57a8f..8ba199e5e46 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx index 03de976829f..5187a83a8e2 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx index 377596fb595..1bd6a3ad4f6 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % \maketitle % % \begin{documentation} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx index eefc4c7ccb0..d602e51b6e1 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx @@ -40,7 +40,7 @@ % {latex-team@latex-project.org}^^A % }^^A % } -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx index 51f1a30c184..2f2dddd6457 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx @@ -49,7 +49,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx index 5d48154ae8c..96a02296b86 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3int.dtx b/Master/texmf-dist/source/latex/l3kernel/l3int.dtx index f246d8400be..90ae8a493ea 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3int.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3int.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx b/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx index 7c9d2372d99..fc9f8bf2c43 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -461,6 +461,51 @@ % \end{macro} % \end{macro} % +% \begin{macro}[rEXP]{\__kernel_intarray_range_to_clist:Nnn, \@@_range_to_clist:ww} +% Loop through part of the array. +% \begin{macrocode} +\cs_new:Npn \__kernel_intarray_range_to_clist:Nnn #1#2#3 + { + \exp_last_unbraced:Nf \use_none:n + { + \exp_after:wN \@@_range_to_clist:ww + \int_value:w \int_eval:w #2 \exp_after:wN ; + \int_value:w \int_eval:w #3 ; + #1 \prg_break_point: + } + } +\cs_new:Npn \@@_range_to_clist:ww #1 ; #2 ; #3 + { + \if_int_compare:w #1 > #2 \exp_stop_f: + \prg_break:n + \fi: + , \__kernel_intarray_item:Nn #3 {#1} + \exp_after:wN \@@_range_to_clist:ww + \int_value:w \int_eval:w #1 + \c_one_int ; #2 ; #3 + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\__kernel_intarray_gset_range_from_clist:Nnn, \@@_gset_range:Nw} +% Loop through part of the array. +% \begin{macrocode} +\cs_new_protected:Npn \__kernel_intarray_gset_range_from_clist:Nnn #1#2#3 + { + \int_set:Nn \l_@@_loop_int {#2} + \@@_gset_range:Nw #1 #3 , , \prg_break_point: + } +\cs_new_protected:Npn \@@_gset_range:Nw #1 #2 , + { + \if_catcode:w \scan_stop: \tl_to_str:n {#2} \scan_stop: + \prg_break:n + \fi: + \__kernel_intarray_gset:Nnn #1 \l_@@_loop_int {#2} + \int_incr:N \l_@@_loop_int + \@@_gset_range:Nw #1 + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\intarray_show:N, \intarray_show:c, \intarray_log:N, \intarray_log:c} % Convert the list to a comma list (with spaces after each comma) % \begin{macrocode} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx b/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx index b844c89385a..76778ce5c29 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -207,6 +207,31 @@ % the \cs{intarray_count:N}, lest a low-level \TeX{} error occur. % \end{function} % +% \begin{function}[rEXP, added = 2020-07-12]{\__kernel_intarray_range_to_clist:Nnn} +% \begin{syntax} +% \cs{__kernel_intarray_range_to_clist:Nnn} \meta{intarray~var} \Arg{start index} \Arg{end index} +% \end{syntax} +% Converts to integer denotations separated by commas the entries of +% the \meta{intarray} from positions \meta{start index} to \meta{end +% index} included. The \meta{start index} and \meta{end index} must +% be suitable for a direct assignment to a \TeX{} count register, must +% be between $1$ and the \cs{intarray_count:N}, and be suitably +% ordered. All tokens have category code other. +% \end{function} +% +% \begin{function}[added = 2020-07-12]{\__kernel_intarray_gset_range_from_clist:Nnn} +% \begin{syntax} +% \cs{__kernel_intarray_gset_range_from_clist:Nnn} \meta{intarray~var} \Arg{start index} \Arg{integer clist} +% \end{syntax} +% Stores the entries of the \meta{clist} as entries of the +% \meta{intarray~var} starting from the \meta{start index}, upwards. +% This is done without any bound checking. The \meta{start index} and +% all entries of the \meta{integer comma list} (which do not undergo +% space trimming and brace stripping as in normal clist mappings) must +% be suitable for a direct assignment to a \TeX{} count register. An +% empty entry may stop the loop. +% \end{function} +% % \begin{function}{\__kernel_ior_open:Nn, \__kernel_ior_open:No} % \begin{syntax} % \cs{__kernel_ior_open:Nn} \meta{stream} \Arg{file name} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx b/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx index ea9b1f9d04f..495346fa204 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx b/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx index 041c39c3335..f640d04cd16 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx index 880eb79f9e1..891b7e06547 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx b/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx index cde386e371f..96bc2ca4476 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3names.dtx b/Master/texmf-dist/source/latex/l3kernel/l3names.dtx index b75c20eb08a..5333720124d 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3names.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3names.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx b/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx index 3f1280f23e9..b5b1e4246e7 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx b/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx index 9419a5ce6c3..84b41be5a05 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx b/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx index 16a9b0c447c..487393eb6ab 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx index 2086b5c3ec0..5e64d739805 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -713,7 +713,7 @@ % |{n,}| quantifier? (I think not.) % \item Quantifiers for |\u| and assertions. % \item When matching, keep track of an explicit stack of -% \texttt{current_state} and \texttt{current_submatches}. +% \texttt{curr_state} and \texttt{curr_submatches}. % \item If possible, when a state is reused by the same thread, kill % other subthreads. % \item Use an array rather than \cs[no-index]{l__regex_balance_tl} @@ -845,12 +845,14 @@ % names). % \begin{itemize} % \item \emph{Group}: index of the capturing group, $-1$ for -% non-capturing groups. +% non-capturing groups. ^^A start/end index? % \item \emph{Position}: each token in the query is labelled by an % integer \meta{position}, with $\texttt{min_pos} - 1 \leq % \meta{position} \leq \texttt{max_pos}$. The lowest and highest -% positions correspond to imaginary begin and end markers (with -% inaccessible category code and character code). +% positions $\texttt{min_pos} - 1$ and $\texttt{max_pos}$ +% correspond to imaginary begin and end markers (with +% non-existent category code and character code). +% $\texttt{max_pos}$ is only set quite late in the processing. % \item \emph{Query}: the token list to which we apply the regular % expression. % \item \emph{State}: each state of the \textsc{nfa} is labelled by an @@ -865,8 +867,8 @@ % unique id for all the steps of the matching algorithm. % \end{itemize} % -% We use \pkg{l3intarray} to manipulate arrays of integers (stored into -% some dimension registers in scaled points). We also abuse \TeX{}'s +% We use \pkg{l3intarray} to manipulate arrays of integers. +% We also abuse \TeX{}'s % \tn{toks} registers, by accessing them directly by number rather than % tying them to control sequence using the \tn{newtoks} allocation % functions. Specifically, these arrays and \tn{toks} are used as @@ -876,21 +878,14 @@ % \begin{itemize} % \item \cs{g_@@_state_active_intarray} holds the last \meta{step} in % which each \meta{state} was active. -% \item \cs{g_@@_thread_state_intarray} maps each \meta{thread} (with -% $\texttt{min_active} \leq \meta{thread} < \texttt{max_active}$) to -% the \meta{state} in which the \meta{thread} currently is. The +% \item \cs{g_@@_thread_info_intarray} consists of blocks for each +% \meta{thread} (with $\texttt{min_thread} \leq \meta{thread} < +% \texttt{max_thread}$). Each block has +% $1+2\cs{l_@@_capturing_group_int}$ entries: the \meta{state} in +% which the \meta{thread} currently is, followed by the beginnings +% of all submatches, and then the ends of all submatches. The % \meta{threads} are ordered starting from the best to the least % preferred. -% \item \tn{toks}\meta{thread} holds the submatch information for the -% \meta{thread}, as the contents of a property list. -% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the -% character codes and category codes of tokens at each -% \meta{position} in the query. -% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and -% end-group character tokens which appear before that point in the -% token list. -% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}- -% and \texttt{x}-expand to the \meta{position}-th token in the query. % \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray} % and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would % be extracted by \cs{regex_extract_all:nnN}), the place where the @@ -901,6 +896,14 @@ % block corresponding to one match with all its submatches stored in % consecutive entries. % \end{itemize} +% When actually building the result, +% \begin{itemize} +% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}- +% and \texttt{x}-expand to the \meta{position}-th token in the query. +% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and +% end-group character tokens which appear before that point in the +% token list. +% \end{itemize} % % The code is structured as follows. Variables are introduced in the % relevant section. First we present some generic helper functions. Then @@ -944,7 +947,7 @@ { \@@_toks_set:Nn #1 { } } \cs_new_eq:NN \@@_toks_set:Nn \tex_toks:D \cs_new_protected:Npn \@@_toks_set:No #1 - { \@@_toks_set:Nn #1 \exp_after:wN } + { \tex_toks:D #1 \exp_after:wN } % \end{macrocode} % \end{macro} % @@ -975,13 +978,13 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_toks_put_left:Nx #1#2 { - \cs_set:Npx \@@_tmp:w { #2 } + \cs_set_nopar:Npx \@@_tmp:w { #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 } } \cs_new_protected:Npn \@@_toks_put_right:Nx #1#2 { - \cs_set:Npx \@@_tmp:w {#2} + \cs_set_nopar:Npx \@@_tmp:w {#2} \tex_toks:D #1 \exp_after:wN { \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w } } @@ -1000,11 +1003,38 @@ \cs_new:Npn \@@_curr_cs_to_str: { \exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N - \tex_the:D \tex_toks:D \l_@@_curr_pos_int + \l_@@_curr_token_tl } % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_intarray_item:NnF, \@@_intarray_item_aux:nNF} +% Item of intarray, with a default value. +% \begin{macrocode} +\cs_new:Npn \@@_intarray_item:NnF #1#2 + { \exp_args:Nf \@@_intarray_item_aux:nNF { \int_eval:n {#2} } #1 } +\cs_new:Npn \@@_intarray_item_aux:nNF #1#2 + { + \if_int_compare:w #1 > \c_zero_int + \exp_after:wN \use_i:nn + \else: + \exp_after:wN \use_ii:nn + \fi: + { \__kernel_intarray_item:Nn #2 {#1} } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_maplike_break:} +% Analogous to \cs{tl_map_break:}, this correctly exits +% \cs{tl_map_inline:nn} and similar constructions and jumps to the +% matching \cs{prg_break_point:Nn} \cs{@@_maplike_break:} |{| |}|. +% \begin{macrocode} +\cs_new:Npn \@@_maplike_break: + { \prg_map_break:Nn \@@_maplike_break: { } } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Constants and variables} % % \begin{macro}{\@@_tmp:w} @@ -1056,19 +1086,6 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray} -% The first thing we do when matching is to go once through the query -% token list and store the information for each token into -% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks} -% registers. We also store the balance of begin-group/end-group -% characters into \cs{g_@@_balance_intarray}. -% \begin{macrocode} -\intarray_new:Nn \g_@@_charcode_intarray { 65536 } -\intarray_new:Nn \g_@@_catcode_intarray { 65536 } -\intarray_new:Nn \g_@@_balance_intarray { 65536 } -% \end{macrocode} -% \end{variable} -% % \begin{variable}{\l_@@_balance_int} % During this phase, \cs{l_@@_balance_int} counts the balance of % begin-group and end-group character tokens which appear before a @@ -1079,15 +1096,6 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_cs_name_tl} -% This variable is used in \cs{@@_item_cs:n} to store the csname of -% the currently-tested token when the regex contains a sub-regex for -% testing csnames. -% \begin{macrocode} -\tl_new:N \l_@@_cs_name_tl -% \end{macrocode} -% \end{variable} -% % \subsubsection{Testing characters} % % \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int} @@ -1201,7 +1209,7 @@ % \begin{macro} % {\@@_item_caseless_equal:n, \@@_item_caseless_range:nn} % For caseless matching, we perform the test both on the -% \texttt{current_char} and on the \texttt{case_changed_char}. Before +% \texttt{curr_char} and on the \texttt{case_changed_char}. Before % doing the second set of tests, we make sure that % \texttt{case_changed_char} has been computed. % \begin{macrocode} @@ -1308,7 +1316,8 @@ % % \begin{macro}{\@@_item_exact:nn, \@@_item_exact_cs:n} % This matches an exact \meta{category}-\meta{character code} pair, or -% an exact control sequence, more precisely one of several possible control sequences. +% an exact control sequence, more precisely one of several possible +% control sequences, separated by \cs{scan_stop:}. % \begin{macrocode} \cs_new_protected:Npn \@@_item_exact:nn #1#2 { @@ -1337,24 +1346,19 @@ % Match a control sequence (the argument is a compiled regex). % First test the catcode of the current token to be zero. % Then perform the matching test, and break if the csname -% indeed matches. The three \cs{exp_after:wN} expand the contents -% of the \tn{toks}\meta{current position} (of the form \cs{exp_not:n} -% \Arg{control sequence}) to \meta{control sequence}. -% We store the cs name before building states for the cs, as those -% states may overlap with toks registers storing the user's input. +% indeed matches. % \begin{macrocode} \cs_new_protected:Npn \@@_item_cs:n #1 { \int_compare:nNnT \l_@@_curr_catcode_int = 0 { \group_begin: - \__kernel_tl_set:Nx \l_@@_cs_name_tl { \@@_curr_cs_to_str: } \@@_single_match: \@@_disable_submatches: \@@_build_for_cs:n {#1} \bool_set_eq:NN \l_@@_saved_success_bool \g_@@_success_bool - \exp_args:NV \@@_match_cs:n \l_@@_cs_name_tl + \exp_args:Nx \@@_match_cs:n { \@@_curr_cs_to_str: } \if_meaning:w \c_true_bool \g_@@_success_bool \group_insert_after:N \@@_break_true:w \fi: @@ -1822,7 +1826,7 @@ % \item \cs{@@_command_K:} % \item \cs{@@_assertion:Nn} \meta{boolean} \Arg{assertion test}, % where the \meta{assertion test} is \cs{@@_b_test:} or -% |{|\cs{@@_anchor:N} \meta{integer}|}| +% \cs{@@_Z_test:} or \cs{@@_A_test:} or \cs{@@_G_test:} % \end{itemize} % Tests can be the following: % \begin{itemize} @@ -2663,73 +2667,56 @@ % % \subsubsection{Anchoring and simple assertions} % -% \begin{macro}{\@@_compile_anchor:NF} +% \begin{macro}{\@@_compile_anchor_letter:NNN} +% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:, \@@_compile_/Z:, \@@_compile_/z:, \@@_compile_/b:, \@@_compile_/B:} % \begin{macro}+\@@_compile_^:+ -% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:} % \begin{macro}+\@@_compile_$:+ -% \begin{macro}{\@@_compile_/Z:, \@@_compile_/z:} -% In modes where assertions are allowed, anchor to the start of the -% query, the start of the match, or the end of the query, depending on -% the integer |#1|. In other modes, |#2| treats the character as raw, -% with an error for escaped letters (|$| is valid in a class, but |\A| -% is definitely a mistake on the user's part). +% In modes where assertions are forbidden, anchors such as |\A| +% produce an error (|\A|~is invalid in classes); otherwise they add an +% \cs{@@_assertion:Nn} test as appropriate (the only negative +% assertion is~|\B|). The test functions are defined later. The +% implementation for +% |$| and |^| is only different from |\A| etc because these are valid +% in a class. % \begin{macrocode} -\cs_new_protected:Npn \@@_compile_anchor:NF #1#2 +\cs_new_protected:Npn \@@_compile_anchor_letter:NNN #1#2#3 { - \@@_if_in_class_or_catcode:TF {#2} + \@@_if_in_class_or_catcode:TF { \@@_compile_raw_error:N #1 } { \tl_build_put_right:Nn \l_@@_build_tl - { \@@_assertion:Nn \c_true_bool { \@@_anchor:N #1 } } + { \@@_assertion:Nn #2 {#3} } } } -\cs_set_protected:Npn \@@_tmp:w #1#2 - { - \cs_new_protected:cpn { @@_compile_/#1: } - { \@@_compile_anchor:NF #2 { \@@_compile_raw_error:N #1 } } - } -\@@_tmp:w A \l_@@_min_pos_int -\@@_tmp:w G \l_@@_start_pos_int -\@@_tmp:w Z \l_@@_max_pos_int -\@@_tmp:w z \l_@@_max_pos_int +\cs_new_protected:cpn { @@_compile_/A: } + { \@@_compile_anchor_letter:NNN A \c_true_bool \@@_A_test: } +\cs_new_protected:cpn { @@_compile_/G: } + { \@@_compile_anchor_letter:NNN G \c_true_bool \@@_G_test: } +\cs_new_protected:cpn { @@_compile_/Z: } + { \@@_compile_anchor_letter:NNN Z \c_true_bool \@@_Z_test: } +\cs_new_protected:cpn { @@_compile_/z: } + { \@@_compile_anchor_letter:NNN z \c_true_bool \@@_Z_test: } +\cs_new_protected:cpn { @@_compile_/b: } + { \@@_compile_anchor_letter:NNN b \c_true_bool \@@_b_test: } +\cs_new_protected:cpn { @@_compile_/B: } + { \@@_compile_anchor_letter:NNN B \c_false_bool \@@_b_test: } \cs_set_protected:Npn \@@_tmp:w #1#2 { \cs_new_protected:cpn { @@_compile_#1: } - { \@@_compile_anchor:NF #2 { \@@_compile_raw:N #1 } } + { + \@@_if_in_class_or_catcode:TF { \@@_compile_raw:N #1 } + { + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_assertion:Nn \c_true_bool {#2} } + } + } } -\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } \l_@@_min_pos_int -\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } \l_@@_max_pos_int +\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } { \@@_A_test: } +\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } { \@@_Z_test: } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} -% \end{macro} -% -% \begin{macro}{\@@_compile_/b:, \@@_compile_/B:} -% Contrarily to |^| and |$|, which could be implemented without really -% knowing what precedes in the token list, this requires more -% information, namely, the knowledge of the last character code. -% \begin{macrocode} -\cs_new_protected:cpn { @@_compile_/b: } - { - \@@_if_in_class_or_catcode:TF - { \@@_compile_raw_error:N b } - { - \tl_build_put_right:Nn \l_@@_build_tl - { \@@_assertion:Nn \c_true_bool { \@@_b_test: } } - } - } -\cs_new_protected:cpn { @@_compile_/B: } - { - \@@_if_in_class_or_catcode:TF - { \@@_compile_raw_error:N B } - { - \tl_build_put_right:Nn \l_@@_build_tl - { \@@_assertion:Nn \c_false_bool { \@@_b_test: } } - } - } -% \end{macrocode} -% \end{macro} % % \subsubsection{Character classes} % @@ -3512,7 +3499,9 @@ { \bool_if:NF ##1 { negative~ } assertion:~##2 } } \cs_set:Npn \@@_b_test: { word~boundary } - \cs_set_eq:NN \@@_anchor:N \@@_show_anchor_to_str:N + \cs_set:Npn \@@_Z_test: { anchor~at~end~(\iow_char:N\\Z) } + \cs_set:Npn \@@_A_test: { anchor~at~start~(\iow_char:N\\A) } + \cs_set:Npn \@@_G_test: { anchor~at~start~of~match~(\iow_char:N\\G) } \cs_set_protected:Npn \@@_item_caseful_equal:n ##1 { \@@_show_one:n { char~code~\int_eval:n{##1} } } \cs_set_protected:Npn \@@_item_caseful_range:nn ##1##2 @@ -3655,24 +3644,6 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[rEXP]{\@@_show_anchor_to_str:N} -% The argument is an integer telling us where the anchor is. We -% convert that to the relevant info. -% \begin{macrocode} -\cs_new:Npn \@@_show_anchor_to_str:N #1 - { - anchor~at~ - \str_case:nnF { #1 } - { - { \l_@@_min_pos_int } { start~(\iow_char:N\\A) } - { \l_@@_start_pos_int } { start~of~match~(\iow_char:N\\G) } - { \l_@@_max_pos_int } { end~(\iow_char:N\\Z) } - } - { <error:~'#1'~not~recognized> } - } -% \end{macrocode} -% \end{macro} -% % \begin{macro}{\@@_show_item_catcode:NnT} % Produce a sequence of categories which the catcode bitmap |#2| % contains, and show it, indenting the tests on which this catcode @@ -3757,8 +3728,9 @@ % Each state of the \textsc{nfa} is stored in a \tn{toks}. The % operations which can appear in the \tn{toks} are % \begin{itemize} -% \item \cs{@@_action_start_wildcard:} inserted at the start -% of the regular expression to make it unanchored. +% \item \cs{@@_action_start_wildcard:N} \meta{boolean} inserted at the +% start of the regular expression, where a \texttt{true} +% \meta{boolean} makes it unanchored. % \item \cs{@@_action_success:} marks the exit state of the % \textsc{nfa}. % \item \cs{@@_action_cost:n} \Arg{shift} is a transition from the @@ -3772,10 +3744,11 @@ % how they detect and avoid infinite loops. For now, we just need to % know that the \texttt{group} variant must be used for transitions % back to the start of a group. -% \item \cs{@@_action_submatch:n} \Arg{key} where the \meta{key} is -% a group number followed by |<| or |>| for the beginning or end of -% group. This causes the current position in the query to be stored -% as the \meta{key} submatch boundary. +% \item \cs{@@_action_submatch:nN} \Arg{group} \meta{key} where the +% \meta{key} is |<| or |>| for the beginning or end of group +% numbered \meta{group}. This causes the current position in the +% query to be stored as the \meta{key} submatch boundary. +% \item One of these actions, within a conditional. % \end{itemize} % % We strive to preserve the following properties while building. @@ -3793,20 +3766,26 @@ % corresponding end-points of nested groups. % \end{itemize} % -% \begin{macro}{\@@_build:n, \@@_build:N} +% \begin{macro}{\@@_build:n, \@@_build_aux:Nn, \@@_build:N, \@@_build_aux:NN} % The \texttt{n}-type function first compiles its argument. Reset some % variables. Allocate two states, and put a wildcard in state $0$ % (transitions to state $1$ and $0$ state). Then build the regex % within a (capturing) group numbered $0$ (current % value of \texttt{capturing_group}). Finally, if the match reaches the -% last state, it is successful. +% last state, it is successful. A \texttt{false} boolean for argument +% |#1| for the auxiliaries will suppress the wildcard and make the +% match anchored: used for \cs{peek_regex:nTF} and similar. % \begin{macrocode} -\cs_new_protected:Npn \@@_build:n #1 +\cs_new_protected:Npn \@@_build:n + { \@@_build_aux:Nn \c_true_bool } +\cs_new_protected:Npn \@@_build:N + { \@@_build_aux:NN \c_true_bool } +\cs_new_protected:Npn \@@_build_aux:Nn #1#2 { - \@@_compile:n {#1} - \@@_build:N \l_@@_internal_regex + \@@_compile:n {#2} + \@@_build_aux:NN #1 \l_@@_internal_regex } -\cs_new_protected:Npn \@@_build:N #1 +\cs_new_protected:Npn \@@_build_aux:NN #1#2 { \@@_standard_escapechar: \int_zero:N \l_@@_capturing_group_int @@ -3814,8 +3793,8 @@ \@@_build_new_state: \@@_build_new_state: \@@_toks_put_right:Nn \l_@@_left_state_int - { \@@_action_start_wildcard: } - \@@_group:nnnN {#1} { 1 } { 0 } \c_false_bool + { \@@_action_start_wildcard:N #1 } + \@@_group:nnnN {#2} { 1 } { 0 } \c_false_bool \@@_toks_put_right:Nn \l_@@_right_state_int { \@@_action_success: } } @@ -3826,19 +3805,14 @@ % The matching code relies on some global intarray variables, but only % uses a range of their entries. Specifically, % \begin{itemize} -% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int} -% to $\cs{l_@@_max_state_int}-1$; -% \item \cs{g_@@_thread_state_intarray} from \cs{l_@@_min_active_int} -% to $\cs{l_@@_max_active_int}-1$. +% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int} +% to $\cs{l_@@_max_state_int}-1$; % \end{itemize} -% In fact, some data is stored in \tn{toks} registers (local) in the -% same ranges so these ranges mustn't overlap. This is done by -% setting \cs{l_@@_min_active_int} to \cs{l_@@_max_state_int} after -% building the \textsc{nfa}. Here, in this nested call to the -% matching code, we need the new versions of these ranges to involve +% Here, in this nested call to the +% matching code, we need the new versions of this range to involve % completely new entries of the intarray variables, so we begin by % setting (the new) \cs{l_@@_min_state_int} to (the old) -% \cs{l_@@_max_active_int} to use higher entries. +% \cs{l_@@_max_state_int} to use higher entries. % % When using a regex to match a cs, we don't insert a wildcard, we % anchor at the end, and since we ignore submatches, there is no need @@ -3848,8 +3822,7 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_build_for_cs:n #1 { - \int_set_eq:NN \l_@@_min_state_int \l_@@_max_active_int - \int_set_eq:NN \l_@@_max_state_int \l_@@_min_state_int + \int_set_eq:NN \l_@@_min_state_int \l_@@_max_state_int \@@_build_new_state: \@@_build_new_state: \@@_push_lr_states: @@ -3857,7 +3830,7 @@ \@@_pop_lr_states: \@@_toks_put_right:Nn \l_@@_right_state_int { - \if_int_compare:w \l_@@_curr_pos_int = \l_@@_max_pos_int + \if_int_compare:w -2 = \l_@@_curr_char_int \exp_after:wN \@@_action_success: \fi: } @@ -4195,8 +4168,8 @@ \cs_new_protected:Npn \@@_group_submatches:nNN #1#2#3 { \if_int_compare:w #1 > - 1 \exp_stop_f: - \@@_toks_put_left:Nx #2 { \@@_action_submatch:n { #1 < } } - \@@_toks_put_left:Nx #3 { \@@_action_submatch:n { #1 > } } + \@@_toks_put_left:Nx #2 { \@@_action_submatch:nN {#1} < } + \@@_toks_put_left:Nx #3 { \@@_action_submatch:nN {#1} > } \fi: } % \end{macrocode} @@ -4345,16 +4318,14 @@ % % \subsubsection{Others} % -% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_anchor:N} +% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_A_test:, \@@_G_test:, \@@_Z_test:} % Usage: \cs{@@_assertion:Nn} \meta{boolean} \Arg{test}, where the % \meta{test} is either of the two other functions. Add a free % transition to a new state, conditionally to the assertion test. The % \cs{@@_b_test:} test is used by the |\b| and |\B| escape: check % if the last character was a word character or not, and do the same % to the current character. The boundary-markers of the string are -% non-word characters for this purpose. Anchors at the start or end -% of match use \cs{@@_anchor:N}, with a position controlled by the -% integer |#1|. +% non-word characters for this purpose. % \begin{macrocode} \cs_new_protected:Npn \@@_assertion:Nn #1#2 { @@ -4374,12 +4345,6 @@ \bool_if:NT #1 { { } } } } -\cs_new_protected:Npn \@@_anchor:N #1 - { - \if_int_compare:w #1 = \l_@@_curr_pos_int - \exp_after:wN \@@_break_true:w - \fi: - } \cs_new_protected:Npn \@@_b_test: { \group_begin: @@ -4389,6 +4354,24 @@ { \group_end: \@@_item_reverse:n \@@_prop_w: } { \group_end: \@@_prop_w: } } +\cs_new_protected:Npn \@@_Z_test: + { + \if_int_compare:w -2 = \l_@@_curr_char_int + \exp_after:wN \@@_break_true:w + \fi: + } +\cs_new_protected:Npn \@@_A_test: + { + \if_int_compare:w -2 = \l_@@_last_char_int + \exp_after:wN \@@_break_true:w + \fi: + } +\cs_new_protected:Npn \@@_G_test: + { + \if_int_compare:w \l_@@_curr_pos_int = \l_@@_start_pos_int + \exp_after:wN \@@_break_true:w + \fi: + } % \end{macrocode} % \end{macro} % @@ -4401,7 +4384,7 @@ \@@_build_new_state: \@@_toks_put_right:Nx \l_@@_left_state_int { - \@@_action_submatch:n { 0< } + \@@_action_submatch:nN { 0 } < \bool_set_true:N \l_@@_fresh_thread_bool \@@_action_free:n { @@ -4423,7 +4406,8 @@ % transitions, the instruction at the new state of the \textsc{nfa} is % performed immediately. When a transition consumes a character, the % new state is appended to a list of \enquote{active states}, stored in -% \cs{g_@@_thread_state_intarray}: this thread is made active again when the next +% \cs{g_@@_thread_info_intarray} (together with submatch information): +% this thread is made active again when the next % token is read from the query. At every step (for each token in the % query), we unpack that list of active states and the corresponding % submatch props, and empty those. @@ -4467,11 +4451,10 @@ % } % The tokens in the query are indexed from \texttt{min_pos} for the % first to $\texttt{max_pos}-1$ for the last, and their information is -% stored in several arrays and \tn{toks} registers with those numbers. We -% don't start from $0$ because the \tn{toks} registers with low -% numbers are used to hold the states of the \textsc{nfa}. We match +% stored in several arrays and \tn{toks} registers with those numbers. +% We match % without backtracking, keeping all threads in lockstep at the -% \texttt{current_pos} in the query. The starting point of the current +% \texttt{curr_pos} in the query. The starting point of the current % match attempt is \texttt{start_pos}, and \texttt{success_pos}, % updated whenever a thread succeeds, is used as the next starting % position. @@ -4488,20 +4471,26 @@ % { % \l_@@_curr_char_int, % \l_@@_curr_catcode_int, +% \l_@@_curr_token_tl, % \l_@@_last_char_int, +% \l_@@_last_char_success_int, % \l_@@_case_changed_char_int % } % The character and category codes of the token at the current -% position; the character code of the token at the previous position; +% position and a token list expanding to that token; the character +% code of the token at the previous position; +% the character code of the token just before a successful match; % and the character code of the result of changing the case of the % current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is % only computed when necessary, and is otherwise \cs{c_max_int}. The -% \texttt{current_char} variable is also used in various other phases +% \texttt{curr_char} variable is also used in various other phases % to hold a character code. % \begin{macrocode} \int_new:N \l_@@_curr_char_int \int_new:N \l_@@_curr_catcode_int +\tl_new:N \l_@@_curr_token_tl \int_new:N \l_@@_last_char_int +\int_new:N \l_@@_last_char_success_int \int_new:N \l_@@_case_changed_char_int % \end{macrocode} % \end{variable} @@ -4517,18 +4506,17 @@ % \end{variable} % % \begin{variable} -% {\l_@@_curr_submatches_prop, \l_@@_success_submatches_prop} +% {\l_@@_curr_submatches_tl, \l_@@_success_submatches_tl} % The submatches for the thread which is currently active are stored -% in the \texttt{current_submatches} property list variable. This -% property list is stored by \cs{@@_action_cost:n} into the -% \tn{toks} register for the target state of the transition, to be -% retrieved when matching at the next position. When a thread -% succeeds, this property list is copied to -% \cs{l_@@_success_submatches_prop}: only the last successful thread +% in the \texttt{curr_submatches} list, which is almost a comma list, +% but ends with a comma. This list is stored by \cs{@@_store_state:n} +% into an intarray variable, to be retrieved when matching at the next +% position. When a thread succeeds, this list is copied to +% \cs{l_@@_success_submatches_tl}: only the last successful thread % remains there. % \begin{macrocode} -\prop_new:N \l_@@_curr_submatches_prop -\prop_new:N \l_@@_success_submatches_prop +\tl_new:N \l_@@_curr_submatches_tl +\tl_new:N \l_@@_success_submatches_tl % \end{macrocode} % \end{variable} % @@ -4551,28 +4539,44 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_min_active_int, \l_@@_max_active_int} +% \begin{variable}{\l_@@_min_thread_int, \l_@@_max_thread_int} % All the currently active threads are kept in order of precedence in -% \cs{g_@@_thread_state_intarray}, and the corresponding submatches in the -% \tn{toks}. For our purposes, those serve as an array, indexed from -% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded). -% At the start of every step, the whole array is unpacked, so that the -% space can immediately be reused, and \texttt{max_active} is reset to -% \texttt{min_active}, effectively clearing the array. +% \cs{g_@@_thread_info_intarray} together with the corresponding +% submatch information. Data in this intarray is organized as blocks +% from \texttt{min_thread} (included) to \texttt{max_thread} +% (excluded). At the start of every step, the whole array is +% unpacked, so that the space can immediately be reused, and +% \texttt{max_thread} is reset to \texttt{min_thread}, effectively +% clearing the array. % \begin{macrocode} -\int_new:N \l_@@_min_active_int -\int_new:N \l_@@_max_active_int +\int_new:N \l_@@_min_thread_int +\int_new:N \l_@@_max_thread_int % \end{macrocode} % \end{variable} % -% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_state_intarray} +% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_info_intarray} % \cs{g_@@_state_active_intarray} stores the last \meta{step} in which -% each \meta{state} was active. \cs{g_@@_thread_state_intarray} stores +% each \meta{state} was active. \cs{g_@@_thread_info_intarray} stores % threads to be considered in the next step, more precisely the % states in which these threads are. % \begin{macrocode} \intarray_new:Nn \g_@@_state_active_intarray { 65536 } -\intarray_new:Nn \g_@@_thread_state_intarray { 65536 } +\intarray_new:Nn \g_@@_thread_info_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_matched_analysis_tl, \l_@@_curr_analysis_tl} +% The list \cs{l_@@_curr_analysis_tl} consists of a brace group +% containing three brace groups corresponding to the current token, +% with the same syntax as \cs{tl_analysis_map_inline:nn}. The list +% \cs{l_@@_matched_analysis_tl} (constructed under the +% \texttt{tl\_build} machinery) has one item for each token that has +% already been treated so far in a given match attempt: each item +% consists of three brace groups with the same syntax as +% \cs{tl_analysis_map_inline:nn}. +% \begin{macrocode} +\tl_new:N \l_@@_matched_analysis_tl +\tl_new:N \l_@@_curr_analysis_tl % \end{macrocode} % \end{variable} % @@ -4635,8 +4639,7 @@ % % \begin{macro}{\@@_match:n, \@@_match_cs:n} % \begin{macro}{\@@_match_init:} -% First store the query into \tn{toks} registers and arrays (see -% \cs{@@_query_set:nnn}). Then initialize the variables that should +% Initialize the variables that should % be set once for each user function (even for multiple % matches). Namely, the overall matching is not yet successful; none of % the states should be marked as visited (\cs{g_@@_state_active_intarray}), and @@ -4647,38 +4650,26 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_match:n #1 { - \int_zero:N \l_@@_balance_int - \int_set:Nn \l_@@_curr_pos_int { 2 * \l_@@_max_state_int } - \@@_query_set:nnn { } { -1 } { -2 } - \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int - \tl_analysis_map_inline:nn {#1} - { \@@_query_set:nnn {##1} {"##3} {##2} } - \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int - \@@_query_set:nnn { } { -1 } { -2 } \@@_match_init: - \@@_match_once: + \@@_match_once_init: + \tl_analysis_map_inline:nn {#1} + { \@@_match_one_token:nnN {##1} {##2} ##3 } + \@@_match_one_token:nnN { } { -2 } F + \prg_break_point:Nn \@@_maplike_break: { } } \cs_new_protected:Npn \@@_match_cs:n #1 { - \int_zero:N \l_@@_balance_int - \int_set:Nn \l_@@_curr_pos_int - { - \int_max:nn { 2 * \l_@@_max_state_int - \l_@@_min_state_int } - { \l_@@_max_pos_int } - + 1 - } - \@@_query_set:nnn { } { -1 } { -2 } - \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int + \int_set_eq:NN \l_@@_min_thread_int \l_@@_max_thread_int + \@@_match_init: + \@@_match_once_init: \str_map_inline:nn {#1} { - \@@_query_set:nnn { \exp_not:n {##1} } - { \tl_if_blank:nTF {##1} { 10 } { 12 } } - { `##1 } + \tl_if_blank:nTF {##1} + { \@@_match_one_token:nnN {##1} {`##1} A } + { \@@_match_one_token:nnN {##1} {`##1} C } } - \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int - \@@_query_set:nnn { } { -1 } { -2 } - \@@_match_init: - \@@_match_once: + \@@_match_one_token:nnN { } { -2 } F + \prg_break_point:Nn \@@_maplike_break: { } } \cs_new_protected:Npn \@@_match_init: { @@ -4689,11 +4680,13 @@ \__kernel_intarray_gset:Nnn \g_@@_state_active_intarray {##1} { 1 } } - \int_set_eq:NN \l_@@_min_active_int \l_@@_max_state_int \int_zero:N \l_@@_step_int + \int_set:Nn \l_@@_min_pos_int { 2 } \int_set_eq:NN \l_@@_success_pos_int \l_@@_min_pos_int - \int_set:Nn \l_@@_min_submatch_int - { 2 * \l_@@_max_state_int } + \int_set:Nn \l_@@_last_char_success_int { -2 } + \tl_build_begin:N \l_@@_matched_analysis_tl + \tl_clear:N \l_@@_curr_analysis_tl + \int_set:Nn \l_@@_min_submatch_int { 1 } \int_set_eq:NN \l_@@_submatch_int \l_@@_min_submatch_int \bool_set_false:N \l_@@_empty_success_bool } @@ -4701,21 +4694,24 @@ % \end{macro} % \end{macro} % -% \begin{macro}{\@@_match_once:} -% This function finds one match, then does some action defined by the -% \texttt{every_match} token list, which may recursively call -% \cs{@@_match_once:}. First initialize some variables: set the +% \begin{macro}{\@@_match_once_init:} +% This function resets various variables used when finding one match. +% It is called before the loop through characters, and every time we +% find a match, before searching for another match (this is controlled +% by the \texttt{every_match} token list). +% +% First initialize some variables: set the % conditional which detects identical empty matches; this match % attempt starts at the previous \texttt{success_pos}, is not yet % successful, and has no submatches yet; clear the array of active % threads, and put the starting state $0$ in it. We are then almost % ready to read our first token in the query, but we actually start -% one position earlier than the start, and \texttt{get} that token, to -% set \texttt{last_char} properly for word -% boundaries. Then call \cs{@@_match_loop:}, which runs through the -% query until the end or until a successful match breaks early. +% one position earlier than the start because +% \cs{@@_match_one_token:nnN} increments \cs{l_@@_curr_pos_int} and +% saves \cs{l_@@_curr_char_int} as the \texttt{last_char} so that word +% boundaries can be correctly identified. % \begin{macrocode} -\cs_new_protected:Npn \@@_match_once: +\cs_new_protected:Npn \@@_match_once_init: { \if_meaning:w \c_true_bool \l_@@_empty_success_bool \cs_set:Npn \@@_if_two_empty_matches:F @@ -4728,14 +4724,24 @@ \fi: \int_set_eq:NN \l_@@_start_pos_int \l_@@_success_pos_int \bool_set_false:N \l_@@_match_success_bool - \prop_clear:N \l_@@_curr_submatches_prop - \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int + \tl_set:Nx \l_@@_curr_submatches_tl + { \prg_replicate:nn { 2 * \l_@@_capturing_group_int } { 0 , } } + \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int \@@_store_state:n { \l_@@_min_state_int } \int_set:Nn \l_@@_curr_pos_int { \l_@@_start_pos_int - 1 } - \@@_query_get: - \@@_match_loop: - \l_@@_every_match_tl + \int_set_eq:NN \l_@@_curr_char_int \l_@@_last_char_success_int + \tl_build_get:NN \l_@@_matched_analysis_tl \l_@@_internal_a_tl + \exp_args:NNf \@@_match_once_init_aux: + \tl_map_inline:nn + { \exp_after:wN \l_@@_internal_a_tl \l_@@_curr_analysis_tl } + { \@@_match_one_token:nnN ##1 } + \prg_break_point:Nn \@@_maplike_break: { } + } +\cs_new_protected:Npn \@@_match_once_init_aux: + { + \tl_build_clear:N \l_@@_matched_analysis_tl + \tl_clear:N \l_@@_curr_analysis_tl } % \end{macrocode} % \end{macro} @@ -4753,110 +4759,80 @@ \bool_gset_eq:NN \g_@@_success_bool \l_@@_match_success_bool + \@@_maplike_break: } } \cs_new_protected:Npn \@@_multi_match:n #1 { \tl_set:Nn \l_@@_every_match_tl { - \if_meaning:w \c_true_bool \l_@@_match_success_bool - \bool_gset_true:N \g_@@_success_bool - #1 - \exp_after:wN \@@_match_once: + \if_meaning:w \c_false_bool \l_@@_match_success_bool + \exp_after:wN \@@_maplike_break: \fi: + \bool_gset_true:N \g_@@_success_bool + #1 + \@@_match_once_init: } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_match_loop:} +% \begin{macro}{\@@_match_one_token:nnN} % \begin{macro}[rEXP]{\@@_match_one_active:n} % At each new position, set some variables and get the new character % and category from the query. Then unpack the array of active % threads, and clear it by resetting its length -% (\texttt{max_active}). This results in a sequence of -% \cs{@@_use_state_and_submatches:nn} \Arg{state} \Arg{prop}, and +% (\texttt{max_thread}). This results in a sequence of +% \cs{@@_use_state_and_submatches:w} \meta{state}|,|\meta{submatch-clist}|;| and % we consider those states one by one in order. As soon as a thread % succeeds, exit the step, and, if there are threads to consider at the % next position, and we have not reached the end of the string, -% repeat the loop. Otherwise, the last thread that succeeded is what -% \cs{@@_match_once:} matches. We explain the \texttt{fresh_thread} -% business when describing \cs{@@_action_wildcard:}. +% repeat the loop. Otherwise, the last thread that succeeded is the +% match. We explain the \texttt{fresh_thread} business when +% describing \cs{@@_action_wildcard:}. % \begin{macrocode} -\cs_new_protected:Npn \@@_match_loop: +\cs_new_protected:Npn \@@_match_one_token:nnN #1#2#3 { \int_add:Nn \l_@@_step_int { 2 } \int_incr:N \l_@@_curr_pos_int \int_set_eq:NN \l_@@_last_char_int \l_@@_curr_char_int \int_set_eq:NN \l_@@_case_changed_char_int \c_max_int - \@@_query_get: + \tl_set:Nn \l_@@_curr_token_tl {#1} + \int_set:Nn \l_@@_curr_char_int {#2} + \int_set:Nn \l_@@_curr_catcode_int { "#3 } + \tl_build_put_right:Nx \l_@@_matched_analysis_tl + { \exp_not:o \l_@@_curr_analysis_tl } + \tl_set:Nn \l_@@_curr_analysis_tl { { {#1} {#2} #3 } } \use:x { - \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int + \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int \int_step_function:nnN - { \l_@@_min_active_int } - { \l_@@_max_active_int - 1 } + { \l_@@_min_thread_int } + { \l_@@_max_thread_int - 1 } \@@_match_one_active:n } \prg_break_point: \bool_set_false:N \l_@@_fresh_thread_bool - \if_int_compare:w \l_@@_max_active_int > \l_@@_min_active_int - \if_int_compare:w \l_@@_curr_pos_int < \l_@@_max_pos_int - \exp_after:wN \exp_after:wN \exp_after:wN \@@_match_loop: + \if_int_compare:w \l_@@_max_thread_int > \l_@@_min_thread_int + \if_int_compare:w -2 < \l_@@_curr_char_int + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n \fi: \fi: + \l_@@_every_match_tl } \cs_new:Npn \@@_match_one_active:n #1 { - \@@_use_state_and_submatches:nn - { \__kernel_intarray_item:Nn \g_@@_thread_state_intarray {#1} } - { \@@_toks_use:w #1 } + \@@_use_state_and_submatches:w + \__kernel_intarray_range_to_clist:Nnn + \g_@@_thread_info_intarray + { 1 + #1 * (\l_@@_capturing_group_int * 2 + 1) } + { (1 + #1) * (\l_@@_capturing_group_int * 2 + 1) } + ; } % \end{macrocode} % \end{macro} % \end{macro} % -% \begin{macro}{\@@_query_set:nnn} -% The arguments are: tokens that \texttt{o} and \texttt{x} expand to -% one token of the query, the catcode, and the character code. Store -% those, and the current brace balance (used later to check for -% overall brace balance) in a \tn{toks} register and some arrays, -% then update the \texttt{balance}. -% \begin{macrocode} -\cs_new_protected:Npn \@@_query_set:nnn #1#2#3 - { - \__kernel_intarray_gset:Nnn \g_@@_charcode_intarray - { \l_@@_curr_pos_int } {#3} - \__kernel_intarray_gset:Nnn \g_@@_catcode_intarray - { \l_@@_curr_pos_int } {#2} - \__kernel_intarray_gset:Nnn \g_@@_balance_intarray - { \l_@@_curr_pos_int } { \l_@@_balance_int } - \@@_toks_set:Nn \l_@@_curr_pos_int {#1} - \int_incr:N \l_@@_curr_pos_int - \if_case:w #2 \exp_stop_f: - \or: \int_incr:N \l_@@_balance_int - \or: \int_decr:N \l_@@_balance_int - \fi: - } -% \end{macrocode} -% \end{macro} -% -% \begin{macro}{\@@_query_get:} -% Extract the current character and category codes at the current -% position from the appropriate arrays. -% \begin{macrocode} -\cs_new_protected:Npn \@@_query_get: - { - \l_@@_curr_char_int - = \__kernel_intarray_item:Nn \g_@@_charcode_intarray - { \l_@@_curr_pos_int } \scan_stop: - \l_@@_curr_catcode_int - = \__kernel_intarray_item:Nn \g_@@_catcode_intarray - { \l_@@_curr_pos_int } \scan_stop: - } -% \end{macrocode} -% \end{macro} -% % \subsubsection{Using states of the \textsc{nfa}} % % \begin{macro}{\@@_use_state:} @@ -4879,20 +4855,20 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_use_state_and_submatches:nn} +% \begin{macro}{\@@_use_state_and_submatches:w} % This function is called as one item in the array of active threads % after that array has been unpacked for a new step. Update the -% \texttt{current_state} and \texttt{current_submatches} and use the +% \texttt{curr_state} and \texttt{curr_submatches} and use the % state if it has not yet been encountered at this step. % \begin{macrocode} -\cs_new_protected:Npn \@@_use_state_and_submatches:nn #1 #2 +\cs_new_protected:Npn \@@_use_state_and_submatches:w #1 , #2 ; { \int_set:Nn \l_@@_curr_state_int {#1} \if_int_compare:w \__kernel_intarray_item:Nn \g_@@_state_active_intarray { \l_@@_curr_state_int } < \l_@@_step_int - \tl_set:Nn \l_@@_curr_submatches_prop {#2} + \tl_set:Nn \l_@@_curr_submatches_tl { #2 , } \exp_after:wN \@@_use_state: \fi: \scan_stop: @@ -4902,20 +4878,20 @@ % % \subsubsection{Actions when matching} % -% \begin{macro}{\@@_action_start_wildcard:} +% \begin{macro}{\@@_action_start_wildcard:N} % For an unanchored match, state $0$ has a free transition to the next % and a costly one to itself, to repeat at the next position. To catch % repeated identical empty matches, we need to know if a successful % thread corresponds to an empty match. The instruction resetting % \cs{l_@@_fresh_thread_bool} may be skipped by a successful -% thread, hence we had to add it to \cs{@@_match_loop:} too. +% thread, hence we had to add it to \cs{@@_match_one_token:nnN} too. % \begin{macrocode} -\cs_new_protected:Npn \@@_action_start_wildcard: +\cs_new_protected:Npn \@@_action_start_wildcard:N #1 { \bool_set_true:N \l_@@_fresh_thread_bool \@@_action_free:n {1} \bool_set_false:N \l_@@_fresh_thread_bool - \@@_action_cost:n {0} + \bool_if:NT #1 { \@@_action_cost:n {0} } } % \end{macrocode} % \end{macro} @@ -4954,8 +4930,8 @@ } \int_set:Nn \l_@@_curr_state_int { \int_use:N \l_@@_curr_state_int } - \tl_set:Nn \exp_not:N \l_@@_curr_submatches_prop - { \exp_not:o \l_@@_curr_submatches_prop } + \tl_set:Nn \exp_not:N \l_@@_curr_submatches_tl + { \exp_not:o \l_@@_curr_submatches_tl } } } % \end{macrocode} @@ -4978,21 +4954,26 @@ % % \begin{macro}{\@@_store_state:n} % \begin{macro}{\@@_store_submatches:} -% Put the given state in \cs{g_@@_thread_state_intarray}, and increment -% the length of the array. Also store the current submatch in the -% appropriate \tn{toks}. +% Put the given state and current submatch information in +% \cs{g_@@_thread_info_intarray}, and increment the length of the +% array. % \begin{macrocode} \cs_new_protected:Npn \@@_store_state:n #1 { - \@@_store_submatches: - \__kernel_intarray_gset:Nnn \g_@@_thread_state_intarray - { \l_@@_max_active_int } {#1} - \int_incr:N \l_@@_max_active_int + \exp_args:No \@@_store_submatches:nn + \l_@@_curr_submatches_tl {#1} + \int_incr:N \l_@@_max_thread_int } -\cs_new_protected:Npn \@@_store_submatches: +\cs_new_protected:Npn \@@_store_submatches:nn #1#2 { - \@@_toks_set:No \l_@@_max_active_int - { \l_@@_curr_submatches_prop } + \__kernel_intarray_gset_range_from_clist:Nnn + \g_@@_thread_info_intarray + { + \@@_int_eval:w + 1 + \l_@@_max_thread_int * + (\l_@@_capturing_group_int * 2 + 1) + } + { #2 , #1 } } % \end{macrocode} % \end{macro} @@ -5006,21 +4987,37 @@ % \begin{macrocode} \cs_new_protected:Npn \@@_disable_submatches: { - \cs_set_protected:Npn \@@_store_submatches: { } - \cs_set_protected:Npn \@@_action_submatch:n ##1 { } + \cs_set_protected:Npn \@@_store_submatches:n ##1 { } + \cs_set_protected:Npn \@@_action_submatch:nN ##1##2 { } } % \end{macrocode} % \end{macro} % -% \begin{macro}{\@@_action_submatch:n} +% \begin{macro}{\@@_action_submatch:nN, \@@_action_submatch_aux:w, \@@_action_submatch_auxii:w, \@@_action_submatch_auxiii:w, \@@_action_submatch_auxiv:w} % Update the current submatches with the information from the current % position. Maybe a bottleneck. % \begin{macrocode} -\cs_new_protected:Npn \@@_action_submatch:n #1 +\cs_new_protected:Npn \@@_action_submatch:nN #1#2 + { + \exp_after:wN \@@_action_submatch_aux:w + \l_@@_curr_submatches_tl ; {#1} #2 + } +\cs_new_protected:Npn \@@_action_submatch_aux:w #1 ; #2#3 { - \prop_put:Nno \l_@@_curr_submatches_prop {#1} - { \int_use:N \l_@@_curr_pos_int } + \tl_set:Nx \l_@@_curr_submatches_tl + { + \prg_replicate:nn + { #2 \if_meaning:w > #3 + \l_@@_capturing_group_int \fi: } + { \@@_action_submatch_auxii:w } + \@@_action_submatch_auxiii:w + #1 + } } +\cs_new:Npn \@@_action_submatch_auxii:w + #1 \@@_action_submatch_auxiii:w #2 , + { #2 , #1 \@@_action_submatch_auxiii:w } +\cs_new:Npn \@@_action_submatch_auxiii:w #1 , + { \int_use:N \l_@@_curr_pos_int , } % \end{macrocode} % \end{macro} % @@ -5042,8 +5039,10 @@ \bool_set_eq:NN \l_@@_empty_success_bool \l_@@_fresh_thread_bool \int_set_eq:NN \l_@@_success_pos_int \l_@@_curr_pos_int - \prop_set_eq:NN \l_@@_success_submatches_prop - \l_@@_curr_submatches_prop + \int_set_eq:NN \l_@@_last_char_success_int \l_@@_last_char_int + \tl_build_clear:N \l_@@_matched_analysis_tl + \tl_set_eq:NN \l_@@_success_submatches_tl + \l_@@_curr_submatches_tl \prg_break: } } @@ -5134,6 +5133,14 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_replacement_exp_not:V} +% This is used for the implementation of~|\u|, and it gets redefined +% for \cs{peek_regex_replace_once:nnTF}. +% \begin{macrocode} +\cs_new_eq:NN \@@_replacement_exp_not:V \exp_not:V +% \end{macrocode} +% \end{macro} +% % \subsubsection{Query and brace balance} % % \begin{macro}[rEXP]{\@@_query_range:nn} @@ -5189,42 +5196,25 @@ % range is the difference between the brace balances at the % \meta{max~pos} and \meta{min~pos}. These two positions are found in % the corresponding \enquote{submatch} arrays. -%^^A todo: understand when these int_compare are needed % \begin{macrocode} \cs_new_protected:Npn \@@_submatch_balance:n #1 { \int_eval:n - { - \int_compare:nNnTF - { - \__kernel_intarray_item:Nn - \g_@@_submatch_end_intarray {#1} - } - = 0 - { 0 } - { - \__kernel_intarray_item:Nn \g_@@_balance_intarray - { - \__kernel_intarray_item:Nn - \g_@@_submatch_end_intarray {#1} - } - } - - - \int_compare:nNnTF - { - \__kernel_intarray_item:Nn - \g_@@_submatch_begin_intarray {#1} - } - = 0 - { 0 } - { - \__kernel_intarray_item:Nn \g_@@_balance_intarray - { - \__kernel_intarray_item:Nn - \g_@@_submatch_begin_intarray {#1} - } - } - } + { + \@@_intarray_item:NnF \g_@@_balance_intarray + { + \__kernel_intarray_item:Nn + \g_@@_submatch_end_intarray {#1} + } + { 0 } + - + \@@_intarray_item:NnF \g_@@_balance_intarray + { + \__kernel_intarray_item:Nn + \g_@@_submatch_begin_intarray {#1} + } + { 0 } + } } % \end{macrocode} % \end{macro} @@ -5305,6 +5295,14 @@ % \end{macro} % \end{macro} % +% \begin{macro}{\@@_replacement_put:n} +% This gets redefined for \cs{peek_regex_replace_once:nnTF}. +% \begin{macrocode} +\cs_new_protected:Npn \@@_replacement_put:n + { \tl_build_put_right:Nn \l_@@_build_tl } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\@@_replacement_normal:n} % Most characters are simply sent to the output by % \cs{tl_build_put_right:Nn}, unless a particular category code has been @@ -5314,11 +5312,13 @@ % sequence is non-empty there: it contains an empty entry % corresponding to the initial value of % \cs{l_@@_replacement_category_tl}. +% The argument |#1| can be a space, otherwise it is a single +% character. % \begin{macrocode} \cs_new_protected:Npn \@@_replacement_normal:n #1 { \tl_if_empty:NTF \l_@@_replacement_category_tl - { \tl_build_put_right:Nn \l_@@_build_tl {#1} } + { \@@_replacement_put:n {#1} } { % ( \token_if_eq_charcode:NNTF #1 ) { @@ -5361,7 +5361,7 @@ % % \subsubsection{Submatches} % -% \begin{macro}{\@@_replacement_put_submatch:n} +% \begin{macro}{\@@_replacement_put_submatch:n, \@@_replacement_put_submatch_aux:n} % Insert a submatch in the replacement text. This is dropped if the % submatch number is larger than the number of capturing groups. % Unless the submatch appears inside a |\c{...}| or |\u{...}| @@ -5374,15 +5374,19 @@ \cs_new_protected:Npn \@@_replacement_put_submatch:n #1 { \if_int_compare:w #1 < \l_@@_capturing_group_int - \tl_build_put_right:Nn \l_@@_build_tl - { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } } - \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f: - \tl_put_right:Nn \l_@@_balance_tl - { - + \@@_submatch_balance:n - { \exp_not:N \int_eval:n { #1 + ##1 } } - } - \fi: + \@@_replacement_put_submatch_aux:n {#1} + \fi: + } +\cs_new_protected:Npn \@@_replacement_put_submatch_aux:n #1 + { + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } } + \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f: + \tl_put_right:Nn \l_@@_balance_tl + { + + \@@_submatch_balance:n + { \exp_not:N \int_eval:n { #1 + ##1 } } + } \fi: } % \end{macrocode} @@ -5488,7 +5492,7 @@ { \@@_two_if_eq:NNNNTF #1 #2 \@@_replacement_normal:n \c_left_brace_str - { \@@_replacement_cu_aux:Nw \exp_not:V } + { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:V } { \@@_replacement_error:NNN u #1#2 } } % \end{macrocode} @@ -5579,7 +5583,7 @@ \cs_new_protected:Npn \@@_replacement_char:nNN #1#2#3 { \tex_lccode:D 0 = `#3 \scan_stop: - \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {#1} } + \tex_lowercase:D { \@@_replacement_put:n {#1} } } % \end{macrocode} % \end{macro} @@ -5625,7 +5629,7 @@ \cs_new_protected:Npn \@@_replacement_c_C:w #1#2 { \tl_build_put_right:Nn \l_@@_build_tl - { \exp_not:N \exp_not:N \exp_not:c {#2} } + { \exp_not:N \@@_replacement_exp_not:N \exp_not:c {#2} } } % \end{macrocode} % \end{macro} @@ -5712,7 +5716,7 @@ \__kernel_msg_error:nn { kernel } { replacement-null-space } \fi: \tex_lccode:D `\ = `#2 \scan_stop: - \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {~} } + \tex_lowercase:D { \@@_replacement_put:n {~} } } % \end{macrocode} % \end{macro} @@ -5940,6 +5944,14 @@ % \end{macrocode} % \end{variable} % +% \begin{variable}{\g_@@_balance_intarray} +% The first thing we do when matching is to store the balance of +% begin-group/end-group characters into \cs{g_@@_balance_intarray}. +% \begin{macrocode} +\intarray_new:Nn \g_@@_balance_intarray { 65536 } +% \end{macrocode} +% \end{variable} +% % \begin{macro}{\@@_return:} % This function triggers either \cs{prg_return_false:} or % \cs{prg_return_true:} as appropriate to whether a match was found or @@ -5956,6 +5968,36 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}{\@@_query_set:n, \@@_query_set_aux:nN} +% To easily extract subsets of the input once we found the positions +% at which to cut, store the input tokens one by one into successive +% \tn{toks} registers. Also store the brace balance (used to check +% for overall brace balance) in an array. +% \begin{macrocode} +\cs_new_protected:Npn \@@_query_set:n #1 + { + \int_zero:N \l_@@_balance_int + \int_zero:N \l_@@_curr_pos_int + \@@_query_set_aux:nN { } F + \tl_analysis_map_inline:nn {#1} + { \@@_query_set_aux:nN {##1} ##3 } + \@@_query_set_aux:nN { } F + \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int + } +\cs_new_protected:Npn \@@_query_set_aux:nN #1#2 + { + \int_incr:N \l_@@_curr_pos_int + \@@_toks_set:Nn \l_@@_curr_pos_int {#1} + \__kernel_intarray_gset:Nnn \g_@@_balance_intarray + { \l_@@_curr_pos_int } { \l_@@_balance_int } + \if_case:w "#2 \exp_stop_f: + \or: \int_incr:N \l_@@_balance_int + \or: \int_decr:N \l_@@_balance_int + \fi: + } +% \end{macrocode} +% \end{macro} +% % \subsubsection{Matching} % % \begin{macro}{\@@_if_match:nn} @@ -6011,6 +6053,7 @@ #1 \@@_match:n {#2} \@@_extract: + \@@_query_set:n {#2} \@@_group_end_extract_seq:N #3 } \cs_new_protected:Npn \@@_extract_all:nnN #1#2#3 @@ -6019,6 +6062,7 @@ \@@_multi_match:n { \@@_extract: } #1 \@@_match:n {#2} + \@@_query_set:n {#2} \@@_group_end_extract_seq:N #3 } % \end{macrocode} @@ -6058,7 +6102,7 @@ } #1 \@@_match:n {#2} -%<assert>\assert_int:n { \l_@@_curr_pos_int = \l_@@_max_pos_int } + \@@_query_set:n {#2} \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray { \l_@@_submatch_int } { 0 } \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray @@ -6143,18 +6187,15 @@ % \end{macrocode} % \end{macro} % -% \begin{macro} -% {\@@_extract:, \@@_extract_b:wn, \@@_extract_e:wn} -% Our task here is to extract from the property list -% \cs{l_@@_success_submatches_prop} the list of end-points of -% submatches, and store them in appropriate array entries, from -% \cs{l_@@_zeroth_submatch_int} upwards. We begin by emptying those -% entries. Then for each \meta{key}--\meta{value} pair in -% the property list update the appropriate entry. This -% is somewhat a hack: the \meta{key} is a non-negative integer -% followed by |<| or |>|, which we use in a comparison to $-1$. At the -% end, store the information about the position at which the match -% attempt started, in \cs{g_@@_submatch_prev_intarray}. +% \begin{macro}{\@@_extract:} +% Our task here is to store the list of end-points of submatches, and +% store them in appropriate array entries, from +% \cs{l_@@_zeroth_submatch_int} upwards. First, we store in +% \cs{g_@@_submatch_prev_intarray} the position at which the match +% attempt started. We extract the rest from the comma list +% \cs{l_@@_success_submatches_tl}, which starts with entries to be +% stored in \cs{g_@@_submatch_begin_intarray} and continues with +% entries for \cs{g_@@_submatch_end_intarray}. % \begin{macrocode} \cs_new_protected:Npn \@@_extract: { @@ -6162,34 +6203,26 @@ \int_set_eq:NN \l_@@_zeroth_submatch_int \l_@@_submatch_int \prg_replicate:nn \l_@@_capturing_group_int { - \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray - { \l_@@_submatch_int } { 0 } - \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray - { \l_@@_submatch_int } { 0 } \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray { \l_@@_submatch_int } { 0 } \int_incr:N \l_@@_submatch_int } - \prop_map_inline:Nn \l_@@_success_submatches_prop + \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray + { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int } + \int_zero:N \l_@@_internal_a_int + \clist_map_inline:Nn \l_@@_success_submatches_tl { - \if_int_compare:w ##1 - 1 \exp_stop_f: - \exp_after:wN \@@_extract_e:wn \int_value:w + \if_int_compare:w \l_@@_internal_a_int < \l_@@_capturing_group_int + \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray + { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int } {##1} \else: - \exp_after:wN \@@_extract_b:wn \int_value:w + \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray + { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int - \l_@@_capturing_group_int } {##1} \fi: - \@@_int_eval:w \l_@@_zeroth_submatch_int + ##1 {##2} + \int_incr:N \l_@@_internal_a_int } - \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray - { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int } \fi: } -\cs_new_protected:Npn \@@_extract_b:wn #1 < #2 - { - \__kernel_intarray_gset:Nnn - \g_@@_submatch_begin_intarray {#1} {#2} - } -\cs_new_protected:Npn \@@_extract_e:wn #1 > #2 - { \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray {#1} {#2} } % \end{macrocode} % \end{macro} % @@ -6214,12 +6247,13 @@ \group_begin: \@@_single_match: #1 - \@@_replacement:n {#2} - \exp_args:No \@@_match:n { #3 } + \exp_args:No \@@_match:n {#3} \if_meaning:w \c_false_bool \g_@@_success_bool \group_end: \else: \@@_extract: + \exp_args:No \@@_query_set:n {#3} + \@@_replacement:n {#2} \int_set:Nn \l_@@_balance_int { \@@_replacement_balance_one_match:n @@ -6259,8 +6293,9 @@ \group_begin: \@@_multi_match:n { \@@_extract: } #1 - \@@_replacement:n {#2} \exp_args:No \@@_match:n {#3} + \exp_args:No \@@_query_set:n {#3} + \@@_replacement:n {#2} \int_set:Nn \l_@@_balance_int { 0 @@ -6320,7 +6355,371 @@ % \end{macrocode} % \end{macro} % -% \subsubsection{Storing and showing compiled patterns} +% \subsubsection{Peeking ahead} +% +% \begin{variable}{\l_@@_peek_true_tl, \l_@@_peek_false_tl} +% True/false code arguments of \cs{peek_regex:nTF} or similar. +% \begin{macrocode} +\tl_new:N \l_@@_peek_true_tl +\tl_new:N \l_@@_peek_false_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_replacement_tl} +% When peeking in \cs{peek_regex_replace_once:nnTF} we need to store +% the replacement text. +% \begin{macrocode} +\tl_new:N \l_@@_replacement_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_input_tl} +% \begin{macro}{\@@_input_item:n} +% Stores each token found as \cs{@@_input_item:n} \Arg{tokens}, where +% the \meta{tokens} \texttt{o}-expand to the token found, as for +% \cs{tl_analysis_map_inline:nn}. +% \begin{macrocode} +\tl_new:N \l_@@_input_tl +\cs_new_eq:NN \@@_input_item:n ? +% \end{macrocode} +% \end{macro} +% \end{variable} +% +% \begin{macro}[TF] +% {\peek_regex:n, \peek_regex:N, \peek_regex_remove_once:n, \peek_regex_remove_once:N} +% The |T| and |F| functions just call the corresponding |TF| function. +% The four |TF| functions differ along two axes: whether to remove the +% token or not, distinguished by using \cs{@@_peek_end:} or +% \cs{@@_peek_remove_end:n} (the latter case needs an argument, as we +% will see), and whether the regex has to be compiled or is already in +% an |N|-type variable, distinguished by calling \cs{@@_build_aux:Nn} +% or \cs{@@_build_aux:NN}. The first argument of these functions is +% \cs{c_false_bool} to indicate that there should be no implicit +% insertion of a wildcard at the start of the pattern: otherwise the +% code would keep looking further into the input stream until matching +% the regex. +% \begin{macrocode} +\cs_new_protected:Npn \peek_regex:nTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:Nn \c_false_bool {#1} } + { \@@_peek_end: } + } +\cs_new_protected:Npn \peek_regex:nT #1#2 + { \peek_regex:nTF {#1} {#2} { } } +\cs_new_protected:Npn \peek_regex:nF #1 { \peek_regex:nTF {#1} { } } +\cs_new_protected:Npn \peek_regex:NTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:NN \c_false_bool #1 } + { \@@_peek_end: } + } +\cs_new_protected:Npn \peek_regex:NT #1#2 + { \peek_regex:NTF #1 {#2} { } } +\cs_new_protected:Npn \peek_regex:NF #1 { \peek_regex:NTF {#1} { } } +\cs_new_protected:Npn \peek_regex_remove_once:nTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:Nn \c_false_bool {#1} } + { \@@_peek_remove_end:n {##1} } + } +\cs_new_protected:Npn \peek_regex_remove_once:nT #1#2 + { \peek_regex_remove_once:nTF {#1} {#2} { } } +\cs_new_protected:Npn \peek_regex_remove_once:nF #1 + { \peek_regex_remove_once:nTF {#1} { } } +\cs_new_protected:Npn \peek_regex_remove_once:NTF #1 + { + \@@_peek:nnTF + { \@@_build_aux:NN \c_false_bool #1 } + { \@@_peek_remove_end:n {##1} } + } +\cs_new_protected:Npn \peek_regex_remove_once:NT #1#2 + { \peek_regex_remove_once:NTF #1 {#2} { } } +\cs_new_protected:Npn \peek_regex_remove_once:NF #1 + { \peek_regex_remove_once:NTF #1 { } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek:nnTF, \@@_peek_aux:nnTF} +% Store the user's true/false codes (plus \cs{group_end:}) into two +% token lists. Then build the automaton with |#1|, without submatch +% tracking, and aiming for a single match. Then start matching by +% setting up a few variables like for any regex matching like +% \cs{regex_match:nnTF}, with the addition of \cs{l_@@_input_tl} +% that keeps track of the tokens seen, to reinsert them at the +% end. Instead of \cs{tl_analysis_map_inline:nn} on the input, we +% call \cs{peek_analysis_map_inline:n} to go through tokens in the +% input stream. Since \cs{@@_match_one_token:nnN} calls +% \cs{@@_maplike_break:} we need to catch that and break the +% \cs{peek_analysis_map_inline:n} loop instead. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek:nnTF #1 + { + \@@_peek_aux:nnTF + { + \@@_disable_submatches: + #1 + } + } +\cs_new_protected:Npn \@@_peek_aux:nnTF #1#2#3#4 + { + \group_begin: + \tl_set:Nn \l_@@_peek_true_tl { \group_end: #3 } + \tl_set:Nn \l_@@_peek_false_tl { \group_end: #4 } + \@@_single_match: + #1 + \@@_match_init: + \tl_build_clear:N \l_@@_input_tl + \@@_match_once_init: + \peek_analysis_map_inline:n + { + \tl_build_put_right:Nn \l_@@_input_tl + { \@@_input_item:n {##1} } + \@@_match_one_token:nnN {##1} {##2} ##3 + \use_none:nnn + \prg_break_point:Nn \@@_maplike_break: + { \peek_analysis_map_break:n {#2} } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_end:, \@@_peek_remove_end:n} +% Once the regex matches (or permanently fails to match) we call +% \cs{@@_peek_end:}, or \cs{@@_peek_remove_end:n} with argument the +% last token seen (or rather tokens that \texttt{o}-expand and +% \texttt{x}-expand to it). For \cs{peek_regex:nTF} we reinsert +% tokens seen by calling \cs{@@_peek_reinsert:N} regardless of the +% result of the match. For \cs{peek_regex_remove_once:nTF} we reinsert the +% tokens seen only if the match failed; otherwise we just reinsert the +% tokens~|#1|, with one expansion. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_end: + { + \bool_if:NTF \g_@@_success_bool + { \@@_peek_reinsert:N \l_@@_peek_true_tl } + { \@@_peek_reinsert:N \l_@@_peek_false_tl } + } +\cs_new_protected:Npn \@@_peek_remove_end:n #1 + { + \bool_if:NTF \g_@@_success_bool + { \exp_after:wN \l_@@_peek_true_tl #1 } + { \@@_peek_reinsert:N \l_@@_peek_false_tl } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_reinsert:N, \@@_reinsert_item:n} +% Insert the true/false code |#1|, followed by the tokens found, which +% were stored in \cs{l_@@_input_tl}. For this, loop through that +% token list using \cs{@@_reinsert_item:n}, which expands |#1| once to +% get a single token, and jumps over it to expand what follows, with +% suitable \cs{exp:w} and \cs{exp_end:}. We cannot just use +% \cs{use:e} on the whole token list because the result may be +% unbalanced, which would stop the primitive prematurely, or let it +% continue beyond where we would like. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_reinsert:N #1 + { + \tl_build_end:N \l_@@_input_tl + \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n + \exp_after:wN #1 \exp:w \l_@@_input_tl \exp_end: + } +\cs_new_protected:Npn \@@_reinsert_item:n #1 + { + \exp_after:wN \exp_after:wN + \exp_after:wN \exp_end: + \exp_after:wN \exp_after:wN + #1 + \exp:w + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[noTF] +% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn} +% Similar to \cs{peek_regex:nTF} above. +% \begin{macrocode} +\cs_new_protected:Npn \peek_regex_replace_once:nnTF #1 + { \@@_peek_replace:nnTF { \@@_build_aux:Nn \c_false_bool {#1} } } +\cs_new_protected:Npn \peek_regex_replace_once:nnT #1#2#3 + { \peek_regex_replace_once:nnTF {#1} {#2} {#3} { } } +\cs_new_protected:Npn \peek_regex_replace_once:nnF #1#2 + { \peek_regex_replace_once:nnTF {#1} {#2} { } } +\cs_new_protected:Npn \peek_regex_replace_once:nn #1#2 + { \peek_regex_replace_once:nnTF {#1} {#2} { } { } } +\cs_new_protected:Npn \peek_regex_replace_once:NnTF #1 + { \@@_peek_replace:nnTF { \@@_build_aux:NN \c_false_bool #1 } } +\cs_new_protected:Npn \peek_regex_replace_once:NnT #1#2#3 + { \peek_regex_replace_once:NnTF #1 {#2} {#3} { } } +\cs_new_protected:Npn \peek_regex_replace_once:NnF #1#2 + { \peek_regex_replace_once:NnTF #1 {#2} { } } +\cs_new_protected:Npn \peek_regex_replace_once:Nn #1#2 + { \peek_regex_replace_once:NnTF #1 {#2} { } { } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replace:nnTF} +% Same as \cs{@@_peek:nnTF} (used for \cs{peek_regex:nTF} above), but +% without disabling submatches, and with a different end. The +% replacement text |#2| is stored, to be analyzed later. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replace:nnTF #1#2 + { + \tl_set:Nn \l_@@_replacement_tl {#2} + \@@_peek_aux:nnTF {#1} { \@@_peek_replace_end: } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replace_end:} +% If the match failed \cs{@@_peek_reinsert:N} reinserts the tokens +% found. Otherwise, finish storing the submatch information using +% \cs{@@_extract:}, and store the input into \tn{toks}. Redefine a +% few auxiliaries to change slightly their expansion behaviour as +% explained below. Analyse the replacement text with +% \cs{@@_replacement:n}, which as usual defines +% \cs{@@_replacement_do_one_match:n} to insert the tokens from the +% start of the match attempt to the beginning of the match, followed +% by the replacement text. The \cs{use:x} expands for instance the +% trailing \cs{@@_query_range:nn} down to a sequence of +% \cs{@@_reinsert_item:n} \Arg{tokens} where \meta{tokens} +% \texttt{o}-expand to a single token that we want to insert. After +% \texttt{x}-expansion, \cs{use:x} does \cs{use:n}, so we have +% \cs{exp_after:wN} \cs{l_@@_peek_true_tl} \cs{exp:w} \ldots{} +% \cs{exp_end:}. This is set up such as to obtain +% \cs{l_@@_peek_true_tl} followed by the replaced tokens (possibly +% unbalanced) in the input stream. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replace_end: + { + \bool_if:NTF \g_@@_success_bool + { + \@@_extract: + \@@_query_set_from_input_tl: + \cs_set_eq:NN \@@_replacement_put:n \@@_peek_replacement_put:n + \cs_set_eq:NN \@@_replacement_put_submatch_aux:n + \@@_peek_replacement_put_submatch_aux:n + \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n + \cs_set_eq:NN \@@_replacement_exp_not:N \@@_peek_replacement_token:n + \cs_set_eq:NN \@@_replacement_exp_not:V \@@_peek_replacement_var:N + \exp_args:No \@@_replacement:n { \l_@@_replacement_tl } + \use:x + { + \exp_not:n { \exp_after:wN \l_@@_peek_true_tl \exp:w } + \@@_replacement_do_one_match:n + { \l_@@_zeroth_submatch_int } + \@@_query_range:nn + { + \__kernel_intarray_item:Nn \g_@@_submatch_end_intarray + { \l_@@_zeroth_submatch_int } + } + { \l_@@_max_pos_int } + \exp_end: + } + } + { \@@_peek_reinsert:N \l_@@_peek_false_tl } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_query_set_from_input_tl:, \@@_query_set_item:n} +% The input was stored into \cs{l_@@_input_tl} as successive items +% \cs{@@_input_item:n} \Arg{tokens}. Store that in successive +% \tn{toks}. It's not clear whether the empty entries before and +% after are both useful. +% \begin{macrocode} +\cs_new_protected:Npn \@@_query_set_from_input_tl: + { + \tl_build_end:N \l_@@_input_tl + \int_zero:N \l_@@_curr_pos_int + \cs_set_eq:NN \@@_input_item:n \@@_query_set_item:n + \@@_query_set_item:n { } + \l_@@_input_tl + \@@_query_set_item:n { } + \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int + } +\cs_new_protected:Npn \@@_query_set_item:n #1 + { + \int_incr:N \l_@@_curr_pos_int + \@@_toks_set:Nn \l_@@_curr_pos_int { \@@_input_item:n {#1} } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_put:n} +% While building the replacement function +% \cs{@@_replacement_do_one_match:n}, we often want to put simple +% material, given as |#1|, whose \texttt{x}-expansion +% \texttt{o}-expands to a single token. Normally we can just add the +% token to \cs{l_@@_build_tl}, but for +% \cs{peek_regex_replace_once:nnTF} we eventually want to do some +% strange expansion that is basically using \cs{exp_after:wN} to jump +% through numerous tokens (we cannot use \texttt{x}-expansion like for +% \cs{regex_replace_once:nnNTF} because it is ok for the result to be +% unbalanced since we insert it in the input stream rather than +% storing it. When within a csname we don't do any such shenanigan +% because \cs{cs:w} \ldots{} \cs{cs_end:} does all the expansion we +% need. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_put:n #1 + { + \if_case:w \l_@@_replacement_csnames_int + \tl_build_put_right:Nn \l_@@_build_tl + { \exp_not:N \@@_reinsert_item:n {#1} } + \else: + \tl_build_put_right:Nn \l_@@_build_tl {#1} + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_token:n} +% When hit with \cs{exp:w}, \cs{@@_peek_replacement_token:n} +% \Arg{token} stops \cs{exp_end:} and does \cs{exp_after:wN} +% \meta{token} \cs{exp:w} to continue expansion after it. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_token:n #1 + { \exp_after:wN \exp_end: \exp_after:wN #1 \exp:w } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_put_submatch_aux:n} +% While analyzing the replacement we also have to insert submatches +% found in the query. Since query items \cs{@@_input_item:n} +% \Arg{tokens} expand correctly only when surrounded by \cs{exp:w} +% \ldots{} \cs{exp_end:}, and since these expansion controls are not +% there within csnames (because \cs{cs:w} \ldots{} \cs{cs_end:} make +% them unnecessary in most cases), we have to put \cs{exp:w} and +% \cs{exp_end:} by hand here. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_put_submatch_aux:n #1 + { + \if_case:w \l_@@_replacement_csnames_int + \tl_build_put_right:Nn \l_@@_build_tl + { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } } + \else: + \tl_build_put_right:Nn \l_@@_build_tl + { \exp:w \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } \exp_end: } + \fi: + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_peek_replacement_var:N} +% This is used for |\u| outside csnames. It makes sure to continue +% expansion with \cs{exp:w} before expanding the variable~|#1| and +% stopping the \cs{exp:w} that precedes. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_replacement_var:N #1 + { + \exp_after:wN \exp_last_unbraced:NV + \exp_after:wN \exp_end: + \exp_after:wN #1 + \exp:w + } +% \end{macrocode} +% \end{macro} % % \subsection{Messages} % @@ -6726,8 +7125,6 @@ %^^A NOT IMPLEMENTED %^^A \p{xx} a character with the xx property %^^A \P{xx} a character without the xx property -%^^A [[:xxx:]] positive POSIX named set -%^^A [[:^xxx:]] negative POSIX named set %^^A (?=...) positive look ahead %^^A (?!...) negative look ahead %^^A (?<=...) positive look behind diff --git a/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx b/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx index 737a4c776af..28df8592d9d 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx b/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx index aeef5ae45c4..6336081c2cb 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx b/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx index 30d2b9737ed..b47a9d73d63 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx b/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx index 2acfdcf0aa1..e3ebadbcdd0 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3str.dtx b/Master/texmf-dist/source/latex/l3kernel/l3str.dtx index bdde8616550..02083ae9b5f 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3str.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3str.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx b/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx index 4e5e3816c7a..8ba6b2f2e6c 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx b/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx index fd41c6bdd31..b52d3488672 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx b/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx index 1bc377ed02a..4a4abbf256a 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -396,6 +396,8 @@ \tiny } { \text_declare_purify_equivalent:Nn #1 { } } +\exp_args:Nc \text_declare_purify_equivalent:Nn + { @protected@testopt } { \use_none:nnn } % \end{macrocode} % Environments have to be handled by pure expansion. % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3text.dtx b/Master/texmf-dist/source/latex/l3kernel/l3text.dtx index 893c43da59a..9ab4ae6ef52 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3text.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3text.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -634,6 +634,8 @@ % \begin{macro}[EXP]{\@@_expand_protect:N} % \begin{macro}[EXP]{\@@_expand_protect:nN} % \begin{macro}[EXP]{\@@_expand_protect:Nw} +% \begin{macro}[EXP]{\@@_expand_testopt:N} +% \begin{macro}[EXP]{\@@_expand_testopt:NNn} % \begin{macro}[EXP]{\@@_expand_replace:N} % \begin{macro}[EXP]{\@@_expand_replace:n} % \begin{macro}[EXP]{\@@_expand_cs_expand:N} @@ -917,7 +919,8 @@ % \LaTeXe{}'s \cs{protect} makes life interesting. Where possible, we % simply remove it and replace with the \enquote{parent} command; of course, % the \cs{protect} might be explicit, in which case we need to leave it alone -% if it's required. +% if it's required. There is also the case of a straight \tn{@protected@testopt} +% to cover. % \begin{macrocode} \cs_new:Npx \@@_expand_cs:N #1 { @@ -927,7 +930,7 @@ \bool_lazy_and:nnTF { \cs_if_exist_p:N \fmtname } { \str_if_eq_p:Vn \fmtname { LaTeX2e } } - { \exp_not:N \@@_expand_encoding:N #1 } + { \exp_not:N \@@_expand_testopt:N #1 } { \exp_not:N \@@_expand_replace:N #1 } } } @@ -949,6 +952,17 @@ { \@@_expand_store:n { \protect #1 } } \@@_expand_loop:w } +\cs_new:Npn \@@_expand_testopt:N #1 + { + \str_if_eq:nnTF {#1} { \@protected@testopt } + { \@@_expand_testopt:NNn } + { \@@_expand_encoding:N #1 } + } +\cs_new:Npn \@@_expand_testopt:NNn #1#2#3 + { + \@@_expand_store:n {#1} + \@@_expand_loop:w + } % \end{macrocode} % Deal with encoding-specific commands % \begin{macrocode} @@ -1036,6 +1050,8 @@ % \end{macro} % \end{macro} % \end{macro} +% \end{macro} +% \end{macro} % % \begin{macro} % { diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx index be4bd2406e8..cf61a8f7028 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx @@ -44,7 +44,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -52,10 +52,30 @@ % % \section{\pkg{l3tl-analysis} documentation} % -% This module mostly provides internal functions for use in the -% \pkg{l3regex} module. However, it provides as a side-effect a user -% debugging function, very similar to the \cs{ShowTokens} macro from the -% \pkg{ted} package. +% This module provides functions that are particularly useful in the +% \pkg{l3regex} module for mapping through a token list one \meta{token} +% at a time (including begin-group/end-group tokens). For +% \cs{tl_analysis_map_inline:Nn} or \cs{tl_analysis_map_inline:nn}, the +% token list is given as an argument; the analogous function +% \cs{peek_analysis_map_inline:n} documented in \pkg{l3token} finds +% tokens in the input stream instead. In both cases the user provides +% \meta{inline code} that receives three arguments for each +% \meta{token}: +% \begin{itemize} +% \item \meta{tokens}, which both \texttt{o}-expand and +% \texttt{x}-expand to the \meta{token}. The detailed form of +% \meta{tokens} may change in later releases. +% \item \meta{char code}, a decimal representation of the character +% code of the \meta{token}, $-1$ if it is a control sequence. +% \item \meta{catcode}, a capital hexadecimal digit which denotes the +% category code of the \meta{token} (0:~control sequence, +% 1:~begin-group, 2:~end-group, 3:~math shift, 4:~alignment tab, +% 6:~parameter, 7:~superscript, 8:~subscript, A:~space, B:~letter, +% C:~other, D:~active). This can be converted to an integer by +% writing |"|\meta{catcode}. +% \end{itemize} +% In addition, there is a debugging function \cs{tl_analysis_show:n}, +% very similar to the \cs{ShowTokens} macro from the \pkg{ted} package. % % \begin{function}[added = 2018-04-09]{\tl_analysis_show:N, \tl_analysis_show:n} % \begin{syntax} @@ -74,23 +94,9 @@ % \end{syntax} % Applies the \meta{inline function} to each individual \meta{token} % in the \meta{token list}. The \meta{inline function} receives three -% arguments: -% \begin{itemize} -% \item \meta{tokens}, which both \texttt{o}-expand and -% \texttt{x}-expand to the \meta{token}. The detailed form of -% \meta{token} may change in later releases. -% \item \meta{char code}, a decimal representation of the character -% code of the token, $-1$ if it is a control sequence (with -% \meta{catcode} $0$). -% \item \meta{catcode}, a capital hexadecimal digit which denotes -% the category code of the \meta{token} (0: control sequence, 1: -% begin-group, 2: end-group, 3: math shift, 4: alignment tab, 6: -% parameter, 7: superscript, 8: subscript, A: space, B: letter, -% C:other, D:active). -% \end{itemize} -% As all other mappings the mapping is done at the current group -% level, \emph{i.e.}~any local assignments made by the \meta{inline -% function} remain in effect after the loop. +% arguments as explained above. As all other mappings the mapping is +% done at the current group level, \emph{i.e.}~any local assignments +% made by the \meta{inline function} remain in effect after the loop. % \end{function} % % \end{documentation} @@ -182,18 +188,54 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_@@_analysis_token} -% \begin{variable}{\l_@@_analysis_char_token} +% \begin{variable} +% {\l_@@_analysis_token, \l_@@_analysis_char_token, \l_@@_analysis_next_token} % The tokens in the token list are probed with the \TeX{} primitive % \tn{futurelet}. We use \cs{l_@@_analysis_token} in that % construction. In some cases, we convert the following token to a % string before probing it: then the token variable used is -% \cs{l_@@_analysis_char_token}. +% \cs{l_@@_analysis_char_token}. When getting tokens from the input +% stream we may need to look two tokens ahead, for which we use +% \cs{l_@@_analysis_next_token}. % \begin{macrocode} \cs_new_eq:NN \l_@@_analysis_token ? \cs_new_eq:NN \l_@@_analysis_char_token ? +\cs_new_eq:NN \l_@@_analysis_next_token ? +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_peek_code_tl} +% Holds some code to be run once the next token has been fully +% analysed in \cs{peek_analysis_map_inline:n}. +% \begin{macrocode} +\tl_new:N \l_@@_peek_code_tl % \end{macrocode} % \end{variable} +% +% \begin{variable}{\c_@@_peek_catcodes_tl} +% A token list containing the character number~$32$ (space) with all +% possible category codes except $1$ and $2$ (begin-group and +% end-group). Why $32$? Because some \LuaTeX{} versions only allow +% creation of catcode~$10$ (space) tokens with this character code, +% and because even in other engines it is much easier to produce since +% \cs{char_generate:nn} refuses to produce spaces. +% \begin{macrocode} +\group_begin: +\char_set_active_eq:NN \ \scan_stop: +\tl_const:Nx \c_@@_peek_catcodes_tl + { + \char_generate:nn { 32 } { 3 } 3 + \char_generate:nn { 32 } { 4 } 4 + # \char_generate:nn { 32 } { 6 } 6 + \char_generate:nn { 32 } { 7 } 7 + \char_generate:nn { 32 } { 8 } 8 + \c_space_tl \token_to_str:N A + \char_generate:nn { 32 } { 11 } \token_to_str:N B + \char_generate:nn { 32 } { 12 } \token_to_str:N C + \char_generate:nn { 32 } { 13 } \token_to_str:N D + } +\group_end: +% \end{macrocode} % \end{variable} % % \begin{variable}{\l_@@_analysis_normal_int} @@ -1075,6 +1117,358 @@ % \end{macro} % \end{macro} % +% \subsection{Peeking ahead} +% +% \begin{macro}[EXP]{\peek_analysis_map_break:, \peek_analysis_map_break:n} +% The break statements use the general \cs{prg_map_break:Nn}. +% \begin{macrocode} +\cs_new:Npn \peek_analysis_map_break: + { \prg_map_break:Nn \peek_analysis_map_break: { } } +\cs_new:Npn \peek_analysis_map_break:n + { \prg_map_break:Nn \peek_analysis_map_break: } +% \end{macrocode} +% \end{macro} +% +% \begin{variable}{\l_@@_peek_charcode_int} +% \begin{macrocode} +\int_new:N \l_@@_peek_charcode_int +% \end{macrocode} +% \end{variable} +% +% \begin{macro}{\@@_analysis_char_arg:Nw, \@@_analysis_char_arg_aux:Nw} +% After a call to \tn{futurelet} \cs{l_@@_analysis_token} followed by +% a stringified character token (either explicit space or catcode +% other character), grab the argument and pass it to |#1|. We only +% need to do anything in the case of a space. +% \begin{macrocode} +\cs_new:Npn \@@_analysis_char_arg:Nw + { + \if_meaning:w \l_@@_analysis_token \c_space_token + \exp_after:wN \@@_analysis_char_arg_aux:Nw + \fi: + } +\cs_new:Npn \@@_analysis_char_arg_aux:Nw #1 ~ { #1 { ~ } } +% \end{macrocode} +% \end{macro} +% +% \begin{macro} +% { +% \peek_analysis_map_inline:n, +% \@@_peek_analysis_loop:NNn, \@@_peek_analysis_test:, +% \@@_peek_analysis_normal:N, \@@_peek_analysis_cs:, +% \@@_peek_analysis_char:N, \@@_peek_analysis_char:nN, +% \@@_peek_analysis_special:, \@@_peek_analysis_retest:, +% \@@_peek_analysis_next:, \@@_peek_analysis_str:, +% \@@_peek_analysis_str:w, \@@_peek_analysis_str:n, +% \@@_peek_analysis_active_str:n, \@@_peek_analysis_explicit:n, +% \@@_peek_analysis_escape:, \@@_peek_analysis_collect:w, +% \@@_peek_analysis_collect:n, \@@_peek_analysis_collect_loop:, +% \@@_peek_analysis_collect_test:, \@@_peek_analysis_collect_end:NNN +% } +% Save the user's code in a control sequence that is suitable for +% nested maps. We may wish to pass to this function an \tn{outer} +% control sequence or active character; for this we will undefine +% potentially-\tn{outer} tokens within a group, closed after the +% function receives its arguments. This user's code function also +% calls the loop auxiliary, and includes the trailing +% \cs{prg_break_point:Nn} for when the user wants to stop the loop. +% The loop auxiliary must remove that break point because it must look +% at the input stream. +% \begin{macrocode} +\cs_new_protected:Npn \peek_analysis_map_inline:n #1 + { + \int_gincr:N \g__kernel_prg_map_int + \cs_set_protected:cpn + { @@_analysis_map_ \int_use:N \g__kernel_prg_map_int :nnN } + ##1##2##3 + { + \group_end: + #1 + \@@_peek_analysis_loop:NNn + \prg_break_point:Nn \peek_analysis_map_break: { } + } + \@@_peek_analysis_loop:NNn ? ? ? + } +% \end{macrocode} +% The loop starts a group (closed by the user-code function defined +% above) with a normalized escape character, and checks if the next +% token is special or \texttt{N}-type. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_loop:NNn #1#2#3 + { + \group_begin: + \tl_set:Nx \l_@@_peek_code_tl + { + \exp_not:c + { @@_analysis_map_ \int_use:N \g__kernel_prg_map_int :nnN } + } + \int_set:Nn \tex_escapechar:D { `\\ } + \peek_after:Nw \@@_peek_analysis_test: + } +\cs_new_protected:Npn \@@_peek_analysis_test: + { + \if_int_odd:w + \if_catcode:w \exp_not:N \l_peek_token { 1 \exp_stop_f: \fi: + \if_catcode:w \exp_not:N \l_peek_token } 1 \exp_stop_f: \fi: + \if_meaning:w \l_peek_token \c_space_token 1 \exp_stop_f: \fi: + 0 \exp_stop_f: + \exp_after:wN \@@_peek_analysis_special: + \else: + \exp_after:wN \exp_after:wN + \exp_after:wN \@@_peek_analysis_normal:N + \exp_after:wN \exp_not:N + \fi: + } +% \end{macrocode} +% Normal tokens are not too hard, but can be \tn{outer}, hence the +% \cs{exp_not:N} in the code above. If the token is expandable then +% it might be an \tn{outer} or a \TeX{} conditional, so to be safe we +% set it to \cs{scan_stop:} (the assignment is local and stopped by +% the \cs{group_end:} upon calling the user's code). Then distinguish +% characters (including active ones and macro parameter characters) +% from control sequences (whose string representation is more than one +% character because the escape character is printable). For a control +% sequence call the user code with suitable arguments. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_normal:N #1 + { + \exp_after:wN \reverse_if:N \exp_after:wN \if_meaning:w + \exp_not:N #1 #1 + \tex_let:D #1 \scan_stop: + \tl_put_right:Nn \l_@@_peek_code_tl { { \exp_not:N #1 } } + \else: + \tl_put_right:Nn \l_@@_peek_code_tl { { \exp_not:n {#1} } } + \fi: + \if_charcode:w + \scan_stop: + \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing: + \scan_stop: + \exp_after:wN \@@_peek_analysis_char:N + \exp_after:wN #1 + \else: + \exp_after:wN \@@_peek_analysis_cs: + \fi: + } +\cs_new_protected:Npn \@@_peek_analysis_cs: + { \l_@@_peek_code_tl { -1 } 0 } +\cs_new_protected:Npn \@@_peek_analysis_char:N #1 + { + \char_set_lccode:nn { `#1 } { 32 } + \tex_lowercase:D { \@@_peek_analysis_char:nN {#1} } #1 + } +\cs_new_protected:Npn \@@_peek_analysis_char:nN #1#2 + { + \cs_set_protected:Npn \@@_tmp:w ##1 #1 ##2 ##3 \scan_stop: + { \exp_args:No \l_@@_peek_code_tl { \int_value:w `#2 } ##2 } + \exp_after:wN \@@_tmp:w \c_@@_peek_catcodes_tl \scan_stop: + } +% \end{macrocode} +% For special characters the idea is to eventually act with +% \cs{token_to_str:N}, then pick up one by one the characters of this +% string representation until hitting the token that follows. First +% determine the character code of (the meaning of) the \meta{token} +% (which we know is a special token), make sure the escape character +% is different from it, normalize the meanings of two active +% characters and the empty control sequence, and filter out these +% cases in \cs{@@_peek_analysis_retest:}. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_special: + { + \tex_let:D \l_@@_analysis_token = ~ \l_peek_token + \int_set:Nn \l_@@_peek_charcode_int + { \@@_analysis_extract_charcode: } + \if_int_compare:w \l_@@_peek_charcode_int = \tex_escapechar:D + \int_set:Nn \tex_escapechar:D { `\/ } + \fi: + \char_set_active_eq:nN { \l_@@_peek_charcode_int } \scan_stop: + \char_set_active_eq:nN { \tex_escapechar:D } \scan_stop: + \cs_set_eq:cN { } \scan_stop: + \tex_futurelet:D \l_@@_analysis_token + \@@_peek_analysis_retest: + } +\cs_new_protected:Npn \@@_peek_analysis_retest: + { + \if_meaning:w \l_@@_analysis_token \scan_stop: + \exp_after:wN \@@_peek_analysis_normal:N + \else: + \exp_after:wN \@@_peek_analysis_next: + \fi: + } +% \end{macrocode} +% At this point we know the meaning of the \meta{token} in the input +% stream is \cs{l_peek_token}, either a space (32, 10) or a +% begin-group or end-group token (catcode $1$ or~$2$), and we excluded +% a few cases that would be difficult later (empty control sequence, +% active character with the same character code as its meaning or as +% the escape character). Now look at the \meta{next token} following +% it using a combination of \tn{afterassignment} and \tn{futurelet}. +% The syntax of this primitive is \tn{futurelet} \meta{peek token} +% \meta{first token} \meta{next token}, and it sets \meta{peek token} +% equal to \meta{next token}. Traditionally, one takes \meta{first +% token} to be some macro that regains control of the code and, e.g., +% analyses \meta{peek token}. Here, both \meta{first token} and +% \meta{next token} are mostly unknown tokens in the input stream (but +% we know the \meta{first token} has catcode $1$, $2$ or $10$), where +% \meta{first token} was already stored as \cs{l_peek_token}, and we +% regain control using \tn{afterassignment}, which inserts its +% argument after the assignment, hence after \meta{peek token} but +% before \meta{first token}. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_next: + { + \tl_if_empty:oT { \tex_the:D \tex_everyeof:D } + { \tex_everyeof:D { \scan_stop: } } + \tex_afterassignment:D \@@_peek_analysis_str: + \tex_futurelet:D \l_@@_analysis_next_token + } +% \end{macrocode} +% We then hit the \meta{first token} with \cs{token_to_str:N} and grab +% characters until finding \cs{l_@@_analysis_next_token}. More +% precisely, by looking at the first character in the string +% representation of the \meta{first token} we distinguish three cases: +% a stringified control sequence starts with the escape character; for +% an explicit character we find that same character; for an explicit +% character we find anything else (we made sure to exclude the case of +% an active character whose string representation coincides with the +% other two cases). +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_str: + { + \exp_after:wN \tex_futurelet:D + \exp_after:wN \l_@@_analysis_token + \exp_after:wN \@@_peek_analysis_str:w + \token_to_str:N + } +\cs_new_protected:Npn \@@_peek_analysis_str:w + { \@@_analysis_char_arg:Nw \@@_peek_analysis_str:n } +\cs_new_protected:Npn \@@_peek_analysis_str:n #1 + { + \int_case:nnF { `#1 } + { + { \l_@@_peek_charcode_int } + { \@@_peek_analysis_explicit:n {#1} } + { \tex_escapechar:D } { \@@_peek_analysis_escape: } + } + { \@@_peek_analysis_active_str:n {#1} } + } +% \end{macrocode} +% When |#1| is a stringified active character we pass appropriate +% arguments to the user's code; thankfully \cs{char_generate:nn} +% can make active characters. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_active_str:n #1 + { + \tl_put_right:Nx \l_@@_peek_code_tl + { + { \char_generate:nn { `#1 } { 13 } } + { \int_value:w `#1 } + \token_to_str:N D + } + \l_@@_peek_code_tl + } +% \end{macrocode} +% When |#1| matches the character we had extracted from the meaning of +% \cs{l_peek_token}, the token was an explicit character, which can be +% a standard space, or a begin-group or end-group character with some +% character code. In the latter two cases we call +% \cs{char_generate:nn} with suitable arguments and put suitable +% \cs{if_false:} \cs{fi:} constructions to make the result balanced +% and such that \texttt{o}-expanding or \texttt{x}-expanding gives +% back a single (unbalanced) begin-group or end-group character. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_explicit:n #1 + { + \tl_put_right:Nx \l_@@_peek_code_tl + { + \if_meaning:w \l_peek_token \c_space_token + { ~ } { 32 } \token_to_str:N A + \else: + \if_catcode:w \l_peek_token \c_group_begin_token + { + \exp_not:N \exp_after:wN + \char_generate:nn { `#1 } { 1 } + \exp_not:N \if_false: + \if_false: { \fi: } + \exp_not:N \fi: + } + { \int_value:w `#1 } + 1 + \else: + { + \exp_not:N \if_false: + { \if_false: } \fi: + \exp_not:N \fi: + \char_generate:nn { `#1 } { 2 } + } + { \int_value:w `#1 } + 2 + \fi: + \fi: + } + \l_@@_peek_code_tl + } +% \end{macrocode} +% Finally there is the case of a special token whose string +% representation starts with an escape character, namely the token was +% a control sequence. In that case we could have grabbed the token +% directly as an \texttt{N}-type argument, but of course we couldn't +% know that until we had run all the various tests including +% stringifying the token. We are thus left with the hard work of +% picking up one by one the characters in the csname (being careful +% about spaces), until finding a token that matches the \meta{next +% token} picked up earlier (which was not stringified), such that the +% control sequence that we found so far indeed has the expected +% meaning \cs{l_peek_token}. This comparison with \cs{l_peek_token} +% catches a reasonably common case like \cs{c_group_begin_token} |_| +% in which the trailing |_| has category code other: without +% comparison of the constructed csname with \cs{l_peek_token} +% collection would stop at \cs[no-index]{c}, which is wrong. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_escape: + { + \tl_clear:N \l_@@_internal_a_tl + \tex_futurelet:D \l_@@_analysis_token + \@@_peek_analysis_collect:w + } +\cs_new_protected:Npn \@@_peek_analysis_collect:w + { \@@_analysis_char_arg:Nw \@@_peek_analysis_collect:n } +\cs_new_protected:Npn \@@_peek_analysis_collect:n #1 + { + \tl_put_right:Nn \l_@@_internal_a_tl {#1} + \@@_peek_analysis_collect_loop: + } +\cs_new_protected:Npn \@@_peek_analysis_collect_loop: + { + \tex_futurelet:D \l_@@_analysis_token + \@@_peek_analysis_collect_test: + } +\cs_new_protected:Npn \@@_peek_analysis_collect_test: + { + \if_meaning:w \l_@@_analysis_token \l_@@_analysis_next_token + \exp_after:wN \if_meaning:w \cs:w \l_@@_internal_a_tl \cs_end: \l_peek_token + \@@_peek_analysis_collect_end:NNN + \fi: + \fi: + \@@_peek_analysis_collect:w + } +% \end{macrocode} +% End by calling the user code with suitable arguments (here |#1|, +% |#2| are \cs{fi:}), which closes the group begun early on. +% \begin{macrocode} +\cs_new_protected:Npn \@@_peek_analysis_collect_end:NNN #1#2#3 + { + #1 #2 + \tl_put_right:Nx \l_@@_peek_code_tl + { + { \exp_not:N \exp_not:n { \exp_not:c { \l_@@_internal_a_tl } } } + { -1 } + 0 + } + \l_@@_peek_code_tl + } +% \end{macrocode} +% \end{macro} +% % \subsection{Messages} % % \begin{variable}{\c_@@_analysis_show_etc_str} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx index b942e97703c..f88abe05c7a 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx index 676c55747ae..6105f3aab05 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % @@ -754,14 +754,42 @@ % this includes primitive-like commands defined using |{token.set_lua}|. % \end{function} % +% \begin{function}[added = 2020-12-03, EXP, noTF] +% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn} +% \begin{syntax} +% \cs{token_case_meaning:NnTF} \meta{test token} \\ +% ~~"{" \\ +% ~~~~\meta{token case_1} \Arg{code case_1} \\ +% ~~~~\meta{token case_2} \Arg{code case_2} \\ +% ~~~~\ldots \\ +% ~~~~\meta{token case_n} \Arg{code case_n} \\ +% ~~"}" \\ +% ~~\Arg{true code} +% ~~\Arg{false code} +% \end{syntax} +% This function compares the \meta{test token} in turn with each of +% the \meta{token cases}. If the two are equal (as described for +% \cs{token_if_eq_catcode:NNTF}, \cs{token_if_eq_charcode:NNTF} and +% \cs{token_if_eq_meaning:NNTF}, respectively) then the associated +% \meta{code} is left in the input stream and other cases are +% discarded. If any of the cases are matched, the \meta{true code} is +% also inserted into the input stream (after the code for the +% appropriate case), while if none match then the \meta{false code} is +% inserted. The functions \cs{token_case_catcode:Nn}, +% \cs{token_case_charcode:Nn}, and \cs{token_case_meaning:Nn}, which +% do nothing if there is no match, are also available. +% \end{function} +% % \section{Peeking ahead at the next token} % % There is often a need to look ahead at the next token in the input % stream while leaving it in place. This is handled using the -% \enquote{peek} functions. The generic \cs{peek_after:Nw} is -% provided along with a family of predefined tests for common cases. -% As peeking ahead does \emph{not} skip spaces the predefined tests -% include both a space-respecting and space-skipping version. +% \enquote{peek} functions. The generic \cs{peek_after:Nw} is provided +% along with a family of predefined tests for common cases. As peeking +% ahead does \emph{not} skip spaces the predefined tests include both a +% space-respecting and space-skipping version. In addition, using +% \cs{peek_analysis_map_inline:n}, one can map through the following +% tokens in the input stream and repeatedly perform some tests. % % \begin{function}{\peek_after:Nw} % \begin{syntax} @@ -980,6 +1008,128 @@ % (as appropriate to the result of the test). % \end{function} % +% \begin{function}[added = 2020-12-03]{\peek_analysis_map_inline:n} +% \begin{syntax} +% \cs{peek_analysis_map_inline:n} \Arg{inline function} +% \end{syntax} +% Repeatedly removes one \meta{token} from the input stream and +% applies the \meta{inline function} to it, until +% \cs{peek_analysis_map_break:} is called. The \meta{inline function} +% receives three arguments for each \meta{token} in the input stream: +% \begin{itemize} +% \item \meta{tokens}, which both \texttt{o}-expand and +% \texttt{x}-expand to the \meta{token}. The detailed form of +% \meta{tokens} may change in later releases. +% \item \meta{char code}, a decimal representation of the character +% code of the \meta{token}, $-1$ if it is a control sequence. +% \item \meta{catcode}, a capital hexadecimal digit which denotes the +% category code of the \meta{token} (0:~control sequence, +% 1:~begin-group, 2:~end-group, 3:~math shift, 4:~alignment tab, +% 6:~parameter, 7:~superscript, 8:~subscript, A:~space, B:~letter, +% C:~other, D:~active). This can be converted to an integer by +% writing |"|\meta{catcode}. +% \end{itemize} +% These arguments are the same as for \cs{tl_analysis_map_inline:nn} +% defined in \pkg{l3tl-analysis}. The \meta{char code} and +% \meta{catcode} do not take the meaning of a control sequence or +% active character into account: for instance, upon encountering the +% token \cs{c_group_begin_token} in the input stream, +% \cs{peek_analysis_map_inline:n} calls the \meta{inline function} +% with |#1| being \cs{exp_not:n} |{| \cs{c_group_begin_token} |}| +% (with the current implementation), +% |#2|~being~$-1$, and +% |#3|~being~$0$, as for any other control sequence. In contrast, +% upon encountering an explicit begin-group token~|{|, % ^^A |}| +% the \meta{inline function} is called with arguments +% \cs{exp_after:wN} |{| \cs{if_false:} |}| \cs{fi:}, $123$ and~$1$. +% +% The mapping is done at the current group level, \emph{i.e.}~any +% local assignments made by the \meta{inline function} remain in +% effect after the loop. Within the code, \cs{l_peek_token} is set +% equal (as a token, not a token list) to the token under +% consideration. +% \end{function} +% +% \begin{function}[added = 2020-12-03] +% {\peek_analysis_map_break:, \peek_analysis_map_break:n} +% \begin{syntax} +% \cs{peek_analysis_map_inline:n} +% |{| \dots{} \cs{peek_analysis_map_break:n} \Arg{code} |}| +% \end{syntax} +% Stops the \cs{peek_analysis_map_inline:n} loop from seeking more +% tokens, and inserts \meta{code} in the input stream (empty for +% \cs{peek_analysis_map_break:}). +% \end{function} +% +% \begin{function}[added = 2020-12-03, TF]{\peek_regex:n, \peek_regex:N} +% \begin{syntax} +% \cs{peek_regex:nTF} \Arg{regex} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{tokens} that follow in the input stream match the +% \meta{regular expression}. Any \meta{tokens} that have been read +% are left in the input stream after the \meta{true code} or +% \meta{false code} (as appropriate to the result of the test). See +% \pkg{l3regex} for documentation of the syntax of regular +% expressions. The \meta{regular expression} is implicitly anchored +% at the start, so for instance \cs{peek_regex:nTF}~|{|~|a|~|}| is +% essentially equivalent to \cs{peek_charcode:NTF}~|a|. +% \begin{texnote} +% Implicit character tokens are correctly considered by +% \cs{peek_regex:nTF} as control sequences, while functions that +% inspect individual tokens (for instance \cs{peek_charcode:NTF}) +% only take into account their meaning. +% \end{texnote} +% \end{function} +% +% \begin{function}[added = 2020-12-03, TF] +% {\peek_regex_remove_once:n, \peek_regex_remove_once:N} +% \begin{syntax} +% \cs{peek_regex_remove_once:nTF} \Arg{regex} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests if the \meta{tokens} that follow in the input stream match the +% \meta{regex}. If the test is true, the \meta{tokens} are removed +% from the input stream and the \meta{true code} is inserted, while if +% the test is false, the \meta{false code} is inserted followed by the +% \meta{tokens} that were originally in the input stream. +% See \pkg{l3regex} for documentation of the syntax of +% regular expressions. The \meta{regular expression} is implicitly +% anchored at the start, so for instance +% \cs{peek_regex_remove_once:nTF}~|{|~|a|~|}| is essentially equivalent to +% \cs{peek_charcode_remove:NTF}~|a|. +% \begin{texnote} +% Implicit character tokens are correctly considered by +% \cs{peek_regex_remove_once:nTF} as control sequences, while functions +% that inspect individual tokens (for instance +% \cs{peek_charcode:NTF}) only take into account their meaning. +% \end{texnote} +% \end{function} +% +% \begin{function}[added = 2020-12-03, noTF] +% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn} +% \begin{syntax} +% \cs{peek_regex_replace_once:nnTF} \Arg{regex} \Arg{replacement} \Arg{true code} \Arg{false code} +% \end{syntax} +% If the \meta{tokens} that follow in the input stream match the +% \meta{regex}, replaces them according to the \meta{replacement} as +% for \cs{regex_replace_once:nnN}, and leaves the result in the input +% stream, after the \meta{true code}. Otherwise, leaves \meta{false +% code} followed by the \meta{tokens} that were originally in the +% input stream, with no modifications. See \pkg{l3regex} for +% documentation of the syntax of regular expressions and of the +% \meta{replacement}: for instance |\0| in the \meta{replacement} is +% replaced by the tokens that were matched in the input stream. The +% \meta{regular expression} is implicitly anchored at the start. In +% contrast to \cs{regex_replace_once:nnN}, no error arises if the +% \meta{replacement} leads to an unbalanced token list: the tokens are +% inserted into the input stream without issue. +% \begin{texnote} +% Implicit character tokens are correctly considered by +% \cs{peek_regex_replace_once:nnTF} as control sequences, while +% functions that inspect individual tokens (for instance +% \cs{peek_charcode:NTF}) only take into account their meaning. +% \end{texnote} +% \end{function} +% % \section{Description of all possible tokens} % \label{sec:l3token:all-tokens} % @@ -1897,9 +2047,10 @@ %<@@=token> % \end{macrocode} % -% \begin{variable}{\s_@@_stop} +% \begin{variable}{\s_@@_mark, \s_@@_stop} % Internal scan marks. % \begin{macrocode} +\scan_new:N \s_@@_mark \scan_new:N \s_@@_stop % \end{macrocode} % \end{variable} @@ -2566,6 +2717,72 @@ end % \end{macro} % \end{macro} % +% \begin{macro}[EXP, noTF] +% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn} +% \begin{macro}[EXP]{\@@_case:NNnTF, \@@_case:NNw, \@@_case_end:nw} +% The aim here is to allow the case statement to be evaluated +% using a known number of expansion steps (two), and without +% needing to use an explicit \enquote{end of recursion} marker. +% That is achieved by using the test input as the final case, +% as this is always true. The trick is then to tidy up +% the output such that the appropriate case code plus either +% the \texttt{true} or \texttt{false} branch code is inserted. +% \begin{macrocode} +\cs_new:Npn \token_case_catcode:Nn #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } { } } +\cs_new:Npn \token_case_catcode:NnT #1#2#3 + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} {#3} { } } +\cs_new:Npn \token_case_catcode:NnF #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } } +\cs_new:Npn \token_case_catcode:NnTF + { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF } +\cs_new:Npn \token_case_charcode:Nn #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } { } } +\cs_new:Npn \token_case_charcode:NnT #1#2#3 + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} {#3} { } } +\cs_new:Npn \token_case_charcode:NnF #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } } +\cs_new:Npn \token_case_charcode:NnTF + { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF } +\cs_new:Npn \token_case_meaning:Nn #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } { } } +\cs_new:Npn \token_case_meaning:NnT #1#2#3 + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} {#3} { } } +\cs_new:Npn \token_case_meaning:NnF #1#2 + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } } +\cs_new:Npn \token_case_meaning:NnTF + { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF } +\cs_new:Npn \@@_case:NNnTF #1#2#3#4#5 + { + \@@_case:NNw #1 #2 #3 #2 { } + \s_@@_mark {#4} + \s_@@_mark {#5} + \s_@@_stop + } +\cs_new:Npn \@@_case:NNw #1#2#3#4 + { + #1 #2 #3 + { \@@_case_end:nw {#4} } + { \@@_case:NNw #1 #2 } + } +% \end{macrocode} +% To tidy up the recursion, there are two outcomes. If there was a hit to +% one of the cases searched for, then |#1| is the code to insert, +% |#2| is the \emph{next} case to check on and |#3| is all of +% the rest of the cases code. That means that |#4| is the \texttt{true} +% branch code, and |#5| tidies up the spare \cs{s_@@_mark} and the +% \texttt{false} branch. On the other hand, if none of the cases matched +% then we arrive here using the \enquote{termination} case of comparing +% the search with itself. That means that |#1| is empty, |#2| is +% the first \cs{s_@@_mark} and so |#4| is the \texttt{false} code (the +% \texttt{true} code is mopped up by |#3|). +% \begin{macrocode} +\cs_new:Npn \@@_case_end:nw #1#2#3 \s_@@_mark #4#5 \s_@@_stop + { \exp_end: #1 #4 } +% \end{macrocode} +% \end{macro} +% \end{macro} +% % \subsection{Peeking ahead at the next token} % % \begin{macrocode} diff --git a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx index 14467d96177..c69fa555ab0 100644 --- a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx +++ b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx @@ -43,7 +43,7 @@ % }^^A % } % -% \date{Released 2020-10-27} +% \date{Released 2020-12-03} % % \maketitle % |