summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3kernel
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2020-12-04 22:23:08 +0000
committerKarl Berry <karl@freefriends.org>2020-12-04 22:23:08 +0000
commit308fffa9446e3b5d28b6e48b89be81575b565c70 (patch)
treeb33f5e1cf009bf510dd974451299d08dc0ffab2a /Master/texmf-dist/source/latex/l3kernel
parentc4c23071af2870bf72f421f1aae5c9544ff3a87d (diff)
l3kernel (4dec20)
git-svn-id: svn://tug.org/texlive/trunk@57066 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3kernel')
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/expl3.dtx30
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3basics.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3box.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3clist.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3debug.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3doc.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3expan.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3file.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3flag.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fp.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3int.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx47
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx27
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3keys.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3msg.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3names.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3prg.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3prop.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3quark.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3regex.dtx1219
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3seq.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3skip.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3sort.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3str.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3sys.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx4
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3text.dtx22
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx444
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3tl.dtx2
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3token.dtx229
-rw-r--r--Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx2
55 files changed, 1610 insertions, 506 deletions
diff --git a/Master/texmf-dist/source/latex/l3kernel/expl3.dtx b/Master/texmf-dist/source/latex/l3kernel/expl3.dtx
index 59b8cad3a78..fc49c0aa90a 100644
--- a/Master/texmf-dist/source/latex/l3kernel/expl3.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/expl3.dtx
@@ -24,7 +24,7 @@
%
%<*driver|generic|package|2ekernel>
%</driver|generic|package|2ekernel>
-\def\ExplFileDate{2020-10-27}%
+\def\ExplFileDate{2020-12-03}%
%<*driver>
\documentclass[full]{l3doc}
\usepackage{graphicx}
@@ -51,7 +51,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -526,9 +526,20 @@
% \end{arg-description}
% There are two other specifiers with more general meanings:
% \begin{arg-description}
-% \item[D] This means: \textbf{Do not use}. This special case is used
-% for \TeX{} primitives. Programmers outside the kernel team should
-% not use these functions!
+% \item[D] Stands for \textbf{Do not use}. This special case is used
+% for \TeX{} primitives. These functions have no standardized
+% syntax, they are engine dependent and their name can change
+% without warning, thus their use is \emph{strongly discouraged} in
+% package code: programmers should instead use the interfaces
+% documented in \href{interface3.pdf}{interface3.pdf}%^^A
+% \footnote{If a primitive offers a functionality not yet in the
+% kernel, programmers and users are encouraged to write to the
+% \texttt{LaTeX-L} mailing list
+% (\url{mailto:LATEX-L@listserv.uni-heidelberg.de}) describing
+% their use-case and intended behaviour, so that a possible
+% interface can be discussed. Temporarily, while an interface is
+% not provided, programmers may use the procedure described in the
+% \href{l3styleguide.pdf}{l3styleguide.pdf}.}.
% \item[w] This means that the argument syntax is \enquote{weird} in that it
% does not follow any standard rule. It is used for functions with
% arguments that take non standard forms: examples are \TeX{}-level
@@ -1072,13 +1083,10 @@
% \begin{macrocode}
%<*2ekernel|generic>
\begingroup
- \catcode`\>=12
- \def\aux#1>{}
- \def\auxi{c__kernel_expl_date_tl}%
- \edef\auxi{\expandafter\aux\meaning\auxi}%
+ \catcode`\_=11
\expandafter
- \ifx\csname\auxi\endcsname\relax
- \global\expandafter\let\csname\auxi\endcsname\ExplFileDate
+ \ifx\csname c__kernel_expl_date_tl\endcsname\relax
+ \global\let\c__kernel_expl_date_tl\ExplFileDate
\fi
\endgroup
%</2ekernel|generic>
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx b/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx
index e7f2c4fd634..025f016eb83 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3basics.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx b/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx
index eba03dca568..2139dfae9fd 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3bootstrap.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3box.dtx b/Master/texmf-dist/source/latex/l3kernel/l3box.dtx
index a866d6324b2..3bd2f1de85a 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3box.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3box.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx b/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx
index 17f7f33891d..f376f0e0a41 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3candidates.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx b/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx
index 6e57d4a1bf0..2918f8cae3d 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3cctab.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx b/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx
index 779062a3cff..f45ff58a92b 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3clist.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx b/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx
index 0c0c3c776c0..f4d6810d9a5 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3coffins.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx b/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx
index 1d88e3d6ff4..fb04d8eee0c 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3color-base.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx b/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx
index 30f0f78b5eb..d05991e37d2 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3debug.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx b/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx
index 6d13ae1b38c..e60766e876f 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3deprecation.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx b/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx
index 25ad20f7ddf..3eff299b538 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3doc.dtx
@@ -79,7 +79,7 @@ and all files in that bundle must be distributed together.
%
% \title{The \cls{l3doc} class}
% \author{\Team}
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
% \maketitle
% \tableofcontents
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx b/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx
index 2e455845db1..7dc8dbe3554 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3docstrip.dtx
@@ -63,7 +63,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx b/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx
index 49fafac2964..544887627c4 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3expan.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3file.dtx b/Master/texmf-dist/source/latex/l3kernel/l3file.dtx
index fc6f7260a0b..132741ccdaa 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3file.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3file.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx b/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx
index 2475241f91c..f2df290cb5a 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3flag.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx
index 918c7b9941b..b629576584e 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-assign.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
% \maketitle
%
% \begin{documentation}
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx
index 60662e35601..ece95374b5d 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-aux.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx
index 4fa89b63556..634c5f6f118 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-basics.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx
index 1b28754a82d..ccfaba64751 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-convert.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx
index b30fd883bd1..77bc35d84de 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-expo.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx
index 2ff5305b4c8..337875215c5 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-extended.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx
index 28ed6bba76e..c2422c1a5e2 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-logic.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx
index 894cf439d97..cc8e6520d37 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-parse.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx
index e8682c57a8f..8ba199e5e46 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-random.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx
index 03de976829f..5187a83a8e2 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-round.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx
index 377596fb595..1bd6a3ad4f6 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-traps.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
% \maketitle
%
% \begin{documentation}
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx
index eefc4c7ccb0..d602e51b6e1 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp-trig.dtx
@@ -40,7 +40,7 @@
% {latex-team@latex-project.org}^^A
% }^^A
% }
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx
index 51f1a30c184..2f2dddd6457 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fp.dtx
@@ -49,7 +49,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx b/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx
index 5d48154ae8c..96a02296b86 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3fparray.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3int.dtx b/Master/texmf-dist/source/latex/l3kernel/l3int.dtx
index f246d8400be..90ae8a493ea 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3int.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3int.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx b/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx
index 7c9d2372d99..fc9f8bf2c43 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3intarray.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -461,6 +461,51 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}[rEXP]{\__kernel_intarray_range_to_clist:Nnn, \@@_range_to_clist:ww}
+% Loop through part of the array.
+% \begin{macrocode}
+\cs_new:Npn \__kernel_intarray_range_to_clist:Nnn #1#2#3
+ {
+ \exp_last_unbraced:Nf \use_none:n
+ {
+ \exp_after:wN \@@_range_to_clist:ww
+ \int_value:w \int_eval:w #2 \exp_after:wN ;
+ \int_value:w \int_eval:w #3 ;
+ #1 \prg_break_point:
+ }
+ }
+\cs_new:Npn \@@_range_to_clist:ww #1 ; #2 ; #3
+ {
+ \if_int_compare:w #1 > #2 \exp_stop_f:
+ \prg_break:n
+ \fi:
+ , \__kernel_intarray_item:Nn #3 {#1}
+ \exp_after:wN \@@_range_to_clist:ww
+ \int_value:w \int_eval:w #1 + \c_one_int ; #2 ; #3
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\__kernel_intarray_gset_range_from_clist:Nnn, \@@_gset_range:Nw}
+% Loop through part of the array.
+% \begin{macrocode}
+\cs_new_protected:Npn \__kernel_intarray_gset_range_from_clist:Nnn #1#2#3
+ {
+ \int_set:Nn \l_@@_loop_int {#2}
+ \@@_gset_range:Nw #1 #3 , , \prg_break_point:
+ }
+\cs_new_protected:Npn \@@_gset_range:Nw #1 #2 ,
+ {
+ \if_catcode:w \scan_stop: \tl_to_str:n {#2} \scan_stop:
+ \prg_break:n
+ \fi:
+ \__kernel_intarray_gset:Nnn #1 \l_@@_loop_int {#2}
+ \int_incr:N \l_@@_loop_int
+ \@@_gset_range:Nw #1
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}{\intarray_show:N, \intarray_show:c, \intarray_log:N, \intarray_log:c}
% Convert the list to a comma list (with spaces after each comma)
% \begin{macrocode}
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx b/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx
index b844c89385a..76778ce5c29 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3kernel-functions.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -207,6 +207,31 @@
% the \cs{intarray_count:N}, lest a low-level \TeX{} error occur.
% \end{function}
%
+% \begin{function}[rEXP, added = 2020-07-12]{\__kernel_intarray_range_to_clist:Nnn}
+% \begin{syntax}
+% \cs{__kernel_intarray_range_to_clist:Nnn} \meta{intarray~var} \Arg{start index} \Arg{end index}
+% \end{syntax}
+% Converts to integer denotations separated by commas the entries of
+% the \meta{intarray} from positions \meta{start index} to \meta{end
+% index} included. The \meta{start index} and \meta{end index} must
+% be suitable for a direct assignment to a \TeX{} count register, must
+% be between $1$ and the \cs{intarray_count:N}, and be suitably
+% ordered. All tokens have category code other.
+% \end{function}
+%
+% \begin{function}[added = 2020-07-12]{\__kernel_intarray_gset_range_from_clist:Nnn}
+% \begin{syntax}
+% \cs{__kernel_intarray_gset_range_from_clist:Nnn} \meta{intarray~var} \Arg{start index} \Arg{integer clist}
+% \end{syntax}
+% Stores the entries of the \meta{clist} as entries of the
+% \meta{intarray~var} starting from the \meta{start index}, upwards.
+% This is done without any bound checking. The \meta{start index} and
+% all entries of the \meta{integer comma list} (which do not undergo
+% space trimming and brace stripping as in normal clist mappings) must
+% be suitable for a direct assignment to a \TeX{} count register. An
+% empty entry may stop the loop.
+% \end{function}
+%
% \begin{function}{\__kernel_ior_open:Nn, \__kernel_ior_open:No}
% \begin{syntax}
% \cs{__kernel_ior_open:Nn} \meta{stream} \Arg{file name}
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx b/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx
index ea9b1f9d04f..495346fa204 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3keys.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx b/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx
index 041c39c3335..f640d04cd16 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3legacy.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx
index 880eb79f9e1..891b7e06547 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3luatex.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx b/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx
index cde386e371f..96bc2ca4476 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3msg.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3names.dtx b/Master/texmf-dist/source/latex/l3kernel/l3names.dtx
index b75c20eb08a..5333720124d 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3names.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3names.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx b/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx
index 3f1280f23e9..b5b1e4246e7 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3prg.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx b/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx
index 9419a5ce6c3..84b41be5a05 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3prop.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx b/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx
index 16a9b0c447c..487393eb6ab 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3quark.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx
index 2086b5c3ec0..5e64d739805 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3regex.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -713,7 +713,7 @@
% |{n,}| quantifier? (I think not.)
% \item Quantifiers for |\u| and assertions.
% \item When matching, keep track of an explicit stack of
-% \texttt{current_state} and \texttt{current_submatches}.
+% \texttt{curr_state} and \texttt{curr_submatches}.
% \item If possible, when a state is reused by the same thread, kill
% other subthreads.
% \item Use an array rather than \cs[no-index]{l__regex_balance_tl}
@@ -845,12 +845,14 @@
% names).
% \begin{itemize}
% \item \emph{Group}: index of the capturing group, $-1$ for
-% non-capturing groups.
+% non-capturing groups. ^^A start/end index?
% \item \emph{Position}: each token in the query is labelled by an
% integer \meta{position}, with $\texttt{min_pos} - 1 \leq
% \meta{position} \leq \texttt{max_pos}$. The lowest and highest
-% positions correspond to imaginary begin and end markers (with
-% inaccessible category code and character code).
+% positions $\texttt{min_pos} - 1$ and $\texttt{max_pos}$
+% correspond to imaginary begin and end markers (with
+% non-existent category code and character code).
+% $\texttt{max_pos}$ is only set quite late in the processing.
% \item \emph{Query}: the token list to which we apply the regular
% expression.
% \item \emph{State}: each state of the \textsc{nfa} is labelled by an
@@ -865,8 +867,8 @@
% unique id for all the steps of the matching algorithm.
% \end{itemize}
%
-% We use \pkg{l3intarray} to manipulate arrays of integers (stored into
-% some dimension registers in scaled points). We also abuse \TeX{}'s
+% We use \pkg{l3intarray} to manipulate arrays of integers.
+% We also abuse \TeX{}'s
% \tn{toks} registers, by accessing them directly by number rather than
% tying them to control sequence using the \tn{newtoks} allocation
% functions. Specifically, these arrays and \tn{toks} are used as
@@ -876,21 +878,14 @@
% \begin{itemize}
% \item \cs{g_@@_state_active_intarray} holds the last \meta{step} in
% which each \meta{state} was active.
-% \item \cs{g_@@_thread_state_intarray} maps each \meta{thread} (with
-% $\texttt{min_active} \leq \meta{thread} < \texttt{max_active}$) to
-% the \meta{state} in which the \meta{thread} currently is. The
+% \item \cs{g_@@_thread_info_intarray} consists of blocks for each
+% \meta{thread} (with $\texttt{min_thread} \leq \meta{thread} <
+% \texttt{max_thread}$). Each block has
+% $1+2\cs{l_@@_capturing_group_int}$ entries: the \meta{state} in
+% which the \meta{thread} currently is, followed by the beginnings
+% of all submatches, and then the ends of all submatches. The
% \meta{threads} are ordered starting from the best to the least
% preferred.
-% \item \tn{toks}\meta{thread} holds the submatch information for the
-% \meta{thread}, as the contents of a property list.
-% \item \cs{g_@@_charcode_intarray} and \cs{g_@@_catcode_intarray} hold the
-% character codes and category codes of tokens at each
-% \meta{position} in the query.
-% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
-% end-group character tokens which appear before that point in the
-% token list.
-% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
-% and \texttt{x}-expand to the \meta{position}-th token in the query.
% \item \cs{g_@@_submatch_prev_intarray}, \cs{g_@@_submatch_begin_intarray}
% and \cs{g_@@_submatch_end_intarray} hold, for each submatch (as would
% be extracted by \cs{regex_extract_all:nnN}), the place where the
@@ -901,6 +896,14 @@
% block corresponding to one match with all its submatches stored in
% consecutive entries.
% \end{itemize}
+% When actually building the result,
+% \begin{itemize}
+% \item \tn{toks}\meta{position} holds \meta{tokens} which \texttt{o}-
+% and \texttt{x}-expand to the \meta{position}-th token in the query.
+% \item \cs{g_@@_balance_intarray} holds the balance of begin-group and
+% end-group character tokens which appear before that point in the
+% token list.
+% \end{itemize}
%
% The code is structured as follows. Variables are introduced in the
% relevant section. First we present some generic helper functions. Then
@@ -944,7 +947,7 @@
{ \@@_toks_set:Nn #1 { } }
\cs_new_eq:NN \@@_toks_set:Nn \tex_toks:D
\cs_new_protected:Npn \@@_toks_set:No #1
- { \@@_toks_set:Nn #1 \exp_after:wN }
+ { \tex_toks:D #1 \exp_after:wN }
% \end{macrocode}
% \end{macro}
%
@@ -975,13 +978,13 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_toks_put_left:Nx #1#2
{
- \cs_set:Npx \@@_tmp:w { #2 }
+ \cs_set_nopar:Npx \@@_tmp:w { #2 }
\tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
{ \exp_after:wN \@@_tmp:w \tex_the:D \tex_toks:D #1 }
}
\cs_new_protected:Npn \@@_toks_put_right:Nx #1#2
{
- \cs_set:Npx \@@_tmp:w {#2}
+ \cs_set_nopar:Npx \@@_tmp:w {#2}
\tex_toks:D #1 \exp_after:wN
{ \tex_the:D \tex_toks:D \exp_after:wN #1 \@@_tmp:w }
}
@@ -1000,11 +1003,38 @@
\cs_new:Npn \@@_curr_cs_to_str:
{
\exp_after:wN \exp_after:wN \exp_after:wN \cs_to_str:N
- \tex_the:D \tex_toks:D \l_@@_curr_pos_int
+ \l_@@_curr_token_tl
}
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_intarray_item:NnF, \@@_intarray_item_aux:nNF}
+% Item of intarray, with a default value.
+% \begin{macrocode}
+\cs_new:Npn \@@_intarray_item:NnF #1#2
+ { \exp_args:Nf \@@_intarray_item_aux:nNF { \int_eval:n {#2} } #1 }
+\cs_new:Npn \@@_intarray_item_aux:nNF #1#2
+ {
+ \if_int_compare:w #1 > \c_zero_int
+ \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \use_ii:nn
+ \fi:
+ { \__kernel_intarray_item:Nn #2 {#1} }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_maplike_break:}
+% Analogous to \cs{tl_map_break:}, this correctly exits
+% \cs{tl_map_inline:nn} and similar constructions and jumps to the
+% matching \cs{prg_break_point:Nn} \cs{@@_maplike_break:} |{| |}|.
+% \begin{macrocode}
+\cs_new:Npn \@@_maplike_break:
+ { \prg_map_break:Nn \@@_maplike_break: { } }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Constants and variables}
%
% \begin{macro}{\@@_tmp:w}
@@ -1056,19 +1086,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_@@_charcode_intarray, \g_@@_catcode_intarray, \g_@@_balance_intarray}
-% The first thing we do when matching is to go once through the query
-% token list and store the information for each token into
-% \cs{g_@@_charcode_intarray}, \cs{g_@@_catcode_intarray} and \tn{toks}
-% registers. We also store the balance of begin-group/end-group
-% characters into \cs{g_@@_balance_intarray}.
-% \begin{macrocode}
-\intarray_new:Nn \g_@@_charcode_intarray { 65536 }
-\intarray_new:Nn \g_@@_catcode_intarray { 65536 }
-\intarray_new:Nn \g_@@_balance_intarray { 65536 }
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}{\l_@@_balance_int}
% During this phase, \cs{l_@@_balance_int} counts the balance of
% begin-group and end-group character tokens which appear before a
@@ -1079,15 +1096,6 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_cs_name_tl}
-% This variable is used in \cs{@@_item_cs:n} to store the csname of
-% the currently-tested token when the regex contains a sub-regex for
-% testing csnames.
-% \begin{macrocode}
-\tl_new:N \l_@@_cs_name_tl
-% \end{macrocode}
-% \end{variable}
-%
% \subsubsection{Testing characters}
%
% \begin{macro}{\c_@@_ascii_min_int, \c_@@_ascii_max_control_int, \c_@@_ascii_max_int}
@@ -1201,7 +1209,7 @@
% \begin{macro}
% {\@@_item_caseless_equal:n, \@@_item_caseless_range:nn}
% For caseless matching, we perform the test both on the
-% \texttt{current_char} and on the \texttt{case_changed_char}. Before
+% \texttt{curr_char} and on the \texttt{case_changed_char}. Before
% doing the second set of tests, we make sure that
% \texttt{case_changed_char} has been computed.
% \begin{macrocode}
@@ -1308,7 +1316,8 @@
%
% \begin{macro}{\@@_item_exact:nn, \@@_item_exact_cs:n}
% This matches an exact \meta{category}-\meta{character code} pair, or
-% an exact control sequence, more precisely one of several possible control sequences.
+% an exact control sequence, more precisely one of several possible
+% control sequences, separated by \cs{scan_stop:}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_item_exact:nn #1#2
{
@@ -1337,24 +1346,19 @@
% Match a control sequence (the argument is a compiled regex).
% First test the catcode of the current token to be zero.
% Then perform the matching test, and break if the csname
-% indeed matches. The three \cs{exp_after:wN} expand the contents
-% of the \tn{toks}\meta{current position} (of the form \cs{exp_not:n}
-% \Arg{control sequence}) to \meta{control sequence}.
-% We store the cs name before building states for the cs, as those
-% states may overlap with toks registers storing the user's input.
+% indeed matches.
% \begin{macrocode}
\cs_new_protected:Npn \@@_item_cs:n #1
{
\int_compare:nNnT \l_@@_curr_catcode_int = 0
{
\group_begin:
- \__kernel_tl_set:Nx \l_@@_cs_name_tl { \@@_curr_cs_to_str: }
\@@_single_match:
\@@_disable_submatches:
\@@_build_for_cs:n {#1}
\bool_set_eq:NN \l_@@_saved_success_bool
\g_@@_success_bool
- \exp_args:NV \@@_match_cs:n \l_@@_cs_name_tl
+ \exp_args:Nx \@@_match_cs:n { \@@_curr_cs_to_str: }
\if_meaning:w \c_true_bool \g_@@_success_bool
\group_insert_after:N \@@_break_true:w
\fi:
@@ -1822,7 +1826,7 @@
% \item \cs{@@_command_K:}
% \item \cs{@@_assertion:Nn} \meta{boolean} \Arg{assertion test},
% where the \meta{assertion test} is \cs{@@_b_test:} or
-% |{|\cs{@@_anchor:N} \meta{integer}|}|
+% \cs{@@_Z_test:} or \cs{@@_A_test:} or \cs{@@_G_test:}
% \end{itemize}
% Tests can be the following:
% \begin{itemize}
@@ -2663,73 +2667,56 @@
%
% \subsubsection{Anchoring and simple assertions}
%
-% \begin{macro}{\@@_compile_anchor:NF}
+% \begin{macro}{\@@_compile_anchor_letter:NNN}
+% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:, \@@_compile_/Z:, \@@_compile_/z:, \@@_compile_/b:, \@@_compile_/B:}
% \begin{macro}+\@@_compile_^:+
-% \begin{macro}{\@@_compile_/A:, \@@_compile_/G:}
% \begin{macro}+\@@_compile_$:+
-% \begin{macro}{\@@_compile_/Z:, \@@_compile_/z:}
-% In modes where assertions are allowed, anchor to the start of the
-% query, the start of the match, or the end of the query, depending on
-% the integer |#1|. In other modes, |#2| treats the character as raw,
-% with an error for escaped letters (|$| is valid in a class, but |\A|
-% is definitely a mistake on the user's part).
+% In modes where assertions are forbidden, anchors such as |\A|
+% produce an error (|\A|~is invalid in classes); otherwise they add an
+% \cs{@@_assertion:Nn} test as appropriate (the only negative
+% assertion is~|\B|). The test functions are defined later. The
+% implementation for
+% |$| and |^| is only different from |\A| etc because these are valid
+% in a class.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_compile_anchor:NF #1#2
+\cs_new_protected:Npn \@@_compile_anchor_letter:NNN #1#2#3
{
- \@@_if_in_class_or_catcode:TF {#2}
+ \@@_if_in_class_or_catcode:TF { \@@_compile_raw_error:N #1 }
{
\tl_build_put_right:Nn \l_@@_build_tl
- { \@@_assertion:Nn \c_true_bool { \@@_anchor:N #1 } }
+ { \@@_assertion:Nn #2 {#3} }
}
}
-\cs_set_protected:Npn \@@_tmp:w #1#2
- {
- \cs_new_protected:cpn { @@_compile_/#1: }
- { \@@_compile_anchor:NF #2 { \@@_compile_raw_error:N #1 } }
- }
-\@@_tmp:w A \l_@@_min_pos_int
-\@@_tmp:w G \l_@@_start_pos_int
-\@@_tmp:w Z \l_@@_max_pos_int
-\@@_tmp:w z \l_@@_max_pos_int
+\cs_new_protected:cpn { @@_compile_/A: }
+ { \@@_compile_anchor_letter:NNN A \c_true_bool \@@_A_test: }
+\cs_new_protected:cpn { @@_compile_/G: }
+ { \@@_compile_anchor_letter:NNN G \c_true_bool \@@_G_test: }
+\cs_new_protected:cpn { @@_compile_/Z: }
+ { \@@_compile_anchor_letter:NNN Z \c_true_bool \@@_Z_test: }
+\cs_new_protected:cpn { @@_compile_/z: }
+ { \@@_compile_anchor_letter:NNN z \c_true_bool \@@_Z_test: }
+\cs_new_protected:cpn { @@_compile_/b: }
+ { \@@_compile_anchor_letter:NNN b \c_true_bool \@@_b_test: }
+\cs_new_protected:cpn { @@_compile_/B: }
+ { \@@_compile_anchor_letter:NNN B \c_false_bool \@@_b_test: }
\cs_set_protected:Npn \@@_tmp:w #1#2
{
\cs_new_protected:cpn { @@_compile_#1: }
- { \@@_compile_anchor:NF #2 { \@@_compile_raw:N #1 } }
+ {
+ \@@_if_in_class_or_catcode:TF { \@@_compile_raw:N #1 }
+ {
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \@@_assertion:Nn \c_true_bool {#2} }
+ }
+ }
}
-\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } \l_@@_min_pos_int
-\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } \l_@@_max_pos_int
+\exp_args:Nx \@@_tmp:w { \iow_char:N \^ } { \@@_A_test: }
+\exp_args:Nx \@@_tmp:w { \iow_char:N \$ } { \@@_Z_test: }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-%
-% \begin{macro}{\@@_compile_/b:, \@@_compile_/B:}
-% Contrarily to |^| and |$|, which could be implemented without really
-% knowing what precedes in the token list, this requires more
-% information, namely, the knowledge of the last character code.
-% \begin{macrocode}
-\cs_new_protected:cpn { @@_compile_/b: }
- {
- \@@_if_in_class_or_catcode:TF
- { \@@_compile_raw_error:N b }
- {
- \tl_build_put_right:Nn \l_@@_build_tl
- { \@@_assertion:Nn \c_true_bool { \@@_b_test: } }
- }
- }
-\cs_new_protected:cpn { @@_compile_/B: }
- {
- \@@_if_in_class_or_catcode:TF
- { \@@_compile_raw_error:N B }
- {
- \tl_build_put_right:Nn \l_@@_build_tl
- { \@@_assertion:Nn \c_false_bool { \@@_b_test: } }
- }
- }
-% \end{macrocode}
-% \end{macro}
%
% \subsubsection{Character classes}
%
@@ -3512,7 +3499,9 @@
{ \bool_if:NF ##1 { negative~ } assertion:~##2 }
}
\cs_set:Npn \@@_b_test: { word~boundary }
- \cs_set_eq:NN \@@_anchor:N \@@_show_anchor_to_str:N
+ \cs_set:Npn \@@_Z_test: { anchor~at~end~(\iow_char:N\\Z) }
+ \cs_set:Npn \@@_A_test: { anchor~at~start~(\iow_char:N\\A) }
+ \cs_set:Npn \@@_G_test: { anchor~at~start~of~match~(\iow_char:N\\G) }
\cs_set_protected:Npn \@@_item_caseful_equal:n ##1
{ \@@_show_one:n { char~code~\int_eval:n{##1} } }
\cs_set_protected:Npn \@@_item_caseful_range:nn ##1##2
@@ -3655,24 +3644,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[rEXP]{\@@_show_anchor_to_str:N}
-% The argument is an integer telling us where the anchor is. We
-% convert that to the relevant info.
-% \begin{macrocode}
-\cs_new:Npn \@@_show_anchor_to_str:N #1
- {
- anchor~at~
- \str_case:nnF { #1 }
- {
- { \l_@@_min_pos_int } { start~(\iow_char:N\\A) }
- { \l_@@_start_pos_int } { start~of~match~(\iow_char:N\\G) }
- { \l_@@_max_pos_int } { end~(\iow_char:N\\Z) }
- }
- { <error:~'#1'~not~recognized> }
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}{\@@_show_item_catcode:NnT}
% Produce a sequence of categories which the catcode bitmap |#2|
% contains, and show it, indenting the tests on which this catcode
@@ -3757,8 +3728,9 @@
% Each state of the \textsc{nfa} is stored in a \tn{toks}. The
% operations which can appear in the \tn{toks} are
% \begin{itemize}
-% \item \cs{@@_action_start_wildcard:} inserted at the start
-% of the regular expression to make it unanchored.
+% \item \cs{@@_action_start_wildcard:N} \meta{boolean} inserted at the
+% start of the regular expression, where a \texttt{true}
+% \meta{boolean} makes it unanchored.
% \item \cs{@@_action_success:} marks the exit state of the
% \textsc{nfa}.
% \item \cs{@@_action_cost:n} \Arg{shift} is a transition from the
@@ -3772,10 +3744,11 @@
% how they detect and avoid infinite loops. For now, we just need to
% know that the \texttt{group} variant must be used for transitions
% back to the start of a group.
-% \item \cs{@@_action_submatch:n} \Arg{key} where the \meta{key} is
-% a group number followed by |<| or |>| for the beginning or end of
-% group. This causes the current position in the query to be stored
-% as the \meta{key} submatch boundary.
+% \item \cs{@@_action_submatch:nN} \Arg{group} \meta{key} where the
+% \meta{key} is |<| or |>| for the beginning or end of group
+% numbered \meta{group}. This causes the current position in the
+% query to be stored as the \meta{key} submatch boundary.
+% \item One of these actions, within a conditional.
% \end{itemize}
%
% We strive to preserve the following properties while building.
@@ -3793,20 +3766,26 @@
% corresponding end-points of nested groups.
% \end{itemize}
%
-% \begin{macro}{\@@_build:n, \@@_build:N}
+% \begin{macro}{\@@_build:n, \@@_build_aux:Nn, \@@_build:N, \@@_build_aux:NN}
% The \texttt{n}-type function first compiles its argument. Reset some
% variables. Allocate two states, and put a wildcard in state $0$
% (transitions to state $1$ and $0$ state). Then build the regex
% within a (capturing) group numbered $0$ (current
% value of \texttt{capturing_group}). Finally, if the match reaches the
-% last state, it is successful.
+% last state, it is successful. A \texttt{false} boolean for argument
+% |#1| for the auxiliaries will suppress the wildcard and make the
+% match anchored: used for \cs{peek_regex:nTF} and similar.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_build:n #1
+\cs_new_protected:Npn \@@_build:n
+ { \@@_build_aux:Nn \c_true_bool }
+\cs_new_protected:Npn \@@_build:N
+ { \@@_build_aux:NN \c_true_bool }
+\cs_new_protected:Npn \@@_build_aux:Nn #1#2
{
- \@@_compile:n {#1}
- \@@_build:N \l_@@_internal_regex
+ \@@_compile:n {#2}
+ \@@_build_aux:NN #1 \l_@@_internal_regex
}
-\cs_new_protected:Npn \@@_build:N #1
+\cs_new_protected:Npn \@@_build_aux:NN #1#2
{
\@@_standard_escapechar:
\int_zero:N \l_@@_capturing_group_int
@@ -3814,8 +3793,8 @@
\@@_build_new_state:
\@@_build_new_state:
\@@_toks_put_right:Nn \l_@@_left_state_int
- { \@@_action_start_wildcard: }
- \@@_group:nnnN {#1} { 1 } { 0 } \c_false_bool
+ { \@@_action_start_wildcard:N #1 }
+ \@@_group:nnnN {#2} { 1 } { 0 } \c_false_bool
\@@_toks_put_right:Nn \l_@@_right_state_int
{ \@@_action_success: }
}
@@ -3826,19 +3805,14 @@
% The matching code relies on some global intarray variables, but only
% uses a range of their entries. Specifically,
% \begin{itemize}
-% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int}
-% to $\cs{l_@@_max_state_int}-1$;
-% \item \cs{g_@@_thread_state_intarray} from \cs{l_@@_min_active_int}
-% to $\cs{l_@@_max_active_int}-1$.
+% \item \cs{g_@@_state_active_intarray} from \cs{l_@@_min_state_int}
+% to $\cs{l_@@_max_state_int}-1$;
% \end{itemize}
-% In fact, some data is stored in \tn{toks} registers (local) in the
-% same ranges so these ranges mustn't overlap. This is done by
-% setting \cs{l_@@_min_active_int} to \cs{l_@@_max_state_int} after
-% building the \textsc{nfa}. Here, in this nested call to the
-% matching code, we need the new versions of these ranges to involve
+% Here, in this nested call to the
+% matching code, we need the new versions of this range to involve
% completely new entries of the intarray variables, so we begin by
% setting (the new) \cs{l_@@_min_state_int} to (the old)
-% \cs{l_@@_max_active_int} to use higher entries.
+% \cs{l_@@_max_state_int} to use higher entries.
%
% When using a regex to match a cs, we don't insert a wildcard, we
% anchor at the end, and since we ignore submatches, there is no need
@@ -3848,8 +3822,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_build_for_cs:n #1
{
- \int_set_eq:NN \l_@@_min_state_int \l_@@_max_active_int
- \int_set_eq:NN \l_@@_max_state_int \l_@@_min_state_int
+ \int_set_eq:NN \l_@@_min_state_int \l_@@_max_state_int
\@@_build_new_state:
\@@_build_new_state:
\@@_push_lr_states:
@@ -3857,7 +3830,7 @@
\@@_pop_lr_states:
\@@_toks_put_right:Nn \l_@@_right_state_int
{
- \if_int_compare:w \l_@@_curr_pos_int = \l_@@_max_pos_int
+ \if_int_compare:w -2 = \l_@@_curr_char_int
\exp_after:wN \@@_action_success:
\fi:
}
@@ -4195,8 +4168,8 @@
\cs_new_protected:Npn \@@_group_submatches:nNN #1#2#3
{
\if_int_compare:w #1 > - 1 \exp_stop_f:
- \@@_toks_put_left:Nx #2 { \@@_action_submatch:n { #1 < } }
- \@@_toks_put_left:Nx #3 { \@@_action_submatch:n { #1 > } }
+ \@@_toks_put_left:Nx #2 { \@@_action_submatch:nN {#1} < }
+ \@@_toks_put_left:Nx #3 { \@@_action_submatch:nN {#1} > }
\fi:
}
% \end{macrocode}
@@ -4345,16 +4318,14 @@
%
% \subsubsection{Others}
%
-% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_anchor:N}
+% \begin{macro}{\@@_assertion:Nn, \@@_b_test:, \@@_A_test:, \@@_G_test:, \@@_Z_test:}
% Usage: \cs{@@_assertion:Nn} \meta{boolean} \Arg{test}, where the
% \meta{test} is either of the two other functions. Add a free
% transition to a new state, conditionally to the assertion test. The
% \cs{@@_b_test:} test is used by the |\b| and |\B| escape: check
% if the last character was a word character or not, and do the same
% to the current character. The boundary-markers of the string are
-% non-word characters for this purpose. Anchors at the start or end
-% of match use \cs{@@_anchor:N}, with a position controlled by the
-% integer |#1|.
+% non-word characters for this purpose.
% \begin{macrocode}
\cs_new_protected:Npn \@@_assertion:Nn #1#2
{
@@ -4374,12 +4345,6 @@
\bool_if:NT #1 { { } }
}
}
-\cs_new_protected:Npn \@@_anchor:N #1
- {
- \if_int_compare:w #1 = \l_@@_curr_pos_int
- \exp_after:wN \@@_break_true:w
- \fi:
- }
\cs_new_protected:Npn \@@_b_test:
{
\group_begin:
@@ -4389,6 +4354,24 @@
{ \group_end: \@@_item_reverse:n \@@_prop_w: }
{ \group_end: \@@_prop_w: }
}
+\cs_new_protected:Npn \@@_Z_test:
+ {
+ \if_int_compare:w -2 = \l_@@_curr_char_int
+ \exp_after:wN \@@_break_true:w
+ \fi:
+ }
+\cs_new_protected:Npn \@@_A_test:
+ {
+ \if_int_compare:w -2 = \l_@@_last_char_int
+ \exp_after:wN \@@_break_true:w
+ \fi:
+ }
+\cs_new_protected:Npn \@@_G_test:
+ {
+ \if_int_compare:w \l_@@_curr_pos_int = \l_@@_start_pos_int
+ \exp_after:wN \@@_break_true:w
+ \fi:
+ }
% \end{macrocode}
% \end{macro}
%
@@ -4401,7 +4384,7 @@
\@@_build_new_state:
\@@_toks_put_right:Nx \l_@@_left_state_int
{
- \@@_action_submatch:n { 0< }
+ \@@_action_submatch:nN { 0 } <
\bool_set_true:N \l_@@_fresh_thread_bool
\@@_action_free:n
{
@@ -4423,7 +4406,8 @@
% transitions, the instruction at the new state of the \textsc{nfa} is
% performed immediately. When a transition consumes a character, the
% new state is appended to a list of \enquote{active states}, stored in
-% \cs{g_@@_thread_state_intarray}: this thread is made active again when the next
+% \cs{g_@@_thread_info_intarray} (together with submatch information):
+% this thread is made active again when the next
% token is read from the query. At every step (for each token in the
% query), we unpack that list of active states and the corresponding
% submatch props, and empty those.
@@ -4467,11 +4451,10 @@
% }
% The tokens in the query are indexed from \texttt{min_pos} for the
% first to $\texttt{max_pos}-1$ for the last, and their information is
-% stored in several arrays and \tn{toks} registers with those numbers. We
-% don't start from $0$ because the \tn{toks} registers with low
-% numbers are used to hold the states of the \textsc{nfa}. We match
+% stored in several arrays and \tn{toks} registers with those numbers.
+% We match
% without backtracking, keeping all threads in lockstep at the
-% \texttt{current_pos} in the query. The starting point of the current
+% \texttt{curr_pos} in the query. The starting point of the current
% match attempt is \texttt{start_pos}, and \texttt{success_pos},
% updated whenever a thread succeeds, is used as the next starting
% position.
@@ -4488,20 +4471,26 @@
% {
% \l_@@_curr_char_int,
% \l_@@_curr_catcode_int,
+% \l_@@_curr_token_tl,
% \l_@@_last_char_int,
+% \l_@@_last_char_success_int,
% \l_@@_case_changed_char_int
% }
% The character and category codes of the token at the current
-% position; the character code of the token at the previous position;
+% position and a token list expanding to that token; the character
+% code of the token at the previous position;
+% the character code of the token just before a successful match;
% and the character code of the result of changing the case of the
% current token (|A-Z|$\leftrightarrow$|a-z|). This last integer is
% only computed when necessary, and is otherwise \cs{c_max_int}. The
-% \texttt{current_char} variable is also used in various other phases
+% \texttt{curr_char} variable is also used in various other phases
% to hold a character code.
% \begin{macrocode}
\int_new:N \l_@@_curr_char_int
\int_new:N \l_@@_curr_catcode_int
+\tl_new:N \l_@@_curr_token_tl
\int_new:N \l_@@_last_char_int
+\int_new:N \l_@@_last_char_success_int
\int_new:N \l_@@_case_changed_char_int
% \end{macrocode}
% \end{variable}
@@ -4517,18 +4506,17 @@
% \end{variable}
%
% \begin{variable}
-% {\l_@@_curr_submatches_prop, \l_@@_success_submatches_prop}
+% {\l_@@_curr_submatches_tl, \l_@@_success_submatches_tl}
% The submatches for the thread which is currently active are stored
-% in the \texttt{current_submatches} property list variable. This
-% property list is stored by \cs{@@_action_cost:n} into the
-% \tn{toks} register for the target state of the transition, to be
-% retrieved when matching at the next position. When a thread
-% succeeds, this property list is copied to
-% \cs{l_@@_success_submatches_prop}: only the last successful thread
+% in the \texttt{curr_submatches} list, which is almost a comma list,
+% but ends with a comma. This list is stored by \cs{@@_store_state:n}
+% into an intarray variable, to be retrieved when matching at the next
+% position. When a thread succeeds, this list is copied to
+% \cs{l_@@_success_submatches_tl}: only the last successful thread
% remains there.
% \begin{macrocode}
-\prop_new:N \l_@@_curr_submatches_prop
-\prop_new:N \l_@@_success_submatches_prop
+\tl_new:N \l_@@_curr_submatches_tl
+\tl_new:N \l_@@_success_submatches_tl
% \end{macrocode}
% \end{variable}
%
@@ -4551,28 +4539,44 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_min_active_int, \l_@@_max_active_int}
+% \begin{variable}{\l_@@_min_thread_int, \l_@@_max_thread_int}
% All the currently active threads are kept in order of precedence in
-% \cs{g_@@_thread_state_intarray}, and the corresponding submatches in the
-% \tn{toks}. For our purposes, those serve as an array, indexed from
-% \texttt{min_active} (inclusive) to \texttt{max_active} (excluded).
-% At the start of every step, the whole array is unpacked, so that the
-% space can immediately be reused, and \texttt{max_active} is reset to
-% \texttt{min_active}, effectively clearing the array.
+% \cs{g_@@_thread_info_intarray} together with the corresponding
+% submatch information. Data in this intarray is organized as blocks
+% from \texttt{min_thread} (included) to \texttt{max_thread}
+% (excluded). At the start of every step, the whole array is
+% unpacked, so that the space can immediately be reused, and
+% \texttt{max_thread} is reset to \texttt{min_thread}, effectively
+% clearing the array.
% \begin{macrocode}
-\int_new:N \l_@@_min_active_int
-\int_new:N \l_@@_max_active_int
+\int_new:N \l_@@_min_thread_int
+\int_new:N \l_@@_max_thread_int
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_state_intarray}
+% \begin{variable}{\g_@@_state_active_intarray, \g_@@_thread_info_intarray}
% \cs{g_@@_state_active_intarray} stores the last \meta{step} in which
-% each \meta{state} was active. \cs{g_@@_thread_state_intarray} stores
+% each \meta{state} was active. \cs{g_@@_thread_info_intarray} stores
% threads to be considered in the next step, more precisely the
% states in which these threads are.
% \begin{macrocode}
\intarray_new:Nn \g_@@_state_active_intarray { 65536 }
-\intarray_new:Nn \g_@@_thread_state_intarray { 65536 }
+\intarray_new:Nn \g_@@_thread_info_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_matched_analysis_tl, \l_@@_curr_analysis_tl}
+% The list \cs{l_@@_curr_analysis_tl} consists of a brace group
+% containing three brace groups corresponding to the current token,
+% with the same syntax as \cs{tl_analysis_map_inline:nn}. The list
+% \cs{l_@@_matched_analysis_tl} (constructed under the
+% \texttt{tl\_build} machinery) has one item for each token that has
+% already been treated so far in a given match attempt: each item
+% consists of three brace groups with the same syntax as
+% \cs{tl_analysis_map_inline:nn}.
+% \begin{macrocode}
+\tl_new:N \l_@@_matched_analysis_tl
+\tl_new:N \l_@@_curr_analysis_tl
% \end{macrocode}
% \end{variable}
%
@@ -4635,8 +4639,7 @@
%
% \begin{macro}{\@@_match:n, \@@_match_cs:n}
% \begin{macro}{\@@_match_init:}
-% First store the query into \tn{toks} registers and arrays (see
-% \cs{@@_query_set:nnn}). Then initialize the variables that should
+% Initialize the variables that should
% be set once for each user function (even for multiple
% matches). Namely, the overall matching is not yet successful; none of
% the states should be marked as visited (\cs{g_@@_state_active_intarray}), and
@@ -4647,38 +4650,26 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_match:n #1
{
- \int_zero:N \l_@@_balance_int
- \int_set:Nn \l_@@_curr_pos_int { 2 * \l_@@_max_state_int }
- \@@_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int
- \tl_analysis_map_inline:nn {#1}
- { \@@_query_set:nnn {##1} {"##3} {##2} }
- \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
- \@@_query_set:nnn { } { -1 } { -2 }
\@@_match_init:
- \@@_match_once:
+ \@@_match_once_init:
+ \tl_analysis_map_inline:nn {#1}
+ { \@@_match_one_token:nnN {##1} {##2} ##3 }
+ \@@_match_one_token:nnN { } { -2 } F
+ \prg_break_point:Nn \@@_maplike_break: { }
}
\cs_new_protected:Npn \@@_match_cs:n #1
{
- \int_zero:N \l_@@_balance_int
- \int_set:Nn \l_@@_curr_pos_int
- {
- \int_max:nn { 2 * \l_@@_max_state_int - \l_@@_min_state_int }
- { \l_@@_max_pos_int }
- + 1
- }
- \@@_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_@@_min_pos_int \l_@@_curr_pos_int
+ \int_set_eq:NN \l_@@_min_thread_int \l_@@_max_thread_int
+ \@@_match_init:
+ \@@_match_once_init:
\str_map_inline:nn {#1}
{
- \@@_query_set:nnn { \exp_not:n {##1} }
- { \tl_if_blank:nTF {##1} { 10 } { 12 } }
- { `##1 }
+ \tl_if_blank:nTF {##1}
+ { \@@_match_one_token:nnN {##1} {`##1} A }
+ { \@@_match_one_token:nnN {##1} {`##1} C }
}
- \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
- \@@_query_set:nnn { } { -1 } { -2 }
- \@@_match_init:
- \@@_match_once:
+ \@@_match_one_token:nnN { } { -2 } F
+ \prg_break_point:Nn \@@_maplike_break: { }
}
\cs_new_protected:Npn \@@_match_init:
{
@@ -4689,11 +4680,13 @@
\__kernel_intarray_gset:Nnn
\g_@@_state_active_intarray {##1} { 1 }
}
- \int_set_eq:NN \l_@@_min_active_int \l_@@_max_state_int
\int_zero:N \l_@@_step_int
+ \int_set:Nn \l_@@_min_pos_int { 2 }
\int_set_eq:NN \l_@@_success_pos_int \l_@@_min_pos_int
- \int_set:Nn \l_@@_min_submatch_int
- { 2 * \l_@@_max_state_int }
+ \int_set:Nn \l_@@_last_char_success_int { -2 }
+ \tl_build_begin:N \l_@@_matched_analysis_tl
+ \tl_clear:N \l_@@_curr_analysis_tl
+ \int_set:Nn \l_@@_min_submatch_int { 1 }
\int_set_eq:NN \l_@@_submatch_int \l_@@_min_submatch_int
\bool_set_false:N \l_@@_empty_success_bool
}
@@ -4701,21 +4694,24 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\@@_match_once:}
-% This function finds one match, then does some action defined by the
-% \texttt{every_match} token list, which may recursively call
-% \cs{@@_match_once:}. First initialize some variables: set the
+% \begin{macro}{\@@_match_once_init:}
+% This function resets various variables used when finding one match.
+% It is called before the loop through characters, and every time we
+% find a match, before searching for another match (this is controlled
+% by the \texttt{every_match} token list).
+%
+% First initialize some variables: set the
% conditional which detects identical empty matches; this match
% attempt starts at the previous \texttt{success_pos}, is not yet
% successful, and has no submatches yet; clear the array of active
% threads, and put the starting state $0$ in it. We are then almost
% ready to read our first token in the query, but we actually start
-% one position earlier than the start, and \texttt{get} that token, to
-% set \texttt{last_char} properly for word
-% boundaries. Then call \cs{@@_match_loop:}, which runs through the
-% query until the end or until a successful match breaks early.
+% one position earlier than the start because
+% \cs{@@_match_one_token:nnN} increments \cs{l_@@_curr_pos_int} and
+% saves \cs{l_@@_curr_char_int} as the \texttt{last_char} so that word
+% boundaries can be correctly identified.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_match_once:
+\cs_new_protected:Npn \@@_match_once_init:
{
\if_meaning:w \c_true_bool \l_@@_empty_success_bool
\cs_set:Npn \@@_if_two_empty_matches:F
@@ -4728,14 +4724,24 @@
\fi:
\int_set_eq:NN \l_@@_start_pos_int \l_@@_success_pos_int
\bool_set_false:N \l_@@_match_success_bool
- \prop_clear:N \l_@@_curr_submatches_prop
- \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int
+ \tl_set:Nx \l_@@_curr_submatches_tl
+ { \prg_replicate:nn { 2 * \l_@@_capturing_group_int } { 0 , } }
+ \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int
\@@_store_state:n { \l_@@_min_state_int }
\int_set:Nn \l_@@_curr_pos_int
{ \l_@@_start_pos_int - 1 }
- \@@_query_get:
- \@@_match_loop:
- \l_@@_every_match_tl
+ \int_set_eq:NN \l_@@_curr_char_int \l_@@_last_char_success_int
+ \tl_build_get:NN \l_@@_matched_analysis_tl \l_@@_internal_a_tl
+ \exp_args:NNf \@@_match_once_init_aux:
+ \tl_map_inline:nn
+ { \exp_after:wN \l_@@_internal_a_tl \l_@@_curr_analysis_tl }
+ { \@@_match_one_token:nnN ##1 }
+ \prg_break_point:Nn \@@_maplike_break: { }
+ }
+\cs_new_protected:Npn \@@_match_once_init_aux:
+ {
+ \tl_build_clear:N \l_@@_matched_analysis_tl
+ \tl_clear:N \l_@@_curr_analysis_tl
}
% \end{macrocode}
% \end{macro}
@@ -4753,110 +4759,80 @@
\bool_gset_eq:NN
\g_@@_success_bool
\l_@@_match_success_bool
+ \@@_maplike_break:
}
}
\cs_new_protected:Npn \@@_multi_match:n #1
{
\tl_set:Nn \l_@@_every_match_tl
{
- \if_meaning:w \c_true_bool \l_@@_match_success_bool
- \bool_gset_true:N \g_@@_success_bool
- #1
- \exp_after:wN \@@_match_once:
+ \if_meaning:w \c_false_bool \l_@@_match_success_bool
+ \exp_after:wN \@@_maplike_break:
\fi:
+ \bool_gset_true:N \g_@@_success_bool
+ #1
+ \@@_match_once_init:
}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_match_loop:}
+% \begin{macro}{\@@_match_one_token:nnN}
% \begin{macro}[rEXP]{\@@_match_one_active:n}
% At each new position, set some variables and get the new character
% and category from the query. Then unpack the array of active
% threads, and clear it by resetting its length
-% (\texttt{max_active}). This results in a sequence of
-% \cs{@@_use_state_and_submatches:nn} \Arg{state} \Arg{prop}, and
+% (\texttt{max_thread}). This results in a sequence of
+% \cs{@@_use_state_and_submatches:w} \meta{state}|,|\meta{submatch-clist}|;| and
% we consider those states one by one in order. As soon as a thread
% succeeds, exit the step, and, if there are threads to consider at the
% next position, and we have not reached the end of the string,
-% repeat the loop. Otherwise, the last thread that succeeded is what
-% \cs{@@_match_once:} matches. We explain the \texttt{fresh_thread}
-% business when describing \cs{@@_action_wildcard:}.
+% repeat the loop. Otherwise, the last thread that succeeded is the
+% match. We explain the \texttt{fresh_thread} business when
+% describing \cs{@@_action_wildcard:}.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_match_loop:
+\cs_new_protected:Npn \@@_match_one_token:nnN #1#2#3
{
\int_add:Nn \l_@@_step_int { 2 }
\int_incr:N \l_@@_curr_pos_int
\int_set_eq:NN \l_@@_last_char_int \l_@@_curr_char_int
\int_set_eq:NN \l_@@_case_changed_char_int \c_max_int
- \@@_query_get:
+ \tl_set:Nn \l_@@_curr_token_tl {#1}
+ \int_set:Nn \l_@@_curr_char_int {#2}
+ \int_set:Nn \l_@@_curr_catcode_int { "#3 }
+ \tl_build_put_right:Nx \l_@@_matched_analysis_tl
+ { \exp_not:o \l_@@_curr_analysis_tl }
+ \tl_set:Nn \l_@@_curr_analysis_tl { { {#1} {#2} #3 } }
\use:x
{
- \int_set_eq:NN \l_@@_max_active_int \l_@@_min_active_int
+ \int_set_eq:NN \l_@@_max_thread_int \l_@@_min_thread_int
\int_step_function:nnN
- { \l_@@_min_active_int }
- { \l_@@_max_active_int - 1 }
+ { \l_@@_min_thread_int }
+ { \l_@@_max_thread_int - 1 }
\@@_match_one_active:n
}
\prg_break_point:
\bool_set_false:N \l_@@_fresh_thread_bool
- \if_int_compare:w \l_@@_max_active_int > \l_@@_min_active_int
- \if_int_compare:w \l_@@_curr_pos_int < \l_@@_max_pos_int
- \exp_after:wN \exp_after:wN \exp_after:wN \@@_match_loop:
+ \if_int_compare:w \l_@@_max_thread_int > \l_@@_min_thread_int
+ \if_int_compare:w -2 < \l_@@_curr_char_int
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_none:n
\fi:
\fi:
+ \l_@@_every_match_tl
}
\cs_new:Npn \@@_match_one_active:n #1
{
- \@@_use_state_and_submatches:nn
- { \__kernel_intarray_item:Nn \g_@@_thread_state_intarray {#1} }
- { \@@_toks_use:w #1 }
+ \@@_use_state_and_submatches:w
+ \__kernel_intarray_range_to_clist:Nnn
+ \g_@@_thread_info_intarray
+ { 1 + #1 * (\l_@@_capturing_group_int * 2 + 1) }
+ { (1 + #1) * (\l_@@_capturing_group_int * 2 + 1) }
+ ;
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\@@_query_set:nnn}
-% The arguments are: tokens that \texttt{o} and \texttt{x} expand to
-% one token of the query, the catcode, and the character code. Store
-% those, and the current brace balance (used later to check for
-% overall brace balance) in a \tn{toks} register and some arrays,
-% then update the \texttt{balance}.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_query_set:nnn #1#2#3
- {
- \__kernel_intarray_gset:Nnn \g_@@_charcode_intarray
- { \l_@@_curr_pos_int } {#3}
- \__kernel_intarray_gset:Nnn \g_@@_catcode_intarray
- { \l_@@_curr_pos_int } {#2}
- \__kernel_intarray_gset:Nnn \g_@@_balance_intarray
- { \l_@@_curr_pos_int } { \l_@@_balance_int }
- \@@_toks_set:Nn \l_@@_curr_pos_int {#1}
- \int_incr:N \l_@@_curr_pos_int
- \if_case:w #2 \exp_stop_f:
- \or: \int_incr:N \l_@@_balance_int
- \or: \int_decr:N \l_@@_balance_int
- \fi:
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}{\@@_query_get:}
-% Extract the current character and category codes at the current
-% position from the appropriate arrays.
-% \begin{macrocode}
-\cs_new_protected:Npn \@@_query_get:
- {
- \l_@@_curr_char_int
- = \__kernel_intarray_item:Nn \g_@@_charcode_intarray
- { \l_@@_curr_pos_int } \scan_stop:
- \l_@@_curr_catcode_int
- = \__kernel_intarray_item:Nn \g_@@_catcode_intarray
- { \l_@@_curr_pos_int } \scan_stop:
- }
-% \end{macrocode}
-% \end{macro}
-%
% \subsubsection{Using states of the \textsc{nfa}}
%
% \begin{macro}{\@@_use_state:}
@@ -4879,20 +4855,20 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_use_state_and_submatches:nn}
+% \begin{macro}{\@@_use_state_and_submatches:w}
% This function is called as one item in the array of active threads
% after that array has been unpacked for a new step. Update the
-% \texttt{current_state} and \texttt{current_submatches} and use the
+% \texttt{curr_state} and \texttt{curr_submatches} and use the
% state if it has not yet been encountered at this step.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_use_state_and_submatches:nn #1 #2
+\cs_new_protected:Npn \@@_use_state_and_submatches:w #1 , #2 ;
{
\int_set:Nn \l_@@_curr_state_int {#1}
\if_int_compare:w
\__kernel_intarray_item:Nn \g_@@_state_active_intarray
{ \l_@@_curr_state_int }
< \l_@@_step_int
- \tl_set:Nn \l_@@_curr_submatches_prop {#2}
+ \tl_set:Nn \l_@@_curr_submatches_tl { #2 , }
\exp_after:wN \@@_use_state:
\fi:
\scan_stop:
@@ -4902,20 +4878,20 @@
%
% \subsubsection{Actions when matching}
%
-% \begin{macro}{\@@_action_start_wildcard:}
+% \begin{macro}{\@@_action_start_wildcard:N}
% For an unanchored match, state $0$ has a free transition to the next
% and a costly one to itself, to repeat at the next position. To catch
% repeated identical empty matches, we need to know if a successful
% thread corresponds to an empty match. The instruction resetting
% \cs{l_@@_fresh_thread_bool} may be skipped by a successful
-% thread, hence we had to add it to \cs{@@_match_loop:} too.
+% thread, hence we had to add it to \cs{@@_match_one_token:nnN} too.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_action_start_wildcard:
+\cs_new_protected:Npn \@@_action_start_wildcard:N #1
{
\bool_set_true:N \l_@@_fresh_thread_bool
\@@_action_free:n {1}
\bool_set_false:N \l_@@_fresh_thread_bool
- \@@_action_cost:n {0}
+ \bool_if:NT #1 { \@@_action_cost:n {0} }
}
% \end{macrocode}
% \end{macro}
@@ -4954,8 +4930,8 @@
}
\int_set:Nn \l_@@_curr_state_int
{ \int_use:N \l_@@_curr_state_int }
- \tl_set:Nn \exp_not:N \l_@@_curr_submatches_prop
- { \exp_not:o \l_@@_curr_submatches_prop }
+ \tl_set:Nn \exp_not:N \l_@@_curr_submatches_tl
+ { \exp_not:o \l_@@_curr_submatches_tl }
}
}
% \end{macrocode}
@@ -4978,21 +4954,26 @@
%
% \begin{macro}{\@@_store_state:n}
% \begin{macro}{\@@_store_submatches:}
-% Put the given state in \cs{g_@@_thread_state_intarray}, and increment
-% the length of the array. Also store the current submatch in the
-% appropriate \tn{toks}.
+% Put the given state and current submatch information in
+% \cs{g_@@_thread_info_intarray}, and increment the length of the
+% array.
% \begin{macrocode}
\cs_new_protected:Npn \@@_store_state:n #1
{
- \@@_store_submatches:
- \__kernel_intarray_gset:Nnn \g_@@_thread_state_intarray
- { \l_@@_max_active_int } {#1}
- \int_incr:N \l_@@_max_active_int
+ \exp_args:No \@@_store_submatches:nn
+ \l_@@_curr_submatches_tl {#1}
+ \int_incr:N \l_@@_max_thread_int
}
-\cs_new_protected:Npn \@@_store_submatches:
+\cs_new_protected:Npn \@@_store_submatches:nn #1#2
{
- \@@_toks_set:No \l_@@_max_active_int
- { \l_@@_curr_submatches_prop }
+ \__kernel_intarray_gset_range_from_clist:Nnn
+ \g_@@_thread_info_intarray
+ {
+ \@@_int_eval:w
+ 1 + \l_@@_max_thread_int *
+ (\l_@@_capturing_group_int * 2 + 1)
+ }
+ { #2 , #1 }
}
% \end{macrocode}
% \end{macro}
@@ -5006,21 +4987,37 @@
% \begin{macrocode}
\cs_new_protected:Npn \@@_disable_submatches:
{
- \cs_set_protected:Npn \@@_store_submatches: { }
- \cs_set_protected:Npn \@@_action_submatch:n ##1 { }
+ \cs_set_protected:Npn \@@_store_submatches:n ##1 { }
+ \cs_set_protected:Npn \@@_action_submatch:nN ##1##2 { }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\@@_action_submatch:n}
+% \begin{macro}{\@@_action_submatch:nN, \@@_action_submatch_aux:w, \@@_action_submatch_auxii:w, \@@_action_submatch_auxiii:w, \@@_action_submatch_auxiv:w}
% Update the current submatches with the information from the current
% position. Maybe a bottleneck.
% \begin{macrocode}
-\cs_new_protected:Npn \@@_action_submatch:n #1
+\cs_new_protected:Npn \@@_action_submatch:nN #1#2
+ {
+ \exp_after:wN \@@_action_submatch_aux:w
+ \l_@@_curr_submatches_tl ; {#1} #2
+ }
+\cs_new_protected:Npn \@@_action_submatch_aux:w #1 ; #2#3
{
- \prop_put:Nno \l_@@_curr_submatches_prop {#1}
- { \int_use:N \l_@@_curr_pos_int }
+ \tl_set:Nx \l_@@_curr_submatches_tl
+ {
+ \prg_replicate:nn
+ { #2 \if_meaning:w > #3 + \l_@@_capturing_group_int \fi: }
+ { \@@_action_submatch_auxii:w }
+ \@@_action_submatch_auxiii:w
+ #1
+ }
}
+\cs_new:Npn \@@_action_submatch_auxii:w
+ #1 \@@_action_submatch_auxiii:w #2 ,
+ { #2 , #1 \@@_action_submatch_auxiii:w }
+\cs_new:Npn \@@_action_submatch_auxiii:w #1 ,
+ { \int_use:N \l_@@_curr_pos_int , }
% \end{macrocode}
% \end{macro}
%
@@ -5042,8 +5039,10 @@
\bool_set_eq:NN \l_@@_empty_success_bool
\l_@@_fresh_thread_bool
\int_set_eq:NN \l_@@_success_pos_int \l_@@_curr_pos_int
- \prop_set_eq:NN \l_@@_success_submatches_prop
- \l_@@_curr_submatches_prop
+ \int_set_eq:NN \l_@@_last_char_success_int \l_@@_last_char_int
+ \tl_build_clear:N \l_@@_matched_analysis_tl
+ \tl_set_eq:NN \l_@@_success_submatches_tl
+ \l_@@_curr_submatches_tl
\prg_break:
}
}
@@ -5134,6 +5133,14 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_replacement_exp_not:V}
+% This is used for the implementation of~|\u|, and it gets redefined
+% for \cs{peek_regex_replace_once:nnTF}.
+% \begin{macrocode}
+\cs_new_eq:NN \@@_replacement_exp_not:V \exp_not:V
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Query and brace balance}
%
% \begin{macro}[rEXP]{\@@_query_range:nn}
@@ -5189,42 +5196,25 @@
% range is the difference between the brace balances at the
% \meta{max~pos} and \meta{min~pos}. These two positions are found in
% the corresponding \enquote{submatch} arrays.
-%^^A todo: understand when these int_compare are needed
% \begin{macrocode}
\cs_new_protected:Npn \@@_submatch_balance:n #1
{
\int_eval:n
- {
- \int_compare:nNnTF
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_end_intarray {#1}
- }
- = 0
- { 0 }
- {
- \__kernel_intarray_item:Nn \g_@@_balance_intarray
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_end_intarray {#1}
- }
- }
- -
- \int_compare:nNnTF
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_begin_intarray {#1}
- }
- = 0
- { 0 }
- {
- \__kernel_intarray_item:Nn \g_@@_balance_intarray
- {
- \__kernel_intarray_item:Nn
- \g_@@_submatch_begin_intarray {#1}
- }
- }
- }
+ {
+ \@@_intarray_item:NnF \g_@@_balance_intarray
+ {
+ \__kernel_intarray_item:Nn
+ \g_@@_submatch_end_intarray {#1}
+ }
+ { 0 }
+ -
+ \@@_intarray_item:NnF \g_@@_balance_intarray
+ {
+ \__kernel_intarray_item:Nn
+ \g_@@_submatch_begin_intarray {#1}
+ }
+ { 0 }
+ }
}
% \end{macrocode}
% \end{macro}
@@ -5305,6 +5295,14 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}{\@@_replacement_put:n}
+% This gets redefined for \cs{peek_regex_replace_once:nnTF}.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_replacement_put:n
+ { \tl_build_put_right:Nn \l_@@_build_tl }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}{\@@_replacement_normal:n}
% Most characters are simply sent to the output by
% \cs{tl_build_put_right:Nn}, unless a particular category code has been
@@ -5314,11 +5312,13 @@
% sequence is non-empty there: it contains an empty entry
% corresponding to the initial value of
% \cs{l_@@_replacement_category_tl}.
+% The argument |#1| can be a space, otherwise it is a single
+% character.
% \begin{macrocode}
\cs_new_protected:Npn \@@_replacement_normal:n #1
{
\tl_if_empty:NTF \l_@@_replacement_category_tl
- { \tl_build_put_right:Nn \l_@@_build_tl {#1} }
+ { \@@_replacement_put:n {#1} }
{ % (
\token_if_eq_charcode:NNTF #1 )
{
@@ -5361,7 +5361,7 @@
%
% \subsubsection{Submatches}
%
-% \begin{macro}{\@@_replacement_put_submatch:n}
+% \begin{macro}{\@@_replacement_put_submatch:n, \@@_replacement_put_submatch_aux:n}
% Insert a submatch in the replacement text. This is dropped if the
% submatch number is larger than the number of capturing groups.
% Unless the submatch appears inside a |\c{...}| or |\u{...}|
@@ -5374,15 +5374,19 @@
\cs_new_protected:Npn \@@_replacement_put_submatch:n #1
{
\if_int_compare:w #1 < \l_@@_capturing_group_int
- \tl_build_put_right:Nn \l_@@_build_tl
- { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } }
- \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f:
- \tl_put_right:Nn \l_@@_balance_tl
- {
- + \@@_submatch_balance:n
- { \exp_not:N \int_eval:n { #1 + ##1 } }
- }
- \fi:
+ \@@_replacement_put_submatch_aux:n {#1}
+ \fi:
+ }
+\cs_new_protected:Npn \@@_replacement_put_submatch_aux:n #1
+ {
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } }
+ \if_int_compare:w \l_@@_replacement_csnames_int = 0 \exp_stop_f:
+ \tl_put_right:Nn \l_@@_balance_tl
+ {
+ + \@@_submatch_balance:n
+ { \exp_not:N \int_eval:n { #1 + ##1 } }
+ }
\fi:
}
% \end{macrocode}
@@ -5488,7 +5492,7 @@
{
\@@_two_if_eq:NNNNTF
#1 #2 \@@_replacement_normal:n \c_left_brace_str
- { \@@_replacement_cu_aux:Nw \exp_not:V }
+ { \@@_replacement_cu_aux:Nw \@@_replacement_exp_not:V }
{ \@@_replacement_error:NNN u #1#2 }
}
% \end{macrocode}
@@ -5579,7 +5583,7 @@
\cs_new_protected:Npn \@@_replacement_char:nNN #1#2#3
{
\tex_lccode:D 0 = `#3 \scan_stop:
- \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {#1} }
+ \tex_lowercase:D { \@@_replacement_put:n {#1} }
}
% \end{macrocode}
% \end{macro}
@@ -5625,7 +5629,7 @@
\cs_new_protected:Npn \@@_replacement_c_C:w #1#2
{
\tl_build_put_right:Nn \l_@@_build_tl
- { \exp_not:N \exp_not:N \exp_not:c {#2} }
+ { \exp_not:N \@@_replacement_exp_not:N \exp_not:c {#2} }
}
% \end{macrocode}
% \end{macro}
@@ -5712,7 +5716,7 @@
\__kernel_msg_error:nn { kernel } { replacement-null-space }
\fi:
\tex_lccode:D `\ = `#2 \scan_stop:
- \tex_lowercase:D { \tl_build_put_right:Nn \l_@@_build_tl {~} }
+ \tex_lowercase:D { \@@_replacement_put:n {~} }
}
% \end{macrocode}
% \end{macro}
@@ -5940,6 +5944,14 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\g_@@_balance_intarray}
+% The first thing we do when matching is to store the balance of
+% begin-group/end-group characters into \cs{g_@@_balance_intarray}.
+% \begin{macrocode}
+\intarray_new:Nn \g_@@_balance_intarray { 65536 }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}{\@@_return:}
% This function triggers either \cs{prg_return_false:} or
% \cs{prg_return_true:} as appropriate to whether a match was found or
@@ -5956,6 +5968,36 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@@_query_set:n, \@@_query_set_aux:nN}
+% To easily extract subsets of the input once we found the positions
+% at which to cut, store the input tokens one by one into successive
+% \tn{toks} registers. Also store the brace balance (used to check
+% for overall brace balance) in an array.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_query_set:n #1
+ {
+ \int_zero:N \l_@@_balance_int
+ \int_zero:N \l_@@_curr_pos_int
+ \@@_query_set_aux:nN { } F
+ \tl_analysis_map_inline:nn {#1}
+ { \@@_query_set_aux:nN {##1} ##3 }
+ \@@_query_set_aux:nN { } F
+ \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
+ }
+\cs_new_protected:Npn \@@_query_set_aux:nN #1#2
+ {
+ \int_incr:N \l_@@_curr_pos_int
+ \@@_toks_set:Nn \l_@@_curr_pos_int {#1}
+ \__kernel_intarray_gset:Nnn \g_@@_balance_intarray
+ { \l_@@_curr_pos_int } { \l_@@_balance_int }
+ \if_case:w "#2 \exp_stop_f:
+ \or: \int_incr:N \l_@@_balance_int
+ \or: \int_decr:N \l_@@_balance_int
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Matching}
%
% \begin{macro}{\@@_if_match:nn}
@@ -6011,6 +6053,7 @@
#1
\@@_match:n {#2}
\@@_extract:
+ \@@_query_set:n {#2}
\@@_group_end_extract_seq:N #3
}
\cs_new_protected:Npn \@@_extract_all:nnN #1#2#3
@@ -6019,6 +6062,7 @@
\@@_multi_match:n { \@@_extract: }
#1
\@@_match:n {#2}
+ \@@_query_set:n {#2}
\@@_group_end_extract_seq:N #3
}
% \end{macrocode}
@@ -6058,7 +6102,7 @@
}
#1
\@@_match:n {#2}
-%<assert>\assert_int:n { \l_@@_curr_pos_int = \l_@@_max_pos_int }
+ \@@_query_set:n {#2}
\__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
{ \l_@@_submatch_int } { 0 }
\__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
@@ -6143,18 +6187,15 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}
-% {\@@_extract:, \@@_extract_b:wn, \@@_extract_e:wn}
-% Our task here is to extract from the property list
-% \cs{l_@@_success_submatches_prop} the list of end-points of
-% submatches, and store them in appropriate array entries, from
-% \cs{l_@@_zeroth_submatch_int} upwards. We begin by emptying those
-% entries. Then for each \meta{key}--\meta{value} pair in
-% the property list update the appropriate entry. This
-% is somewhat a hack: the \meta{key} is a non-negative integer
-% followed by |<| or |>|, which we use in a comparison to $-1$. At the
-% end, store the information about the position at which the match
-% attempt started, in \cs{g_@@_submatch_prev_intarray}.
+% \begin{macro}{\@@_extract:}
+% Our task here is to store the list of end-points of submatches, and
+% store them in appropriate array entries, from
+% \cs{l_@@_zeroth_submatch_int} upwards. First, we store in
+% \cs{g_@@_submatch_prev_intarray} the position at which the match
+% attempt started. We extract the rest from the comma list
+% \cs{l_@@_success_submatches_tl}, which starts with entries to be
+% stored in \cs{g_@@_submatch_begin_intarray} and continues with
+% entries for \cs{g_@@_submatch_end_intarray}.
% \begin{macrocode}
\cs_new_protected:Npn \@@_extract:
{
@@ -6162,34 +6203,26 @@
\int_set_eq:NN \l_@@_zeroth_submatch_int \l_@@_submatch_int
\prg_replicate:nn \l_@@_capturing_group_int
{
- \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray
- { \l_@@_submatch_int } { 0 }
- \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
- { \l_@@_submatch_int } { 0 }
\__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
{ \l_@@_submatch_int } { 0 }
\int_incr:N \l_@@_submatch_int
}
- \prop_map_inline:Nn \l_@@_success_submatches_prop
+ \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
+ { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int }
+ \int_zero:N \l_@@_internal_a_int
+ \clist_map_inline:Nn \l_@@_success_submatches_tl
{
- \if_int_compare:w ##1 - 1 \exp_stop_f:
- \exp_after:wN \@@_extract_e:wn \int_value:w
+ \if_int_compare:w \l_@@_internal_a_int < \l_@@_capturing_group_int
+ \__kernel_intarray_gset:Nnn \g_@@_submatch_begin_intarray
+ { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int } {##1}
\else:
- \exp_after:wN \@@_extract_b:wn \int_value:w
+ \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray
+ { \@@_int_eval:w \l_@@_zeroth_submatch_int + \l_@@_internal_a_int - \l_@@_capturing_group_int } {##1}
\fi:
- \@@_int_eval:w \l_@@_zeroth_submatch_int + ##1 {##2}
+ \int_incr:N \l_@@_internal_a_int
}
- \__kernel_intarray_gset:Nnn \g_@@_submatch_prev_intarray
- { \l_@@_zeroth_submatch_int } { \l_@@_start_pos_int }
\fi:
}
-\cs_new_protected:Npn \@@_extract_b:wn #1 < #2
- {
- \__kernel_intarray_gset:Nnn
- \g_@@_submatch_begin_intarray {#1} {#2}
- }
-\cs_new_protected:Npn \@@_extract_e:wn #1 > #2
- { \__kernel_intarray_gset:Nnn \g_@@_submatch_end_intarray {#1} {#2} }
% \end{macrocode}
% \end{macro}
%
@@ -6214,12 +6247,13 @@
\group_begin:
\@@_single_match:
#1
- \@@_replacement:n {#2}
- \exp_args:No \@@_match:n { #3 }
+ \exp_args:No \@@_match:n {#3}
\if_meaning:w \c_false_bool \g_@@_success_bool
\group_end:
\else:
\@@_extract:
+ \exp_args:No \@@_query_set:n {#3}
+ \@@_replacement:n {#2}
\int_set:Nn \l_@@_balance_int
{
\@@_replacement_balance_one_match:n
@@ -6259,8 +6293,9 @@
\group_begin:
\@@_multi_match:n { \@@_extract: }
#1
- \@@_replacement:n {#2}
\exp_args:No \@@_match:n {#3}
+ \exp_args:No \@@_query_set:n {#3}
+ \@@_replacement:n {#2}
\int_set:Nn \l_@@_balance_int
{
0
@@ -6320,7 +6355,371 @@
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Storing and showing compiled patterns}
+% \subsubsection{Peeking ahead}
+%
+% \begin{variable}{\l_@@_peek_true_tl, \l_@@_peek_false_tl}
+% True/false code arguments of \cs{peek_regex:nTF} or similar.
+% \begin{macrocode}
+\tl_new:N \l_@@_peek_true_tl
+\tl_new:N \l_@@_peek_false_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_replacement_tl}
+% When peeking in \cs{peek_regex_replace_once:nnTF} we need to store
+% the replacement text.
+% \begin{macrocode}
+\tl_new:N \l_@@_replacement_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_input_tl}
+% \begin{macro}{\@@_input_item:n}
+% Stores each token found as \cs{@@_input_item:n} \Arg{tokens}, where
+% the \meta{tokens} \texttt{o}-expand to the token found, as for
+% \cs{tl_analysis_map_inline:nn}.
+% \begin{macrocode}
+\tl_new:N \l_@@_input_tl
+\cs_new_eq:NN \@@_input_item:n ?
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+%
+% \begin{macro}[TF]
+% {\peek_regex:n, \peek_regex:N, \peek_regex_remove_once:n, \peek_regex_remove_once:N}
+% The |T| and |F| functions just call the corresponding |TF| function.
+% The four |TF| functions differ along two axes: whether to remove the
+% token or not, distinguished by using \cs{@@_peek_end:} or
+% \cs{@@_peek_remove_end:n} (the latter case needs an argument, as we
+% will see), and whether the regex has to be compiled or is already in
+% an |N|-type variable, distinguished by calling \cs{@@_build_aux:Nn}
+% or \cs{@@_build_aux:NN}. The first argument of these functions is
+% \cs{c_false_bool} to indicate that there should be no implicit
+% insertion of a wildcard at the start of the pattern: otherwise the
+% code would keep looking further into the input stream until matching
+% the regex.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_regex:nTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:Nn \c_false_bool {#1} }
+ { \@@_peek_end: }
+ }
+\cs_new_protected:Npn \peek_regex:nT #1#2
+ { \peek_regex:nTF {#1} {#2} { } }
+\cs_new_protected:Npn \peek_regex:nF #1 { \peek_regex:nTF {#1} { } }
+\cs_new_protected:Npn \peek_regex:NTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:NN \c_false_bool #1 }
+ { \@@_peek_end: }
+ }
+\cs_new_protected:Npn \peek_regex:NT #1#2
+ { \peek_regex:NTF #1 {#2} { } }
+\cs_new_protected:Npn \peek_regex:NF #1 { \peek_regex:NTF {#1} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:nTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:Nn \c_false_bool {#1} }
+ { \@@_peek_remove_end:n {##1} }
+ }
+\cs_new_protected:Npn \peek_regex_remove_once:nT #1#2
+ { \peek_regex_remove_once:nTF {#1} {#2} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:nF #1
+ { \peek_regex_remove_once:nTF {#1} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:NTF #1
+ {
+ \@@_peek:nnTF
+ { \@@_build_aux:NN \c_false_bool #1 }
+ { \@@_peek_remove_end:n {##1} }
+ }
+\cs_new_protected:Npn \peek_regex_remove_once:NT #1#2
+ { \peek_regex_remove_once:NTF #1 {#2} { } }
+\cs_new_protected:Npn \peek_regex_remove_once:NF #1
+ { \peek_regex_remove_once:NTF #1 { } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek:nnTF, \@@_peek_aux:nnTF}
+% Store the user's true/false codes (plus \cs{group_end:}) into two
+% token lists. Then build the automaton with |#1|, without submatch
+% tracking, and aiming for a single match. Then start matching by
+% setting up a few variables like for any regex matching like
+% \cs{regex_match:nnTF}, with the addition of \cs{l_@@_input_tl}
+% that keeps track of the tokens seen, to reinsert them at the
+% end. Instead of \cs{tl_analysis_map_inline:nn} on the input, we
+% call \cs{peek_analysis_map_inline:n} to go through tokens in the
+% input stream. Since \cs{@@_match_one_token:nnN} calls
+% \cs{@@_maplike_break:} we need to catch that and break the
+% \cs{peek_analysis_map_inline:n} loop instead.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek:nnTF #1
+ {
+ \@@_peek_aux:nnTF
+ {
+ \@@_disable_submatches:
+ #1
+ }
+ }
+\cs_new_protected:Npn \@@_peek_aux:nnTF #1#2#3#4
+ {
+ \group_begin:
+ \tl_set:Nn \l_@@_peek_true_tl { \group_end: #3 }
+ \tl_set:Nn \l_@@_peek_false_tl { \group_end: #4 }
+ \@@_single_match:
+ #1
+ \@@_match_init:
+ \tl_build_clear:N \l_@@_input_tl
+ \@@_match_once_init:
+ \peek_analysis_map_inline:n
+ {
+ \tl_build_put_right:Nn \l_@@_input_tl
+ { \@@_input_item:n {##1} }
+ \@@_match_one_token:nnN {##1} {##2} ##3
+ \use_none:nnn
+ \prg_break_point:Nn \@@_maplike_break:
+ { \peek_analysis_map_break:n {#2} }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_end:, \@@_peek_remove_end:n}
+% Once the regex matches (or permanently fails to match) we call
+% \cs{@@_peek_end:}, or \cs{@@_peek_remove_end:n} with argument the
+% last token seen (or rather tokens that \texttt{o}-expand and
+% \texttt{x}-expand to it). For \cs{peek_regex:nTF} we reinsert
+% tokens seen by calling \cs{@@_peek_reinsert:N} regardless of the
+% result of the match. For \cs{peek_regex_remove_once:nTF} we reinsert the
+% tokens seen only if the match failed; otherwise we just reinsert the
+% tokens~|#1|, with one expansion.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_end:
+ {
+ \bool_if:NTF \g_@@_success_bool
+ { \@@_peek_reinsert:N \l_@@_peek_true_tl }
+ { \@@_peek_reinsert:N \l_@@_peek_false_tl }
+ }
+\cs_new_protected:Npn \@@_peek_remove_end:n #1
+ {
+ \bool_if:NTF \g_@@_success_bool
+ { \exp_after:wN \l_@@_peek_true_tl #1 }
+ { \@@_peek_reinsert:N \l_@@_peek_false_tl }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_reinsert:N, \@@_reinsert_item:n}
+% Insert the true/false code |#1|, followed by the tokens found, which
+% were stored in \cs{l_@@_input_tl}. For this, loop through that
+% token list using \cs{@@_reinsert_item:n}, which expands |#1| once to
+% get a single token, and jumps over it to expand what follows, with
+% suitable \cs{exp:w} and \cs{exp_end:}. We cannot just use
+% \cs{use:e} on the whole token list because the result may be
+% unbalanced, which would stop the primitive prematurely, or let it
+% continue beyond where we would like.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_reinsert:N #1
+ {
+ \tl_build_end:N \l_@@_input_tl
+ \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n
+ \exp_after:wN #1 \exp:w \l_@@_input_tl \exp_end:
+ }
+\cs_new_protected:Npn \@@_reinsert_item:n #1
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \exp_end:
+ \exp_after:wN \exp_after:wN
+ #1
+ \exp:w
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[noTF]
+% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn}
+% Similar to \cs{peek_regex:nTF} above.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_regex_replace_once:nnTF #1
+ { \@@_peek_replace:nnTF { \@@_build_aux:Nn \c_false_bool {#1} } }
+\cs_new_protected:Npn \peek_regex_replace_once:nnT #1#2#3
+ { \peek_regex_replace_once:nnTF {#1} {#2} {#3} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:nnF #1#2
+ { \peek_regex_replace_once:nnTF {#1} {#2} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:nn #1#2
+ { \peek_regex_replace_once:nnTF {#1} {#2} { } { } }
+\cs_new_protected:Npn \peek_regex_replace_once:NnTF #1
+ { \@@_peek_replace:nnTF { \@@_build_aux:NN \c_false_bool #1 } }
+\cs_new_protected:Npn \peek_regex_replace_once:NnT #1#2#3
+ { \peek_regex_replace_once:NnTF #1 {#2} {#3} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:NnF #1#2
+ { \peek_regex_replace_once:NnTF #1 {#2} { } }
+\cs_new_protected:Npn \peek_regex_replace_once:Nn #1#2
+ { \peek_regex_replace_once:NnTF #1 {#2} { } { } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replace:nnTF}
+% Same as \cs{@@_peek:nnTF} (used for \cs{peek_regex:nTF} above), but
+% without disabling submatches, and with a different end. The
+% replacement text |#2| is stored, to be analyzed later.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replace:nnTF #1#2
+ {
+ \tl_set:Nn \l_@@_replacement_tl {#2}
+ \@@_peek_aux:nnTF {#1} { \@@_peek_replace_end: }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replace_end:}
+% If the match failed \cs{@@_peek_reinsert:N} reinserts the tokens
+% found. Otherwise, finish storing the submatch information using
+% \cs{@@_extract:}, and store the input into \tn{toks}. Redefine a
+% few auxiliaries to change slightly their expansion behaviour as
+% explained below. Analyse the replacement text with
+% \cs{@@_replacement:n}, which as usual defines
+% \cs{@@_replacement_do_one_match:n} to insert the tokens from the
+% start of the match attempt to the beginning of the match, followed
+% by the replacement text. The \cs{use:x} expands for instance the
+% trailing \cs{@@_query_range:nn} down to a sequence of
+% \cs{@@_reinsert_item:n} \Arg{tokens} where \meta{tokens}
+% \texttt{o}-expand to a single token that we want to insert. After
+% \texttt{x}-expansion, \cs{use:x} does \cs{use:n}, so we have
+% \cs{exp_after:wN} \cs{l_@@_peek_true_tl} \cs{exp:w} \ldots{}
+% \cs{exp_end:}. This is set up such as to obtain
+% \cs{l_@@_peek_true_tl} followed by the replaced tokens (possibly
+% unbalanced) in the input stream.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replace_end:
+ {
+ \bool_if:NTF \g_@@_success_bool
+ {
+ \@@_extract:
+ \@@_query_set_from_input_tl:
+ \cs_set_eq:NN \@@_replacement_put:n \@@_peek_replacement_put:n
+ \cs_set_eq:NN \@@_replacement_put_submatch_aux:n
+ \@@_peek_replacement_put_submatch_aux:n
+ \cs_set_eq:NN \@@_input_item:n \@@_reinsert_item:n
+ \cs_set_eq:NN \@@_replacement_exp_not:N \@@_peek_replacement_token:n
+ \cs_set_eq:NN \@@_replacement_exp_not:V \@@_peek_replacement_var:N
+ \exp_args:No \@@_replacement:n { \l_@@_replacement_tl }
+ \use:x
+ {
+ \exp_not:n { \exp_after:wN \l_@@_peek_true_tl \exp:w }
+ \@@_replacement_do_one_match:n
+ { \l_@@_zeroth_submatch_int }
+ \@@_query_range:nn
+ {
+ \__kernel_intarray_item:Nn \g_@@_submatch_end_intarray
+ { \l_@@_zeroth_submatch_int }
+ }
+ { \l_@@_max_pos_int }
+ \exp_end:
+ }
+ }
+ { \@@_peek_reinsert:N \l_@@_peek_false_tl }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_query_set_from_input_tl:, \@@_query_set_item:n}
+% The input was stored into \cs{l_@@_input_tl} as successive items
+% \cs{@@_input_item:n} \Arg{tokens}. Store that in successive
+% \tn{toks}. It's not clear whether the empty entries before and
+% after are both useful.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_query_set_from_input_tl:
+ {
+ \tl_build_end:N \l_@@_input_tl
+ \int_zero:N \l_@@_curr_pos_int
+ \cs_set_eq:NN \@@_input_item:n \@@_query_set_item:n
+ \@@_query_set_item:n { }
+ \l_@@_input_tl
+ \@@_query_set_item:n { }
+ \int_set_eq:NN \l_@@_max_pos_int \l_@@_curr_pos_int
+ }
+\cs_new_protected:Npn \@@_query_set_item:n #1
+ {
+ \int_incr:N \l_@@_curr_pos_int
+ \@@_toks_set:Nn \l_@@_curr_pos_int { \@@_input_item:n {#1} }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_put:n}
+% While building the replacement function
+% \cs{@@_replacement_do_one_match:n}, we often want to put simple
+% material, given as |#1|, whose \texttt{x}-expansion
+% \texttt{o}-expands to a single token. Normally we can just add the
+% token to \cs{l_@@_build_tl}, but for
+% \cs{peek_regex_replace_once:nnTF} we eventually want to do some
+% strange expansion that is basically using \cs{exp_after:wN} to jump
+% through numerous tokens (we cannot use \texttt{x}-expansion like for
+% \cs{regex_replace_once:nnNTF} because it is ok for the result to be
+% unbalanced since we insert it in the input stream rather than
+% storing it. When within a csname we don't do any such shenanigan
+% because \cs{cs:w} \ldots{} \cs{cs_end:} does all the expansion we
+% need.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_put:n #1
+ {
+ \if_case:w \l_@@_replacement_csnames_int
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \exp_not:N \@@_reinsert_item:n {#1} }
+ \else:
+ \tl_build_put_right:Nn \l_@@_build_tl {#1}
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_token:n}
+% When hit with \cs{exp:w}, \cs{@@_peek_replacement_token:n}
+% \Arg{token} stops \cs{exp_end:} and does \cs{exp_after:wN}
+% \meta{token} \cs{exp:w} to continue expansion after it.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_token:n #1
+ { \exp_after:wN \exp_end: \exp_after:wN #1 \exp:w }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_put_submatch_aux:n}
+% While analyzing the replacement we also have to insert submatches
+% found in the query. Since query items \cs{@@_input_item:n}
+% \Arg{tokens} expand correctly only when surrounded by \cs{exp:w}
+% \ldots{} \cs{exp_end:}, and since these expansion controls are not
+% there within csnames (because \cs{cs:w} \ldots{} \cs{cs_end:} make
+% them unnecessary in most cases), we have to put \cs{exp:w} and
+% \cs{exp_end:} by hand here.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_put_submatch_aux:n #1
+ {
+ \if_case:w \l_@@_replacement_csnames_int
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } }
+ \else:
+ \tl_build_put_right:Nn \l_@@_build_tl
+ { \exp:w \@@_query_submatch:n { \int_eval:n { #1 + ##1 } } \exp_end: }
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_peek_replacement_var:N}
+% This is used for |\u| outside csnames. It makes sure to continue
+% expansion with \cs{exp:w} before expanding the variable~|#1| and
+% stopping the \cs{exp:w} that precedes.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_replacement_var:N #1
+ {
+ \exp_after:wN \exp_last_unbraced:NV
+ \exp_after:wN \exp_end:
+ \exp_after:wN #1
+ \exp:w
+ }
+% \end{macrocode}
+% \end{macro}
%
% \subsection{Messages}
%
@@ -6726,8 +7125,6 @@
%^^A NOT IMPLEMENTED
%^^A \p{xx} a character with the xx property
%^^A \P{xx} a character without the xx property
-%^^A [[:xxx:]] positive POSIX named set
-%^^A [[:^xxx:]] negative POSIX named set
%^^A (?=...) positive look ahead
%^^A (?!...) negative look ahead
%^^A (?<=...) positive look behind
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx b/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx
index 737a4c776af..28df8592d9d 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3seq.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx b/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx
index aeef5ae45c4..6336081c2cb 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3skip.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx b/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx
index 30d2b9737ed..b47a9d73d63 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3sort.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx b/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx
index 2acfdcf0aa1..e3ebadbcdd0 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3str-convert.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3str.dtx b/Master/texmf-dist/source/latex/l3kernel/l3str.dtx
index bdde8616550..02083ae9b5f 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3str.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx b/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx
index 4e5e3816c7a..8ba6b2f2e6c 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3sys.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx b/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx
index fd41c6bdd31..b52d3488672 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3text-case.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx b/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx
index 1bc377ed02a..4a4abbf256a 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3text-purify.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -396,6 +396,8 @@
\tiny
}
{ \text_declare_purify_equivalent:Nn #1 { } }
+\exp_args:Nc \text_declare_purify_equivalent:Nn
+ { @protected@testopt } { \use_none:nnn }
% \end{macrocode}
% Environments have to be handled by pure expansion.
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3text.dtx b/Master/texmf-dist/source/latex/l3kernel/l3text.dtx
index 893c43da59a..9ab4ae6ef52 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3text.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3text.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -634,6 +634,8 @@
% \begin{macro}[EXP]{\@@_expand_protect:N}
% \begin{macro}[EXP]{\@@_expand_protect:nN}
% \begin{macro}[EXP]{\@@_expand_protect:Nw}
+% \begin{macro}[EXP]{\@@_expand_testopt:N}
+% \begin{macro}[EXP]{\@@_expand_testopt:NNn}
% \begin{macro}[EXP]{\@@_expand_replace:N}
% \begin{macro}[EXP]{\@@_expand_replace:n}
% \begin{macro}[EXP]{\@@_expand_cs_expand:N}
@@ -917,7 +919,8 @@
% \LaTeXe{}'s \cs{protect} makes life interesting. Where possible, we
% simply remove it and replace with the \enquote{parent} command; of course,
% the \cs{protect} might be explicit, in which case we need to leave it alone
-% if it's required.
+% if it's required. There is also the case of a straight \tn{@protected@testopt}
+% to cover.
% \begin{macrocode}
\cs_new:Npx \@@_expand_cs:N #1
{
@@ -927,7 +930,7 @@
\bool_lazy_and:nnTF
{ \cs_if_exist_p:N \fmtname }
{ \str_if_eq_p:Vn \fmtname { LaTeX2e } }
- { \exp_not:N \@@_expand_encoding:N #1 }
+ { \exp_not:N \@@_expand_testopt:N #1 }
{ \exp_not:N \@@_expand_replace:N #1 }
}
}
@@ -949,6 +952,17 @@
{ \@@_expand_store:n { \protect #1 } }
\@@_expand_loop:w
}
+\cs_new:Npn \@@_expand_testopt:N #1
+ {
+ \str_if_eq:nnTF {#1} { \@protected@testopt }
+ { \@@_expand_testopt:NNn }
+ { \@@_expand_encoding:N #1 }
+ }
+\cs_new:Npn \@@_expand_testopt:NNn #1#2#3
+ {
+ \@@_expand_store:n {#1}
+ \@@_expand_loop:w
+ }
% \end{macrocode}
% Deal with encoding-specific commands
% \begin{macrocode}
@@ -1036,6 +1050,8 @@
% \end{macro}
% \end{macro}
% \end{macro}
+% \end{macro}
+% \end{macro}
%
% \begin{macro}
% {
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx
index be4bd2406e8..cf61a8f7028 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3tl-analysis.dtx
@@ -44,7 +44,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -52,10 +52,30 @@
%
% \section{\pkg{l3tl-analysis} documentation}
%
-% This module mostly provides internal functions for use in the
-% \pkg{l3regex} module. However, it provides as a side-effect a user
-% debugging function, very similar to the \cs{ShowTokens} macro from the
-% \pkg{ted} package.
+% This module provides functions that are particularly useful in the
+% \pkg{l3regex} module for mapping through a token list one \meta{token}
+% at a time (including begin-group/end-group tokens). For
+% \cs{tl_analysis_map_inline:Nn} or \cs{tl_analysis_map_inline:nn}, the
+% token list is given as an argument; the analogous function
+% \cs{peek_analysis_map_inline:n} documented in \pkg{l3token} finds
+% tokens in the input stream instead. In both cases the user provides
+% \meta{inline code} that receives three arguments for each
+% \meta{token}:
+% \begin{itemize}
+% \item \meta{tokens}, which both \texttt{o}-expand and
+% \texttt{x}-expand to the \meta{token}. The detailed form of
+% \meta{tokens} may change in later releases.
+% \item \meta{char code}, a decimal representation of the character
+% code of the \meta{token}, $-1$ if it is a control sequence.
+% \item \meta{catcode}, a capital hexadecimal digit which denotes the
+% category code of the \meta{token} (0:~control sequence,
+% 1:~begin-group, 2:~end-group, 3:~math shift, 4:~alignment tab,
+% 6:~parameter, 7:~superscript, 8:~subscript, A:~space, B:~letter,
+% C:~other, D:~active). This can be converted to an integer by
+% writing |"|\meta{catcode}.
+% \end{itemize}
+% In addition, there is a debugging function \cs{tl_analysis_show:n},
+% very similar to the \cs{ShowTokens} macro from the \pkg{ted} package.
%
% \begin{function}[added = 2018-04-09]{\tl_analysis_show:N, \tl_analysis_show:n}
% \begin{syntax}
@@ -74,23 +94,9 @@
% \end{syntax}
% Applies the \meta{inline function} to each individual \meta{token}
% in the \meta{token list}. The \meta{inline function} receives three
-% arguments:
-% \begin{itemize}
-% \item \meta{tokens}, which both \texttt{o}-expand and
-% \texttt{x}-expand to the \meta{token}. The detailed form of
-% \meta{token} may change in later releases.
-% \item \meta{char code}, a decimal representation of the character
-% code of the token, $-1$ if it is a control sequence (with
-% \meta{catcode} $0$).
-% \item \meta{catcode}, a capital hexadecimal digit which denotes
-% the category code of the \meta{token} (0: control sequence, 1:
-% begin-group, 2: end-group, 3: math shift, 4: alignment tab, 6:
-% parameter, 7: superscript, 8: subscript, A: space, B: letter,
-% C:other, D:active).
-% \end{itemize}
-% As all other mappings the mapping is done at the current group
-% level, \emph{i.e.}~any local assignments made by the \meta{inline
-% function} remain in effect after the loop.
+% arguments as explained above. As all other mappings the mapping is
+% done at the current group level, \emph{i.e.}~any local assignments
+% made by the \meta{inline function} remain in effect after the loop.
% \end{function}
%
% \end{documentation}
@@ -182,18 +188,54 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_@@_analysis_token}
-% \begin{variable}{\l_@@_analysis_char_token}
+% \begin{variable}
+% {\l_@@_analysis_token, \l_@@_analysis_char_token, \l_@@_analysis_next_token}
% The tokens in the token list are probed with the \TeX{} primitive
% \tn{futurelet}. We use \cs{l_@@_analysis_token} in that
% construction. In some cases, we convert the following token to a
% string before probing it: then the token variable used is
-% \cs{l_@@_analysis_char_token}.
+% \cs{l_@@_analysis_char_token}. When getting tokens from the input
+% stream we may need to look two tokens ahead, for which we use
+% \cs{l_@@_analysis_next_token}.
% \begin{macrocode}
\cs_new_eq:NN \l_@@_analysis_token ?
\cs_new_eq:NN \l_@@_analysis_char_token ?
+\cs_new_eq:NN \l_@@_analysis_next_token ?
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_peek_code_tl}
+% Holds some code to be run once the next token has been fully
+% analysed in \cs{peek_analysis_map_inline:n}.
+% \begin{macrocode}
+\tl_new:N \l_@@_peek_code_tl
% \end{macrocode}
% \end{variable}
+%
+% \begin{variable}{\c_@@_peek_catcodes_tl}
+% A token list containing the character number~$32$ (space) with all
+% possible category codes except $1$ and $2$ (begin-group and
+% end-group). Why $32$? Because some \LuaTeX{} versions only allow
+% creation of catcode~$10$ (space) tokens with this character code,
+% and because even in other engines it is much easier to produce since
+% \cs{char_generate:nn} refuses to produce spaces.
+% \begin{macrocode}
+\group_begin:
+\char_set_active_eq:NN \ \scan_stop:
+\tl_const:Nx \c_@@_peek_catcodes_tl
+ {
+ \char_generate:nn { 32 } { 3 } 3
+ \char_generate:nn { 32 } { 4 } 4
+ # \char_generate:nn { 32 } { 6 } 6
+ \char_generate:nn { 32 } { 7 } 7
+ \char_generate:nn { 32 } { 8 } 8
+ \c_space_tl \token_to_str:N A
+ \char_generate:nn { 32 } { 11 } \token_to_str:N B
+ \char_generate:nn { 32 } { 12 } \token_to_str:N C
+ \char_generate:nn { 32 } { 13 } \token_to_str:N D
+ }
+\group_end:
+% \end{macrocode}
% \end{variable}
%
% \begin{variable}{\l_@@_analysis_normal_int}
@@ -1075,6 +1117,358 @@
% \end{macro}
% \end{macro}
%
+% \subsection{Peeking ahead}
+%
+% \begin{macro}[EXP]{\peek_analysis_map_break:, \peek_analysis_map_break:n}
+% The break statements use the general \cs{prg_map_break:Nn}.
+% \begin{macrocode}
+\cs_new:Npn \peek_analysis_map_break:
+ { \prg_map_break:Nn \peek_analysis_map_break: { } }
+\cs_new:Npn \peek_analysis_map_break:n
+ { \prg_map_break:Nn \peek_analysis_map_break: }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}{\l_@@_peek_charcode_int}
+% \begin{macrocode}
+\int_new:N \l_@@_peek_charcode_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}{\@@_analysis_char_arg:Nw, \@@_analysis_char_arg_aux:Nw}
+% After a call to \tn{futurelet} \cs{l_@@_analysis_token} followed by
+% a stringified character token (either explicit space or catcode
+% other character), grab the argument and pass it to |#1|. We only
+% need to do anything in the case of a space.
+% \begin{macrocode}
+\cs_new:Npn \@@_analysis_char_arg:Nw
+ {
+ \if_meaning:w \l_@@_analysis_token \c_space_token
+ \exp_after:wN \@@_analysis_char_arg_aux:Nw
+ \fi:
+ }
+\cs_new:Npn \@@_analysis_char_arg_aux:Nw #1 ~ { #1 { ~ } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {
+% \peek_analysis_map_inline:n,
+% \@@_peek_analysis_loop:NNn, \@@_peek_analysis_test:,
+% \@@_peek_analysis_normal:N, \@@_peek_analysis_cs:,
+% \@@_peek_analysis_char:N, \@@_peek_analysis_char:nN,
+% \@@_peek_analysis_special:, \@@_peek_analysis_retest:,
+% \@@_peek_analysis_next:, \@@_peek_analysis_str:,
+% \@@_peek_analysis_str:w, \@@_peek_analysis_str:n,
+% \@@_peek_analysis_active_str:n, \@@_peek_analysis_explicit:n,
+% \@@_peek_analysis_escape:, \@@_peek_analysis_collect:w,
+% \@@_peek_analysis_collect:n, \@@_peek_analysis_collect_loop:,
+% \@@_peek_analysis_collect_test:, \@@_peek_analysis_collect_end:NNN
+% }
+% Save the user's code in a control sequence that is suitable for
+% nested maps. We may wish to pass to this function an \tn{outer}
+% control sequence or active character; for this we will undefine
+% potentially-\tn{outer} tokens within a group, closed after the
+% function receives its arguments. This user's code function also
+% calls the loop auxiliary, and includes the trailing
+% \cs{prg_break_point:Nn} for when the user wants to stop the loop.
+% The loop auxiliary must remove that break point because it must look
+% at the input stream.
+% \begin{macrocode}
+\cs_new_protected:Npn \peek_analysis_map_inline:n #1
+ {
+ \int_gincr:N \g__kernel_prg_map_int
+ \cs_set_protected:cpn
+ { @@_analysis_map_ \int_use:N \g__kernel_prg_map_int :nnN }
+ ##1##2##3
+ {
+ \group_end:
+ #1
+ \@@_peek_analysis_loop:NNn
+ \prg_break_point:Nn \peek_analysis_map_break: { }
+ }
+ \@@_peek_analysis_loop:NNn ? ? ?
+ }
+% \end{macrocode}
+% The loop starts a group (closed by the user-code function defined
+% above) with a normalized escape character, and checks if the next
+% token is special or \texttt{N}-type.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_loop:NNn #1#2#3
+ {
+ \group_begin:
+ \tl_set:Nx \l_@@_peek_code_tl
+ {
+ \exp_not:c
+ { @@_analysis_map_ \int_use:N \g__kernel_prg_map_int :nnN }
+ }
+ \int_set:Nn \tex_escapechar:D { `\\ }
+ \peek_after:Nw \@@_peek_analysis_test:
+ }
+\cs_new_protected:Npn \@@_peek_analysis_test:
+ {
+ \if_int_odd:w
+ \if_catcode:w \exp_not:N \l_peek_token { 1 \exp_stop_f: \fi:
+ \if_catcode:w \exp_not:N \l_peek_token } 1 \exp_stop_f: \fi:
+ \if_meaning:w \l_peek_token \c_space_token 1 \exp_stop_f: \fi:
+ 0 \exp_stop_f:
+ \exp_after:wN \@@_peek_analysis_special:
+ \else:
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \@@_peek_analysis_normal:N
+ \exp_after:wN \exp_not:N
+ \fi:
+ }
+% \end{macrocode}
+% Normal tokens are not too hard, but can be \tn{outer}, hence the
+% \cs{exp_not:N} in the code above. If the token is expandable then
+% it might be an \tn{outer} or a \TeX{} conditional, so to be safe we
+% set it to \cs{scan_stop:} (the assignment is local and stopped by
+% the \cs{group_end:} upon calling the user's code). Then distinguish
+% characters (including active ones and macro parameter characters)
+% from control sequences (whose string representation is more than one
+% character because the escape character is printable). For a control
+% sequence call the user code with suitable arguments.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_normal:N #1
+ {
+ \exp_after:wN \reverse_if:N \exp_after:wN \if_meaning:w
+ \exp_not:N #1 #1
+ \tex_let:D #1 \scan_stop:
+ \tl_put_right:Nn \l_@@_peek_code_tl { { \exp_not:N #1 } }
+ \else:
+ \tl_put_right:Nn \l_@@_peek_code_tl { { \exp_not:n {#1} } }
+ \fi:
+ \if_charcode:w
+ \scan_stop:
+ \exp_after:wN \use_none:n \token_to_str:N #1 \prg_do_nothing:
+ \scan_stop:
+ \exp_after:wN \@@_peek_analysis_char:N
+ \exp_after:wN #1
+ \else:
+ \exp_after:wN \@@_peek_analysis_cs:
+ \fi:
+ }
+\cs_new_protected:Npn \@@_peek_analysis_cs:
+ { \l_@@_peek_code_tl { -1 } 0 }
+\cs_new_protected:Npn \@@_peek_analysis_char:N #1
+ {
+ \char_set_lccode:nn { `#1 } { 32 }
+ \tex_lowercase:D { \@@_peek_analysis_char:nN {#1} } #1
+ }
+\cs_new_protected:Npn \@@_peek_analysis_char:nN #1#2
+ {
+ \cs_set_protected:Npn \@@_tmp:w ##1 #1 ##2 ##3 \scan_stop:
+ { \exp_args:No \l_@@_peek_code_tl { \int_value:w `#2 } ##2 }
+ \exp_after:wN \@@_tmp:w \c_@@_peek_catcodes_tl \scan_stop:
+ }
+% \end{macrocode}
+% For special characters the idea is to eventually act with
+% \cs{token_to_str:N}, then pick up one by one the characters of this
+% string representation until hitting the token that follows. First
+% determine the character code of (the meaning of) the \meta{token}
+% (which we know is a special token), make sure the escape character
+% is different from it, normalize the meanings of two active
+% characters and the empty control sequence, and filter out these
+% cases in \cs{@@_peek_analysis_retest:}.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_special:
+ {
+ \tex_let:D \l_@@_analysis_token = ~ \l_peek_token
+ \int_set:Nn \l_@@_peek_charcode_int
+ { \@@_analysis_extract_charcode: }
+ \if_int_compare:w \l_@@_peek_charcode_int = \tex_escapechar:D
+ \int_set:Nn \tex_escapechar:D { `\/ }
+ \fi:
+ \char_set_active_eq:nN { \l_@@_peek_charcode_int } \scan_stop:
+ \char_set_active_eq:nN { \tex_escapechar:D } \scan_stop:
+ \cs_set_eq:cN { } \scan_stop:
+ \tex_futurelet:D \l_@@_analysis_token
+ \@@_peek_analysis_retest:
+ }
+\cs_new_protected:Npn \@@_peek_analysis_retest:
+ {
+ \if_meaning:w \l_@@_analysis_token \scan_stop:
+ \exp_after:wN \@@_peek_analysis_normal:N
+ \else:
+ \exp_after:wN \@@_peek_analysis_next:
+ \fi:
+ }
+% \end{macrocode}
+% At this point we know the meaning of the \meta{token} in the input
+% stream is \cs{l_peek_token}, either a space (32, 10) or a
+% begin-group or end-group token (catcode $1$ or~$2$), and we excluded
+% a few cases that would be difficult later (empty control sequence,
+% active character with the same character code as its meaning or as
+% the escape character). Now look at the \meta{next token} following
+% it using a combination of \tn{afterassignment} and \tn{futurelet}.
+% The syntax of this primitive is \tn{futurelet} \meta{peek token}
+% \meta{first token} \meta{next token}, and it sets \meta{peek token}
+% equal to \meta{next token}. Traditionally, one takes \meta{first
+% token} to be some macro that regains control of the code and, e.g.,
+% analyses \meta{peek token}. Here, both \meta{first token} and
+% \meta{next token} are mostly unknown tokens in the input stream (but
+% we know the \meta{first token} has catcode $1$, $2$ or $10$), where
+% \meta{first token} was already stored as \cs{l_peek_token}, and we
+% regain control using \tn{afterassignment}, which inserts its
+% argument after the assignment, hence after \meta{peek token} but
+% before \meta{first token}.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_next:
+ {
+ \tl_if_empty:oT { \tex_the:D \tex_everyeof:D }
+ { \tex_everyeof:D { \scan_stop: } }
+ \tex_afterassignment:D \@@_peek_analysis_str:
+ \tex_futurelet:D \l_@@_analysis_next_token
+ }
+% \end{macrocode}
+% We then hit the \meta{first token} with \cs{token_to_str:N} and grab
+% characters until finding \cs{l_@@_analysis_next_token}. More
+% precisely, by looking at the first character in the string
+% representation of the \meta{first token} we distinguish three cases:
+% a stringified control sequence starts with the escape character; for
+% an explicit character we find that same character; for an explicit
+% character we find anything else (we made sure to exclude the case of
+% an active character whose string representation coincides with the
+% other two cases).
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_str:
+ {
+ \exp_after:wN \tex_futurelet:D
+ \exp_after:wN \l_@@_analysis_token
+ \exp_after:wN \@@_peek_analysis_str:w
+ \token_to_str:N
+ }
+\cs_new_protected:Npn \@@_peek_analysis_str:w
+ { \@@_analysis_char_arg:Nw \@@_peek_analysis_str:n }
+\cs_new_protected:Npn \@@_peek_analysis_str:n #1
+ {
+ \int_case:nnF { `#1 }
+ {
+ { \l_@@_peek_charcode_int }
+ { \@@_peek_analysis_explicit:n {#1} }
+ { \tex_escapechar:D } { \@@_peek_analysis_escape: }
+ }
+ { \@@_peek_analysis_active_str:n {#1} }
+ }
+% \end{macrocode}
+% When |#1| is a stringified active character we pass appropriate
+% arguments to the user's code; thankfully \cs{char_generate:nn}
+% can make active characters.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_active_str:n #1
+ {
+ \tl_put_right:Nx \l_@@_peek_code_tl
+ {
+ { \char_generate:nn { `#1 } { 13 } }
+ { \int_value:w `#1 }
+ \token_to_str:N D
+ }
+ \l_@@_peek_code_tl
+ }
+% \end{macrocode}
+% When |#1| matches the character we had extracted from the meaning of
+% \cs{l_peek_token}, the token was an explicit character, which can be
+% a standard space, or a begin-group or end-group character with some
+% character code. In the latter two cases we call
+% \cs{char_generate:nn} with suitable arguments and put suitable
+% \cs{if_false:} \cs{fi:} constructions to make the result balanced
+% and such that \texttt{o}-expanding or \texttt{x}-expanding gives
+% back a single (unbalanced) begin-group or end-group character.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_explicit:n #1
+ {
+ \tl_put_right:Nx \l_@@_peek_code_tl
+ {
+ \if_meaning:w \l_peek_token \c_space_token
+ { ~ } { 32 } \token_to_str:N A
+ \else:
+ \if_catcode:w \l_peek_token \c_group_begin_token
+ {
+ \exp_not:N \exp_after:wN
+ \char_generate:nn { `#1 } { 1 }
+ \exp_not:N \if_false:
+ \if_false: { \fi: }
+ \exp_not:N \fi:
+ }
+ { \int_value:w `#1 }
+ 1
+ \else:
+ {
+ \exp_not:N \if_false:
+ { \if_false: } \fi:
+ \exp_not:N \fi:
+ \char_generate:nn { `#1 } { 2 }
+ }
+ { \int_value:w `#1 }
+ 2
+ \fi:
+ \fi:
+ }
+ \l_@@_peek_code_tl
+ }
+% \end{macrocode}
+% Finally there is the case of a special token whose string
+% representation starts with an escape character, namely the token was
+% a control sequence. In that case we could have grabbed the token
+% directly as an \texttt{N}-type argument, but of course we couldn't
+% know that until we had run all the various tests including
+% stringifying the token. We are thus left with the hard work of
+% picking up one by one the characters in the csname (being careful
+% about spaces), until finding a token that matches the \meta{next
+% token} picked up earlier (which was not stringified), such that the
+% control sequence that we found so far indeed has the expected
+% meaning \cs{l_peek_token}. This comparison with \cs{l_peek_token}
+% catches a reasonably common case like \cs{c_group_begin_token} |_|
+% in which the trailing |_| has category code other: without
+% comparison of the constructed csname with \cs{l_peek_token}
+% collection would stop at \cs[no-index]{c}, which is wrong.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_escape:
+ {
+ \tl_clear:N \l_@@_internal_a_tl
+ \tex_futurelet:D \l_@@_analysis_token
+ \@@_peek_analysis_collect:w
+ }
+\cs_new_protected:Npn \@@_peek_analysis_collect:w
+ { \@@_analysis_char_arg:Nw \@@_peek_analysis_collect:n }
+\cs_new_protected:Npn \@@_peek_analysis_collect:n #1
+ {
+ \tl_put_right:Nn \l_@@_internal_a_tl {#1}
+ \@@_peek_analysis_collect_loop:
+ }
+\cs_new_protected:Npn \@@_peek_analysis_collect_loop:
+ {
+ \tex_futurelet:D \l_@@_analysis_token
+ \@@_peek_analysis_collect_test:
+ }
+\cs_new_protected:Npn \@@_peek_analysis_collect_test:
+ {
+ \if_meaning:w \l_@@_analysis_token \l_@@_analysis_next_token
+ \exp_after:wN \if_meaning:w \cs:w \l_@@_internal_a_tl \cs_end: \l_peek_token
+ \@@_peek_analysis_collect_end:NNN
+ \fi:
+ \fi:
+ \@@_peek_analysis_collect:w
+ }
+% \end{macrocode}
+% End by calling the user code with suitable arguments (here |#1|,
+% |#2| are \cs{fi:}), which closes the group begun early on.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_peek_analysis_collect_end:NNN #1#2#3
+ {
+ #1 #2
+ \tl_put_right:Nx \l_@@_peek_code_tl
+ {
+ { \exp_not:N \exp_not:n { \exp_not:c { \l_@@_internal_a_tl } } }
+ { -1 }
+ 0
+ }
+ \l_@@_peek_code_tl
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsection{Messages}
%
% \begin{variable}{\c_@@_analysis_show_etc_str}
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
index b942e97703c..f88abe05c7a 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3tl.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
index 676c55747ae..6105f3aab05 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3token.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%
@@ -754,14 +754,42 @@
% this includes primitive-like commands defined using |{token.set_lua}|.
% \end{function}
%
+% \begin{function}[added = 2020-12-03, EXP, noTF]
+% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn}
+% \begin{syntax}
+% \cs{token_case_meaning:NnTF} \meta{test token} \\
+% ~~"{" \\
+% ~~~~\meta{token case_1} \Arg{code case_1} \\
+% ~~~~\meta{token case_2} \Arg{code case_2} \\
+% ~~~~\ldots \\
+% ~~~~\meta{token case_n} \Arg{code case_n} \\
+% ~~"}" \\
+% ~~\Arg{true code}
+% ~~\Arg{false code}
+% \end{syntax}
+% This function compares the \meta{test token} in turn with each of
+% the \meta{token cases}. If the two are equal (as described for
+% \cs{token_if_eq_catcode:NNTF}, \cs{token_if_eq_charcode:NNTF} and
+% \cs{token_if_eq_meaning:NNTF}, respectively) then the associated
+% \meta{code} is left in the input stream and other cases are
+% discarded. If any of the cases are matched, the \meta{true code} is
+% also inserted into the input stream (after the code for the
+% appropriate case), while if none match then the \meta{false code} is
+% inserted. The functions \cs{token_case_catcode:Nn},
+% \cs{token_case_charcode:Nn}, and \cs{token_case_meaning:Nn}, which
+% do nothing if there is no match, are also available.
+% \end{function}
+%
% \section{Peeking ahead at the next token}
%
% There is often a need to look ahead at the next token in the input
% stream while leaving it in place. This is handled using the
-% \enquote{peek} functions. The generic \cs{peek_after:Nw} is
-% provided along with a family of predefined tests for common cases.
-% As peeking ahead does \emph{not} skip spaces the predefined tests
-% include both a space-respecting and space-skipping version.
+% \enquote{peek} functions. The generic \cs{peek_after:Nw} is provided
+% along with a family of predefined tests for common cases. As peeking
+% ahead does \emph{not} skip spaces the predefined tests include both a
+% space-respecting and space-skipping version. In addition, using
+% \cs{peek_analysis_map_inline:n}, one can map through the following
+% tokens in the input stream and repeatedly perform some tests.
%
% \begin{function}{\peek_after:Nw}
% \begin{syntax}
@@ -980,6 +1008,128 @@
% (as appropriate to the result of the test).
% \end{function}
%
+% \begin{function}[added = 2020-12-03]{\peek_analysis_map_inline:n}
+% \begin{syntax}
+% \cs{peek_analysis_map_inline:n} \Arg{inline function}
+% \end{syntax}
+% Repeatedly removes one \meta{token} from the input stream and
+% applies the \meta{inline function} to it, until
+% \cs{peek_analysis_map_break:} is called. The \meta{inline function}
+% receives three arguments for each \meta{token} in the input stream:
+% \begin{itemize}
+% \item \meta{tokens}, which both \texttt{o}-expand and
+% \texttt{x}-expand to the \meta{token}. The detailed form of
+% \meta{tokens} may change in later releases.
+% \item \meta{char code}, a decimal representation of the character
+% code of the \meta{token}, $-1$ if it is a control sequence.
+% \item \meta{catcode}, a capital hexadecimal digit which denotes the
+% category code of the \meta{token} (0:~control sequence,
+% 1:~begin-group, 2:~end-group, 3:~math shift, 4:~alignment tab,
+% 6:~parameter, 7:~superscript, 8:~subscript, A:~space, B:~letter,
+% C:~other, D:~active). This can be converted to an integer by
+% writing |"|\meta{catcode}.
+% \end{itemize}
+% These arguments are the same as for \cs{tl_analysis_map_inline:nn}
+% defined in \pkg{l3tl-analysis}. The \meta{char code} and
+% \meta{catcode} do not take the meaning of a control sequence or
+% active character into account: for instance, upon encountering the
+% token \cs{c_group_begin_token} in the input stream,
+% \cs{peek_analysis_map_inline:n} calls the \meta{inline function}
+% with |#1| being \cs{exp_not:n} |{| \cs{c_group_begin_token} |}|
+% (with the current implementation),
+% |#2|~being~$-1$, and
+% |#3|~being~$0$, as for any other control sequence. In contrast,
+% upon encountering an explicit begin-group token~|{|, % ^^A |}|
+% the \meta{inline function} is called with arguments
+% \cs{exp_after:wN} |{| \cs{if_false:} |}| \cs{fi:}, $123$ and~$1$.
+%
+% The mapping is done at the current group level, \emph{i.e.}~any
+% local assignments made by the \meta{inline function} remain in
+% effect after the loop. Within the code, \cs{l_peek_token} is set
+% equal (as a token, not a token list) to the token under
+% consideration.
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03]
+% {\peek_analysis_map_break:, \peek_analysis_map_break:n}
+% \begin{syntax}
+% \cs{peek_analysis_map_inline:n}
+% |{| \dots{} \cs{peek_analysis_map_break:n} \Arg{code} |}|
+% \end{syntax}
+% Stops the \cs{peek_analysis_map_inline:n} loop from seeking more
+% tokens, and inserts \meta{code} in the input stream (empty for
+% \cs{peek_analysis_map_break:}).
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03, TF]{\peek_regex:n, \peek_regex:N}
+% \begin{syntax}
+% \cs{peek_regex:nTF} \Arg{regex} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{tokens} that follow in the input stream match the
+% \meta{regular expression}. Any \meta{tokens} that have been read
+% are left in the input stream after the \meta{true code} or
+% \meta{false code} (as appropriate to the result of the test). See
+% \pkg{l3regex} for documentation of the syntax of regular
+% expressions. The \meta{regular expression} is implicitly anchored
+% at the start, so for instance \cs{peek_regex:nTF}~|{|~|a|~|}| is
+% essentially equivalent to \cs{peek_charcode:NTF}~|a|.
+% \begin{texnote}
+% Implicit character tokens are correctly considered by
+% \cs{peek_regex:nTF} as control sequences, while functions that
+% inspect individual tokens (for instance \cs{peek_charcode:NTF})
+% only take into account their meaning.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03, TF]
+% {\peek_regex_remove_once:n, \peek_regex_remove_once:N}
+% \begin{syntax}
+% \cs{peek_regex_remove_once:nTF} \Arg{regex} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{tokens} that follow in the input stream match the
+% \meta{regex}. If the test is true, the \meta{tokens} are removed
+% from the input stream and the \meta{true code} is inserted, while if
+% the test is false, the \meta{false code} is inserted followed by the
+% \meta{tokens} that were originally in the input stream.
+% See \pkg{l3regex} for documentation of the syntax of
+% regular expressions. The \meta{regular expression} is implicitly
+% anchored at the start, so for instance
+% \cs{peek_regex_remove_once:nTF}~|{|~|a|~|}| is essentially equivalent to
+% \cs{peek_charcode_remove:NTF}~|a|.
+% \begin{texnote}
+% Implicit character tokens are correctly considered by
+% \cs{peek_regex_remove_once:nTF} as control sequences, while functions
+% that inspect individual tokens (for instance
+% \cs{peek_charcode:NTF}) only take into account their meaning.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}[added = 2020-12-03, noTF]
+% {\peek_regex_replace_once:nn, \peek_regex_replace_once:Nn}
+% \begin{syntax}
+% \cs{peek_regex_replace_once:nnTF} \Arg{regex} \Arg{replacement} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% If the \meta{tokens} that follow in the input stream match the
+% \meta{regex}, replaces them according to the \meta{replacement} as
+% for \cs{regex_replace_once:nnN}, and leaves the result in the input
+% stream, after the \meta{true code}. Otherwise, leaves \meta{false
+% code} followed by the \meta{tokens} that were originally in the
+% input stream, with no modifications. See \pkg{l3regex} for
+% documentation of the syntax of regular expressions and of the
+% \meta{replacement}: for instance |\0| in the \meta{replacement} is
+% replaced by the tokens that were matched in the input stream. The
+% \meta{regular expression} is implicitly anchored at the start. In
+% contrast to \cs{regex_replace_once:nnN}, no error arises if the
+% \meta{replacement} leads to an unbalanced token list: the tokens are
+% inserted into the input stream without issue.
+% \begin{texnote}
+% Implicit character tokens are correctly considered by
+% \cs{peek_regex_replace_once:nnTF} as control sequences, while
+% functions that inspect individual tokens (for instance
+% \cs{peek_charcode:NTF}) only take into account their meaning.
+% \end{texnote}
+% \end{function}
+%
% \section{Description of all possible tokens}
% \label{sec:l3token:all-tokens}
%
@@ -1897,9 +2047,10 @@
%<@@=token>
% \end{macrocode}
%
-% \begin{variable}{\s_@@_stop}
+% \begin{variable}{\s_@@_mark, \s_@@_stop}
% Internal scan marks.
% \begin{macrocode}
+\scan_new:N \s_@@_mark
\scan_new:N \s_@@_stop
% \end{macrocode}
% \end{variable}
@@ -2566,6 +2717,72 @@ end
% \end{macro}
% \end{macro}
%
+% \begin{macro}[EXP, noTF]
+% {\token_case_catcode:Nn, \token_case_charcode:Nn, \token_case_meaning:Nn}
+% \begin{macro}[EXP]{\@@_case:NNnTF, \@@_case:NNw, \@@_case_end:nw}
+% The aim here is to allow the case statement to be evaluated
+% using a known number of expansion steps (two), and without
+% needing to use an explicit \enquote{end of recursion} marker.
+% That is achieved by using the test input as the final case,
+% as this is always true. The trick is then to tidy up
+% the output such that the appropriate case code plus either
+% the \texttt{true} or \texttt{false} branch code is inserted.
+% \begin{macrocode}
+\cs_new:Npn \token_case_catcode:Nn #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } { } }
+\cs_new:Npn \token_case_catcode:NnT #1#2#3
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} {#3} { } }
+\cs_new:Npn \token_case_catcode:NnF #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF #1 {#2} { } }
+\cs_new:Npn \token_case_catcode:NnTF
+ { \exp:w \@@_case:NNnTF \token_if_eq_catcode:NNTF }
+\cs_new:Npn \token_case_charcode:Nn #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } { } }
+\cs_new:Npn \token_case_charcode:NnT #1#2#3
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} {#3} { } }
+\cs_new:Npn \token_case_charcode:NnF #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF #1 {#2} { } }
+\cs_new:Npn \token_case_charcode:NnTF
+ { \exp:w \@@_case:NNnTF \token_if_eq_charcode:NNTF }
+\cs_new:Npn \token_case_meaning:Nn #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } { } }
+\cs_new:Npn \token_case_meaning:NnT #1#2#3
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} {#3} { } }
+\cs_new:Npn \token_case_meaning:NnF #1#2
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF #1 {#2} { } }
+\cs_new:Npn \token_case_meaning:NnTF
+ { \exp:w \@@_case:NNnTF \token_if_eq_meaning:NNTF }
+\cs_new:Npn \@@_case:NNnTF #1#2#3#4#5
+ {
+ \@@_case:NNw #1 #2 #3 #2 { }
+ \s_@@_mark {#4}
+ \s_@@_mark {#5}
+ \s_@@_stop
+ }
+\cs_new:Npn \@@_case:NNw #1#2#3#4
+ {
+ #1 #2 #3
+ { \@@_case_end:nw {#4} }
+ { \@@_case:NNw #1 #2 }
+ }
+% \end{macrocode}
+% To tidy up the recursion, there are two outcomes. If there was a hit to
+% one of the cases searched for, then |#1| is the code to insert,
+% |#2| is the \emph{next} case to check on and |#3| is all of
+% the rest of the cases code. That means that |#4| is the \texttt{true}
+% branch code, and |#5| tidies up the spare \cs{s_@@_mark} and the
+% \texttt{false} branch. On the other hand, if none of the cases matched
+% then we arrive here using the \enquote{termination} case of comparing
+% the search with itself. That means that |#1| is empty, |#2| is
+% the first \cs{s_@@_mark} and so |#4| is the \texttt{false} code (the
+% \texttt{true} code is mopped up by |#3|).
+% \begin{macrocode}
+\cs_new:Npn \@@_case_end:nw #1#2#3 \s_@@_mark #4#5 \s_@@_stop
+ { \exp_end: #1 #4 }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
% \subsection{Peeking ahead at the next token}
%
% \begin{macrocode}
diff --git a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx
index 14467d96177..c69fa555ab0 100644
--- a/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx
+++ b/Master/texmf-dist/source/latex/l3kernel/l3unicode.dtx
@@ -43,7 +43,7 @@
% }^^A
% }
%
-% \date{Released 2020-10-27}
+% \date{Released 2020-12-03}
%
% \maketitle
%