summaryrefslogtreecommitdiff
path: root/Master/texmf-dist
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2012-02-06 01:22:53 +0000
committerKarl Berry <karl@freefriends.org>2012-02-06 01:22:53 +0000
commita851d6714a00e2a09b93ba3e251d8665290c0ab8 (patch)
tree3ba045c487cc9e5e28e73714cb6312d0e556b924 /Master/texmf-dist
parenta9533a6794104fad593912e3481304047dcd0025 (diff)
l3experimental 3287
git-svn-id: svn://tug.org/texlive/trunk@25308 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist')
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/README9
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3dt/l3dt.pdfbin0 -> 517694 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdfbin408973 -> 419092 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdfbin611358 -> 641253 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdfbin753705 -> 773069 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdfbin504006 -> 504777 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdfbin481796 -> 481798 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdfbin626321 -> 633867 bytes
-rw-r--r--Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdfbin566327 -> 566303 bytes
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx1351
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.ins40
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx257
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx2855
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx3692
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins44
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx367
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex380
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex281
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex185
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx163
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/xgalley/xgalley.dtx25
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3dt/l3dt.sty459
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty66
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty1597
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty1040
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty46
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty112
-rw-r--r--Master/texmf-dist/tex/latex/l3experimental/xgalley/xgalley.sty20
28 files changed, 7618 insertions, 5371 deletions
diff --git a/Master/texmf-dist/doc/latex/l3experimental/README b/Master/texmf-dist/doc/latex/l3experimental/README
index 9acac689ec6..979f15c6bc1 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/README
+++ b/Master/texmf-dist/doc/latex/l3experimental/README
@@ -28,10 +28,19 @@ are under active development and the interfaces may change.
Currently included in the CTAN release of l3experimental are the following
bundles:
+ * l3dt
* l3str
* xcoffins
* xgalley
+l3dt
+----
+
+A 'data table' is a method of storing data in a spreadsheet-like format,
+with rows and fields. This module provides the basic management structures
+needed to work with data tables, including the ability to map to fields
+on a row-by-row basis.
+
l3str
-----
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3dt/l3dt.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3dt/l3dt.pdf
new file mode 100644
index 00000000000..fef8e6c0cbc
--- /dev/null
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3dt/l3dt.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf
index a8357c3a9d1..b192a46feda 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3flag.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf
index 9e9589d110b..a515a7db835 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf
index ba1ea7f9ae4..fd12914e028 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf
index 2c899da282e..73b04f9da8a 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3tl-analysis.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf
index 5e2c94527c5..e7d17de3454 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf
index f9890ba242e..58bce6c6648 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf
index c9470898b78..6729e85c3a7 100644
--- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf
+++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf
Binary files differ
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx
new file mode 100644
index 00000000000..e5063618228
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.dtx
@@ -0,0 +1,1351 @@
+% \iffalse meta-comment
+%
+%% File l3dt.dtx Copyright (C) 2011,2012 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX Project Team.
+%%
+%% -----------------------------------------------------------------------
+%%
+%
+%<*driver|package>
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3dt.dtx 3251 2012-01-28 09:42:44Z joseph $
+ {L3 Experimental data tables}
+%</driver|package>
+%<*driver>
+\documentclass[full]{l3doc}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+% \title{^^A
+% The \pkg{l3dt} package\\ Data tables^^A
+% \thanks{This file describes v\ExplFileVersion,
+% last revised \ExplFileDate.}^^A
+% }
+%
+% \author{^^A
+% The \LaTeX3 Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released \ExplFileDate}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% \LaTeX3 implements a \enquote{data table} variable type, which is made up of
+% a series of rows each of which contain a number of key--value pairs. Thus
+% a data table is in effect an array of property lists. The rows of the table
+% are stored in a fixed order, and are numbered consecutively from zero.
+% In the same way, the order of keys (columns) is recorded in a sequence-like
+% manner, again indexed from zero.
+%
+% Within each row in a data table each entry must have a unique \meta{key}: if
+% an entry is added to a row within a data table which already contains the
+% \meta{key} then the new entry will overwrite the existing one. The
+% \meta{keys} are compared on a string basis, using the same method as
+% \cs{str_if_eq:nn}.
+%
+% \section{Creating and initialising data tables}
+%
+% \begin{function}{\dt_new:N}
+% \begin{syntax}
+% \cs{dt_new:N} \meta{data table}
+% \end{syntax}
+% Creates a new \meta{data table} or raises an error if the name is
+% already taken. The declaration is global. The \meta{property lists} will
+% initially contain no entries.
+% \end{function}
+%
+% \begin{function}{\dt_clear:N, \dt_gclear:N}
+% \begin{syntax}
+% \cs{dt_clear:N} \meta{data table}
+% \end{syntax}
+% Clears all entries and keys from the \meta{data table}.
+% \end{function}
+%
+% \begin{function}{\dt_clear_new:N, \dt_gclear_new:N}
+% \begin{syntax}
+% \cs{dt_clear_new:N} \meta{data table}
+% \end{syntax}
+% Ensures that the \meta{data table} exists globally by applying
+% \cs{dt_new:N} if necessary, then applies \cs{dt_(g)clear:N} to leave
+% the table empty.
+% \end{function}
+%
+% \begin{function}{\dt_set_eq:NN, \dt_gset_eq:NN}
+% \begin{syntax}
+% \cs{dt_set_eq:NN} \meta{data table1} \meta{data table2}
+% \end{syntax}
+% Sets the content of \meta{data table1} equal to that of
+% \meta{data table2}.
+% \end{function}
+%
+% \section{Adding data}
+%
+% \begin{function}{\dt_add_key:Nn, \dt_gadd_key:Nn}
+% \begin{syntax}
+% \cs{dt_add_key:Nn} \meta{dt} \Arg{key}
+% \end{syntax}
+% Adds the \meta{key} to the list of those in the \meta{data table}. The
+% \meta{key} will be converted to a string using \cs{tl_to_str:n}, and thus
+% category codes in the \meta{key} are ignored. If the \meta{key} is already
+% present in the \meta{data table} then no action is taken.
+% \end{function}
+%
+% \begin{function}{\dt_add_row:N, \dt_gadd_row:N}
+% \begin{syntax}
+% \cs{dt_add_row:N} \meta{dt}
+% \end{syntax}
+% Adds a new row to the \meta{data table}. This will initially contain
+% no entries: all keys will be be blank.
+% \end{function}
+%
+% \begin{function}{\dt_put:Nnn, \dt_gput:Nnn}
+% \begin{syntax}
+% \cs{dt_put:Nnn} \meta{dt} \Arg{key} \Arg{value}
+% \end{syntax}
+% Adds an entry to the current row of the \meta{data table} which may be
+% accessed using the \meta{key} and which has \meta{value}. Both the
+% \meta{key} and \meta{value} may contain any \meta{balanced text}. The
+% \meta{key} is stored after processing with \cs{tl_to_str:n}, meaning
+% that category codes are ignored. If the \meta{key} is already present
+% in the current row of the \meta{data table}, the existing entry is
+% overwritten by the new \meta{value}.
+% \end{function}
+%
+% \begin{function}{\dt_put:Nnnn, \dt_gput:Nnnn}
+% \begin{syntax}
+% \cs{dt_put:Nnnn} \meta{dt} \Arg{row} \Arg{key} \Arg{value}
+% \end{syntax}
+% Adds an entry to the \meta{row} of the \meta{data table} which may be
+% accessed using the \meta{key} and which has \meta{value}. Both the
+% \meta{key} and \meta{value} may contain any \meta{balanced text}. The
+% \meta{key} is stored after processing with \cs{tl_to_str:n}, meaning
+% that category codes are ignored. If the \meta{key} is already present
+% in the \meta{row} of the \meta{data table}, the existing entry is
+% overwritten by the new \meta{value}. The \meta{row} should be given as
+% an \meta{integer expression}.
+% \end{function}
+%
+% \section{Removing data}
+%
+% \begin{function}{\dt_del:Nn, \dt_gdel:Nn}
+% \begin{syntax}
+% \cs{dt_del:Nn} \meta{dt} \Arg{key}
+% \end{syntax}
+% Deletes any entry from the current row of the \meta{data table} with
+% the \meta{key}. The \meta{key} is compared after processing with
+% \cs{tl_to_str:n}, meaning that category codes are ignored. Deleting of
+% all entries from a row does not delete the row itself.
+% \end{function}
+%
+% \begin{function}{\dt_del:Nnn, \dt_gdel:Nnn}
+% \begin{syntax}
+% \cs{dt_del:Nnn} \meta{dt} \Arg{row} \Arg{key}
+% \end{syntax}
+% Deletes any entry from the \meta{row} of the \meta{data table} with
+% the \meta{key}. The \meta{key} is compared after processing with
+% \cs{tl_to_str:n}, meaning that category codes are ignored. The \meta{row}
+% may be given as an \meta{integer expression}. Deleting of
+% all entries from a row does not delete the row itself.
+% \end{function}
+%
+% \begin{function}{\dt_remove_key:Nn, \dt_gremove_key:Nn}
+% \begin{syntax}
+% \cs{dt_remove_key:N} \meta{data table} \Arg{key}
+% \end{syntax}
+% Removes the \meta{key} from the \meta{data table} if it is present.
+% The \meta{key} and any associated \meta{value} will be removed from any
+% row that it is found in.
+% \end{function}
+%
+% \begin{function}{\dt_remove_row:Nn, \dt_gremove_row:Nn}
+% \begin{syntax}
+% \cs{dt_remove_row:Nn} \meta{data table} \Arg{row}
+% \end{syntax}
+% Removes the \meta{row} (given as an \meta{integer expressions}) from the
+% \meta{data table}. The remaining rows of the table will be renumbered
+% such that they are sequential.
+% \end{function}
+%
+% \section{Recovering information}
+%
+% \begin{function}[EXP]{\dt_keys:N}
+% \begin{syntax}
+% \cs{dt_keys:N} \meta{dt}
+% \end{syntax}
+% Leaves the number of keys in the \meta{data table} in the input
+% stream as an \meta{integer denotation}.
+% \end{function}
+%
+% \begin{function}[EXP]{\dt_rows:N}
+% \begin{syntax}
+% \cs{dt_rows:N} \meta{dt}
+% \end{syntax}
+% Leaves the number of rows in the \meta{data table} in the input
+% stream as an \meta{integer denotation}.
+% \end{function}
+%
+% \begin{function}{\dt_get:NnN}
+% \begin{syntax}
+% \cs{dt_get:NnnN} \meta{dt} \Arg{key} \meta{tl var}
+% \end{syntax}
+% Recovers the \meta{value} stored with \meta{key} from the current row in
+% the \meta{data table}, and places this in the \meta{token list variable}.
+% If the \meta{key} is not found in the \meta{row} of the \meta{data table}
+% then the \meta{token list variable} will contain the special marker
+% \cs{q_no_value}. The \meta{token list variable} is set within the
+% current \TeX{} group. The \meta{row} should be given as an
+% \meta{integer expression}. See also \cs{dt_get:NnNTF}.
+% \end{function}
+%
+% \begin{function}[TF]{\dt_get:NnN}
+% \begin{syntax}
+% \cs{dt_get:NnnNTF} \meta{dt} \Arg{key} \meta{tl var} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Recovers the \meta{value} stored with \meta{key} from the current row in
+% the \meta{data table}, and places this in the \meta{token list variable}.
+% If the \meta{key} is not found in the \meta{row} of the \meta{data table}
+% then the \meta{token list variable} will contain the special marker
+% \cs{q_no_value}. The \meta{token list variable} is set within the
+% current \TeX{} group. The \meta{row} should be given as an
+% \meta{integer expression}. Once the \meta{token list variable} has been
+% assigned either the \meta{true code} or \meta{false code} will be left in
+% the input stream, depending on whether the \meta{key} was found.
+% See also \cs{dt_get:NnN}.
+% \end{function}
+%
+% \begin{function}{\dt_get:NnnN}
+% \begin{syntax}
+% \cs{dt_get:NnnN} \meta{dt} \Arg{row} \Arg{key} \meta{tl var}
+% \end{syntax}
+% Recovers the \meta{value} stored with \meta{key} from \meta{row} in the
+% \meta{data table}, and places this in the \meta{token list variable}. If
+% the \meta{key} is not found in the \meta{row} of the \meta{data table}
+% then the \meta{token list variable} will contain the special marker
+% \cs{q_no_value}. The \meta{token list variable} is set within the
+% current \TeX{} group. The \meta{row} should be given as an
+% \meta{integer expression}. See also \cs{dt_get:NnnNTF}.
+% \end{function}
+%
+% \begin{function}[TF]{\dt_get:NnnN}
+% \begin{syntax}
+% \cs{dt_get:NnnNTF} \meta{dt} \Arg{row} \Arg{key} \meta{tl var} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Recovers the \meta{value} stored with \meta{key} from \meta{row} in the
+% \meta{data table}, and places this in the \meta{token list variable}. If
+% the \meta{key} is not found in the \meta{row} of the \meta{data table}
+% then the \meta{token list variable} will contain the special marker
+% \cs{q_no_value}. The \meta{token list variable} is set within the
+% current \TeX{} group. The \meta{row} should be given as an
+% \meta{integer expression}. Once the \meta{token list variable} has been
+% assigned either the \meta{true code} or \meta{false code} will be left in
+% the input stream, depending on whether the \meta{key} was found.
+% See also \cs{dt_get:NnnN}.
+% \end{function}
+%
+% \section{Mapping to data tables}
+%
+% \begin{function}{\dt_map_variables:Nnn}
+% \begin{syntax}
+% \cs{dt_map_variables:Nnn} \meta{data table} \Arg{key--variable mapping} \Arg{code}
+% \end{syntax}
+% Applies the \meta{code} to each \meta{row} of the \meta{data table}. The
+% \meta{keys} of the \meta{data table} are mapped to variables by the
+% \meta{key--variable mapping}, which should be a key--value list of the form
+% \begin{verbatim}
+% key-a = \l_a_tl ,
+% key-b = \l_b_tl
+% ...
+% \end{verbatim}
+% It is not necessary to map all of the \meta{keys} in a \meta{data table}
+% to variables. If there is not \meta{value} for a \meta{key} in a row,
+% the variable will contain the marker \cs{q_no_value}. Assignment of the
+% \meta{variables} is local to the current \TeX{} group. The mapping
+% to rows is ordered.
+% \end{function}
+%
+% \begin{variable}{\g_dt_map_level_int}
+% The nesting level of the data table mapping is available as
+% \cs{g_dt_map_level_int}. Within a mapping, the \texttt{int} variable
+% \cs{l_dt_map_\meta{level}_row_int} is available so that the row
+% number being operated on is available. Thus
+% \begin{verbatim}
+% \int_use:c { l_dt_map_ \int_use:N \g_dt_map_level_int _row_int }
+% \end{verbatim}
+% will give the current row for the current mapping.
+% \end{variable}
+%
+% \begin{function}[rEXP]{\dt_map_break:, \dt_map_break:n}
+% \begin{syntax}
+% \cs{dt_map_break:}
+% \cs{dt_map_break:n} \Arg{tokens}
+% \end{syntax}
+% Used to terminate a \cs{dt_map_\ldots} function before all
+% entries in the \meta{data table} have been processed. This will
+% normally take place within a conditional statement, for example
+% \begin{verbatim}
+% \dt_map_variables:Nn \l_my_dt { a = \l_my_tl }
+% {
+% \str_if_eq:VnTF \l_my_tl { bingo }
+% { \dt_map_break: }
+% {
+% % Do something useful
+% }
+% }
+% \end{verbatim}
+% The \texttt{:n} variant will insert the \meta{tokens} into the input stream
+% after the mapping terminates. Use outside of a \cs{dt_map_\ldots} scenario
+% will lead low level \TeX{} errors.
+% \end{function}
+%
+% \section{Data table conditionals}
+%
+% \begin{function}[EXP, pTF]{\dt_if_empty:N}
+% \begin{syntax}
+% \cs{dt_if_empty_p:N} \meta{dt}
+% \cs{dt_if_empty:NTF} \meta{dt} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{dt} is empty, containing no keys and no rows.
+% \end{function}
+%
+% \begin{function}[EXP, pTF]{\dt_if_in:Nn}
+% \begin{syntax}
+% \cs{dt_if_in_p:Nn} \meta{dt} \Arg{key}
+% \cs{dt_if_in:NnTF} \meta{dt} \Arg{key} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{key} is present in the \meta{data table} at all,
+% \emph{i.e.} if it is one of the columns of the table. This test will be
+% \texttt{true} even if none of the rows contain an entry for the \meta{key}.
+% \end{function}
+%
+% \begin{function}[EXP, pTF]{\dt_if_in_row:Nnn}
+% \begin{syntax}
+% \cs{dt_if_in_row_p:Nnn} \meta{dt} \Arg{row} \Arg{key}
+% \cs{dt_if_in_row:NnnTF} \meta{dt} \Arg{row} \Arg{key} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{key} is present in the \meta{row} of the
+% \meta{data table}. The \meta{row} may be given as an \meta{integer
+% expression}.
+% \end{function}
+%
+% \begin{function}[EXP, pTF]{\dt_if_in_row:Nn}
+% \begin{syntax}
+% \cs{dt_if_in_row_p:Nn} \meta{dt} \Arg{key}
+% \cs{dt_if_in_row:NnTF} \meta{dt} \Arg{key} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests if the \meta{key} is present in the current row of
+% \meta{data table}.
+% \end{function}
+%
+% \section{Variables}
+%
+% \begin{variable}{\c_empty_dt}
+% A permanently empty data table.
+% \end{variable}
+%
+% \begin{variable}{\l_tmpa_dt, \l_tmpb_dt, \g_tmpa_dt, \g_tmpb_dt}
+% Scratch data tables for general use: these are never used by the kernel.
+% \end{variable}
+%
+% \section{Internal function}
+%
+% \begin{variable}{\q_dt, \q_dt_header, \q_dt_row}
+% Quarks used to construct the data table format.
+% \end{variable}
+%
+% \begin{function}[EXP]{\dt_split_header:NT}
+% \begin{syntax}
+% \cs{dt_split_header:Nn} \meta{dt} \Arg{code}
+% \end{syntax}
+% Splits the \meta{data table} into the header part (containing the total
+% number of rows present and the key list) and the body (containing the
+% rows). The \meta{code} is then inserted, and should absorb the two parts
+% of the split table as arguments.
+% \end{function}
+%
+% \begin{function}{\dt_split_key:nnTF}
+% \begin{syntax}
+% \cs{dt_split_key:nnTF} \Arg{row} \Arg{key} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Searches the \meta{row} for the \meta{key}, using the
+% comparison method as described for \cs{str_if_eq:nn}. If the \meta{key} is
+% present, the \meta{true code} is left in the input stream followed by
+% three \meta{balanced text} arguments
+% \begin{enumerate}
+% \item the partial \meta{row} for all keys before the \meta{key},
+% \item the \meta{value} for the \meta{key} and
+% \item the partial \meta{row} for all keys after the \meta{key}.
+% \end{enumerate}
+% Thus the \meta{true code} must absorb three arguments.
+% The two partial tables are structured such they may be recombined directly
+% to produce a valid row lacking the entry for the \meta{key}.
+%
+% If the \meta{key} is not found in the \meta{row}, then the \meta{false code}
+% is left in the input stream with no arguments.
+% \end{function}
+%
+% \begin{function}{\dt_split_key_list:NnTF}
+% \begin{syntax}
+% \cs{dt_split_key_list:NnTF} \meta{dt} \Arg{key} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Searches the key list of the \meta{dt} for the \meta{key}, using the
+% comparison method as described for \cs{str_if_eq:nn}. If the \meta{key} is
+% present, the \meta{true code} is left in the input stream followed by
+% five \meta{balanced text} arguments
+% \begin{enumerate}
+% \item the partial table for all keys before the \meta{key},
+% \item the \meta{id} for the \meta{key},
+% \item the \meta{type} for the \meta{key},
+% \item the \meta{header} for the \meta{key} and
+% \item the partial table for all keys after the \meta{key}.
+% \end{enumerate}
+% Thus the \meta{true code} must absorb five arguments.
+% The two partial tables are structured such they may be recombined directly
+% to produce a valid key table lacking the entry for the \meta{key}.
+%
+% If the \meta{key} is not found in the key table, then the \meta{false code}
+% is left in the input stream with no arguments.
+% \end{function}
+%
+% \begin{function}{\dt_split_row:NnTF}
+% \begin{syntax}
+% \cs{dt_split_row:NnTF} \meta{dt} \Arg{row} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Searches the key list of the \meta{dt} for the \meta{row}
+% (an \meta{integer expression}). If the
+% \meta{row} is present, the \meta{true code} is left in the input stream
+% followed by three \meta{balanced text} arguments
+% \begin{enumerate}
+% \item the partial table before the \meta{row},
+% \item the content of the \meta{row}, starting and ending with the
+% row number,
+% \item the partial table after the \meta{row}.
+% \end{enumerate}
+% Thus the \meta{true code} must absorb three arguments.
+% The two partial tables are structured such they may be recombined directly
+% to produce a valid key table lacking the entry for the \meta{row}.
+%
+% If the \meta{row} is not found in the key table, then the \meta{false code}
+% is left in the input stream with no arguments.
+% \end{function}
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3dt} implementation}
+%
+% \begin{macrocode}
+%<*initex|package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*package>
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\package_check_loaded_expl:
+%</package>
+% \end{macrocode}
+%
+% \subsection{Structures}
+%
+% The structure of a data table must allow each row (record) to contain only
+% some of the keys, and for the keys to be removed after the table
+% is initialised. It also needs to ensure that a unique match can be made to
+% every item in the table. At the same time, it is desirable to keep all of
+% the information about the table in a single \TeX{} macro. This can be
+% achieved by packing the data into a structure in which each key and row is
+% numbered:
+% \begin{quote}
+% \Arg{rows} \\
+% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \meta{key$_1$} \cs{q_dt} \ldots \\
+% \cs{q_nil} \\
+% \cs{q_dt_header} \\
+% \cs{q_dt_row} \\
+% \meta{row$_0$} \\
+% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{\0,\0}$} \\
+% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{\0,\1}$} \\
+% \ldots \\
+% \cs{q_dt} \\
+% \cs{q_nil} \\
+% \cs{q_dt_row} \\
+% \meta{row$_2$} \\
+% \cs{q_dt} \meta{key$_0$} \cs{q_dt} \Arg{data$_{\1,\0}$} \\
+% \cs{q_dt} \meta{key$_1$} \cs{q_dt} \Arg{data$_{\1,\1}$} \\
+% \ldots \\
+% \cs{q_dt} \\
+% \cs{q_nil} \\
+% \cs{q_dt_row} \\
+% \ldots \\
+% \cs{q_dt_row}
+% \end{quote}
+%
+% \begin{variable}{\q_dt, \q_dt_row, \q_dt_header}
+% The quarks are set up.
+% \begin{macrocode}
+\quark_new:N \q_dt
+\quark_new:N \q_dt_row
+\quark_new:N \q_dt_header
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\c_empty_dt}
+% A permanently-empty data table, which therefore contains only the minimum
+% number of items necessary to comply with the structure above.
+% \begin{macrocode}
+\tl_const:Nn \c_empty_dt
+ {
+ { 0 }
+ \q_dt
+ \q_nil
+ \q_dt_header
+ \q_dt_row
+ }
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Allocation and initialisation}
+%
+% \begin{macro}{\dt_new:N}
+% Internally, data tables are token lists, but an empty dt
+% is not an empty tl.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_new:N #1 { \cs_new_eq:NN #1 \c_empty_dt }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\dt_clear:N, \dt_gclear:N}
+% The same idea for clearing.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_clear:N #1 { \cs_set_eq:NN #1 \c_empty_dt }
+\cs_new_protected:Npn \dt_gclear:N #1 { \cs_gset_eq:NN #1 \c_empty_dt }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {\dt_clear_new:N, \dt_gclear_new:N}
+% Once again a simple copy from the token list functions.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_clear_new:N #1
+ { \cs_if_exist:NTF #1 { \dt_clear:N #1 } { \dt_new:N #1 } }
+\cs_new_protected:Npn \dt_gclear_new:N #1
+ { \cs_if_exist:NTF #1 { \dt_gclear:N #1 } { \dt_new:N #1 } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\dt_set_eq:NN, \dt_gset_eq:NN}
+% Once again, these are simply copies from the token list functions.
+% \begin{macrocode}
+\cs_new_eq:NN \dt_set_eq:NN \tl_set_eq:NN
+\cs_new_eq:NN \dt_gset_eq:NN \tl_gset_eq:NN
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}{\l_tmpa_dt, \l_tmpb_dt, \g_tmpa_dt, \g_tmpb_dt}
+% Scratch tables.
+% \begin{macrocode}
+\dt_new:N \l_tmpa_dt
+\dt_new:N \l_tmpb_dt
+\dt_new:N \g_tmpa_dt
+\dt_new:N \g_tmpb_dt
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Splitting functions}
+%
+% \begin{macro}[aux]{\dt_split_aux:nnnn}
+% \begin{macro}[aux]{\dt_split_aux:w}
+% Two general auxiliaries. The \texttt{nnnn} function is used to apply the
+% \texttt{T} branch if a match is found and the \texttt{F} branch otherwise.
+% The \texttt{w} function is general purpose, and is used to define the
+% matching parameter set.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_split_aux:nnnn #1#2#3#4 { #3 #2 }
+\cs_new_protected:Npn \dt_split_aux:w { }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int,EXP]{\dt_split_header:NT}
+% \begin{macro}[aux,EXP]{\dt_split_header_aux:wn}
+% Splits the header from the table, inserting the code required to then
+% process the split table. The \cs{q_nil} is also removed from the end of the
+% header, as it is essentially a distraction here.
+% \begin{macrocode}
+\cs_new:Npn \dt_split_header:NT #1#2
+ { \exp_after:wN \dt_split_header_aux:wn #1 \q_stop {#2} }
+\cs_new:Npn \dt_split_header_aux:wn #1 \q_nil \q_dt_header #2 \q_stop #3
+ { #3 {#1} { \q_dt_header #2 } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\dt_split_key:nnTF}
+% \begin{macro}[aux]{\dt_split_key_aux:nnTF}
+% Here, the split is made for a partial list within a row. The row is
+% basically the same as a property list, so the split here is almost
+% identical to that in \cs{prop_split_aux:NnTF}. The row-end data is set up
+% such that it will not interfere with this process.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_split_key:nnTF #1#2
+ { \exp_args:No \dt_split_key_aux:nnTF { \tl_to_str:n {#2} } {#1} }
+\cs_new_protected:Npn \dt_split_key_aux:nnTF #1#2
+ {
+ \cs_set_protected:Npn \dt_split_aux:w
+ ##1 \q_dt #1 \q_dt ##2##3##4 \q_mark ##5 \q_stop
+ { \dt_split_aux:nnnn ##3 { { ##1 \q_dt } {##2} {##4} } }
+ \dt_split_aux:w #2 \q_mark
+ \q_dt #1 \q_dt { } { ? \use_ii:nn { } } \q_mark \q_stop
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\dt_split_key_list:NnTF}
+% \begin{macro}[aux]{\dt_split_key_list_aux:NnTF}
+% Finding a key in the header uses a similar approach to finding a key in
+% a property list. Here, if the key is found there will always be at least
+% one token between \cs{q_dt} and \cs{q_dt_header} due to the \cs{q_nil}
+% which is part of a new table. The use of |##1##2| in \cs{dt_split_aux:w}
+% here is to deal with the overall number of rows. The set up here means
+% that this will always be unbraced then rebraced: simply grabbing |##1|
+% to include this and anything before the key of interest will give variable
+% results depending on whether the match is to the very first key or not.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_split_key_list:NnTF #1#2
+ { \exp_args:NNo \dt_split_key_list_aux:NnTF #1 { \tl_to_str:n {#2} } }
+\cs_new_protected:Npn \dt_split_key_list_aux:NnTF #1#2
+ {
+ \cs_set_protected:Npn \dt_split_aux:w
+ ##1##2 \q_dt #2 \q_dt ##3##4 \q_dt_header ##5 \q_mark ##6 \q_stop
+ {
+ \dt_split_aux:nnnn ##3
+ { { {##1} ##2 \q_dt } { ##3##4 \q_dt_header ##5 } }
+ }
+ \exp_after:wN \dt_split_aux:w #1 \q_mark
+ \q_dt #2 \q_dt { ? \use_ii:nn { } } \q_dt_header \q_mark \q_stop
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\dt_split_row:NnTF}
+% \begin{macro}[aux]{\dt_split_row_aux:NnTF, \dt_split_row_aux:NfTF}
+% The usual approach, here using the fact that each row start with row number
+% and ends with \cs{q_nil} so there will always be at least one token to be
+% absorbed as |##2|. The only odd thing to watch here is that the row
+% number is evaluated so that higher-level functions in the main do not
+% need to have an \texttt{f}-type variant.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_split_row:NnTF #1#2
+ { \dt_split_row_aux:NfTF #1 { \int_eval:n {#2} } }
+\cs_new_protected:Npn \dt_split_row_aux:NnTF #1#2
+ {
+ \cs_set_protected:Npn \dt_split_aux:w
+ ##1 \q_dt_row #2 \q_dt ##2##3 \q_dt_row ##4 \q_mark ##5 \q_stop
+ {
+ \dt_split_aux:nnnn ##2
+ { { ##1 \q_dt_row } { #2 \q_dt ##2##3 } {##4} }
+ }
+ \exp_after:wN \dt_split_aux:w #1 \q_mark
+ \q_dt_row #2 \q_dt { ? \use_ii:nn { } } \q_dt_row \q_mark \q_stop
+ }
+\cs_generate_variant:Nn \dt_split_row_aux:NnTF { Nf }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Adding and removing data}
+%
+% \begin{macro}{\dt_add_key:Nn, \dt_gadd_key:Nn}
+% \begin{macro}[aux]{\dt_add_key_aux:NNn}
+% \begin{macro}[aux]{\dt_add_key_aux:NNnnn}
+% Here, there are two stages. If the key is already present in the list of
+% known keys then no action is taken, and the split list is thrown away.
+% On the other hand, if the key is not present then the header and body
+% are separated and the key is added to the end of the list of known keys
+% (hence keys are ordered). The \cs{dt_split_header:Nn} function will have
+% removed the \cs{q_nil} from the header, and so it is put back in here.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \dt_add_key:Nn { \dt_add_key_aux:NNn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gadd_key:Nn { \dt_add_key_aux:NNn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_add_key_aux:NNn #1#2#3
+ {
+ \dt_split_key_list:NnTF #2 {#3}
+ { \use_none:nn }
+ {
+ \dt_split_header:NT #2
+ { \dt_add_key_aux:NNnnn #1 #2 {#3} }
+ }
+ }
+\cs_new_protected:Npn \dt_add_key_aux:NNnnn #1#2#3#4#5
+ {
+ #1 #2
+ {
+ \exp_not:n {#4}
+ \tl_to_str:n {#3}
+ \exp_not:n { \q_dt \q_nil #5 }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\dt_add_row:N, \dt_gadd_row:N}
+% \begin{macro}[aux]{\dt_add_row_aux:NN}
+% \begin{macro}[aux]{\dt_add_row_aux:NnN}
+% \begin{macro}[aux,EXP]{\dt_add_row_aux:nw}
+% Adding a row means incrementing the total number and adding the structure
+% of an empty row. As finding the rows will get slow for large tables, this
+% is only done once.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \dt_add_row:N { \dt_add_row_aux:NN \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gadd_row:N { \dt_add_row_aux:NN \tl_gset:Nx }
+\cs_new_protected:Npn \dt_add_row_aux:NN #1#2
+ { \exp_args:NNf \dt_add_row_aux:NnN #1 { \dt_rows:N #2 } #2 }
+\cs_new_protected:Npn \dt_add_row_aux:NnN #1#2#3
+ {
+ #1 #3
+ {
+ { \int_eval:n { #2 + \c_one } }
+ \exp_after:wN \dt_add_row_aux:nw #3 \q_stop
+ #2
+ \exp_not:n { \q_dt \q_nil \q_dt_row }
+ }
+ }
+\cs_new:Npn \dt_add_row_aux:nw #1#2 \q_stop { \exp_not:n {#2} }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\dt_put:Nnn, \dt_gput:Nnn}
+% Adding to the current row is simply a special case of adding to an
+% arbitrary row.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_put:Nnn #1
+ { \dt_put:Nnnn #1 { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_gput:Nnn #1
+ { \dt_gput:Nnnn #1 { \dt_rows:N #1 - \c_one } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\dt_put:Nnnn, \dt_gput:Nnnn}
+% \begin{macro}[aux]{\dt_put_aux:NNNnnn}
+% \begin{macro}[aux]{\dt_put_aux:NNnnnnn}
+% \begin{macro}[aux]{\dt_put_update:NNnnnnnnn}
+% \begin{macro}[aux]{\dt_put_add_to_row:NNnnnnn}
+% \begin{macro}[aux, EXP]{\dt_put_add_to_row_aux:w}
+% Adding to a row is a slightly complex procedure. The lead-off is the
+% standard combination across the local and global routes.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \dt_put:Nnnn
+ { \dt_put_aux:NNNnnn \dt_add_key:Nn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gput:Nnnn
+ { \dt_put_aux:NNNnnn \dt_gadd_key:Nn \tl_gset:Nx }
+% \end{macrocode}
+% Add the key to the list those known, if necessary, then check that the
+% row requested makes sense.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_put_aux:NNNnnn #1#2#3#4#5#6
+ {
+ #1 #3 {#5}
+ \dt_split_row:NnTF #3 {#4}
+ { \dt_put_aux:NNnnnnn #2 #3 {#5} {#6} }
+ {
+ \msg_kernel_error:nnxxx { dt } { unknown-row }
+ { \token_to_str:N #3 } { \int_eval:n {#4} } { \dt_rows:N #3 }
+ }
+ }
+% \end{macrocode}
+% At this stage, the arguments are
+% \begin{enumerate}
+% \item the set function \cs{tl_(g)set:Nx},
+% \item the data table,
+% \item the key,
+% \item the value,
+% \item the data table before the row,
+% \item the extracted data table row,
+% \item the data table after the row.
+% \end{enumerate}
+% Splitting on the key will then leave three further items in the input
+% stack if the key is already present. So there is some care needed sending
+% the parameters forward without running out of \TeX{} arguments.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_put_aux:NNnnnnn #1#2#3#4#5#6#7
+ {
+ \dt_split_key:nnTF {#6} {#3}
+ { \dt_put_update:NNnnnnnnn #1 #2 {#3} {#4} {#5} {#7} }
+ { \dt_put_add_to_row:NNnnnnn #1 #2 {#3} {#4} {#5} {#6} {#7} }
+ }
+% \end{macrocode}
+% The arguments here are
+% \begin{enumerate}
+% \item the set function \cs{tl_(g)set:Nx},
+% \item the data table,
+% \item the key,
+% \item the value,
+% \item the data table before the row,
+% \item the data table after the row,
+% \item the row before the key,
+% \item the current value for the key
+% \item the row after the key.
+% \end{enumerate}
+% What happens here is a reconstruction of the table: everything except
+% |#8| is needed. To try to keep things clear, there are a few more
+% \cs{exp_not:n} here than formally required.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_put_update:NNnnnnnnn #1#2#3#4#5#6#7#8#9
+ {
+ #1 #2
+ {
+ \exp_not:n { #5 #7 }
+ \tl_to_str:n {#3}
+ \exp_not:n { \q_dt {#4} \q_dt #9 \q_dt_row #6 }
+ }
+ }
+% \end{macrocode}
+% A slightly more complex case when adding an item. The arguments here are
+% identical to those for \cs{dt_put_aux:NNnnnnnn}. The row has not been
+% split, so the \cs{q_nil} there is removed and re-added to come after the
+% new content.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_put_add_to_row:NNnnnnn #1#2#3#4#5#6#7
+ {
+ #1 #2
+ {
+ \exp_not:n {#5}
+ \exp_not:o { \dt_put_add_row_aux:w #6 }
+ \tl_to_str:n {#3}
+ \exp_not:n { \q_dt {#4} \q_dt \q_nil \q_dt_row #7 }
+ }
+ }
+\cs_new:Npn \dt_put_add_row_aux:w #1 \q_nil {#1}
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[EXP]{\dt_keys:N}
+% \begin{macro}[aux, EXP]{\dt_keys_aux:nn}
+% \begin{macro}[aux, EXP]{\dt_keys_aux:wN}
+% A quick mapping is needed to count keys. The \cs{use_none:nn} here
+% is used to remove the number of rows and initial \cs{q_dt}. This could
+% also be handled by starting from $-1$ rather than $0$, but this makes
+% the logic hopefully slightly clearer.
+% \begin{macrocode}
+\cs_new:Npn \dt_keys:N #1
+ { \dt_split_header:NT #1 { \dt_keys_aux:nn } }
+\cs_new:Npn \dt_keys_aux:nn #1#2
+ {
+ \int_eval:n
+ {
+ 0
+ \exp_after:wN \dt_keys_aux:wN \use_none:nn #1 \q_recursion_tail \q_dt
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \dt_keys_aux:wN #1 \q_dt
+ {
+ \if_meaning:w \q_recursion_tail #1
+ \exp_after:wN \prg_map_break:
+ \fi:
+ +1
+ \dt_keys_aux:wN
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[EXP]{\dt_rows:N}
+% The number of rows in a dt is the very first entry.
+% \begin{macrocode}
+\cs_new:Npn \dt_rows:N #1
+ { \exp_after:wN \use_i_delimit_by_q_stop:nw #1 \q_stop }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Removing data}
+%
+% \begin{macro}{\dt_del:Nn,\dt_gdel:Nn}
+% Deleting to the current row is simply a special case of deleting to an
+% arbitrary row.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_del:Nn #1 { \dt_del:Nnn #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_gdel:Nn #1 { \dt_gdel:Nnn #1
+ { \dt_rows:N #1 - \c_one } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\dt_del:Nnn, \dt_gdel:Nnn}
+% \begin{macro}[aux]{\dt_del_aux:NNnn}
+% \begin{macro}[aux]{\dt_del_aux:NNnnnn}
+% \begin{macro}[aux]{\dt_del_aux:NNnnnnn}
+% Deleting a single entry from a single row means first splitting by row,
+% then splitting by key, and finally doing the assignment. If the row or the
+% key are not present then the entire function does nothing at all.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \dt_del:Nnn { \dt_del_aux:NNnn \tl_set:Nn }
+\cs_new_protected_nopar:Npn \dt_gdel:Nnn { \dt_del_aux:NNnn \tl_gset:Nn }
+\cs_new_protected:Npn \dt_del_aux:NNnn #1#2#3#4
+ {
+ \dt_split_row:NnTF #2 {#3}
+ { \dt_del_aux:NNnnnn #1 #2 {#4} }
+ { }
+ }
+\cs_new_protected:Npn \dt_del_aux:NNnnnn #1#2#3#4#5#6
+ {
+ \dt_split_key:nnTF {#5} {#3}
+ { \dt_del_aux:NNnnnnn #1 #2 {#4} {#6} }
+ { }
+ }
+\cs_new_protected:Npn \dt_del_aux:NNnnnnn #1#2#3#4#5#6#7
+ { #1 #2 { #3 #5 #7 #4 } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\dt_remove_key:Nn, \dt_gremove_key:Nn}
+% \begin{macro}[aux]{\dt_remove_key_aux:NNn}
+% \begin{macro}[aux]{\dt_remove_key_aux:nNNnn}
+% \begin{macro}[aux, EXP]{\dt_remove_key_aux:w}
+% Deleting a key also removes from the table itself, so that there is no
+% need to do any awkward checks when extracting data from the table. (It's
+% likely that there will be more cases of accessing data than deleting
+% rows). The deletion mapping ignores rows entirely and just pulls out
+% matching key--value pairs, as this reduces the number of matches needed
+% to a minimum.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \dt_remove_key:Nn
+ { \dt_remove_key_aux:NNn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gremove_key:Nn
+ { \dt_remove_key_aux:NNn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_remove_key_aux:NNn #1#2#3
+ {
+ \dt_split_key_list:NnTF #2 {#3}
+ { \exp_args:No \dt_remove_key_aux:nNNnn { \tl_to_str:n {#3} } #1 #2 }
+ { }
+ }
+\cs_new_protected:Npn \dt_remove_key_aux:nNNnn #1#2#3#4#5
+ {
+ \cs_set:Npn \dt_remove_key_aux:w ##1 \q_dt #1 \q_dt ##2 ##3
+ {
+ \exp_not:n {##1}
+ \if_meaning:w \q_recursion_tail ##3
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \dt_remove_key_aux:w ##3
+ }
+ #2 #3
+ {
+ \exp_not:n {#4}
+ \dt_remove_key_aux:w #5 \q_dt #1 \q_dt { } \q_recursion_tail
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \dt_remove_key_aux:w { }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\dt_remove_row:Nn, \dt_gremove_row:Nn}
+% \begin{macro}[aux]{\dt_remove_row_aux:NNn}
+% \begin{macro}[aux]{\dt_remove_row_aux:NNnnnn}
+% \begin{macro}[aux, EXP]{\dt_remove_row_aux:nw}
+% \begin{macro}[aux, EXP]{\dt_remove_row_loop:nw}
+% Removing a row is a slightly complex operation as there are two stages.
+% The row itself is easy enough to remove, but then all later rows have to
+% be renumbers.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \dt_remove_row:Nn
+ { \dt_remove_row_aux:NNn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gremove_row:Nn
+ { \dt_remove_row_aux:NNn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_remove_row_aux:NNn #1#2#3
+ {
+ \dt_split_row:NnTF #2 {#3}
+ { \dt_remove_row_aux:NNnnn #1 #2 }
+ { }
+ }
+% \end{macrocode}
+% If the code gets here, then |#3| is the table before the removed row,
+% |#4| is the removed row and |#5| is everything afterwards. The first stage
+% is to work out the new number of rows, then include all of |#3| except
+% the old number of rows. The removed row |#4| is thrown away, and then there
+% is a loop to recalculate the row numbers for all of the later rows.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_remove_row_aux:NNnnn #1#2#3#4#5
+ {
+ #1 #2
+ {
+ { \int_eval:n { \dt_rows:N #2 - \c_one } }
+ \dt_remove_row_aux:nw #3 \q_stop
+ \dt_remove_row_loop:nw #5 \q_recursion_tail \q_dt_row
+ \prg_break_point:n { }
+ }
+ }
+\cs_new_eq:NN \dt_remove_row_aux:nw \dt_add_row_aux:nw
+\cs_new:Npn \dt_remove_row_loop:nw #1#2 \q_dt_row
+ {
+ \if_meaning:w \q_recursion_tail #1
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \int_eval:n { #1 - \c_one }
+ \exp_not:n { #2 \q_dt_row }
+ \dt_remove_row_loop:nw
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Accessing data in data tables}
+%
+% \begin{macro}{\dt_get:NnnN}
+% \begin{macro}[aux]{\dt_get_aux:nNnnn}
+% \begin{macro}[aux]{\dt_get_aux:nNnnn}
+% Recovering a value from a row means doing two splits: first find the row,
+% then find the key. Nothing exciting, just a question of tracking the
+% returned items.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_get:NnnN #1#2#3#4
+ {
+ \dt_split_row:NnTF #1 {#2}
+ { \dt_get_aux:nNnnn {#3} #4 }
+ { \tl_set:Nn #4 { \q_no_value } }
+ }
+\cs_new_protected:Npn \dt_get_aux:nNnnn #1#2#3#4#5
+ {
+ \dt_split_key:nnTF {#4} {#1}
+ { \dt_get_aux:Nnnn #2 }
+ { \tl_set:Nn #2 { \q_no_value } }
+ }
+\cs_new_protected:Npn \dt_get_aux:Nnnn #1#2#3#4 { \tl_set:Nn #1 {#3} }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[TF]{\dt_get:NnnN}
+% \begin{macro}[aux]{\dt_get_aux_true:nNnnn}
+% \begin{macro}[aux]{\dt_get_aux_true:Nnnn}
+% The same idea as the standard method, but built as a conditional.
+% \begin{macrocode}
+\prg_new_protected_conditional:Npnn \dt_get:NnnN #1#2#3#4 { T , F , TF }
+ {
+ \dt_split_row:NnTF #1 {#2}
+ { \dt_get_aux_true:nNnnn {#3} #4 }
+ { \prg_return_false: }
+ }
+\cs_new_protected:Npn \dt_get_aux_true:nNnnn #1#2#3#4#5
+ {
+ \dt_split_key:nnTF {#4} {#1}
+ { \dt_get_aux_true:Nnnn #2 }
+ { \prg_return_false: }
+ }
+\cs_new_protected:Npn \dt_get_aux_true:Nnnn #1#2#3#4
+ {
+ \tl_set:Nn #1 {#3}
+ \prg_return_true:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\dt_get:NnN}
+% \begin{macro}[TF]{\dt_get:NnN}
+% Simple wrappers.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_get:NnN #1 { \dt_get:NnnN #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_get:NnNT #1 { \dt_get:NnnNF #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_get:NnNF #1 { \dt_get:NnnNF #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_get:NnNTF #1 { \dt_get:NnnNTF #1
+ { \dt_rows:N #1 - \c_one } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Mapping to data tables}
+%
+% \begin{variable}{\g_dt_map_level_int}
+% Unlike other mappings, the mapping level here has to be available
+% and so linked to the module.
+% \begin{macrocode}
+\int_new:N \g_dt_map_level_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}{\dt_map_variables:Nnn}
+% \begin{macro}[aux]{\dt_map_variables_key:nn}
+% \begin{macro}[aux]{\dt_map_variables_aux:nnn}
+% \begin{macro}[aux]{\dt_map_variables_aux:nNNw}
+% \begin{macro}[aux]{\dt_map_variables_aux:nnw}
+% Mapping across a data table is more complex than other cases as there
+% are two \enquote{dimensions} to worry about: the rows and the keys.
+% The first stage of the mapping is to convert the key--variable mapping
+% into a sequence that can be used later. This is done with the assumption
+% that any key without a variable can simply be dropped entirely. The
+% header of the table is then split from the body.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_map_variables:Nnn #1#2#3
+ {
+ \int_gincr:N \g_dt_map_level_int
+ \seq_gclear_new:c { g_dt_map_ \int_use:N \g_dt_map_level_int _seq }
+ \keyval_parse:NNn \use_none:n \dt_map_variables_key:nn {#2}
+ \dt_split_header:NT #1 { \dt_map_variables_aux:nnn {#3} }
+ }
+\cs_new_protected:Npn \dt_map_variables_key:nn #1#2
+ {
+ \seq_gput_right:cn { g_dt_map_ \int_use:N \g_dt_map_level_int _seq }
+ { {#1} #2 }
+ }
+% \end{macrocode}
+% As \cs{dt_split_header:NT} will leave a couple of tokens at the front
+% of the body part of the split, there is a quick piece of tidying up
+% to remove them.
+% \begin{macrocode}
+\cs_new_protected:Npn \dt_map_variables_aux:nnn #1#2#3
+ { \dt_map_variables_aux:nNNw {#1} #3 \q_stop }
+\cs_new_protected:Npn \dt_map_variables_aux:nNNw
+ #1 \q_dt_header \q_dt_row #2 \q_stop
+ {
+ \int_zero_new:c { l_dt_map_ \int_use:N \g_dt_map_level_int _row_int }
+ \dt_map_variables_aux:nnw {#1} #2 { } \q_recursion_tail \q_dt_row
+ \prg_break_point:n { \int_gdecr:N \g_dt_map_level_int }
+ }
+\cs_new_protected:Npn \dt_map_variables_aux:nnw #1#2#3#4 \q_dt_row
+ {
+ \if_meaning:w \q_recursion_tail #3
+ \exp_after:wN \dt_map_break:
+ \fi:
+ \seq_map_inline:cn { g_dt_map_ \int_use:N \g_dt_map_level_int _seq }
+ { \dt_get_aux:nNnnn ##1 { } {#3#4} { } }
+ #1
+ \int_incr:c { l_dt_map_ \int_use:N \g_dt_map_level_int _row_int }
+ \dt_map_variables_aux:nnw {#1}
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[rEXP]{\dt_map_break:}
+% \begin{macro}[rEXP]{\dt_map_break:n}
+% The break statements are simply copies.
+% \begin{macrocode}
+\cs_new_eq:NN \dt_map_break: \prg_map_break:
+\cs_new_eq:NN \dt_map_break:n \prg_map_break:n
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Data table conditionals}
+%
+% \begin{macro}[pTF, EXP]{\dt_if_empty:N}
+% An empty data table has not only no rows but also no keys. (The number of
+% rows can be tested using \cs{dt_rows:N} and an \texttt{int} test.)
+% \begin{macrocode}
+\prg_new_conditional:Npnn \dt_if_empty:N #1 { T , F , TF , p }
+ {
+ \if_meaning:w #1 \c_empty_dt
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[pTF, EXP]{\dt_if_in:Nn}
+% \begin{macro}[aux, EXP]{\dt_if_in_aux:nnn}
+% \begin{macro}[aux, EXP]{\dt_if_in_aux:nwN}
+% \begin{macro}[aux, EXP]{\dt_if_in_aux:n}
+% Expandably checking for the presence of a key in the table as a whole
+% requires a mapping to the header. The idea is the usual recursion set
+% up with a string-based comparison only after checking for the end of
+% the loop.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \dt_if_in:Nn #1#2 { p , T , F , TF }
+ { \dt_split_header:NT #1 { \dt_if_in_aux:nnn {#2} } }
+\cs_new:Npn \dt_if_in_aux:nnn #1#2#3
+ {
+ \exp_last_unbraced:Nno \dt_if_in_aux:nwN {#1} { \use_none:nn #2 }
+ \q_recursion_tail \q_dt
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \dt_if_in_aux:nwN #1#2 \q_dt
+ {
+ \if_meaning:w \q_recursion_tail #2
+ \exp_after:wN \prg_map_break:n
+ \else:
+ \exp_after:wN \use_none:n
+ \fi:
+ { \prg_return_false: }
+ \str_if_eq:nnTF {#1} {#2}
+ { \prg_map_break:n { \prg_return_true: } }
+ { \dt_if_in_aux:nwN {#1} }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[pTF, EXP]{\dt_if_in_row:Nnn}
+% \begin{macro}[aux, EXP]{\dt_if_in_row_aux:nw}
+% \begin{macro}[aux, EXP]{\dt_if_in_row_aux:nn}
+% \begin{macro}[aux, EXP]{\dt_if_in_row_aux:nwn}
+% \begin{macro}[aux, EXP]{\dt_if_in_row_aux:N}
+% Finding a key in a single row in an expandable way requires two mappings.
+% To start of with, there is a search for the row. This uses for termination
+% the fact that each row starts \cs{q_dt_row} and ends \cs{q_nil}, and
+% always contains at least the row number as the first \meta{balanced
+% text}. That can be replaced by the tail marker to terminate iteration:
+% all that is then needed is the correct placement of the clean-up code.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \dt_if_in_row:Nnn #1#2#3 { p , T , F , TF }
+ {
+ \exp_last_unbraced:Nno \dt_if_in_row_aux:nw {#2} #1
+ \q_recursion_tail \q_nil
+ \prg_break_point:n { }
+ { \tl_to_str:n {#3} }
+ }
+% \end{macrocode}
+% The row iteration does a numerical comparison to see if the target row has
+% been found. That means that the row argument does not need to be converted
+% to a number earlier.
+% \begin{macrocode}
+\cs_new:Npn \dt_if_in_row_aux:nw #1#2 \q_dt_row #3#4 \q_nil
+ {
+ \if_meaning:w \q_recursion_tail #3
+ \exp_after:wN \prg_map_break:n
+ \else:
+ \exp_after:wN \use_none:n
+ \fi:
+ {
+ \use_i:nn
+ \prg_return_false:
+ }
+ \int_compare:nNnTF {#1} = {#3}
+ { \prg_map_break:n { \exp_args:Nno \dt_if_in_row_aux:nn {#4} } }
+ { \dt_if_in_row_aux:nw {#1} }
+ }
+% \end{macrocode}
+% The second iteration is along the row. This is basically the same as
+% \cs{prop_if_in:NnTF} with the \cs{q_dt} in place of \cs{q_prop}.
+% \begin{macrocode}
+\cs_new:Npn \dt_if_in_row_aux:nn #1#2
+ {
+ \dt_if_in_row_aux:nwn {#2} #1 {#2} \q_dt { } \q_recursion_tail
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \dt_if_in_row_aux:nwn #1 \q_dt #2 \q_dt #3
+ {
+ \str_if_eq:xxTF {#1} {#2}
+ { \dt_if_in_row_aux:N }
+ { \dt_if_in_row_aux:nwn {#1} }
+ }
+\cs_new:Npn \dt_if_in_row_aux:N #1
+ {
+ \if_meaning:w \q_dt #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ \prg_map_break:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[pTF, EXP]{\dt_if_in_row:Nn}
+% Simple wrappers.
+% \begin{macrocode}
+\cs_new:Npn \dt_if_in_row_p:Nn #1 { \dt_if_in_row_p:Nnn #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new:Npn \dt_if_in_row:NnT #1 { \dt_if_in_row:NnnT #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new:Npn \dt_if_in_row:NnF #1 { \dt_if_in_row:NnnF #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new:Npn \dt_if_in_row:NnTF #1 { \dt_if_in_row:NnnTF #1
+ { \dt_rows:N #1 - \c_one } }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Messages}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnnn { dt } { unknown-row }
+ { Data~table~#1~does~not~contain~a~row~'#2'. }
+ {
+ Data~table~#1~contains~#3~rows.~These~must~be~accessed~by~number:~row~
+ #2~is~not~present~in~the~table.
+ }
+% \end{macrocode}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+% \PrintIndex \ No newline at end of file
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.ins b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.ins
new file mode 100644
index 00000000000..5b84dfca3dc
--- /dev/null
+++ b/Master/texmf-dist/source/latex/l3experimental/l3dt/l3dt.ins
@@ -0,0 +1,40 @@
+\iffalse meta-comment
+
+File l3dt.ins Copyright (C) 2011 The LaTeX3 Project
+
+It may be distributed and/or modified under the conditions of the
+LaTeX Project Public License (LPPL), either version 1.3c of this
+license or (at your option) any later version. The latest version
+of this license is in the file
+
+ http://www.latex-project.org/lppl.txt
+
+This file is part of the "l3trial bundle" (The Work in LPPL)
+and all files in that bundle must be distributed together.
+
+The released version of this bundle is available from CTAN.
+
+\fi
+
+\input docstrip.tex
+\askforoverwritefalse
+
+\preamble
+
+EXPERIMENTAL CODE
+
+Do not distribute this file without also distributing the
+source files specified above.
+
+Do not distribute a modified version of this file.
+
+\endpreamble
+% stop docstrip adding \endinput
+\postamble
+\endpostamble
+
+\keepsilent
+
+\generate{\file{l3dt.sty}{\from{l3dt.dtx}{package}}}
+
+\endbatchfile
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
index 46bd4c80cf7..717065d7d27 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3flag.dtx
@@ -69,64 +69,106 @@
%
% \begin{documentation}
%
-% Flags are the only data structure on which \TeX{} can perform
-% assignments in expansion-only contexts. This module is meant
-% mostly for kernel use: in almost all cases, booleans should
-% be preferred to flags, because they are faster.
-%
-% A flag can be in two states: \enquote{lowered} or \enquote{raised}.
-% The status of a flag can be tested expandably (just like booleans).
-% A flag can also be \emph{raised} expandably. However, a flag cannot
-% be lowered expandably.
-%
-% Flag variables are always local.
-%
-% Testing for the existence of flags with \cs{cs_if_exist:NTF} will
-% always result in a negative answer: from the point of view of
-% \TeX{}, flags are either undefined control sequences, or let to
-% \cs{scan_stop:}. This has a second consequence: flags don't need
-% to be declared with \cs{flag_new:N}, although this has error-checking
-% benefits. The ability of using flags without declaring them is used
-% in \pkg{l3rand}.
-%
-% \begin{function}{\flag_new:N, \flag_new:c}
+% Flags are the only data-type on which \TeX{} can perform
+% assignments in expansion-only contexts. This module is meant mostly
+% for kernel use: in almost all cases, booleans or integers should be
+% preferred to flags, because they are faster.
+%
+% A flag can hold any non-negative value, which we call its
+% \meta{height}. In expansion-only contexts, a flag can only be
+% \enquote{raised}: this normally increases the \meta{height} by $1$,
+% but can be configured by defining specific traps. The \meta{height}
+% can also be queried expandably. However, decreasing it, or setting it
+% to zero requires non-expandable assignments.
+%
+% Flag variables are always local. They are referenced by a \meta{name}
+% of the form \meta{package}\texttt{_}\meta{flag name}, for instance,
+% \texttt{str_missing}.
+%
+% \section{Setting up flags}
+%
+% \begin{function}{\flag_new:n}
+% \begin{syntax}
+% \cs{flag_new:n} \Arg{flag name}
+% \end{syntax}
+% Creates a new \meta{flag} with a name given by \meta{flag name}, or
+% raises an error if the name is already taken. The \meta{flag name}
+% must consist of character tokens only. The declaration is global,
+% but flags are always local variables. The \meta{flag} will initially
+% have zero height.
+% \end{function}
+%
+% \begin{function}{\flag_clear:n}
% \begin{syntax}
-% \cs{flag_new:N} \meta{flag}
+% \cs{flag_clear:n} \Arg{flag name}
% \end{syntax}
-% Creates a new \meta{flag} or raises an error if the
-% name is already taken. The declaration is global,
-% but \meta{flags} are always local variables.
-% The \meta{flag} will initially be lowered.
+% The \meta{flag}'s height is set to zero. The assignment is local.
% \end{function}
%
-% \begin{function}[EXP,pTF]{\flag_test:N, \flag_test:c}
+% \begin{function}{\flag_clear_new:n}
% \begin{syntax}
-% \cs{flag_test:N} \meta{flag}
+% \cs{flag_clear_new:n} \Arg{flag name}
% \end{syntax}
-% This function returns \texttt{true} if the \meta{flag} is raised,
-% and \texttt{false} if the \meta{flag} is lowered.
+% Ensures that the \meta{flag} exists globally by applying
+% \cs{flag_new:n} if necessary, then applies \cs{flag_zero:n}, setting
+% the height to zero locally.
% \end{function}
%
-% \begin{function}[EXP]{\flag_raise:N}
+% \begin{function}{\flag_set_trap:nn}
% \begin{syntax}
-% \cs{flag_raise:N} \meta{flag}
+% \cs{flag_set_trap:nn} \Arg{flag name} \Arg{inline function}
% \end{syntax}
-% The \meta{flag} is raised. The assignment is local.
-% This function is expandable, despite being an assignment.
+% Changes the action that is taken when the \meta{flag} is raised
+% using \cs{flag_raise:n}. Instead of the default action which is to
+% increase the \meta{flag}'s height by $1$, the \meta{inline function}
+% will be called, receiving the current flag's height as |#1|. The
+% \meta{inline function} should expand to nothing; \emph{e.g.}, it
+% could call \cs{msg_expandable_error:n}. This function is very
+% experimental.
% \end{function}
%
-% \begin{function}{\flag_lower:N}
+% \section{Expandable flag commands}
+%
+% \begin{function}[EXP,pTF]{\flag_if_exist:n}
% \begin{syntax}
-% \cs{flag_lower:N} \meta{flag}
+% \cs{flag_if_exist:n} \Arg{flag name}
% \end{syntax}
-% The \meta{flag} is lowered. The assignment is local.
+% This function returns \texttt{true} if the \meta{flag name}
+% references a flag that has been defined previously, and
+% \texttt{false} otherwise.
+% \end{function}
+%
+% \begin{function}[EXP,pTF]{\flag_if_raised:n}
+% \begin{syntax}
+% \cs{flag_if_raised:n} \Arg{flag name}
+% \end{syntax}
+% This function returns \texttt{true} if the \meta{flag} has non-zero
+% height, and \texttt{false} if the \meta{flag} has zero height.
+% \end{function}
+%
+% \begin{function}[EXP]{\flag_height:n}
+% \begin{syntax}
+% \cs{flag_height:n} \Arg{flag name}
+% \end{syntax}
+% Expands to the height of the \meta{flag} as an integer denotation.
+% \end{function}
+%
+% \begin{function}[EXP]{\flag_raise:n}
+% \begin{syntax}
+% \cs{flag_raise:n} \Arg{flag name}
+% \end{syntax}
+% The \meta{flag}'s trap is performed, taking the current height as
+% its argument. The default behaviour is to increase the \meta{flag}'s
+% height by $1$ locally. This function is expandable, as long as the
+% trap is expandable (the default trap is expandable, despite being an
+% assignment).
% \end{function}
%
% \end{documentation}
%
% \begin{implementation}
%
-% \section{\pkg{l3str} implementation}
+% \section{\pkg{l3flag} implementation}
%
% \begin{macrocode}
%<*initex|package>
@@ -137,51 +179,78 @@
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
% \end{macrocode}
%
-% \begin{variable}{\g_flag_list_tl}
-% We keep track of the list of all defined flags in a token list.
-% This is only used by \cs{flag_new:N} to check that the flag
-% has not yet been defined.
-% \begin{macrocode}
-\tl_new:N \g_flag_list_tl
-% \end{macrocode}
-% \end{variable}
-%
-% \begin{macro}{\flag_new:N, \flag_new:c}
-% If the control sequence is already defined, then we raise an error.
-% Otherwise we check if it is already a flag, raising an appropriate
-% error, finally lower the flag globally (this is the only global
-% assignment to a flag\ldots{} perhaps an error?).
-% The \texttt{c} variant can be created without worrying for this function.
+% \subsection{Non-expandable flag commands}
+%
+% \begin{macro}{\flag_new:n}
+% For each flag, we define a \enquote{trap} function, which by default
+% simply increases the flag by $1$.
% \begin{macrocode}
-\cs_new_protected:Npn \flag_new:N #1
+\cs_new_protected:Npn \flag_new:n #1
{
- \cs_new_eq:NN #1 \c_undefined:D
- \tl_if_in:NnTF \g_flag_list_tl {#1}
- {
- \msg_kernel_error:nnx
- { flag } { already-defined }
- { \token_to_str:N #1 }
- }
- { \tl_gput_right:Nn \g_flag_list_tl {#1} }
+ \cs_new:cpn { flag_trap_#1:w } ##1 ;
+ { \exp_after:wN \use_none:n \cs:w l_#1_##1_flag \cs_end: }
}
-\cs_generate_variant:Nn \flag_new:N { c }
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[EXP, pTF]{\flag_test:N, \flag_test:c}
-% Test if the control sequence is \texttt{undefined} or not.
+% \begin{macro}{\flag_clear:n}
+% \begin{macro}[aux]{\flag_clear_aux:ww}
+% Undefine control sequences, starting from the |_0| flag, upwards,
+% until reaching an undefined control sequence.
% \begin{macrocode}
-\prg_new_conditional:Npnn \flag_test:N #1 { p , T , F , TF }
+\cs_new_protected:Npn \flag_clear:n #1
+ { \flag_clear_aux:ww 0 ; #1 \q_stop }
+\cs_new_protected:Npn \flag_clear_aux:ww #1 ; #2 \q_stop
{
- \if_cs_exist:N #1
- \prg_return_true:
+ \if_cs_exist:w l_#2_#1_flag \cs_end:
\else:
- \prg_return_false:
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
+ \cs_set_eq:cN { l_#2_#1_flag } \c_undefined:D
+ \exp_after:wN \flag_clear_aux:ww
+ \int_use:N \int_eval:w \c_one + #1 ;
+ #2 \q_stop
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\flag_clear_new:n}
+% A flag exist if \cs{flag_trap_\meta{flag name}:n} exists.
+% \begin{macrocode}
+\cs_new_protected:Npn \flag_clear_new:n #1
+ { \flag_if_exist:nTF {#1} { \flag_clear:n } { \flag_new:n } {#1} }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\flag_set_trap:nn}
+% Should that \pkg{l3flag} function check whether the flag exists?
+% \begin{macrocode}
+\cs_new_protected:Npn \flag_set_trap:nn #1#2
+ { \cs_set:cpn { flag_trap_#1:w } ##1 ; {#2} }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Expandable flag commands}
+%
+% \begin{macro}[EXP, pTF]{\flag_if_exist:n}
+% A \meta{flag} is defined if the corresponding \enquote{trap} is
+% defined.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \flag_if_exist:n #1 { p , T , F , TF }
+ {
+ \cs_if_exist:cTF { flag_trap_#1:w }
+ { \prg_return_true: } { \prg_return_false: }
}
-\prg_new_conditional:Npnn \flag_test:c #1 { p , T , F , TF }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[EXP, pTF]{\flag_if_raised:n}
+% Test if the flag is non-zero, by checking the |_0| control sequence.
+% \begin{macrocode}
+\prg_new_conditional:Npnn \flag_if_raised:n #1 { p , T , F , TF }
{
- \if_cs_exist:w #1 \cs_end:
+ \if_cs_exist:w l_#1_0_flag \cs_end:
\prg_return_true:
\else:
\prg_return_false:
@@ -190,40 +259,36 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[EXP]{\flag_raise:N, \flag_raise:c}
-% Raising a flag expandably relies on the fact that \TeX{}
-% automatically lets undefined control sequences to \tn{relax}
-% when building an unknown csname. We build such a csname, then
-% remove it with \cs{use_none:n}, essentially doing
-% \cs{use_none:c}, but optimized slightly. When a flag is given
-% as an \texttt{N}-type argument, a little more work is needed,
-% converting to a csname before raising that.
+% \begin{macro}[EXP]{\flag_height:n}
+% \begin{macro}[EXP, aux]{\flag_height_loop:ww, \flag_height_end:ww}
+% Extract the value of the flag by going through all of the
+% |_|\meta{integer} control sequences starting from $0$.
% \begin{macrocode}
-\cs_new:Npn \flag_raise:N #1
- { \exp_after:wN \use_none:n \cs:w \cs_to_str:N #1 \cs_end: }
-\cs_new:Npn \flag_raise:c #1
- { \exp_after:wN \use_none:n \cs:w #1 \cs_end: }
+\cs_new:Npn \flag_height:n #1 { \flag_height_loop:ww 0; #1 \q_stop }
+\cs_new:Npn \flag_height_loop:ww #1 ; #2 \q_stop
+ {
+ \if_cs_exist:w l_#2_#1_flag \cs_end:
+ \exp_after:wN \flag_height_loop:ww \int_use:N \int_eval:w \c_one +
+ \else:
+ \exp_after:wN \flag_height_end:ww
+ \fi:
+ #1 ; #2 \q_stop
+ }
+\cs_new:Npn \flag_height_end:ww #1 ; #2 \q_stop { #1 }
% \end{macrocode}
% \end{macro}
-%
-% \begin{macro}{\flag_lower:N, \flag_lower:c}
-% Simply undefine the control sequence, locally.
-% \begin{macrocode}
-\cs_new_protected:Npn \flag_lower:N #1
- { \cs_set_eq:NN #1 \c_undefined:D }
-\cs_generate_variant:Nn \flag_lower:N { c }
-% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[EXP]{\flag_raise:n}
+% Simply apply the trap to the height, after expanding the latter.
% \begin{macrocode}
-\msg_kernel_new:nnnn { flag } { already-defined }
- { The~control~sequence~#1~is~already~declared~as~a~flag. }
+\cs_new:Npn \flag_raise:n #1
{
- LaTeX~was~asked~to~define~the~flag~#1,~but~it~has~already~
- been~defined~as~a~flag.~The~flag~module~is~mostly~meant~
- for~kernel~use,~and~booleans~should~be~preferred.
+ \cs:w flag_trap_#1:w \exp_after:wN \cs_end:
+ \int_value:w \flag_height:n {#1} ;
}
% \end{macrocode}
+% \end{macro}
%
% \begin{macrocode}
%</initex|package>
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
index d921b0b7e56..9c2b82c248d 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 3200 2012-01-16 03:50:54Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3259 2012-01-29 20:50:48Z bruno $
{L3 Experimental Regular Expressions}
%</driver|package>
%<*driver>
@@ -77,7 +77,7 @@
% extraction of submatches, splitting, and replacement, all acting
% on token lists. The syntax of regular expressions is mostly a subset
% of the PCRE syntax (and very close to POSIX), with some additions
-% due to the fact that we act on lists of tokens rather than characters.
+% due to the fact that \TeX{} manipulates tokens rather than characters.
% For performance reasons, only a limited set of features are implemented.
% Notably, back-references are not supported.
%
@@ -101,12 +101,12 @@
% (here, a word).
%
% If a regular expression is to be used several times,
-% it can be compiled once, and stored in a token list
+% it can be compiled once, and stored in a regex
% variable using \cs{regex_const:Nn}. For example,
% \begin{verbatim}
-% \regex_const:Nn \c_foo_regex_tl { \c{begin} \cB. (\c[^BE].*) \cE. }
+% \regex_const:Nn \c_foo_regex { \c{begin} \cB. (\c[^BE].*) \cE. }
% \end{verbatim}
-% stores in \cs{c_foo_regex_tl} a regular expression which matches the
+% stores in \cs{c_foo_regex} a regular expression which matches the
% starting marker for an environment: \cs{begin}, followed by a
% begin-group token (|\cB.|), then any number of tokens which are
% neither begin-group nor end-group character tokens (|\c[^BE].*|),
@@ -145,8 +145,9 @@
% using the |\c|\Arg{regex} syntax (see below).
%
% Any special character which appears at a place where its special
-% behaviour cannot apply matches itself instead (for instance,
-% a quantifier appearing at the beginning of a string).
+% behaviour cannot apply matches itself instead (for instance, a
+% quantifier appearing at the beginning of a string), after raising a
+% warning.
%
% Characters.
% \begin{l3regex-syntax}
@@ -179,6 +180,7 @@
% \item[\\S] Any token not matched by |\s|.
% \item[\\V] Any token not matched by |\v|.
% \item[\\W] Any token not matched by |\w|.
+% \item[\\N] Any token not matched by |\n|.
% \end{l3regex-syntax}
% Of those, |.|, |\D|, |\H|, |\N|, |\S|, |\V|, and |\W| will match arbitrary
% control sequences.
@@ -282,6 +284,14 @@
% digits, namely digits with category other, or uppercase letters from
% |A| to |F| with category either letter or other.
%
+% The |\u| escape sequence allows to insert the contents of a token list
+% directly into a regular expression or a replacement, avoiding the need
+% to escape special characters. Namely, |\u|\Arg{tl~var~name} matches
+% the exact contents of the token list \meta{tl~var}. Within a |\c{...}|
+% control sequence matching, the |\u| escape sequence only expands its
+% argument once, in effect performing \cs{tl_to_str:v}. Quantifiers are
+% not supported directly: use a group.
+%
% Options can be set with |(?|\meta{option}|)| and
% unset with |(?-|\meta{option}|)|. Options are local
% to the group in which they are set, and revert to their
@@ -336,9 +346,7 @@
% which must be one of |CBEMTPUDSLOA|.
% \item[\\c\Arg{text}] Produces the control sequence with csname
% \meta{text}. The \meta{text} may contain references to the submatches
-% |\0|, |\1| \emph{etc.} As an experimental feature, |\c{...}| can be
-% nested, but the behaviour is not yet fully specified. Use at own risks.
-%^^A todo: decide what the right behaviour should be.
+% |\0|, |\1| \emph{etc.}
% \end{l3regex-syntax}
%
% \subsection{Precompiling regular expressions}
@@ -346,50 +354,36 @@
% If a regular expression is to be used several times,
% it is better to compile it once rather than doing it
% each time the regular expression is used. The precompiled
-% regular expression is stored as a token list variable. All
+% regular expression is stored in a variable. All
% of the \pkg{l3regex} module's functions can be given their
% regular expression argument either as an explicit string
% or as a precompiled regular expression.
%
+% \begin{function}{\regex_new:N}
+% \begin{syntax}
+% \cs{regex_new:N} \meta{regex~var}
+% \end{syntax}
+% Creates a new \meta{regex~var} or raises an error if the
+% name is already taken. The declaration is global. The
+% \meta{regex~var} will initially be such that it never matches.
+% \end{function}
+%
% \begin{function}{\regex_set:Nn, \regex_gset:Nn, \regex_const:Nn}
% \begin{syntax}
-% \cs{regex_set:Nn} \meta{tl var} \Arg{regex}
+% \cs{regex_set:Nn} \meta{regex~var} \Arg{regex}
% \end{syntax}
% Stores a precompiled version of the \meta{regular expression}
-% in the \meta{tl var}. For instance, this function can be used
+% in the \meta{regex~var}. For instance, this function can be used
% as
% \begin{verbatim}
-% \tl_new:N \l_my_regex_tl
-% \regex_set:Nn \l_my_regex_tl { my\ (simple\ )? reg(ex|ular\ expression) }
+% \regex_new:N \l_my_regex
+% \regex_set:Nn \l_my_regex { my\ (simple\ )? reg(ex|ular\ expression) }
% \end{verbatim}
% The assignment is local for \cs{regex_set:Nn} and global for
% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions
% which will never change.
% \end{function}
%
-% \begin{function}{\regex_to_str:N}
-% \begin{syntax}
-% \cs{regex_to_str:N} \meta{regex var}
-% \end{syntax}
-% Converts the \meta{regex var}, previously defined using
-% \cs{regex_(g)set:Nn}, to a string, which can be safely
-% read back when the \LaTeX3 syntax is active
-% (as triggered by \cs{ExplSyntaxOn}).
-% Line breaks are inserted at various places, to \emph{try}
-% and keep the line length short.
-% For instance, if \cs{l_my_stream} is an open stream for writing,
-% and \cs{l_regex_tl} is a regex variable, you can save its definition
-% using
-% \begin{verbatim}
-% \iow_now:Nx \l_my_stream
-% {
-% \tl_to_str:n { \tl_set:Nn \l_my_regex_tl } { \iow_newline:
-% \regex_to_str:N \l_my_regex_tl
-% }
-% }
-% \end{verbatim}
-% \end{function}
-%
% \subsection{Matching}
%
% All regular expression functions are available in both |:n| and |:N|
@@ -542,13 +536,9 @@
%
% The following need to be done now.
% \begin{itemize}
-% \item The \texttt{\{\}} quantifiers are only partially implemented.
-% \item Clean up the use of messages.
-% \item Triple check magic numbers.
-% \item Test \cs{par}.
-% \item Add tests for every regex and replacement feature.
-% \item Make sure the documentation is correct, and the code comments
-% are as well.
+% \item Clean up the use of messages.
+% \item Code comments.
+% \item Rewrite the documentation in a clearer way.
% \end{itemize}
%
% Code improvements to come.
@@ -559,14 +549,15 @@
% \item Use \tn{dimen} registers rather than \cs{l_regex_nesting_tl}
% to build \cs{regex_nesting:n}.
% \item Reduce the number of epsilon-transitions in alternatives.
-% \item Improve nesting of |\c| in the replacement text.
% \item Move the \enquote{reconstruction} part of \pkg{l3regex}
% to \pkg{l3tl-analysis}.
% \item Optimize regexes for csnames when the regex is a simple string.
-% \item Optimize simple strings: use less states
-% (|abcade| should give two states, for |abc| and |ade|).
+% \item Optimize simple strings: use less states (|abcade| should give
+% two states, for |abc| and |ade|). [Does that really make sense?]
% \item Optimize groups with no alternative.
% \item Optimize the use of \cs{prg_stepwise_...} functions.
+% \item Decide what |\c{\c{...}}| should do in the replacement text.
+% \item Fix the |\c{\cBx}| bug in replacement text.
% \end{itemize}
%
% The following features are likely to be implemented at some point
@@ -638,52 +629,164 @@
\RequirePackage{l3str, l3tl-analysis, l3flag}
% \end{macrocode}
%
+% \subsection{Plan of attack}
+%
% Most regex engines use backtracking. This allows to provide very
% powerful features (back-references come to mind first), but it is
-% costly. Since \TeX{} is not first and foremost a programming language,
-% complicated code tends to run slowly, and we must use faster, albeit
-% slightly more restrictive, techniques, coming from automata theory.
+% costly, and raises the problem of catastrophic backtracking. Since
+% \TeX{} is not first and foremost a programming language, complicated
+% code tends to run slowly, and we must use faster, albeit slightly more
+% restrictive, techniques, coming from automata theory.
%
% Given a regular expression of $n$ characters, we build a
% non-deterministic finite automaton (NFA) with roughly $n$ states,
-% which accepts precisely those token lists matching that regular expression.
-% We then run the token list through the NFA, and check the return value.
+% which accepts precisely those token lists matching that regular
+% expression. Then loop through the query token list one token (one
+% \enquote{index}) at a time, exploring in parallel every possible path
+% through the NFA. We keep track of an array of the states currently
+% \enquote{active}, which are to be considered in order when the next
+% token is read.
+%
+% We use the following vocabulary in the code comments (and in variable
+% names).
+% \begin{itemize}
+% \item \emph{Query}: the token list to which we apply the regular
+% expression.
+% \item \emph{Index}: each token in the query is labelled by an integer
+% \meta{index}, with $\cs{l_regex_min_index_int} - 1 \leq \meta{index}
+% \leq \cs{l_regex_max_index_int}$. The lowest and highest indexes
+% correspond to imaginary begin and end markers (with inacessible
+% category code and character code).
+% \item \emph{Step}:^^A todo: fill
+% \item \emph{State}: each state of the NFA is labelled by an integer
+% \meta{state} with $0 \leq \meta{state} <
+% \cs{l_regex_max_state_int}$.
+% \item \emph{Active state}: state of the NFA that is reached when
+% reading the query string for the matching. Those states are
+% ordered according to the greediness of quantifiers.
+% \end{itemize}
+%
+% To acheive a good performance, we abuse \TeX{}'s registers in two
+% ways. We access registers directly by number rather than tying them
+% to control sequence using \cs{int_new:N} and other allocation
+% functions. And we store integers in \tn{dimen} registers in scaled
+% points (\texttt{sp}), using \TeX{}'s implicit conversion from
+% dimensions to integers in some contexts. Specifically, the registers
+% are used as follows.
+% \begin{itemize}
+% \item \tn{toks}\meta{state} has two parts: a property list which
+% holds the submatch information, followed by the tests and actions
+% to perform in the \meta{state} of the NFA.
+% \item \tn{dimen}\meta{state} is equal to the last step in which
+% the \meta{state} was active.
+% \item \tn{skip}$i$ holds the \meta{state} number corresponding to
+% the $i$-th active \meta{state} (in order of precedence: smaller is
+% more important) and has no shrink or stretch components.
+% \item \tn{toks}\meta{index} holds \meta{tokens} which \texttt{o}-
+% and \texttt{x}-expand to the \meta{index}-th token in the query.
+% \end{itemize}
+% \tn{count} registers are not abused, which means that we can safely
+% use named integers.
%
-% The code is structured as follows. Various helper functions are
-% introduced in the next subsection, to limit the clutter in later
-% parts. Then functions pertaining to parsing the regular expression
-% are introduced: that part is rather long because of the many bells
-% and whistles that we need to cater for. The next subsection takes
-% care of running the NFA, and describes how the various \TeX{}
-% registers are (ab)used in this module. Finally, user functions.
+% The code is structured as follows. Variables, constants and various
+% helper functions are introduced first, to limit the clutter in later
+% parts. Then functions pertaining to parsing the regular expression are
+% defined: that part is rather long because of the many bells and
+% whistles of the regex notation. The next subsection takes care of
+% running the NFA. Finally, user functions.
%
% \subsection{Constants and variables}
%
+% \subsubsection{Generic}
+%
% \begin{macro}{\regex_tmp:w}
-% \begin{variable}{\l_regex_tmpa_tl, \l_regex_tmpb_tl, \l_regex_tmpc_tl}
-% \begin{variable}{\l_regex_tmpa_int, \l_regex_tmpb_int}
-% Temporary variables.
+% \begin{variable}
+% {\l_regex_internal_a_int, \l_regex_internal_b_int, \l_regex_internal_c_int}
+% \begin{variable}{\l_regex_internal_a_tl, \l_regex_internal_b_tl}
+% \begin{variable}{\g_regex_internal_tl}
+% Temporary variables used for various purposes.
% \begin{macrocode}
\cs_new:Npn \regex_tmp:w { }
-\tl_new:N \l_regex_tmpa_tl
-\tl_new:N \l_regex_tmpb_tl
-\tl_new:N \l_regex_tmpc_tl
-\int_new:N \l_regex_tmpa_int
-\int_new:N \l_regex_tmpb_int
+\tl_new:N \l_regex_internal_a_tl
+\tl_new:N \l_regex_internal_b_tl
+\int_new:N \l_regex_internal_a_int
+\int_new:N \l_regex_internal_b_int
+\int_new:N \l_regex_internal_c_int
+\tl_new:N \g_regex_internal_tl
% \end{macrocode}
% \end{variable}
% \end{variable}
+% \end{variable}
% \end{macro}
%
-% \begin{variable}{\l_regex_group_begin_flag}
-% \begin{variable}{\l_regex_group_end_flag}
-% Those flags are raised to indicate extra begin-group
-% or end-group tokens when extracting submatches.
+% \begin{variable}
+% {
+% \c_regex_d_tl, \c_regex_D_tl, \c_regex_h_tl, \c_regex_H_tl,
+% \c_regex_s_tl, \c_regex_S_tl, \c_regex_v_tl, \c_regex_V_tl,
+% \c_regex_w_tl, \c_regex_W_tl, \c_regex_N_tl
+% }
+%
+% These constant token lists encode which characters are recognized by
+% |\d|, |\D|, |\w|, \emph{etc.} in regular expressions. Namely,
+% |\d=[0-9]|, |\w=[0-9A-Z_a-z]|, \verb*+\s=[\ \^^I\^^J\^^L\^^M]+,
+% \verb*+\h=[\ \^^I]+, |\v=[\^^J-\^^M]|, and the upper case
+% counterparts match anything that the lower case does not match. The
+% order in which the various tests appear is optimized for usual
+% mostly lower case letter text.
% \begin{macrocode}
-\flag_new:N \l_regex_group_begin_flag
-\flag_new:N \l_regex_group_end_flag
+\tl_const:Nn \c_regex_d_tl
+ { \regex_item_range:nn \c_forty_eight { 57 } } % 0--9
+\tl_const:Nn \c_regex_h_tl
+ {
+ \regex_item_equal:n \c_thirty_two % space
+ \regex_item_equal:n \c_nine % tab
+ }
+\tl_const:Nn \c_regex_s_tl
+ {
+ \regex_item_equal:n \c_thirty_two % space
+ \regex_item_equal:n \c_nine % tab
+ \regex_item_equal:n \c_ten % lf
+ \regex_item_equal:n \c_twelve % ff
+ \regex_item_equal:n \c_thirteen % cr
+ }
+\tl_const:Nn \c_regex_v_tl
+ { \regex_item_range:nn \c_ten \c_thirteen } % lf, vtab, ff, cr
+\tl_const:Nn \c_regex_w_tl
+ {
+ \regex_item_range:nn \c_ninety_seven { 122 } % a--z
+ \regex_item_range:nn \c_sixty_five { 90 } % A--Z
+ \regex_item_range:nn \c_forty_eight { 57 } % 0--9
+ \regex_item_equal:n { 95 } % _
+ }
+\tl_const:Nn \c_regex_D_tl
+ { \c_regex_d_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_H_tl
+ { \c_regex_h_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_S_tl
+ { \c_regex_s_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_V_tl
+ { \c_regex_v_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_W_tl
+ { \c_regex_w_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_N_tl
+ {
+ \regex_item_equal:n \c_ten
+ \regex_break_point:TF { } { \regex_break_true:w }
+ }
% \end{macrocode}
% \end{variable}
+%
+% \begin{variable}{\c_regex_._tl}
+% The dot meta-character matches any character, except the end marker,
+% whose character code is $-2$.
+% \begin{macrocode}
+\tl_const:cn { c_regex_._tl }
+ {
+ \if_num:w \l_regex_current_char_int > - \c_two
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
+% \end{macrocode}
% \end{variable}
%
% \subsubsection{Variables used while building}
@@ -746,6 +849,10 @@
% This number is used when a branch of the alternation ends.
% Capturing groups can be arbitrarily nested, and we keep track of
% the stack of ID numbers in \cs{l_regex_capturing_group_seq}.
+% The \texttt{max_int} variable is used in the case of the special
+% groups \verb+(?|...|...)+, which reset the capturing group number in
+% each alternative: at the end of the group, we must use a number
+% larger than every capturing group appearing in any alternative.
% \begin{macrocode}
\int_new:N \l_regex_capturing_group_int
\seq_new:N \l_regex_capturing_group_seq
@@ -784,7 +891,9 @@
% all allowed catcodes $c$. The latter is local to each capturing
% group, and we reset \cs{l_regex_catcodes_int} to that value after
% each character or class, changing it only when encountering a |\c|
-% escape.
+% escape. Errata: the \cs{l_regex_catcodes_default_int} is not used
+% yet, because the idea of having \verb+\cL(...|...)+ act on the whole
+% contents of the group is not yet implemented.
% \begin{macrocode}
\int_new:N \l_regex_catcodes_int
\int_new:N \l_regex_catcodes_default_int
@@ -814,194 +923,108 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_regex_tmpa_regex_tl}
-% This holds a temporary pre-compiled regular expression
-% when matching a control sequence name.
-% \begin{macrocode}
-\tl_new:N \l_regex_tmpa_regex_tl
+% \begin{variable}{\l_regex_tests_bool,\l_regex_tests_tl}
+% The tests which should be performed on an item of the token list are
+% stored in \cs{l_regex_tests_tl}. In the case of character classes,
+% \cs{l_regex_tests_bool} is \texttt{false} for negative character
+% classes. In the other case, \cs{l_regex_tests_tl} is directly
+% converted to a state of the NFA. The two variables are cleared
+% (boolean set to \texttt{true}) after looking for a quantifier for
+% the class or single character or property. It would seem cleaner to
+% clear them instead before grabbing the next character or class, but
+% that doesn't interact correctly with constructions like |\cL[...]|
+% where |\cL| inserts some tests in \cs{l_regex_tests_tl}, which
+% should not be eliminated before finding the class.
+% \begin{macrocode}
+\bool_new:N \l_regex_tests_bool
+\tl_new:N \l_regex_tests_tl
% \end{macrocode}
% \end{variable}
%
-% \subsubsection{Character classes}
-%
-% \begin{variable}{\l_regex_class_bool,\l_regex_class_tl}
-% \begin{variable}{\l_regex_class_saved_bool,\l_regex_class_saved_tl}
-% \cs{l_regex_class_bool} is false for negative character classes.
-% \cs{l_regex_class_tl} holds the tests which should be performed to
-% decide whether the \cs{l_regex_current_char_int} matches that
-% character class. In nested class, which can only occur as
-% |[\c[...]]|, the two variables from the outer class must be saved
-% while processing the inner class.
+% \begin{variable}
+% {
+% \l_regex_tests_saved_bool,
+% \l_regex_tests_saved_tl,
+% \l_regex_catcodes_saved_int
+% }
+% In nested class, which can only occur as |[\cA[...]]|, the variables
+% \cs{l_regex_tests_bool}, \cs{l_regex_tests_tl}, and
+% \cs{l_regex_catcodes_int} from the outer class must be saved while
+% processing the inner class.
% \begin{macrocode}
-\bool_new:N \l_regex_class_bool
-\tl_new:N \l_regex_class_tl
-\bool_new:N \l_regex_class_saved_bool
-\tl_new:N \l_regex_class_saved_tl
+\bool_new:N \l_regex_tests_saved_bool
+\tl_new:N \l_regex_tests_saved_tl
+\int_new:N \l_regex_catcodes_saved_int
% \end{macrocode}
% \end{variable}
-% \end{variable}
%
-% \begin{variable}{\c_regex_d_tl,\c_regex_D_tl}
-% \begin{variable}{\c_regex_h_tl,\c_regex_H_tl}
-% \begin{variable}{\c_regex_s_tl,\c_regex_S_tl}
-% \begin{variable}{\c_regex_v_tl,\c_regex_V_tl}
-% \begin{variable}{\c_regex_w_tl,\c_regex_W_tl}
-% \begin{variable}{\c_regex_N_tl}
-% These constant token lists encode which characters
-% are recognized by |\d|, |\D|, |\w|, \emph{etc.}
-% in regular expressions. Namely, |\d=[0-9]|,
-% |\w=[0-9A-Z_a-z]|, |\s=[\ \^^I\^^J\^^L\^^M]|,
-% |\h=[\ \^^I]|, |\v=[\^^J-\^^M]|, and the upper case
-% counterparts match anything that the lower case
-% does not match.
-% The order in which the various ranges appear is
-% optimized for usual mostly lower case letter text.
+% \subsubsection{Variables used when matching}
+%
+% \begin{variable}{\l_regex_min_index_int}
+% \begin{variable}{\l_regex_max_index_int}
+% The tokens in the query are indexed from \cs{l_regex_min_index_int}
+% for the first to $\cs{l_regex_max_index_int}-1$ for the last, and
+% their information is stored in \tn{muskip} and \tn{toks} registers
+% with those numbers. We don't start from $0$ because the \tn{toks}
+% registers with low numbers are used to hold the states of the NFA.
% \begin{macrocode}
-\tl_const:Nn \c_regex_d_tl
- {
- \regex_item_range:nn { \c_forty_eight } { 57 } % 0--9
- }
-\tl_const:Nn \c_regex_D_tl
- {
- \regex_item_geq:n { \c_fifty_eight } % > `9
- \regex_item_range:nn { \c_zero } { 47 } % 0
- }
-\tl_const:Nn \c_regex_h_tl
- {
- \regex_item_equal:n { \c_thirty_two } % space
- \regex_item_equal:n { \c_nine } % tab
- }
-\tl_const:Nn \c_regex_H_tl
- {
- \regex_item_geq:n { 33 } % > space
- \regex_item_range:nn { \c_ten } { 31 } % tab < ... < space
- \regex_item_range:nn { \c_zero } { \c_eight } % < tab
- }
-\tl_const:Nn \c_regex_s_tl
- {
- \regex_item_equal:n { \c_thirty_two } % space
- \regex_item_range:nn { \c_nine } { \c_ten } % tab, lf
- \regex_item_range:nn { \c_twelve } { \c_thirteen } % ff, cr
- }
-\tl_const:Nn \c_regex_S_tl
- {
- \regex_item_geq:n { 33 } % > space
- \regex_item_range:nn { \c_fourteen } { 31 } % tab < ... < space
- \regex_item_range:nn { \c_zero } { \c_eight } % < tab
- \regex_item_equal:n { \c_eleven } % vtab
- }
-\tl_const:Nn \c_regex_v_tl
- {
- \regex_item_range:nn { \c_ten } { \c_thirteen } % lf, vtab, ff, cr
- }
-\tl_const:Nn \c_regex_V_tl
- {
- \regex_item_geq:n { \c_fourteen } % >cr
- \regex_item_range:nn { \c_zero } { \c_nine } % < lf
- }
-\tl_const:Nn \c_regex_w_tl
- {
- \regex_item_range:nn { \c_ninety_seven } { 122 } % a--z
- \regex_item_range:nn { \c_sixty_five } { 90 } % A--Z
- \regex_item_range:nn { \c_forty_eight } { 57 } % 0--9
- \regex_item_equal:n { 95 } % _
- }
-\tl_const:Nn \c_regex_W_tl
- {
- \regex_item_range:nn { \c_zero } { 47 } % <`0
- \regex_item_range:nn { \c_fifty_eight } { 64 } % (`9+1)--(`A-1)
- \regex_item_range:nn { \c_ninety_one } { 94 } % (`Z+1)--(`_-1)
- \regex_item_equal:n { 96 } % `
- \regex_item_geq:n { \c_one_hundred_twenty_three } % z
- }
-\tl_const:Nn \c_regex_N_tl
- {
- \regex_item_geq:n { \c_eleven } % > lf
- \regex_item_range:nn { \c_zero } { \c_nine } % < lf
- }
+\int_new:N \l_regex_min_index_int
+\int_new:N \l_regex_max_index_int
% \end{macrocode}
% \end{variable}
% \end{variable}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-% \end{variable}
-%
-% \subsubsection{Variables used when matching}
%
% \begin{variable}{\l_regex_nesting_int}
-% This integer is used to keep track of begin and end-group tokens.
+% The first phase when matching is to go once through the query token
+% list and store the information for each token as \tn{muskip} and
+% \tn{toks} registers. During this phase, \cs{l_regex_nesting_int}
+% counts the balance of begin-group and end-group character tokens
+% which appear before a given point in the string, and is stored as
+% the shrink component of the \tn{muskip} registers. This integer is
+% also used to keep track of the balance in the replacement text.
% \begin{macrocode}
\int_new:N \l_regex_nesting_int
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_regex_min_step_int}
-% \begin{variable}{\l_regex_max_step_int}
-% This integer holds the value of the step corresponding to
-% the left end of the token list, \emph{i.e.}, when the token
-% list is stored in \tn{toks} registers, the \tn{toks} register
-% \cs{l_regex_min_step_int} holds the first token in the token
-% list. In fact, this number is always one more than
-% \cs{l_regex_max_state_int}, but it is more practical to give
-% that a name.
-% \begin{macrocode}
-\int_new:N \l_regex_min_step_int
-\int_new:N \l_regex_max_step_int
-% \end{macrocode}
-% \end{variable}
-% \end{variable}
-%
-% \begin{variable}{\l_regex_current_step_int}
-% \begin{variable}{\l_regex_start_step_int}
-% \begin{variable}{\l_regex_success_step_int}
+% \begin{variable}{\l_regex_current_index_int}
+% \begin{variable}{\l_regex_start_index_int}
+% \begin{variable}{\l_regex_success_index_int}
% While reading through the query token list,
-% \cs{l_regex_current_step_int} is the position in the
-% token list, starting at \cs{l_regex_min_step_int} for
-% the left-most token.
-% Each match begins at the position
-% given by \cs{l_regex_start_step_int}. Whenever an execution thread
-% succeeds, the corresponding step is stored into
-% \cs{l_regex_success_step_int}, which will be the next starting step
-% (except in the case of empty matches).
+% \cs{l_regex_current_index_int} is the position in the token list,
+% starting at \cs{l_regex_min_index_int} for the first token.
+% Each match begins at the position given by
+% \cs{l_regex_start_index_int}. Whenever an execution thread succeeds,
+% the corresponding index is stored into \cs{l_regex_success_index_int},
+% which will be the next starting index (except in the case of empty
+% matches).
% \begin{macrocode}
-\int_new:N \l_regex_current_step_int
-\int_new:N \l_regex_start_step_int
-\int_new:N \l_regex_success_step_int
+\int_new:N \l_regex_current_index_int
+\int_new:N \l_regex_start_index_int
+\int_new:N \l_regex_success_index_int
% \end{macrocode}
% \end{variable}
% \end{variable}
% \end{variable}
%
-% \begin{variable}{\l_regex_unique_id_int}
-% In the case of repeated matches, \cs{l_regex_current_step_int}
-% is reset to the end-position of the previous match. In contrast,
-% \cs{l_regex_unique_id_int} is simply incremented to provide
-% a unique number for each iteration of the matching loop. This
-% is handy to attach each set of submatch information to a given
-% iteration (and automatically discard it when it corresponds to
-% a past iteration).
-% \begin{macrocode}
-\int_new:N \l_regex_unique_id_int
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}{\l_regex_current_char_int}
% \begin{variable}{\l_regex_current_catcode_int}
% \begin{variable}{\l_regex_current_token_tl}
% \begin{variable}{\l_regex_last_char_int}
% \begin{variable}{\l_regex_case_changed_char_int}
-% The character codes of the character at the current position
-% in the token list, and at the previous position, and the current
-% character with its case changed (|A-Z|$\leftrightarrow$|a-z|).
-% The \cs{l_regex_last_char_int} is used to test for word boundaries
-% (|\b| and |\B|). The \cs{l_regex_case_changed_char_int} is
-% only computed if the \enquote{case insensitive} option |(?i)|
-% is used in the regex.
+% The character and category codes of the token at the current
+% position; \meta{tokens} which \texttt{o}- and \texttt{x}-expand to
+% the token (as provided by \cs{tl_analysis:n}); the character code of
+% the token at the previous position; and the character code of the
+% result of changing the case of the current token
+% (|A-Z|$\leftrightarrow$|a-z|). This last integer is only computed if
+% the \enquote{case insensitive} option |(?i)| is used in the regex.
+% The \cs{l_regex_current_char_int} variable is also used in various
+% other phases to hold a character code.
% \begin{macrocode}
\int_new:N \l_regex_current_char_int
\int_new:N \l_regex_current_catcode_int
-\tl_new:N \l_regex_current_token_tl
+\tl_new:N \l_regex_current_token_tl
\int_new:N \l_regex_last_char_int
\int_new:N \l_regex_case_changed_char_int
% \end{macrocode}
@@ -1025,7 +1048,7 @@
% The variable \cs{l_regex_current_state_int} holds the state
% of the NFA which is currently considered: transitions are then
% given as shifts relative to the current state.
-% In the case of groups with quantifiers,
+% In some cases of groups with quantifiers,
% \cs{l_regex_current_state_int} is shifted to a fake value for
% transitions to point to the correct states.
% \begin{macrocode}
@@ -1050,40 +1073,77 @@
% \end{variable}
% \end{variable}
%
-% \begin{variable}{\l_regex_max_index_int}
+% \begin{variable}{\l_regex_step_int}
+% In the case of repeated matches, \cs{l_regex_current_index_int}
+% is reset to the end-position of the previous match. In contrast,
+% \cs{l_regex_step_int} is simply incremented to provide
+% a unique number for each iteration of the matching loop. This
+% is handy to attach each set of submatch information to a given
+% iteration (and automatically discard it when it corresponds to
+% a past iteration).
+% \begin{macrocode}
+\int_new:N \l_regex_step_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_max_active_int}
% All the currently active states are kept in order of precedence
% in the \tn{skip} registers, which for our purpose serve as an array:
% the $i$th item of the array is \tn{skip}$i$. The largest index used
-% after treating the previous character is \cs{l_regex_max_index_int}.
+% after treating the previous character is \cs{l_regex_max_active_int}.
% At the start of every step, the whole array is unpacked, so that the
-% space can immediately be reused, and \cs{l_regex_max_index_int} reset
-% to zero, effectively clearing the array.
+% space can immediately be reused, and \cs{l_regex_max_active_int} is
+% reset to zero, effectively clearing the array.
% \begin{macrocode}
-\int_new:N \l_regex_max_index_int
+\int_new:N \l_regex_max_active_int
% \end{macrocode}
% \end{variable}
%
-% \begin{macro}[int]{\l_regex_every_match_tl}
-% Every time a match is found, this token list is used.
-% For single matching, the token list is set to removing
-% the remainder of the query token list. For multiple matching,
-% the token list is set to repeat the matching.
+% \begin{variable}{\l_regex_every_match_tl}
+% Every time a match is found, this token list is used. For single
+% matching, the token list is empty. For multiple matching, the token
+% list is set to repeat the matching.
% \begin{macrocode}
\tl_new:N \l_regex_every_match_tl
% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_regex_fresh_thread_bool}
+% \begin{variable}{\l_regex_success_empty_bool}
+% \begin{macro}{\regex_if_two_empty_matches:F}
+% When doing multiple matches, we need to avoid infinite loops where
+% each iteration matches the same empty token list. When an empty
+% token list is matched, the next successful match of the same empty
+% token list is suppressed. We detect empty matches by setting
+% \cs{l_regex_fresh_thread_bool} to \texttt{true} for threads which
+% directly come from the start of the regular expression, and testing
+% that boolean whenever a thread succeeds. The function
+% \cs{regex_if_two_empty_matches:F} is redefined at every match
+% attempt, depending on whether the previous match was empty or not:
+% if it was, then the function must cancel a purported success if it
+% is empty and at the same spot as the previous match; otherwise, we
+% definitely don't have two identical empty matches, so the function
+% is \cs{use:n}.
+% \begin{macrocode}
+\bool_new:N \l_regex_fresh_thread_bool
+\bool_new:N \l_regex_success_empty_bool
+\cs_new_eq:NN \regex_if_two_empty_matches:F \use:n
+% \end{macrocode}
% \end{macro}
+% \end{variable}
+% \end{variable}
%
% \begin{variable}{\g_regex_success_bool}
% \begin{variable}{\l_regex_saved_success_bool}
% \begin{variable}{\l_regex_success_match_bool}
-% The boolean \cs{g_regex_success_bool} is true if there was
-% at least one match, and \cs{l_regex_success_match_bool} is
-% true if the current match attempt was successful.
-% The variable \cs{g_regex_success_bool} is the only global
-% variable in this whole module. When nesting \cs{regex}
-% functions internally, the value of \cs{g_regex_success_bool}
-% is saved into \cs{l_regex_saved_success_bool}, which is local,
-% hence not affected by the changes due to inner regex functions.
+% The boolean \cs{g_regex_success_bool} is true if there was at least
+% one successful match, and \cs{l_regex_success_match_bool} is true if
+% the current match attempt was successful. The variable
+% \cs{g_regex_success_bool} is the only global variable in this whole
+% module. When nesting \cs{regex} functions internally, the value of
+% \cs{g_regex_success_bool} is saved into
+% \cs{l_regex_saved_success_bool}, which is local, hence not affected
+% by the changes due to inner regex functions.
% \begin{macrocode}
\bool_new:N \g_regex_success_bool
\bool_new:N \l_regex_saved_success_bool
@@ -1093,55 +1153,63 @@
% \end{variable}
% \end{variable}
%
-% \begin{macro}{\regex_last_match_empty:F}
-% \begin{macro}[aux]{\regex_last_match_empty_no:F}
-% \begin{macro}[aux]{\regex_last_match_empty_yes:F}
-% When doing multiple matches, we need to avoid infinite loops where
-% each iteration matches the same empty token list. When we detect such
-% a situation, the next match attempt is shifted by one character.
-% Namely, an empty match is discarded if it follows an empty match
-% at the same position. If the previous match was non-empty,
-% \cs{regex_last_match_empty:F} is simply \cs{use:n}, and keeps
-% the match. If it was empty, then we test whether the new match
-% has moved or not: if it has not, then the success is discarded.
+% \subsubsection{Regular expression variables}
+%
+% \begin{variable}{\c_regex_no_match_regex}
+% This regular expression matches nothing, but is still a valid
+% regular expression. It is used as the initial value for regular
+% expressions declared using \cs{regex_new:N}.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_last_match_empty_no:F #1 {#1}
-\cs_new_protected:Npn \regex_last_match_empty_yes:F
- { \int_compare:nNnF \l_regex_start_step_int = \l_regex_current_step_int }
-\cs_new_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F
+\tl_const:Nn \c_regex_no_match_regex
+ {
+ \regex_nfa:Nw \c_regex_no_match_regex
+ \l_regex_max_state_int = \c_one
+ \l_regex_capturing_group_int = \c_zero
+ \tex_toks:D \c_zero { \s_stop }
+ \s_stop
+ }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}{\regex_new:N}
+% As for many data types deriving from the \texttt{tl} data type,
+% creating a new one is simply a matter of checking that it does not
+% exist yet, and setting it equal to a default value.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_new:N #1
+ {
+ \chk_if_free_cs:N #1
+ \cs_gset_eq:NN #1 \c_regex_no_match_regex
+ }
% \end{macrocode}
-% \end{macro}
-% \end{macro}
% \end{macro}
%
-% \begin{variable}{\l_regex_success_empty_bool}
-% \begin{variable}{\l_regex_fresh_thread_bool}
-% When a match succeeds, \cs{l_regex_success_empty_bool}
-% records whether it is empty. This information is used
-% to initialize \cs{regex_last_match_empty:F} before
-% starting the next match attempt.
-% The boolean \cs{l_regex_fresh_thread_bool} is true
-% when the current thread has started from the beginning of the
-% regular expression at this character.
-% This is probably suboptimal. Improvements welcome.
+% \begin{variable}{\l_regex_internal_regex}
+% This holds a temporary pre-compiled regular expression
+% when matching a control sequence name.
% \begin{macrocode}
-\bool_new:N \l_regex_success_empty_bool
-\bool_new:N \l_regex_fresh_thread_bool
+\regex_new:N \l_regex_internal_regex
% \end{macrocode}
% \end{variable}
-% \end{variable}
%
-% \subsubsection{Variables used when building the replacement}
+% \subsubsection{Other variables}
+%^^A todo: move?
+%^^A todo: run with the check-declarations option.
%
-% \begin{variable}{\l_regex_replacement_int}
+% \begin{variable}{\l_regex_replacement_csnames_int}
+% The behaviour of closing braces inside a replacement text depends on
+% whether a sequences |\c{| or |\u{| has been encountered. The number
+% of \enquote{open} such sequences that should be closed by |}| is
+% stored in \cs{l_regex_replacement_csnames_int}, and decreased by
+% $1$ by each |}|.
% \begin{macrocode}
-\int_new:N \l_regex_replacement_int
+\int_new:N \l_regex_replacement_csnames_int
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_regex_replacement_csnames_int}
+% \begin{variable}{\l_regex_replacement_int}
% \begin{macrocode}
-\int_new:N \l_regex_replacement_csnames_int
+\int_new:N \l_regex_replacement_int
% \end{macrocode}
% \end{variable}
%
@@ -1154,8 +1222,6 @@
% \end{variable}
% \end{variable}
%
-% \subsubsection{Variables used for user functions}
-%
% \begin{variable}{\l_regex_match_count_int}
% The number of matches found so far is stored
% in \cs{l_regex_match_count_int}. This is only used
@@ -1165,23 +1231,29 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{regex_begin, regex_end}
+% Those flags are raised to indicate extra begin-group
+% or end-group tokens when extracting submatches.
+% \begin{macrocode}
+\flag_new:n { regex_begin }
+\flag_new:n { regex_end }
+% \end{macrocode}
+% \end{variable}
+%
% \subsection{Helpers}
%
% \subsubsection{Toks}
%
-% When performing the matching, the \tn{toks} registers hold submatch
-% information, followed by the instruction for a given state of the NFA.
-% The two parts are separated by \cs{s_stop}.
+% Two unrelated sets of functions for manipulating \tn{toks} registers.
%
% \begin{macro}[int]{\regex_toks_put_left:Nx}
% \begin{macro}[int]{\regex_toks_put_right:Nx}
% During the building phase, every \tn{toks} register starts with
-% \cs{s_stop}, and we wish to add \texttt{x}-expanded material
-% to those registers. The expansion is done \enquote{by hand} for
+% \cs{s_stop}, and we wish to add \texttt{x}-expanded material to
+% those registers. The expansion is done \enquote{by hand} for
% optimization (these operations are used quite a lot). When adding
% material to the left, we define \cs{regex_tmp:w} to remove the
-% \cs{s_stop} marker and put it back to the left of the new
-% material.
+% \cs{s_stop} marker and put it back to the left of the new material.
% \begin{macrocode}
\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
{
@@ -1199,20 +1271,52 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Extracting parts of the query token list}
-%
-% \begin{macro}[int]{\regex_query_substr:nn}
-% \begin{macro}[int]{\regex_query_submatch:nn}
-% \begin{macro}[int]{\regex_query_submatch:w}
-% The token list is stored in \tn{toks} registers:
-% the first is \cs{l_regex_min_step_int}, the last is (we don't care yet).
+% \begin{macro}[int, rEXP]{\regex_toks_range:nn}
+% \begin{macro}[aux, rEXP]{\regex_toks_range:ww}
+% Some non-expandable functions store pieces of their result in
+% \tn{toks} registers. Those pieces are concatenated using
+% \cs{regex_toks_range:nn}, which expects two integers, \meta{start}
+% and \meta{end}, and expands to the contents of the \tn{toks}
+% registers from \meta{start} (inclusive) to \meta{end}
+% (exclusive). This correctly expands to nothing if \meta{start}
+% $\geq$ \meta{end}.
% \begin{macrocode}
-\cs_new:Npn \regex_query_substr:nn #1#2
+\cs_new:Npn \regex_toks_range:nn #1#2
{
- \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end:
- \str_aux_toks_range:nn {#1} {#2}
+ \exp_after:wN \regex_toks_range:ww
+ \int_use:N \int_eval:w #1 \exp_after:wN ;
+ \int_use:N \int_eval:w #2 ;
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \regex_toks_range:ww #1 ; #2 ;
+ {
+ \if_num:w #1 < #2 \exp_stop_f:
+ \else:
+ \exp_after:wN \prg_map_break:
\fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \exp_after:wN \regex_toks_range:ww
+ \int_use:N \int_eval:w #1 + \c_one ; #2 ;
}
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% On the one hand, when performing the matching, the \tn{toks} registers
+% hold submatch information, followed by the instruction for a given
+% state of the NFA. The two parts are separated by \cs{s_stop}. On the
+% other hand, we provide functions to unpack the contents from a range
+% of \tn{toks} within an \texttt{x}-expanding assignment.
+%
+% When it is time to extract submatches from the token list, the various
+% tokens are stored in \tn{toks} registers numbered from
+% \cs{l_regex_min_index_int} inclusive to \cs{l_regex_max_index_int}
+% exclusive. Furthermore, the \tn{skip} registers hold\ldots{} ^^A ??
+%
+% \begin{macro}[int]{\regex_query_submatch:nn}
+% \begin{macro}[int]{\regex_query_submatch:w}
+%^^A ??
+% \begin{macrocode}
\cs_new:Npn \regex_query_submatch:nn #1#2
{
\if_num:w #1 < \l_regex_capturing_group_int
@@ -1222,14 +1326,13 @@
}
\cs_new:Npn \regex_query_submatch:w #1 ;
{
- \regex_query_substr:nn
+ \regex_toks_range:nn
{ \tex_skip:D #1 }
{ \etex_gluestretch:D \tex_skip:D #1 }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \subsubsection{Sequences}
%
@@ -1243,13 +1346,13 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2
{
- \seq_pop:NN #1 \l_regex_tmpa_tl
- \int_set:Nn #2 \l_regex_tmpa_tl
+ \seq_pop:NN #1 \l_regex_internal_a_tl
+ \int_set:Nn #2 \l_regex_internal_a_tl
}
\cs_new_protected:Npn \regex_seq_get_int:NN #1#2
{
- \seq_get:NN #1 \l_regex_tmpa_tl
- \int_set:Nn #2 \l_regex_tmpa_tl
+ \seq_get:NN #1 \l_regex_internal_a_tl
+ \int_set:Nn #2 \l_regex_internal_a_tl
}
\cs_new_protected:Npn \regex_seq_push_int:NN #1#2
{ \seq_push:No #1 { \int_use:N #2 } }
@@ -1267,13 +1370,54 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_seq_pop_use:N #1
{
- \seq_pop:NN #1 \l_regex_tmpa_tl
- \l_regex_tmpa_tl
+ \seq_pop:NN #1 \l_regex_internal_a_tl
+ \l_regex_internal_a_tl
}
\cs_new_protected:Npn \regex_seq_get_use:N #1
{
- \seq_get:NN #1 \l_regex_tmpa_tl
- \l_regex_tmpa_tl
+ \seq_get:NN #1 \l_regex_internal_a_tl
+ \l_regex_internal_a_tl
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Grabbing digits}
+%
+% \begin{macro}[int]{\regex_get_digits:nw}
+% \begin{macro}[aux]{\regex_get_digits_loop:N,\regex_get_digits_end:w}
+% When parsing the \texttt{\{} quantifier, we need a tool to grab
+% digits until reaching the first non-digit. This
+% \cs{regex_get_digits:nw} function places whatever digits it found as
+% a brace group after |#1|.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_get_digits:nw #1
+ {
+ \tex_afterassignment:D \regex_tmp:w
+ \cs_set_nopar:Npx \regex_tmp:w
+ {
+ \exp_not:n {#1}
+ { \if_false: } } \fi:
+ \regex_get_digits_aux:NN
+ }
+\cs_new:Npn \regex_get_digits_aux:NN #1#2
+ {
+ \if_meaning:w \regex_build_raw:N #1
+ \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f:
+ #2
+ \else:
+ \regex_get_digits_end:w #1 #2
+ \fi:
+ \else:
+ \regex_get_digits_end:w #1 #2
+ \fi:
+ \regex_get_digits_aux:NN
+ }
+\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
+ {
+ \fi: #2
+ \if_false: { { \fi: } }
+ #1
}
% \end{macrocode}
% \end{macro}
@@ -1303,19 +1447,6 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\regex_item_dot:}
-% The dot meta-character matches any character,
-% except the end marker.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_item_dot:
- {
- \if_num:w \l_regex_current_char_int > - \c_two
- \exp_after:wN \regex_break_true:w
- \fi:
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}[int]{\regex_item_caseful_equal:n}
% \begin{macro}[int]{\regex_item_caseful_range:nn}
% \begin{macro}[int]{\regex_item_caseful_geq:n}
@@ -1394,7 +1525,7 @@
% \begin{macro}[int]{\regex_item_geq:n}
% By default, matching takes the letter case into account.
% Note that those functions are not protected:
-% they will expand at the building step, hard-coding which
+% they will expand at the building phase, hard-coding which
% states take care of caseless versus caseful matching.
% \begin{macrocode}
\cs_new:Npn \regex_item_equal:n { \regex_item_caseful_equal:n }
@@ -1407,7 +1538,7 @@
%
% \begin{macro}[int]{\regex_build_caseless:,\regex_build_caseful:}
% Switch between caseful and caseless matching.
-% This is only done during the building step.
+% This is only done during the building phase.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_caseless:
{
@@ -1426,23 +1557,27 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_item_catcode:nT}
+% \begin{macro}[int]{\regex_item_catcode:nT}
+% \begin{macro}[aux]{\regex_item_catcode_aux:}
% The argument is a sum of powers of $4$ with exponents given by the
% allowed category codes (between $0$ and $13$). Dividing by a given
% power of $4$ gives an odd result if and only if that category code
% is allowed. If the catcode does not match, then skip the character
% code tests which follow.
% \begin{macrocode}
+\cs_new_protected:Npn \regex_item_catcode_aux:
+ {
+ "
+ \if_case:w \l_regex_current_catcode_int
+ 1 \or: 4 \or: 10 \or: 40
+ \or: 100 \or: \or: 1000 \or: 4000
+ \or: 10000 \or: \or: 100000 \or: 400000
+ \or: 1000000 \or: 4000000 \else: 1*\c_zero
+ \fi:
+ }
\cs_new_protected:Npn \regex_item_catcode:nT #1
{
- \if_int_odd:w \int_eval:w #1 / "
- \if_case:w \l_regex_current_catcode_int
- 1 \or: 4 \or: 10 \or: 40
- \or: 100 \or: \or: 1000 \or: 4000
- \or: 10000 \or: \or: 100000 \or: 400000
- \or: 1000000 \or: 4000000 \else: 1*\c_zero
- \fi:
- \int_eval_end:
+ \if_int_odd:w \int_eval:w #1 / \regex_item_catcode_aux: \int_eval_end:
\exp_after:wN \use:n
\else:
\exp_after:wN \use_none:n
@@ -1450,6 +1585,7 @@
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
% \begin{macro}[int]{\regex_item_cs:n}
% Match a control sequence (the argument is a pre-compiled regex).
@@ -1463,9 +1599,9 @@
{
\int_compare:nNnT \l_regex_current_catcode_int = \c_zero
{
- \tl_set:Nn \l_regex_tmpa_regex_tl {#1}
+ \tl_set:Nn \l_regex_internal_regex {#1}
\bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool
- \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl
+ \exp_args:NNx \regex_match:NnTF \l_regex_internal_regex
{
\exp_after:wN \exp_after:wN
\exp_after:wN \cs_to_str:N \l_regex_current_token_tl
@@ -1480,67 +1616,246 @@
% \end{macrocode}
% \end{macro}
%
-% \subsubsection{Grabbing digits}
-%
-% \begin{macro}[int]{\regex_get_digits:nw}
-% \begin{macro}[aux]{\regex_get_digits_loop:N,\regex_get_digits_end:w}
-% Grabs digits (of category code other), skipping any intervening
-% space, until encountering a non-digit, and places the result
-% in a brace group after |#1|. This is used when parsing the \texttt{\{}
-% quantifier.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_get_digits:nw #1
+% \subsubsection{Simple character escape}
+%
+% Before actually parsing the regular expression or the replacement
+% text, we go through them once, converting |\n| to the character $10$,
+% \emph{etc.} In this pass, we also convert any special character
+% (\texttt{*}, \texttt{?}, \texttt{\{}, etc.) or escaped alphanumeric
+% character into a marker indicating that this was a special sequence,
+% and replace escaped special characters and non-escaped alphanumeric
+% characters by markers indicating that those were \enquote{raw}
+% characters. The rest of the code can then avoid caring about escaping
+% issues (those can become quite complex to handle in combination with
+% ranges in character classes).
+%
+% Usage: \cs{regex_escape_use:nnnn} \meta{inline~1} \meta{inline~2}
+% \meta{inline~3} \Arg{token list} The \meta{token list} is converted to
+% a string, then read from left to right, interpreting backslashes as
+% escaping the next character. Unescaped characters are fed to the
+% function \meta{fn1}, and escaped characters are fed to the function
+% \meta{fn2} within an \texttt{x}-expansion context (typically those
+% functions perform some tests on their argument to decide how to output
+% them). The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and
+% |\x| are recognized, and those are replaced by the corresponding
+% character, then fed to \meta{fn3}. The result is then left in the
+% input stream.
+%
+% The idea is to feed unescaped characters to one function,
+% escaped characters to another, and feed |\a|, |\e|, |\f|,
+% |\n|, |\r|, |\t| and |\x| converted to the appropriate
+% character to a third function. Spaces are ignored unless
+% escaped.
+%
+% \begin{macro}[int]{\regex_escape_use:nnnn}
+% \begin{macro}[aux]{\regex_escape_loop:N}
+% \begin{macro}[aux]+\regex_escape_\:w+
+% Go through |#4| once, applying |#1|, |#2|, or |#3| to each character
+% (after de-escaping it), then leave the result in the input
+% stream. Most of the work is done within an \texttt{x}-expanding
+% assignment, but the |\x| escape sequence cannot be done in that
+% way. Therefore, we interrupt the assignment at each |\x| escape
+% sequence, store the partial result in a \tn{toks} register, and at
+% the end unpack all of the \tn{toks} registers to the left of the
+% last chunk of \cs{l_regex_internal_a_tl}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_escape_use:nnnn #1#2#3#4
{
- \tex_afterassignment:D \regex_tmp:w
- \cs_set_nopar:Npx \regex_tmp:w
- {
- \exp_not:n {#1}
- { \if_false: } } \fi:
- \regex_get_digits_aux:NN
+ \group_begin:
+ \cs_set_nopar:Npn \regex_escape_unescaped:N ##1 { #1 }
+ \cs_set_nopar:Npn \regex_escape_escaped:N ##1 { #2 }
+ \cs_set_nopar:Npn \regex_escape_raw:N ##1 { #3 }
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \str_gset_other:Nn \g_regex_internal_tl {#4}
+ \int_zero:N \l_regex_internal_a_int
+ \tl_set:Nx \l_regex_internal_a_tl
+ {
+ \exp_after:wN \regex_escape_loop:N \g_regex_internal_tl
+ { break } \prg_break_point:n { }
+ }
+ \use:x
+ {
+ \group_end:
+ \regex_toks_range:nn \c_zero \l_regex_internal_a_int
+ \exp_not:o \l_regex_internal_a_tl
+ }
}
-\cs_new:Npn \regex_get_digits_aux:NN #1#2
+\cs_new:Npn \regex_escape_loop:N #1
{
- \if_meaning:w \regex_build_raw:N #1
- \if_charcode:w \c_space_token \exp_not:N #2
- \else:
- \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f:
- #2
- \else:
- \regex_get_digits_end:w #1 #2
- \fi:
- \fi:
- \else:
- \regex_get_digits_end:w #1 #2
- \fi:
- \regex_get_digits_aux:NN
+ \cs_if_exist_use:cF { regex_escape_\token_to_str:N #1:w }
+ { \regex_escape_unescaped:N #1 }
+ \regex_escape_loop:N
}
-\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
+\cs_new_nopar:cpn { regex_escape_ \c_backslash_str :w }
+ \regex_escape_loop:N #1
{
- \fi: #2
- \if_false: { { \fi: } }
- #1
+ \cs_if_exist_use:cF { regex_escape_/\token_to_str:N #1:w }
+ { \regex_escape_escaped:N #1 }
+ \regex_escape_loop:N
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
%
-% \subsubsection{More character testing}
-%
-% \begin{macro}[EXP,pTF]{\regex_token_if_other_digit:N}
-% In the replacement text, |\g{|\meta{int}|}| denotes the \meta{int}-th
-% submatch. Parsing this construction robustly requires a test of whether
-% a token is a digit or not.
+% \begin{macro}[aux]{\regex_escape_unescaped:N}
+% \begin{macro}[aux]{\regex_escape_escaped:N}
+% \begin{macro}[aux]{\regex_escape_raw:N}
+% Those functions are never called before being given a new meaning,
+% so their definitions here don't matter.
% \begin{macrocode}
-\prg_new_conditional:Npnn \regex_token_if_other_digit:N #1 { TF }
+\cs_new_eq:NN \regex_escape_unescaped:N ?
+\cs_new_eq:NN \regex_escape_escaped:N ?
+\cs_new_eq:NN \regex_escape_raw:N ?
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]+\regex_escape_\q_recursion_tail:w+
+% \begin{macro}[aux]+\regex_escape_/\q_recursion_tail:w+
+% \begin{macro}[aux]+\regex_escape_ :w+
+% \begin{macro}[aux]
+% {
+% \regex_escape_/a:w, \regex_escape_/e:w, \regex_escape_/f:w,
+% \regex_escape_/n:w, \regex_escape_/r:w, \regex_escape_/t:w
+% }
+% The loop is ended upon seeing \cs{q_recursion_tail}.
+% Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, |\t| take
+% their meaning here.
+% \begin{macrocode}
+\cs_new_eq:NN \regex_escape_break:w \prg_map_break:
+\cs_new_nopar:cpn { regex_escape_/break:w }
+ {
+ \if_false: { \fi: }
+ \msg_kernel_error:nn { regex } { trailing-backslash }
+ \exp_after:wN \use_none:n \exp_after:wN { \if_false: } \fi:
+ }
+\cs_new_nopar:cpn { regex_escape_~:w } { }
+\cs_new_nopar:cpx { regex_escape_/a:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^G }
+\cs_new_nopar:cpx { regex_escape_/t:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^I }
+\cs_new_nopar:cpx { regex_escape_/n:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^J }
+\cs_new_nopar:cpx { regex_escape_/f:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^L }
+\cs_new_nopar:cpx { regex_escape_/r:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^M }
+\cs_new_nopar:cpx { regex_escape_/e:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^[ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_escape_/x:w}
+% \begin{macro}[aux]{\regex_escape_x_test:N}
+% \begin{macro}[aux]{\regex_escape_x_unbraced_i:N}
+% \begin{macro}[aux]{\regex_escape_x_unbraced_ii:N}
+% \begin{macro}[aux]{\regex_escape_x_braced_loop:N}
+% \begin{macro}[aux]{\regex_escape_x_braced_end:N}
+% \begin{macro}[aux]{\regex_escape_x_end:}
+% When |\x| is encountered, interrupt the assignment,
+% and distinguish the cases of a braced or unbraced syntax.
+% In the braced case, collect arbitrarily many hexadecimal digits,
+% building the number in \cs{l_regex_current_char_int} (using
+% \cs{str_aux_hexadecimal_use:NTF}), and check that
+% the run of digits was stopped by a closing brace.
+% In the unbraced case, collect up to two hexadecimal digits,
+% possibly less, building the character number in \cs{l_regex_current_char_int}.
+% In both cases, once all digits have been collected, use
+% the \TeX{} primitive \tn{lowercase} to produce that character,
+% and use an \cs{if_false:} trick to restart the assignment.
+% \begin{macrocode}
+\cs_new_nopar:cpn { regex_escape_/x:w } \regex_escape_loop:N
+ {
+ \if_false: { \fi: }
+ \tex_toks:D \l_regex_internal_a_int
+ \exp_after:wN { \l_regex_internal_a_tl }
+ \int_incr:N \l_regex_internal_a_int
+ \int_zero:N \l_regex_current_char_int
+ \regex_escape_x_test:N
+ }
+\cs_new_protected:Npx \regex_escape_x_test:N #1
+ {
+ \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
+ { \exp_not:N \regex_escape_x_test:N }
+ {
+ \l_regex_current_char_int = "0
+ \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
+ { \exp_not:N \regex_escape_x_braced_loop:N }
+ { \exp_not:N \regex_escape_x_unbraced_i:N #1 }
+ }
+ }
+\cs_new:Npn \regex_escape_x_unbraced_i:N #1
{
- \if_num:w \c_nine < 1 \exp_not:N #1 \exp_stop_f:
- \prg_return_true: \else: \prg_return_false: \fi:
+ \str_aux_hexadecimal_use:NTF #1
+ { \regex_escape_x_unbraced_ii:N }
+ { \exp_stop_f: \regex_escape_x_end: #1 }
}
+\cs_new:Npn \regex_escape_x_unbraced_ii:N #1
+ {
+ \token_if_eq_charcode:NNTF \c_space_token #1
+ { \regex_escape_x_unbraced_ii:N }
+ {
+ \str_aux_hexadecimal_use:NTF #1
+ { \exp_stop_f: \regex_escape_x_end: }
+ { \exp_stop_f: \regex_escape_x_end: #1 }
+ }
+ }
+\cs_new:Npn \regex_escape_x_braced_loop:N #1
+ {
+ \token_if_eq_charcode:NNTF \c_space_token #1
+ { \regex_escape_x_braced_loop:N }
+ {
+ \str_aux_hexadecimal_use:NTF #1
+ { \regex_escape_x_braced_loop:N }
+ { \exp_stop_f: \regex_escape_x_braced_end:N #1 }
+ }
+ }
+\cs_new_protected:Npx \regex_escape_x_braced_end:N #1
+ {
+ \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
+ { \exp_not:N \regex_escape_x_end: }
+ {
+ \msg_kernel_error:nnx { regex } { x-missing-rbrace }
+ { \int_use:N \l_regex_current_char_int }
+ \exp_not:N \regex_escape_x_end: #1
+ }
+ }
+\group_begin:
+ \char_set_catcode_other:N \^^@
+ \cs_new_protected_nopar:Npn \regex_escape_x_end:
+ {
+ \if_num:w \l_regex_current_char_int > \c_max_char_int
+ \msg_kernel_error:nnx { regex } { x-overflow }
+ { \int_use:N \l_regex_current_char_int }
+ \exp_after:wN \use:n
+ \else:
+ \tex_lccode:D \c_zero \l_regex_current_char_int
+ \exp_after:wN \tl_to_lowercase:n
+ \fi:
+ {
+ \tl_set:Nx \l_regex_internal_a_tl
+ { \if_false: } \fi:
+ \regex_escape_raw:N ^^@
+ \regex_escape_loop:N
+ }
+ }
+\group_end:
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}[EXP,aux]{\regex_aux_char_if_alphanumeric:NTF}
-% \begin{macro}[EXP,aux]{\regex_aux_char_if_special:NTF}
+% \begin{macro}[EXP,aux]{\regex_char_if_alphanumeric:NTF}
+% \begin{macro}[EXP,aux]{\regex_char_if_special:NTF}
% These two tests are used in the first pass when parsing a
% regular expression. That pass is responsible for finding
% escaped and non-escaped characters, and recognizing which
@@ -1564,7 +1879,7 @@
% but printable ascii, from space (hex \texttt{20}) to
% \texttt{del} (hex \texttt{7E}).
% \begin{macrocode}
-\prg_new_conditional:Npnn \regex_aux_char_if_special:N #1 { TF }
+\prg_new_conditional:Npnn \regex_char_if_special:N #1 { TF }
{
\if_num:w `#1 < \c_ninety_one
\if_num:w `#1 < \c_fifty_eight
@@ -1586,7 +1901,7 @@
\fi:
\fi:
}
-\prg_new_conditional:Npnn \regex_aux_char_if_alphanumeric:N #1 { TF }
+\prg_new_conditional:Npnn \regex_char_if_alphanumeric:N #1 { TF }
{
\if_num:w `#1 < \c_ninety_one
\if_num:w `#1 < \c_fifty_eight
@@ -1617,41 +1932,41 @@
% When building the NFA corresponding to a given regex, we can be in
% ten distinct modes, which we label by some magic numbers:
% \begin{itemize}
-% \item[-6] |[\c{...}]| control sequence in a class,
-% \item[-2] |\c{...}| control sequence,
-% \item[0] |...| outer,
-% \item[2] |\c...| catcode test,
-% \item[6] |[\c...]| catcode test in a class,
-% \item[-63] |[\c{[...]}]| class inside mode $-6$,
-% \item[-23] |\c{[...]}| class inside mode $-2$,
-% \item[3] |[...]| class inside mode $-3$,
-% \item[23] |\c[...]| class inside mode $2$,
-% \item[63] |[\c[...]]| class inside mode $6$.
+% \item[-6] |[\c{...}]| control sequence in a class,
+% \item[-2] |\c{...}| control sequence,
+% \item[0] |...| outer,
+% \item[2] |\c...| catcode test,
+% \item[6] |[\c...]| catcode test in a class,
+% \item[-63] |[\c{[...]}]| class inside mode $-6$,
+% \item[-23] |\c{[...]}| class inside mode $-2$,
+% \item[3] |[...]| class inside mode $-3$,
+% \item[23] |\c[...]| class inside mode $2$,
+% \item[63] |[\c[...]]| class inside mode $6$.
% \end{itemize}
% This list is exhaustive, because |\c| escape sequences cannot be
% nested, and character classes cannot be nested directly. The choice
% of numbers is such as to optimize the most useful tests, and make
% transitions from one mode to another as simple as possible.
% \begin{itemize}
-% \item Even modes mean that we are not directly in a character class.
-% In this case, a left bracket appends $3$ to the mode. In a character
-% class, a right bracket changes the mode as $m\to (m-15)/13$,
-% truncated.
-% \item Grouping, assertion, and anchors are allowed in non-positive
-% even modes ($0$, $-2$, $-6$), and do not change the mode. Otherwise,
-% they trigger an error.
-% \item A left bracket is special in even modes, appending $3$ to the
-% mode; in those modes, quantifiers and the dot are recognized, and
-% the right bracket is normal. In odd modes (within classes), the left
-% bracket is normal, but the right bracket ends the class, changing
-% the mode from $m$ to $(m-15)/13$, truncated; also, ranges are
-% recognized.
-% \item In non-negative modes, left and right braces are normal. In
-% negative modes, however, left braces trigger a warning; right braces
-% end the control sequence, going from $-2$ to $0$ or $-6$ to $3$,
-% with error recovery for odd modes.
-% \item Properties (such as the |\d| character class) can appear in any
-% mode.
+% \item Even modes mean that we are not directly in a character class.
+% In this case, a left bracket appends $3$ to the mode. In a
+% character class, a right bracket changes the mode as $m\to
+% (m-15)/13$, truncated.
+% \item Grouping, assertion, and anchors are allowed in non-positive
+% even modes ($0$, $-2$, $-6$), and do not change the
+% mode. Otherwise, they trigger an error.
+% \item A left bracket is special in even modes, appending $3$ to the
+% mode; in those modes, quantifiers and the dot are recognized, and
+% the right bracket is normal. In odd modes (within classes), the
+% left bracket is normal, but the right bracket ends the class,
+% changing the mode from $m$ to $(m-15)/13$, truncated; also, ranges
+% are recognized.
+% \item In non-negative modes, left and right braces are normal. In
+% negative modes, however, left braces trigger a warning; right
+% braces end the control sequence, going from $-2$ to $0$ or $-6$ to
+% $3$, with error recovery for odd modes.
+% \item Properties (such as the |\d| character class) can appear in
+% any mode.
% \end{itemize}
%
% \begin{macro}[int, EXP]{\regex_build_if_in_class:TF}
@@ -1671,13 +1986,32 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\regex_build_if_assertions_forbidden:TF}
+% Assertions are only allowed in modes $0$, $-2$, and $-6$,
+% \emph{i.e.}, even, non-positive modes.
+% \begin{macrocode}
+\cs_new_nopar:Npn \regex_build_if_assertions_forbidden:TF
+ {
+ \if_int_odd:w \l_regex_build_mode_int
+ \exp_after:wN \use_i:nn
+ \else:
+ \if_int_compare:w \l_regex_build_mode_int > \c_zero
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_ii:nn
+ \fi:
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Helpers for building an NFA}
%
% \begin{macro}[int]{\regex_build_new_state:}
-% Here, we add a new state to the NFA. At the end of the building
-% phase, we want every \tn{toks} register to start with
-% \cs{s_stop}, hence initialize the new register appropriately.
-% Then set \cs{l_regex_left/right_state_int} to their new values.
+% Add a new state to the NFA. At the end of the building phase, we
+% want every \tn{toks} register to start with \cs{s_stop}, hence
+% initialize the new register appropriately. Then update
+% the left, right, and max \meta{states}.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_new_state:
{
@@ -1715,84 +2049,125 @@
%
% \subsubsection{From regex to NFA: framework}
%
-%^^A In order for the construction \verb"ab|cd" to work, we enclose the
-%^^A whole pattern within parentheses (in the code below,
-%^^A \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}). These
-%^^A have the added benefit to form a capturing group: hence we get the
-%^^A data of the whole match for free.
-%
-% \begin{macro}[int]{\regex_build:n}
-% First, reset a few variables. Then use the generic framework defined
-% in \pkg{l3str} to parse the regular expression once, recognizing
-% which characters are raw characters, and which have special meanings.
-% The search is not anchored: to achieve that, we insert state(s)
-% responsible for repeating the match attempt on every token
-% in the token list.
-% The trailing \cs{prg_do_nothing:} ensure that the look-ahead done by
-% some of the operations is harmless.
-% Finally, \cs{regex_build_end:} adds the finishing code
-% (checking that parentheses are properly nested, for instance).
+% \begin{macro}[int]{\regex_build:w, \regex_build_end:}
+% \begin{macro}[int]{\regex_build_exit_class:, \regex_build_exit_groups:}
+% There are two situations where we want to convert a regular
+% expression given as a string to an NFA. The first case is when the
+% regex is given directly by the user, as an \texttt{n} argument to
+% one of the \pkg{l3regex} functions, and the second case is when
+% building the regex given as an argument of the |\c{...}| escape
+% sequence within another regular expression. In both cases, some
+% variables must be reset before starting to build the NFA, using
+% \cs{regex_build:w}. Once building has ended (for instance when
+% encountering the closing brace in |\c{...}|), we make sure to close
+% any dangling class or open group, then add
+% \cs{regex_action_success:} to make the match successful if it
+% reaches the last state. The capturing group number is incremented:
+% then its value tells us how many capturing group there are,
+% including $0$.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build:n #1
+\cs_new_protected_nopar:Npn \regex_build:w
{
- \int_set_eq:NN \l_regex_build_mode_int \c_zero
- \regex_build:w
+ \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
+ \int_set_eq:NN \l_regex_capturing_group_int \c_zero
+ \tl_clear:N \l_regex_tests_tl
+ \bool_set_true:N \l_regex_tests_bool
+ \int_zero:N \l_regex_max_state_int
\regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- { \regex_action_start_wildcard: }
- \regex_build_open_aux:
- \str_escape_use:NNNn
- \regex_build_i_unescaped:N
- \regex_build_i_escaped:N
- \regex_build_i_raw:N
- { #1 }
- \prg_do_nothing: \prg_do_nothing:
- \prg_do_nothing: \prg_do_nothing:
- \regex_build_end:
}
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_i_unescaped:N}
-% \begin{macro}[aux]{\regex_build_i_escaped:N}
-% \begin{macro}[aux]{\regex_build_i_raw:N}
-% The \pkg{l3str} function \cs{str_escape_use:NNNn} goes through
-% the regular expression and finds the |\a|, |\e|, |\f|, |\n|, |\r|,
-% |\t|, and |\x| escape sequences, then distinguishes three cases:
-% non-escaped characters, escaped characters, and \enquote{raw}
-% characters coming from one of the escape sequences.
-% In the particular case of regular expressions, escaped alphanumerics
-% and non-escaped non-alphanumeric printable ascii characters may have
-% special meanings, while everything else should be treated as a raw
-% character.
-% \begin{macrocode}
-\cs_new:Npn \regex_build_i_unescaped:N #1
+\cs_new_protected_nopar:Npn \regex_build_end:
{
- \regex_aux_char_if_special:NTF #1
- { \exp_not:N \regex_build_special:N #1 }
- { \exp_not:N \regex_build_raw:N #1 }
+ \regex_build_exit_class:
+ \regex_build_exit_groups:
+ \regex_toks_put_right:Nx \l_regex_right_state_int
+ { \regex_action_success: }
+ \int_incr:N \l_regex_capturing_group_int
}
-\cs_new:Npn \regex_build_i_escaped:N #1
+\cs_new_protected_nopar:Npn \regex_build_exit_class:
{
- \regex_aux_char_if_alphanumeric:NTF #1
- { \exp_not:N \regex_build_escaped:N #1 }
- { \exp_not:N \regex_build_raw:N #1 }
+ \regex_build_if_in_class:TF
+ {
+ \msg_kernel_error:nn { regex } { missing-rbrack }
+ \use:c { regex_build_]: }
+ }
+ { }
+ }
+\cs_new_protected_nopar:Npn \regex_build_exit_groups:
+ {
+ \seq_if_empty:NF \l_regex_capturing_group_seq
+ {
+ \msg_kernel_error:nnx { regex } { missing-rparen }
+ { \seq_length:N \l_regex_capturing_group_seq }
+ \prg_replicate:nn
+ { \seq_length:N \l_regex_capturing_group_seq }
+ { \regex_build_close_aux: \regex_build_group_: }
+ }
}
-\cs_new:Npn \regex_build_i_raw:N #1
- { \exp_not:N \regex_build_raw:N #1 }
% \end{macrocode}
% \end{macro}
% \end{macro}
+%
+% \begin{macro}[int]{\regex_build:n}
+% Setup some variables with \cs{regex_build:w}, and additionally set
+% the build mode to $0$: we are neither in a class nor a |\c|
+% construction. Initiate the regular expression by a wildcard: the
+% search is unanchored and should be tried at every index in the token
+% list. We surround the regular expression by parentheses with
+% \cs{regex_build_open_aux:} and \cs{regex_build_close_aux:}, so that
+% any alternative appears within a group (\emph{e.g.}, \verb"ab|cd"
+% will now be within the external group); this also forms a capturing
+% group (labelled $0$), which gives us the whole match as the $0$-th
+% submatch. The regular expression itself is parsed by using the
+% generic framework of \cs{regex_escape_use:nnnn} to recognize special
+% characters, escaped ones, and escape sequences such as |\n| or
+% |\x{02f}|. This results in successive \cs{regex_build_raw:N},
+% \cs{regex_build_escaped:N}, \cs{regex_build_special:N} followed by
+% their arguments. The trailing \cs{prg_do_nothing:} ensure that the
+% look-ahead done by some of the operations is harmless. Finally,
+% \cs{regex_build_end:} adds the finishing code (checking that
+% parentheses are properly nested, for instance).
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build:n #1
+ {
+ \regex_build:w
+ \int_set_eq:NN \l_regex_build_mode_int \c_zero
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
+ { \regex_action_start_wildcard: }
+ \regex_build_open_aux:
+ \regex_escape_use:nnnn
+ {
+ \regex_char_if_special:NTF ##1
+ \regex_build_special:N \regex_build_raw:N ##1
+ }
+ {
+ \regex_char_if_alphanumeric:NTF ##1
+ \regex_build_escaped:N \regex_build_raw:N ##1
+ }
+ { \regex_build_raw:N ##1 }
+ { #1 }
+ \prg_do_nothing: \prg_do_nothing:
+ \prg_do_nothing: \prg_do_nothing:
+ \int_compare:nNnT \l_regex_build_mode_int < \c_zero
+ {
+ \msg_kernel_error:nn { regex } { c-missing-rbrace }
+ \exp_after:wN \regex_build_special:N \c_rbrace_str
+ }
+ \seq_put_right:Nn \l_regex_capturing_group_seq {0}
+ \regex_build_close_aux: \regex_build_group_:
+ \regex_build_end:
+ }
+% \end{macrocode}
% \end{macro}
%
% \begin{macro}[aux]{\regex_build_special:N}
% \begin{macro}[aux]{\regex_build_escaped:N}
-% If the control character has a particular meaning in regexes, the
-% corresponding function is used. Otherwise, it is interpreted as a
-% raw character. We distinguish special characters from escaped
-% alphanumeric characters because in character classes, unknown
-% escaped alphanumeric characters raise an error, while special
-% charcters are silently converted to raw characters.
+% If the special character or escaped alphanumeric has a particular
+% meaning in regexes, the corresponding function is used. Otherwise,
+% it is interpreted as a raw character. We distinguish special
+% characters from escaped alphanumeric characters because they behave
+% differently when appearing as an end-point of a range.
% \begin{macrocode}
\cs_new_protected:Npn \regex_build_special:N #1
{
@@ -1808,64 +2183,24 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\regex_build:w}
-% Hopefully, we didn't forget to initialize anything here.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build:w
- {
- \int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int
- \int_set_eq:NN \l_regex_capturing_group_int \c_zero
- \int_zero:N \l_regex_max_state_int
- \regex_build_new_state:
- \tl_clear:N \l_regex_class_tl
- \bool_set_true:N \l_regex_class_bool
- \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[int]{\regex_build_end:}
-% If parentheses are not nested properly, an error is raised,
-% and the correct number of parentheses is closed.
-% After that, we insert an instruction for the match to succeed.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_end:
- {
- \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
- \regex_build_close_aux: \regex_build_group_:
- \seq_if_empty:NF \l_regex_capturing_group_seq
- {
- \msg_kernel_error:nnx { regex } { missing-rparen }
- { \seq_length:N \l_regex_capturing_group_seq }
- \prg_replicate:nn
- { \seq_length:N \l_regex_capturing_group_seq }
- { \regex_build_close_aux: \regex_build_group_: }
- }
- \regex_toks_put_right:Nx \l_regex_right_state_int
- { \regex_action_success: }
- \int_incr:N \l_regex_capturing_group_int
- }
-% \end{macrocode}
-% \end{macro}
-%
% \begin{macro}[int]{\regex_build_one:n, \regex_build_one:x}
% In a class, add the argument to the current class. Outside a class,
-% this argument is the whole \enquote{class}, and we look for
-% quantifiers.
+% this argument is the only test, and we look for quantifiers.
% \begin{macrocode}
\cs_new_protected:Npn \regex_build_one:n #1
{ \regex_build_one:x { \exp_not:n {#1} } }
\cs_new_protected:Npn \regex_build_one:x #1
{
- \tl_put_right:Nx \l_regex_class_tl
+ \tl_put_right:Nx \l_regex_tests_tl
{
- \if_num:w \l_regex_catcodes_int = \c_regex_catcodes_all_int
- \exp_after:wN \use:n
- \else:
+ \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int
\regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ \else:
+ \exp_after:wN \use:n
\fi:
{#1}
}
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
\if_num:w \l_regex_build_mode_int = \c_two
\l_regex_build_mode_int = \c_zero
\else:
@@ -1880,51 +2215,34 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\regex_build_tmp_class:n}
-% ^^A todo: rename, not tmp, and used for class & single.
+% \begin{macro}{\regex_tests_action_cost:n}
% The argument is the target state if the test succeeds.
% \begin{macrocode}
-\cs_new:Npn \regex_build_tmp_class:n #1
+\cs_new:Npn \regex_tests_action_cost:n #1
{
- \exp_not:o \l_regex_class_tl
- \bool_if:NTF \l_regex_class_bool
+ \exp_not:o \l_regex_tests_tl
+ \bool_if:NTF \l_regex_tests_bool
{ \regex_break_point:TF { \regex_action_cost:n {#1} } { } }
{ \regex_break_point:TF { } { \regex_action_cost:n {#1} } }
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\regex_class_put:N}
-% \begin{macro}[int]{\regex_class_put:NN}
-% This function is used to rapidly put a character in a character
-% class being built, avoiding some tests implied by \cs{regex_build_one:x}.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_class_put:N #1
- {
- \tl_put_right:Nx \l_regex_class_tl
- { \regex_item_equal:n { \int_value:w `#1 ~ } }
- }
-\cs_new_protected:Npn \regex_class_put:NN #1#2
- {
- \tl_put_right:Nx \l_regex_class_tl
- { \regex_item_range:nn { \int_value:w `#1 ~ } { \int_value:w `#2 ~ } }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
+% \subsubsection{Raw characters}
%
-% \begin{macro}[int]{\regex_build_raw_alphanum_is_error:N}
-% Within character classes, some special character and escape
-% sequences do not have any meaning. Special characters are
-% interpreted as their \enquote{raw} counterpart, but escape sequences
-% without a meaning, \emph{e.g.}, |\A|, are most likely an error:
-% this should be |A|.
+% \begin{macro}[int]{\regex_build_raw_error:N}
+% Within character classes, some escaped alphanumeric sequences such
+% as |\b| do not have any meaning. They are replaced by a raw
+% character, after spitting out an error.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_raw_alphanum_is_error:N #1
+\cs_new_protected:Npn \regex_build_raw_error:N #1
{
- \regex_aux_char_if_alphanumeric:NTF #1
- { \msg_kernel_error:nnx { regex } { class-bad-escape } { #1 } }
- { \regex_build_raw:N #1 }
+ \if_int_odd:w \l_regex_build_mode_int
+ \msg_kernel_error:nnx { regex } { class-bad-escape } {#1}
+ \else:
+ \msg_kernel_error:nnx { regex } { catcode-bad-escape } {#1}
+ \fi:
+ \regex_build_raw:N #1
}
% \end{macrocode}
% \end{macro}
@@ -1961,21 +2279,28 @@
\token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [
{ \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } }
{
- \msg_kernel_warning:nnx { regex } { end-range-missing } % [
- { #1 - \if_meaning:w #3 ] \else: \c_backslash_str \fi: #3 }
- \regex_class_put:N #1
- \regex_class_put:N -
+ \msg_kernel_warning:nnxx { regex } { range-missing-end } % [
+ {#1} { \token_if_eq_charcode:NNF #3 ] { \c_backslash_str } #3 }
+ \tl_put_right:Nx \l_regex_tests_tl
+ {
+ \regex_item_equal:n { \int_value:w `#1 ~ }
+ \regex_item_equal:n { \int_value:w `- ~ }
+ }
#2#3
}
{
\if_num:w `#1 > `#3 \exp_stop_f:
- \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#3}
+ \msg_kernel_error:nnxx { regex } { range-backwards } {#1} {#3}
\else:
- \if_num:w `#1 = `#3 \exp_stop_f:
- \regex_class_put:N #3
- \else:
- \regex_class_put:NN #1#3
- \fi:
+ \tl_put_right:Nx \l_regex_tests_tl
+ {
+ \if_num:w `#1 = `#3 \exp_stop_f:
+ \regex_item_equal:n
+ \else:
+ \regex_item_range:nn { \int_value:w `#1 ~ }
+ \fi:
+ { \int_value:w `#3 ~ }
+ }
\fi:
}
}
@@ -1986,14 +2311,14 @@
%
% \begin{macro}[aux]{\regex_build_.:}
% In a class, the dot has no special meaning. Outside, insert
-% \cs{regex_item_dot:}, which matches any character or control
+% \cs{c_regex_._tl}, which matches any character or control
% sequence, and refuses $-2$, which marks the end of the token list.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_.: }
+\cs_new_protected_nopar:cpx { regex_build_.: }
{
- \regex_build_if_in_class:TF
+ \exp_not:N \regex_build_if_in_class:TF
{ \regex_build_raw:N . }
- { \regex_build_one:x \regex_item_dot: }
+ { \regex_build_one:n \exp_not:c { c_regex_._tl } }
}
% \end{macrocode}
% \end{macro}
@@ -2024,16 +2349,18 @@
%
% \subsubsection{Anchoring and simple assertions}
%
-% \begin{macro}[aux]{\regex_build_simple_assertion:Nn}
-% Assertions are not allowed within character classes: the raw
-% character |#1| is inserted instead. Otherwise, use the test |#2|;
-% if the assertion is successful, move from the \texttt{left} state to
-% the \texttt{right} state.
+% \begin{macro}[aux]{\regex_build_simple_assertion:nn}
+% Assertions are only allowed in modes $0$, $-2$ and $-6$. In
+% character classes, or immediately following a category code test,
+% escaped letters cause errors, while the meaning of special
+% characters is ignored: |#1| is used instead. In modes where the
+% assertion is allowed, we insert the test |#2|; if the assertion is
+% successful, move from the \texttt{left} state to the \texttt{right}
+% state.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_simple_assertion:Nn #1#2
+\cs_new_protected:Npn \regex_build_simple_assertion:nn #1#2
{
- \regex_build_if_in_class:TF
- { \regex_build_raw_alphanum_is_error:N #1 }
+ \regex_build_if_assertions_forbidden:TF {#1}
{
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
@@ -2047,7 +2374,7 @@
}
}
}
- %^^A \regex_assertion_quantifier:
+ %^^A todo: add \regex_assertion_quantifier:
}
}
% \end{macrocode}
@@ -2061,38 +2388,41 @@
% \begin{macro}[aux]{\regex_build_/z:}
% Anchoring at the start corresponds to checking that the current
% character is the first in the token list. Anchoring to the beginning
-% of the match attempt uses \cs{l_regex_start_step_int} instead of
+% of the match attempt uses \cs{l_regex_start_index_int} instead of
% \cs{c_zero}. End anchors match the end of the token list, marked by
% a character code of $-2$.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_^: }
{
- \regex_build_simple_assertion:Nn ^
- { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int }
+ \regex_build_simple_assertion:nn { \regex_build_raw:N ^ }
+ { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int }
}
\cs_new_protected_nopar:cpn { regex_build_/A: }
{
- \regex_build_simple_assertion:Nn A
- { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int }
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N A }
+ { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int }
}
\cs_new_protected_nopar:cpn { regex_build_/G: }
{
- \regex_build_simple_assertion:Nn G
- { \int_compare:nNnT \l_regex_start_step_int = \l_regex_current_step_int }
- }
-\cs_new_protected_nopar:cpn { regex_build_$: } % $
- {
- \regex_build_simple_assertion:Nn $ % $
- { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N G }
+ { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int }
}
+\group_begin:
+ \char_set_catcode_other:N \$
+ \cs_new_protected_nopar:cpn { regex_build_$: } % $
+ {
+ \regex_build_simple_assertion:nn { \regex_build_raw:N $ } % $
+ { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ }
+\group_end:
\cs_new_protected_nopar:cpn { regex_build_/Z: }
{
- \regex_build_simple_assertion:Nn Z
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N Z }
{ \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
}
\cs_new_protected_nopar:cpn { regex_build_/z: }
{
- \regex_build_simple_assertion:Nn z
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N z }
{ \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
}
% \end{macrocode}
@@ -2113,12 +2443,12 @@
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_/b: }
{
- \regex_build_simple_assertion:Nn b
- { \regex_if_word_boundary:TF \use:n \use_none:n }
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N b }
+ { \regex_if_word_boundary:T }
}
\cs_new_protected_nopar:cpn { regex_build_/B: }
{
- \regex_build_simple_assertion:Nn B
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N B }
{ \regex_if_word_boundary:TF { } }
}
\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF
@@ -2131,6 +2461,8 @@
{ \group_end: \c_regex_w_tl }
\regex_break_point:TF
}
+\cs_new_protected_nopar:Npn \regex_if_word_boundary:T
+ { \regex_if_word_boundary:TF \use:n \use_none:n }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -2156,7 +2488,7 @@
% the mode ($m\to (m-15)/13$, truncated) to reflect the fact that we
% are leaving the class. If we are still in a class after leaving one,
% then this is the case |[...\cL[...]...]|, and we insert the relevant
-% closing material in \cs{l_regex_class_tl}. Otherwise look for
+% closing material in \cs{l_regex_tests_tl}. Otherwise look for
% quantifiers.
% \begin{macrocode}
\cs_new_protected:cpn { regex_build_]: }
@@ -2164,21 +2496,22 @@
\regex_build_if_in_class:TF
{
\if_num:w \l_regex_build_mode_int > \c_sixteen
- \tl_set:Nx \l_regex_class_tl
+ \tl_set:Nx \l_regex_tests_tl
{
- \exp_not:o \l_regex_class_saved_tl
- \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int
- \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ \exp_not:o \l_regex_tests_saved_tl
+ \if_num:w \l_regex_catcodes_saved_int < \c_regex_catcodes_all_int
+ \regex_item_catcode:nT
+ { \int_use:N \l_regex_catcodes_saved_int }
\else:
\exp_after:wN \use:n
\fi:
{
- \exp_not:o \l_regex_class_tl
- \bool_if:NF \l_regex_class_bool
- { \regex_break_point:TF { } { \regex_break_true:w } }
+ \exp_not:o \l_regex_tests_tl
+ \bool_if:NF \l_regex_tests_bool
+ { \regex_break_point:TF { } \regex_break_true:w }
}
}
- \bool_set_eq:NN \l_regex_class_bool \l_regex_class_saved_bool
+ \bool_set_eq:NN \l_regex_tests_bool \l_regex_tests_saved_bool
\fi:
\tex_advance:D \l_regex_build_mode_int - \c_fifteen
\tex_divide:D \l_regex_build_mode_int \c_thirteen
@@ -2191,17 +2524,15 @@
% \end{macrocode}
% \end{macro}
%
-%^^A todo: ignore spaces everywhere!!
-%
% \begin{macro}[aux]{\regex_class_first:NNNN}
% This starts a class. Change the mode by appending $3$ to it, and
-% reset the variables \cs{l_regex_class_tl} and
+% reset the variables \cs{l_regex_tests_tl} and
% \cs{l_regex_bool_tl}. In the special case of mode $63$
% (|[\c[...]]|), we open a group, to avoid overriding the setting of
-% \cs{l_regex_class_bool} and \cs{l_regex_class_tl}; the group ends at
+% \cs{l_regex_tests_bool} and \cs{l_regex_tests_tl}; the group ends at
% the matching right bracket. If the first character is |^|, then the
% class is inverted. We keep track of this in
-% \cs{l_regex_class_bool}. If the next character is a right bracket,
+% \cs{l_regex_tests_bool}. If the next character is a right bracket,
% then it should be changed to a raw one (dirty hack here; the |F|
% argument of \cs{str_if_eq:nnTF} is the trailing |#3|).
% \begin{macrocode}
@@ -2209,14 +2540,16 @@
{
\l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ %
\if_num:w \l_regex_build_mode_int > \c_sixteen
- \tl_set_eq:NN \l_regex_class_saved_tl \l_regex_class_tl
- \bool_set_eq:NN \l_regex_class_saved_bool \l_regex_class_bool
+ \tl_set_eq:NN \l_regex_tests_saved_tl \l_regex_tests_tl
+ \bool_set_eq:NN \l_regex_tests_saved_bool \l_regex_tests_bool
+ \int_set_eq:NN \l_regex_catcodes_saved_int \l_regex_catcodes_int
+ \int_set_eq:NN \l_regex_catcodes_int \c_regex_catcodes_all_int
\fi:
\token_if_eq_meaning:NNTF #1 \regex_build_special:N
{
\token_if_eq_charcode:NNTF #2 ^
{
- \bool_set_false:N \l_regex_class_bool % [
+ \bool_set_false:N \l_regex_tests_bool % [
\str_if_eq:nnTF {#3#4} { \regex_build_special:N ] }
{ \regex_build_raw:N }
}
@@ -2269,8 +2602,7 @@
}
{ \cs_if_exist_use:cF { regex_build_c_#3: } }
{
- \msg_kernel_error:nnxx { regex } { c-command }
- { regular~expression } { #3 }
+ \msg_kernel_error:nnx { regex } { c-missing-category } {#3}
#2 #3
}
}
@@ -2292,11 +2624,12 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_c_[:} ^^A]
+% \begin{macro}[aux]{\regex_build_c_[:}
% \begin{macro}[aux]{\regex_build_c_lbrack_loop:NN}
% \begin{macro}[aux]{\regex_build_c_lbrack_end:}
% \begin{macro}[aux]{\regex_build_add:N}
-%
+% When encountering |\c[|, the task is to collect uppercase letters
+% representing character categories.
% \begin{macrocode}
\cs_new_protected:cpn { regex_build_c_[: } #1#2
{
@@ -2328,8 +2661,7 @@
{ \regex_build_c_lbrack_end: }
}
{
- \msg_kernel_error:nnxx { regex } { c-command }
- { regular-expression } { #2 }
+ \msg_kernel_error:nnx { regex } { c-missing-rbrack } {#2}
\regex_build_c_lbrack_end:
}
}
@@ -2356,24 +2688,30 @@
% \end{macro}
%
% ^^A{
-% \begin{macro}+\regex_build}:+
-% Non-escaped right braces are only special if they appear
-% when building the regular expression for a csname. Otherwise,
+% \begin{macro}+\regex_build_}:+
+% Non-escaped right braces are only special if they appear when
+% building the regular expression for a csname. In that case, make
+% sure that any character class and group is closed, anchor the match
+% at the end, add the code for making the match successful, and expand
+% the contents of the regular expression we just built. Otherwise,
% replace the brace with an escaped brace.
% \begin{macrocode}
\cs_new_protected:cpn { regex_build_ \c_rbrace_str : }
- { %^^A incorrect recovery for \c{...(...[...} with missing ) or ].
+ {
\int_compare:nNnTF \l_regex_build_mode_int < \c_zero
{
- \use:c { regex_build_/Z: }
- \regex_toks_put_right:Nx \l_regex_right_state_int
- { \regex_action_success: }
- \int_incr:N \l_regex_capturing_group_int
+ \regex_build_exit_class:
+ \regex_build_exit_groups:
+ \regex_build_escaped:N Z
+ \regex_build_end:
\use:x
{
\group_end:
\regex_build_one:n
- { \regex_item_cs:n { \regex_set_aux:N ? } }
+ {
+ \regex_item_cs:n
+ { \regex_set_aux:N \l_regex_internal_regex }
+ }
}
}
{ \exp_after:wN \regex_build_raw:N \c_rbrace_str }
@@ -2488,7 +2826,7 @@
\tl_if_empty:nTF {#2}
{ \regex_build_quantifier_aux:nnNN { {#1} } { n* } }
{ \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } }
- } %^^A todo: catch the case m>n
+ }
{
\regex_build_quantifier_end:nn { } { }
\use:x
@@ -2513,9 +2851,8 @@
\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2
{
\use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2
- \tl_clear:N \l_regex_class_tl
- \bool_set_true:N \l_regex_class_bool
- \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
+ \tl_clear:N \l_regex_tests_tl
+ \bool_set_true:N \l_regex_tests_bool
}
% \end{macrocode}
% \end{macro}
@@ -2536,138 +2873,131 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[aux]{\regex_build_one_n:, \regex_build_one_n?:}
% \begin{macro}[aux]{\regex_build_one_:}
-% If no quantifier is found, then the character or character class
-% should just be built into a transition from the current
-% \enquote{right} state to a new state.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_one_:
- {
- \regex_build_transition:NN
- \regex_build_tmp_class:n \l_regex_right_state_int
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_one_?:}
-% \begin{macro}[aux]{\regex_build_one_??:}
-% The two transitions are a costly transition controlled by
-% the character class, and a free transition, both going to
-% a common new state. The only difference between the greedy
-% and lazy operators is the order of transitions.
-% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_one_?: }
- {
- \regex_build_transitions:NNNN
- \regex_build_tmp_class:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_one_??: }
- {
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_tmp_class:n \l_regex_right_state_int
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_one_*:}
-% \begin{macro}[aux]{\regex_build_one_*?:}
-% Build a costly transition going from the current state to itself,
-% and a free transition moving to a new state.
+% This function is called in case the syntax is
+% \texttt{\{\meta{int}\}}. Greedy and lazy operators are identical,
+% since the number of repetitions is fixed. Build one state for each
+% repetition, with a transition controlled by the tests that we have
+% collected. The case where we find no quantifier is the special case
+% $n=1$.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_one_*: }
- {
- \regex_build_transitions:NNNN
- \regex_build_tmp_class:n \l_regex_left_state_int
- \regex_action_free:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_one_*?: }
+\cs_new_protected:Npn \regex_build_one_n: #1
{
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_tmp_class:n \l_regex_left_state_int
+ \prg_replicate:nn {#1}
+ {
+ \regex_build_transition:NN
+ \regex_tests_action_cost:n \l_regex_right_state_int
+ }
}
+\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n:
+\cs_new_protected_nopar:Npn \regex_build_one_:
+ { \regex_build_one_n: \c_one }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_one_+:}
-% \begin{macro}[aux]{\regex_build_one_+?:}
-% Build a transition from the current state to a new state,
-% controlled by the character class, then build two transitions
-% from this new state to the original state (for repetition)
-% and to another new state (to move on to the rest of the pattern).
+% \begin{macro}[aux]{\regex_build_one_n*:, \regex_build_one_n*?:}
+% \begin{macro}[aux]{\regex_build_one_*:, \regex_build_one_*?:}
+% \begin{macro}[aux]{\regex_build_one_+:, \regex_build_one_+?:}
+% Those functions are called in case the syntax is
+% \texttt{\{\meta{int},\}}. The |*| and |+| quantifiers (greedy or
+% lazy) are special cases for $n=0$ or $n=1$. In the case $n=0$,
+% build a costly transition going from the current state to itself,
+% and a free transition moving to a new state. In the case $n\geq 1$,
+% reuse \cs{regex_build_one_n:} to match $n$ repetition, then add free
+% transitions from the last state to the previous one, and from the
+% last state to a new one. The greedy and lazy versions only differ in
+% the order of transitions.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_one_+: }
+\cs_new_protected:cpn { regex_build_one_n*: } #1
{
- \regex_build_one_:
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_tmpa_int
- \regex_action_free:n \l_regex_right_state_int
+ \int_compare:nNnTF {#1} = \c_zero
+ {
+ \regex_build_transitions:NNNN
+ \regex_tests_action_cost:n \l_regex_left_state_int
+ \regex_action_free:n \l_regex_right_state_int
+ }
+ {
+ \regex_build_one_n: {#1}
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_internal_a_int
+ \regex_action_free:n \l_regex_right_state_int
+ }
}
-\cs_new_protected_nopar:cpn { regex_build_one_+?: }
+\cs_new_protected:cpn { regex_build_one_n*?: } #1
{
- \regex_build_one_:
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_tmpa_int
+ \int_compare:nNnTF {#1} = \c_zero
+ {
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_right_state_int
+ \regex_tests_action_cost:n \l_regex_left_state_int
+ }
+ {
+ \regex_build_one_n: {#1}
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_right_state_int
+ \regex_action_free:n \l_regex_internal_a_int
+ }
}
+\cs_new_protected_nopar:cpx { regex_build_one_*: }
+ { \exp_not:c { regex_build_one_n*: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_one_*?: }
+ { \exp_not:c { regex_build_one_n*?: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_one_+: }
+ { \exp_not:c { regex_build_one_n*: } \c_one }
+\cs_new_protected_nopar:cpx { regex_build_one_+?: }
+ { \exp_not:c { regex_build_one_n*?: } \c_one }
% \end{macrocode}
% \end{macro}
% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_one_n:}
-% \begin{macro}[aux]{\regex_build_one_n?:}
-% This function is called in case the syntax is
-% \texttt{\{\meta{int}\}}. Greedy and lazy operators
-% are identical, since the number of repetitions is fixed.
-% Simply repeat |#1| times the effect of \cs{regex_build_one_:}.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_one_n: #1
- { \prg_replicate:nn {#1} { \regex_build_one_: } }
-\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n:
-% \end{macrocode}
-% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_one_n*:}
-% \begin{macro}[aux]{\regex_build_one_n*?:}
-% This function is called in case the syntax is
-% \texttt{\{\meta{int},\}}.
+% \begin{macro}[aux]{\regex_build_one_nn:, \regex_build_one_nn?:}
+% \begin{macro}[aux]{\regex_build_one_?:, \regex_build_one_??:}
+% \begin{macro}[aux]{\regex_build_one_nn_aux:nnnn}
+% Those functions are called when the syntax is
+% \texttt{\{\meta{int},\meta{int}\}}, or for the |?| quantifier, with
+% $m=0$ and $n=1$.
% \begin{macrocode}
-\cs_new_protected:cpx { regex_build_one_n*: } #1
+\cs_new_protected_nopar:Npn \regex_build_one_nn:
{
- \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: }
- \exp_not:c { regex_build_one_*: }
+ \regex_build_one_nn_aux:nnnn { }
+ {
+ \regex_build_transitions:NNNN
+ \regex_tests_action_cost:n \l_regex_right_state_int
+ \regex_action_free:n \l_regex_internal_a_int
+ }
}
-\cs_new_protected:cpx { regex_build_one_n*?: } #1
+\cs_new_protected_nopar:cpn { regex_build_one_nn?: }
{
- \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: }
- \exp_not:c { regex_build_one_*?: }
+ \regex_build_one_nn_aux:nnnn {?}
+ {
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_internal_a_int
+ \regex_tests_action_cost:n \l_regex_right_state_int
+ }
}
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_one_nn:}
-% \begin{macro}[aux]{\regex_build_one_nn?:}
-% \begin{macro}[aux]{\regex_build_one_nn_aux:Nnn}
-% This function is called when the syntax is
-% \texttt{\{\meta{int},\meta{int}\}}.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_one_nn_aux:Nnn #1#2#3
+\cs_new_protected:Npn \regex_build_one_nn_aux:nnnn #1#2#3#4
{
- \prg_replicate:nn {#2} { \regex_build_one_: }
- \prg_replicate:nn {#3-#2} {#1}
+ \regex_build_one_n: {#3}
+ \int_compare:nNnTF {#3} > {#4}
+ {
+ \msg_kernel_error:nnxxx
+ { regex } { quantifier-backwards } {#3} {#4} {#1}
+ }
+ {
+ \int_set:Nn \l_regex_internal_a_int
+ { \l_regex_right_state_int + #4 - #3 }
+ \prg_replicate:nn { #4 - #3 } { #2 }
+ }
}
-\cs_new_protected_nopar:Npx \regex_build_one_nn:
- { \regex_build_one_nn_aux:Nnn \exp_not:c { regex_build_one_?: } }
-\cs_new_protected_nopar:cpx { regex_build_one_nn?: }
- { \regex_build_one_nn_aux:Nnn \exp_not:c { regex_build_one_??: } }
+\cs_new_protected_nopar:cpn { regex_build_one_?: }
+ { \regex_build_one_nn: \c_zero \c_one }
+\cs_new_protected_nopar:cpx { regex_build_one_??: }
+ { \exp_not:c { regex_build_one_nn?: } \c_zero \c_one }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -2751,7 +3081,7 @@
\regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int
\regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int
\regex_seq_pop_use:N \l_regex_end_group_seq
- \seq_pop:NN \l_regex_end_alternation_seq \l_regex_tmpa_tl
+ \seq_pop:NN \l_regex_end_alternation_seq \l_regex_internal_a_tl
}
% \end{macrocode}
% Building each branch.
@@ -2798,7 +3128,7 @@
{
\cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : }
{
- \msg_kernel_error:nnx { regex } { unsupported }
+ \msg_kernel_error:nnx { regex } { special-group-unknown }
{ (? \token_to_str:N #2 } %)
\regex_build_special:N ( % )
\regex_build_raw:N ?
@@ -2888,7 +3218,7 @@
\token_if_eq_meaning:NNTF \regex_build_raw:N #2
{
\cs_if_exist_use:cF { regex_build_option_#1#3: }
- { \msg_error:nnx { regex } { unknown-option } { #3 } }
+ { \msg_kernel_error:nnx { regex } { unknown-option } { #3 } }
\regex_build_options:NNN #1
}
{
@@ -2897,7 +3227,7 @@
{ ) } { }
{ - } { \regex_build_options:NNN - }
}
- { \msg_error:nnx { regex } { invalid-in-option } { #3 } }
+ { \msg_kernel_error:nnx { regex } { invalid-in-option } { #3 } }
}
}
\cs_new_protected_nopar:cpn { regex_build_option_+i: }
@@ -2938,88 +3268,76 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2
{
- \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl
- \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero
+ \seq_pop:NN \l_regex_capturing_group_seq \l_regex_internal_a_tl
+ \int_compare:nNnF { \l_regex_internal_a_tl } < \c_zero
{
\regex_toks_put_left:Nx #1
- { \regex_action_submatch:n { \l_regex_tmpa_tl < } }
+ { \regex_action_submatch:n { \l_regex_internal_a_tl < } }
\regex_toks_put_left:Nx #2
- { \regex_action_submatch:n { \l_regex_tmpa_tl > } }
+ { \regex_action_submatch:n { \l_regex_internal_a_tl > } }
}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_:}
-% When there is no quantifier, the group is simply inserted as is,
-% and we only need to track submatches, and move to a new state.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_build_group_:
- {
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \regex_build_transition:NN
- \regex_action_free:n \l_regex_right_state_int
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_group_shift:N}
% Most quantifiers require to add an extra state before the group.
% This is done by shifting the current contents of the \cs{tex_toks:D}
-% \cs{l_regex_tmpa_int} to a new state.
-% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_group_shift:N #1
- {
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_new_state:
- \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_tmpa_int
- \regex_toks_put_left:Nx \l_regex_right_state_int
- {
- \int_set:Nn \l_regex_current_state_int
- { \int_use:N \l_regex_tmpa_int } % ^^A here we lie!
- }
- \cs_set:Npx \regex_tmp:w
- {
- \tex_toks:D \l_regex_tmpa_int
- {
- \s_stop
- #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
- }
- }
- \regex_tmp:w
- \regex_build_group_submatches:NN
- \l_regex_right_state_int \l_regex_left_state_int
- }
-% \end{macrocode}
-% \end{macro}
+% \cs{l_regex_internal_a_int} to a new state.
%
% \begin{macro}[aux]{\regex_build_group_qs_aux:NN}
% \begin{macro}[aux]{\regex_build_group_?:}
% \begin{macro}[aux]{\regex_build_group_??:}
-% \begin{macro}[aux]{\regex_build_group_*:}
-% \begin{macro}[aux]{\regex_build_group_*?:}
% Shift the state at which the group begins using
% \cs{regex_build_group_shift:N}, then add two transitions.
% The first transition is taken once the group has been
% traversed: in the case of \texttt{?} and \texttt{??},
% we should exit by going to \cs{l_regex_right_state_int},
% while for \texttt{*} and \texttt{*?} we loop by going to
-% \cs{l_regex_tmpa_int}.
+% \cs{l_regex_internal_a_int}.
% The second transition corresponds to skipping the group;
% it has lower priority (\texttt{put_right}) for greedy
% operators, and higher priority (\texttt{put_left}) for
% lazy operators.
+% We shift the left state to avoid interactions with
+% \texttt{action_submatch}.
% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:
+ {
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \regex_build_new_state:
+ \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_internal_a_int
+ \regex_toks_put_left:Nx \l_regex_right_state_int
+ {
+ \int_sub:Nn \l_regex_current_state_int
+ {
+ \int_eval:n % ^^A here we lie!
+ { \l_regex_right_state_int - \l_regex_internal_a_int }
+ }
+ }
+ \use:x
+ {
+ \tex_toks:D \l_regex_internal_a_int
+ {
+ \s_stop
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_right_state_int - \l_regex_internal_a_int }
+ }
+ }
+ }
+ \regex_build_group_submatches:NN
+ \l_regex_right_state_int \l_regex_left_state_int
+ }
\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2
{
- \regex_build_group_shift:N \regex_action_free:n
+ \regex_build_group_qs_aux:
\int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
\regex_build_transition:NN \regex_action_free:n #1
- #2 \l_regex_tmpa_int
+ #2 \l_regex_internal_a_int
{
\regex_action_free:n
- { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
+ { \int_eval:n { \l_regex_right_state_int - \l_regex_internal_a_int } }
}
}
\cs_new_protected_nopar:cpn { regex_build_group_?: }
@@ -3032,112 +3350,83 @@
\regex_build_group_qs_aux:NN
\l_regex_right_state_int \regex_toks_put_left:Nx
}
-\cs_new_protected_nopar:cpn { regex_build_group_*: }
- {
- \regex_build_group_qs_aux:NN
- \l_regex_tmpa_int \regex_toks_put_right:Nx
- }
-\cs_new_protected_nopar:cpn { regex_build_group_*?: }
- {
- \regex_build_group_qs_aux:NN
- \l_regex_tmpa_int \regex_toks_put_left:Nx
- }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[aux]{\regex_build_group_+:}
-% \begin{macro}[aux]{\regex_build_group_+?:}
-% Insert the submatch tracking code, then add two transitions
-% from the current state to the left end of the group (repeating the group),
-% and to a new state (to carry on with the rest of the regular expression).
-% \begin{macrocode}
-\cs_new_protected_nopar:cpn { regex_build_group_+: }
- {
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_tmpa_int
- \regex_action_free:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_group_+?: }
- {
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_tmpa_int
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
%
% \begin{macro}[aux]{\regex_build_group_n_aux:n}
-% The braced quantifiers rely on replicating the states
-% corresponding to the group that has just been built,
-% and joining the right state of each copy to the left state
-% of the next copy. Once this function has been run,
-% \cs{l_regex_tmpa_int} points to the last copy of the initial
-% left-most state, \cs{l_regex_left_state_int} has its initial
-% value, and \cs{l_regex_right_state_int} points to the last
-% copy of the initial right-most state. Furthermore,
-% \cs{l_regex_max_state_int} is set appropriately to the largest
-% allocated \tn{toks} register.
+% \begin{macro}[aux]{\regex_build_group_transition_right_max:}
+% The braced quantifiers rely on replicating the states corresponding
+% to the group that has just been built, and joining the right state
+% of each copy to the left state of the next copy. Once this function
+% has been run, \cs{l_regex_internal_a_int} points to the last copy of
+% the initial left-most state, \cs{l_regex_left/right_state_int} have
+% their initial values. Furthermore, \cs{l_regex_max_state_int} is set
+% appropriately to the largest allocated \tn{toks} register plus $1$.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_group_n_aux:n #1
+\cs_new_protected:Npn \regex_build_group_transition_right_max:
{
\regex_toks_put_right:Nx \l_regex_right_state_int
{
\regex_action_free:n
- {
- \int_eval:n %^^A todo: document why that value.
- { \l_regex_max_state_int - \c_one - \l_regex_left_state_int }
- }
+ { \int_eval:n { \l_regex_max_state_int - \l_regex_right_state_int } }
}
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int
+ }
+\cs_new_protected:Npn \regex_build_group_n_aux:n #1
+ {
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \int_set_eq:NN \l_regex_internal_b_int \l_regex_max_state_int
+ \int_add:Nn \l_regex_right_state_int
+ { ( #1 - 1 ) * ( \l_regex_max_state_int - \l_regex_left_state_int ) }
\int_set:Nn \l_regex_max_state_int
{
\l_regex_left_state_int
- + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int )
+ + (#1) * ( \l_regex_max_state_int - \l_regex_left_state_int )
}
- \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int
+ \int_while_do:nNnn \l_regex_internal_b_int < \l_regex_max_state_int
{
- \tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int
- \int_incr:N \l_regex_tmpa_int
- \int_incr:N \l_regex_tmpb_int
+ \tex_toks:D \l_regex_internal_b_int
+ = \tex_toks:D \l_regex_internal_a_int
+ \int_incr:N \l_regex_internal_a_int
+ \int_incr:N \l_regex_internal_b_int
}
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_n:}
-% \begin{macro}[aux]{\regex_build_group_n?:}
+% \begin{macro}[aux]{\regex_build_group_n:, \regex_build_group_n?:}
+% \begin{macro}[aux]{\regex_build_group_:}
% These functions are called in case the syntax is
-% \texttt{\{\meta{int}\}}. Greedy and lazy operators
-% are identical, since the number of repetitions is fixed.
-% We only record the submatch information at the last repetition.
+% \texttt{\{\meta{int}\}}, or in the absence of quantifier (with
+% $n=1$). Greedy and lazy operators are identical, since the number of
+% repetitions is fixed. We only record the submatch information at
+% the last repetition; and of course, in the case $n=0$, we record
+% nothing.
% \begin{macrocode}
\cs_new_protected:Npn \regex_build_group_n: #1
- { % ^^A todo: catch case #1 <= 0.
+ {
+ \regex_build_group_transition_right_max:
\regex_build_group_n_aux:n {#1}
- \regex_build_transition:NN
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_group_submatches:NN
- \l_regex_tmpa_int \l_regex_left_state_int
+ \regex_build_new_state:
+ \int_compare:nNnF {#1} = \c_zero
+ {
+ \regex_build_group_submatches:NN
+ \l_regex_internal_a_int \l_regex_left_state_int
+ }
}
\cs_new_eq:cN { regex_build_group_n?: } \regex_build_group_n:
+\cs_new_protected_nopar:Npn \regex_build_group_:
+ { \regex_build_group_n: \c_one }
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_n*:}
-% \begin{macro}[aux]{\regex_build_group_n*?:}
+% \begin{macro}[aux]{\regex_build_group_n*:, \regex_build_group_n*?:}
+% \begin{macro}[aux]{\regex_build_group_*:, \regex_build_group_*?:}
+% \begin{macro}[aux]{\regex_build_group_+:, \regex_build_group_+?:}
+% \begin{macro}[aux]{\regex_build_group_n_star_aux:NNn}
% These functions are called in case the syntax is
% \texttt{\{\meta{int},\}}. They are somewhat hybrid between
% the \texttt{\{\meta{int}\}} and the \texttt{*} quantifiers.
@@ -3146,61 +3435,271 @@
% case no iteration was made, here, we know that the submatch
% information is overwritten in any case.
% \begin{macrocode}
-\cs_new_protected:cpn { regex_build_group_n*: } #1
- { % ^^A todo: catch case #1 <= 0.
- \regex_build_group_n_aux:n {#1}
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_tmpa_int
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_group_submatches:NN
- \l_regex_tmpa_int \l_regex_left_state_int
+\cs_new_protected_nopar:cpn { regex_build_group_n*: }
+ {
+ \regex_build_group_n_star_aux:NNn
+ \regex_toks_put_right:Nx \regex_toks_put_left:Nx
}
-\cs_new_protected:cpn { regex_build_group_n*?: } #1
- { % ^^A todo: catch case #1 <= 0.
- \regex_build_group_n_aux:n {#1}
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_tmpa_int
- \regex_build_group_submatches:NN
- \l_regex_tmpa_int \l_regex_left_state_int
+\cs_new_protected_nopar:cpn { regex_build_group_n*?: }
+ {
+ \regex_build_group_n_star_aux:NNn
+ \regex_toks_put_left:Nx \regex_toks_put_right:Nx
}
+\cs_new_protected:Npn \regex_build_group_n_star_aux:NNn #1#2#3
+ {
+ \int_compare:nNnTF {#3} = \c_zero
+ { \regex_build_group_qs_aux:NN \l_regex_internal_a_int #1 }
+ {
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n {#3}
+ \regex_build_new_state:
+ #2 \l_regex_left_state_int
+ {
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_internal_a_int - \l_regex_left_state_int }
+ }
+ }
+ \regex_build_group_submatches:NN
+ \l_regex_internal_a_int \l_regex_left_state_int
+ }
+ }
+\cs_new_protected_nopar:cpx { regex_build_group_*: }
+ { \exp_not:c { regex_build_group_n*: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_group_*?: }
+ { \exp_not:c { regex_build_group_n*?: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_group_+: }
+ { \exp_not:c { regex_build_group_n*: } \c_one }
+\cs_new_protected_nopar:cpx { regex_build_group_+?: }
+ { \exp_not:c { regex_build_group_n*?: } \c_one }
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}[aux]{\regex_build_group_nn:}
-% \begin{macro}[aux]{\regex_build_group_nn?:}
+% \begin{macro}[aux]{\regex_build_group_nn:, \regex_build_group_nn?:}
+% \begin{macro}[aux]{\regex_build_group_nn_aux:nnnn}
+% \begin{macro}[aux, rEXP]{\regex_build_group_nn_aux_ii:w}
% These functions are called when the syntax is either
% \texttt{\{\meta{int},\}} or \texttt{\{\meta{int},\meta{int}\}}.
% \begin{macrocode}
-\cs_new_protected:Npn \regex_build_group_nn: #1#2
- { % ^^A Not Implemented Yet!
- \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet }
- \use:c { regex_build_group_n*: } {#1}
+\cs_new_protected_nopar:Npn \regex_build_group_nn:
+ {
+ \regex_build_group_nn_aux:nnnn { }
+ \regex_build_group_transition_right_max:
+ }
+\cs_new_protected_nopar:cpn { regex_build_group_nn?: }
+ {
+ \regex_build_group_nn_aux:nnnn { }
+ {
+ \use:x
+ {
+ \tex_toks:D \l_regex_right_state_int
+ {
+ \exp_after:wN \regex_build_group_nn_aux_ii:w
+ \tex_the:D \tex_toks:D \l_regex_right_state_int
+ }
+ }
+ }
+ }
+\cs_new_protected:Npn \regex_build_group_nn_aux:nnnn #1#2#3#4
+ {
+ \int_compare:nNnTF {#3} > {#4}
+ {
+ \msg_kernel_error:nnxxx
+ { regex } { quantifier-backwards } {#3} {#4} {#1}
+ }
+ {
+ \int_compare:nNnTF {#3} = \c_zero
+ {
+ \regex_build_group_qs_aux:
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n { #4 - #3 }
+ \int_set_eq:NN \l_regex_internal_c_int \l_regex_right_state_int
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
+ \prg_replicate:nn { #4 - #3 }
+ {
+ #2
+ \int_add:Nn \l_regex_right_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ }
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_internal_c_int
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_new_state:
+ }
+ {
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n {#3}
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_group_submatches:NN
+ \l_regex_left_state_int \l_regex_right_state_int
+ \regex_build_group_n_aux:n { #4 - #3 + \c_one }
+ \int_sub:Nn \l_regex_right_state_int
+ { \l_regex_internal_a_int - \l_regex_left_state_int }
+ \prg_replicate:nn { #4 - #3 }
+ {
+ #2
+ \int_add:Nn \l_regex_right_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ }
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_new_state:
+ }
+ }
+ }
+\cs_new_protected:Npn \regex_build_group_nn_aux_ii:w #1 \regex_action_free:n
+ {
+ #1
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_max_state_int - \l_regex_right_state_int }
+ }
+ \regex_action_free:n
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Matching raw token lists with \cs{u}}
+%
+%^^A todo: document interactions between caselessness and |\u{...}|
+%^^A todo: clarify when l_regex_catcodes_int is reset.
+%
+% \begin{macro}[aux]{\regex_build_/u:}
+% \begin{macro}[aux]{\regex_build_u_loop:NN}
+% The |\u| escape is invalid in classes nad directly following a
+% catcode test. Otherwise, it must be followed by a left brace. We
+% then collect the characters for the argument of |\u| within an
+% \texttt{x}-expanding assignment. In principle we could just wait to
+% encounter a right brace, but this is unsafe: if the right brace is
+% missing, then we will reach the end-markers of the regex, and
+% continue, leading to obscure fatal errors. Instead, we only allow
+% raw and special characters, and stop when encountering a special
+% right brace.
+% \begin{macrocode}
+\cs_new_protected:cpn { regex_build_/u: } #1#2
+ {
+ \regex_build_if_assertions_forbidden:TF
+ { \regex_build_raw_error:N u #1#2 }
+ {
+ \str_if_eq:xxTF {#1#2} { \regex_build_special:N \c_lbrace_str }
+ {
+ \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi:
+ \regex_build_u_loop:NN
+ }
+ {
+ \msg_kernel_error:nn { regex } { u-missing-lbrace }
+ #1#2
+ }
+ }
}
-\cs_new_protected:cpn { regex_build_group_nn?: } #1#2
- { % ^^A Not Implemented Yet!
- \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet }
- \use:c { regex_build_group_n*?: } {#1}
+\cs_new:Npn \regex_build_u_loop:NN #1#2
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_build_raw:N
+ { #2 \regex_build_u_loop:NN }
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_build_special:N
+ {
+ \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #2
+ { \if_false: { \fi: } \regex_build_u_end: }
+ { #2 \regex_build_u_loop:NN }
+ }
+ {
+ \if_false: { \fi: }
+ \msg_kernel_error:nnx { regex } { u-missing-rbrace } {#2}
+ \regex_build_u_end:
+ #1#2
+ }
+ }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \subsection{Matching}
+% \begin{macro}[aux]{\regex_build_u_end:}
+% Once we have extracted the variable's name, we store the contents of
+% that variable in \cs{l_regex_internal_a_tl}. The behaviour of |\u|
+% then depends on whether we are within a |\c{...}| escape (in this
+% case, the variable is turned to a string), or not. Afterwards, clean
+% up the value of \cs{l_regex_tests_tl}, but not
+% \cs{l_regex_tests_bool}, because this is guaranteed to be
+% \texttt{true} at the start, and is not affected by anything we do
+% here.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_u_end:
+ {
+ \tl_set:Nv \l_regex_internal_a_tl \l_regex_internal_a_tl
+ \if_num:w \l_regex_build_mode_int = \c_zero
+ \regex_build_u_not_cs:
+ \else:
+ \regex_build_u_in_cs:
+ \fi:
+ \tl_clear:N \l_regex_tests_tl
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[aux]{\regex_build_u_in_cs:}
+% When |\u| appears within a control sequence, the variable is
+% converted to a string, and we add to the NFA one state per
+% character. We use \cs{regex_build_one_:} directly, as this avoids
+% having to search for a quantifier.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_u_in_cs:
+ {
+ \exp_args:NNo \str_gset_other:Nn \g_regex_internal_tl
+ { \l_regex_internal_a_tl }
+ \tl_map_inline:Nn \g_regex_internal_tl
+ {
+ \tl_set:Nx \l_regex_tests_tl { \regex_item_equal:n { `##1 } }
+ \regex_build_one_:
+ }
+ }
+% \end{macrocode}
+% \end{macro}
%
-% \subsubsection{Use of \TeX{} registers when matching}
+% \begin{macro}[aux]{\regex_build_u_not_cs:}
+% In mode $0$, the |\u| escape adds one state to the NFA for each
+% token in \cs{l_regex_internal_a_tl}. The tests corresponding to a
+% given \meta{token} are: first test the category, then, if the
+% \meta{token} is a control sequence, compare the string
+% representations, otherwise compare the character codes. We could
+% compare the string representations in all cases, but that doesn't
+% interact well with caseless matching.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_build_u_not_cs:
+ {
+ \exp_args:No \tl_analysis_map_inline:nn { \l_regex_internal_a_tl }
+ {
+ \tl_set:Nx \l_regex_tests_tl
+ {
+ \exp_not:N \int_compare:nNnT
+ \l_regex_current_catcode_int = { \int_value:w "##2 }
+ {
+ \int_compare:nNnTF { "##2 } = \c_zero
+ {
+ \exp_not:n
+ { \str_if_eq:xxT { \l_regex_current_token_tl } {##1} }
+ { \regex_break_true:w }
+ }
+ { \regex_item_equal:n {##3} }
+ }
+ }
+ \regex_build_one_:
+ }
+ }
+% \end{macrocode}
+% \end{macro}
%
-% The first step in matching a regular expression is to build
-% the corresponding NFA and store its states in the \tn{toks}
-% registers. Then loop through the query token list one character
-% (one \enquote{step}) at a time, exploring in parallel every
-% possible path through the NFA. We keep track of an array of
-% the states currently \enquote{active}. More precisely,
-% \tn{skip} registers hold the state numbers to be considered
-% when the next token is read.
+% \subsection{Matching}
%
-% At every step, we unpack that array of active states and
+% At every index, we unpack that array of active states and
% empty it. Then loop over all active states, and perform the
% instruction at that state of the NFA. This can involve
% \enquote{free} transitions to other states, or transitions
@@ -3255,27 +3754,19 @@
% a value which uniquely identifies at which step it was last
% inserted.
%
-% The current approach means that stretch and shrink components
-% of \tn{skip} registers,
-%^^A todo: update: submatches' end-points are muskips.
-% as well as all \tn{muskip} registers are unused. It could seem that
-% \tn{count} registers are also free for use, but we still want to be
-% able to safely use integers, which are implemented as \tn{count}
-% registers.
-%
% \subsubsection{Helpers for running the NFA}
%
% \begin{macro}[aux]{\regex_store_state:n}
% Put the given state in the array of \tn{skip} registers.
% This is done by increasing the pointer
-% \cs{l_regex_max_index_int}, and converting the integer
+% \cs{l_regex_max_active_int}, and converting the integer
% to a dimension (suitable for a \tn{skip} assignment) in
% scaled points.
% \begin{macrocode}
\cs_new_protected:Npn \regex_store_state:n #1
{
- \int_incr:N \l_regex_max_index_int
- \tex_skip:D \l_regex_max_index_int #1 sp \scan_stop:
+ \int_incr:N \l_regex_max_active_int
+ \tex_skip:D \l_regex_max_active_int #1 sp \scan_stop:
\regex_store_submatches:n {#1}
}
% \end{macrocode}
@@ -3291,7 +3782,7 @@
% but not by \cs{regex_state_use_with_submatches:}.
% A state is free if it is not marker as taken, namely
% if the corresponding \tn{dimen} register is not
-% \cs{l_regex_unique_id_int} in \texttt{sp}.
+% \cs{l_regex_step_int} in \texttt{sp}.
% The primitive conditional is ended before unpacking
% the \tn{toks} register.
% \begin{macrocode}
@@ -3301,10 +3792,9 @@
{ \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } }
\cs_new_protected:Npn \regex_state_use_aux:n #1
{
- \if_num:w \tex_dimen:D \l_regex_current_state_int
- < \l_regex_unique_id_int
+ \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_step_int
\tex_dimen:D \l_regex_current_state_int
- = \l_regex_unique_id_int sp \scan_stop:
+ = \l_regex_step_int sp \scan_stop:
#1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int
\fi:
\scan_stop:
@@ -3364,11 +3854,7 @@
{
\exp_after:wN \c_zero
\exp_after:wN \regex_state_submatches:nn \exp_after:wN
- {
- \int_value:w \int_eval:w
- \l_regex_unique_id_int + \c_one
- \exp_after:wN
- }
+ { \int_value:w \int_eval:w \l_regex_step_int + \c_one \exp_after:wN }
\exp_after:wN { \l_regex_current_submatches_prop }
\regex_state_submatches:nn {#1} {#2}
\s_stop
@@ -3385,7 +3871,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_state_submatches:nn #1#2
{
- \if_num:w #1 = \l_regex_unique_id_int
+ \if_num:w #1 = \l_regex_step_int
\tl_set:Nn \l_regex_current_submatches_prop { #2 }
\fi:
}
@@ -3403,51 +3889,42 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_match:n #1
{
- \tl_set_analysis:Nn \l_regex_tmpa_tl {#1}
-
\int_zero:N \l_regex_nesting_int
- \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int
+ \int_set_eq:NN \l_regex_current_index_int \l_regex_max_state_int
\regex_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int
- \exp_after:wN \regex_query_set_loop:ww
- \l_regex_tmpa_tl
- \s_tl { ? \prg_map_break: } \s_tl
- \prg_break_point:n { }
- \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int
+ \int_set_eq:NN \l_regex_min_index_int \l_regex_current_index_int
+ \tl_analysis_map_inline:nn {#1}
+ {
+ \regex_query_set:nnn {##1} {"##2} {##3}
+ \if_case:w "##2 \exp_stop_f:
+ \or: \int_incr:N \l_regex_nesting_int
+ \or: \int_decr:N \l_regex_nesting_int
+ \fi:
+ }
+ \int_set_eq:NN \l_regex_max_index_int \l_regex_current_index_int
\regex_query_set:nnn { } { -1 } { -2 }
-
\regex_match_initial_setup:
\regex_match_once:
}
-\cs_new:Npn \regex_query_set_loop:ww #1 \s_tl #2#3 \s_tl
- {
- \use_none:n #2
- \regex_query_set:nnn {#1} {"#2} {#3}
- \if_case:w "#2 \exp_stop_f:
- \or: \int_incr:N \l_regex_nesting_int
- \or: \int_decr:N \l_regex_nesting_int
- \fi:
- \regex_query_set_loop:ww
- }
\cs_new_protected:Npn \regex_query_set:nnn #1#2#3
{
- \tex_muskip:D \l_regex_current_step_int
+ \tex_muskip:D \l_regex_current_index_int
= \etex_gluetomu:D
#3 sp
plus #2 sp
minus \l_regex_nesting_int sp
\scan_stop:
- \tex_toks:D \l_regex_current_step_int {#1}
- \int_incr:N \l_regex_current_step_int
+ \tex_toks:D \l_regex_current_index_int {#1}
+ \int_incr:N \l_regex_current_index_int
}
\cs_new_protected_nopar:Npn \regex_query_get:
{
\tl_set:Nx \l_regex_current_token_tl
- { \tex_the:D \tex_toks:D \l_regex_current_step_int }
+ { \tex_the:D \tex_toks:D \l_regex_current_index_int }
\l_regex_current_char_int
- = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ = \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int
\l_regex_current_catcode_int = \etex_gluestretch:D
- \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int
}
% \end{macrocode}
% \end{macro}
@@ -3482,13 +3959,14 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_match_initial_setup:
{
- \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one }
- { \tex_dimen:D ##1 \c_minus_one sp \scan_stop: }
- \int_set_eq:NN \l_regex_unique_id_int \c_minus_one
- \int_set:Nn \l_regex_start_step_int
- { \l_regex_min_step_int - \c_one }
- \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int
- \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int
+ \int_set_eq:NN \l_regex_step_int \c_minus_one
+ \prg_stepwise_inline:nnnn
+ \c_zero \c_one { \l_regex_max_state_int - \c_one }
+ { \tex_dimen:D ##1 \l_regex_step_int sp \scan_stop: }
+ \int_set:Nn \l_regex_start_index_int
+ { \l_regex_min_index_int - \c_one }
+ \int_set_eq:NN \l_regex_current_index_int \l_regex_min_index_int
+ \int_set_eq:NN \l_regex_success_index_int \l_regex_min_index_int
\int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int
\bool_set_false:N \l_regex_success_empty_bool
\bool_gset_false:N \g_regex_success_bool
@@ -3504,21 +3982,21 @@
{
\prop_clear:N \l_regex_current_submatches_prop
\bool_if:NTF \l_regex_success_empty_bool
- { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F }
- { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F }
- \int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int
- \int_set:Nn \l_regex_current_step_int
- { \l_regex_start_step_int - \c_one }
+ { \cs_set_eq:NN \regex_if_two_empty_matches:F \regex_if_match_empty:F }
+ { \cs_set_eq:NN \regex_if_two_empty_matches:F \use:n }
+ \int_set_eq:NN \l_regex_start_index_int \l_regex_success_index_int
+ \int_set:Nn \l_regex_current_index_int
+ { \l_regex_start_index_int - \c_one }
\bool_set_false:N \l_regex_success_match_bool
- \int_zero:N \l_regex_max_index_int
+ \int_zero:N \l_regex_max_active_int
\regex_store_state:n {0} %^^A _state_int!
}
% \end{macrocode}
% \end{macro}
%
% \begin{macro}[aux]{\regex_match_loop:}
-% \begin{macro}[aux]{\regex_match_one_index:n}
-% \begin{macro}[aux]{\regex_match_one_index_aux:n}
+% \begin{macro}[aux]{\regex_match_one_active:n}
+% \begin{macro}[aux]{\regex_match_one_active_aux:n}
% Setup what needs to be reset at every character,
% then set \cs{l_regex_current_char_int} to the
% character code of the token that is read
@@ -3532,38 +4010,37 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_match_loop:
{
- \int_incr:N \l_regex_current_step_int
- \int_incr:N \l_regex_unique_id_int
+ \int_incr:N \l_regex_current_index_int
+ \int_incr:N \l_regex_step_int
\bool_set_false:N \l_regex_fresh_thread_bool
\int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
\regex_query_get:
\regex_match_loop_case_hook:
- \cs_set_nopar:Npx \regex_tmp:w
+ \use:x
{
- \int_zero:N \l_regex_max_index_int
- \regex_match_one_index:w 1 ; \prg_break_point:n { }
+ \int_zero:N \l_regex_max_active_int
+ \regex_match_one_active:w 1 ; \prg_break_point:n { }
\exp_not:N \prg_break_point:n { }
}
- \regex_tmp:w
\if_num:w \l_regex_current_char_int < \c_minus_one
\exp_after:wN \prg_map_break:
\fi:
- \if_num:w \l_regex_max_index_int = \c_zero
+ \if_num:w \l_regex_max_active_int = \c_zero
\exp_after:wN \prg_map_break:
\fi:
\regex_match_loop:
}
-\cs_new:Npn \regex_match_one_index:w #1;
+\cs_new:Npn \regex_match_one_active:w #1;
{
- \if_num:w #1 > \l_regex_max_index_int
+ \if_num:w #1 > \l_regex_max_active_int
\exp_after:wN \prg_map_break:
\fi:
- \regex_match_one_index_aux:n
+ \regex_match_one_active_aux:n
{ \int_value:w \tex_skip:D #1 }
- \exp_after:wN \regex_match_one_index:w
+ \exp_after:wN \regex_match_one_active:w
\int_use:N \int_eval:w #1 + \c_one ;
}
-\cs_new_protected:Npn \regex_match_one_index_aux:n #1
+\cs_new_protected:Npn \regex_match_one_active_aux:n #1
{
\int_set:Nn \l_regex_current_state_int {#1}
\prop_clear:N \l_regex_current_submatches_prop
@@ -3604,7 +4081,7 @@
%
% \subsubsection{Actions when matching}
%
-% \begin{macro}[aux]{\regex_action_start_wildcard:}
+% \begin{macro}[int]{\regex_action_start_wildcard:}
%: the first state has a free transition to the second
% state, where the regular expression really begins, and a costly
% transition to itself, to try again at the next character. ^^A ??
@@ -3627,7 +4104,7 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_action_cost:n}
+% \begin{macro}[int]{\regex_action_cost:n}
% A transition which consumes the current character and moves
% to state |#1|.
% \begin{macrocode}
@@ -3639,7 +4116,7 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_action_success:}
+% \begin{macro}[int]{\regex_action_success:}
% There is a successful match when an execution path reaches
% the end of the regular expression. Then store the current
% step and submatches. The current step is then interrupted
@@ -3650,12 +4127,12 @@
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_action_success:
{
- \regex_last_match_empty:F
+ \regex_if_two_empty_matches:F
{
\bool_set_true:N \l_regex_success_match_bool
\bool_set_eq:NN \l_regex_success_empty_bool
\l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int
+ \int_set_eq:NN \l_regex_success_index_int \l_regex_current_index_int
\prop_set_eq:NN \l_regex_success_submatches_prop
\l_regex_current_submatches_prop
\prg_map_break:
@@ -3664,7 +4141,7 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_action_free:n}
+% \begin{macro}[int]{\regex_action_free:n}
% To copy a thread, check whether the program state has already
% been used at this character. If not, store submatches in the
% new state, and insert the instructions for that state in the
@@ -3674,7 +4151,7 @@
% \begin{macrocode}
\cs_new_protected:Npn \regex_action_free:n #1
{
- \cs_set_nopar:Npx \regex_tmp:w
+ \use:x
{
\int_add:Nn \l_regex_current_state_int {#1}
\regex_state_use:
@@ -3683,23 +4160,34 @@
\tl_set:Nn \exp_not:N \l_regex_current_submatches_prop
{ \exp_not:o \l_regex_current_submatches_prop }
}
- \regex_tmp:w
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\regex_action_submatch:n}
+% \begin{macro}[int]{\regex_action_submatch:n}
% Update the current submatches with the information
-% from the current step.
+% from the current index.
% \begin{macrocode}
\cs_new_protected:Npn \regex_action_submatch:n #1
{
\prop_put:Nno \l_regex_current_submatches_prop {#1}
- { \int_use:N \l_regex_current_step_int }
+ { \int_use:N \l_regex_current_index_int }
}
% \end{macrocode}
% \end{macro}
%
+% \subsubsection{??}%^^A todo: title
+%
+% \begin{macro}{\regex_if_match_empty:T, \regex_if_match_empty:F}
+%
+% \begin{macrocode}
+\cs_new:Npn \regex_if_match_empty:T
+ { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int }
+\cs_new:Npn \regex_if_match_empty:F
+ { \int_compare:nNnF \l_regex_start_index_int = \l_regex_current_index_int }
+% \end{macrocode}
+% \end{macro}
+%
% \subsection{Replacement}
%
% \begin{macro}[rEXP]{\regex_submatch_nesting_aux:n}
@@ -3717,7 +4205,7 @@
% \begin{macro}[int]{\regex_replacement:n}
% Our goal here is to analyse the replacement text. First take care
% of detecting escaped and non-escaped characters using
-% \cs{str_escape_use:NNNn} with three protected arguments.
+% \cs{regex_escape_use:nnnn} with three protected arguments.
% This inserts in the input stream a token list of the form
% \meta{fn 1} \meta{char 1} \ldots{} \meta{fn $N$} \meta{char $N$},
% where \meta{fn $i$} is one of the three functions, and \meta{char $i$}
@@ -3730,10 +4218,10 @@
\int_zero:N \l_regex_replacement_int
\int_zero:N \l_regex_nesting_int
\tl_clear:N \l_regex_nesting_tl
- \str_escape_use:NNNn
- \regex_replacement_unescaped:N
- \regex_replacement_escaped:N
- \regex_replacement_raw:N
+ \regex_escape_use:nnnn
+ { \regex_replacement_unescaped:N ##1 }
+ { \regex_replacement_escaped:N ##1 }
+ { \regex_replacement_raw:N ##1 }
{#1}
\prg_do_nothing: \prg_do_nothing:
\cs_set:Npx \regex_nesting:n ##1
@@ -3745,9 +4233,9 @@
\use:x
{
\exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 }
- { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int }
+ { \regex_toks_range:nn \c_zero \l_regex_replacement_int }
}
- % ^^A rename! Careful with \cP\#.
+ % ^^A rename!
}
% \end{macrocode}
% \end{macro}
@@ -3788,19 +4276,14 @@
}
\cs_new_protected:Npn \regex_replacement_escaped:N #1
{
- \if_charcode:w c #1
- \exp_after:wN \regex_replacement_c:w
- \else:
- \if_charcode:w g #1
- \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w
- \else:
+ \cs_if_exist_use:cF { regex_replacement_#1:w }
+ {
\if_num:w \c_one < 1#1 \exp_stop_f:
\regex_replacement_put_submatch:n {#1}
\else:
\regex_replacement_put:n #1
\fi:
- \fi:
- \fi:
+ }
}
\cs_new_protected:Npn \regex_replacement_put_submatch:n #1
{
@@ -3827,7 +4310,7 @@
{ \exp_not:n { #1#2 } }
{ \regex_replacement_unescaped:N \c_lbrace_str }
{
- \int_zero:N \l_regex_tmpa_int
+ \int_zero:N \l_regex_internal_a_int
\regex_replacement_g_digits:NN
}
{ \regex_replacement_error:NNN g #1 #2 }
@@ -3837,13 +4320,13 @@
\token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
{
\if_num:w \c_one < 1#2 \exp_stop_f:
- \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 }
+ \int_set:Nn \l_regex_internal_a_int { \c_ten * \l_regex_internal_a_int + #2 }
\exp_after:wN \use_i:nnn
\exp_after:wN \regex_replacement_g_digits:NN
\else:
\if_charcode:w \c_rbrace_str #2
\exp_args:No \regex_replacement_put_submatch:n
- { \int_use:N \l_regex_tmpa_int }
+ { \int_use:N \l_regex_internal_a_int }
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
\else:
\exp_after:wN \exp_after:wN
@@ -3855,6 +4338,16 @@
{ \regex_replacement_error:NNN g }
#1 #2
}
+\cs_new_protected:Npn \regex_replacement_u:w #1#2
+ {
+ \str_if_eq:xxTF { #1#2 } { \regex_replacement_unescaped:N \c_lbrace_str }
+ {
+ \int_incr:N \l_regex_replacement_csnames_int
+ \regex_replacement_put:n
+ { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } }
+ }
+ { \regex_replacement_error:NNN u #1#2 }
+ }
\cs_new_protected:Npn \regex_replacement_c:w #1#2
{
\token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
@@ -3870,8 +4363,6 @@
\regex_replacement_put:n
{ \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
}
-\cs_new_eq:cc { regex_replacement_c_C:w }
- { regex_replacement_c_ \c_lbrace_str :w }
\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
\group_begin:
@@ -3989,11 +4480,11 @@
{
\group_begin:
\regex_build:n {#3}
- \cs_set_nopar:Npx \regex_tmp:w
- { #1 \exp_not:N #2 { \regex_set_aux:N #2 } }
- \exp_after:wN
- \group_end:
- \regex_tmp:w
+ \use:x
+ {
+ \group_end:
+ #1 \exp_not:N #2 { \regex_set_aux:N #2 }
+ }
}
% \end{macrocode}
% \end{macro}
@@ -4026,7 +4517,7 @@
\regex_match_loop_caseless_hook:
}
\prg_stepwise_function:nnnN
- {0} {1} {\l_regex_max_state_int - \c_one }
+ \c_zero \c_one {\l_regex_max_state_int - \c_one }
\regex_set_aux:n
\s_stop
}
@@ -4050,29 +4541,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int, rEXP]{\regex_to_str:N}
-% \begin{macro}[int, rEXP]{\regex_to_str_aux:w}
-% Of course, we could simply set \cs{regex_to_str:N} equal
-% to \cs{tl_to_str:N}. After all, regex variables are in particular
-% token list variables. We do some more processing to start each
-% line with \cs{tex_toks:D} instead of a single very long line.
-% \begin{macrocode}
-\cs_new:Npn \regex_to_str:N #1
- {
- \exp_after:wN \regex_to_str_aux:Nw #1
- \tex_toks:D \q_stop \prg_map_break: \tex_toks:D
- \prg_break_point:n { }
- }
-\cs_new:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D
- {
- \use_none_delimit_by_q_stop:w #2 \q_stop
- \tl_to_str:n {#1#2} \iow_newline:
- \regex_to_str_aux:Nw \tex_toks:D
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
% \subsubsection{Generic auxiliary functions}
%
% Most of \pkg{l3regex}'s work is done within a group.
@@ -4236,9 +4704,9 @@
\cs_new_protected:Npn \regex_group_end_extract_seq:N #1
{
\cs_set_eq:NN \seq_item:n \scan_stop:
- \flag_lower:N \l_regex_group_begin_flag
- \flag_lower:N \l_regex_group_end_flag
- \tl_set:Nx \l_regex_tmpa_tl
+ \flag_clear:n { regex_begin }
+ \flag_clear:n { regex_end }
+ \tl_set:Nx \l_regex_internal_a_tl
{
\prg_stepwise_function:nnnN
\l_regex_max_state_int
@@ -4246,12 +4714,13 @@
{ \l_regex_submatch_int - \c_one }
\regex_extract_seq_aux:n
}
- \flag_test:NF \l_regex_group_begin_flag
- { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } }
- \msg_kernel_error:nn { regex } { sequence-unbalanced }
- \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \int_compare:nNnF
+ { \flag_height:n { regex_begin } + \flag_height:n { regex_end } }
+ = \c_zero
+ { \msg_kernel_error:nn { regex } { sequence-unbalanced } }
+ \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
\exp_args:NNNo \group_end:
- \tl_set:Nn #1 \l_regex_tmpa_tl
+ \tl_set:Nn #1 \l_regex_internal_a_tl
}
\cs_new:Npn \regex_extract_seq_aux:n #1
{
@@ -4262,14 +4731,14 @@
}
}
\cs_new:Npn \regex_extract_seq_aux:ww #1; #2;
- { %^^A todo: use flags
+ {
\if_num:w #1 < \c_zero
- \flag_raise:N \l_regex_group_end_flag
+ \flag_raise:n { regex_end }
\prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } }
\fi:
\regex_query_submatch:w #2;
\if_num:w #1 > \c_zero
- \flag_raise:N \l_regex_group_begin_flag
+ \flag_raise:n { regex_begin }
\prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } }
\fi:
}
@@ -4406,20 +4875,20 @@
\group_begin:
\tl_set:Nn \l_regex_every_match_tl
{
- \if_num:w \l_regex_start_step_int < \l_regex_success_step_int
+ \if_num:w \l_regex_start_index_int < \l_regex_success_index_int
\regex_extract:
\tex_skip:D \l_regex_submatch_start_int
- = \l_regex_start_step_int sp
+ = \l_regex_start_index_int sp
plus \tex_skip:D \l_regex_submatch_start_int \scan_stop:
\fi:
\regex_match_once:
}
#1 {#2} {#3}
\tex_skip:D \l_regex_submatch_int
- = \l_regex_start_step_int sp
- plus \l_regex_current_step_int sp \scan_stop:
+ = \l_regex_start_index_int sp
+ plus \l_regex_current_index_int sp \scan_stop:
\int_incr:N \l_regex_submatch_int
- \if_num:w \l_regex_start_step_int = \l_regex_current_step_int
+ \if_num:w \l_regex_start_index_int = \l_regex_current_index_int
\if_meaning:w \c_true_bool \l_regex_success_empty_bool
\int_decr:N \l_regex_submatch_int
\fi:
@@ -4448,34 +4917,34 @@
\exp_args:Nno #1 {#2} #4
\if_meaning:w \c_true_bool \g_regex_success_bool
\regex_extract:
- \int_set:Nn \l_regex_tmpa_int
+ \int_set:Nn \l_regex_internal_a_int
{ \regex_nesting:n { \l_regex_submatch_start_int } }
- \if_num:w \l_regex_tmpa_int = \c_zero
+ \if_num:w \l_regex_internal_a_int = \c_zero
\else:
\msg_kernel_error:nnx { regex } { replace-unbalanced }
- { \l_regex_tmpa_int }
+ { \l_regex_internal_a_int }
\fi:
- \tl_set:Nx \l_regex_tmpa_tl
+ \tl_set:Nx \l_regex_internal_a_tl
{
- \if_num:w \l_regex_tmpa_int < \c_zero
- \prg_replicate:nn { - \l_regex_tmpa_int }
+ \if_num:w \l_regex_internal_a_int < \c_zero
+ \prg_replicate:nn { - \l_regex_internal_a_int }
{ \exp_not:n { { \if_false: } \fi: } }
\fi:
- \regex_query_substr:nn
- { \l_regex_min_step_int }
+ \regex_toks_range:nn
+ { \l_regex_min_index_int }
{ \tex_skip:D \l_regex_submatch_start_int }
\regex_replacement_tl:n { \l_regex_submatch_start_int }
- \regex_query_substr:nn
+ \regex_toks_range:nn
{ \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int }
- { \l_regex_max_step_int }
- \if_num:w \l_regex_tmpa_int > \c_zero
- \prg_replicate:nn { \l_regex_tmpa_int }
+ { \l_regex_max_index_int }
+ \if_num:w \l_regex_internal_a_int > \c_zero
+ \prg_replicate:nn { \l_regex_internal_a_int }
{ \exp_not:n { \if_false: { \fi: } } }
\fi:
}
- \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
\exp_args:NNNo \group_end:
- \tl_set:Nn #4 \l_regex_tmpa_tl
+ \tl_set:Nn #4 \l_regex_internal_a_tl
\else:
\group_end:
\fi:
@@ -4500,12 +4969,12 @@
\regex_extract:
\tex_skip:D \l_regex_submatch_start_int
= \tex_the:D \tex_skip:D \l_regex_submatch_start_int
- minus \l_regex_start_step_int sp \scan_stop:
+ minus \l_regex_start_index_int sp \scan_stop:
\regex_match_once:
}
\regex_replacement:n {#3}
\exp_args:Nno #1 {#2} #4
- \int_set:Nn \l_regex_tmpa_int
+ \int_set:Nn \l_regex_internal_a_int
{
0
\prg_stepwise_function:nnnN
@@ -4514,15 +4983,15 @@
{ \l_regex_submatch_int - \c_one }
\regex_nesting:n
}
- \if_num:w \l_regex_tmpa_int = \c_zero
+ \if_num:w \l_regex_internal_a_int = \c_zero
\else:
\msg_kernel_error:nnx { regex } { replace-unbalanced }
- { \l_regex_tmpa_int }
+ { \l_regex_internal_a_int }
\fi:
- \tl_set:Nx \l_regex_tmpa_tl
+ \tl_set:Nx \l_regex_internal_a_tl
{
- \if_num:w \l_regex_tmpa_int < \c_zero
- \prg_replicate:nn { - \l_regex_tmpa_int }
+ \if_num:w \l_regex_internal_a_int < \c_zero
+ \prg_replicate:nn { - \l_regex_internal_a_int }
{ \exp_not:n { { \if_false: } \fi: } }
\fi:
\prg_stepwise_function:nnnN
@@ -4530,20 +4999,20 @@
\l_regex_capturing_group_int
{ \l_regex_submatch_int - \c_one }
\regex_replace_all_aux:n
- \regex_query_substr:nn
- \l_regex_start_step_int \l_regex_max_step_int
- \if_num:w \l_regex_tmpa_int > \c_zero
- \prg_replicate:nn { \l_regex_tmpa_int }
+ \regex_toks_range:nn
+ \l_regex_start_index_int \l_regex_max_index_int
+ \if_num:w \l_regex_internal_a_int > \c_zero
+ \prg_replicate:nn { \l_regex_internal_a_int }
{ \exp_not:n { \if_false: { \fi: } } }
\fi:
}
- \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
\exp_args:NNNo \group_end:
- \tl_set:Nn #4 \l_regex_tmpa_tl
+ \tl_set:Nn #4 \l_regex_internal_a_tl
}
\cs_new:Npn \regex_replace_all_aux:n #1
{
- \regex_query_substr:nn
+ \regex_toks_range:nn
{ \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 }
\regex_replacement_tl:n {#1}
}
@@ -4552,14 +5021,152 @@
%
% \subsection{Messages}
%
+% Messages for the preparsing phase.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { trailing-backslash }
+ { Trailing~escape~character~\iow_char:N\\. }
+ {
+ A~regular~expression~or~its~replacement~text~ends~with~
+ the~escape~character~\iow_char:N\\.~It~will~be~ignored.
+ }
+\msg_kernel_new:nnnn { regex } { x-missing-rbrace }
+ { Missing~closing~brace~in~\iow_char:N\\x~hexadecimal~sequence. }
+ {
+ You~wrote~something~like~
+ `\iow_char:N\\x\{\int_to_hexadecimal:n{#1}'.~
+ The~closing~brace~is~missing.
+ }
+\msg_kernel_new:nnnn { regex } { x-overflow }
+ { Character~code~'#1'~too~large~in~\iow_char:N\\x~hexadecimal~sequence. }
+ {
+ You~wrote~something~like~
+ `\iow_char:N\\x\{\int_to_hexadecimal:n{#1}\}'.~
+ The~character~code~'#1'~is~larger~than~\int_use:N \c_max_char_int.
+ }
+% \end{macrocode}
+%
+% Messages for missing or extra closing brackets and parentheses, with
+% some fancy singular/plural handling for the case of parentheses.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { missing-rbrack }
+ { Missing~right~bracket~inserted~in~regular~expression. }
+ {
+ LaTeX~was~given~a~regular~expression~where~a~character~class~
+ was~started~with~'[',~but~the~matching~']'~is~missing.
+ }
+\msg_kernel_new:nnnn { regex } { missing-rparen }
+ {
+ Missing~right~parenthes\int_compare:nTF{#1=1}{i}{e}s~
+ inserted~in~regular~expression.
+ }
+ {
+ LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~
+ more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~
+ parenthes\int_compare:nTF{#1=1}{i}{e}s.
+ }
+\msg_kernel_new:nnnn { regex } { extra-rparen }
+ { Extra~right~parenthesis~ignored~in~regular~expression. }
+ {
+ LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~
+ was~open.~The~parenthesis~will~be~ignored.
+ }
+% \end{macrocode}
+%
+% Sometimes escaped alphanumerics are not allowed everywhere.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { class-bad-escape }
+ { Invalid~escape~\c_backslash_str #1~in~character~class. }
+ {
+ The~escape~sequence~\iow_char:N\\#1~may~not~appear~within~
+ a~character~class.~For~instance,~assertions~(which~match~zero~
+ characters)~would~not~make~sense~there.
+ }
+\msg_kernel_new:nnnn { regex } { catcode-bad-escape }
+ { Invalid~escape~\c_backslash_str #1~following~category~test. }
+ {
+ The~escape~sequence~\iow_char:N\\#1~may~not~appear~following~
+ a~category~test~such~as~\iow_char:N\\cL~or~
+ \iow_char:N\\c[\iow_char:N\^BE].
+ }
+% \end{macrocode}
+%
+% Range errors.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { range-missing-end }
+ { Invalid~end-point~for~range~'#1-#2'~in~character~class. }
+ {
+ The~end-point~'#2'~of~the~range~'#1-#2'~may~not~serve~as~an~
+ end-point~for~a~range:~alphanumeric~characters~should~not~be~
+ escaped,~and~non-alphanumeric~characters~should~be~escaped.
+ }
+\msg_kernel_new:nnnn { regex } { range-backwards }
+ { Range~[#1-#2]~out~of~order~in~character~class. }
+ {
+ In~ranges~of~characters~[x-y]~appearing~in~character~classes,~
+ the~first~character~code~must~not~be~larger~than~the~second.~
+ Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~
+ character~code~\int_eval:n {`#2}.
+ }
+% \end{macrocode}
+%
+% Errors related to |\c| and |\u|.
+% \begin{macrocode}
+\msg_kernel_new:nnnn { regex } { c-bad-mode }
+ { Invalid~nested~\iow_char:N\\c~escape~in~regular~expression. }
+ {
+ The~\iow_char:N\\c~escape~cannot~be~used~within~
+ a~control~sequence~test~'\iow_char:N\\c{...}'.~
+ To~combine~several~category~tests,~use~'\iow_char:N\\c[...]'.
+ }
+\msg_kernel_new:nnnn { regex } { c-missing-rbrace }
+ { Missing~right~brace~inserted~for~\iow_char:N\\c~escape. }
+ {
+ LaTeX~was~given~a~regular~expression~where~a~
+ '\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~
+ with~a~closing~brace~'\iow_char:N\}'.
+ }
+\msg_kernel_new:nnnn { regex } { c-missing-rbrack }
+ { Missing~right~bracket~inserted~for~\iow_char:N\\c~escape. }
+ {
+ A~construction~'\iow_char:N\\c[...'~appears~in~a~
+ regular~expression,~but~the~closing~']'~is~not~present.
+ }
+\msg_kernel_new:nnnn { regex } { c-missing-category }
+ { Invalid~character~'#1'~following~\iow_char\\c~escape. }
+ {
+ In~regular~expressions,~the~\iow_char:N\\c~escape~sequence~
+ may~only~be~followed~by~a~left~brace,~a~left~bracket,~or~a~
+ capital~letter~representing~a~character~category,~namely~
+ one~of~ABCDELMOPSTU.
+ }
+\msg_kernel_new:nnnn { regex } { u-missing-lbrace }
+ { Missing~left~brace~following~\iow_char:N\\u~escape. }
+ {
+ The~\iow_char:N\\u~escape~sequence~must~be~followed~by~
+ a~brace~group~with~the~name~of~the~variable~to~use.
+ }
+\msg_kernel_new:nnnn { regex } { u-missing-rbrace }
+ { Missing~right~brace~inserted~for~\iow_char:N\\u~escape. }
+ {
+ LaTeX~
+ \tl_if_empty:xTF {#2}
+ { reached~the~end~of~the~string~ }
+ { encountered~an~escaped~alphanumeric~character '\iow_char:N\\#2'~ }
+ when~parsing~the~argument~of~an~'\iow_char:N\\u\iow_char:N\{...\}'~escape.
+ }
+% \end{macrocode}
+%
+% In various cases, the result of a \pkg{l3regex} operation can leave us
+% with an unbalanced token list, which we must re-balance by adding
+% begin-group or end-group character tokens.
% \begin{macrocode}
\msg_kernel_new:nnnn { regex } { sequence-unbalanced }
{
Missing~
- \flag_test:NTF \l_regex_group_end_flag
+ \flag_if_raised:nTF { regex_end }
{
left~
- \flag_test:NTF \l_regex_group_begin_flag
+ \flag_if_raised:nTF { regex_begin }
{ and~right~braces } { brace }
}
{ right~brace }
@@ -4591,35 +5198,8 @@
}
% \end{macrocode}
%
+% Messages related to NFA variables.
% \begin{macrocode}
-\msg_kernel_new:nnnn { regex } { missing-rparen }
- { Missing~right~parenthesis~added~in~regular~expression. }
- {
- LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~
- more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~
- parenthes\int_compare:nTF{#1=1}{i}{e}s.
- }
-\msg_kernel_new:nnnn { regex } { extra-rparen }
- { Extra~right~parenthesis~ignored~in~regular~expression. }
- {
- LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~
- was~open.~The~parenthesis~will~be~ignored.
- }
-\msg_kernel_new:nnnn { regex } { backwards-range }
- { Range~[#1-#2]~out~of~order~in~character~class. }
- {
- In~ranges~of~characters~[x-y]~appearing~in~character~classes,~
- the~first~character~code~must~not~be~larger~than~the~second.~
- Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~
- character~code~\int_eval:n {`#2}.
- }
-\msg_kernel_new:nnnn { regex } { unsupported }
- { Unsupported~construction~`#1'. }
- {
- The~construction~`#1'~is~not~supported~by~the~LaTeX~
- regular~expression~module.~Perhaps~some~character~should~
- have~been~escaped?
- }
\msg_kernel_new:nnnn { regex } { not-nfa }
{ This~is~not~a~regular~expression~variable. }
{
@@ -4628,29 +5208,8 @@
is~\\\\
\ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 .
}
-% \end{macrocode}
-%
-% \begin{macrocode}
-% \msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern }
-% \msg_new:nnn { regex } { 2 } { \iow_char:N\\c~at~end~of~pattern }
-% \msg_new:nnn { regex } { 4 }
-% { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. }
-% \msg_new:nnn { regex } { 6 }
-% { Missing~terminating~\iow_char:N\]~for~character~class }
-% \msg_new:nnn { regex } { 7 }
-% { Invalid~escape~sequence~in~character~class }
-% \msg_new:nnn { regex } { 34 }
-% { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large }
-% \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string }
-% \msg_new:nnn { regex } { 46 }
-% { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence }
-% \msg_new:nnn { regex } { 47 }
-% { Unknown~property~after~\iow_char:N\\P~or\iow_char:N\\p }
-% \msg_new:nnn { regex } { 68 }
-% { \iow_char:N\\c~must~be~followed~by~an~ASCII~character }
-% \end{macrocode}
-%
-% \begin{macrocode}
+\msg_kernel_new:nnn { regex } { nfa-misused }
+ { Automaton~#1 used~incorrectly. }
% \end{macrocode}
%
% \begin{macrocode}
@@ -4683,17 +5242,21 @@
% \end{macrocode}
%
% \begin{macrocode}
-\msg_kernel_new:nnn { regex } { nfa-misused }
- { Automaton~#1 used~incorrectly. }
-% \end{macrocode}
-%
-% \begin{macrocode}
%</package>
% \end{macrocode}
%
% \end{implementation}
%
% \endinput
+%^^A
+%^^A \msg_new:nnn { regex } { 4 }
+%^^A { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. }
+%^^A \msg_new:nnn { regex } { 44 } { Invalid~UTF-8~string }
+%^^A \msg_new:nnn { regex } { 46 }
+%^^A { Malformed~\iow_char:N\\P~or\iow_char:N\\p~sequence }
+%^^A \msg_new:nnn { regex } { 47 }
+%^^A { Unknown~property~after~\iow_char:N\\P~or\iow_char:N\\p }
+%^^A
%^^A NOT IMPLEMENTED
%^^A \cx "control-x", where x is any ASCII character
%^^A \C one byte, even in UTF-8 mode (best avoided)
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
index cddd7349adc..3d73ed10b8f 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
@@ -35,13 +35,14 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3159 2012-01-09 00:51:58Z bruno $
+\GetIdInfo$Id: l3str.dtx 3257 2012-01-29 13:37:03Z bruno $
{L3 Experimental Strings}
%</driver|package>
%<*driver>
\documentclass[full]{l3doc}
\usepackage{amsmath}
\begin{document}
+ \tableofcontents
\DocInput{\jobname.dtx}
\end{document}
%</driver>
@@ -75,25 +76,23 @@
% characters.
%
% String variables are simply specialised token lists, but by convention
-% should be named with the suffix \texttt{\ldots str}. Such variables should
-% contain characters with category code $12$ (other), except spaces, which
-% have category code $10$ (blank space). All the functions in
-% this module first convert their argument to a string for internal processing,
-% and will not treat a token list or the corresponding string representation
-% differently.
+% should be named with the suffix \texttt{\ldots str}. Such variables
+% should contain characters with category code $12$ (other), except
+% spaces, which have category code $10$ (blank space). All the functions
+% in this module first convert their argument to a string for internal
+% processing, and will not treat a token list or the corresponding
+% string representation differently.
%
% Most functions in this module come in three flavours:
% \begin{itemize}
% \item \cs{str_...:N...}, which expect a token list or string
% variable as their argument;
-% \item \cs{str_...:n...}, taking any token list (or string) as an argument;
-% \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered
-% during the operation: these functions are faster than those
-% which take care of escaping spaces appropriately;
+% \item \cs{str_...:n...}, taking any token list (or string) as an
+% argument;
+% \item \cs{str_..._ignore_spaces:n...}, which ignores any space
+% encountered during the operation: these functions are faster than
+% those which take care of escaping spaces appropriately;
% \end{itemize}
-% When performance is critical, the internal \cs{str_..._unsafe:n...}
-% functions, which expect an \enquote{other string} in which spaces
-% have category code $12$ instead of $10$, might be useful.
%
% \section{Building strings}
%
@@ -106,8 +105,9 @@
% \c_tilde_str,
% \c_percent_str
% }
-% Constant strings, containing a single character, with category code $12$.
-% Any character can be accessed as \cs{iow_char:N} |\|\meta{character}.
+% Constant strings, containing a single character, with category code
+% $12$. Any character can be accessed as \cs{iow_char:N}
+% |\|\meta{character}.
% \end{variable}
%
% \begin{function}[EXP]{\tl_to_str:N, \tl_to_str:n}
@@ -115,28 +115,36 @@
% \cs{tl_to_str:N} \meta{tl var}
% \cs{tl_to_str:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}, leaving the resulting
-% tokens in the input stream.
+% Converts the \meta{token list} to a \meta{string}, leaving the
+% resulting tokens in the input stream.
% \begin{texnote}
-% The string representation of a token list may depend
-% on the category codes in effect when it is measured,
-% and the value of the \tn{escapechar}: for instance
-% |\tl_to_str:n {\a}| may produce $1$, $2$ or $3$
-% characters depending on the escape character, and
-% the category code of \texttt{a}. This impacts almost
-% all functions in the module, which use \cs{tl_to_str:n}
-% internally.
+% The string representation of a token list may depend on the
+% category codes in effect when it is evaluated, and the value of
+% the \tn{escapechar}: for instance |\tl_to_str:n {\a}| normally
+% produces the three character \enquote{backslash},
+% \enquote{lower-case a}, \enquote{space}, but it may also produce
+% $1$ or $2$ characters depending on the escape character, and the
+% category code of \texttt{a}. This impacts almost all functions in
+% the module, which use \cs{tl_to_str:n} internally.
% \end{texnote}
% \end{function}
%
+% \begin{function}{\str_new:N, \str_new:c}
+% \begin{syntax}
+% \cs{str_new:N} \meta{str~var}
+% \end{syntax}
+% Creates a new \meta{str~var} or raises an error if the name is
+% already taken. The declaration is global. The \meta{str~var} will
+% initially be empty.
+% \end{function}
+%
% \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx}
% \begin{syntax}
% \cs{str_const:Nn} \meta{str~var} \Arg{token list}
% \end{syntax}
-% Creates a new constant \meta{str~var} or raises an error
-% if the name is already taken. The value of the
-% \meta{str~var} will be set globally to the
-% \meta{token list}, converted to a string.
+% Creates a new constant \meta{str~var} or raises an error if the name
+% is already taken. The value of the \meta{str~var} will be set
+% globally to the \meta{token list}, converted to a string.
% \end{function}
%
% \begin{function}
@@ -147,8 +155,8 @@
% \begin{syntax}
% \cs{str_set:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and stores the result in \meta{str var}.
+% Converts the \meta{token list} to a \meta{string}, and stores the
+% result in \meta{str var}.
% \end{function}
%
% \begin{function}
@@ -161,10 +169,9 @@
% \begin{syntax}
% \cs{str_put_left:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and prepends the result to \meta{str var}.
-% The current contents of the \meta{str var} are not
-% automatically converted to a string.
+% Converts the \meta{token list} to a \meta{string}, and prepends the
+% result to \meta{str var}. The current contents of the \meta{str
+% var} are not automatically converted to a string.
% \end{function}
%
% \begin{function}
@@ -177,50 +184,9 @@
% \begin{syntax}
% \cs{str_put_right:Nn} \meta{str var} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and appends the result to \meta{str var}.
-% The current contents of the \meta{str var} are not
-% automatically converted to a string.
-% \end{function}
-%
-% \begin{function}{\str_input:Nn, \str_ginput:Nn}
-% \begin{syntax}
-% \cs{str_input:Nn} \meta{str var} \Arg{token list}
-% \end{syntax}
-% Converts the \meta{token list} into a \meta{string}, and stores
-% it in the \meta{str var}.
-% Special characters can be input by
-% escaping them with a backslash.
-% \begin{itemize}
-% \item Spaces are ignored unless escaped with a backslash.
-% \item |\xhh| produces the character with code \texttt{hh}
-% in hexadecimal: when |\x| is encountered, up to two hexadecimal digits
-% (\texttt{0}--\texttt{9}, \texttt{a}--\texttt{f},
-% \texttt{A}--\texttt{F})
-% are read to give a number between $0$ and $255$.
-% \item |\x{hh...}| produces the character with code \texttt{hh...}
-% (an arbitrary number of hexadecimal digits are read):
-% this is mostly useful for Unicode-aware engines.
-% \item |\a|, |\e|, |\f|, |\n|, |\r|, |\t| stand for specific characters:
-% \begin{center}
-% \begin{tabular}{cccc}
-% |\a| & |\^^G| & alarm & hex \texttt{07} \\
-% |\e| & |\^^[| & escape & hex \texttt{1B} \\
-% |\f| & |\^^L| & form feed & hex \texttt{0C} \\
-% |\n| & |\^^J| & new line & hex \texttt{0A} \\
-% |\r| & |\^^M| & carriage return & hex \texttt{0D} \\
-% |\t| & |\^^I| & horizontal tab & hex \texttt{09} \\
-% \end{tabular}
-% \end{center}
-% \end{itemize}
-% For instance,
-% \begin{quote}
-% \cs{tl_new:N} \cs{l_my_str} \\
-% \cs{str_input:Nn} \cs{l_my_str} |{\x3C \\ \# abc\ def\^\n}|
-% \end{quote}
-% results in \cs{l_my_str} containing the characters |<\#abc def^|,
-% followed by a newline character (hex \texttt{0A})
-% since |<| has \textsc{ascii} code \texttt{3C} (in hexadecimal).
+% Converts the \meta{token list} to a \meta{string}, and appends the
+% result to \meta{str var}. The current contents of the \meta{str
+% var} are not automatically converted to a string.
% \end{function}
%
% \section{Characters given by their position}
@@ -231,63 +197,67 @@
% \cs{str_length:n} \Arg{token list}
% \end{syntax}
% Leaves the length of the string representation of \meta{token list}
-% in the input stream as an integer denotation. The functions differ in
-% their treatment of spaces.
-% In the case of \cs{str_length:N} and \cs{str_length:n},
-% all characters including spaces are counted.
-% The \cs{str_length_ignore_spaces:n} leaves the number of non-space
+% in the input stream as an integer denotation. The functions differ
+% in their treatment of spaces. In the case of \cs{str_length:N} and
+% \cs{str_length:n}, all characters including spaces are counted. The
+% \cs{str_length_ignore_spaces:n} leaves the number of non-space
% characters in the input stream.
% \end{function}
%
+% \begin{function}[EXP]{\str_count_spaces:N, \str_count_spaces:n}
+% \begin{syntax}
+% \cs{str_count_spaces:n} \Arg{token list}
+% \end{syntax}
+% Leaves in the input stream the number of space characters in the
+% string representation of \meta{token list}, as an integer
+% denotation. Of course, this function has no \texttt{_ignore_spaces}
+% variant.
+% \end{function}
+%
% \begin{function}[EXP]{\str_head:N, \str_head:n, \str_head_ignore_spaces:n}
% \begin{syntax}
% \cs{str_head:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} into a \meta{string}.
-% The first character in the \meta{string} is then
-% left in the input stream, with category code \enquote{other}.
-% The functions differ in their treatment of spaces.
-% In the case of \cs{str_head:N} and \cs{str_head:n},
-% a leading space is returned with category code $10$ (blank space).
-% The \cs{str_head_ignore_spaces:n} function leaves the first
-% non-space character in the input stream.
-% If the \meta{token list} is empty (or blank in the case of the
-% \texttt{_ignore_spaces} variant), then nothing is left on the
-% input stream.
+% Converts the \meta{token list} into a \meta{string}. The first
+% character in the \meta{string} is then left in the input stream,
+% with category code \enquote{other}. The functions differ in their
+% treatment of spaces. In the case of \cs{str_head:N} and
+% \cs{str_head:n}, a leading space is returned with category code $10$
+% (blank space). The \cs{str_head_ignore_spaces:n} function leaves
+% the first non-space character in the input stream. If the
+% \meta{token list} is empty (or blank in the case of the
+% \texttt{_ignore_spaces} variant), then nothing is left on the input
+% stream.
% \end{function}
%
% \begin{function}[EXP]{\str_tail:N, \str_tail:n, \str_tail_ignore_spaces:n}
% \begin{syntax}
% \cs{str_tail:n} \Arg{token list}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% removes the first character, and leaves
-% the remaining characters (if any) in the input stream,
-% with category codes $12$ and $10$ (for spaces).
-% The functions differ in the case where the first character
-% is a space: \cs{str_tail:N} and \cs{str_tail:n} will trim
-% only that space, while \cs{str_tail_ignore_spaces:n} removes
-% the first non-space character and any space before it.
-% If the \meta{token list} is empty (or blank in the case of the
-% \texttt{_ignore_spaces} variant), then nothing is left on the
-% input stream.
+% Converts the \meta{token list} to a \meta{string}, removes the first
+% character, and leaves the remaining characters (if any) in the input
+% stream, with category codes $12$ and $10$ (for spaces). The
+% functions differ in the case where the first character is a space:
+% \cs{str_tail:N} and \cs{str_tail:n} will trim only that space, while
+% \cs{str_tail_ignore_spaces:n} removes the first non-space character
+% and any space before it. If the \meta{token list} is empty (or
+% blank in the case of the \texttt{_ignore_spaces} variant), then
+% nothing is left on the input stream.
% \end{function}
%
% \begin{function}[EXP]{\str_item:Nn, \str_item:nn, \str_item_ignore_spaces:nn}
% \begin{syntax}
% \cs{str_item:nn} \Arg{token list} \Arg{integer expression}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}, and
-% leaves in the input stream the character in position
-% \meta{integer expression} of the \meta{string}.
-% In the case of \cs{str_item:Nn} and \cs{str_item:nn},
-% all characters including spaces are taken into account.
-% The \cs{str_item_ignore_spaces:nn} function skips spaces
-% in its argument.
-% If the \meta{integer expression} is negative, characters
-% are counted from the end of the \meta{string}. Hence, $-1$ is
-% the right-most character, \emph{etc.}, while $0$ is the first
-% (left-most) character.
+% Converts the \meta{token list} to a \meta{string}, and leaves in the
+% input stream the character in position \meta{integer expression} of
+% the \meta{string}. In the case of \cs{str_item:Nn} and
+% \cs{str_item:nn}, all characters including spaces are taken into
+% account. The \cs{str_item_ignore_spaces:nn} function skips spaces
+% in its argument. If the \meta{integer expression} is negative,
+% characters are counted from the end of the \meta{string}. Hence,
+% $-1$ is the right-most character, \emph{etc.}, while $0$ is the
+% first (left-most) character.
% \end{function}
%
% \begin{function}[EXP]
@@ -295,16 +265,16 @@
% \begin{syntax}
% \cs{str_substr:nnn} \Arg{token list} \Arg{start index} \Arg{end index}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string},
-% and leaves in the input stream the characters
-% between \meta{start index} (inclusive) and \meta{end index} (exclusive).
+% Converts the \meta{token list} to a \meta{string}, and leaves in the
+% input stream the characters from the \meta{start index} inclusive to
+% the \meta{end index} exclusive. Note that the length of the
+% substring is equal to the difference between the two \meta{indices}.
% If either of \meta{start index} or \meta{end index} is negative,
-% then it is incremented by the length of the list.
-% If either of \meta{start index} or \meta{end index} is empty,
-% it is replaced by the corresponding end-point of the string.
-% Both \meta{start index} and \meta{end index} count from $0$
-% for the first (left most) character.
-% For instance,
+% then it is incremented by the length of the list. If either of
+% \meta{start index} or \meta{end index} is empty, it is replaced by
+% the corresponding end-point of the string. Both \meta{start index}
+% and \meta{end index} count from $0$ for the first (left most)
+% character. For instance,
% \begin{verbatim}
% \iow_term:x { \str_substr:nnn { abcdef } { 1 } { 4 } }
% \iow_term:x { \str_substr:nnn { abcdef } { -4 } { } }
@@ -330,8 +300,8 @@
% \begin{verbatim}
% \str_if_eq_p:xx { abc } { \tl_to_str:n { abc } }
% \end{verbatim}
-% is logically \texttt{true}. All versions of these functions are fully
-% expandable (including those involving an \texttt{x}-type
+% is logically \texttt{true}. All versions of these functions are
+% fully expandable (including those involving an \texttt{x}-type
% expansion).
% \end{function}
%
@@ -340,29 +310,31 @@
% Traditionally, string encodings only specify how strings of characters
% should be stored as bytes. However, the resulting lists of bytes are
% often to be used in contexts where only a restricted subset of bytes
-% are permitted (\textsc{pdf} string objects, \textsc{url}s).
-% Hence, storing strings of characters has two components.
+% are permitted (\emph{e.g.}, \textsc{pdf} string objects,
+% \textsc{url}s). Hence, storing a string of characters is done in two
+% steps.
% \begin{itemize}
-% \item The code points (\enquote{character codes})
-% are expressed as bytes following a given
-% \enquote{encoding}. This can be \textsc{utf-16},
-% \textsc{iso 8859-1}, \emph{etc.}
-% See Table~\ref{tab:encodings} for a list
-% of encodings supported.\footnote{Encodings and escapings
-% will be added as they are requested.}
-% \item Bytes are translated to \TeX{} tokens through
-% a given \enquote{escaping}. Those are defined for
-% the most part by the \texttt{pdf} file format.
-% See Table~\ref{tab:escapings} for a list
-% of escaping methods supported.\footnotemark
+% \item The code points (\enquote{character codes}) are expressed as
+% bytes following a given \enquote{encoding}. This can be
+% \textsc{utf-16}, \textsc{iso 8859-1}, \emph{etc.} See
+% Table~\ref{tab:encodings} for a list of supported
+% encodings.\footnote{Encodings and escapings will be added as they
+% are requested.}
+% \item Bytes are translated to \TeX{} tokens through a given
+% \enquote{escaping}. Those are defined for the most part by the
+% \texttt{pdf} file format. See Table~\ref{tab:escapings} for a
+% list of escaping methods supported.\footnotemark
% \end{itemize}
%
% \begin{table}\centering
% \caption{\label{tab:encodings}Supported encodings.
% Non-alphanumeric characters are ignored,
-% and capital letters are lower-cased.}
+% and capital letters are lower-cased
+% before searching for the encoding in this list.}
% \begin{tabular}{cc}
% \toprule
+% \meta{Encoding} & description \\
+% \midrule
% \texttt{utf8} & \textsc{utf-8} \\
% \texttt{utf16} & \textsc{utf-16}, with byte-order mark \\
% \texttt{utf16be} & \textsc{utf-16}, big-endian \\
@@ -386,8 +358,7 @@
% \texttt{iso885915}, \texttt{latin9} & \textsc{iso 8859-15} \\
% \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\
% \midrule
-% \texttt{native} & Native Unicode string. \\
-% \texttt{internal} & For curious programmers. \\
+% Empty & Native (Unicode) string. \\
% \bottomrule
% \end{tabular}
% \end{table}
@@ -395,9 +366,12 @@
% \begin{table}\centering
% \caption{\label{tab:escapings}Supported escapings.
% Non-alphanumeric characters are ignored,
-% and capital letters are lower-cased.}
+% and capital letters are lower-cased
+% before searching for the escaping in this list.}
% \begin{tabular}{cc}
% \toprule
+% \meta{Escaping} & description \\
+% \midrule
% \texttt{bytes}, or empty
% & arbitrary bytes \\
% \texttt{hex}, \texttt{hexadecimal}
@@ -412,86 +386,91 @@
% \end{tabular}
% \end{table}
%
-% \begin{function}{\str_set_convert:Nnnn}
+% \begin{function}{\str_set_convert:Nnnn,\str_gset_convert:Nnnn}
% \begin{syntax}
-% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string}
-% ~~\Arg{name 1} \Arg{name 2}
+% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string} \Arg{name~1} \Arg{name~2}
% \end{syntax}
-% This function converts the \meta{string} from the encoding
-% given by \meta{name 1} to the encoding given by \meta{name 2},
-% and stores the result in the \meta{str~var}.
-% Each \meta{name} can have the form \meta{encoding} or
-% \meta{encoding}\texttt{/}\meta{escaping}, where the possible values
-% of \meta{encoding} and \meta{escaping} are given in
-% Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively.
-% The default escaping is to input and output bytes directly.
-% The special encodings \texttt{native} and \texttt{internal}
-% do not support a \meta{escaping}, since those formats do not
-% correspond to strings of bytes.
+% This function converts the \meta{string} from the encoding given by
+% \meta{name~1} to the encoding given by \meta{name~2}, and stores the
+% result in the \meta{str~var}. Each \meta{name} can have the form
+% \meta{encoding} or \meta{encoding}\texttt{/}\meta{escaping}, where
+% the possible values of \meta{encoding} and \meta{escaping} are given
+% in Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively.
+% The default escaping is to input and output bytes directly. The
+% special case of an empty \meta{name} indicates the use of
+% \enquote{native} strings, 8-bit for pdf\TeX{}, and Unicode strings
+% for the other two engines.
%
% For example,
% \begin{verbatim}
-% \str_set_convert:Nnnn \l_foo_str { Hello! }
-% { native } { utf16/hex }
+% \str_set_convert:Nnnn \l_foo_str { Hello! } { } { utf16/hex }
% \end{verbatim}
% results in the variable \cs{l_foo_str} holding the string
-% \texttt{FEFF00480065006C006C006F0021}. This is obtained
-% by converting each character in the string \texttt{Hello!}
-% to the \textsc{utf-16} encoding, and expressing each byte
-% as a pair of hexadecimal digits. Note the presence of a
-% (big-endian) byte order mark \hexnum{FEFF}, which can be
-% avoided by specifying the encoding \texttt{utf16be/hex}.
-%
-% Parts of the \meta{string} which cannot be converted to bytes
-% using the \meta{escaping 1} are silently ignored (including non-byte
-% characters which could be present in the string).
-% Sequences of bytes which are not valid in the \meta{encoding 1}
-% are converted to the replacement character \hexnum{FFFD}, and raise
-% an error.
-% Characters which cannot be encoded in the \meta{encoding 2}
-% are silently ignored.
+% \texttt{FEFF00480065006C006C006F0021}. This is obtained by
+% converting each character in the (native) string \texttt{Hello!} to
+% the \textsc{utf-16} encoding, and expressing each byte as a pair of
+% hexadecimal digits. Note the presence of a (big-endian) byte order
+% mark \hexnum{FEFF}, which can be avoided by specifying the encoding
+% \texttt{utf16be/hex}.
+%
+% An error is raised if the \meta{string} is not valid according to
+% the \meta{escaping~1} and \meta{encoding~1}, or if it cannot be
+% reencoded in the \meta{encoding~2} and \meta{escaping~2} (for
+% instance, if a character does not exist in the \meta{encoding~2}).
+% Erroneous input is replaced by the Unicode replacement character
+% \hexnum{FFFD}, and characters which cannot be reencoded are replaced
+% by either the replacement character \hexnum{FFFD} if it exists in
+% the \meta{encoding~2}, or an encoding-specific replacement
+% character, or the question mark character.
% \end{function}
%
-% \section{Internal string functions}
-%
-% \begin{function}{\str_escape_use:NNNn}
+% \begin{function}[TF]{\str_set_convert:Nnnn,\str_gset_convert:Nnnn}
% \begin{syntax}
-% \cs{str_escape_use:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list}
+% \cs{str_set_convert:NnnnTF} \meta{str~var} \Arg{string} \Arg{name~1} \Arg{name~2} \Arg{true code} \Arg{false code}
% \end{syntax}
-% The \meta{token list} is converted to a string, then read from
-% left to right, interpreting backslashes as escaping the next character.
-% Unescaped characters are fed to the function \meta{fn1},
-% and escaped characters are fed to the function \meta{fn2}
-% within an \texttt{x}-expansion context (typically those functions
-% perform some tests on their argument to decide how to output them).
-% The escape sequences |\a|, |\e|, |\f|, |\n|, |\r|, |\t| and |\x|
-% are recognized as described for \cs{str_input:Nn}, and those
-% are replaced by the corresponding character, then fed to
-% \meta{fn3}. The result is then left in the input stream.
+% As \cs{str_set_convert:Nnnn}, converts the \meta{string} from the
+% encoding given by \meta{name~1} to the encoding given by
+% \meta{name~2}, and assigns the result to \meta{str~var}. Contrarily
+% to \cs{str_set_convert:Nnnn}, the conditional variant does not raise
+% errors in case the \meta{string} is not valid according to the
+% \meta{name~1} encoding, or cannot be expressed in the \meta{name~2}
+% encoding. Instead, the \meta{false code} is performed.
% \end{function}
%
-% \begin{function}[rEXP]{\str_aux_toks_range:nn}
-% \begin{syntax}
-% \cs{str_aux_toks_range:nn} \meta{start} \meta{end}
-% \end{syntax}
-% Expands to the contents of the \tn{toks} registers numbered
-% from \meta{start} (inclusive) to \meta{end} (exclusive).
-% This function is used in \pkg{l3regex}.
-% \end{function}
+% \section{Internal string functions}
+%
+% \begin{variable}{\c_max_char_int}
+% The maximum valid character code, $255$ for pdf\TeX{}, and $1114111$
+% for the two other engines.
+% \end{variable}
%
% \begin{function}{\str_aux_hexadecimal_use:NTF}
% \begin{syntax}
-% \cs{str_aux_hexadecimal_use:NTF} \meta{token}
-% ~~\Arg{true code} \Arg{false code}
+% \cs{str_aux_hexadecimal_use:NTF} \meta{token} \Arg{true code} \Arg{false code}
% \end{syntax}
-% If the \meta{token} is a hexdecimal digit (upper case or lower case),
-% its upper-case version is left in the input stream, \emph{followed}
-% by the \meta{true code}. Otherwise, the \meta{false code} is left
-% in the input stream.
+% If the \meta{token} is a hexdecimal digit (upper case or lower
+% case), its upper-case version is left in the input stream,
+% \emph{followed} by the \meta{true code}. Otherwise, the \meta{false
+% code} is left in the input stream.
% \begin{texnote}
% This function will fail if the escape character is a hexadecimal
% digit, or has a character code in the range $[0,5]$. We are thus
-% careful to set the escape character to a known value before using it.
+% careful to set the escape character to a known value before using
+% it.
+% \end{texnote}
+% \end{function}
+%
+% \begin{function}{\str_aux_octal_use:NTF}
+% \begin{syntax}
+% \cs{str_aux_octal_use:NTF} \meta{token} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% If the \meta{token} is an octal digit, it is left in the input
+% stream, \emph{followed} by the \meta{true code}. Otherwise, the
+% \meta{false code} is left in the input stream.
+% \begin{texnote}
+% This function will fail if the escape character is an octal
+% digit. We are thus careful to set the escape character to a known
+% value before using it.
% \end{texnote}
% \end{function}
%
@@ -502,9 +481,10 @@
% Converts the \meta{token list} to a \meta{other string}, where
% spaces have category code \enquote{other}.
% \begin{texnote}
-% These functions can be \texttt{f}-expanded without fear of losing
-% a leading space, since spaces do not have category code $10$ in
-% their result.
+% This function can be \texttt{f}-expanded without fear of losing a
+% leading space, since spaces do not have category code $10$ in its
+% result. This function takes a time quadratic in the length of the
+% string; \cs{str_gset_other:Nn} is faster but not expandable.
% \end{texnote}
% \end{function}
%
@@ -513,8 +493,8 @@
% \cs{str_gset_other:Nn} \meta{tl~var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} to an \meta{other string}, where
-% spaces have category code \enquote{other}, and assigns the result
-% to the \meta{tl~var}, globally.
+% spaces have category code \enquote{other}, and assigns the result to
+% the \meta{tl~var}, globally.
% \end{function}
%
% \begin{function}[EXP]
@@ -526,24 +506,56 @@
% \begin{syntax}
% \cs{str_if_contains_char:nNTF} \Arg{token list} \meta{char}
% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}
-% and tests whether the \meta{char} is present in the \meta{string}.
-% Spaces are ignored.
+% Converts the \meta{token list} to a \meta{string} and tests whether
+% the \meta{char} is present in the \meta{string}. Spaces are
+% ignored.
+% \end{function}
+%
+% \begin{function}{\str_declare_eight_bit_encoding:nnn}
+% \begin{syntax}
+% \cs{str_declare_eight_bit_encoding:nnn} \Arg{name} \Arg{mapping} \Arg{missing}
+% \end{syntax}
+% This declares the encoding \meta{name} to map bytes to Unicode
+% characters according to the \meta{mapping}, and map those bytes
+% which are not mentionned in the \meta{mapping} either to the
+% replacement character (if they appear in \meta{missing}), or to
+% themselves.
% \end{function}
%
% \section{Possibilities, and things to do}
%
+% Encoding/escaping-related tasks.
+% \begin{itemize}
+% \item Describe the internal format in the code comments. Refuse code
+% points in $[\hexnum{D800}, \hexnum{DFFF}]$ in the internal
+% representation?
+% \item Add documentation about each encoding and escaping method, and
+% add examples.
+% \item The \texttt{hex} unescaping should raise an error for
+% odd-length strings.
+% \item Decide what bytes should be escaped in the \texttt{url}
+% escaping. Perhaps |!'()*-./0123456789_| are safe, and all other
+% characters should be escaped?
+% \item Automate generation of 8-bit mapping files.
+% \item Change the framework for 8-bit encodings: for decoding from
+% 8-bit to Unicode, use $256$ integer registers; for encoding, use a
+% tree-box.
+% \item More encodings (see Heiko's \pkg{stringenc}). CESU?
+% \item More escapings: shell escapes, lua escapes, etc?
+% \end{itemize}
+%
+% Other string tasks.
% \begin{itemize}
-% \item Use flags rather than expandable errors.
-% \item Add documentation about each encoding and escaping method,
-% and add examples.
-% \item Perhaps use the pdf\TeX{} primitives for escapings when available.
-% But this changes error detection between engines.
-% \item More encodings (see Heiko's \pkg{stringenc}). CESU.
-% \item More escapings: shell escapes, lua escapes, etc?
-% \item \cs{str_if_head_eq:nN}
-% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}?
-% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}?
+% \item \cs{str_if_head_eq:nN}
+% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}?
+% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}?
+% \item Should \cs{str_substr:Nnn} be \cs{str_range:Nnn}?
+% \item Introduce \cs{str_slice:Nnnn} with a third \enquote{step}
+% argument? Or should we simply have \cs{str_slice:Nn}
+% \meta{string} \Arg{clist}, where the \meta{clist}'s items are
+% either one integer expression, two integer expressions separated
+% by |:|, or three integer expressions separated by |:|, \emph{cf.}
+% Python's extended slice syntax?
% \end{itemize}
%
% \end{documentation}
@@ -559,13 +571,14 @@
% \begin{macrocode}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3tl-analysis}
+\RequirePackage{l3tl-analysis,l3flag}
% \end{macrocode}
%
-% Those string-related functions are defined in \pkg{l3kernel}.
+% The following string-related functions are currently defined in
+% \pkg{l3kernel}.
% \begin{itemize}
% \item \cs{str_if_eq:nn}[pTF] and variants,
-% \item \cs{str_if_eq_return:on},
+% \item \cs{str_if_eq_return:on}, \cs{str_if_eq_return:xx}
% \item \cs{tl_to_str:n}, \cs{tl_to_str:N}, \cs{tl_to_str:c},
% \item \cs{token_to_str:N}, \cs{cs_to_str:N}
% \item \cs{str_head:n}, \cs{str_head_aux:w}, (copied here)
@@ -576,6 +589,16 @@
%
% \subsection{Helpers}
%
+% \subsubsection{A function unrelated to strings}
+%
+% \begin{macro}[EXP,aux]{\use_ii_i:nn}
+% A function used to swap its arguments.
+% \begin{macrocode}
+\cs_if_exist:NF \use_ii_i:nn
+ { \cs_new:Npn \use_ii_i:nn #1#2 { #2 #1 } }
+% \end{macrocode}
+% \end{macro}
+%
% \subsubsection{Assigning strings}
%
% \begin{macro}
@@ -597,65 +620,58 @@
% }
% Simply convert the token list inputs to \meta{strings}.
% \begin{macrocode}
-\group_begin:
- \cs_set_protected_nopar:Npn \str_tmp:w #1
- {
- \cs_new_protected:cpx { str_ #1 :Nn } ##1##2
- { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } }
- \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx }
- }
- \str_tmp:w { set }
- \str_tmp:w { gset }
- \str_tmp:w { const }
- \str_tmp:w { put_left }
- \str_tmp:w { gput_left }
- \str_tmp:w { put_right }
- \str_tmp:w { gput_right }
-\group_end:
+\tl_map_inline:nn
+ {
+ { set }
+ { gset }
+ { const }
+ { put_left }
+ { gput_left }
+ { put_right }
+ { gput_right }
+ }
+ {
+ \cs_new_protected:cpx { str_ #1 :Nn } ##1##2
+ { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } }
+ \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx }
+ }
% \end{macrocode}
% \end{macro}
%
% \subsubsection{Variables and constants}
%
% \begin{macro}{\str_tmp:w}
-% \begin{variable}{\l_str_tmpa_tl}
+% \begin{variable}{\l_str_internal_int}
+% \begin{variable}{\l_str_internal_tl}
% Internal scratch space for some functions.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_tmp:w { }
-\tl_new:N \l_str_tmpa_tl
-\int_new:N \l_str_tmpa_int
+\tl_new:N \l_str_internal_tl
+\int_new:N \l_str_internal_int
% \end{macrocode}
% \end{variable}
+% \end{variable}
% \end{macro}
%
% \begin{variable}{\g_str_result_tl}
-% The \cs{g_str_result_tl} variable is used to hold the result
-% of various internal string operations which are typically
-% performed in a group. The variable is global so that it remains
-% defined outside the group, to be assigned to a user provided variable.
+% The \cs{g_str_result_tl} variable is used to hold the result of
+% various internal string operations (mostly conversions) which are
+% typically performed in a group. The variable is global so that it
+% remains defined outside the group, to be assigned to a user-provided
+% variable.
% \begin{macrocode}
\tl_new:N \g_str_result_tl
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_str_char_int}
-% When converting from various forms to characters,
-% \cs{l_str_char_int} is the character code of the
-% character which should be created.
-% \begin{macrocode}
-\int_new:N \l_str_char_int
-% \end{macrocode}
-% \end{variable}
-%
% \begin{variable}
% {
% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one,
% \c_ninety_seven, \c_one_hundred_twenty_three,
% \c_one_hundred_twenty_seven
% }
-% We declare here some integer values which delimit ranges of
-% ASCII characters of various types. This is mostly used in
-% \pkg{l3regex}.
+% We declare here some integer values which delimit ranges of ASCII
+% characters of various types. This is mostly used in \pkg{l3regex}.
% \begin{macrocode}
\int_const:Nn \c_forty_eight { 48 }
\int_const:Nn \c_fifty_eight { 58 }
@@ -668,10 +684,11 @@
% \end{variable}
%
% \begin{variable}{\c_max_char_int}
-% The maximum character code depends on the engine.
+% The maximum valid character code is $255$ for pdf\TeX{}, and
+% $1114111$ for other engines.
% \begin{macrocode}
\int_const:Nn \c_max_char_int
- { \pdftex_if_engine:TF { 255 } { 1114111 } }
+ { \pdftex_if_engine:TF { "FF } { "10FFFF } }
% \end{macrocode}
% \end{variable}
%
@@ -692,8 +709,8 @@
% \c_tilde_str,
% \c_percent_str
% }
-% For all of those strings, \cs{cs_to_str:N} produce characters
-% with the correct category code.
+% For all of those strings, \cs{cs_to_str:N} produces characters with
+% the correct category code.
% \begin{macrocode}
\tl_const:Nx \c_backslash_str { \cs_to_str:N \\ }
\tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ }
@@ -704,22 +721,46 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\g_str_aliases_prop}
-% To avoid needing one file per encoding/escaping alias,
-% we keep track of those in a property list.
+% \begin{variable}{\g_str_file_prop}
+% To avoid needing one file per encoding/escaping alias, we keep track
+% of those in a property list.
+% \begin{macrocode}
+\prop_new:N \g_str_file_prop
+\prop_gput:Nnn \g_str_file_prop { latin1 } { iso88591 }
+\prop_gput:Nnn \g_str_file_prop { latin2 } { iso88592 }
+\prop_gput:Nnn \g_str_file_prop { latin3 } { iso88593 }
+\prop_gput:Nnn \g_str_file_prop { latin4 } { iso88594 }
+\prop_gput:Nnn \g_str_file_prop { latin5 } { iso88599 }
+\prop_gput:Nnn \g_str_file_prop { latin6 } { iso885910 }
+\prop_gput:Nnn \g_str_file_prop { latin7 } { iso885913 }
+\prop_gput:Nnn \g_str_file_prop { latin8 } { iso885914 }
+\prop_gput:Nnn \g_str_file_prop { latin9 } { iso885915 }
+\prop_gput:Nnn \g_str_file_prop { latin10 } { iso885916 }
+\prop_gput:Nnn \g_str_file_prop { utf16le } { utf16 }
+\prop_gput:Nnn \g_str_file_prop { utf16be } { utf16 }
+\prop_gput:Nnn \g_str_file_prop { utf32le } { utf32 }
+\prop_gput:Nnn \g_str_file_prop { utf32be } { utf32 }
+\prop_gput:Nnn \g_str_file_prop { hexadecimal } { hex }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_str_error_bool}
+% In conversion functions with a built-in conditional, errors are not
+% reported directly to the user, but the information is collected in
+% this boolean, used at the end to decide on which branch of the
+% conditional to take.
+% \begin{macrocode}
+\bool_new:N \g_str_error_bool
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{str_byte, str_error}
+% Conversions from one \meta{encoding}/\meta{escaping} pair to another
+% are done within \texttt{x}-expanding assignments. Errors are
+% signalled by raising the relevant flag.
% \begin{macrocode}
-\prop_new:N \g_str_aliases_prop
-\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 }
-\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 }
-\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 }
-\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 }
-\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 }
-\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 }
-\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 }
-\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 }
-\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 }
-\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 }
-\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex }
+\flag_new:n { str_byte }
+\flag_new:n { str_error }
% \end{macrocode}
% \end{variable}
%
@@ -727,10 +768,10 @@
%
% \begin{macro}[EXP]{\tl_to_other_str:n}
% \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w}
-% Replaces all spaces by \enquote{other} spaces, after converting
-% the token list to a string via \cs{tl_to_str:n}.
-% This function is \texttt{f}-expandable, storing the part of the string
-% with escaped spaces between the \cs{q_mark} and \cs{q_stop} markers.
+% Replaces all spaces by \enquote{other} spaces, after converting the
+% token list to a string via \cs{tl_to_str:n}. This function is
+% \texttt{f}-expandable, storing the part of the string with escaped
+% spaces between the \cs{q_mark} and \cs{q_stop} markers.
% \begin{macrocode}
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
@@ -743,7 +784,7 @@
\exp_after:wN \tl_to_other_str_loop:w \tl_to_str:n {#1} ~ %
A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
}
- \cs_new_nopar:Npn \tl_to_other_str_loop:w
+ \cs_new:Npn \tl_to_other_str_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 \q_stop
{
\if_meaning:w A #8
@@ -752,7 +793,7 @@
\tl_to_other_str_loop:w
#9 #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * \q_stop
}
- \cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
+ \cs_new:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
{ \fi: #2 }
}
% \end{macrocode}
@@ -762,12 +803,11 @@
% \begin{macro}[int]{\str_gset_other:Nn}
% \begin{macro}[aux,EXP]{\str_gset_other_loop:w}
% \begin{macro}[aux,EXP]{\str_gset_other_end:w}
-% This function could be done by using \cs{tl_to_other_str:n}
-% within an \texttt{x}-expansion, but that would take a time
-% quadratic in the size of the string. Instead, we can
-% \enquote{leave the result behind us} in the input stream,
-% to be captured into the expanding assignment. This gives us
-% a linear time.
+% This function could be done by using \cs{tl_to_other_str:n} within
+% an \texttt{x}-expansion, but that would take a time quadratic in the
+% size of the string. Instead, we can \enquote{leave the result behind
+% us} in the input stream, to be captured into the expanding
+% assignment. This gives us a linear time.
% \begin{macrocode}
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
@@ -780,10 +820,10 @@
\tl_gset:Nx #1
{
\exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ %
- A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
+ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_stop
}
}
- \cs_new_nopar:Npn \str_gset_other_loop:w
+ \cs_new:Npn \str_gset_other_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~
{
\if_meaning:w A #9
@@ -792,7 +832,7 @@
#1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9
\str_gset_other_loop:w *
}
- \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
+ \cs_new:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
{ \fi: #1 }
}
% \end{macrocode}
@@ -800,66 +840,70 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Functions unrelated to strings}
-%
-% \begin{macro}[EXP,aux]{\use_i:nnnnnnnn}
-% A function which may already be defined elsewhere.
-% \begin{macrocode}
-\cs_if_exist:NF \use_i:nnnnnnnn
- { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
-% \end{macrocode}
-% \end{macro}
+% \subsection{Characters given by their position}
%
-% \begin{macro}[int, rEXP]{\str_aux_toks_range:nn}
-% \begin{macro}[aux, rEXP]{\str_aux_toks_range:ww}
-% Some non-expandable functions (in this module,
-% \cs{str_aux_escape:NNNn}, which is used by \cs{str_(g)input:Nn})
-% store pieces of their result in \tn{toks} registers.
-% Those pieces are concatenated using \cs{str_aux_toks_range:nn},
-% which expects two integer registers, \meta{start} and \meta{end},
-% and expands to the contents of the \tn{toks} registers from
-% \meta{start} (inclusive) to \meta{end} (exclusive).
-% This function is also used in the \pkg{l3regex} module.
+% \begin{macro}[EXP]{\str_count_spaces:N}
+% \begin{macro}[EXP]{\str_count_spaces:n}
+% \begin{macro}[EXP,aux]{\str_count_spaces_loop:wwwwwwwww}
+% To speed up this function, we grab $9$ spaces in each step. The
+% loop stops when the last argument is one of the trailing
+% |X|\meta{number}, and that \meta{number} is added to the sum of $9$
+% that preceeds, to adjust the result.
% \begin{macrocode}
-\cs_new:Npn \str_aux_toks_range:nn #1#2
+\cs_new:Npn \str_count_spaces:N
+ { \exp_args:No \str_count_spaces:n }
+\cs_new:Npn \str_count_spaces:n #1
{
- \exp_after:wN \str_aux_toks_range:ww
- \int_use:N \int_eval:w #1 \exp_after:wN ;
- \int_use:N \int_eval:w #2 ;
- \prg_break_point:n { }
+ \int_eval:n
+ {
+ \exp_after:wN \str_count_spaces_loop:wwwwwwwww
+ \tl_to_str:n {#1} ~
+ X 7 ~ X 6 ~ X 5 ~ X 4 ~ X 3 ~ X 2 ~ X 1 ~ X 0 ~ X -1 ~
+ \q_stop
+ }
}
-\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ;
+\cs_new:Npn \str_count_spaces_loop:wwwwwwwww #1~#2~#3~#4~#5~#6~#7~#8~#9~
{
- \if_num:w #1 = #2 \exp_stop_f:
- \exp_after:wN \prg_map_break:
+ \if_meaning:w X #9
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
- \tex_the:D \tex_toks:D #1 \exp_stop_f:
- \exp_after:wN \str_aux_toks_range:ww
- \int_use:N \int_eval:w #1 + \c_one ; #2 ;
+ \c_nine + \str_count_spaces_loop:wwwwwwwww
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-%
-% \subsection{Characters given by their position}
+% \end{macro}
%
% \begin{macro}[EXP]{\str_length:N}
% \begin{macro}[EXP]{\str_length:n}
-% \begin{macro}[EXP]{\str_length_ignore_spaces:n}
% \begin{macro}[EXP]{\str_length_unsafe:n}
-% \begin{macro}[EXP,aux]{\str_length_aux:n,\str_length_loop:NNNNNNNNN}
-% The length of a string is found by first changing all spaces
-% to other spaces using \cs{tl_to_other_str:n}, then counting
-% characters $9$ at a time. When the end is reached, |#9|
-% has the form |X|\meta{digit}, the catcode test is true,
-% the digit gets added to the sum, and the loop is terminated
-% by \cs{use_none_delimit_by_q_stop:w}.
+% \begin{macro}[EXP]{\str_length_ignore_spaces:n}
+% \begin{macro}[EXP,aux]{\str_length_aux:n, \str_length_loop:NNNNNNNNN}
+% To measure the length of a string we could first escape all spaces
+% using \cs{tl_to_other_str:N}, then measure the length of this token
+% list. However, this would be quadratic in the length of the string,
+% and we can do better. Namely, add the number of spaces (counted
+% using the functions defined above) to the length ignoring spaces.
+% To measure the length ignoring spaces we use the same technique as
+% for counting spaces: loop, grabbing $9$ characters at each step, and
+% end as soon as we reach one of the $9$ trailing items. The
+% \texttt{_unsafe} variant expects a token list consisting entirely of
+% category code $12$ characters.
% \begin{macrocode}
\cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n }
\cs_new:Npn \str_length:n #1
- { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } }
-\cs_new_nopar:Npn \str_length_unsafe:n #1
- { \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } }
+ {
+ \str_length_aux:n
+ {
+ \str_count_spaces:n {#1}
+ + \exp_after:wN \str_length_loop:NNNNNNNNN \tl_to_str:n {#1}
+ }
+ }
+\cs_new:Npn \str_length_unsafe:n #1
+ {
+ \str_length_aux:n
+ { \str_length_loop:NNNNNNNNN #1 }
+ }
\cs_new:Npn \str_length_ignore_spaces:n #1
{
\str_length_aux:n
@@ -878,7 +922,7 @@
}
\cs_set:Npn \str_length_loop:NNNNNNNNN #1#2#3#4#5#6#7#8#9
{
- \if_catcode:w X #9
+ \if_meaning:w X #9
\exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
\c_nine + \str_length_loop:NNNNNNNNN
@@ -893,21 +937,18 @@
% \begin{macro}[EXP]{\str_head:N}
% \begin{macro}[EXP]{\str_head:n}
% \begin{macro}[EXP]{\str_head_ignore_spaces:n}
-% \begin{macro}[EXP]{\str_head_unsafe:n}
% \begin{macro}[EXP,aux]{\str_head_aux:w}
-% The cases of \cs{str_head_ignore_spaces:n} and
-% \cs{str_head_unsafe:n} are mostly identical to
-% \cs{tl_head:n}.
-% As usual, \cs{str_head:N} expands its argument and hands it to
-% \cs{str_head:n}. To circumvent the fact that \TeX{} skips spaces
-% when grabbing undelimited macro parameters, \cs{str_head_aux:w}
-% takes an argument delimited by a space. If |#1| starts with a
-% non-space character, \cs{use_i_delimit_by_q_stop:nw} leaves that
-% in the input stream. On the other hand, if |#1| starts with a
-% space, the \cs{str_head_aux:w} takes an empty argument, and
+% The \texttt{_ignore_spaces} variant is almost identical to
+% \cs{tl_head:n}. As usual, \cs{str_head:N} expands its argument and
+% hands it to \cs{str_head:n}. To circumvent the fact that \TeX{}
+% skips spaces when grabbing undelimited macro parameters,
+% \cs{str_head_aux:w} takes an argument delimited by a space. If |#1|
+% starts with a non-space character, \cs{use_i_delimit_by_q_stop:nw}
+% leaves that in the input stream. On the other hand, if |#1| starts
+% with a space, the \cs{str_head_aux:w} takes an empty argument, and
% the single (braced) space in the definition of \cs{str_head_aux:w}
-% makes its way to the output. Finally, for an empty argument,
-% the (braced) empty brace group in the definition of \cs{str_head:n}
+% makes its way to the output. Finally, for an empty argument, the
+% (braced) empty brace group in the definition of \cs{str_head:n}
% gives an empty result after passing through
% \cs{use_i_delimit_by_q_stop:nw}.
% \begin{macrocode}
@@ -918,36 +959,33 @@
\tl_to_str:n {#1}
{ { } } ~ \q_stop
}
-\cs_set_nopar:Npn \str_head_aux:w #1 ~ %
+\cs_set:Npn \str_head_aux:w #1 ~ %
{ \use_i_delimit_by_q_stop:nw #1 { ~ } }
\cs_new:Npn \str_head_ignore_spaces:n #1
- { \exp_after:wN \use_i_delimit_by_q_stop:nw \tl_to_str:n {#1} { } \q_stop }
-\cs_new_nopar:Npn \str_head_unsafe:n #1
- { \use_i_delimit_by_q_stop:nw #1 { } \q_stop }
+ {
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tl_to_str:n {#1} { } \q_stop
+ }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
% \begin{macro}[EXP]{\str_tail:N}
% \begin{macro}[EXP]{\str_tail:n}
% \begin{macro}[EXP]{\str_tail_ignore_spaces:n}
-% \begin{macro}[EXP]{\str_tail_unsafe:n}
% \begin{macro}[EXP,aux]{\str_tail_aux:w}
% \begin{macro}[EXP,aux]{\str_tail_aux_ii:w}
-% As when fetching the head of a string, the cases
-% \enquote{\texttt{ignore_spaces:n}} and \enquote{\texttt{unsafe:n}}
-% are similar to \cs{tl_tail:n}.
-% The more commonly used \cs{str_tail:n} function is a little bit
-% more convoluted: hitting the front of the string with
-% \cs{reverse_if:N} \cs{if_charcode:w} \cs{scan_stop:}
-% removes the first character (which necessarily makes the test true,
-% since it cannot match \cs{scan_stop:}). The auxiliary function inserts
-% the required \cs{fi:} to close the conditional, and leaves the tail
-% of the string in the input string. The details are such that an empty
-% string has an empty tail.
+% As when fetching the head of a string, the \texttt{_ignore_spaces}
+% variant is similar to \cs{tl_tail:n}. The more commonly used
+% \cs{str_tail:n} function is a little bit more convoluted: hitting
+% the front of the string with \cs{reverse_if:N} \cs{if_charcode:w}
+% \cs{scan_stop:} removes the first character (which necessarily makes
+% the test true, since it cannot match \cs{scan_stop:}). The auxiliary
+% function inserts the required \cs{fi:} to close the conditional, and
+% leaves the tail of the string in the input string. The details are
+% such that an empty string has an empty tail.
% \begin{macrocode}
\cs_new_nopar:Npn \str_tail:N { \exp_args:No \str_tail:n }
\cs_set:Npn \str_tail:n #1
@@ -956,113 +994,103 @@
\reverse_if:N \if_charcode:w
\scan_stop: \tl_to_str:n {#1} X X \q_stop
}
-\cs_set_nopar:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 }
+\cs_set:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 }
\cs_new:Npn \str_tail_ignore_spaces:n #1
{
\exp_after:wN \str_tail_aux_ii:w
- \tl_to_str:n {#1} X { } X \q_stop
+ \tl_to_str:n {#1} X X \q_stop
}
-\cs_new_nopar:Npn \str_tail_unsafe:n #1
- { \str_tail_aux_ii:w #1 X { } X \q_stop }
-\cs_new_nopar:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 }
+\cs_new:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[EXP,int]{\str_skip_do:nn}
-% \begin{macro}[EXP,aux]{\str_skip_aux:nnnnnnnnn}
-% \begin{macro}[EXP,aux]{\str_skip_end:nn}
-% \begin{macro}[EXP,aux]{\str_skip_end_ii:nwn}
-% Removes |max(#1,0)| characters then leaves |#2| in the input stream.
-% We remove characters $7$ at a time. When the number of characters
-% to remove is not a multiple of $7$, we need to remove less than
-% $7$ characters in the last step. This is done by inserting a number
-% of \texttt{X}, which are discarded as if they were part of the string.
+% \begin{macro}[EXP, int]{\str_skip_c_zero:w}
+% \begin{macro}[EXP, aux]{\str_skip_aux:wNNNNNNNN}
+% \begin{macro}[EXP, aux]{\str_skip_end:w, \str_skip_end_ii:NNNNNNNN}
+% Removes |max(#1,0)| characters from the input stream, and then
+% leaves \cs{c_zero}. This should be expanded using
+% \cs{tex_romannumeral:D}. We remove characters $8$ at a time until
+% there are at most $8$ to remove. Then we do a dirty trick: the
+% \cs{if_case:w} construction leaves between $0$ and $8$ times the
+% \cs{or:} control sequence, and those \cs{or:} become arguments of
+% \cs{str_skip_end_ii:NNNNNNNN}. If the number of characters to remove
+% is $6$, say, then there are two \cs{or:} left, and the $8$ arguments
+% of \cs{str_skip_end_ii:NNNNNNNN} are the two \cs{or:}, and $6$
+% characters from the input stream, exactly what we wanted to
+% remove. Then close the \cs{if_case:w} conditional with \cs{fi:}, and
+% stop the initial expansion with \cs{c_zero} (see places where
+% \cs{str_skip_c_zero:w} is called).
% \begin{macrocode}
-\cs_new:Npn \str_skip_do:nn #1
+\cs_new:Npn \str_skip_c_zero:w #1;
{
- \if_num:w \int_eval:w #1 > \c_seven
- \exp_after:wN \str_skip_aux:nnnnnnnnn
+ \if_num:w \int_eval:w #1 > \c_eight
+ \exp_after:wN \str_skip_aux:wNNNNNNNN
\else:
- \exp_after:wN \str_skip_end:n
+ \exp_after:wN \str_skip_end:w
+ \int_use:N \int_eval:w
\fi:
- {#1}
+ #1 ;
}
-\cs_new:Npn \str_skip_aux:nnnnnnnnn #1#2#3#4#5#6#7#8#9
- { \exp_args:Nf \str_skip_do:nn { \int_eval:n { #1 - \c_seven } } {#2} }
-\cs_new:Npn \str_skip_end:n #1
+\cs_new:Npn \str_skip_aux:wNNNNNNNN #1; #2#3#4#5#6#7#8#9
+ { \exp_after:wN \str_skip_c_zero:w \int_use:N \int_eval:w #1 - \c_eight ; }
+\cs_new:Npn \str_skip_end:w #1 ;
{
- \if_case:w \int_eval:w #1 \int_eval_end:
- \str_skip_end_ii:nwn { XXXXXXX }
- \or: \str_skip_end_ii:nwn { XXXXXX }
- \or: \str_skip_end_ii:nwn { XXXXX }
- \or: \str_skip_end_ii:nwn { XXXX }
- \or: \str_skip_end_ii:nwn { XXX }
- \or: \str_skip_end_ii:nwn { XX }
- \or: \str_skip_end_ii:nwn { X }
- \or: \str_skip_end_ii:nwn { }
- \else: \str_skip_end_ii:nwn { XXXXXXX }
- \fi:
+ \exp_after:wN \str_skip_end_ii:NNNNNNNN
+ \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f:
+ \or: \or: \or: \or: \or: \or: \or: \or:
}
-\cs_new:Npn \str_skip_end_ii:nwn #1#2 \fi: #3
- { \fi: \use_i:nnnnnnnn {#3} #1 }
+\cs_new:Npn \str_skip_end_ii:NNNNNNNN #1#2#3#4#5#6#7#8 { \fi: \c_zero }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[EXP,int]{\str_collect_do:nn}
-% \begin{macro}[EXP,aux]{\str_collect_aux:n,\str_collect_aux:nnNNNNNNN}
+% \begin{macro}[EXP,int]{\str_collect_delimit_by_q_stop:w}
+% \begin{macro}[EXP,aux]{\str_collect_aux:wn, \str_collect_aux:wnNNNNNNN}
% \begin{macro}[EXP,aux]
-% {
-% \str_collect_end:nn,
-% \str_collect_end_ii:nwn,
-% \str_collect_end_iii:nwNNNNNNN
-% }
-% Collects |max(#1,0)| characters, and feeds them as an argument to |#2|.
-% Again, we grab $7$ characters at a time. Instead of inserting
-% a string of \texttt{X} to fill in to a multiple of $7$, we insert
-% empty groups, so that they disappear in this context where arguments
-% are accumulated.
+% {\str_collect_end:wn, \str_collect_end_ii:nnnnnnnnw} Collects
+% |max(#1,0)| characters, and removes everything else until
+% \cs{q_stop}. This is somewhat similar to \cs{str_skip_c_zero:w}, but
+% this time we can only grab $7$ characters at a time. At the end, we
+% use an \cs{if_case:w} trick again, so that the $8$ first arguments
+% of \cs{str_collect_end_ii:nnnnnnnnw} are some \cs{or:}, followed by
+% an \cs{if:}, followed by |#1| characters from the input
+% stream. Simply leaving this in the input stream will close the
+% conditional properly and the \cs{or:} disappear.
% \begin{macrocode}
-\cs_new:Npn \str_collect_do:nn #1#2
- { \str_collect_aux:n {#1} { \str_collect_end_iii:nwNNNNNNN {#2} } }
-\cs_new:Npn \str_collect_aux:n #1
+\cs_new:Npn \str_collect_delimit_by_q_stop:w #1;
{
- \int_compare:nNnTF {#1} > \c_seven
- { \str_collect_aux:nnNNNNNNN }
- { \str_collect_end:n }
- {#1}
+ \exp_after:wN \str_collect_aux:wn
+ \int_use:N \int_eval:w #1 ;
+ { }
}
-\cs_new:Npn \str_collect_aux:nnNNNNNNN #1#2 #3#4#5#6#7#8#9
+\cs_new:Npn \str_collect_aux:wn #1 ;
{
- \exp_args:Nf \str_collect_aux:n
- { \int_eval:n { #1 - \c_seven } }
- { #2 #3#4#5#6#7#8#9 }
+ \if_num:w #1 > \c_seven
+ \exp_after:wN \str_collect_aux:wnNNNNNNN
+ \else:
+ \exp_after:wN \str_collect_end:wn
+ \fi:
+ #1 ;
}
-\cs_new:Npn \str_collect_end:n #1
+\cs_new:Npn \str_collect_aux:wnNNNNNNN #1; #2 #3#4#5#6#7#8#9
{
- \if_case:w \int_eval:w #1 \int_eval_end:
- \str_collect_end_ii:nwn { { } { } { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } }
- \or: \str_collect_end_ii:nwn { { } }
- \or: \str_collect_end_ii:nwn { }
- \else: \str_collect_end_ii:nwn { { } { } { } { } { } { } { } }
- \fi:
+ \exp_after:wN \str_collect_aux:wn
+ \int_use:N \int_eval:w #1 - \c_seven ;
+ { #2 #3#4#5#6#7#8#9 }
}
-\cs_new:Npn \str_collect_end_ii:nwn #1#2 \fi: #3
- { \fi: #3 \q_stop #1 }
-\cs_new:Npn \str_collect_end_iii:nwNNNNNNN #1 #2 \q_stop #3#4#5#6#7#8#9
- { #1 {#2#3#4#5#6#7#8#9} }
+\cs_new:Npn \str_collect_end:wn #1 ;
+ {
+ \exp_after:wN \str_collect_end_ii:nnnnnnnnw
+ \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f:
+ \or: \or: \or: \or: \or: \or: \fi:
+ }
+\cs_new:Npn \str_collect_end_ii:nnnnnnnnw #1#2#3#4#5#6#7#8 #9 \q_stop
+ { #1#2#3#4#5#6#7#8 }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -1072,48 +1100,47 @@
% \begin{macro}[EXP]{\str_item:nn}
% \begin{macro}[EXP]{\str_item_ignore_spaces:nn}
% \begin{macro}[EXP]{\str_item_unsafe:nn}
-% \begin{macro}[EXP,aux]{\str_item_aux:nn}
-% This is mostly shuffling arguments around to avoid measuring
-% the length of the string more than once, and make sure that
-% the parameters given to \cs{str_skip_do:nn} are necessarily
-% within the bounds of the length of the string.
-% The \cs{str_item_ignore_spaces:nn} function cheats a little bit
-% in that it doesn't hand to \cs{str_item_unsafe:nn} an
-% \enquote{other string}. This is alright, as everything else
-% is done with undelimited arguments.
+% \begin{macro}[EXP,aux]{\str_item_aux:ww}
+% This is mostly shuffling arguments around to avoid measuring the
+% length of the string more than once, and make sure that the
+% parameters given to \cs{str_skip_c_zero:w} are necessarily within
+% the bounds of the length of the string. The \texttt{_ignore_spaces}
+% function cheats a little bit in that it doesn't hand to
+% \cs{str_item_unsafe:nn} an \enquote{other string}. This is alright,
+% as everything else is done with undelimited arguments.
% \begin{macrocode}
\cs_new_nopar:Npn \str_item:Nn { \exp_args:No \str_item:nn }
-\cs_new:Npn \str_item:nn #1#2
+\cs_new:Npn \str_item:nn #1
{
- \exp_args:Nf \tl_to_str:n
- { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} }
+ \exp_last_unbraced:Nf \token_to_str:N
+ \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} }
}
\cs_new:Npn \str_item_ignore_spaces:nn #1
{ \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } }
-\cs_new_nopar:Npn \str_item_unsafe:nn #1#2
+\cs_new:Npn \str_item_unsafe:nn #1#2
{
- \exp_args:Nff \str_item_aux:nn
- { \int_eval:n {#2} }
- { \str_length_unsafe:n {#1} }
+ \exp_after:wN \str_item_aux:ww
+ \int_use:N \int_eval:w #2 \exp_after:wN ;
+ \int_value:w \str_length_unsafe:n {#1} ;
#1
\q_stop
}
-\cs_new_nopar:Npn \str_item_aux:nn #1#2
+\cs_new:Npn \str_item_aux:ww #1; #2;
{
\int_compare:nNnTF {#1} < \c_zero
{
\int_compare:nNnTF {#1} < {-#2}
{ \use_none_delimit_by_q_stop:w }
{
- \str_skip_do:nn { #1 + #2 }
- { \use_i_delimit_by_q_stop:nw }
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tex_romannumeral:D \str_skip_c_zero:w #1 + #2 ;
}
}
{
\int_compare:nNnTF {#1} < {#2}
{
- \str_skip_do:nn {#1}
- { \use_i_delimit_by_q_stop:nw }
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tex_romannumeral:D \str_skip_c_zero:w #1 ;
}
{ \use_none_delimit_by_q_stop:w }
}
@@ -1128,24 +1155,27 @@
% \begin{macro}[EXP]{\str_substr:Nnn}
% \begin{macro}[EXP]{\str_substr:nnn}
% \begin{macro}[EXP]{\str_substr_ignore_spaces:nnn}
-% \begin{macro}[EXP]{\str_substr_unsafe:nnn}
-% \begin{macro}[EXP,aux]{\str_substr_aux:www}
-% \begin{macro}[EXP,aux]{\str_substr_aux:nnw}
-% \begin{macro}[EXP,aux]{\str_aux_normalize_range:nn}
-% Sanitize the string. Then evaluate the arguments, replacing
-% them by \cs{c_zero} or \cs{c_max_int} if they are empty.
-% This is done by using the construction
+% \begin{macro}[EXP,aux]
+% {
+% \str_substr_unsafe:nnn,
+% \str_substr_aux:nN,
+% \str_substr_aux:www,
+% \str_substr_aux:nnw,
+% \str_substr_normalize_range:nn
+% }
+% Sanitize the string. Then evaluate the arguments, replacing them by
+% \cs{c_zero} or \cs{c_max_int} if they are empty. This is done by
+% using the construction
% \begin{verbatim}
% \int_use:N \int_eval:w #2 \c_zero \c_zero ;
% \end{verbatim}
% which expands to the value of |#2|, followed by |\c_zero \c_zero ;|
% if |#2| is an expression, and expands to |0\c_zero ;| otherwise.
-% The same is done to the end-point of the range.
-% Then limit the range to be at most the length of the string
-% (this avoids needing to check for the end of the string when
-% grabbing characters).
-% Afterwards, skip characters, then keep some more, and finally
-% drop the end of the string.
+% The same is done to the end-point of the range. Then limit the
+% range to be at most the length of the string (this avoids needing to
+% check for the end of the string when grabbing characters).
+% Afterwards, skip characters, then keep some more, and finally drop
+% the end of the string.
% \begin{macrocode}
\cs_new_nopar:Npn \str_substr:Nnn { \exp_args:No \str_substr:nnn }
\cs_new:Npn \str_substr:nnn #1#2#3
@@ -1163,28 +1193,26 @@
\exp_after:wN \str_substr_aux:www
\int_value:w \str_length_unsafe:n {#1} \exp_after:wN ;
\int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ;
- \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ;
+ \int_use:N \int_eval:w
+ \exp_args:Nf \str_substr_aux:nN {#3} \c_max_int ;
#1
\q_stop
}
-\cs_new:Npn \str_substr_unsafe:nN #1 #2
+\cs_new:Npn \str_substr_aux:nN #1 #2
{ \tl_if_empty:nTF {#1} {#2} {#1} }
\cs_new:Npn \str_substr_aux:www #1; #2; #3;
{
\exp_args:Nf \str_substr_aux:nnw
- { \str_aux_normalize_range:nn {#2} {#1} }
- { \str_aux_normalize_range:nn {#3} {#1} }
+ { \str_substr_normalize_range:nn {#2} {#1} }
+ { \str_substr_normalize_range:nn {#3} {#1} }
}
\cs_new:Npn \str_substr_aux:nnw #1#2
{
- \str_skip_do:nn {#1}
- {
- \exp_args:Nf \str_collect_do:nn
- { \int_eval:n { #2 - #1 } }
- { \use_i_delimit_by_q_stop:nw }
- }
+ \exp_after:wN \str_collect_delimit_by_q_stop:w
+ \int_use:N \int_eval:w #2 - #1 \exp_after:wN ;
+ \tex_romannumeral:D \str_skip_c_zero:w #1 ;
}
-\cs_new:Npn \str_aux_normalize_range:nn #1#2
+\cs_new:Npn \str_substr_normalize_range:nn #1#2
{
\int_eval:n
{
@@ -1208,18 +1236,14 @@
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% \subsection{String conditionals}
%
% \begin{macro}[EXP,pTF]{\str_if_eq:NN}
% \begin{macro}[EXP,pTF]{\str_if_eq:nn,\str_if_eq:xx}
-% The \texttt{nn} and \texttt{xx} variants are already
-% defined in \pkg{l3basics}. Note that \cs{str_if_eq:NN}
-% is different from \cs{tl_if_eq:NN} because it needs to
-% ignore category codes.
+% The \texttt{nn} and \texttt{xx} variants are already defined in
+% \pkg{l3basics}. Note that \cs{str_if_eq:NN} is different from
+% \cs{tl_if_eq:NN} because it needs to ignore category codes.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_if_eq:NN #1#2 { p , TF , T , F }
{
@@ -1233,17 +1257,15 @@
% \begin{macro}[EXP]{\str_if_contains_char:NNT, \str_if_contains_char:NNTF}
% \begin{macro}[EXP]{\str_if_contains_char:nNTF}
% \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN}
-% \begin{macro}[EXP,aux]{\str_if_contains_char_end:}
+% \begin{macro}[EXP,aux]{\str_if_contains_char_true:}
% Loop over the characters of the string, comparing character codes.
-% We allow |#2| to be a single-character control sequence, hence the
-% use of \cs{int_compare:nNnT} rather than \cs{token_if_eq_charcode:NNT}.
% The loop is broken if character codes match. Otherwise we return
% \enquote{false}.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF }
{
\exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
- #1 { \prg_map_break:n { ? \fi: } }
+ \tl_to_str:n \exp_after:wN {#1} { \prg_map_break:n { ? \fi: } }
\prg_break_point:n { }
\prg_return_false:
}
@@ -1254,14 +1276,14 @@
\prg_break_point:n { }
\prg_return_false:
}
-\cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2
+\cs_new:Npn \str_if_contains_char_aux:NN #1#2
{
\if_charcode:w #1 #2
- \exp_after:wN \str_if_contains_char_end:
+ \exp_after:wN \str_if_contains_char_true:
\fi:
\str_if_contains_char_aux:NN #1
}
-\cs_new_nopar:Npn \str_if_contains_char_end:
+\cs_new_nopar:Npn \str_if_contains_char_true:
{ \prg_map_break:n { \prg_return_true: \use_none:n } }
% \end{macrocode}
% \end{macro}
@@ -1270,11 +1292,10 @@
% \end{macro}
%
% \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF}
-% \TeX{} dutifully detects octal digits for us: if |#1|
-% is an octal digit, then the right-hand side of the
-% comparison is |'1#1|, greater than $1$. Otherwise,
-% the right-hand side stops as |'1|, and the conditional
-% takes the \texttt{false} branch.
+% \TeX{} dutifully detects octal digits for us: if |#1| is an octal
+% digit, then the right-hand side of the comparison is |'1#1|, greater
+% than $1$. Otherwise, the right-hand side stops as |'1|, and the
+% conditional takes the \texttt{false} branch.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF }
{
@@ -1288,10 +1309,9 @@
% \end{macro}
%
% \begin{macro}[aux, rEXP]{\str_aux_hexadecimal_use:NTF}
-% \TeX{} detects uppercase hexadecimal digits for us
-% (see \cs{str_aux_octal_use:NTF}, but not the
-% lowercase letters, which we need to detect and replace
-% by their uppercase counterpart.
+% \TeX{} detects uppercase hexadecimal digits for us (see
+% \cs{str_aux_octal_use:NTF}), but not the lowercase letters, which we
+% need to detect and replace by their uppercase counterpart.
% \begin{macrocode}
\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF }
{
@@ -1321,25 +1341,24 @@
%
% \subsubsection{Producing one byte or character}
%
-% \begin{variable}{\c_str_byte_0_tl}
+% \begin{variable}{\c_str_byte_0_tl, \c_str_byte_1_tl, \c_str_byte_255_tl}
% \begin{variable}{\c_str_byte_-1_tl}
% \begin{variable}{\c_str_positive_bytes_tl}
-% For each integer $N$ in the range $[0,255]$, we create
-% a constant token list which holds three character tokens
-% with category code other: the character with character
-% code $N$, followed by the representation of $N$ as
-% two hexadecimal digits.
-% The value $-1$ is given a default token list which ensures
-% that later functions give an empty result for the input $-1$.
-% Simultaneously, we build a list of all bytes (from which we
-% remove the null byte) in \cs{c_str_positive_bytes_tl}.
+% For each integer $N$ in the range $[0,255]$, we create a constant
+% token list which holds three character tokens with category code
+% other: the character with character code $N$, followed by the
+% representation of $N$ as two hexadecimal digits. The value $-1$ is
+% given a default token list which ensures that later functions give
+% an empty result for the input $-1$. Simultaneously, we build a list
+% of all bytes (from which we remove the null byte) in
+% \cs{c_str_positive_bytes_tl}.
% \begin{macrocode}
\group_begin:
- \tl_clear:N \l_str_tmpa_tl
+ \tl_clear:N \l_str_internal_tl
\char_set_catcode_other:n { \c_zero }
\tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
- \tl_map_inline:Nn \g_str_result_tl
- { \char_set_lccode:nn {`#1} {`#1} }
+ \exp_args:No \tl_map_inline:nn { \g_str_result_tl " }
+ { \char_set_lccode:nn {`#1} { \c_zero } }
\tl_map_inline:Nn \g_str_result_tl
{
\tl_map_inline:Nn \g_str_result_tl
@@ -1350,12 +1369,12 @@
\tl_const:cx
{ c_str_byte_ \int_eval:n {"#1##1} _tl }
{ ^^@ #1 ##1 }
- \tl_put_right:Nn \l_str_tmpa_tl { ^^@ }
+ \tl_put_right:Nn \l_str_internal_tl { ^^@ }
}
}
}
\tl_const:Nx \c_str_positive_bytes_tl
- { \exp_after:wN \use_none:n \l_str_tmpa_tl }
+ { \exp_after:wN \use_none:n \l_str_internal_tl }
\group_end:
\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
% \end{macrocode}
@@ -1364,15 +1383,17 @@
% \end{variable}
%
% \begin{macro}[int, EXP]{\str_output_byte:n}
-% \begin{macro}[aux, EXP]{\str_output_byte:w}
+% \begin{macro}[int, EXP]{\str_output_byte:w}
% \begin{macro}[int, EXP]{\str_output_hexadecimal:n}
-% \begin{macro}[aux, EXP]{\str_output_hexadecimal:w}
-% \begin{macro}[aux, EXP]{\str_output_end:}
-% For now, no error detection, we simply pick either
-% the byte or the hexadecimal representation of it
-% from the three-character token list corresponding
-% to the argument.
-% The value $-1$ produces an empty result.
+% \begin{macro}[int, EXP]{\str_output_hexadecimal:w}
+% \begin{macro}[int, EXP]{\str_output_end:}
+% Those functions must be used carefully: feeding them a value outside
+% the range $[-1,255]$ will attempt to use the undefined token list
+% variable \cs{c_str_byte_\meta{number}_tl}. Assuming that the
+% argument is in the right range, we expand the corresponding token
+% list, and pick either the byte (first token) or the hexadecimal
+% representations (second and third tokens). The value $-1$ produces
+% an empty result in both cases.
% \begin{macrocode}
\cs_new:Npn \str_output_byte:n #1
{ \str_output_byte:w #1 \str_output_end: }
@@ -1399,445 +1420,346 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Mapping functions for encoding conversion}
+% \begin{macro}[int, rEXP]{\str_output_byte_pair_be:n}
+% \begin{macro}[int, rEXP]{\str_output_byte_pair_le:n}
+% \begin{macro}[aux, rEXP]{\str_output_byte_pair_aux:nnN}
+% Convert a number in the range $[0,65535]$ to a pair of bytes, either
+% big-endian or little-endian.
+% \begin{macrocode}
+\cs_new:Npn \str_output_byte_pair_be:n #1
+ {
+ \exp_args:Nf \str_output_byte_pair_aux:nnN
+ { \int_div_truncate:nn { #1 } { "100 } } {#1} \use:nn
+ }
+\cs_new:Npn \str_output_byte_pair_le:n #1
+ {
+ \exp_args:Nf \str_output_byte_pair_aux:nnN
+ { \int_div_truncate:nn { #1 } { "100 } } {#1} \use_ii_i:nn
+ }
+\cs_new:Npn \str_output_byte_pair_aux:nnN #1#2#3
+ {
+ #3
+ { \str_output_byte:n { #1 } }
+ { \str_output_byte:n { #2 - #1 * "100 } }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Mapping functions for conversions}
%
-% \begin{macro}{\str_aux_gmap_result:N}
-% \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN}
-% This maps the function |#1| over all characters
-% in \cs{g_str_result_tl}, which should be a byte
-% string in most cases, sometimes a native string.
+% \begin{macro}{\str_convert_gmap:N}
+% \begin{macro}[aux, rEXP]{\str_convert_gmap_loop:NN}
+% This maps the function |#1| over all characters in
+% \cs{g_str_result_tl}, which should be a byte string in most cases,
+% sometimes a native string.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1
+\cs_new_protected:Npn \str_convert_gmap:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_result_loop:NN
+ \exp_after:wN \str_convert_gmap_loop:NN
\exp_after:wN #1
\g_str_result_tl { ? \prg_map_break: }
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2
+\cs_new:Npn \str_convert_gmap_loop:NN #1#2
{
\use_none:n #2
#1#2
- \str_aux_gmap_result_loop:NN #1
+ \str_convert_gmap_loop:NN #1
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}{\str_aux_gmap_internal_result:N}
-% \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw}
-% This maps the function |#1| over all character codes
-% in \cs{g_str_result_tl}, which must be in the internal
-% representation.
+% \begin{macro}{\str_convert_gmap_internal:N}
+% \begin{macro}[aux, rEXP]{\str_convert_gmap_internal_loop:Nw}
+% This maps the function |#1| over all character codes in
+% \cs{g_str_result_tl}, which must be in the internal representation.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1
+\cs_new_protected:Npn \str_convert_gmap_internal:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_internal_result_loop:Nww
+ \exp_after:wN \str_convert_gmap_internal_loop:Nww
\exp_after:wN #1
\g_str_result_tl \s_tl \q_stop \prg_map_break: \s_tl
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1 #2 \s_tl #3 \s_tl
+\cs_new:Npn \str_convert_gmap_internal_loop:Nww #1 #2 \s_tl #3 \s_tl
{
\use_none_delimit_by_q_stop:w #3 \q_stop
#1 {#3}
- \str_aux_gmap_internal_result_loop:Nww #1
+ \str_convert_gmap_internal_loop:Nww #1
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \subsubsection{Unescape user input}
-%
-% The code of this section is used both here for \cs{str_(g)input:Nn},
-% and in the regular expression module to go through the regular expression
-% once before actually parsing it. The goal in that case is to turn any
-% character with a meaning in regular expressions (\texttt{*}, \texttt{?},
-% \texttt{\{}, etc.) into a marker indicating that this was a special
-% character,
-% and replace any escaped character by the corresponding unescaped character,
-% so that the \pkg{l3regex} code can avoid caring about escaping issues
-% (those can become quite complex to handle in combination with ranges
-% in character classes).
-%
-% The idea is to feed unescaped characters to one function,
-% escaped characters to another, and feed |\a|, |\e|, |\f|,
-% |\n|, |\r|, |\t| and |\x| converted to the appropriate
-% character to a third function. Spaces are ignored unless
-% escaped.
-% For the \cs{str_(g)input:Nn} application, all the functions are simply
-% \cs{token_to_str:N} (this ensures that spaces correctly get assigned
-% category code $10$).
-% For the \pkg{l3regex} applications, the functions do some further
-% tests on the character they receive.
-%
-% \begin{macro}{\str_input:Nn,\str_ginput:Nn}
-% Simple wrappers around the internal \cs{str_aux_escape:NNNn}.
-% \begin{macrocode}
-\cs_new_protected:Npn \str_input:Nn #1#2
- {
- \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
- \tl_set_eq:NN #1 \g_str_result_tl
- }
-\cs_new_protected:Npn \str_ginput:Nn #1#2
- {
- \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
- \tl_gset_eq:NN #1 \g_str_result_tl
- }
-% \end{macrocode}
-% \end{macro}
+% \subsubsection{Error-reporting during conversion}
%
-% \begin{macro}[int]{\str_escape_use:NNNn}
-% Use the internal \cs{str_aux_escape:NNNn}, then leave the result
-% in the input stream. This is used in \pkg{l3regex}.
+% \begin{macro}[int]{\str_if_flag_error:nnx}
+% \begin{macro}[aux]{\str_if_flag_no_error:nnx}
+% When converting using the function \cs{str_set_convert:Nnnn}, errors
+% should be reported to the user after each step in the
+% conversion. Errors are signalled by raising some flag (typically
+% \texttt{str_error}), so here we test that flag: if it is raised,
+% give the user an error, otherwise remove the arguments. On the other
+% hand, in the conditional functions \cs{str_set_convert:NnnnTF},
+% errors should be suppressed. This is done by changing
+% \cs{str_if_flag_error:nnx} into \cs{str_if_flag_no_error:nnx}
+% locally.
% \begin{macrocode}
-\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4
+\cs_new_protected:Npn \str_if_flag_error:nnx #1
{
- \str_aux_escape:NNNn #1#2#3 {#4}
- \g_str_result_tl
+ \flag_if_raised:nTF {#1}
+ { \msg_kernel_error:nnx { str } }
+ { \use_none:nn }
}
+\cs_new_protected:Npn \str_if_flag_no_error:nnx #1#2#3
+ { \flag_if_raised:nT {#1} { \bool_gset_true:N \g_str_error_bool } }
% \end{macrocode}
% \end{macro}
-%
-% \begin{macro}[int]{\str_aux_escape:NNNn}
-% \begin{macro}[aux]{\str_aux_escape_loop:N}
-% \begin{macro}[aux]+\str_aux_escape_\:w+
-% Most of the work is done within an \texttt{x}-expanding assignment,
-% but the |\x| escape sequence cannot be done in that way. Therefore,
-% we interrupt the assignment at each |\x| escape sequence, store the
-% partial result in a \tn{toks} register, and at the end unpack all
-% of the \tn{toks} registers to the left of the last chunk of
-% \cs{g_str_result_tl}.
-% \begin{macrocode}
-\cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4
- {
- \group_begin:
- \cs_set_nopar:Npn \str_aux_escape_unescaped:N { #1 }
- \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 }
- \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 }
- \int_set:Nn \tex_escapechar:D { 92 }
- \str_gset_other:Nn \g_str_result_tl {#4}
- \int_zero:N \l_str_tmpa_int
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl
- \q_recursion_tail \q_recursion_stop
- }
- \tl_gput_left:Nx \g_str_result_tl
- { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int }
- \group_end:
- }
-\cs_new_nopar:Npn \str_aux_escape_loop:N #1
- {
- \cs_if_exist_use:cF { str_aux_escape_\token_to_str:N #1:w }
- { \str_aux_escape_unescaped:N #1 }
- \str_aux_escape_loop:N
- }
-\cs_new_nopar:cpn { str_aux_escape_ \c_backslash_str :w }
- \str_aux_escape_loop:N #1
- {
- \cs_if_exist_use:cF { str_aux_escape_/\token_to_str:N #1:w }
- { \str_aux_escape_escaped:N #1 }
- \str_aux_escape_loop:N
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
% \end{macro}
%
-% \begin{macro}[aux]{\str_aux_escape_unescaped:N}
-% \begin{macro}[aux]{\str_aux_escape_escaped:N}
-% \begin{macro}[aux]{\str_aux_escape_raw:N}
-% Those functions are never called before being given a new meaning,
-% so their definitions here don't matter.
+% \begin{macro}[int]{\str_if_flag_times:nT}
+% At the end of each conversion step, we raise all relevant errors as
+% one error message, built on the fly. The height of each flag
+% indicates how many times a given error was encountered. This
+% function prints |#2| followed by the number of occurrences of an
+% error if it occurred, nothing otherwise.
% \begin{macrocode}
-\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing:
-\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing:
-\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing:
+\cs_new_protected:Npn \str_if_flag_times:nT #1#2
+ { \flag_if_raised:nT {#1} { #2~(x \flag_height:n {#1} ) } }
% \end{macrocode}
% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux]+\str_aux_escape_\q_recursion_tail:w+
-% \begin{macro}[aux]+\str_aux_escape_/\q_recursion_tail:w+
-% \begin{macro}[aux]+\str_aux_escape_ :w+
-% \begin{macro}[aux]
-% {
-% \str_aux_escape_/a:w, \str_aux_escape_/e:w, \str_aux_escape_/f:w,
-% \str_aux_escape_/n:w, \str_aux_escape_/r:w, \str_aux_escape_/t:w
-% }
-% The loop is ended upon seeing \cs{q_recursion_tail}.
-% Spaces are ignored, and |\a|, |\e|, |\f|, |\n|, |\r|, |\t| take
-% their meaning here.
-% \begin{macrocode}
-\cs_new_eq:cN
- { str_aux_escape_ \c_backslash_str q_recursion_tail :w }
- \use_none_delimit_by_q_recursion_stop:w
-\cs_new_eq:cN
- { str_aux_escape_ / \c_backslash_str q_recursion_tail :w }
- \use_none_delimit_by_q_recursion_stop:w
-\cs_new_nopar:cpn { str_aux_escape_~:w } { }
-\cs_new_nopar:cpx { str_aux_escape_/a:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^G }
-\cs_new_nopar:cpx { str_aux_escape_/t:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^I }
-\cs_new_nopar:cpx { str_aux_escape_/n:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^J }
-\cs_new_nopar:cpx { str_aux_escape_/f:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^L }
-\cs_new_nopar:cpx { str_aux_escape_/r:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^M }
-\cs_new_nopar:cpx { str_aux_escape_/e:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^[ }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
+% \subsubsection{Framework for conversions}
+%
+% Most functions in this module expect to be working with
+% \enquote{native} strings. Strings can also be stored as bytes, in one
+% of many encodings, for instance \textsc{utf8}. The bytes themselves
+% can be expressed in various ways in terms of \TeX{} tokens, for
+% instance as pairs of hexadecimal digits. The questions of going from
+% arbitrary Unicode code points to bytes, and from bytes to tokens are
+% mostly independent.
%
-% \begin{macro}[aux]{\str_aux_escape_/x:w}
-% \begin{macro}[aux]{\str_aux_escape_x_test:N}
-% \begin{macro}[aux]{\str_aux_escape_x_unbraced_i:N}
-% \begin{macro}[aux]{\str_aux_escape_x_unbraced_ii:N}
-% \begin{macro}[aux]{\str_aux_escape_x_braced_loop:N}
-% \begin{macro}[aux]{\str_aux_escape_x_braced_end:N}
-% \begin{macro}[aux]{\str_aux_escape_x_end:}
-% When |\x| is encountered, interrupt the assignment,
-% and distinguish the cases of a braced or unbraced syntax.
-% In the braced case, collect arbitrarily many hexadecimal digits,
-% building the number in \cs{l_str_char_int} (using
-% \cs{str_aux_hexadecimal_use:NTF}), and check that
-% the run of digits was stopped by a closing brace.
-% In the unbraced case, collect up to two hexadecimal digits,
-% possibly less, building the character number in \cs{l_str_char_int}.
-% In both cases, once all digits have been collected, use
-% the \TeX{} primitive \tn{lowercase} to produce that character,
-% and use an \cs{if_false:} trick to restart the assignment.
+% Conversions are done in four steps:
+% \begin{itemize}
+% \item \enquote{unescape} produces a string of bytes;
+% \item \enquote{decode} takes in a string of bytes, and converts it
+% to a list of Unicode characters in an internal representation,
+% with items of the form
+% \begin{quote}
+% \meta{bytes} \cs{s_tl} \meta{Unicode code point} \cs{s_tl}
+% \end{quote}
+% where we have collected the \meta{bytes} which combined to form
+% this particular Unicode character, and the \meta{Unicode code
+% point} is in the range $[0,\hexnum{10FFFF}]$.
+% \item \enquote{encode} encodes the internal list of code points as a
+% byte string in the new encoding;
+% \item \enquote{escape} escapes bytes as requested.
+% \end{itemize}
+% The process is modified in case one of the encoding is empty (or the
+% conversion function has been set equal to the empty encoding because
+% it was not found): then the unescape or escape step is ignored, and
+% the decode or encode steps work on tokens instead of bytes. Otherwise,
+% each step must ensure that it passes a correct byte string or internal
+% string to the next step.
+%
+% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
+% \begin{macro}[TF]{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
+% \begin{macro}[aux]{\str_convert_aux_i:nNNnnn}
+% The input string is stored in \cs{g_str_result_tl}, then we:
+% unescape and decode; encode and escape; exit the group and store the
+% result in the user's variable. The various conversion functions all
+% act on \cs{g_str_result_tl}. Errors are silenced for the conditional
+% functions by redefining \cs{str_if_flag_error:nnx} locally.
% \begin{macrocode}
-\cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N
- {
- \if_false: { \fi: }
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl }
- \int_incr:N \l_str_tmpa_int
- \int_zero:N \l_str_char_int
- \str_aux_escape_x_test:N
- }
-\cs_new_protected_nopar:Npx \str_aux_escape_x_test:N #1
- {
- \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
- { \exp_not:N \str_aux_escape_x_test:N }
- {
- \l_str_char_int = "0
- \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
- { \exp_not:N \str_aux_escape_x_braced_loop:N }
- { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
- }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
+\cs_new_protected_nopar:Npn \str_set_convert:Nnnn
+ { \str_convert_aux_i:nNNnnn { } \tl_set_eq:NN }
+\cs_new_protected_nopar:Npn \str_gset_convert:Nnnn
+ { \str_convert_aux_i:nNNnnn { } \tl_gset_eq:NN }
+\prg_new_protected_conditional:Npnn
+ \str_set_convert:Nnnn #1#2#3#4 { T , F , TF }
{
- \str_aux_hexadecimal_use:NTF #1
- { \str_aux_escape_x_unbraced_ii:N }
- { \exp_stop_f: \str_aux_escape_x_end: #1 }
+ \bool_gset_false:N \g_str_error_bool
+ \str_convert_aux_i:nNNnnn
+ { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx }
+ \tl_set_eq:NN #1 {#2} {#3} {#4}
+ \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true:
}
-\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
+\prg_new_protected_conditional:Npnn
+ \str_gset_convert:Nnnn #1#2#3#4 { T , F , TF }
{
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \str_aux_escape_x_unbraced_ii:N }
- {
- \str_aux_hexadecimal_use:NTF #1
- { \exp_stop_f: \str_aux_escape_x_end: }
- { \exp_stop_f: \str_aux_escape_x_end: #1 }
- }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1
- {
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \str_aux_escape_x_braced_loop:N }
- {
- \str_aux_hexadecimal_use:NTF #1
- { \str_aux_escape_x_braced_loop:N }
- { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 }
- }
+ \bool_gset_false:N \g_str_error_bool
+ \str_convert_aux_i:nNNnnn
+ { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx }
+ \tl_gset_eq:NN #1 {#2} {#3} {#4}
+ \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true:
}
-\cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1
+\cs_new_protected:Npn \str_convert_aux_i:nNNnnn #1#2#3#4#5#6
{
- \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
- { \exp_not:N \str_aux_escape_x_end: }
- {
- \msg_kernel_error:nn { str } { x-missing-brace }
- \exp_not:N \str_aux_escape_x_end: #1
- }
+ \group_begin:
+ #1
+ \str_gset_other:Nn \g_str_result_tl {#4}
+ \exp_after:wN \str_convert_aux_ii:wwwnn
+ \tl_to_str:n {#5} /// \q_stop
+ { decode } { unescape }
+ \prg_do_nothing:
+ \str_convert_decode_:
+ \exp_after:wN \str_convert_aux_ii:wwwnn
+ \tl_to_str:n {#6} /// \q_stop
+ { encode } { escape }
+ \use_ii_i:nn
+ \str_convert_encode_:
+ \group_end:
+ #2 #3 \g_str_result_tl
}
-\group_begin:
- \char_set_catcode_other:N \^^@
- \cs_new_protected_nopar:Npn \str_aux_escape_x_end:
- {
- \tex_lccode:D \c_zero \l_str_char_int
- \tl_to_lowercase:n
- {
- \tl_gset:Nx \g_str_result_tl
- { \if_false: } \fi:
- \str_aux_escape_raw:N ^^@
- \str_aux_escape_loop:N
- }
- }
-\group_end:
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Framework for encoding conversions}
%
-% Most functions in this module expect to be working with
-% \enquote{native} strings. Strings can also be stored as
-% bytes, in one of many encodings, for instance \textsc{utf8}.
-% The bytes themselves can be expressed as character
-% tokens with the relevant character code, or as pairs
-% of hexadecimal digits, \emph{etc.} The questions of going
-% from arbitrary characters to bytes, and from bytes to \TeX{}
-% tokens are mostly independent.
-%
-% Conversions are done in four steps:
-% \begin{itemize}
-% \item \enquote{input} produces a string of bytes;
-% \item \enquote{from} interprets the byte string in the old encoding,
-% and converts it to a comma-list, each character becoming
-% \enquote{\ttfamily \meta{character code},};
-% \item \enquote{to} encodes the internal comma list as a byte string
-% in the new encoding;
-% \item \enquote{output} escapes bytes as requested.
-% \end{itemize}
-% The process is modified in case one of the encoding is
-% \texttt{internal} or \texttt{native}: then the input or
-% output step is ignored.
-%
-% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
-% \begin{macro}[aux]{\str_convert_aux_i:NNnnn}
% \begin{macro}[aux]{\str_convert_aux_ii:wwwnn}
-% \begin{macro}[aux]{\str_convert_aux_iii:nnF}
-% \begin{macro}[aux]{\str_convert_aux_iv:nnnF}
-% \begin{macro}[aux]{\str_convert_aux_v:NNnNN}
-% The input string is stored in \cs{g_str_result_tl},
-% then we call the various conversion functions, which all
-% act on \cs{g_str_result_tl}. The arguments |#3| and |#4|
-% of the \texttt{aux_i} function are split into their
-% \meta{encoding} and \meta{escaping} parts by \texttt{aux_ii}.
-% The conversion function names are built by \texttt{aux_iii},
-% which first tries to use the name as supplied by the user,
-% then filters it with \cs{str_aux_lowercase_alphanum:n},
-% and calls \texttt{aux_iv} to load the relevant definition file.
-% In all cases, \texttt{aux_iii} ensures that the two
-% required conversion functions are defined, with
-% a fall-back to the native encoding with no escaping.
-% Once this is done, \texttt{aux_v} does the main work:
+% \begin{macro}[aux]{\str_convert_aux_iii:NNnNN}
+% The task of \cs{str_convert_aux_ii:wwwnn} is to split
+% \meta{encoding}/\meta{escaping} pairs into their components, |#1|
+% and |#2|. Calls to \cs{str_convert_aux_iv:nnn} ensure that the
+% corresponding conversion functions are defined. The third auxiliary
+% does the main work.
% \begin{itemize}
-% \item |#1| is the encoding conversion function;
-% \item |#2| is the escaping function;
-% \item |#3| is the escaping name;
-% \item |#4| is the \enquote{native} encoding function
-% (either \enquote{from} or \enquote{to});
-% \item |#5| is \cs{use_i:nn} or \cs{use_ii:nn}.
+% \item |#1| is the encoding conversion function;
+% \item |#2| is the escaping function;
+% \item |#3| is the escaping name for use in an error message;
+% \item |#4| is \cs{prg_do_nothing:} for unescaping/decoding, and
+% \cs{use_ii_i:nn} for encoding/escaping;
+% \item |#5| is the default encoding function (either
+% \enquote{decode} or \enquote{encode}), for which there should be
+% no escaping.
% \end{itemize}
-% When converting to the internal format, the escaping must be done
-% first, then the encoding (this is controlled by |#5|).
-% If the encoding is native, then the escaping is ignored and
-% must be empty. Otherwise, the escaping |#2| is performed.
+% Let us ignore the native encoding for a second. In the
+% unescaping/decoding phase, we want to do |#2#1| in this order, and
+% in the encoding/escaping phase, the order should be reversed:
+% |#4#2#1| does exactly that. If one of the encodings is the default
+% (native), then the escaping should be ignored, with an error if any
+% was given, and only the encoding, |#1|, should be performed.
% \begin{macrocode}
-\cs_new_protected:Npn \str_set_convert:Nnnn
- { \str_convert_aux_i:NNnnn \tl_set_eq:NN }
-\cs_new_protected:Npn \str_gset_convert:Nnnn
- { \str_convert_aux_i:NNnnn \tl_gset_eq:NN }
-\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5
+\cs_new_protected:Npn \str_convert_aux_ii:wwwnn
+ #1 / #2 // #3 \q_stop #4#5
{
- \group_begin:
- \str_gset_other:Nn \g_str_result_tl {#5}
-
- \exp_after:wN \str_convert_aux_ii:wwwnn
- \tl_to_str:n {#3} /// \q_stop
- { from } { input } \str_convert_from_native: \use_i:nn
-
- \exp_after:wN \str_convert_aux_ii:wwwnn
- \tl_to_str:n {#4} /// \q_stop
- { to } { output } \str_convert_to_native: \use_ii:nn
-
- \group_end:
- #1 #2 \g_str_result_tl
- }
-\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5
- {
- \str_convert_aux_iii:nnF {#4} {#1} {native}
- \str_convert_aux_iii:nnF {#5} {#2} { }
- \exp_args:Ncc \str_convert_aux_v:NNnNN
+ \str_convert_aux_iv:nnn {enc} {#4} {#1}
+ \str_convert_aux_iv:nnn {esc} {#5} {#2}
+ \exp_args:Ncc \str_convert_aux_iii:NNnNN
{ str_convert_#4_#1: } { str_convert_#5_#2: } {#2}
}
-\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3
+\cs_new_protected:Npn \str_convert_aux_iii:NNnNN #1#2#3#4#5
{
- \cs_if_exist:cF { str_convert_#1_#2: }
- {
- \exp_args:Nx \str_convert_aux_iv:nnnF
- { \str_aux_lowercase_alphanum:n {#2} }
- {#1} {#2} {#3}
- }
+ \if_meaning:w #1 #5
+ \tl_if_empty:nF {#3}
+ { \msg_kernel_error:nnx { str } { native-escaping } {#3} }
+ #1
+ \else:
+ #4 #2 #1
+ \fi:
}
-\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[aux]{\str_convert_aux_iv:nnn}
+% \begin{macro}[aux]{\str_convert_aux_v:nnnn}
+% The arguments of \cs{str_convert_aux_iv:nnn} are: \texttt{enc} or
+% \texttt{esc}, used to build filenames, the type of the conversion
+% (unescape, decode, encode, escape), and the encoding or escaping
+% name. If the function is already defined, no need to do anything.
+% Otherwise, filter out all non-alphanumerics in the name, and
+% lowercase it. Feed that, and the same three arguments, to
+% \cs{str_convert_aux_v:nnnn}. The task is then to make sure that the
+% conversion function |#3_#1| corresponding to the type |#3| and
+% filtered name |#1| is defined, then set our initial conversion
+% function |#3_#4| equal to that.
+%
+% How do we get the |#3_#1| conversion to be defined if it isn't?
+% Two main cases.
+%
+% First, if |#1| is a key in \cs{g_str_file_prop}, then the value
+% \cs{l_str_internal_tl} tells us what file to load. Loading is
+% skipped if the file was already read, \emph{i.e.}, if the conversion
+% command based on \cs{l_str_internal_tl} already exists. Otherwise,
+% try to load the file; if that fails, there is an error, use the
+% default empty name instead.
+%
+% Second, |#1| may be absent from the property list. The
+% \cs{cs_if_exist:cF} test is automatically false, and we search for a
+% file defining the encoding or escaping |#1| (this should allow
+% third-party \texttt{.def} files). If the file is not found, there is
+% an error, use the default empty name instead.
+%
+% In all cases, the conversion based on \cs{l_str_internal_tl} is
+% defined, so we can set the |#3_#1| function equal to that. In some
+% cases (\emph{e.g.}, \texttt{utf16be}), the |#3_#1| function is
+% actually defined within the file we just loaded, and it is different
+% from the \cs{l_str_internal_tl}-based function: we mustn't clobber
+% that different definition.
+% \begin{macrocode}
+\cs_new_protected:Npn \str_convert_aux_iv:nnn #1#2#3
{
- \cs_if_exist:cF { str_convert_#2_#1: }
- {
- \group_begin:
- \str_load_catcodes:
- \str_load_one:n {#1}
- \group_end:
- }
- \cs_if_exist:cTF { str_convert_#2_#1: }
- { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } }
+ \cs_if_exist:cF { str_convert_#2_#3: }
{
- \msg_kernel_error:nnx { str } { unknown-#2 } {#3}
- \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: }
+ \exp_args:Nx \str_convert_aux_v:nnnn
+ { \str_convert_lowercase_alphanum:n {#3} }
+ {#1} {#2} {#3}
}
}
-\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5
+\cs_new_protected:Npn \str_convert_aux_v:nnnn #1#2#3#4
{
- #5 { } {#1}
- \cs_if_eq:NNTF #1 #4
+ \cs_if_exist:cF { str_convert_#3_#1: }
{
- \tl_if_empty:nF {#3}
- { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} }
+ \prop_get:NnNF \g_str_file_prop {#1} \l_str_internal_tl
+ { \tl_set:Nn \l_str_internal_tl {#1} }
+ \cs_if_exist:cF { str_convert_#3_ \l_str_internal_tl : }
+ {
+ \file_if_exist:nTF { l3str-#2- \l_str_internal_tl .def }
+ {
+ \group_begin:
+ \str_load_catcodes:
+ \file_input:n { l3str-#2- \l_str_internal_tl .def }
+ \group_end:
+ }
+ {
+ \tl_clear:N \l_str_internal_tl
+ \msg_kernel_error:nnxx { str } { unknown-#2 } {#4} {#1}
+ }
+ }
+ \cs_if_exist:cF { str_convert_#3_#1: }
+ {
+ \cs_gset_eq:cc { str_convert_#3_#1: }
+ { str_convert_#3_ \l_str_internal_tl : }
+ }
}
- { #2 }
- #5 {#1} { }
+ \cs_gset_eq:cc { str_convert_#3_#4: } { str_convert_#3_#1: }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int, rEXP]{\str_aux_lowercase_alphanum:n}
-% \begin{macro}[aux, rEXP]{\str_aux_lowercase_alphanum_loop:N}
-% This function keeps only letters and digits, with upper case
-% letters converted to lower case.
+% \begin{macro}[int, rEXP]{\str_convert_lowercase_alphanum:n}
+% \begin{macro}[aux, rEXP]{\str_convert_lowercase_alphanum_loop:N}
+% This function keeps only letters and digits, with upper case letters
+% converted to lower case.
% \begin{macrocode}
-\cs_new:Npn \str_aux_lowercase_alphanum:n #1
+\cs_new:Npn \str_convert_lowercase_alphanum:n #1
{
- \exp_after:wN \str_aux_lowercase_alphanum_loop:N
+ \exp_after:wN \str_convert_lowercase_alphanum_loop:N
\tl_to_str:n {#1} { ? \prg_map_break: }
\prg_break_point:n { }
}
-\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1
+\cs_new:Npn \str_convert_lowercase_alphanum_loop:N #1
{
\use_none:n #1
\if_num:w `#1 < \c_ninety_one
@@ -1856,149 +1778,331 @@
\fi:
\fi:
\fi:
- \str_aux_lowercase_alphanum_loop:N
+ \str_convert_lowercase_alphanum_loop:N
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_from_internal:}
-% \begin{macro}[int]{\str_convert_to_internal:}
-% Converting from the internal format to itself is
-% of course trivial.
+% \begin{macro}[int]{\str_load_catcodes:}
+% Since encoding files may be loaded at arbitrary places in a \TeX{}
+% document, including within verbatim mode, we set the catcodes of all
+% characters appearing in any encoding definition file.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_from_internal: { }
-\cs_new_protected_nopar:Npn \str_convert_to_internal: { }
+\cs_new_protected:Npn \str_load_catcodes:
+ {
+ \char_set_catcode_escape:N \\
+ \char_set_catcode_group_begin:N \{
+ \char_set_catcode_group_end:N \}
+ \char_set_catcode_math_toggle:N \$
+ \char_set_catcode_alignment:N \&
+ \char_set_catcode_parameter:N \#
+ \char_set_catcode_math_superscript:N \^
+ \char_set_catcode_ignore:N \ %
+ \char_set_catcode_space:N \~
+ \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFILNPSTUX }
+ \char_set_catcode_letter:N
+ \tl_map_function:nN { 0123456789"'?*+-.(),`!/<>[];= }
+ \char_set_catcode_other:N
+ \char_set_catcode_comment:N \%
+ \int_set:Nn \tex_endlinechar:D {32}
+ }
% \end{macrocode}
% \end{macro}
-% \end{macro}
%
-% \subsubsection{Loading encoding files}
+% \subsubsection{Byte unescape and escape}
%
-% \begin{macro}{\str_load:n}
-% \begin{macro}[int]{\str_load_one:n}
-% \begin{macro}[aux]{\str_load_alias:n, \str_load_alias_aux:nnn}
-% Loading encoding or escaping files can be done in bulk by
-% specifying a comma-list of encodings and escapings. Each
-% name is normalized: only alphanumeric characters are kept,
-% lowercased. The relevant file is loaded if it was not loaded
-% already.
+% Strings of bytes may need to be stored in auxiliary files in safe
+% \enquote{escaping} formats. Each such escaping is only loaded as
+% needed. By default, on input any non-byte is filtered out, while the
+% output simply consists in letting bytes through.
+%
+% \begin{macro}[int, rEXP]{\str_filter_bytes:n}
+% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N}
+% In the case of pdf\TeX{}, every character is a byte. For
+% Unicode-aware engines, test the character code; non-bytes cause us
+% to raise the flag \texttt{str_byte}. Spaces have already been given
+% the correct category code when this function is called.
% \begin{macrocode}
-\cs_new_protected:Npn \str_load:n #1
- {
- \group_begin:
- \str_load_catcodes:
- \clist_map_inline:nn {#1}
- {
- \exp_args:Nx \str_load_one:n
- { \str_aux_lowercase_alphanum:n {#1} }
- }
- \group_end:
- }
-\cs_new_protected:Npn \str_load_one:n #1
+\pdftex_if_engine:TF
+ { \cs_new_eq:NN \str_filter_bytes:n \use:n }
{
- \cs_if_exist:cF { str_convert_from_ #1 : }
+ \cs_new:Npn \str_filter_bytes:n #1
{
- \cs_if_exist:cF { str_convert_input_#1: }
- {
- \file_if_exist:nTF { l3str- #1 .def }
- { \file_input:n { l3str- #1 .def } }
- { \str_load_alias:n {#1} }
- }
+ \str_filter_bytes_aux:N #1
+ { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \cs_new:Npn \str_filter_bytes_aux:N #1
+ {
+ \use_none:n #1
+ \if_num:w `#1 < 256 \exp_stop_f:
+ #1
+ \else:
+ \flag_raise:n { str_byte }
+ \fi:
+ \str_filter_bytes_aux:N
}
}
-\cs_new_protected:Npn \str_load_alias:n #1
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_unescape_:}
+% \begin{macro}[int]{\str_convert_unescape_bytes:}
+% The simplest unescaping method removes non-bytes from
+% \cs{g_str_result_tl}.
+% \begin{macrocode}
+\pdftex_if_engine:TF
+ { \cs_new_protected_nopar:Npn \str_convert_unescape_: { } }
{
- \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl
+ \cs_new_protected_nopar:Npn \str_convert_unescape_:
{
- \str_load_one:n { \l_str_tmpa_tl }
- \cs_if_exist:cTF
- { str_convert_from_ \l_str_tmpa_tl : }
- {
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from }
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to }
- }
- {
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input }
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output }
- }
+ \flag_clear:n { str_byte }
+ \tl_gset:Nx \g_str_result_tl
+ { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
+ \str_if_flag_error:nnx { str_byte } { non-byte } { bytes }
}
- { \msg_kernel_error:nnx { str } { file-not-found } {#1} }
}
-\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3
- { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } }
+\cs_new_eq:NN \str_convert_unescape_bytes: \str_convert_unescape_:
% \end{macrocode}
% \end{macro}
% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_escape_:}
+% \begin{macro}[int]{\str_convert_escape_bytes:}
+% The simplest form of escape leaves the bytes from the previous step
+% of the conversion unchanged.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_convert_escape_: { }
+\cs_new_eq:NN \str_convert_escape_bytes: \str_convert_escape_:
+% \end{macrocode}
+% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_load_catcodes:}
-% Since encoding files may be loaded at arbitrary places
-% in a \TeX{} document, including within verbatim mode,
-% we set the catcodes of all characters appearing in any
-% encoding definition file.
+% \subsubsection{Native strings}
+%
+% \begin{macro}[int]{\str_convert_decode_:}
+% \begin{macro}[aux, rEXP]{\str_decode_native_char:N}
+% Convert each character to its character code, one at a time.
% \begin{macrocode}
-\cs_new_protected:Npn \str_load_catcodes:
+\cs_new_protected_nopar:Npn \str_convert_decode_:
+ { \str_convert_gmap:N \str_decode_native_char:N }
+\cs_new:Npn \str_decode_native_char:N #1
+ { #1 \s_tl \int_value:w `#1 \s_tl }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_encode_:}
+% The conversion from an internal string to native character tokens is
+% very different in pdf\TeX{} and in other engines. For Unicode-aware
+% engines, we need the definitions to be read when the null byte has
+% category code $12$, so we set that inside a group.
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_other:n { 0 }
+ \pdftex_if_engine:TF
+% \end{macrocode}
+% \begin{macro}[aux, EXP]{\str_encode_native_char:n}
+% Since pdf\TeX{} only supports 8-bit characters, and we have a table
+% of all bytes, the conversion can be done in linear time within an
+% \texttt{x}-expanding assignment. Look out for character codes larger
+% than $255$, those characters are replaced by |?|, and raise a flag,
+% which then triggers a pdf\TeX{}-specific error.
+% \begin{macrocode}
+ {
+ \cs_new_protected_nopar:Npn \str_convert_encode_:
+ {
+ \flag_clear:n { str_error }
+ \str_convert_gmap_internal:N \str_encode_native_char:n
+ \str_if_flag_error:nnx { str_error }
+ { pdfTeX-native-overflow } { }
+ }
+ \cs_new:Npn \str_encode_native_char:n #1
+ {
+ \if_num:w #1 < \c_two_hundred_fifty_six
+ \str_output_byte:n {#1}
+ \else:
+ \flag_raise:n { str_error }
+ ?
+ \fi:
+ }
+ \msg_kernel_new:nnnn { str } { pdfTeX-native-overflow }
+ { Character~code~too~large~for~pdfTeX. }
+ {
+ The~pdfTeX~engine~only~supports~8-bit~characters:~
+ valid~character~codes~are~in~the~range~[0,255].~
+ To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \begin{macro}[aux]{\str_encode_native_loop:w}
+% \begin{macro}[aux]{\str_encode_native_flush:}
+% \begin{macro}[aux, rEXP]{\str_encode_native_filter:N}
+% In Unicode-aware engines, since building particular characters
+% cannot be done expandably in \TeX{}, we cannot hope to get a
+% linear-time function. However, we get quite close by building the
+% result one block of $255$ characters at a time. Set the lowercase
+% code of the \textsc{ascii} character $1$ to the first character code
+% in the block, \emph{etc.}, then lowercase the contents of
+% \cs{c_str_positive_bytes_tl}. Unfortunately, this is complicated by
+% the fact that \TeX{} won't lowercase a character to the null
+% character, thus the need for filtering: whenever the lowercase code
+% of a given character is zero, it is replaced \enquote{by hand} by
+% the null character. Unicode-aware engines will never incur an
+% overflow because the internal string is guaranteed to only contain
+% code points in $[0,\hexnum{10FFFF}]$.
+% \begin{macrocode}
+ {
+ \cs_new_protected_nopar:Npn \str_convert_encode_:
+ {
+ \tex_lccode:D \c_zero \c_zero
+ \int_zero:N \l_str_internal_int
+ \tl_set_eq:NN \l_str_internal_tl \g_str_result_tl
+ \tl_gclear:N \g_str_result_tl
+ \exp_after:wN \str_encode_native_loop:w
+ \l_str_internal_tl \s_tl { \q_stop \prg_map_break: } \s_tl
+ \prg_break_point:n { \str_encode_native_flush: }
+ }
+ \cs_new_protected:Npn \str_encode_native_loop:w #1 \s_tl #2 \s_tl
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \if_num:w \l_str_internal_int = \c_two_hundred_fifty_five
+ \str_encode_native_flush:
+ \l_str_internal_int \c_zero
+ \fi:
+ \tex_advance:D \l_str_internal_int \c_one
+ \tex_lccode:D \l_str_internal_int \int_eval:w #2 \int_eval_end:
+ \str_encode_native_loop:w
+ }
+ \cs_new_protected_nopar:Npn \str_encode_native_flush:
+ {
+ \use:x
+ {
+ \tl_to_lowercase:n
+ {
+ \tl_gput_right:Nx \exp_not:N \g_str_result_tl
+ {
+ \exp_after:wN \str_encode_native_filter:N
+ \c_str_positive_bytes_tl
+ { ? = \c_zero \fi: \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+ }
+ }
+ \cs_new:Npn \str_encode_native_filter:N #1
+ {
+ \if_num:w `#1 > \l_str_internal_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \if_num:w \tex_lccode:D `#1 = \c_zero
+ ^^@
+ \else:
+ #1
+ \fi:
+ \str_encode_native_filter:N
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% End the group to restore the catcode of the null byte.
+% \begin{macrocode}
+\group_end:
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{8-bit encodings}
+%
+% This section will be entirely rewritten: it is not yet clear in what
+% situations 8-bit encodings are used, hence I don't know what exactly
+% should be optimized. The current approach is reasonably efficient to
+% convert long strings, and it scales well when using many different
+% encodings. An approach based on csnames would have a smaller constant
+% load time for each individual conversion, but has a large hash table
+% cost. Using a range of \tn{count} registers works for decoding, but
+% not for encoding: one possibility there would be to use a binary tree
+% for the mapping of Unicode characters to bytes, stored as a box, one
+% per encoding.
+%
+% Since the section is going to be rewritten, documentation lacks.
+%
+% All the 8-bit encodings which \pkg{l3str} supports rely on the same
+% internal functions.
+%
+% \begin{macro}[int]{\str_declare_eight_bit_encoding:nnn}
+% All the 8-bit encoding definition file start with
+% \cs{str_declare_eight_bit_encoding:nnn} \Arg{encoding name}
+% \Arg{mapping} \Arg{missing bytes}. The \meta{mapping} argument is a
+% token list of pairs \Arg{byte} \Arg{Unicode} expressed in uppercase
+% hexadecimal notation. The \meta{missing} argument is a token list
+% of \Arg{byte}. Every \meta{byte} which does not appear in the
+% \meta{mapping} nor the \meta{missing} lists maps to the same code
+% point in Unicode.
+% \begin{macrocode}
+\cs_new_protected:Npn \str_declare_eight_bit_encoding:nnn #1#2#3
{
- \char_set_catcode_escape:N \\
- \char_set_catcode_group_begin:N \{
- \char_set_catcode_group_end:N \}
- \char_set_catcode_math_superscript:N \^
- \char_set_catcode_ignore:N \ %
- \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN }
- \char_set_catcode_letter:N
- \tl_map_function:nN { 0123456789"'?*+ }
- \char_set_catcode_other:N
- \char_set_catcode_comment:N \%
- \int_set:Nn \tex_endlinechar:D {32}
+ \tl_set:Nn \l_str_internal_tl {#1}
+ \cs_new_protected_nopar:cpn { str_convert_decode_#1: }
+ { \str_convert_decode_eight_bit:n {#1} }
+ \cs_new_protected_nopar:cpn { str_convert_encode_#1: }
+ { \str_convert_encode_eight_bit:n {#1} }
+ \tl_const:cn { c_str_encoding_#1_tl } {#2}
+ \tl_const:cn { c_str_encoding_#1_missing_tl } {#3}
}
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_from_eight_bit:n}
-% \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn}
-% \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n}
-% \begin{macro}[aux, EXP]{\str_convert_from_eight_bit_aux:N}
+% \begin{macro}[int]{\str_convert_decode_eight_bit:n}
+% \begin{macro}[aux]{\str_decode_eight_bit_load:nn}
+% \begin{macro}[aux]{\str_decode_eight_bit_load_missing:n}
+% \begin{macro}[aux, EXP]{\str_decode_eight_bit_char:N}
%^^A todo: document
% \begin{macrocode}
-\cs_new_protected:Npn \str_convert_from_eight_bit:n #1
+\cs_new_protected:Npn \str_convert_decode_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_tmpa_int
- \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn
+ \int_zero:N \l_str_internal_int
+ \exp_last_unbraced:Nx \str_decode_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
\prg_break_point:n { }
- \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n
+ \exp_last_unbraced:Nx \str_decode_eight_bit_load_missing:n
{ \tl_use:c { c_str_encoding_#1_missing_tl } }
{ \q_stop \prg_map_break: }
\prg_break_point:n { }
- \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N
+ \flag_clear:n { str_error }
+ \str_convert_gmap:N \str_decode_eight_bit_char:N
+ \str_if_flag_error:nnx { str_error } { decode-8-bit } {#1}
\group_end:
}
-\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2
+\cs_new_protected:Npn \str_decode_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_from_eight_bit_load:nn
+ \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_internal_int \exp_after:wN { \int_value:w "#2 }
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_decode_eight_bit_load:nn
}
-\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1
+\cs_new_protected:Npn \str_decode_eight_bit_load_missing:n #1
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN
+ \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_internal_int \exp_after:wN
{ \int_use:N \c_str_replacement_char_int }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_from_eight_bit_load_missing:n
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_decode_eight_bit_load_missing:n
}
-\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
+\cs_new:Npn \str_decode_eight_bit_char:N #1
{
#1 \s_tl
- \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int
+ \if_num:w \tex_dimen:D `#1 < \l_str_internal_int
\if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D
\fi:
@@ -2011,54 +2115,55 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_to_eight_bit:n}
-% \begin{macro}[aux]{\str_convert_to_eight_bit_load:nn}
-% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux:n}
-% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux_ii:n}
+% \begin{macro}[int]{\str_convert_encode_eight_bit:n}
+% \begin{macro}[aux]{\str_encode_eight_bit_load:nn}
+% \begin{macro}[aux, rEXP]{\str_encode_eight_bit_char:n}
+% \begin{macro}[aux, rEXP]{\str_encode_eight_bit_char_aux:n}
%^^A todo: document
% \begin{macrocode}
-\cs_new_protected:Npn \str_convert_to_eight_bit:n #1
+\cs_new_protected:Npn \str_convert_encode_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_tmpa_int
- \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn
+ \int_zero:N \l_str_internal_int
+ \exp_last_unbraced:Nx \str_encode_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
\prg_break_point:n { }
- \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n
+ \flag_clear:n { str_error }
+ \str_convert_gmap_internal:N \str_encode_eight_bit_char:n
+ \str_if_flag_error:nnx { str_error } { encode-8-bit } {#1}
\group_end:
}
-\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2
+\cs_new_protected:Npn \str_encode_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop:
- \exp_args:NNf \tex_toks:D \l_str_tmpa_int
+ \tex_dimen:D "#2 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#2 sp \scan_stop:
+ \exp_args:NNf \tex_toks:D \l_str_internal_int
{ \str_output_byte:n { "#1 } }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_to_eight_bit_load:nn
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_encode_eight_bit_load:nn
}
-\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
+\cs_new:Npn \str_encode_eight_bit_char:n #1
{
\if_num:w #1 > \c_max_register_int
- \msg_expandable_kernel_error:nnn
- { str } { eight-bit-to-byte-overflow } {#1}
+ \flag_raise:n { str_error }
\else:
- \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int
+ \if_num:w \tex_dimen:D #1 < \l_str_internal_int
\if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
\fi:
\fi:
- \str_convert_to_eight_bit_aux_ii:n {#1}
+ \str_encode_eight_bit_char_aux:n {#1}
\fi:
}
-\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
+\cs_new:Npn \str_encode_eight_bit_char_aux:n #1
{
\if_num:w #1 < \c_two_hundred_fifty_six
\str_output_byte:n {#1}
\else:
- \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte }
+ \flag_raise:n { str_error }
\fi:
}
% \end{macrocode}
@@ -2067,255 +2172,72 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_encoding_eight_bit:n}
-%^^A todo: document
-% \begin{macrocode}
-\cs_new_protected:Npn \str_encoding_eight_bit:n #1
- {
- \cs_new_protected:cpn { str_convert_from_#1: }
- { \str_convert_from_eight_bit:n {#1} }
- \cs_new_protected:cpn { str_convert_to_#1: }
- { \str_convert_to_eight_bit:n {#1} }
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \subsubsection{Byte input and output}
-%
-% Strings of bytes may need to be stored in auxiliary files
-% in safe \enquote{escaping} formats. Each such escaping
-% is only loaded as needed. By default, on input any non-byte
-% is filtered out, while the output simply consists in letting
-% bytes through.
-%
-% \begin{macro}[int, rEXP]{\str_filter_bytes:n}
-% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N}
-% In the case of pdf\TeX{}, every character is a byte.
-% For Unicode-aware engines, test the character code.
-% Spaces have already been given the correct category
-% code when this function is called.
-% \begin{macrocode}
-\pdftex_if_engine:TF
- { \cs_new_eq:NN \str_filter_bytes:n \use:n }
- {
- \cs_new_nopar:Npn \str_filter_bytes:n #1
- {
- \str_filter_bytes_aux:N #1
- { ? \prg_map_break: }
- \prg_break_point:n { }
- }
- \cs_new_nopar:Npn \str_filter_bytes_aux:N #1
- {
- \use_none:n #1
- \if_num:w `#1 < 256 \exp_stop_f: #1 \fi:
- \str_filter_bytes_aux:N
- }
- }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[int]{\str_convert_input_:}
-% \begin{macro}[int]{\str_convert_input_bytes:}
-% The simplest input method removes non-bytes
-% from \cs{g_str_result_tl}.
-% \begin{macrocode}
-\pdftex_if_engine:TF
- { \cs_new_protected_nopar:Npn \str_convert_input_: { } }
- {
- \cs_new_protected_nopar:Npn \str_convert_input_:
- {
- \tl_gset:Nx \g_str_result_tl
- { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
- }
- }
-\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_:
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \begin{macro}[int]{\str_convert_output_:}
-% \begin{macro}[int]{\str_convert_output_bytes:}
-% The simplest form of output leaves the bytes from the previous
-% step of the conversion unchanged.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_output_: { }
-\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_:
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
-% \subsubsection{Convert to and from native strings}
-%
-% \begin{macro}[int]{\str_convert_from_native:}
-% \begin{macro}[aux, rEXP]{\str_convert_from_native_aux:N}
-% Convert each character to its character code, one at a time.
-% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_from_native:
- { \str_aux_gmap_result:N \str_convert_from_native_aux:N }
-\cs_new:Npn \str_convert_from_native_aux:N #1
- { #1 \s_tl \int_value:w `#1 \s_tl }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
+% \subsection{Messages}
%
-% \begin{macro}[int]{\str_convert_to_native:}
-% \begin{macro}[aux]{\str_convert_to_native_aux:w}
-% This function is based on the internal functions
-% \cs{str_aux_convert_to_native:n} and
-% \cs{str_aux_convert_to_native_step:n}.
-% Loop through the comma-list, and call
-% \cs{str_aux_convert_to_native_step:n}
-% with the relevant integer argument.
-% We need to store \cs{g_str_result_tl} in a temporary
-% token list, because that global \enquote{result} is
-% cleared silently by \cs{str_aux_convert_to_native:n}.
-%^^A todo: update
+% General messages, and messages for the encodings and escapings loaded
+% by default (\enquote{native}, and \enquote{bytes}).
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_convert_to_native:
- {
- \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl
- \str_aux_convert_to_native:n
- {
- \exp_after:wN \str_convert_to_native_aux:w
- \l_str_tmpa_tl \s_tl { \q_stop \prg_map_break: } \s_tl
- }
- }
-\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 \s_tl #2 \s_tl
+\msg_kernel_new:nnn { str } { unknown-esc }
+ { Escaping~scheme~'#1'~(filtered:~'#2')~unknown. }
+\msg_kernel_new:nnn { str } { unknown-enc }
+ { Encoding~scheme~'#1'~(filtered:~'#2')~unknown. }
+\msg_kernel_new:nnnn { str } { native-escaping }
+ { The~'native'~encoding~scheme~does~not~support~any~escaping. }
{
- \use_none_delimit_by_q_stop:w #2 \q_stop
- \str_aux_convert_to_native_step:n {#2}
- \str_convert_to_native_aux:w
+ Since~native~strings~do~not~consist~in~bytes,~
+ none~of~the~escaping~methods~make~sense.~
+ The~specified~escaping,~'#1',~will be ignored.
}
+\msg_kernel_new:nnn { str } { file-not-found }
+ { File~'l3str-#1.def'~not~found. }
% \end{macrocode}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\str_aux_convert_to_native:n}
-% \begin{macro}[int]{\str_aux_convert_to_native_step:n}
-% \begin{macro}[aux]{\str_aux_convert_to_native_flush:}
-% \begin{macro}[aux, rEXP]{\str_aux_convert_to_native_filter:N}
-%^^A todo: document.
+% Message used when the \enquote{bytes} unescaping fails because the
+% string given to \cs{str_set_convert:Nnnn} contains a non-byte. This
+% cannot happen for the pdf\TeX{} engine, since that engine only
+% supports 8-bit characters. Messages used for other escapings and
+% encodings are defined in each definition file.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1
- {
- \tex_lccode:D \c_zero \c_zero
- \int_zero:N \l_str_tmpa_int
- \tl_gclear:N \g_str_result_tl
- #1
- \prg_break_point:n { \str_aux_convert_to_native_flush: }
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1
- {
- \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five
- \str_aux_convert_to_native_flush:
- \l_str_tmpa_int \c_zero
- \fi:
- \tex_advance:D \l_str_tmpa_int \c_one
- \if_int_compare:w #1 > \c_max_char_int
- \msg_kernel_error:nnx { str } { overflow } {#1}
- \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int
- \else:
- \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end:
- \fi:
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush:
+\pdftex_if_engine:F
{
- \use:x
+ \msg_kernel_new:nnnn { str } { non-byte }
+ { String~invalid~in~escaping~'#1':~it~may~only~contain~bytes. }
{
- \tl_to_lowercase:n
+ Some~characters~in~the~string~you~asked~to~convert~are~not~
+ 8-bit~characters.~Perhaps~the~string~is~a~'native'~Unicode~string?~
+ If~it~is,~try~using\\
+ \\
+ \iow_indent:n
{
- \tl_gput_right:Nx \exp_not:N \g_str_result_tl
- {
- \exp_after:wN \str_aux_convert_to_native_filter:N
- \c_str_positive_bytes_tl
- { ? = \c_zero \fi: \prg_map_break: }
- \prg_break_point:n { }
- }
+ \iow_char:N\\str_set_convert:Nnnn \\
+ \ \ <str~var>~\{~<string>~\}~\{~native~\}~\{~<target~encoding>~\}
}
}
}
-\group_begin:
- \char_set_catcode_other:n { 0 }
- \cs_new:Npn \str_aux_convert_to_native_filter:N #1
- {
- \if_num:w `#1 > \l_str_tmpa_int
- \exp_after:wN \prg_map_break:
- \fi:
- \if_num:w \tex_lccode:D `#1 = \c_zero
- ^^@
- \else:
- #1
- \fi:
- \str_aux_convert_to_native_filter:N
- }
-\group_end:
% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
-% \subsection{Messages}
%
+% Those messages are used when converting to and from 8-bit encodings.
% \begin{macrocode}
-\msg_kernel_new:nnnn { str } { x-missing-brace }
- { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. }
+\msg_kernel_new:nnnn { str } { decode-8-bit }
+ { Invalid~string~in~encoding~'#1'. }
{
- You~wrote~something~like~
- `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~
- The~closing~brace~is~missing.
+ LaTeX~came~across~a~byte~which~is~not~defined~to~represent~
+ any~character~in~the~encoding~'#1'.
}
-\msg_kernel_new:nnnn { str } { overflow }
- { Character~code~#1~too~big. }
+\msg_kernel_new:nnnn { str } { encode-8-bit }
+ { Unicode~string~cannot~be~converted~to~encoding~'#1'. }
{
- \int_compare:nNnTF {#1} > { 1114111 }
- { The~Unicode~standard~limits~code~points~to~1114111. }
- {
- The~pdfTeX~engine~only~supports~8-bit~characters:~
- valid~character~codes~are~in~the~range~[0,255].~
- To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
- }
+ The~encoding~'#1'~only~contains~a~subset~of~all~Unicode~characters.~
+ LaTeX~was~asked~to~convert~a~string~to~that~encoding,~but~that~
+ string~contains~a~character~that~'#1'~does~not~support.
}
-\msg_kernel_new:nnn { str } { convert-input }
- { Input~scheme~`#1'~unknown. }
-\msg_kernel_new:nnn { str } { convert-from }
- { Encoding~`#1'~unknown. }
-\msg_kernel_new:nnn { str } { convert-to }
- { Encoding~`#1'~unknown.~Using~UTF-8. }
-\msg_kernel_new:nnn { str } { convert-output }
- { Output~scheme~`#1~unknown. }
-\msg_kernel_new:nnn { str } { to-native }
- { Use~\str_set_convert:Nnn to~convert~to~native~strings. }
-\msg_kernel_new:nnn { str } { unicode-surrogate }
- { Code~point~#1~is~an~unpaired~surrogate. }
-\msg_kernel_new:nnn { str } { utf16-surrogate }
- { UTF-16~unpaired~surrogate~#1. }
-\msg_kernel_new:nnn { str } { utf16-odd }
- { UTF-16~dangling~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-extra-conti }
- { UTF-8~extra~continuation~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-missing-conti }
- { UTF-8~missing~continuation~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-invalid-byte }
- { Byte~#1~cannot~appear~in~UTF-8. }
-\msg_kernel_new:nnn { str } { utf8-overlong }
- { Overlong~UTF-8~byte~sequence~for~code~point~#1. }
-\msg_kernel_new:nnn { str } { utf8-premature-end }
- { Incomplete~last~UTF-8~character. }
-\msg_kernel_new:nnn { str } { utf8-overflow }
- { Code~point~#1~too~large~(UTF-8). }
-\msg_kernel_new:nnn { str } { utf32-overflow }
- { Code~point~too~large~(UTF-32~`#1'). }
-\msg_kernel_new:nnn { str } { utf32-truncated }
- { Truncated~UTF-32~string~`...#1'. }
% \end{macrocode}
%
% \subsection{Deprecated string functions}
%
% \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n}
-% The naming scheme is a little bit more consistent
-% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}.
+% The naming scheme is a little bit more consistent with
+% \enquote{ignore_spaces} instead of \enquote{skip_spaces}.
% \begin{macrocode}
\cs_set:Npn \str_length_skip_spaces:N
{ \exp_args:No \str_length_skip_spaces:n }
@@ -2329,102 +2251,116 @@
%
% \subsection{Escaping definition files}
%
-% Several of those encodings are defined by the pdf file format.
-% The following byte storage methods are defined:
+% Several of those encodings are defined by the pdf file format. The
+% following byte storage methods are defined:
% \begin{itemize}
-% \item \texttt{bytes} (default), non-bytes are filtered out,
-% and bytes are left untouched;
-% \item \texttt{hex} or \texttt{hexadecimal},
-% as per the pdf\TeX{} primitive \tn{pdfescapehex}
-% \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename}
-% \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring}
-% \item \texttt{url}, as per the percent encoding of urls.
+% \item \texttt{bytes} (default), non-bytes are filtered out, and
+% bytes are left untouched (this is defined by default);
+% \item \texttt{hex} or \texttt{hexadecimal}, as per the pdf\TeX{}
+% primitive \tn{pdfescapehex}
+% \item \texttt{name}, as per the pdf\TeX{} primitive
+% \tn{pdfescapename}
+% \item \texttt{string}, as per the pdf\TeX{} primitive
+% \tn{pdfescapestring}
+% \item \texttt{url}, as per the percent encoding of urls.
% \end{itemize}
%
-% \subsubsection{Input methods}
-%
-% \begin{macro}[int]{\str_convert_input_hex:}
-% \begin{macro}[int]{\str_convert_input_hexadecimal:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux_ii:N}
-% Take chars two by two, and interpret each pair as
-% the hexadecimal code for a byte. Anything else than
-% hexadecimal digits is ignored.
-% A string which contains an odd number of hexadecimal
-% digits gets |0| appended to it: this is equivalent
-% to appending a |0| in all cases, and dropping it if
-% it is alone.
+% \subsubsection{Unescape methods}
+%
+% \begin{macro}[int]{\str_convert_unescape_hex:}
+% \begin{macro}[aux, rEXP]{\str_unescape_hex_i:N}
+% \begin{macro}[aux, rEXP]{\str_unescape_hex_ii:N}
+% Take chars two by two, and interpret each pair as the hexadecimal
+% code for a byte. Anything else than hexadecimal digits is ignored,
+% raising the flag. A string which contains an odd number of
+% hexadecimal digits gets |0| appended to it: this is equivalent to
+% appending a |0| in all cases, and dropping it if it is alone.
% \begin{macrocode}
%<*hex>
-\cs_new_protected_nopar:Npn \str_convert_input_hex:
+\cs_new_protected_nopar:Npn \str_convert_unescape_hex:
{
\group_begin:
+ \flag_clear:n { str_error }
\int_set:Nn \tex_escapechar:D { 92 }
\tl_gset:Nx \g_str_result_tl
{
\str_output_byte:w "
- \exp_after:wN \str_convert_input_hex_aux:N
- \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
+ \exp_last_unbraced:Nf \str_unescape_hex_i:N
+ { \tl_to_str:N \g_str_result_tl }
+ 0 { ? 0 - \c_one \prg_map_break: }
\prg_break_point:n { \str_output_end: }
}
+ \str_if_flag_error:nnx { str_error } { unescape-hex } { }
\group_end:
}
-\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1
+\cs_new:Npn \str_unescape_hex_i:N #1
{
\use_none:n #1
\str_aux_hexadecimal_use:NTF #1
- \str_convert_input_hex_aux_ii:N
- \str_convert_input_hex_aux:N
+ { \str_unescape_hex_ii:N }
+ {
+ \flag_raise:n { str_error }
+ \str_unescape_hex_i:N
+ }
}
-\cs_new_nopar:Npn \str_convert_input_hex_aux_ii:N #1
+\cs_new:Npn \str_unescape_hex_ii:N #1
{
\use_none:n #1
\str_aux_hexadecimal_use:NTF #1
{
\str_output_end:
- \str_output_byte:w " \str_convert_input_hex_aux:N
+ \str_output_byte:w " \str_unescape_hex_i:N
+ }
+ {
+ \flag_raise:n { str_error }
+ \str_unescape_hex_ii:N
}
- \str_convert_input_hex_aux_ii:N
+ }
+\msg_kernel_new:nnnn { str } { unescape-hex }
+ { String~invalid~in~escaping~'hex':~only~hexadecimal~digits~allowed. }
+ {
+ Some~characters~in~the~string~you~asked~to~convert~are~not~
+ hexadecimal~digits~(0-9,~A-F,~a-f)~nor~spaces.
}
%</hex>
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\str_convert_input_name:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN}
-% \begin{macro}[int]{\str_convert_input_url:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN}
-% The \cs{str_convert_input_name:} function replaces each
-% occurrence of |#| followed by two hexadecimal digits
-% in \cs{g_str_result_tl} by the corresponding byte.
-% The \texttt{url} function is identical, with escape
-% character |%| instead of |#|. Thus we define the two
-% together. The arguments of \cs{str_tmp:w} are the character
-% code of |#| or |%| in hexadecimal, the name of the main
-% function to define, and the name of the auxiliary which
+% \begin{macro}[int]{\str_convert_unescape_name:}
+% \begin{macro}[aux, rEXP]{\str_unescape_name_loop:wNN}
+% \begin{macro}[int]{\str_convert_unescape_url:}
+% \begin{macro}[aux, rEXP]{\str_unescape_url_loop:wNN}
+% The \cs{str_convert_unescape_name:} function replaces each
+% occurrence of |#| followed by two hexadecimal digits in
+% \cs{g_str_result_tl} by the corresponding byte. The \texttt{url}
+% function is identical, with escape character |%| instead of |#|.
+% Thus we define the two together. The arguments of \cs{str_tmp:w} are
+% the character code of |#| or |%| in hexadecimal, the name of the
+% main function to define, and the name of the auxiliary which
% performs the loop.
%
-% The looping auxiliary |#3| finds the next escape character,
-% reads the following two characters, and tests them. The test
-% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit
-% in the input stream, hence we surround the test with
-% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}.
-% If both characters are hexadecimal digits, they should be
-% removed before looping: this is done by \cs{use_i:nnn}.
-% If one of the characters is not a hexadecimal digit,
-% then feed |"#1| to \cs{str_output_byte:w} to produce the
-% escape character, and call the looping function followed
-% by the two characters (remove \cs{use_i:nnn}).
+% The looping auxiliary |#3| finds the next escape character, reads
+% the following two characters, and tests them. The test
+% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit in the
+% input stream, hence we surround the test with
+% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}. If both
+% characters are hexadecimal digits, they should be removed before
+% looping: this is done by \cs{use_i:nnn}. If one of the characters
+% is not a hexadecimal digit, then feed |"#1| to
+% \cs{str_output_byte:w} to produce the escape character, raise the
+% flag, and call the looping function followed by the two characters
+% (remove \cs{use_i:nnn}).
% \begin{macrocode}
%<*name|url>
\cs_set_protected:Npn \str_tmp:w #1#2#3
{
- \cs_new_protected_nopar:Npn #2
+ \cs_new_protected:cpn { str_convert_unescape_#2: }
{
\group_begin:
+ \flag_clear:n { str_byte }
+ \flag_clear:n { str_error }
\int_set:Nn \tex_escapechar:D { 92 }
\tl_gset:Nx \g_str_result_tl
{
@@ -2432,9 +2368,11 @@
#1 ? { ? \prg_map_break: }
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_byte } { non-byte } { #2 }
+ \str_if_flag_error:nnx { str_error } { unescape-#2 } { }
\group_end:
}
- \cs_new_nopar:Npn #3 ##1#1##2##3
+ \cs_new:Npn #3 ##1#1##2##3
{
\str_filter_bytes:n {##1}
\use_none:n ##3
@@ -2443,50 +2381,63 @@
{
\str_aux_hexadecimal_use:NTF ##3
{ }
- { * \c_zero + `#1 \use_i:nn }
+ {
+ \flag_raise:n { str_error }
+ * \c_zero + `#1 \use_i:nn
+ }
+ }
+ {
+ \flag_raise:n { str_error }
+ 0 + `#1 \use_i:nn
}
- { 0 + `#1 \use_i:nn }
\str_output_end:
\use_i:nnn #3 ##2##3
}
+ \msg_kernel_new:nnnn { str } { unescape-#2 }
+ { String~invalid~in~escaping~'#2'. }
+ {
+ LaTeX~came~across~the~escape~character~'#1'~not~followed~by~
+ two~hexadecimal~digits.~This~is~invalid~in~the~escaping~'#2'.
+ }
}
%</name|url>
-%<name>\exp_after:wN \str_tmp:w \c_hash_str
-%<name> \str_convert_input_name: \str_convert_input_name_aux:wNN
-%<url>\exp_after:wN \str_tmp:w \c_percent_str
-%<url> \str_convert_input_url: \str_convert_input_url_aux:wNN
+%<name>\exp_after:wN \str_tmp:w \c_hash_str { name }
+%<name> \str_unescape_name_loop:wNN
+%<url>\exp_after:wN \str_tmp:w \c_percent_str { url }
+%<url> \str_unescape_url_loop:wNN
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_input_string:}
-% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN}
-% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wNNN}
-% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:NNNNNN}
-% The \texttt{string} escaping is somewhat similar to
-% the \texttt{name} and \texttt{url} escapings, with
-% escape character |\|.
-% The first step is to convert all three line endings,
-% |^^J|, |^^M|, and |^^M^^J| to the common |^^J|, as per
-% the \textsc{pdf} specification.
+% \begin{macro}[int]{\str_convert_unescape_string:}
+% \begin{macro}[aux, rEXP]{\str_unescape_string_newlines:wN}
+% \begin{macro}[aux, rEXP]{\str_unescape_string_loop:wNNN}
+% \begin{macro}[aux, rEXP]{\str_unescape_string_repeat:NNNNNN}
+% The \texttt{string} escaping is somewhat similar to the
+% \texttt{name} and \texttt{url} escapings, with escape character |\|.
+% The first step is to convert all three line endings, |^^J|, |^^M|,
+% and |^^M^^J| to the common |^^J|, as per the \textsc{pdf}
+% specification. This step cannot raise the flag.
+%
% Then the following escape sequences are decoded.
% \begin{itemize}\def\makelabel#1{\hss\llap{\ttfamily\string#1}}
-% \item[\n] Line feed ($10$)
-% \item[\r] Carriage return ($13$)
-% \item[\t] Horizontal tab ($9$)
-% \item[\b] Backspace ($8$)
-% \item[\f] Form feed ($12$)
-% \item[\(] Left parenthesis
-% \item[\)] Right parenthesis
-% \item[\\] Backslash
-% \item[\ddd] (backslash followed by $1$ to $3$ octal digits)
-% Byte \texttt{ddd} (octal), subtracting $256$ in case of overflow.
+% \item[\n] Line feed ($10$)
+% \item[\r] Carriage return ($13$)
+% \item[\t] Horizontal tab ($9$)
+% \item[\b] Backspace ($8$)
+% \item[\f] Form feed ($12$)
+% \item[\(] Left parenthesis
+% \item[\)] Right parenthesis
+% \item[\\] Backslash
+% \item[\ddd] (backslash followed by $1$ to $3$ octal digits) Byte
+% \texttt{ddd} (octal), subtracting $256$ in case of overflow.
% \end{itemize}
-% If followed by an end-of-line character, the backslash and
-% the end-of-line are ignored. If followed by anything else,
-% the backslash is ignored.
+% If followed by an end-of-line character, the backslash and the
+% end-of-line are ignored. If followed by anything else, the backslash
+% is ignored, raising the error flag.
+%^^A Be paranoid: \tl_to_lowercase:n is unsafe.
% \begin{macrocode}
%<*string>
\group_begin:
@@ -2495,25 +2446,29 @@
\char_set_catcode_other:N \^^M
\tl_to_lowercase:n
{
- \cs_new_protected_nopar:Npn \str_convert_input_string:
+ \cs_new_protected_nopar:Npn \str_convert_unescape_string:
{
\group_begin:
+ \flag_clear:n { str_byte }
+ \flag_clear:n { str_error }
\int_set:Nn \tex_escapechar:D { 92 }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_convert_input_string_aux:wN
+ \exp_after:wN \str_unescape_string_newlines:wN
\g_str_result_tl \prg_map_break: ^^M ?
\prg_break_point:n { }
}
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_convert_input_string_aux:wNNN
+ \exp_after:wN \str_unescape_string_loop:wNNN
\g_str_result_tl * ?? { ? \prg_map_break: }
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_byte } { non-byte } { string }
+ \str_if_flag_error:nnx { str_error } { unescape-string } { }
\group_end:
}
- \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4
+ \cs_new:Npn \str_unescape_string_loop:wNNN #1 *#2#3#4
}
{
\str_filter_bytes:n {#1}
@@ -2528,11 +2483,11 @@
\if_int_compare:w #2 > \c_three
- 256
\fi:
- \str_convert_input_string_aux:NNNNNN
+ \str_unescape_string_repeat:NNNNNN
}
- { \str_convert_input_string_aux:NNNNNN ? }
+ { \str_unescape_string_repeat:NNNNNN ? }
}
- { \str_convert_input_string_aux:NNNNNN ?? }
+ { \str_unescape_string_repeat:NNNNNN ?? }
}
{
\prg_case_str:xxn {#2}
@@ -2547,20 +2502,29 @@
{ b } { 10 }
{ ^^J } { 0 - \c_one }
}
- { 0 - \c_one \use_i:nn }
+ {
+ \flag_raise:n { str_error }
+ 0 - \c_one \use_i:nn
+ }
}
\str_output_end:
- \use_i:nn \str_convert_input_string_aux:wNNN #2#3#4
+ \use_i:nn \str_unescape_string_loop:wNNN #2#3#4
}
- \cs_new_nopar:Npn \str_convert_input_string_aux:NNNNNN #1#2#3#4#5#6
- { \str_output_end: \str_convert_input_string_aux:wNNN }
- \cs_new_nopar:Npn \str_convert_input_string_aux:wN #1 ^^M #2
+ \cs_new:Npn \str_unescape_string_repeat:NNNNNN #1#2#3#4#5#6
+ { \str_output_end: \str_unescape_string_loop:wNNN }
+ \cs_new:Npn \str_unescape_string_newlines:wN #1 ^^M #2
{
- #1 ^^J
- \if_charcode:w ^^J #2
- \exp_after:wN \use_i:nn
- \fi:
- \str_convert_input_string_aux:wN #2
+ #1
+ \if_charcode:w ^^J #2 \else: ^^J \fi:
+ \str_unescape_string_newlines:wN #2
+ }
+ \msg_kernel_new:nnnn { str } { unescape-string }
+ { String~invalid~in~escaping~'string'. }
+ {
+ LaTeX~came~across~an~escape~character~'\c_backslash_str'~
+ not~followed~by~any~of:~'n',~'r',~'t',~'b',~'f',~'(',~')',~
+ '\c_backslash_str',~one~to~three~octal~digits,~or~the~end~
+ of~a~line.
}
\group_end:
%</string>
@@ -2570,57 +2534,57 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Output methods}
+% \subsubsection{Escape methods}
%
-% \begin{macro}[int]{\str_convert_output_hex:}
-% \begin{macro}[int]{\str_convert_output_hexadecimal:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_hex_aux:N}
+% Currently, none of the escape methods can lead to errors, assuming
+% that their input is made out of bytes.
+%
+% \begin{macro}[int]{\str_convert_escape_hex:}
+% \begin{macro}[aux, rEXP]{\str_escape_hex_char:N}
% Loop and convert each byte to hexadecimal.
% \begin{macrocode}
%<*hex>
-\cs_new_protected_nopar:Npn \str_convert_output_hex:
- { \str_aux_gmap_result:N \str_convert_output_hex_aux:N }
-\cs_new_nopar:Npn \str_convert_output_hex_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_escape_hex:
+ { \str_convert_gmap:N \str_escape_hex_char:N }
+\cs_new:Npn \str_escape_hex_char:N #1
{ \str_output_hexadecimal:n { `#1 } }
%</hex>
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[int]{\str_convert_output_name:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:NTF}
-% \begin{variable}{\c_str_convert_output_name_str}
-% \begin{variable}{\c_str_convert_output_name_not_str}
-% For each byte, test whether it should
-% be output as is, or be \enquote{hash-encoded}.
-% Roughly, bytes outside the range $[\hexnum{2A},\hexnum{7E}]$
-% are hash-encoded. We keep two lists of exceptions:
-% characters in \cs{c_str_convert_output_name_not_str}
-% are not hash-encoded, and characters in the
-% \cs{c_str_convert_output_name_str} are encoded.
+% \begin{macro}[int]{\str_convert_escape_name:}
+% \begin{macro}[aux, rEXP]{\str_escape_name_char:N}
+% \begin{macro}[aux, rEXP]{\str_if_escape_name:NTF}
+% \begin{variable}{\c_str_escape_name_str}
+% \begin{variable}{\c_str_escape_name_not_str}
+% For each byte, test whether it should be output as is, or be
+% \enquote{hash-encoded}. Roughly, bytes outside the range
+% $[\hexnum{2A},\hexnum{7E}]$ are hash-encoded. We keep two lists of
+% exceptions: characters in \cs{c_str_escape_name_not_str} are not
+% hash-encoded, and characters in the \cs{c_str_escape_name_str} are
+% encoded.
% \begin{macrocode}
%<*name>
-\str_const:Nn \c_str_convert_output_name_not_str { ! " $ & ' } %$
-\str_const:Nn \c_str_convert_output_name_str { {}/<>[] }
-\cs_new_protected_nopar:Npn \str_convert_output_name:
- { \str_aux_gmap_result:N \str_convert_output_name_aux:N }
-\cs_new_nopar:Npn \str_convert_output_name_aux:N #1
+\str_const:Nn \c_str_escape_name_not_str { ! " $ & ' } %$
+\str_const:Nn \c_str_escape_name_str { {}/<>[] }
+\cs_new_protected_nopar:Npn \str_convert_escape_name:
+ { \str_convert_gmap:N \str_escape_name_char:N }
+\cs_new:Npn \str_escape_name_char:N #1
{
- \str_convert_output_name_aux:NTF #1 {#1}
+ \str_if_escape_name:NTF #1 {#1}
{ \c_hash_str \str_output_hexadecimal:n {`#1} }
}
-\prg_new_conditional:Npnn \str_convert_output_name_aux:N #1 { TF }
+\prg_new_conditional:Npnn \str_if_escape_name:N #1 { TF }
{
\if_num:w `#1 < "2A \exp_stop_f:
- \str_if_contains_char:NNTF \c_str_convert_output_name_not_str #1
+ \str_if_contains_char:NNTF \c_str_escape_name_not_str #1
\prg_return_true: \prg_return_false:
\else:
\if_num:w `#1 > "7E \exp_stop_f:
\prg_return_false:
\else:
- \str_if_contains_char:NNTF \c_str_convert_output_name_str #1
+ \str_if_contains_char:NNTF \c_str_escape_name_str #1
\prg_return_false: \prg_return_true:
\fi:
\fi:
@@ -2633,25 +2597,25 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_output_string:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:NTF}
-% \begin{variable}{\c_str_convert_output_string_str}
-% Any character below (and including) space, and any character
-% above (and including) \texttt{del}, are converted to octal.
-% One backslash is added before each parenthesis and backslash.
+% \begin{macro}[int]{\str_convert_escape_string:}
+% \begin{macro}[aux, rEXP]{\str_escape_string_char:N}
+% \begin{macro}[aux, rEXP]{\str_if_escape_string:NTF}
+% \begin{variable}{\c_str_escape_string_str}
+% Any character below (and including) space, and any character above
+% (and including) \texttt{del}, are converted to octal. One backslash
+% is added before each parenthesis and backslash.
% \begin{macrocode}
%<*string>
-\str_const:Nx \c_str_convert_output_string_str
+\str_const:Nx \c_str_escape_string_str
{ \c_backslash_str ( ) }
-\cs_new_protected_nopar:Npn \str_convert_output_string:
- { \str_aux_gmap_result:N \str_convert_output_string_aux:N }
-\cs_new_nopar:Npn \str_convert_output_string_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_escape_string:
+ { \str_convert_gmap:N \str_escape_string_char:N }
+\cs_new:Npn \str_escape_string_char:N #1
{
- \str_convert_output_string_aux:NTF #1
+ \str_if_escape_string:NTF #1
{
\str_if_contains_char:NNT
- \c_str_convert_output_string_str #1
+ \c_str_escape_string_str #1
{ \c_backslash_str }
#1
}
@@ -2662,7 +2626,7 @@
\int_mod:nn {`#1} \c_eight
}
}
-\prg_new_conditional:Npnn \str_convert_output_string_aux:N #1 { TF }
+\prg_new_conditional:Npnn \str_if_escape_string:N #1 { TF }
{
\if_num:w `#1 < "21 \exp_stop_f:
\prg_return_false:
@@ -2681,23 +2645,21 @@
% \end{macro}
% \end{macro}
%
-% \begin{macro}[int]{\str_convert_output_url:}
-% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N}
-% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF}
-%^^A todo: Figure out what bytes should be percent encoded! Safe could be
-%^^A !'()*-./0123456789_
-% This function is similar to \cs{str_convert_output_name:},
-% escaping different characters.
+% \begin{macro}[int]{\str_convert_escape_url:}
+% \begin{macro}[aux, rEXP]{\str_escape_url_char:N}
+% \begin{macro}[aux, rEXP]{\str_if_escape_url:NTF}
+% This function is similar to \cs{str_convert_escape_name:}, escaping
+% different characters.
% \begin{macrocode}
%<*url>
-\cs_new_protected_nopar:Npn \str_convert_output_url:
- { \str_aux_gmap_result:N \str_convert_output_url_aux:N }
-\cs_new_nopar:Npn \str_convert_output_url_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_escape_url:
+ { \str_convert_gmap:N \str_escape_url_char:N }
+\cs_new:Npn \str_escape_url_char:N #1
{
- \str_convert_output_url_aux:NTF #1 {#1}
+ \str_if_escape_url:NTF #1 {#1}
{ \c_percent_str \str_output_hexadecimal:n { `#1 } }
}
-\prg_new_conditional:Npnn \str_convert_output_url_aux:N #1 { TF }
+\prg_new_conditional:Npnn \str_if_escape_url:N #1 { TF }
{
\if_num:w `#1 < "41 \exp_stop_f:
\str_if_contains_char:nNTF { "-.<> } #1
@@ -2719,15 +2681,14 @@
%
% \subsection{Encoding definition files}
%
-% The \texttt{native} and \texttt{internal} encodings are automatically
-% defined. Other encodings are loaded as needed. The following encodings
-% are supported:
+% The \texttt{native} encoding is automatically defined. Other encodings
+% are loaded as needed. The following encodings are supported:
% \begin{itemize}
-% \item \textsc{utf-8};
-% \item \textsc{utf-16}, big-endian, little-endian, or with byte order mark;
-% \item \textsc{utf-32}, big-endian, little-endian, or with byte order mark;
-% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$,
-% skipping the inexistent \textsc{iso 8859-12}.
+% \item \textsc{utf-8};
+% \item \textsc{utf-16}, big-, little-endian, or with byte order mark;
+% \item \textsc{utf-32}, big-, little-endian, or with byte order mark;
+% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$,
+% skipping the inexistent \textsc{iso 8859-12}.
% \end{itemize}
%
% \subsubsection{\textsc{utf-8} support}
@@ -2736,74 +2697,235 @@
%<*utf8>
% \end{macrocode}
%
-% \begin{macro}[int]{\str_convert_to_utf8:}
-% \begin{macro}[aux, rEXP]
-% {
-% \str_aux_to_utf_viii:n,
-% \str_aux_to_utf_viii:nnnnw
-% }
-% Loop through the internal string, and convert each character
-% to the \textsc{utf-8} representation. The details are messy.
-%^^A todo: document.
+% \begin{macro}[int]{\str_convert_encode_utf8:}
+% \begin{macro}[aux, rEXP]{\str_encode_utf_viii_char:n}
+% \begin{macro}[aux, rEXP]{\str_encode_utf_viii_loop:wwnnw}
+% Loop through the internal string, and convert each character to its
+% \textsc{utf-8} representation. The representation is built from the
+% right-most (least significant) byte to the left-most (most
+% significant) byte. Continuation bytes are in the range $[128,191]$,
+% taking $64$ different values, hence we roughly want to express the
+% character code in base $64$, shifting the first digit in the
+% representation by some number depending on how many continuation
+% bytes there are. In the range $[0,127]$, output the corresponding
+% byte directly. In the range $[128,2047]$, output the remainder
+% modulo $64$, plus $128$ as a continuation byte, then output the
+% quotient (which is in the range $[0,31]$), shifted by $192$. In the
+% next range, $[2048,65535]$, split the character code into residue
+% and quotient modulo $64$, output the residue as a first continuation
+% byte, then repeat; this leaves us with a quotient in the range
+% $[0,15]$, which we output shifted by $224$. The last range,
+% $[65536,1114111]$, follows the same pattern: once we realize that
+% dividing twice by $64$ leaves us with a number larger than $15$, we
+% repeat, producing a last continuation byte, and offset the quotient
+% by $240$ for the leading byte.
+%
+% How is that implemented? \cs{str_encode_utf_vii_loop:wwnnw} takes
+% successive quotients as its first argument, the quotient from the
+% previous step as its second argument (except in step~$1$), the bound
+% for quotients that trigger one more step or not, and finally the
+% offset used if this step should produce the leading byte. Leading
+% bytes can be in the ranges $[0,127]$, $[192,223]$, $[224,239]$, and
+% $[240,247]$ (really, that last limit should be $244$ because Unicode
+% stops at the code point $1114111$). At each step, if the quotient
+% |#1| is less than the limit |#3| for that range, output the leading
+% byte (|#1| shifted by |#4|) and stop. Otherwise, we need one more
+% step: use the quotient of |#1| by $64$, and |#1| as arguments for
+% the looping auxiliary, and output the continuation byte
+% corresponding to the remainder $|#2|-64|#1|+128$. The bizarre
+% construction |\c_minus_one + \c_zero *| removes the spurious initial
+% continuation byte (better methods welcome).
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { str_convert_to_utf8: }
- { \str_aux_gmap_internal_result:N \str_aux_to_utf_viii:n }
-\cs_new:Npn \str_aux_to_utf_viii:n #1
+\cs_new_protected_nopar:cpn { str_convert_encode_utf8: }
+ { \str_convert_gmap_internal:N \str_encode_utf_viii_char:n }
+\cs_new:Npn \str_encode_utf_viii_char:n #1
{
- \str_aux_to_utf_viii:nnnnw
- {#1} { \c_minus_one + \c_zero * \use_none:n }
+ \str_encode_utf_viii_loop:wwnnw #1 ; \c_minus_one + \c_zero * ;
{ 128 } { \c_zero }
{ 32 } { 192 }
{ 16 } { 224 }
{ 8 } { 240 }
\q_stop
}
-\cs_new_nopar:Npn \str_aux_to_utf_viii:nnnnw #1#2#3#4 #5 \q_stop
+\cs_new:Npn \str_encode_utf_viii_loop:wwnnw #1; #2; #3#4 #5 \q_stop
{
\if_num:w #1 < #3 \exp_stop_f:
\str_output_byte:n { #1 + #4 }
- \else:
- \exp_args:Nf \str_aux_to_utf_viii:nnnnw
- { \int_div_truncate:nn {#1} {64} }
- {#1}
- #5 \q_stop
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
+ \exp_after:wN \str_encode_utf_viii_loop:wwnnw
+ \int_value:w \int_div_truncate:nn {#1} {64} ; #1 ;
+ #5 \q_stop
\str_output_byte:n { #2 - 64 * ( #1 - \c_two ) }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}[int]{\str_convert_from_utf8:}
-% \begin{macro}[aux, rEXP]
+% \begin{variable}
% {
-% \str_aux_from_utf_viii:N,
-% \str_aux_from_utf_viii:wwN,
-% \str_aux_from_utf_viii:wNnnwN
+% \l_str_missing_flag ,
+% \l_str_extra_flag ,
+% \l_str_overlong_flag ,
+% \l_str_overflow_flag ,
% }
+% When decoding a string that is purportedly in the \textsc{utf-8}
+% encoding, four different errors can occur, signalled by a specific
+% flag for each (we define those flags using \cs{flag_clear_new:n}
+% rather than \cs{flag_new:n}, because they are shared with other
+% encoding definition files).
+% \begin{itemize}
+% \item \enquote{Missing continuation byte}: a leading byte is not
+% followed by the right number of continuation bytes.
+% \item \enquote{Extra continuation byte}: a continuation byte
+% appears where it was not expected, \emph{i.e.}, not after an
+% appropriate leading byte.
+% \item \enquote{Overlong}: a Unicode character is expressed using
+% more bytes than necessary, for instance, \hexnum{C0}\hexnum{80}
+% for the code point $0$, instead of a single null byte.
+% \item \enquote{Overflow}: this occurs when decoding produces
+% Unicode code points greater than $1114111$.
+% \end{itemize}
+% We only raise one \LaTeX3 error message, combining all the errors
+% which occurred. In the short message, the leading comma must be
+% removed to get a grammatically correct sentence. In the long text,
+% first remind the user what a correct \textsc{utf-8} string should
+% look like, then add error-specific information.
+% \begin{macrocode}
+\flag_clear_new:n { str_missing }
+\flag_clear_new:n { str_extra }
+\flag_clear_new:n { str_overlong }
+\flag_clear_new:n { str_overflow }
+\msg_kernel_new:nnnn { str } { utf8-decode }
+ {
+ Invalid~UTF-8~string: \exp_last_unbraced:Nf \use_none:n
+ \str_if_flag_times:nT { str_missing } { ,~missing~continuation~byte }
+ \str_if_flag_times:nT { str_extra } { ,~extra~continuation~byte }
+ \str_if_flag_times:nT { str_overlong } { ,~overlong~form }
+ \str_if_flag_times:nT { str_overflow } { ,~code~point~too~large }
+ .
+ }
+ {
+ In~the~UTF-8~encoding,~each~Unicode~character~consists~in~
+ 1~to~4~bytes,~with~the~following~bit~pattern: \\
+ \iow_indent:n
+ {
+ Code~point~\ \ \ \ <~128:~0xxxxxxx \\
+ Code~point~\ \ \ <~2048:~110xxxxx~10xxxxxx \\
+ Code~point~\ \ <~65536:~1110xxxx~10xxxxxx~10xxxxxx \\
+ Code~point~ <~1114112:~11110xxx~10xxxxxx~10xxxxxx~10xxxxxx \\
+ }
+ Bytes~of~the~form~10xxxxxx~are~called~continuation~bytes.
+ \flag_if_raised:nT { str_missing }
+ {
+ \\\\
+ A~leading~byte~(in~the~range~[192,255])~was~not~followed~by~
+ the~appropriate~number~of~continuation~bytes.
+ }
+ \flag_if_raised:nT { str_extra }
+ {
+ \\\\
+ LaTeX~came~across~a~continuation~byte~when~it~was~not~expected.
+ }
+ \flag_if_raised:nT { str_overlong }
+ {
+ \\\\
+ Every~Unicode~code~point~must~be~expressed~in~the~shortest~
+ possible~form.~For~instance,~'0xC0'~'0x83'~is~not~a~valid~
+ representation~for~the~code~point~3.
+ }
+ \flag_if_raised:nT { str_overflow }
+ {
+ \\\\
+ Unicode~limits~code~points~to~the~range~[0,1114111].
+ }
+ }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{macro}[int]{\str_convert_decode_utf8:}
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_from_utf_viii_overflow:w ,
-% \str_aux_from_utf_viii_error:
+% \str_decode_utf_viii_start:N,
+% \str_decode_utf_viii_continuation:wwN,
+% \str_decode_utf_viii_aux:wNnnwN
% }
-% In the \textsc{utf-8} encoding, bytes in the range
-% $[0,\hexnum{7F}]$ stand for themselves, bytes in the range
-% $[\hexnum{80},\hexnum{BF}]$ are continuation bytes,
-% and larger bytes must be followed by a number of continuation bytes.
-% Documentation to come.
-% ^^A todo: refuse "D800 -- "DFFF?
-% ^^A todo: document
+% \begin{macro}[aux, rEXP]
+% {\str_decode_utf_viii_overflow:w, \str_decode_utf_viii_end:}
+% Decoding is significantly harder than encoding. As before, lower
+% some flags, which are tested at the end (in bulk, to trigger at most
+% one \LaTeX3 error, as explained above). We expect successive
+% multi-byte sequences of the form \meta{start byte}
+% \meta{continuation bytes}. The \texttt{_start} auxiliary tests the
+% first byte:
+% \begin{itemize}
+% \item $[0,\hexnum{7F}]$: the byte stands alone, and is converted
+% to its own character code;
+% \item $[\hexnum{80}, \hexnum{BF}]$: unexpected continuation byte,
+% raise the appropriate flag, and convert that byte to the
+% replacement character \hexnum{FFFD};
+% \item $[\hexnum{C0}, \hexnum{FF}]$: this byte should be followed
+% by some continuation byte(s).
+% \end{itemize}
+% In the first two cases, \cs{use_none_delimit_by_q_stop:w} removes
+% data that only the third case requires, namely the limits of ranges
+% of Unicode characters which can be expressed with $1$, $2$, $3$, or
+% $4$ bytes.
+%
+% We can now concentrate on the multi-byte case and the
+% \texttt{_continuation} auxiliary. We expect |#3| to be in the range
+% $[\hexnum{80}, \hexnum{BF}]$. The test for this goes as follows: if
+% the character code is less than \hexnum{80}, we compare it to
+% $-\hexnum{C0}$, yielding \texttt{false}; otherwise to \hexnum{C0},
+% yielding \texttt{true} in the range $[\hexnum{80}, \hexnum{BF}]$ and
+% \texttt{false} otherwise. If we find that the byte is not a
+% continuation range, stop the current slew of bytes, output the
+% replacement character, and continue parsing with the \texttt{_start}
+% auxiliary, starting at the byte we just tested. Once we know that
+% the byte is a continuation byte, leave it behind us in the input
+% stream, compute what code point the bytes read so far would produce,
+% and feed that number to the \texttt{_aux} function.
+%
+% The \texttt{_aux} function tests whether we should look for more
+% continuation bytes or not. If the number it receives as |#1| is less
+% than the maximum |#4| for the current range, then we are done: check
+% for an overlong representation by comparing |#1| with the maximum
+% |#3| for the previous range. Otherwise, we call the
+% \texttt{_continuation} auxiliary again, after shifting the
+% \enquote{current code point} by |#4| (maximum from the range we just
+% checkedd).
+%
+% Two additional tests are needed: if we reach the end of the list of
+% range maxima and we are still not done, then we are faced with an
+% overflow. Clean up, and again insert the code point \hexnum{FFFD}
+% for the replacement character. Also, every time we read a byte, we
+% need to check whether we reached the end of the string. In a correct
+% \textsc{utf-8} string, this happens automatically when the
+% \texttt{_start} auxiliary leaves its first argument in the input
+% stream: the end-marker begins with \cs{prg_map_break:}, which ends
+% the loop. On the other hand, if the end is reached when looking for
+% a continuation byte, the \cs{use_none:n} |#3| construction removes
+% the first token from the end-marker, and leaves the \texttt{_end}
+% auxiliary, which raises the appropriate error flag before ending the
+% mapping.
% \begin{macrocode}
-\cs_new_protected_nopar:cpn { str_convert_from_utf8: }
+\cs_new_protected_nopar:cpn { str_convert_decode_utf8: }
{
+ \flag_clear:n { str_error }
+ \flag_clear:n { str_missing }
+ \flag_clear:n { str_extra }
+ \flag_clear:n { str_overlong }
+ \flag_clear:n { str_overflow }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl
- { \prg_map_break: \str_aux_from_utf_viii_error: }
+ \exp_after:wN \str_decode_utf_viii_start:N \g_str_result_tl
+ { \prg_map_break: \str_decode_utf_viii_end: }
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_error } { utf8-decode } { }
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1
+\cs_new:Npn \str_decode_utf_viii_start:N #1
{
#1
\if_num:w `#1 < "C0 \exp_stop_f:
@@ -2811,78 +2933,73 @@
\if_num:w `#1 < "80 \exp_stop_f:
\int_value:w `#1
\else:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-extra-conti } {#1}
+ \flag_raise:n { str_extra }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\fi:
\else:
- \if_num:w `#1 < "F5 \exp_stop_f:
- \exp_after:wN \str_aux_from_utf_viii:wwN
- \int_value:w \int_eval:n { `#1 - "C0 }
- \else:
- \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf8-invalid-byte } {#1}
- \int_use:N \c_str_replacement_char_int
- \fi:
+ \exp_after:wN \str_decode_utf_viii_continuation:wwN
+ \int_use:N \int_eval:w `#1 - "C0 \exp_after:wN \int_eval_end:
\fi:
\s_tl
\use_none_delimit_by_q_stop:w {"80} {"800} {"10000} {"110000} \q_stop
- \str_aux_from_utf_viii:N
+ \str_decode_utf_viii_start:N
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:wwN
- #1 \s_tl #2 \str_aux_from_utf_viii:N #3
+\cs_new:Npn \str_decode_utf_viii_continuation:wwN
+ #1 \s_tl #2 \str_decode_utf_viii_start:N #3
{
\use_none:n #3
- \if_num:w \int_eval:w ( `#3 + "20 ) / "40 = \c_three
+ \if_num:w `#3 < \if_num:w `#3 < "80 \exp_stop_f: - \fi: "C0 \exp_stop_f:
#3
- \exp_after:wN \str_aux_from_utf_viii:wNnnwN
- \int_value:w \int_eval:n { #1 * "40 + `#3 - "80 }
+ \exp_after:wN \str_decode_utf_viii_aux:wNnnwN
+ \int_use:N \int_eval:w
+ #1 * "40 + `#3 - "80
+ \exp_after:wN \int_eval_end:
\else:
\s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf8-missing-conti } {#3}
+ \flag_raise:n { str_missing }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\fi:
\s_tl
#2
- \str_aux_from_utf_viii:N #3
+ \str_decode_utf_viii_start:N #3
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii:wNnnwN
- #1 \s_tl #2#3#4 #5 \str_aux_from_utf_viii:N #6
+\cs_new:Npn \str_decode_utf_viii_aux:wNnnwN
+ #1 \s_tl #2#3#4 #5 \str_decode_utf_viii_start:N #6
{
- \if_num:w #1 < #4
+ \if_num:w #1 < #4 \exp_stop_f:
\s_tl
\if_num:w #1 < #3 \exp_stop_f:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-overlong } {#1}
+ \flag_raise:n { str_overlong }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\else:
#1
\fi:
\else:
\if_meaning:w \q_stop #5
- \str_aux_from_utf_viii_overflow:w #1
+ \str_decode_utf_viii_overflow:w #1
\fi:
- \exp_after:wN \str_aux_from_utf_viii:wwN
- \int_value:w \int_eval:n { #1 - #4 }
+ \exp_after:wN \str_decode_utf_viii_continuation:wwN
+ \int_use:N \int_eval:w #1 - #4 \exp_after:wN \int_eval_end:
\fi:
\s_tl
#2 {#4} #5
- \str_aux_from_utf_viii:N
+ \str_decode_utf_viii_start:N
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii_overflow:w #1 \fi: #2 \fi:
+\cs_new:Npn \str_decode_utf_viii_overflow:w #1 \fi: #2 \fi:
{
\fi: \fi:
- \msg_expandable_kernel_error:nnn
- { str } { utf8-overflow } {#1}
+ \flag_raise:n { str_overflow }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
}
-\cs_new_nopar:Npn \str_aux_from_utf_viii_error:
+\cs_new_nopar:Npn \str_decode_utf_viii_end:
{
\s_tl
- \msg_expandable_kernel_error:nn
- { str } { utf8-premature-end }
+ \flag_raise:n { str_missing }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int \s_tl
\prg_map_break:
}
@@ -2897,9 +3014,8 @@
%
% \subsubsection{\textsc{utf-16} support}
%
-% The definitions are done in a category code regime where
-% the bytes $254$ and $255$ used by the byte order mark
-% have catcode \enquote{other}.
+% The definitions are done in a category code regime where the bytes
+% $254$ and $255$ used by the byte order mark have catcode~$12$.
% \begin{macrocode}
%<*utf16>
\group_begin:
@@ -2907,53 +3023,61 @@
\char_set_catcode_other:N \^^ff
% \end{macrocode}
%
-% \begin{macro}[int]{\str_convert_to_utf16:}
-% \begin{macro}[int]{\str_convert_to_utf16be:}
-% \begin{macro}[int]{\str_convert_to_utf16le:}
+% \begin{macro}[int]
+% {
+% \str_convert_encode_utf16: ,
+% \str_convert_encode_utf16be: ,
+% \str_convert_encode_utf16le: ,
+% }
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_to_utf_xvi:n,
-% \str_aux_to_utf_xvi_be:n,
-% \str_aux_to_utf_xvi_le:n
+% \str_encode_utf_xvi_aux:N ,
+% \str_encode_utf_xvi_char:n ,
% }
-% Convert characters one by one in a loop, with different
-% behaviours depending on the character code:
+% When the endianness is not specified, it is big-endian by default,
+% and we add a byte-order mark. Convert characters one by one in a
+% loop, with different behaviours depending on the character code.
% \begin{itemize}
-% \item $[0, 55295]$ converted to two bytes;
-% \item $[55296, 57343]$ cannot be converted, replaced by
-% the replacement character;
-% \item $[57344, 65535]$ converted to two bytes;
-% \item $[65536, 1114111]$ converted to a pair of surrogates,
-% each two bytes.
+% \item $[0, \hexnum{D7FF}]$: converted to two bytes;
+% \item $[\hexnum{D800}, \hexnum{DFFF}]$ are used as surrogates:
+% they cannot be converted and are replaced by the replacement
+% character;
+% \item $[\hexnum{E000}, \hexnum{FFFF}]$: converted to two bytes;
+% \item $[\hexnum{10000}, \hexnum{10FFFF}]$: converted to a pair of
+% surrogates, each two bytes.
% \end{itemize}
-% For the duration of this operation, \cs{str_tmp:w} is defined
-% as a function to convert a number in the range $[0, 65536]$
-% to a pair of bytes (either big endian or little endian).
+% For the duration of this operation, \cs{str_tmp:w} is defined as a
+% function to convert a number in the range $[0, \hexnum{FFFF}]$ to a
+% pair of bytes (either big endian or little endian), by feeding the
+% quotient of the division of |#1| by \hexnum{100}, followed by |#1|
+% to \cs{str_encode_utf_xvi_be:nn} or its \texttt{le} analog: those
+% compute the remainder, and output two bytes for the quotient and
+% remainder.
% \begin{macrocode}
- \cs_new_protected_nopar:cpx { str_convert_to_utf16: }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf16: }
{
- \exp_not:c { str_convert_to_utf16be: }
- \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } }
+ \str_encode_utf_xvi_aux:N \str_output_byte_pair_be:n
+ \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff }
}
- \cs_new_protected_nopar:cpn { str_convert_to_utf16be: }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf16be: }
+ { \str_encode_utf_xvi_aux:N \str_output_byte_pair_be:n }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf16le: }
+ { \str_encode_utf_xvi_aux:N \str_output_byte_pair_le:n }
+ \cs_new_protected:Npn \str_encode_utf_xvi_aux:N #1
{
- \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_be:n
- \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n
+ \flag_clear:n { str_error }
+ \cs_set_eq:NN \str_tmp:w #1
+ \str_convert_gmap_internal:N \str_encode_utf_xvi_char:n
+ \str_if_flag_error:nnx { str_error } { utf16-encode } { }
}
- \cs_new_protected_nopar:cpn { str_convert_to_utf16le: }
- {
- \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_le:n
- \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n
- }
- \cs_new_nopar:Npn \str_aux_to_utf_xvi:n #1
+ \cs_new:Npn \str_encode_utf_xvi_char:n #1
{
\if_int_compare:w #1 < "D800 \exp_stop_f:
\str_tmp:w {#1}
\else:
\if_int_compare:w #1 < "10000 \exp_stop_f:
\if_int_compare:w #1 < "E000 \exp_stop_f:
- \msg_expandable_kernel_error:nnn
- { str } { unicode-surrogate } {#1}
+ \flag_raise:n { str_error }
\str_tmp:w { \c_str_replacement_char_int }
\else:
\str_tmp:w {#1}
@@ -2964,120 +3088,208 @@
\fi:
\fi:
}
- \cs_new_nopar:Npn \str_aux_to_utf_xvi_be:n #1
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{variable}
+% {
+% \l_str_missing_flag ,
+% \l_str_extra_flag ,
+% \l_str_end_flag ,
+% }
+% When encoding a Unicode string to \textsc{utf-16}, only one error
+% can occur: code points in the range $[\hexnum{D800},
+% \hexnum{DFFF}]$, corresponding to surrogates, cannot be encoded. We
+% use the all-purpose flag \texttt{str_error} to signal that error.
+%
+% When decoding a Unicode string which is purportedly in
+% \textsc{utf-16}, three errors can occur: a missing trail surrogate,
+% an unexpected trail surrogate, and a string containing an odd number
+% of bytes.
+% \begin{macrocode}
+ \flag_clear_new:n { str_missing }
+ \flag_clear_new:n { str_extra }
+ \flag_clear_new:n { str_end }
+ \msg_kernel_new:nnnn { str } { utf16-encode }
+ { Unicode~string~cannot~be~expressed~in~UTF-16:~surrogate. }
+ {
+ Surrogate~code~points~(in~the~range~[U+D800,~U+DFFF])~
+ can~be~expressed~in~the~UTF-8~and~UTF-32~encodings,~
+ but~not~in~the~UTF-16~encoding.
+ }
+ \msg_kernel_new:nnnn { str } { utf16-decode }
{
- \str_output_byte:n { \int_div_truncate:nn {#1} {"100} }
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ Invalid~UTF-16~string: \exp_last_unbraced:Nf \use_none:n
+ \str_if_flag_times:nT { str_missing } { ,~missing~trail~surrogate }
+ \str_if_flag_times:nT { str_extra } { ,~extra~trail~surrogate }
+ \str_if_flag_times:nT { str_end } { ,~odd~number~of~bytes }
+ .
}
- \cs_new_nopar:Npn \str_aux_to_utf_xvi_le:n #1
{
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
- \str_output_byte:n { \int_div_truncate:nn {#1} {"100} }
+ In~the~UTF-16~encoding,~each~Unicode~character~is~encoded~as~
+ 2~or~4~bytes: \\
+ \iow_indent:n
+ {
+ Code~point~in~[U+0000,~U+D7FF]:~two~bytes \\
+ Code~point~in~[U+D800,~U+DFFF]:~illegal \\
+ Code~point~in~[U+E000,~U+FFFF]:~two~bytes \\
+ Code~point~in~[U+10000,~U+10FFFF]:~
+ a~lead~surrogate~and~a~trail~surrogate \\
+ }
+ Lead~surrogates~are~pairs~of~bytes~in~the~range~[0xD800,~0xDBFF],~
+ and~trail~surrogates~are~in~the~range~[0xDC00,~0xDFFF].
+ \flag_if_raised:nT { str_missing }
+ {
+ \\\\
+ A~lead~surrogate~was~not~followed~by~a~trail~surrogate.
+ }
+ \flag_if_raised:nT { str_extra }
+ {
+ \\\\
+ LaTeX~came~across~a~trail~surrogate~when~it~was~not~expected.
+ }
+ \flag_if_raised:nT { str_end }
+ {
+ \\\\
+ The~string~contained~an~odd~number~of~bytes.~This~is~invalid:~
+ the~basic~code~unit~for~UTF-16~is~16~bits~(2~bytes).
+ }
}
% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
+% \end{variable}
%
% \begin{macro}[int]
% {
-% \str_convert_from_utf16:,
-% \str_convert_from_utf16be:,
-% \str_convert_from_utf16le:
+% \str_convert_decode_utf16: ,
+% \str_convert_decode_utf16be: ,
+% \str_convert_decode_utf16le: ,
% }
-% \begin{macro}[aux]{\str_aux_from_utf_xvi_bom:NNw}
-% \begin{macro}[aux]{\str_aux_from_utf_xvi:No}
-% Define \cs{str_tmp:w} to take two arguments and return the
-% character code of the first one if the string is big-endian,
-% and the second one if the string is little-endian.
-% If the endianness is not known, look for a byte order mark
-% to decide.
+% \begin{macro}[aux]{\str_decode_utf_xvi_bom:NN, \str_decode_utf_xvi:Nw}
+% As for \textsc{utf-8}, decoding \textsc{utf-16} is harder than
+% encoding it. If the endianness is unknown, check the first two
+% bytes: if those are \hexnum{FE} and \hexnum{FF} in either order,
+% remove them and use the corresponding endianness, otherwise assume
+% big-endianness. The three endianness cases are based on a common
+% auxiliary whose first argument is $1$ for big-endian and $2$ for
+% little-endian, and whose second argument, delimited by the scan mark
+% \cs{s_stop}, is expanded once (the string may be long; passing
+% \cs{g_str_result_tl} as an argument before expansion is cheaper).
+%
+% The \cs{strr_decode_utf_xvi:Nw} function defines \cs{str_tmp:w} to
+% take two arguments and return the character code of the first one if
+% the string is big-endian, and the second one if the string is
+% little-endian, then loops over the string using
+% \cs{str_decode_utf_xvi_pair:NN} described below.
% \begin{macrocode}
- \cs_new_protected_nopar:cpn { str_convert_from_utf16be: }
- { \str_aux_from_utf_xvi:No 1 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf16le: }
- { \str_aux_from_utf_xvi:No 2 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf16: }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf16be: }
+ { \str_decode_utf_xvi:Nw 1 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf16le: }
+ { \str_decode_utf_xvi:Nw 2 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf16: }
{
- \exp_after:wN \str_aux_from_utf_xvi_bom:NNw
- \g_str_result_tl \scan_stop: \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xvi_bom:NN
+ \g_str_result_tl \s_stop \s_stop \s_stop
}
- \cs_new_protected_nopar:Npn
- \str_aux_from_utf_xvi_bom:NNw #1#2#3\scan_stop:
+ \cs_new_protected:Npn \str_decode_utf_xvi_bom:NN #1#2
{
- \str_if_eq:nnTF { #1#2 } { ^^ff ^^fe }
- { \str_aux_from_utf_xvi:No 2 {#3} }
+ \str_if_eq:xxTF { #1#2 } { ^^ff ^^fe }
+ { \str_decode_utf_xvi:Nw 2 }
{
- \str_if_eq:nnTF { #1#2 } { ^^fe ^^ff }
- { \str_aux_from_utf_xvi:No 1 {#3} }
- { \str_aux_from_utf_xvi:No 1 {#1#2#3} }
+ \str_if_eq:xxTF { #1#2 } { ^^fe ^^ff }
+ { \str_decode_utf_xvi:Nw 1 }
+ { \str_decode_utf_xvi:Nw 1 #1#2 }
}
}
- \cs_new_protected_nopar:Npn \str_aux_from_utf_xvi:No #1#2
+ \cs_new_protected:Npn \str_decode_utf_xvi:Nw #1#2 \s_stop
{
- \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
+ \flag_clear:n { str_error }
+ \flag_clear:n { str_missing }
+ \flag_clear:n { str_extra }
+ \flag_clear:n { str_end }
+ \cs_set:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_from_utf_xvi:NN
- #2 \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xvi_pair:NN
+ #2 \q_nil \q_nil
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_error } { utf16-decode } { }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
%
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NNwNNN}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNw}
-% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:NNN}
-% When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w}
-% is such that |\str_tmp:w #1#2| expands to |`#1| if the string
-% is big-endian, and |`#2| if the string is little-endian.
-% If the code unit (a pair of characters) does not represent
-% a surrogate, the \cs{if_case:w} construction expands to nothing,
-% and we leave in the input stream \meta{characters} \cs{s_tl}
-% \meta{char code} \cs{s_tl}, and move on to the next code unit.
-% When the end is found, we check for the case of an odd-length
-% string (an error) and end the mapping.
-%
-% Surrogates are characterized by a most significant byte in the
-% ranges $[\hexnum{D8},\hexnum{DB}]$ and $[\hexnum{DC},\hexnum{DF}]$,
-% detected by dividing the character code by $4$ (\eTeX{} rounds to
-% nearest, ties away from zero). Here, a trail surrogate is invalid,
-% replaced by the replacement character. Lead surrogates are treated by
-% \cs{str_aux_from_utf_xvi:NNwNNN}, which grabs the next two
-% characters, checks for a premature end, and checks whether
-% the code unit is a trailing surrogate. If it is, leave
-% \meta{characters} \cs{s_tl} \meta{char code} \cs{s_tl} in the
-% input stream and move on to the next code unit. Otherwise, raise
-% an error, and continue parsing, replacing only the first code unit
-% (the leading surrogate) by the replacement character.
+% \begin{macro}[aux, rEXP]
+% {
+% \str_decode_utf_xvi_pair:NN ,
+% \str_decode_utf_xvi_quad:NNwNN ,
+% \str_decode_utf_xvi_pair_end:Nw ,
+% }
+% \begin{macro}[aux, rEXP]
+% {
+% \str_decode_utf_xvi_error:nNN ,
+% \str_decode_utf_xvi_extra:NNw ,
+% }
+% Bytes are read two at a time. At this stage, |\str_tmp:w #1#2|
+% expands to the character code of the most significant byte, and we
+% distinguish cases depending on which range it lies in:
+% \begin{itemize}
+% \item $[\hexnum{D8}, \hexnum{DB}]$ signals a lead surrogate, and
+% the integer expression yields $1$ (\eTeX{} rounds ties away from
+% zero);
+% \item $[\hexnum{DC}, \hexnum{DF}]$ signals a trail surrogate,
+% unexpected here, and the integer expression yields $2$;
+% \item any other value signals a code point in the Basic
+% Multilingual Plane, which stands for itself, and the
+% \cs{if_case:w} construction expands to nothing (cases other than
+% $1$ or $2$), leaving the relevant material in the input stream,
+% followed by another call to the \texttt{_pair} auxiliary.
+% \end{itemize}
+% The case of a lead surrogate is treated by the \texttt{_quad}
+% auxiliary, whose arguments |#1|, |#2|, |#4| and |#5| are the four
+% bytes. We expect the most significant byte of |#4#5| to be in the
+% range $[\hexnum{DC}, \hexnum{DF}]$ (trail surrogate). The test is
+% similar to the test used for continuation bytes in the
+% \textsc{utf-8} decoding functions. In the case where |#4#5| is
+% indeed a trail surrogate, leave |#1#2#4#5| \cs{s_tl}
+% \meta{code~point} \cs{s_tl}, and remove the pair |#4#5| before
+% looping with \cs{str_decode_utf_xvi_pair:NN}. Otherwise, of course,
+% complain about the missing surrogate.
+%
+% Every time we read a pair of bytes, we test for the end-marker
+% \cs{q_nil}. When reaching the end, we additionally check that the
+% string had an even length. Also, if the end is reached when
+% expecting a trail surrogate, we treat that as a missing surrogate.
% \begin{macrocode}
- \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2
+ \cs_new:Npn \str_decode_utf_xvi_pair:NN #1#2
{
- \if_meaning:w \scan_stop: #2
- \str_aux_from_utf_xvi_end:Nw #1
+ \if_meaning:w \q_nil #2
+ \str_decode_utf_xvi_pair_end:Nw #1
\fi:
\if_case:w
\int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
- \or: \exp_after:wN \str_aux_from_utf_xvi:NNwNNN
- \or: \exp_after:wN \str_aux_from_utf_xvi_error:NNw
+ \or: \exp_after:wN \str_decode_utf_xvi_quad:NNwNN
+ \or: \exp_after:wN \str_decode_utf_xvi_extra:NNw
\fi:
#1#2 \s_tl
\int_eval:n { "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 } \s_tl
- \str_aux_from_utf_xvi:NN
+ \str_decode_utf_xvi_pair:NN
}
- \cs_new:Npn \str_aux_from_utf_xvi:NNwNNN #1#2 \s_tl #3 \s_tl #4#5#6
+ \cs_new:Npn \str_decode_utf_xvi_quad:NNwNN
+ #1#2 #3 \str_decode_utf_xvi_pair:NN #4#5
{
- \if_meaning:w \scan_stop: #6
- \str_aux_from_utf_xvi_error:NNN #1#2#5
+ \if_meaning:w \q_nil #5
+ \str_decode_utf_xvi_error:nNN { missing } #1#2
+ \str_decode_utf_xvi_pair_end:Nw #4
\fi:
- \if_num:w \int_eval:w ( \str_tmp:w #5#6 - "DA ) / \c_four = \c_one
- #1 #2 #5 #6 \s_tl
+ \if_num:w
+ \if_num:w \str_tmp:w #4#5 < "DC \exp_stop_f:
+ \c_zero = \c_one
+ \else:
+ \str_tmp:w #4#5 < "E0 \exp_stop_f:
+ \fi:
+ #1 #2 #4 #5 \s_tl
\int_eval:n
{
( "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 - "D800 ) * "400
@@ -3086,49 +3298,31 @@
\s_tl
\exp_after:wN \use_i:nnn
\else:
- \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl \s_tl
+ \str_decode_utf_xvi_error:nNN { missing } #1#2
\fi:
- #4#5#6
+ \str_decode_utf_xvi_pair:NN #4#5
}
- \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi:
+ \cs_new:Npn \str_decode_utf_xvi_pair_end:Nw #1 \fi:
{
\fi:
- \if_meaning:w \scan_stop: #1
+ \if_meaning:w \q_nil #1
\else:
- #1 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-odd } {#1}
- \int_use:N \c_str_replacement_char_int \s_tl
+ \str_decode_utf_xvi_error:nNN { end } #1 \prg_do_nothing:
\fi:
\prg_map_break:
}
- \cs_new:Npn \str_aux_from_utf_xvi_error:NNw #1#2 \s_tl #3 \s_tl
+ \cs_new:Npn \str_decode_utf_xvi_extra:NNw #1#2 \s_tl #3 \s_tl
+ { \str_decode_utf_xvi_error:nNN { extra } #1#2 }
+ \cs_new:Npn \str_decode_utf_xvi_error:nNN #1#2#3
{
- #1 #2 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-surrogate } { #1#2 }
+ \flag_raise:n { str_error }
+ \flag_raise:n { str_#1 }
+ #2 #3 \s_tl
\int_use:N \c_str_replacement_char_int \s_tl
}
- \cs_new:Npn \str_aux_from_utf_xvi_error:NNN #1#2#3
- {
- \if_meaning:w \scan_stop: #3
- #1#2 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-surrogate } { #1#2 }
- \else:
- #1#2#3 \s_tl
- \msg_expandable_kernel_error:nnn
- { str } { utf16-odd } { #1#2#3 }
- \fi:
- \int_use:N \c_str_replacement_char_int \s_tl
- \prg_map_break:
- }
% \end{macrocode}
% \end{macro}
% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
%
% Restore the original catcodes of bytes $254$ and $255$.
% \begin{macrocode}
@@ -3138,9 +3332,9 @@
%
% \subsubsection{\textsc{utf-32} support}
%
-% The definitions are done in a category code regime where
-% the bytes $0$, $254$ and $255$ used by the byte order mark
-% have catcode \enquote{other}.
+% The definitions are done in a category code regime where the bytes
+% $0$, $254$ and $255$ used by the byte order mark have catcode
+% \enquote{other}.
% \begin{macrocode}
%<*utf32>
\group_begin:
@@ -3151,130 +3345,188 @@
%
% \begin{macro}[int]
% {
-% \str_convert_to_utf32:,
-% \str_convert_to_utf32be:,
-% \str_convert_to_utf32le:
+% \str_convert_encode_utf32: ,
+% \str_convert_encode_utf32be: ,
+% \str_convert_encode_utf32le: ,
% }
% \begin{macro}[aux, rEXP]
% {
-% \str_aux_to_utf_xxxii_be:n,
-% \str_aux_to_utf_xxxii_le:n
+% \str_encode_utf_xxxii_be:n ,
+% \str_encode_utf_xxxii_be_aux:nn ,
+% \str_encode_utf_xxxii_le:n ,
+% \str_encode_utf_xxxii_le_aux:nn ,
% }
-% Convert each integer in the comma-list \cs{g_str_result_tl}
-% to a sequence of four bytes. The functions for big-endian
-% and little-endian encodings are very similar, but the
+% Convert each integer in the comma-list \cs{g_str_result_tl} to a
+% sequence of four bytes. The functions for big-endian and
+% little-endian encodings are very similar, but the
% \cs{str_output_byte:n} instructions are reversed.
% \begin{macrocode}
- \cs_new_protected_nopar:cpx { str_convert_to_utf32: }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf32: }
{
- \exp_not:c { str_convert_to_utf32be: }
- \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } }
+ \str_convert_gmap_internal:N \str_encode_utf_xxxii_be:n
+ \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff }
}
- \cs_new_protected_nopar:cpn { str_convert_to_utf32be: }
- { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_be:n }
- \cs_new_protected_nopar:cpn { str_convert_to_utf32le: }
- { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_le:n }
- \cs_new:Npn \str_aux_to_utf_xxxii_be:n #1
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf32be: }
+ { \str_convert_gmap_internal:N \str_encode_utf_xxxii_be:n }
+ \cs_new_protected_nopar:cpn { str_convert_encode_utf32le: }
+ { \str_convert_gmap_internal:N \str_encode_utf_xxxii_le:n }
+ \cs_new:Npn \str_encode_utf_xxxii_be:n #1
+ {
+ \exp_args:Nf \str_encode_utf_xxxii_be_aux:nn
+ { \int_div_truncate:nn {#1} { "100 } } {#1}
+ }
+ \cs_new:Npn \str_encode_utf_xxxii_be_aux:nn #1#2
{
^^00
- \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } }
- \str_output_byte:n
- { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} }
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ \str_output_byte_pair_be:n {#1}
+ \str_output_byte:n { #2 - #1 * "100 }
}
- \cs_new:Npn \str_aux_to_utf_xxxii_le:n #1
+ \cs_new:Npn \str_encode_utf_xxxii_le:n #1
{
- \str_output_byte:n { \int_mod:nn {#1} {"100} }
- \str_output_byte:n
- { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} }
- \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } }
+ \exp_args:Nf \str_encode_utf_xxxii_le_aux:nn
+ { \int_div_truncate:nn {#1} { "100 } } {#1}
+ }
+ \cs_new:Npn \str_encode_utf_xxxii_le_aux:nn #1#2
+ {
+ \str_output_byte:n { #2 - #1 * "100 }
+ \str_output_byte_pair_le:n {#1}
^^00
}
% \end{macrocode}
% \end{macro}
% \end{macro}
%
+% \begin{variable}{str_overflow, str_end}
+% There can be no error when encoding in \textsc{utf-32}. When
+% decoding, the string may not have length $4n$, or it may contain
+% code points larger than \hexnum{10FFFF}. The latter case often
+% happens if the encoding was in fact not \textsc{utf-32}, because
+% most arbitrary strings are not valid in \textsc{utf-32}.
+% \begin{macrocode}
+ \flag_clear_new:n { str_overflow }
+ \flag_clear_new:n { str_end }
+ \msg_kernel_new:nnnn { str } { utf32-decode }
+ {
+ Invalid~UTF-32~string: \exp_last_unbraced:Nf \use_none:n
+ \str_if_flag_times:nT { str_overflow } { ,~code~point~too~large }
+ \str_if_flag_times:nT { str_end } { ,~truncated~string }
+ .
+ }
+ {
+ In~the~UTF-32~encoding,~every~Unicode~character~
+ (in~the~range~[U+0000,~U+10FFFF])~is~encoded~as~4~bytes.
+ \flag_if_raised:nT { str_overflow }
+ {
+ \\\\
+ LaTeX~came~across~a~code~point~larger~than~1114111,~
+ the~maximum~code~point~defined~by~Unicode.~
+ Perhaps~the~string~was~not~encoded~in~the~UTF-32~encoding?
+ }
+ \flag_if_raised:nT { str_end }
+ {
+ \\\\
+ The~length~of~the~string~is~not~a~multiple~of~4.~
+ Perhaps~the~string~was~truncated?
+ }
+ }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}[int]
% {
-% \str_convert_from_utf32:,
-% \str_convert_from_utf32be:,
-% \str_convert_from_utf32le:
+% \str_convert_decode_utf32: ,
+% \str_convert_decode_utf32be: ,
+% \str_convert_decode_utf32le: ,
% }
% \begin{macro}[aux]
-% {
-% \str_aux_from_utf_xxxii_bom:NNNNw,
-% \str_aux_from_utf_xxxii:No
-% }
+% {\str_decode_utf_xxxii_bom:NNNN, \str_decode_utf_xxxii:Nw}
% \begin{macro}[aux, rEXP]
-% {
-% \str_aux_from_utf_xxxii:NNNN,
-% \str_aux_from_utf_xxxii_end:w
-% }
-% See the conversion functions from \textsc{utf-16} encodings.
-% The \cs{str_aux_from_utf_xxxii:No} auxiliary defines
-% \cs{str_tmp:w} to take two arguments and give the character
-% code of either the first argument (big-endian) or the second
-% argument (little-endian). Then we loop over the string,
-% four bytes at a time, check for overflow, and otherwise
-% produce the number corresponding to the four bytes with
-% the given endianness.
+% {\str_decode_utf_xxxii_loop:NNNN, \str_decode_utf_xxxii_end:w}
+%
+% The structure is similar to \textsc{utf-16} decoding functions. If
+% the endianness is not given, test the first $4$ bytes of the string
+% (possibly \cs{s_stop} if the string is too short) for the presence
+% of a byte-order mark. If there is a byte-order mark, use that
+% endianness, and remove the $4$ bytes, otherwise default to
+% big-endian, and leave the $4$ bytes in place. The
+% \cs{str_decode_utf_xxxii:Nw} auxiliary recieves $1$ or $2$ as its
+% first argument indicating endianness, and the string to convert as
+% its second argument (expanded or not). It sets \cs{str_tmp:w} to
+% expand to the character code of either of its two arguments
+% depending on endianness, then triggers the \texttt{_loop} auxiliary
+% inside an \texttt{x}-expanding assignment to \cs{g_str_result_tl}.
+%
+% The \texttt{_loop} auxiliary first checks for the end-of-string
+% marker \cs{s_stop}, calling the \texttt{_end} auxiliary if
+% appropriate. Otherwise, leave the \meta{4~bytes} \cs{s_tl} behind,
+% then check that the code point is not overflowing: the leading byte
+% must be $0$, and the following byte at most $16$.
+%
+% In the ending code, we check that there remains no byte: there
+% should be nothing left until the first \cs{s_stop}. Break the map.
% \begin{macrocode}
- \cs_new_protected_nopar:cpn { str_convert_from_utf32be: }
- { \str_aux_from_utf_xxxii:No 1 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf32le: }
- { \str_aux_from_utf_xxxii:No 2 { \g_str_result_tl } }
- \cs_new_protected_nopar:cpn { str_convert_from_utf32: }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf32be: }
+ { \str_decode_utf_xxxii:Nw 1 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf32le: }
+ { \str_decode_utf_xxxii:Nw 2 \g_str_result_tl \s_stop }
+ \cs_new_protected_nopar:cpn { str_convert_decode_utf32: }
{
- \exp_after:wN \str_aux_from_utf_xxxii_bom:NNNNw \g_str_result_tl
- \scan_stop: \scan_stop: \scan_stop: \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xxxii_bom:NNNN \g_str_result_tl
+ \s_stop \s_stop \s_stop \s_stop \s_stop
}
- \cs_new_protected_nopar:Npn
- \str_aux_from_utf_xxxii_bom:NNNNw #1#2#3#4#5\scan_stop:
+ \cs_new_protected:Npn \str_decode_utf_xxxii_bom:NNNN #1#2#3#4
{
- \str_if_eq:nnTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 }
- { \str_aux_from_utf_xxxii:No 2 {#5} }
+ \str_if_eq:xxTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 }
+ { \str_decode_utf_xxxii:Nw 2 }
{
- \str_if_eq:nnTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff }
- { \str_aux_from_utf_xxxii:No 1 {#5} }
- { \str_aux_from_utf_xxxii:No 1 {#1#2#3#4#5} }
+ \str_if_eq:xxTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff }
+ { \str_decode_utf_xxxii:Nw 1 }
+ { \str_decode_utf_xxxii:Nw 1 #1#2#3#4 }
}
}
- \cs_new_protected_nopar:Npn \str_aux_from_utf_xxxii:No #1#2
+ \cs_new_protected:Npn \str_decode_utf_xxxii:Nw #1#2 \s_stop
{
- \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
+ \flag_clear:n { str_overflow }
+ \flag_clear:n { str_end }
+ \flag_clear:n { str_error }
+ \cs_set:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_from_utf_xxxii:NNNN
- #2 \scan_stop: \scan_stop: \scan_stop: \scan_stop:
+ \exp_after:wN \str_decode_utf_xxxii_loop:NNNN
+ #2 \s_stop \s_stop \s_stop \s_stop
\prg_break_point:n { }
}
+ \str_if_flag_error:nnx { str_error } { utf32-decode } { }
}
- \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4
+ \cs_new:Npn \str_decode_utf_xxxii_loop:NNNN #1#2#3#4
{
- \if_meaning:w \scan_stop: #4
- \exp_after:wN \str_aux_from_utf_xxxii_end:w
+ \if_meaning:w \s_stop #4
+ \exp_after:wN \str_decode_utf_xxxii_end:w
\fi:
#1#2#3#4 \s_tl
- \if_num:w \int_eval:w
- \str_tmp:w #1#4 * \c_two_hundred_fifty_six
- + \str_tmp:w #2#3
- > \c_sixteen
- \msg_expandable_kernel_error:nnn
- { str } { utf32-overflow } { #1#2#3#4 }
+ \if_num:w \str_tmp:w #1#4 > \c_zero
+ \flag_raise:n { str_overflow }
+ \flag_raise:n { str_error }
\int_use:N \c_str_replacement_char_int
\else:
- \int_eval:n
- { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 }
+ \if_num:w \str_tmp:w #2#3 > \c_sixteen
+ \flag_raise:n { str_overflow }
+ \flag_raise:n { str_error }
+ \int_use:N \c_str_replacement_char_int
+ \else:
+ \int_eval:n
+ { \str_tmp:w #2#3*"10000 + \str_tmp:w #3#2*"100 + \str_tmp:w #4#1 }
+ \fi:
\fi:
\s_tl
- \str_aux_from_utf_xxxii:NNNN
+ \str_decode_utf_xxxii_loop:NNNN
}
- \cs_new:Npn \str_aux_from_utf_xxxii_end:w #1 \scan_stop:
+ \cs_new:Npn \str_decode_utf_xxxii_end:w #1 \s_stop
{
\tl_if_empty:nF {#1}
{
- \msg_expandable_kernel_error:nnn
- { str } { utf32-truncated } { #1 }
+ \flag_raise:n { str_end }
+ \flag_raise:n { str_error }
#1 \s_tl
\int_use:N \c_str_replacement_char_int \s_tl
}
@@ -3292,19 +3544,15 @@
% \end{macrocode}
%
% \subsubsection{\textsc{iso 8859} support}
-%^^A todo: document
%
-% The \textsc{iso-8859-1} encoding exactly matches with the $256$
-% first Unicode characters. For other 8-bit encodings
-% of the \textsc{iso-8859} family, we keep track only of
-% differences, and of unassigned bytes.
+% The \textsc{iso-8859-1} encoding exactly matches with the $256$ first
+% Unicode characters. For other 8-bit encodings of the \textsc{iso-8859}
+% family, we keep track only of differences, and of unassigned bytes.
% \begin{macrocode}
%<*iso88591>
-\str_encoding_eight_bit:n { iso88591 }
-\tl_const:cn { c_str_encoding_iso88591_tl }
+\str_declare_eight_bit_encoding:nnn { iso88591 }
{
}
-\tl_const:cn { c_str_encoding_iso88591_missing_tl }
{
}
%</iso88591>
@@ -3312,8 +3560,7 @@
%
% \begin{macrocode}
%<*iso88592>
-\str_encoding_eight_bit:n { iso88592 }
-\tl_const:cn { c_str_encoding_iso88592_tl }
+\str_declare_eight_bit_encoding:nnn { iso88592 }
{
{ A1 } { 0104 }
{ A2 } { 02D8 }
@@ -3373,7 +3620,6 @@
{ FE } { 0163 }
{ FF } { 02D9 }
}
-\tl_const:cn { c_str_encoding_iso88592_missing_tl }
{
}
%</iso88592>
@@ -3381,8 +3627,7 @@
%
% \begin{macrocode}
%<*iso88593>
-\str_encoding_eight_bit:n { iso88593 }
-\tl_const:cn { c_str_encoding_iso88593_tl }
+\str_declare_eight_bit_encoding:nnn { iso88593 }
{
{ A1 } { 0126 }
{ A2 } { 02D8 }
@@ -3413,7 +3658,6 @@
{ FE } { 015D }
{ FF } { 02D9 }
}
-\tl_const:cn { c_str_encoding_iso88593_missing_tl }
{
{ A5 }
{ AE }
@@ -3428,8 +3672,7 @@
%
% \begin{macrocode}
%<*iso88594>
-\str_encoding_eight_bit:n { iso88594 }
-\tl_const:cn { c_str_encoding_iso88594_tl }
+\str_declare_eight_bit_encoding:nnn { iso88594 }
{
{ A1 } { 0104 }
{ A2 } { 0138 }
@@ -3482,7 +3725,6 @@
{ FE } { 016B }
{ FF } { 02D9 }
}
-\tl_const:cn { c_str_encoding_iso88594_missing_tl }
{
}
%</iso88594>
@@ -3490,8 +3732,7 @@
%
% \begin{macrocode}
%<*iso88595>
-\str_encoding_eight_bit:n { iso88595 }
-\tl_const:cn { c_str_encoding_iso88595_tl }
+\str_declare_eight_bit_encoding:nnn { iso88595 }
{
{ A1 } { 0401 }
{ A2 } { 0402 }
@@ -3588,7 +3829,6 @@
{ FE } { 045E }
{ FF } { 045F }
}
-\tl_const:cn { c_str_encoding_iso88595_missing_tl }
{
}
%</iso88595>
@@ -3596,8 +3836,7 @@
%
% \begin{macrocode}
%<*iso88596>
-\str_encoding_eight_bit:n { iso88596 }
-\tl_const:cn { c_str_encoding_iso88596_tl }
+\str_declare_eight_bit_encoding:nnn { iso88596 }
{
{ AC } { 060C }
{ BB } { 061B }
@@ -3648,7 +3887,6 @@
{ F1 } { 0651 }
{ F2 } { 0652 }
}
-\tl_const:cn { c_str_encoding_iso88596_missing_tl }
{
{ A1 }
{ A2 }
@@ -3688,8 +3926,7 @@
%
% \begin{macrocode}
%<*iso88597>
-\str_encoding_eight_bit:n { iso88597 }
-\tl_const:cn { c_str_encoding_iso88597_tl }
+\str_declare_eight_bit_encoding:nnn { iso88597 }
{
{ A1 } { 2018 }
{ A2 } { 2019 }
@@ -3769,7 +4006,6 @@
{ FD } { 03CD }
{ FE } { 03CE }
}
-\tl_const:cn { c_str_encoding_iso88597_missing_tl }
{
{ AE }
{ D2 }
@@ -3779,8 +4015,7 @@
%
% \begin{macrocode}
%<*iso88598>
-\str_encoding_eight_bit:n { iso88598 }
-\tl_const:cn { c_str_encoding_iso88598_tl }
+\str_declare_eight_bit_encoding:nnn { iso88598 }
{
{ AA } { 00D7 }
{ BA } { 00F7 }
@@ -3815,7 +4050,6 @@
{ FD } { 200E }
{ FE } { 200F }
}
-\tl_const:cn { c_str_encoding_iso88598_missing_tl }
{
{ A1 }
{ BF }
@@ -3858,8 +4092,7 @@
%
% \begin{macrocode}
%<*iso88599>
-\str_encoding_eight_bit:n { iso88599 }
-\tl_const:cn { c_str_encoding_iso88599_tl }
+\str_declare_eight_bit_encoding:nnn { iso88599 }
{
{ D0 } { 011E }
{ DD } { 0130 }
@@ -3868,7 +4101,6 @@
{ FD } { 0131 }
{ FE } { 015F }
}
-\tl_const:cn { c_str_encoding_iso88599_missing_tl }
{
}
%</iso88599>
@@ -3876,8 +4108,7 @@
%
% \begin{macrocode}
%<*iso885910>
-\str_encoding_eight_bit:n { iso885910 }
-\tl_const:cn { c_str_encoding_iso885910_tl }
+\str_declare_eight_bit_encoding:nnn { iso885910 }
{
{ A1 } { 0104 }
{ A2 } { 0112 }
@@ -3926,7 +4157,6 @@
{ F9 } { 0173 }
{ FF } { 0138 }
}
-\tl_const:cn { c_str_encoding_iso885910_missing_tl }
{
}
%</iso885910>
@@ -3934,8 +4164,7 @@
%
% \begin{macrocode}
%<*iso885911>
-\str_encoding_eight_bit:n { iso885911 }
-\tl_const:cn { c_str_encoding_iso885911_tl }
+\str_declare_eight_bit_encoding:nnn { iso885911 }
{
{ A1 } { 0E01 }
{ A2 } { 0E02 }
@@ -4025,7 +4254,6 @@
{ FA } { 0E5A }
{ FB } { 0E5B }
}
-\tl_const:cn { c_str_encoding_iso885911_missing_tl }
{
{ DB }
{ DC }
@@ -4037,8 +4265,7 @@
%
% \begin{macrocode}
%<*iso885913>
-\str_encoding_eight_bit:n { iso885913 }
-\tl_const:cn { c_str_encoding_iso885913_tl }
+\str_declare_eight_bit_encoding:nnn { iso885913 }
{
{ A1 } { 201D }
{ A5 } { 201E }
@@ -4097,7 +4324,6 @@
{ FE } { 017E }
{ FF } { 2019 }
}
-\tl_const:cn { c_str_encoding_iso885913_missing_tl }
{
}
%</iso885913>
@@ -4105,8 +4331,7 @@
%
% \begin{macrocode}
%<*iso885914>
-\str_encoding_eight_bit:n { iso885914 }
-\tl_const:cn { c_str_encoding_iso885914_tl }
+\str_declare_eight_bit_encoding:nnn { iso885914 }
{
{ A1 } { 1E02 }
{ A2 } { 1E03 }
@@ -4140,7 +4365,6 @@
{ F7 } { 1E6B }
{ FE } { 0177 }
}
-\tl_const:cn { c_str_encoding_iso885914_missing_tl }
{
}
%</iso885914>
@@ -4148,8 +4372,7 @@
%
% \begin{macrocode}
%<*iso885915>
-\str_encoding_eight_bit:n { iso885915 }
-\tl_const:cn { c_str_encoding_iso885915_tl }
+\str_declare_eight_bit_encoding:nnn { iso885915 }
{
{ A4 } { 20AC }
{ A6 } { 0160 }
@@ -4160,7 +4383,6 @@
{ BD } { 0153 }
{ BE } { 0178 }
}
-\tl_const:cn { c_str_encoding_iso885915_missing_tl }
{
}
%</iso885915>
@@ -4168,8 +4390,7 @@
%
% \begin{macrocode}
%<*iso885916>
-\str_encoding_eight_bit:n { iso885916 }
-\tl_const:cn { c_str_encoding_iso885916_tl }
+\str_declare_eight_bit_encoding:nnn { iso885916 }
{
{ A1 } { 0104 }
{ A2 } { 0105 }
@@ -4212,7 +4433,6 @@
{ FD } { 0119 }
{ FE } { 021B }
}
-\tl_const:cn { c_str_encoding_iso885916_missing_tl }
{
}
%</iso885916>
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
index 55b4066a306..676a39bea7b 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
@@ -43,36 +43,36 @@ Do not distribute a modified version of this file.
% Escapings.
\generate{%
- \file{l3str-hex.def} {\from{l3str.dtx}{hex}}%
- \file{l3str-name.def} {\from{l3str.dtx}{name}}%
- \file{l3str-string.def} {\from{l3str.dtx}{string}}%
- \file{l3str-url.def} {\from{l3str.dtx}{url}}%
+ \file{l3str-esc-hex.def} {\from{l3str.dtx}{hex}}%
+ \file{l3str-esc-name.def} {\from{l3str.dtx}{name}}%
+ \file{l3str-esc-string.def} {\from{l3str.dtx}{string}}%
+ \file{l3str-esc-url.def} {\from{l3str.dtx}{url}}%
}
% UTF encodings.
\generate{%
- \file{l3str-utf8.def} {\from{l3str.dtx}{utf8}}%
- \file{l3str-utf16.def} {\from{l3str.dtx}{utf16}}%
- \file{l3str-utf32.def} {\from{l3str.dtx}{utf32}}%
+ \file{l3str-enc-utf8.def} {\from{l3str.dtx}{utf8}}%
+ \file{l3str-enc-utf16.def} {\from{l3str.dtx}{utf16}}%
+ \file{l3str-enc-utf32.def} {\from{l3str.dtx}{utf32}}%
}
% ISO-8859 encodings.
\generate{%
- \file{l3str-iso88591.def} {\from{l3str.dtx}{iso88591}}%
- \file{l3str-iso88592.def} {\from{l3str.dtx}{iso88592}}%
- \file{l3str-iso88593.def} {\from{l3str.dtx}{iso88593}}%
- \file{l3str-iso88594.def} {\from{l3str.dtx}{iso88594}}%
- \file{l3str-iso88595.def} {\from{l3str.dtx}{iso88595}}%
- \file{l3str-iso88596.def} {\from{l3str.dtx}{iso88596}}%
- \file{l3str-iso88597.def} {\from{l3str.dtx}{iso88597}}%
- \file{l3str-iso88598.def} {\from{l3str.dtx}{iso88598}}%
- \file{l3str-iso88599.def} {\from{l3str.dtx}{iso88599}}%
- \file{l3str-iso885910.def} {\from{l3str.dtx}{iso885910}}%
- \file{l3str-iso885911.def} {\from{l3str.dtx}{iso885911}}%
- \file{l3str-iso885913.def} {\from{l3str.dtx}{iso885913}}%
- \file{l3str-iso885914.def} {\from{l3str.dtx}{iso885914}}%
- \file{l3str-iso885915.def} {\from{l3str.dtx}{iso885915}}%
- \file{l3str-iso885916.def} {\from{l3str.dtx}{iso885916}}%
+ \file{l3str-enc-iso88591.def} {\from{l3str.dtx}{iso88591}}%
+ \file{l3str-enc-iso88592.def} {\from{l3str.dtx}{iso88592}}%
+ \file{l3str-enc-iso88593.def} {\from{l3str.dtx}{iso88593}}%
+ \file{l3str-enc-iso88594.def} {\from{l3str.dtx}{iso88594}}%
+ \file{l3str-enc-iso88595.def} {\from{l3str.dtx}{iso88595}}%
+ \file{l3str-enc-iso88596.def} {\from{l3str.dtx}{iso88596}}%
+ \file{l3str-enc-iso88597.def} {\from{l3str.dtx}{iso88597}}%
+ \file{l3str-enc-iso88598.def} {\from{l3str.dtx}{iso88598}}%
+ \file{l3str-enc-iso88599.def} {\from{l3str.dtx}{iso88599}}%
+ \file{l3str-enc-iso885910.def} {\from{l3str.dtx}{iso885910}}%
+ \file{l3str-enc-iso885911.def} {\from{l3str.dtx}{iso885911}}%
+ \file{l3str-enc-iso885913.def} {\from{l3str.dtx}{iso885913}}%
+ \file{l3str-enc-iso885914.def} {\from{l3str.dtx}{iso885914}}%
+ \file{l3str-enc-iso885915.def} {\from{l3str.dtx}{iso885915}}%
+ \file{l3str-enc-iso885916.def} {\from{l3str.dtx}{iso885916}}%
}
\endbatchfile
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
index ca9ac5399b0..f9294dbb87c 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3tl-analysis.dtx
@@ -71,96 +71,104 @@
%
% \section{\pkg{l3tl-analysis} documentation}
%
-% This module moslty provides internal functions for use
-% in the \pkg{l3regex} module. However, it provides as a
-% side-effect a user debugging function, very similar to
-% the \cs{ShowTokens} macro from the \pkg{ted} package.
+% This module mostly provides internal functions for use in the
+% \pkg{l3regex} module. However, it provides as a side-effect a user
+% debugging function, very similar to the \cs{ShowTokens} macro from the
+% \pkg{ted} package.
%
% \begin{function}{\tl_show_analysis:N, \tl_show_analysis:n}
% \begin{syntax}
% \cs{tl_show_analysis:n} \Arg{token list}
% \end{syntax}
-% Displays to the terminal the detailed decomposition of
-% the \meta{token list} into tokens,
-% showing the category code of each character token,
-% and the meaning of control sequences and active characters.
+% Displays to the terminal the detailed decomposition of the
+% \meta{token list} into tokens, showing the category code of each
+% character token, and the meaning of control sequences and active
+% characters.
% \end{function}
%
-% \subsection{Internal}
+% \subsection{Internal functions}
%
% \begin{variable}{\s_tl}
-% The format used to store token lists internally uses
-% the scan mark \cs{s_tl} as a delimiter.
+% The format used to store token lists internally uses the scan mark
+% \cs{s_tl} as a delimiter.
% \end{variable}
%
-% The task of the \pkg{l3tl-analysis} module is to convert
-% token lists to an internal format which allows us to extract
-% all the relevant information about individual tokens
-% (category code, character code), as well as reconstruct
-% the token list quickly. This internal format is used in
-% \pkg{l3regex} where we need to support arbitrary tokens,
-% and it is used in conversion functions in \pkg{l3str},
-% where we wish to support clusters of characters instead
-% of single characters.
-%
-% The internal format thus takes the form of a succession
-% of items of the form
-% \begin{quote}
-% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
-% \end{quote}
-% Here, the \meta{tokens} \texttt{o}- and \texttt{x}-expand
-% to the original token in the token list (even special
-% category codes) or to the cluster of tokens corresponding
-% to one Unicode character in the given encoding.
-% The \meta{catcode} is given as a single hexadecimal digit,
-% $0$ for control sequences. The \meta{char code} is given
-% as a decimal number, $-1$ for control sequences.
-%
-% Using delimited arguments lets us build the \meta{tokens}
-% progressively when doing an encoding conversion in \pkg{l3str}.
-% On the other hand, the delimiter \cs{s_tl} may not appear
-% unbraced in \meta{tokens}. This is not a problem because we
-% are careful to wrap control sequences in braces (as an argument
-% to \cs{exp_not:n}) when converting from a general token list
-% to the internal format.
-%
-% \begin{function}{\tl_set_analysis:Nn}
+% \begin{function}{\tl_analysis_map_inline:nn}
% \begin{syntax}
-% \cs{tl_set_analysis:Nn} \meta{tl var} \Arg{token list}
+% \cs{tl_analysis_map_inline:nn} \Arg{token list} \Arg{inline function}
% \end{syntax}
-% This function analyzes each \meta{token} in the \meta{token list},
-% converting each \meta{token} to one item in the internal format.
-% The result of the analysis is then stored in \meta{tl var}.
-% The following transformation is performed:
+% Applies the \meta{inline function} to each individual \meta{token}
+% in the \meta{token list}. The \meta{inline function} receives three
+% arguments:
% \begin{itemize}
-% \item a control sequence |\cs| becomes
-% |\exp_not:n { \cs } \s_tl 0 -1 \s_tl|;
-% \item a begin-group character |{| becomes
-% |\exp_after:wN { \if_false: } \fi: \s_tl 1| \meta{char code} \cs{s_tl};
-% \item an end-group character |}| becomes
-% |\if_false: { \fi: } \s_tl 2| \meta{char code} \cs{s_tl};
-% \item a character with any other category code becomes
-% |\exp_not:n| \Arg{character} \cs{s_tl} \meta{hex catcode}
-% \meta{char code} \cs{s_tl}.
+% \item \meta{tokens}, which both \texttt{o}-expand and
+% \texttt{x}-expand to the \meta{token}. The detailed form of
+% \meta{token} may change in later releases.
+% \item \meta{catcode}, a capital hexadecimal digit which denotes
+% the category code of the \meta{token} (0: control sequence, 1:
+% begin-group, 2: end-group, 3: math shift, 4: alignment tab, 6:
+% parameter, 7: superscript, 8: subscript, A: space, B: letter,
+% C:other, D:active).
+% \item \meta{char code}, a decimal representation of the character
+% code of the token, $-1$ if it is a control sequence (with
+% \meta{catcode} $0$).
% \end{itemize}
-% We can note that in every case the part before \cs{s_tl}
-% both \texttt{o}-expands and \texttt{x}-expands to the token
-% which is represented.
% \end{function}
%
-% \begin{function}{\tl_set_analysis_from_str:Nn}
-% \begin{syntax}
-% \cs{tl_set_analysis:Nn} \meta{tl var} \Arg{token list}
-% \end{syntax}
-% This function analyzes each \meta{token} in the \meta{token list},
-% after conversion to a string.
-% The result of the analysis is then stored in \meta{tl var}.
-% \begin{itemize}
-% \item Spaces become \verb*| |\cs{s_tl} |A32| \cs{s_tl}.
-% \item Other characters become
-% \meta{character} \cs{s_tl} |C| \meta{char code} \cs{s_tl}.
-% \end{itemize}
-% \end{function}
+% For optimizations in \pkg{l3regex} (when matching control sequences),
+% it may be useful to provide a \cs{tl_analysis_from_str_map_inline:nn}
+% function, perhaps named \cs{str_analysis_map_inline:nn}.
+%
+% \subsection{Internal format}
+%
+% The task of the \pkg{l3tl-analysis} module is to convert token lists
+% to an internal format which allows us to extract all the relevant
+% information about individual tokens (category code, character code),
+% as well as reconstruct the token list quickly. This internal format is
+% used in \pkg{l3regex} where we need to support arbitrary tokens, and
+% it is used in conversion functions in \pkg{l3str}, where we wish to
+% support clusters of characters instead of single tokens.
+%
+% We thus need a way to encode any \meta{token} (even begin-group and
+% end-group character tokens) in a way amenable to manipulating tokens
+% individually. The best we can do is to find \meta{tokens} which both
+% \texttt{o}-expand and \texttt{x}-expand to the given
+% \meta{token}. Collecting more information about the category code and
+% character code is also useful for regular expressions, since most
+% regexes are catcode-agnostic. The internal format thus takes the form
+% of a succession of items of the form
+% \begin{quote}
+% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
+% \end{quote}
+% The \meta{tokens} \texttt{o}- \emph{and} \texttt{x}-expand to the
+% original token in the token list or to the cluster of tokens
+% corresponding to one Unicode character in the given encoding (for
+% \pkg{l3str}). The \meta{catcode} is given as a single hexadecimal
+% digit, $0$ for control sequences. The \meta{char code} is given as a
+% decimal number, $-1$ for control sequences.
+%
+% Using delimited arguments lets us build the \meta{tokens}
+% progressively when doing an encoding conversion in \pkg{l3str}. On the
+% other hand, the delimiter \cs{s_tl} may not appear unbraced in
+% \meta{tokens}. This is not a problem because we are careful to wrap
+% control sequences in braces (as an argument to \cs{exp_not:n}) when
+% converting from a general token list to the internal format.
+%
+% The current rule for converting a \meta{token} to a balanced set of
+% \meta{tokens} which both \texttt{o}-expands and \texttt{x}-expands to
+% it is the following.
+% \begin{itemize}
+% \item A control sequence |\cs| becomes |\exp_not:n { \cs }|
+% \cs{s_tl} $0$ $-1$ \cs{s_tl}.
+% \item A begin-group character |{| becomes \cs{exp_after:wN} |{|
+% \cs{if_false:} |}| \cs{fi:} \cs{s_tl} $1$ \meta{char code}
+% \cs{s_tl}.
+% \item An end-group character |}| becomes \cs{if_false:} |{| \cs{fi:}
+% |}| \cs{s_tl} $2$ \meta{char code} \cs{s_tl}.
+% \item A character with any other category code becomes
+% \cs{exp_not:n} \Arg{character} \cs{s_tl} \meta{hex catcode}
+% \meta{char code} \cs{s_tl}.
+% \end{itemize}
%
% ^^A todo: ask LuaTeX list for an \ifx\undefined <active char>
% ^^A which does not add the <active char> in memory.
@@ -185,12 +193,11 @@
%
% \begin{variable}{\s_tl}
% The scan mark \cs{s_tl} is used as a delimiter in the internal
-% format. This is more practical than using a quark, because we
-% would then need to control expansion much more carefully: compare
-% \cs{int_value:w} |`#1| \cs{s_tl} with
-% \cs{int_value:w} |`#1| \cs{exp_stop_f:} \cs{exp_not:N} \cs{q_mark}
-% to extract a character code followed by the delimiter in an
-% \texttt{x}-expansion.
+% format. This is more practical than using a quark, because we would
+% then need to control expansion much more carefully: compare
+% \cs{int_value:w} |`#1| \cs{s_tl} with \cs{int_value:w} |`#1|
+% \cs{exp_stop_f:} \cs{exp_not:N} \cs{q_mark} to extract a character
+% code followed by the delimiter in an \texttt{x}-expansion.
% \begin{macrocode}
\scan_new:N \s_tl
% \end{macrocode}
@@ -198,11 +205,11 @@
%
% \begin{variable}{\l_tl_analysis_token}
% \begin{variable}{\l_tl_analysis_char_token}
-% The tokens in the token list are probed with the \TeX{}
-% primitive \tn{futurelet}. We use \cs{l_tl_analysis_token}
-% in that construction. In some cases, we convert the
-% following token to a string before probing it: then the
-% token variable used is \cs{l_tl_analysis_char_token}.
+% The tokens in the token list are probed with the \TeX{} primitive
+% \tn{futurelet}. We use \cs{l_tl_analysis_token} in that
+% construction. In some cases, we convert the following token to a
+% string before probing it: then the token variable used is
+% \cs{l_tl_analysis_char_token}.
% \begin{macrocode}
\cs_new_eq:NN \l_tl_analysis_token ?
\cs_new_eq:NN \l_tl_analysis_char_token ?
@@ -211,8 +218,8 @@
% \end{variable}
%
% \begin{variable}{\l_tl_analysis_normal_int}
-% The number of normal (\texttt{N}-type argument) tokens
-% since the last special token.
+% The number of normal (\texttt{N}-type argument) tokens since the
+% last special token.
% \begin{macrocode}
\int_new:N \l_tl_analysis_normal_int
% \end{macrocode}
@@ -229,8 +236,8 @@
%
% \begin{variable}{\l_tl_analysis_nesting_int}
% Nesting depth of explicit begin-group and end-group characters
-% during the first pass. This lets us detect the end of the token
-% list without a reserved end-marker.
+% during the first pass. This lets us detect the end of the token list
+% without a reserved end-marker.
% \begin{macrocode}
\int_new:N \l_tl_analysis_nesting_int
% \end{macrocode}
@@ -245,8 +252,8 @@
% \end{variable}
%
% \begin{variable}{\g_tl_analysis_result_tl}
-% The result of the conversion is stored in this token list,
-% with a succession of items of the form
+% The result of the conversion is stored in this token list, with a
+% succession of items of the form
% \begin{quote}
% \meta{tokens} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
% \end{quote}
@@ -258,10 +265,9 @@
% \begin{macro}[int, EXP]{\tl_analysis_extract_charcode:}
% \begin{macro}[aux, EXP]{\tl_analysis_extract_charcode_aux:w}
% Extracting the character code from the meaning of
-% \cs{l_tl_analysis_token}. This has no error checking,
-% and should only be assumed to work for begin-group
-% and end-group character tokens. It produces a number
-% in the form |`|\meta{char}.
+% \cs{l_tl_analysis_token}. This has no error checking, and should
+% only be assumed to work for begin-group and end-group character
+% tokens. It produces a number in the form |`|\meta{char}.
% \begin{macrocode}
\cs_new_nopar:Npn \tl_analysis_extract_charcode:
{
@@ -277,11 +283,10 @@
% \begin{macro}[aux, EXP]{\tl_analysis_cs_space_count:w}
% \begin{macro}[aux, EXP]{\tl_analysis_cs_space_count_end:w}
% Counts the number of spaces in the string representation of its
-% second argument, as well as the number of characters following
-% the last space in that representation, and feeds the two numbers
-% as semicolon-delimited arguments to the first argument.
-% When this function is used, the escape character is printable
-% and non-space.
+% second argument, as well as the number of characters following the
+% last space in that representation, and feeds the two numbers as
+% semicolon-delimited arguments to the first argument. When this
+% function is used, the escape character is printable and non-space.
% \begin{macrocode}
\cs_new:Npn \tl_analysis_cs_space_count:NN #1 #2
{
@@ -314,47 +319,46 @@
% \meta{token N} \cs{s_tl} \meta{catcode N} \meta{char code N} \cs{s_tl}
% \end{quote}
% Most but not all tokens can be grabbed as an undelimited
-% (\texttt{N}-type) argument by \TeX{}.
-% The plan is to have a two pass system. In the first pass,
-% locate special tokens, and store them in various \tn{toks} registers.
-% In the second pass, which is done within an \texttt{x}-expanding
-% assignment, normal tokens are taken in as \texttt{N}-type
-% arguments, and special tokens are retrieved from the \tn{toks}
-% registers, and removed from the input stream by some means.
-%
-% To ease the difficult first pass, we first do some setup
-% with \cs{tl_analysis_setup:n}.
-% Active characters set equal to non-active characters
-% cause trouble, so we disable all active characters by
-% setting them equal to \texttt{undefined} locally.
-% We also set there the escape character to be printable
-% (backslash, but this later oscillates between solidus and backslash):
-% this makes it possible to distinguish characters from
-% control sequences.
-%
-% A token has two characteristics: its \tn{meaning},
-% and what it looks like for \TeX{} when it is in
-% scanning mode (\emph{e.g.}, when capturing parameters
-% for a macro).
-% For our purposes, the token's meaning can be
+% (\texttt{N}-type) argument by \TeX{}. The plan is to have a two pass
+% system. In the first pass, locate special tokens, and store them in
+% various \tn{toks} registers. In the second pass, which is done within
+% an \texttt{x}-expanding assignment, normal tokens are taken in as
+% \texttt{N}-type arguments, and special tokens are retrieved from the
+% \tn{toks} registers, and removed from the input stream by some means.
+% The whole process takes linear time, because we avoid building the
+% result one item at a time.
+%
+% To ease the difficult first pass, we first do some setup with
+% \cs{tl_analysis_setup:n}. Active characters set equal to non-active
+% characters cause trouble, so we disable all active characters by
+% setting them equal to \texttt{undefined} locally. We also set there
+% the escape character to be printable (backslash, but this later
+% oscillates between slash and backslash): this makes it possible to
+% distinguish characters from control sequences.
+%
+% A token has two characteristics: its \tn{meaning}, and what it looks
+% like for \TeX{} when it is in scanning mode (\emph{e.g.}, when
+% capturing parameters for a macro). For our purposes, we distinguish
+% the following meanings:
% \begin{itemize}
-% \item a begin-group token (category code $1$),
-% either space (character code $32$), or non-space;
-% \item an end-group token (category code $2$),
-% either space (character code $32$), or non-space;
-% \item a space token (category code $10$, character code $32$);
-% \item anything else (then the token is always an \texttt{N}-type argument).
+% \item begin-group token (category code $1$), either space (character
+% code $32$), or non-space;
+% \item end-group token (category code $2$), either space (character
+% code $32$), or non-space;
+% \item space token (category code $10$, character code $32$);
+% \item anything else (then the token is always an \texttt{N}-type
+% argument).
% \end{itemize}
% The token itself can \enquote{look like} one of the following
% \begin{itemize}
-% \item a non-active character, in which case its meaning
-% is automatically that associated to its character code
-% and category code, we call it \enquote{true} character;
-% \item an active character (we eliminate those in the setup step);
-% \item a control sequence.
+% \item a non-active character, in which case its meaning is
+% automatically that associated to its character code and category
+% code, we call it \enquote{true} character;
+% \item an active character (we eliminate those in the setup step);
+% \item a control sequence.
% \end{itemize}
-% The only tokens which are not valid \texttt{N}-type arguments are
-% true begin-group characters, true end-group characters, and true spaces.
+% The only tokens which are not valid \texttt{N}-type arguments are true
+% begin-group characters, true end-group characters, and true spaces.
% We will detect those characters by scanning ahead with \tn{futurelet},
% then distinguishing true characters from control sequences set equal
% to them using the \tn{string} representation.
@@ -362,9 +366,9 @@
% The second pass is a simple exercise in expandable loops.
%
% \begin{macro}[int]{\tl_analysis:n}
-% Everything is done within a group, and all definitions will be local.
-% We use \cs{group_align_safe_begin/end:} to avoid problems in case
-% \cs{tl_analysis:n} is used within an alignment and its argument
+% Everything is done within a group, and all definitions will be
+% local. We use \cs{group_align_safe_begin/end:} to avoid problems in
+% case \cs{tl_analysis:n} is used within an alignment and its argument
% contains alignment tab tokens.
% \begin{macrocode}
\cs_new_protected:Npn \tl_analysis:n #1
@@ -380,54 +384,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\tl_set_analysis:Nn}
-% This function is used in \pkg{l3regex}. Simply copy the result
-% of \cs{tl_analysis:n} into the given token list.
-% \begin{macrocode}
-\cs_new_protected:Npn \tl_set_analysis:Nn #1#2
- {
- \tl_analysis:n {#2}
- \tl_set_eq:NN #1 \g_tl_analysis_result_tl
- }
-% \end{macrocode}
-% \end{macro}
-%
-% \begin{macro}[int]{\tl_set_analysis_from_str:Nn}
-% \begin{macro}[aux, EXP]{\tl_analysis_from_str_loop:N}
-% This function, used in \pkg{l3str} could be emulated by passing the
-% token list through \cs{tl_to_str:n} and using
-% \cs{tl_set_analysis:Nn}, but this would be terribly inefficient.
-% Instead, convert to a string with spaces of category code other,
-% and for each character output one item of the analysis:
-% \meta{character} \cs{s_tl} \meta{catcode} \meta{char code} \cs{s_tl}
-% where the \meta{catcode} is either |A| (for spaces) or |C|
-% (for other characters).
-% \begin{macrocode}
-\cs_new_protected:Npn \tl_set_analysis_from_str:Nn #1#2
- {
- \str_gset_other:Nn \g_tl_analysis_result_tl {#2}
- \tl_set:Nx #1
- {
- \exp_after:wN \tl_analysis_from_str_loop:N
- \g_tl_analysis_result_tl \prg_map_break:
- \prg_break_point:n { }
- }
- }
-\group_begin:
- \char_set_catcode_other:N A
- \char_set_catcode_other:N C
- \cs_new_protected:Npn \tl_analysis_from_str_loop:N #1
- {
- #1 \s_tl
- \if_charcode:w #1 ~ A \else: C \fi:
- \int_value:w `#1 \s_tl
- \tl_analysis_from_str_loop:N
- }
-\group_end:
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
% \subsection{Setup}
%
% \begin{macro}[int]{\tl_analysis_setup:n}
@@ -939,6 +895,45 @@
% \end{macro}
% \end{macro}
%
+% \subsection{Mapping through the analysis}
+%
+% \begin{macro}[int]{\tl_analysis_map_inline:nn}
+% \begin{macro}[aux]{\tl_analysis_map_inline_aux:Nn}
+% First obtain the analysis of the token list into
+% \cs{g_tl_analysis_result_tl}. To allow nested mappings, increase the
+% nesting depth \cs{g_prg_map_int} (shared between all modules), then
+% define the looping macro, which has a name specific to that nesting
+% depth. That looping grabs the \meta{tokens}, \meta{catcode} and
+% \meta{char code}; it checks for the end of the loop with
+% \cs{use_none:n} |##2|, normally empty, but which becomes
+% \cs{prg_map_break:} at the end; it then performs the user's code
+% |#2|, and loops by calling itself. When the loop ends, remember to
+% decrease the nesting depth.
+% \begin{macrocode}
+\cs_new_protected:Npn \tl_analysis_map_inline:nn #1
+ {
+ \tl_analysis:n {#1}
+ \int_gincr:N \g_prg_map_int
+ \exp_args:Nc \tl_analysis_map_inline_aux:Nn
+ { tl_analysis_map_inline_ \int_use:N \g_prg_map_int :wNw }
+ }
+\cs_new_protected:Npn \tl_analysis_map_inline_aux:Nn #1#2
+ {
+ \cs_gset_protected:Npn #1 ##1 \s_tl ##2 ##3 \s_tl
+ {
+ \use_none:n ##2
+ #2
+ #1
+ }
+ \exp_after:wN #1
+ \g_tl_analysis_result_tl
+ \s_tl { ? \prg_map_break: } \s_tl
+ \prg_break_point:n { \int_gdecr:N \g_prg_map_int }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
% \subsection{Showing the results}
%
% \begin{macro}{\tl_show_analysis:N}
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex
deleted file mode 100644
index 1ede1770c08..00000000000
--- a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-test.tex
+++ /dev/null
@@ -1,380 +0,0 @@
-
-\documentclass{article}
-
-\usepackage[T1]{fontenc}
-\usepackage{times,multicol}
-\usepackage{xcoffins}
-
-\usepackage{color}
-\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}}
-
-\newcommand\hrulebox [2]{\setbox#1\hbox to#2{{\scriptsize\itshape
- \color{blue}\hrulefill #2\hrulefill}}}
-\newcommand\vrulebox [2]{\setbox#1\vbox to#2{{\hsize 1pt\centering\scriptsize
- \itshape\color{blue}\leaders\vrule\vfill
- \hbox to0pt{\hss #2\hss}\leaders\vrule\vfill\par}}}
-
-
-\addtolength\textwidth{10pt}
-
-\showboxdepth 9999
-\showboxbreadth 9999
-\tracingonline 1
-
-
-\scrollmode
-
-\newbox\zzz
-
-
-\begin{document}
-
-
-\title{Test file for coffins}
-\author{FMi}
-\maketitle
-
-First we add a few test coffins:
-\begin{verbatim}
- \newcoffin \aaa
- \newcoffin \bbb
- \newcoffin \ccc
- \newcoffin \ddd
-\end{verbatim}
- \newcoffin \aaa
- \newcoffin \bbb
- \newcoffin \ccc
- \newcoffin \ddd
-and some boxes not set up as ordinary boxes (without extra poles):
-\begin{verbatim}
- \newbox \xxx
- \newbox \yyy
-\end{verbatim}
- \newbox \xxx
- \newbox \yyy
-
-
-If a coffin receives data a set of ``natural'' default poles are automatically added.
-\begin{verbatim}
-\sbox\aaa{\fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
-\showcoffindata \aaa
-\end{verbatim}
-\sbox\aaa{\fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
-\showcoffindata \aaa
-
-
-In contrast, boxes not declared as coffins have no poles defined. However, if
-used as coffins at least the natural default poles can be used without
-further problems, i.e.,they are changed to coffins automatically on use.
-\begin{verbatim}
-\sbox\xxx{\fbox{Some very looooonggg Caption Text}}
-\showpoles \xxx
-\end{verbatim}
-\sbox\xxx{\fbox{Some very looooonggg Caption Text}}
-\showcoffindata \xxx
-
-
-Using \verb|\setvcoffin| instead of \verb|\sbox| gives us the codes with
-\texttt{T} and \texttt{B}. Similar to \verb|\parbox| this command requires to
-specify a target width of the box:
-\begin{verbatim}
-\setvcoffin \aaa {1.5cm} {\centering
- \fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
-\showcoffindata \aaa
-\end{verbatim}
-\setvcoffin \aaa {1.5cm} {\centering
- \fbox{\begin{tabular}[b]{l}123\\4\\5\end{tabular}}}
-\showcoffindata \aaa
-
-There are a dozen natural poles per box (some with multiple names); you can
-move each of the poles to someplace else (either to an absolute position in
-``box space'' or relative to its previous position). The coordinates can
-refer to box dimensions (using \verb|\height|, \verb|\depth|,
- and \verb|width|). First we move some pole and add one
-additional one:
-\begin{verbatim}
-\setcoffinpole \aaa [h]{t}(\height -3pt)
-\setcoffinpole \aaa [v]{l}(0pt)
-\setcoffinpole \aaa [h]{mybottom}(\depth +2pt)
-\showcoffindata \aaa
-\end{verbatim}
-\setcoffinpole \aaa [h]{t}(\height -3pt)
-\setcoffinpole \aaa [v]{l}(0pt)
-\setcoffinpole \aaa [h]{mybottom}(\depth +2pt)
-\showcoffindata \aaa
-
-
-More interesting in many cases is the relative move of poles. For this the
-poles better exist or one gets an error and \texttt{0pt,0pt} is used:
-\begin{verbatim}
-\adjustcoffinpole \aaa [h]{H}(1pt)
-\adjustcoffinpole \aaa [v]{l}(1pt)
-\adjustcoffinpole \aaa [h]{undefined}(1pt)
-\showcoffindata \aaa
-\end{verbatim}
-\adjustcoffinpole \aaa [h]{H}(1pt)
-\adjustcoffinpole \aaa [v]{l}(1pt)
-\adjustcoffinpole \aaa [h]{undefined}(1pt)
-\showcoffindata \aaa
-
-
-
-\sbox \aaa {\fontsize{14.4}{5.5pc minus .5pc}\sffamily C\,H\,A\,P\,T\,E\,R
- \fontsize{36}{40pt}\selectfont 2}
-
-\setvcoffin \bbb {11cm}
- {\raggedleft\normalfont\fontsize{36}{38pt}\bfseries
- The Structure of a \LaTeX{} Document}
-
-\setvcoffin \ccc {13cm}
- {Some sample blind text to produce paragraph data after a heading.
- Some sample blind text to produce paragraph data after a heading.
- Some sample blind text to produce paragraph data after a heading.
- Some sample blind text to produce paragraph data after a heading.
- \endgraf
- Some more sample blind text to produce paragraph data after a heading.
- Some more sample blind text to produce paragraph data after a heading.
-}
-
-
-Suppose you have the following blocks of text:
-
-\bigskip
-
-\noindent\cbox{\usebox\aaa}
-
-\begin{multicols}{2}
-\ttfamily\tiny
-\ExplSyntaxOn
- \noindent \coffin_print_pole_values:Nn \aaa \\
-\ExplSyntaxOff
-\end{multicols}
-
-\medskip
-
-\noindent\cbox{\usebox\bbb}
-
-\medskip
-
-\noindent\cbox{\usebox\ccc}
-
-\bigskip
-
-produced by:
-\begin{verbatim}
-\sbox \aaa {\fontsize{14.4}{5.5pc minus .5pc}\sffamily
- C\,H\,A\,P\,T\,E\,R
- \fontsize{36}{40pt}\selectfont 2}
-\setvcoffin \bbb {11cm}
- {\raggedleft\normalfont\fontsize{36}{38pt}\bfseries
- The Structure of a \LaTeX{} Document}
-\setvcoffin \ccc {13cm}
- {Some sample blind text to produce paragraph
- data after a heading. ... }
-\end{verbatim}
-
-How do you turn this into a heading of TLC2?
-
-
-\newbox\RBi
-\newbox\RBii
-\newbox\RBiii
-
-\hrulebox\RBi{62pt}
-\vrulebox\RBii{90pt}
-\vrulebox\RBiii{60pt}
-
-\aligncoffins \bbb [T,r] \aaa [H,r](-62pt,60pt)
-\aligncoffins \ccc[T,r] \bbb[B,r](62pt,90pt)
-
- \aligncoffins * \ccc[\bbb-T,\bbb-hc] \RBiii[b,hc](28pt,0pt)
- \aligncoffins * \ccc[\bbb-b,\bbb-r] \RBi[H,r](0pt,-5pt)
- \aligncoffins * \ccc[\bbb-B,\bbb-hc] \RBii[t,r](14pt,0pt)
-
-\newpage
-
-How do you best define/describe the following design?
-
-\medskip
-
-\noindent\cbox{\usebox\ccc}
-
-
-
-
-
-
-\newpage \pagestyle{empty}
-
-\newcoffin \eee
-\newcoffin \fff
-\newcoffin \ggg
-
-\sbox \aaa {\small\scshape les vases communicants}
-\sbox \bbb {\scshape comunicating}
-\sbox \ccc {\fontsize{70pt}{60pt} \bfseries Ve\S els}
-\sbox \ddd {\scshape andr\'e breton}
-\setvcoffin \eee {4.7cm}{\noindent Translated by Mary Ann Caws \&\break
- Geoffrey T.\,Harris, with notes \&\break
- introduction by Mary Ann Caws\parfillskip=0pt\relax
- }
-\sbox \fff {University of Nebraska Press: Lincoln \& London}
-
-\setvcoffin \ggg {100mm}{\noindent\fbox{\parbox{97mm}{\leavevmode\vspace*{228mm}}}}
-
-
-\aligncoffins \ccc [H,r] \ddd [H,r](0pt,-12pt)
-\aligncoffins \ccc [t,l] \bbb (0pt,6pt)
-\aligncoffins \ccc [H,r] \aaa [H,r](0pt,138pt)
-\aligncoffins \eee [T,l] \ccc (0pt,22pc)
-\aligncoffins \fff \eee [B,l](0pt,4pc)
-\aligncoffins \ggg [b,l] \fff [B,l](54pt,11pc)
-
-
-\vspace*{-2cm}
-\noindent\cbox{\usebox\ggg}
-
-
-\newpage
-
-\section{Rotation}
-
-\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\cccccccccccccc\\d
- \\e\\e\\e\\e\\\hline\end{tabular}}
-\sbox \bbb {\fbox{A sample Text}}
-
-% just for the sake of it:
-\adjustcoffinpole \aaa {T}(24pt)
-\displaycoffinpoles \aaa {black}
-x\cbox{\usebox \aaa }x
-%
-\rotatecoffin \aaa {45}
-\displaycoffinhandle \aaa {b}{hc} {blue}
-\displaycoffinhandle \aaa {b}{l} {blue}
-\displaycoffinhandle \aaa {t}{r} {blue}
-\displaycoffinhandle \aaa {vc}{r} {blue}
-\displaycoffinhandle \aaa {vc}{l} {blue}
-\quad
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {45}
-\displaycoffinhandle \aaa {b}{r} {red}
-\displaycoffinhandle \aaa {vc}{l} {red}
-\quad
-x\cbox{\usebox \aaa }x
-
-\vspace{1cm}
-
-\rotatecoffin \aaa {45}
-\displaycoffinhandle \aaa {b}{l} {yellow}
-%
-x\cbox{\usebox \aaa }x
-%
-
-\vspace{1cm}
-
-x\cbox{\usebox \bbb }x
-%
-\rotatecoffin \bbb {45}
-\displaycoffinhandle \bbb {B}{hc} {yellow}
-\quad
-x\cbox{\usebox \bbb }x
-
-
-\section{Rotation + alignment}
-
-\aligncoffins \aaa [b,l] \bbb[B,hc](30pt,0pt)
-x\cbox{\usebox \aaa }x
-
-
-After we have aligned a roated box with some other box we need to decide about
-the bounding box of the new box. Right now this become the enclosing box and
-we do not maintain information about the inner boxes. So when we rotate that
-new box there seems to be unnecessary space in the enclosing bounding box,
-even though that is correct if you think of the aligned box being unrotated.
-
-There is at least the possibility to refer to the handles of the inner boxes
-as one can see by two of the green handles
-
-\medskip
-
-\rotatecoffin \aaa {45}
-\displaycoffinhandle \aaa {vc}{hc} {green}
-\displaycoffinhandle \aaa {H}{l} {green}
-\displaycoffinhandle \aaa {\aaa-T}{\aaa-r} {green}
-\displaycoffinhandle \aaa {\bbb-H}{\bbb-r} {green}
-%\quad
-x\cbox{\usebox \aaa }x
-
-We could do better, if we want to, by actually checking for max and
-min of all inner bounding box corners and construct the result BB box from
-that---but is it worth it?
-
-Perhaps it is! After all, the current implementation shows different results
-depending on when you align boxes and when you rotate, e.g., aligning first
-gives totally different bounding box results.
-
-\newpage
-
-Aligning first and then 135 + 45 rotation gives this:
-
-\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\cccccccccccccc\\d
- \\e\\e\\e\\e\\\hline\end{tabular}}
-\sbox \bbb {\fbox{A sample Text}}
-\rotatecoffin \bbb {90}
-
-\aligncoffins \aaa [b,l] \bbb[B,hc](-30pt,0pt)
-x\cbox{\usebox \aaa }x
-\qquad
-\rotatecoffin \aaa {135}
-x\cbox{\usebox \aaa }x
-\qquad
-\rotatecoffin \aaa {45}
-x\cbox{\usebox \aaa }x
-
-
-
-
-\newpage
-
-\section{Rotation by small amounts}
-
-\sbox \aaa {\tabcolsep0pt\begin{tabular}[c]{|c|}\hline a\\b\\ccccccccc\\d
- \\e\\e\\e\\\hline\end{tabular}}
-
-\subsection{30 + 30 +30 }
-\rotatecoffin \aaa {30}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {30}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {30}
-x\cbox{\usebox \aaa }x
-
-\subsection{6 * 10 + 45 + 45 + 30}
-
-\rotatecoffin \aaa {10}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {10}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {10}
-x\cbox{\usebox \aaa }x
-
-\vspace{1cm}
-\rotatecoffin \aaa {10}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {10}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {10}
-x\cbox{\usebox \aaa }x
-
-\vspace{1cm}
-\rotatecoffin \aaa {45}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {45}
-x\cbox{\usebox \aaa }x
-\rotatecoffin \aaa {30}
-x\cbox{\usebox \aaa }x
-
-
-\end{document}
-
-
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex
deleted file mode 100644
index e83c0b490b6..00000000000
--- a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example-keyval.tex
+++ /dev/null
@@ -1,281 +0,0 @@
-
-\documentclass{article}
-
-\usepackage[a4paper,margin=5pt]{geometry}
-
-\usepackage[T1]{fontenc}
-\usepackage{times,multicol,graphicx}
-\usepackage{xcoffins}
-
-\usepackage{times,color}
-\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}}
-
-\ExplSyntaxOn
-
-\ExplSyntaxOff
-
-
-
-%\scrollmode
-
-\pagestyle{empty}
-
-\begin{document}
-
- \NewCoffin \result
- \NewCoffin \aaa
- \NewCoffin \bbb
- \NewCoffin \ccc
- \NewCoffin \ddd
- \NewCoffin \eee
- \NewCoffin \fff
- \NewCoffin \rulei
- \NewCoffin \ruleii
- \NewCoffin \ruleiii
-
-\SetHorizontalCoffin \result {}
-\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
-\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
-\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
- \quad zeitschrift des bildungsverbandes der
- deutschen buchdrucker leipzig
- \textbullet{} oktoberheft 1925}
-\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
-\SetVerticalCoffin \eee {180pt}
- {\raggedleft\fontsize{31}{36}\sffamily\bfseries
- elementare\\
- typographie}
-\SetVerticalCoffin \fff {140pt}
- {\raggedright \fontsize{13}{14}\sffamily\bfseries
- natan altman \\
- otto baumberger \\
- herbert mayer \\
- max burchartz \\
- el lissitzky \\
- ladislaus moholy-nagy \\
- moln\'ar f.~farkas \\
- johannes molzahn \\
- kurt schwitters \\
- mart stam \\
- ivan tschichold}
-
-\RotateCoffin \bbb {90}
-\RotateCoffin \ccc {270}
-
-\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
-\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
-\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
-
-
-\JoinCoffins \result \aaa
-\JoinCoffins [
- coffin1-hpole = \aaa-t ,
- coffin1-vpole = \aaa-r ,
- coffin2-hpole = b ,
- coffin2-vpole = r ,
- voffset = 2 mm ,
-] \result \rulei
-\JoinCoffins [
- coffin1-hpole = \aaa-b ,
- coffin1-vpole = \aaa-l ,
- coffin2-hpole = B ,
- coffin2-vpole = r ,
- hoffset = 2 pt ,
-] \result \bbb
-\JoinCoffins [
- coffin1-hpole = \bbb-t ,
- coffin1-vpole = \bbb-r ,
- coffin2-hpole = t ,
- coffin2-vpole = r ,
- hoffset = -2 mm ,
-] \result \ruleii
-\JoinCoffins [
- coffin1-hpole = \aaa-B ,
- coffin1-vpole = \aaa-r ,
- coffin2-hpole = B ,
- coffin2-vpole = l ,
- hoffset = 66 pt ,
- voffset = 14 pc ,
-] \result \ccc
-\JoinCoffins [
- coffin1-hpole = \bbb-l ,
- coffin1-vpole = \ccc-B ,
- coffin2-hpole = t ,
- coffin2-vpole = r ,
- hoffset = -2 mm ,
-] \result \fff
-\JoinCoffins [
- coffin1-hpole = \fff-b ,
- coffin1-vpole = \fff-r ,
- coffin2-hpole = b ,
- coffin2-vpole = l ,
- hoffset = 2 mm ,
-] \result \ruleiii
-\JoinCoffins [
- coffin1-hpole = \ccc-r ,
- coffin1-vpole = \fff-l ,
- coffin2-hpole = B ,
- coffin2-vpole = r ,
-] \result \eee
-\JoinCoffins [
- coffin1-hpole = \eee-T ,
- coffin1-vpole = \eee-r ,
- coffin2-hpole = B ,
- coffin2-vpole = r ,
- voffset = 4 pc ,
-] \result \ddd
-
-
-\vspace*{3cm}
-\begin{center}
- {\Large Title page of ``elementare typographie'' by Ivan Tschichold\par}
-
-\large
-\vspace*{1cm}
-
- 1. first the scanned original from 1925
-
-\vspace*{6mm}
-
- 2. then the recreated \TeX{} version from 2010 using coffins---not
- attempting\\ to match the fonts and size but the structure
-
-\vspace*{6mm}
-
- 3. and finally the source code used.
-
- This document uses the new
- implementation by Joseph in \texttt{l3coffins-new}.
-
-\end{center}
-
-\newpage
-
-
-\begin{center}
- \null
- \fbox{\includegraphics[scale=.95]{elementare-typographie-title.jpg}}
-\end{center}
-
-\newpage
-
-\TypesetCoffin \result
-
-\newpage
-
-\vspace*{3cm}
-\begin{center}
- {\Large Code used: \par}
-\vspace*{1cm}
-
-
-\begin{minipage}{14cm}
-\begin{verbatim}
-\SetHorizontalCoffin\result{}
-\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
-\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
-\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
- \quad zeitschrift des bildungsverbandes der
- deutschen buchdrucker leipzig
- \textbullet{} oktoberheft 1925}
-\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
-\SetVerticalCoffin \eee {180pt}
- {\raggedleft\fontsize{31}{36}\sffamily\bfseries
- elementare\\
- typographie}
-\SetVerticalCoffin \fff {140pt}
- {\raggedright \fontsize{13}{14}\sffamily\bfseries
- natan altman \\
- otto baumberger \\
- herbert mayer \\
- max burchartz \\
- el lissitzky \\
- ladislaus moholy-nagy \\
- moln\'ar f.~farkas \\
- jahannes molzahn \\
- kurt schwitters \\
- mart stam \\
- ivan tschichold}
-
-\RotateCoffin \bbb {90}
-\RotateCoffin \ccc {270}
-
-\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
-\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
-\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
-
-\JoinCoffins \result \aaa
-\JoinCoffins [
- coffin1-hpole = \aaa-t ,
- coffin1-vpole = \aaa-r ,
- coffin2-hpole = b ,
- coffin2-vpole = r ,
- voffset = 2 mm ,
-] \result \rulei
-\JoinCoffins [
- coffin1-hpole = \aaa-b ,
- coffin1-vpole = \aaa-l ,
- coffin2-hpole = B ,
- coffin2-vpole = r ,
- hoffset = 2 pt ,
-] \result \bbb
-\JoinCoffins [
- coffin1-hpole = \bbb-t ,
- coffin1-vpole = \bbb-r ,
- coffin2-hpole = t ,
- coffin2-vpole = r ,
- hoffset = -2 mm ,
-] \result \ruleii
-\JoinCoffins [
- coffin1-hpole = \aaa-B ,
- coffin1-vpole = \aaa-r ,
- coffin2-hpole = B ,
- coffin2-vpole = l ,
- hoffset = 66 pt ,
- voffset = 14 pc ,
-] \result \ccc
-\JoinCoffins [
- coffin1-hpole = \bbb-l ,
- coffin1-vpole = \ccc-B ,
- coffin2-hpole = t ,
- coffin2-vpole = r ,
- hoffset = -2 mm ,
-] \result \fff
-\JoinCoffins [
- coffin1-hpole = \fff-b ,
- coffin1-vpole = \fff-r ,
- coffin2-hpole = b ,
- coffin2-vpole = l ,
- hoffset = 2 mm ,
-] \result \ruleiii
-\JoinCoffins [
- coffin1-hpole = \ccc-r ,
- coffin1-vpole = \fff-l ,
- coffin2-hpole = B ,
- coffin2-vpole = r ,
-] \result \eee
-\JoinCoffins [
- coffin1-hpole = \eee-T ,
- coffin1-vpole = \eee-r ,
- coffin2-hpole = B ,
- coffin2-vpole = r ,
- voffset = 4 pc ,
-] \result \ddd
-\SetHorizontalCoffin\result{}
-\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries
- mitteilungen}
-\SetHorizontalCoffin \bbb {\fontsize{52}{50}\rotatebox{90}{\sffamily\bfseries
- typographische}}
-
-\TypesetCoffin \result
-\end{verbatim}
-
-This is not necessarily the final syntax but for now it does its job. For
-example, flexible support for adding ornaments (lines, \ldots) is still
-missing, so above the rules got added as predefined individual coffins.
-
-\end{minipage}
-\end{center}
-
-\end{document}
-
diff --git a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex b/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex
deleted file mode 100644
index 343c9d7e29f..00000000000
--- a/Master/texmf-dist/source/latex/l3experimental/xcoffins/xcoffins-tschichold-example.tex
+++ /dev/null
@@ -1,185 +0,0 @@
-
-\documentclass{article}
-
-\usepackage[a4paper,margin=5pt]{geometry}
-
-\usepackage[T1]{fontenc}
-\usepackage{times,multicol,graphicx}
-\usepackage{xcoffins}
-
-\usepackage{times,color}
-\newcommand\cbox[2][.8]{{\setlength\fboxsep{0pt}\colorbox[gray]{#1}{#2}}}
-
-\ExplSyntaxOn
-
-\ExplSyntaxOff
-
-
-
-%\scrollmode
-
-\pagestyle{empty}
-
-\begin{document}
-
- \NewCoffin \result
- \NewCoffin \aaa
- \NewCoffin \bbb
- \NewCoffin \ccc
- \NewCoffin \ddd
- \NewCoffin \eee
- \NewCoffin \fff
- \NewCoffin \rulei
- \NewCoffin \ruleii
- \NewCoffin \ruleiii
-
-\SetHorizontalCoffin \result {}
-\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
-\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
-\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
- \quad zeitschrift des bildungsverbandes der
- deutschen buchdrucker leipzig
- \textbullet{} oktoberheft 1925}
-\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
-\SetVerticalCoffin \eee {180pt}
- {\raggedleft\fontsize{31}{36}\sffamily\bfseries
- elementare\\
- typographie}
-\SetVerticalCoffin \fff {140pt}
- {\raggedright \fontsize{13}{14}\sffamily\bfseries
- natan altman \\
- otto baumberger \\
- herbert mayer \\
- max burchartz \\
- el lissitzky \\
- ladislaus moholy-nagy \\
- moln\'ar f.~farkas \\
- johannes molzahn \\
- kurt schwitters \\
- mart stam \\
- ivan tschichold}
-
-\RotateCoffin \bbb {90}
-\RotateCoffin \ccc {270}
-
-\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
-\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
-\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
-
-
-\JoinCoffins \result \aaa
-\JoinCoffins \result[\aaa-t,\aaa-r] \rulei [b,r](0pt,2mm)
-\JoinCoffins \result[\aaa-b,\aaa-l] \bbb [B,r](2pt,0pt)
-\JoinCoffins \result[\bbb-t,\bbb-r] \ruleii [t,r](-2mm,0pt)
-\JoinCoffins \result[\aaa-B,\aaa-r] \ccc [B,l](66pt,14pc)
-\JoinCoffins \result[\bbb-l,\ccc-B] \fff [t,r](-2mm,0pt)
-\JoinCoffins \result[\fff-b,\fff-r] \ruleiii [b,l](2mm,0pt)
-\JoinCoffins \result[\ccc-r,\fff-l] \eee [B,r]
-\JoinCoffins \result[\eee-T,\eee-r] \ddd [B,r](0pt,4pc)
-
-
-\vspace*{3cm}
-\begin{center}
- {\Large Title page of ``elementare typographie'' by Ivan Tschichold\par}
-
-\large
-\vspace*{1cm}
-
- 1. first the scanned original from 1925
-
-\vspace*{6mm}
-
- 2. then the recreated \TeX{} version from 2010 using coffins---not
- attempting\\ to match the fonts and size but the structure
-
-\vspace*{6mm}
-
- 3. and finally the source code used.
-
- This document uses the new
- implementation by Joseph in \texttt{l3coffins-new}.
-
-\end{center}
-
-\newpage
-
-
-\begin{center}
- \null
- \fbox{\includegraphics[scale=.95]{elementare-typographie-title.jpg}}
-\end{center}
-
-\newpage
-
-\TypesetCoffin \result
-
-\newpage
-
-\vspace*{3cm}
-\begin{center}
- {\Large Code used: \par}
-\vspace*{1cm}
-
-
-\begin{minipage}{14cm}
-\begin{verbatim}
-\SetHorizontalCoffin\result{}
-\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries mitteilungen}
-\SetHorizontalCoffin \bbb {\fontsize{52}{50}\sffamily\bfseries typographische}
-\SetHorizontalCoffin \ccc {\fontsize{12}{10}\sffamily
- \quad zeitschrift des bildungsverbandes der
- deutschen buchdrucker leipzig
- \textbullet{} oktoberheft 1925}
-\SetHorizontalCoffin \ddd {\fontsize{28}{20}\sffamily sonderheft}
-\SetVerticalCoffin \eee {180pt}
- {\raggedleft\fontsize{31}{36}\sffamily\bfseries
- elementare\\
- typographie}
-\SetVerticalCoffin \fff {140pt}
- {\raggedright \fontsize{13}{14}\sffamily\bfseries
- natan altman \\
- otto baumberger \\
- herbert mayer \\
- max burchartz \\
- el lissitzky \\
- ladislaus moholy-nagy \\
- moln\'ar f.~farkas \\
- jahannes molzahn \\
- kurt schwitters \\
- mart stam \\
- ivan tschichold}
-
-\RotateCoffin \bbb {90}
-\RotateCoffin \ccc {270}
-
-\SetHorizontalCoffin \rulei {\color{red}\rule{6.5in}{1pc}}
-\SetHorizontalCoffin \ruleii {\color{red}\rule{1pc}{23.5cm}}
-\SetHorizontalCoffin \ruleiii{\color{black}\rule{10pt}{152pt}}
-
-\JoinCoffins \result \aaa
-\JoinCoffins \result[\aaa-t,\aaa-r] \rulei [b,r](0pt,2mm)
-\JoinCoffins \result[\aaa-b,\aaa-l] \bbb [B,r](2pt,0pt)
-\JoinCoffins \result[\bbb-t,\bbb-r] \ruleii [t,r](-2mm,0pt)
-\JoinCoffins \result[\aaa-B,\aaa-r] \ccc [B,l](66pt,14pc)
-\JoinCoffins \result[\bbb-l,\ccc-B] \fff [t,r](-2mm,0pt)
-\JoinCoffins \result[\fff-b,\fff-r] \ruleiii [b,l](2mm,0pt)
-\JoinCoffins \result[\ccc-r,\fff-l] \eee [B,r]
-\JoinCoffins \result[\eee-T,\eee-r] \ddd [B,r](0pt,4pc)
-\SetHorizontalCoffin\result{}
-\SetHorizontalCoffin \aaa {\fontsize{52}{50}\sffamily\bfseries
- mitteilungen}
-\SetHorizontalCoffin \bbb {\fontsize{52}{50}\rotatebox{90}{\sffamily\bfseries
- typographische}}
-
-\TypesetCoffin \result
-\end{verbatim}
-
-This is not necessarily the final syntax but for now it does its job. For
-example, flexible support for adding ornaments (lines, \ldots) is still
-missing, so above the rules got added as predefined individual coffins.
-
-\end{minipage}
-\end{center}
-
-\end{document}
-
diff --git a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
index dc6279914af..cb34e8a92aa 100644
--- a/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/xgalley/l3galley.dtx
@@ -1,7 +1,7 @@
% \iffalse
%
%% File: l3galley.dtx Copyright (C) 1999-2001, 2004-2009 Frank Mittelbach
-%% (C) 2010,2011 The LaTeX3 Project
+%% (C) 2010-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -36,8 +36,8 @@
%%
%
%<*driver|package>
-\RequirePackage{l3names}
-\GetIdInfo$Id: l3galley.dtx 3205 2012-01-18 22:09:05Z joseph $
+\RequirePackage{xparse}
+\GetIdInfo$Id: l3galley.dtx 3253 2012-01-28 14:59:55Z joseph $
{L3 Experimental galley code}
%</driver|package>
%<*driver>
@@ -410,6 +410,16 @@
% relational symbol.
% \end{variable}
%
+% \begin{function}{\galley_break_line:Nn}
+% \begin{syntax}
+% \cs{galley_break_line:Nn} \meta{boolean} \Arg{dim expr}
+% \end{syntax}
+% Breaks the current line, filling the remaining space with \texttt{fil}
+% glue. If the \meta{boolean} is \texttt{true} then a page break is possible
+% after the broken line. Vertical space as given by the \meta{dim expr} will
+% be inserted between the broken line and the next line.
+% \end{function}
+%
% \subsection{Paragraph breaking}
%
% \begin{variable}{\l_galley_parbreak_badness_int}
@@ -1180,7 +1190,7 @@
% token list, before setting up the value to the special value needed
% in this one case.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_display_penalty:N #1
+\cs_new_protected:Npn \galley_display_penalty:N #1
{
\tl_if_empty:NF #1
{
@@ -1192,7 +1202,7 @@
\int_gset:Nn \g_galley_penalty_int {#1}
}
}
-\cs_new_protected_nopar:Npn \galley_display_vspace:N #1
+\cs_new_protected:Npn \galley_display_vspace:N #1
{
\tl_if_empty:NF #1
{
@@ -1471,7 +1481,7 @@
% Setting the measure is just a question of adjusting margins, either
% in a relative or absolute sense.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_margins_set_absolute:nn #1#2
+\cs_new_protected:Npn \galley_margins_set_absolute:nn #1#2
{
\dim_set:Nn \l_galley_total_left_margin_dim {#1}
\dim_set:Nn \l_galley_total_right_margin_dim {#2}
@@ -1482,7 +1492,7 @@
- \l_galley_total_right_margin_dim
}
}
-\cs_new_protected_nopar:Npn \galley_margins_set_relative:nn #1#2
+\cs_new_protected:Npn \galley_margins_set_relative:nn #1#2
{
\dim_add:Nn \l_galley_total_left_margin_dim {#1}
\dim_add:Nn \l_galley_total_right_margin_dim {#2}
@@ -1511,7 +1521,7 @@
% later. So this is just a case of saving the various pieces of data
% to the correct locations.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_parshape_fixed_lines:nnn #1#2#3
+\cs_new_protected:Npn \galley_parshape_fixed_lines:nnn #1#2#3
{
\bool_gset_true:N \g_galley_parshape_set_bool
\bool_set_true:N \l_galley_parshape_fixed_lines_bool
@@ -1520,7 +1530,7 @@
\clist_set:Nn \l_galley_parshape_right_indent_clist {#3}
\bool_set_true:N \l_galley_parshape_resume_std_bool
}
-\cs_new_protected_nopar:Npn \galley_parshape_multi_par:nnnN #1#2#3#4
+\cs_new_protected:Npn \galley_parshape_multi_par:nnnN #1#2#3#4
{
\bool_gset_true:N \g_galley_parshape_set_bool
\bool_set_true:N \l_galley_parshape_multipar_bool
@@ -1530,7 +1540,7 @@
\clist_set:Nn \l_galley_parshape_right_indent_clist {#3}
\bool_set_eq:NN \l_galley_parshape_resume_std_bool #4
}
-\cs_new_protected_nopar:Npn \galley_parshape_single_par:nnnN #1#2#3#4
+\cs_new_protected:Npn \galley_parshape_single_par:nnnN #1#2#3#4
{
\bool_gset_true:N \g_galley_parshape_set_bool
\bool_set_false:N \l_galley_parshape_multipar_bool
@@ -1613,13 +1623,12 @@
\dim_use:N \l_galley_text_width_dim
}
}
-\cs_new_nopar:Npn \galley_set_parshape_map:nn #1#2
+\cs_new:Npn \galley_set_parshape_map:nn #1#2
{ \galley_set_parshape_map_aux:nw { } #1 , \q_mark #2 , \q_stop }
\cs_generate_variant:Nn \galley_set_parshape_map:nn { oo }
-\cs_new_nopar:Npn \galley_set_parshape_map_aux:nw
- #1#2 , #3 \q_mark #4 , #5 \q_stop
+\cs_new:Npn \galley_set_parshape_map_aux:nw #1#2 , #3 \q_mark #4 , #5 \q_stop
{
- \bool_if:nTF { \tl_if_empty_p:n {#3} || \tl_if_empty_p:n {#4} }
+ \bool_if:nTF { \tl_if_empty_p:n {#3} || \tl_if_empty_p:n {#5} }
{
#1
\dim_eval:n { \l_galley_total_left_margin_dim + ( #2 ) }
@@ -1671,7 +1680,7 @@
\dim_use:N \l_galley_text_width_dim
}
}
-\cs_new_nopar:Npn \galley_generate_parshape_lines_aux:n #1
+\cs_new:Npn \galley_generate_parshape_lines_aux:n #1
{
\etex_parshapeindent:D #1
~
@@ -1689,9 +1698,9 @@
% form but are stored as token lists. The \texttt{x}-type expansion
% deals with this nicely.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_set_user_penalty:n #1
+\cs_new_protected:Npn \galley_set_user_penalty:n #1
{ \tl_gset:Nx \g_galley_interpar_penalty_user_tl { \int_eval:n {#1} } }
-\cs_new_protected_nopar:Npn \galley_set_user_vspace:n #1
+\cs_new_protected:Npn \galley_set_user_vspace:n #1
{ \tl_gset:Nx \g_galley_interpar_vspace_user_tl { \skip_eval:n {#1} } }
% \end{macrocode}
% \end{macro}
@@ -1760,7 +1769,7 @@
% (non-zero \cs{tex_fontdimen:D} \texttt{3}), then the \cs{teX_spaceskip:D}
% is set to the fixed component from the font.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_set_interword_spacing:N #1
+\cs_new_protected:Npn \galley_set_interword_spacing:N #1
{
\bool_if:NTF #1
{ % TODO Hook for font changes required!
@@ -1776,6 +1785,17 @@
%
% \subsection{Hyphenation}
%
+% \begin{variable}{\l_galley_hyphen_left_int}
+% Currently something of a hack: this links in with language and fonts,
+% so is not so straight-forward to handle.
+% \begin{macrocode}
+\int_new:N \l_galley_hyphen_left_int
+%<*package>
+\int_set:Nn \l_galley_hyphen_left_int { \tex_lefthyphenmin:D }
+%</package>
+% \end{macrocode}
+% \end{variable}
+%
% \subsection{Line breaking}
%
% \begin{variable}
@@ -1808,6 +1828,32 @@
% \end{macrocode}
% \end{variable}
%
+% \begin{macro}{\galley_break_line:Nn}
+% Terminating a line early without a new paragraph requires a few steps.
+% First, any skips are removed, then any additional space to add is
+% places on the surrounding vertical list. Finally, the current line
+% is ended, using a penalty to prevents an overful line ending |\\| giving
+% a totally-blank one in the output. The boolean argument is used to indicate
+% that a break is allowed after the blank line.
+% \begin{macrocode}
+\cs_new_protected:Npn \galley_break_line:Nn #1#2
+ {
+ \mode_if_vertical:TF
+ { \msg_kernel_error:nn { galley } { no-line-to-end } }
+ {
+ \tex_unskip:D
+ \bool_if:NF #1
+ { \tex_vadjust:D { \tex_penalty:D \c_ten_thousand } }
+ \dim_compare:nNnF {#2} = \c_zero_dim
+ { \tex_vadjust:D { \skip_vertical:n {#2} } }
+ \tex_penalty:D \c_ten_thousand
+ \tex_hfil:D
+ \tex_penalty:D -\c_ten_thousand
+ }
+ }
+% \end{macrocode}
+%\end{macro}
+%
% \subsection{Paragraph breaking}
%
% \begin{variable}
@@ -1858,7 +1904,7 @@
% the correct form, and add a $0$ penalty at the end to nullify the effect of
% repeating the last value.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_set_display_widow_penalties:n #1
+\cs_new_protected:Npn \galley_set_display_widow_penalties:n #1
{
\etex_displaywidowpenalties:D
\int_eval:w \clist_length:n {#1} + \c_one \int_eval_end:
@@ -1866,7 +1912,7 @@
\c_zero
}
\cs_generate_variant:Nn \galley_set_display_widow_penalties:n { V , v }
-\cs_new_protected_nopar:Npn \galley_set_widow_penalties:n #1
+\cs_new_protected:Npn \galley_set_widow_penalties:n #1
{
\etex_widowpenalties:D
\int_eval:w \clist_length:n {#1} + \c_one \int_eval_end:
@@ -1874,7 +1920,7 @@
\c_zero
}
\cs_generate_variant:Nn \galley_set_widow_penalties:n { V , v }
-\cs_new_nopar:Npn \galley_set_aux:n #1 { #1 ~ }
+\cs_new:Npn \galley_set_aux:n #1 { #1 ~ }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -1893,13 +1939,13 @@
% the same, but having two takes makes some special effects easier to
% carry out.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_set_club_penalties:n #1
+\cs_new_protected:Npn \galley_set_club_penalties:n #1
{
\clist_set:Nn \l_galley_club_penalties_clist {#1}
\galley_calc_interline_penalties:
}
\cs_generate_variant:Nn \galley_set_club_penalties:n { V , v }
-\cs_new_protected_nopar:Npn \galley_set_interline_penalties:n #1
+\cs_new_protected:Npn \galley_set_interline_penalties:n #1
{
\clist_set:Nn \l_galley_line_penalties_clist {#1}
\galley_calc_interline_penalties:
@@ -1917,7 +1963,7 @@
% Setting the display club penalties means first setting the primitive,
% then recalculating the interline array to allow for these new values.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_set_display_club_penalties:n #1
+\cs_new_protected:Npn \galley_set_display_club_penalties:n #1
{
\etex_clubpenalties:D
\int_eval:w \clist_length:n {#1} + \c_one \int_eval_end:
@@ -1943,7 +1989,7 @@
% generic build system is invoked (in case the
% \cs{etex_interlinepenalties:D} has accidentally been cleared).
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_set_interline_penalty:n #1
+\cs_new_protected:Npn \galley_set_interline_penalty:n #1
{
\int_compare:nNnTF { \etex_interlinepenalties:D \c_zero } = \c_zero
{
@@ -1951,7 +1997,7 @@
\galley_calc_interline_penalties:
}
{
- \cs_set_nopar:Npn \galley_set_interline_penalty_aux_ii:n ##1
+ \cs_set:Npn \galley_set_interline_penalty_aux_ii:n ##1
{
\int_eval:w
\etex_interlinepenalties:D ##1
@@ -1963,7 +2009,7 @@
{ \clist_length:N \l_galley_line_penalties_clist } {#1}
}
}
-\cs_new_protected_nopar:Npn \galley_set_interline_penalty_aux:nn #1#2
+\cs_new_protected:Npn \galley_set_interline_penalty_aux:nn #1#2
{
\etex_interlinepenalties:D
\etex_interlinepenalties:D \c_zero
@@ -1974,9 +2020,9 @@
\galley_set_interline_penalty_aux_ii:n
\int_eval:w #2 \int_eval_end:
}
-\cs_new_nopar:Npn \galley_set_interline_penalty_aux_i:n #1
+\cs_new:Npn \galley_set_interline_penalty_aux_i:n #1
{ \etex_interlinepenalties:D \int_eval:w #1 \int_eval_end: }
-\cs_new_nopar:Npn \galley_set_interline_penalty_aux_ii:n #1 { }
+\cs_new:Npn \galley_set_interline_penalty_aux_ii:n #1 { }
% \end{macrocode}
% \end{macro}
% \end{macro}
@@ -2025,8 +2071,7 @@
% normal lines. At the end of the process, the standard interline
% penalty is always included.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_calc_interline_penalties_aux:nn
- #1#2
+\cs_new_protected:Npn \galley_calc_interline_penalties_aux:nn #1#2
{
\etex_interlinepenalties:D #1 ~
\prg_stepwise_function:nnnN \c_one \c_one {#2}
@@ -2035,8 +2080,7 @@
\galley_calc_interline_penalties_aux_ii:n
\etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
}
-\cs_new_nopar:Npn \galley_calc_interline_penalties_aux_i:n
- #1
+\cs_new:Npn \galley_calc_interline_penalties_aux_i:n #1
{
\int_eval:w
\clist_item:Nn \l_galley_line_penalties_clist { #1 - \c_one }
@@ -2045,8 +2089,7 @@
- \etex_clubpenalties:D #1 ~
\int_eval_end:
}
-\cs_new_nopar:Npn \galley_calc_interline_penalties_aux_ii:n
- #1
+\cs_new:Npn \galley_calc_interline_penalties_aux_ii:n #1
{
\int_eval:w
\etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
@@ -2078,15 +2121,15 @@
% Saving the array penalties varies in complexity depending on how they are
% stored internally. The first two are easy: these are simply copies.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_save_club_penalties:N #1
+\cs_new_protected:Npn \galley_save_club_penalties:N #1
{ \clist_set_eq:NN #1 \l_galley_club_penalties_clist }
-\cs_new_protected_nopar:Npn \galley_save_interline_penalties:N #1
+\cs_new_protected:Npn \galley_save_interline_penalties:N #1
{ \clist_set_eq:NN #1 \l_galley_line_penalties_clist }
% \end{macrocode}
% These all require appropriate mappings, using the fact that
% \cs{clist_set:Nx} will tidy up the excess comma.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \galley_save_display_club_penalties:N #1
+\cs_new_protected:Npn \galley_save_display_club_penalties:N #1
{
\clist_set:Nx #1
{
@@ -2095,9 +2138,9 @@
\galley_save_display_club_penalties:_aux:n
}
}
-\cs_new_nopar:Npn \galley_save_display_club_penalties:_aux:n #1
+\cs_new:Npn \galley_save_display_club_penalties:_aux:n #1
{ \int_use:N \etex_clubpenaltes:D \int_eval:w #1 \int_eval_end: , }
-\cs_new_protected_nopar:Npn \galley_save_display_widow_penalties:N #1
+\cs_new_protected:Npn \galley_save_display_widow_penalties:N #1
{
\clist_set:Nx #1
{
@@ -2106,9 +2149,9 @@
\galley_save_display_widow_penalties:_aux:n
}
}
-\cs_new_nopar:Npn \galley_save_display_widow_penalties:_aux:n #1
+\cs_new:Npn \galley_save_display_widow_penalties:_aux:n #1
{ \int_use:N \etex_displaywidowpenalties:D \int_eval:w #1 \int_eval_end: , }
-\cs_new_protected_nopar:Npn \galley_save_widow_penalties:N #1
+\cs_new_protected:Npn \galley_save_widow_penalties:N #1
{
\clist_set:Nx #1
{
@@ -2117,7 +2160,7 @@
\galley_save_widow_penalties:_aux:n
}
}
-\cs_new_nopar:Npn \galley_save_widow_penalties:_aux:n #1
+\cs_new:Npn \galley_save_widow_penalties:_aux:n #1
{ \int_use:N \etex_widowpenalties:D \int_eval:w #1 \int_eval_end: , }
% \end{macrocode}
% This one is not an array, but is stored in a primitive, so there is
@@ -2131,6 +2174,13 @@
% \end{macro}
% \end{macro}
%
+% \subsection{Messages}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnn { galley } { no-line-to-end }
+ { There's~no~line~here~to~end. }
+% \end{macrocode}
+%
% \subsection{\LaTeXe{} functions}
%
% \begin{macrocode}
@@ -2143,7 +2193,7 @@
% In package mode, some of \LaTeXe{}'s functions are re-implemented using
% the galley system. Not all of the optional arguments currently work!
% \begin{macrocode}
-\cs_set_protected_nopar:Npn \nobreak
+\RenewDocumentCommand \nobreak { }
{ \bool_gset_true:N \g_galley_no_break_next_bool }
% \end{macrocode}
% The \tn{noindent} primitive will causes problems, as it is used by
@@ -2152,12 +2202,12 @@
% need leave vertical mode, at the moment we stick with the primitive as
% well as setting the galley flag.
% \begin{macrocode}
-\cs_set_protected_nopar:Npn \noindent
+\RenewDocumentCommand \noindent { }
{
\tex_noindent:D
\bool_gset_false:N \g_galley_omit_next_indent_bool
}
-\cs_set_protected_nopar:Npn \vspace #1
+\RenewDocumentCommand \vspace { m }
{
\@ifstar
{ \galley_set_user_vspace:n {#1} }
@@ -2168,6 +2218,19 @@
% \end{macro}
% \end{macro}
%
+% \begin{macro}{\\}
+% \begin{macro}{\newline}
+% These functions pass their arguments straight through to the internal
+% implementation (which is currently just the \LaTeXe{} one recoded).
+% \begin{macrocode}
+\RenewDocumentCommand \\ { s O { 0 pt } }
+ { \galley_break_line:Nn #1 {#2} }
+\RenewDocumentCommand \newline { }
+ { \galley_break_line:Nn \c_true_bool { 0 pt } }
+% \end{macrocode}
+%\end{macro}
+%\end{macro}
+%
% \subsection{\LaTeXe{} fixes}
%
% Purely for testing, some internal \LaTeXe{} functions are altered to work
@@ -2217,6 +2280,14 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}{\@normalcr}
+% This is needed as \cs{@parboxrestore} sets |\\| equal to \cs{@normalcr},
+% and the new definition must be used
+% \begin{macrocode}
+\cs_set_eq:Nc \@normalcr { \token_to_str:N \\ }
+% \end{macrocode}
+%\end{macro}
+%
% \begin{macrocode}
%</package>
% \end{macrocode}
diff --git a/Master/texmf-dist/source/latex/l3experimental/xgalley/xgalley.dtx b/Master/texmf-dist/source/latex/l3experimental/xgalley/xgalley.dtx
index f573668fde8..bba0907bbd8 100644
--- a/Master/texmf-dist/source/latex/l3experimental/xgalley/xgalley.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/xgalley/xgalley.dtx
@@ -1,7 +1,7 @@
% \iffalse
%
%% File: xgalley.dtx Copyright (C) 1999-2001, 2004-2009 Frank Mittelbach
-%% (C) 2010,2011 The LaTeX3 Project
+%% (C) 2010-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -37,7 +37,7 @@
%
%<*driver|package>
\RequirePackage{l3names}
-\GetIdInfo$Id: xgalley.dtx 2922 2011-10-17 19:01:04Z joseph $
+\GetIdInfo$Id: xgalley.dtx 3253 2012-01-28 14:59:55Z joseph $
{L3 Experimental galley}
%</driver|package>
%<*driver>
@@ -471,7 +471,7 @@
% similar to the \LaTeXe{} \pkg{ragged2e} \env{RaggedLeft} environment.
% \end{InstanceDescription}
%
-% \begin{InstanceDescription}{justification}{center}{std}
+% \begin{InstanceDescription}{justification}{centering}{std}
% \InstanceKey{end-skip}{0 pt}
% \InstanceKey{fixed-word-spacing}{true}
% \InstanceKey{indent-width}{0 pt}
@@ -994,16 +994,21 @@
\int_set:Nn \tex_lefthyphenmin:D
{
\bool_if:NTF \l_galley_hyphen_enable_bool
- { \l_galley_hyphen_left_int } { 63 }
+ { \l_galley_hyphen_left_int }
+ { 63 }
}
\int_set:Nn \tex_uchyph:D
- { \bool_if:NTF \l_galley_hyphen_uppercase_bool { 1 } { 0 } }
+ {
+ \bool_if:NTF \l_galley_hyphen_uppercase_bool
+ { 1 }
+ { 0 }
+ }
}
% \end{macrocode}
% At this stage, the default hyphenation character should be set and
% hyphenation should be enabled.
% \begin{macrocode}
-%\UseTemplate { hyphenation } { std } { }
+\UseTemplate { hyphenation } { std } { }
\tex_defaulthyphenchar:D 45 \scan_stop:
% \end{macrocode}
%
@@ -1094,7 +1099,7 @@
\bool_if:NTF \l_galley_fixed_spacing_bool
{ \bool_set_true:N \exp_not:N \l_galley_fixed_spacing_bool }
{ \bool_set_false:N \exp_not:N \l_galley_fixed_spacing_bool }
- \galley_set_interword_space:N
+ \galley_set_interword_spacing:N
\exp_not:N \l_galley_fixed_spacing_bool
\dim_set:Nn \exp_not:N \l_galley_par_indent_dim
{ \dim_use:N \l_galley_par_indent_dim }
@@ -1102,8 +1107,8 @@
{ \skip_use:N \l_galley_line_left_skip }
\skip_set:Nn \exp_not:N \l_galley_line_right_skip
{ \skip_use:N \l_galley_line_right_skip }
- \skip_set:Nn \exp_not:N \l_galley_par_start_skip
- { \skip_use:N \l_galley_par_start_skip }
+ \skip_set:Nn \exp_not:N \l_galley_par_begin_skip
+ { \skip_use:N \l_galley_par_begin_skip }
\int_set:Nn \exp_not:N \l_galley_last_line_fit_int
{ \int_use:N \l_galley_last_line_fit_int }
\skip_set:Nn \exp_not:N \@rightskip
@@ -1471,7 +1476,7 @@
\DeclareInstance { paragraph-breaking } { single-noclub } { single }
{
club-penalty = 10 000 ,
- display-club-penalty = 10 000
+ display-club-penalty = 10 000
}
\DeclareInstance { paragraph-breaking } { single-nolone } { single }
{
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3dt/l3dt.sty b/Master/texmf-dist/tex/latex/l3experimental/l3dt/l3dt.sty
new file mode 100644
index 00000000000..5346944521c
--- /dev/null
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3dt/l3dt.sty
@@ -0,0 +1,459 @@
+%%
+%% This is file `l3dt.sty',
+%% generated with the docstrip utility.
+%%
+%% The original source files were:
+%%
+%% l3dt.dtx (with options: `package')
+%%
+%% EXPERIMENTAL CODE
+%%
+%% Do not distribute this file without also distributing the
+%% source files specified above.
+%%
+%% Do not distribute a modified version of this file.
+%%
+%% File l3dt.dtx Copyright (C) 2011,2012 The LaTeX3 Project
+%%
+%% It may be distributed and/or modified under the conditions of the
+%% LaTeX Project Public License (LPPL), either version 1.3c of this
+%% license or (at your option) any later version. The latest version
+%% of this license is in the file
+%%
+%% http://www.latex-project.org/lppl.txt
+%%
+%% This file is part of the "l3experimental bundle" (The Work in LPPL)
+%% and all files in that bundle must be distributed together.
+%%
+%% The released version of this bundle is available from CTAN.
+%%
+%% -----------------------------------------------------------------------
+%%
+%% The development version of the bundle can be found at
+%%
+%% http://www.latex-project.org/svnroot/experimental/trunk/
+%%
+%% for those people who are interested.
+%%
+%%%%%%%%%%%
+%% NOTE: %%
+%%%%%%%%%%%
+%%
+%% Snapshots taken from the repository represent work in progress and may
+%% not work or may contain conflicting material! We therefore ask
+%% people _not_ to put them into distributions, archives, etc. without
+%% prior consultation with the LaTeX Project Team.
+%%
+%% -----------------------------------------------------------------------
+%%
+\RequirePackage{expl3}
+\GetIdInfo$Id: l3dt.dtx 3251 2012-01-28 09:42:44Z joseph $
+ {L3 Experimental data tables}
+\ProvidesExplPackage
+ {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
+\package_check_loaded_expl:
+\quark_new:N \q_dt
+\quark_new:N \q_dt_row
+\quark_new:N \q_dt_header
+\tl_const:Nn \c_empty_dt
+ {
+ { 0 }
+ \q_dt
+ \q_nil
+ \q_dt_header
+ \q_dt_row
+ }
+\cs_new_protected:Npn \dt_new:N #1 { \cs_new_eq:NN #1 \c_empty_dt }
+\cs_new_protected:Npn \dt_clear:N #1 { \cs_set_eq:NN #1 \c_empty_dt }
+\cs_new_protected:Npn \dt_gclear:N #1 { \cs_gset_eq:NN #1 \c_empty_dt }
+\cs_new_protected:Npn \dt_clear_new:N #1
+ { \cs_if_exist:NTF #1 { \dt_clear:N #1 } { \dt_new:N #1 } }
+\cs_new_protected:Npn \dt_gclear_new:N #1
+ { \cs_if_exist:NTF #1 { \dt_gclear:N #1 } { \dt_new:N #1 } }
+\cs_new_eq:NN \dt_set_eq:NN \tl_set_eq:NN
+\cs_new_eq:NN \dt_gset_eq:NN \tl_gset_eq:NN
+\dt_new:N \l_tmpa_dt
+\dt_new:N \l_tmpb_dt
+\dt_new:N \g_tmpa_dt
+\dt_new:N \g_tmpb_dt
+\cs_new_protected:Npn \dt_split_aux:nnnn #1#2#3#4 { #3 #2 }
+\cs_new_protected:Npn \dt_split_aux:w { }
+\cs_new:Npn \dt_split_header:NT #1#2
+ { \exp_after:wN \dt_split_header_aux:wn #1 \q_stop {#2} }
+\cs_new:Npn \dt_split_header_aux:wn #1 \q_nil \q_dt_header #2 \q_stop #3
+ { #3 {#1} { \q_dt_header #2 } }
+\cs_new_protected:Npn \dt_split_key:nnTF #1#2
+ { \exp_args:No \dt_split_key_aux:nnTF { \tl_to_str:n {#2} } {#1} }
+\cs_new_protected:Npn \dt_split_key_aux:nnTF #1#2
+ {
+ \cs_set_protected:Npn \dt_split_aux:w
+ ##1 \q_dt #1 \q_dt ##2##3##4 \q_mark ##5 \q_stop
+ { \dt_split_aux:nnnn ##3 { { ##1 \q_dt } {##2} {##4} } }
+ \dt_split_aux:w #2 \q_mark
+ \q_dt #1 \q_dt { } { ? \use_ii:nn { } } \q_mark \q_stop
+ }
+\cs_new_protected:Npn \dt_split_key_list:NnTF #1#2
+ { \exp_args:NNo \dt_split_key_list_aux:NnTF #1 { \tl_to_str:n {#2} } }
+\cs_new_protected:Npn \dt_split_key_list_aux:NnTF #1#2
+ {
+ \cs_set_protected:Npn \dt_split_aux:w
+ ##1##2 \q_dt #2 \q_dt ##3##4 \q_dt_header ##5 \q_mark ##6 \q_stop
+ {
+ \dt_split_aux:nnnn ##3
+ { { {##1} ##2 \q_dt } { ##3##4 \q_dt_header ##5 } }
+ }
+ \exp_after:wN \dt_split_aux:w #1 \q_mark
+ \q_dt #2 \q_dt { ? \use_ii:nn { } } \q_dt_header \q_mark \q_stop
+ }
+\cs_new_protected:Npn \dt_split_row:NnTF #1#2
+ { \dt_split_row_aux:NfTF #1 { \int_eval:n {#2} } }
+\cs_new_protected:Npn \dt_split_row_aux:NnTF #1#2
+ {
+ \cs_set_protected:Npn \dt_split_aux:w
+ ##1 \q_dt_row #2 \q_dt ##2##3 \q_dt_row ##4 \q_mark ##5 \q_stop
+ {
+ \dt_split_aux:nnnn ##2
+ { { ##1 \q_dt_row } { #2 \q_dt ##2##3 } {##4} }
+ }
+ \exp_after:wN \dt_split_aux:w #1 \q_mark
+ \q_dt_row #2 \q_dt { ? \use_ii:nn { } } \q_dt_row \q_mark \q_stop
+ }
+\cs_generate_variant:Nn \dt_split_row_aux:NnTF { Nf }
+\cs_new_protected_nopar:Npn \dt_add_key:Nn { \dt_add_key_aux:NNn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gadd_key:Nn { \dt_add_key_aux:NNn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_add_key_aux:NNn #1#2#3
+ {
+ \dt_split_key_list:NnTF #2 {#3}
+ { \use_none:nn }
+ {
+ \dt_split_header:NT #2
+ { \dt_add_key_aux:NNnnn #1 #2 {#3} }
+ }
+ }
+\cs_new_protected:Npn \dt_add_key_aux:NNnnn #1#2#3#4#5
+ {
+ #1 #2
+ {
+ \exp_not:n {#4}
+ \tl_to_str:n {#3}
+ \exp_not:n { \q_dt \q_nil #5 }
+ }
+ }
+\cs_new_protected_nopar:Npn \dt_add_row:N { \dt_add_row_aux:NN \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gadd_row:N { \dt_add_row_aux:NN \tl_gset:Nx }
+\cs_new_protected:Npn \dt_add_row_aux:NN #1#2
+ { \exp_args:NNf \dt_add_row_aux:NnN #1 { \dt_rows:N #2 } #2 }
+\cs_new_protected:Npn \dt_add_row_aux:NnN #1#2#3
+ {
+ #1 #3
+ {
+ { \int_eval:n { #2 + \c_one } }
+ \exp_after:wN \dt_add_row_aux:nw #3 \q_stop
+ #2
+ \exp_not:n { \q_dt \q_nil \q_dt_row }
+ }
+ }
+\cs_new:Npn \dt_add_row_aux:nw #1#2 \q_stop { \exp_not:n {#2} }
+\cs_new_protected:Npn \dt_put:Nnn #1
+ { \dt_put:Nnnn #1 { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_gput:Nnn #1
+ { \dt_gput:Nnnn #1 { \dt_rows:N #1 - \c_one } }
+\cs_new_protected_nopar:Npn \dt_put:Nnnn
+ { \dt_put_aux:NNNnnn \dt_add_key:Nn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gput:Nnnn
+ { \dt_put_aux:NNNnnn \dt_gadd_key:Nn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_put_aux:NNNnnn #1#2#3#4#5#6
+ {
+ #1 #3 {#5}
+ \dt_split_row:NnTF #3 {#4}
+ { \dt_put_aux:NNnnnnn #2 #3 {#5} {#6} }
+ {
+ \msg_kernel_error:nnxxx { dt } { unknown-row }
+ { \token_to_str:N #3 } { \int_eval:n {#4} } { \dt_rows:N #3 }
+ }
+ }
+\cs_new_protected:Npn \dt_put_aux:NNnnnnn #1#2#3#4#5#6#7
+ {
+ \dt_split_key:nnTF {#6} {#3}
+ { \dt_put_update:NNnnnnnnn #1 #2 {#3} {#4} {#5} {#7} }
+ { \dt_put_add_to_row:NNnnnnn #1 #2 {#3} {#4} {#5} {#6} {#7} }
+ }
+\cs_new_protected:Npn \dt_put_update:NNnnnnnnn #1#2#3#4#5#6#7#8#9
+ {
+ #1 #2
+ {
+ \exp_not:n { #5 #7 }
+ \tl_to_str:n {#3}
+ \exp_not:n { \q_dt {#4} \q_dt #9 \q_dt_row #6 }
+ }
+ }
+\cs_new_protected:Npn \dt_put_add_to_row:NNnnnnn #1#2#3#4#5#6#7
+ {
+ #1 #2
+ {
+ \exp_not:n {#5}
+ \exp_not:o { \dt_put_add_row_aux:w #6 }
+ \tl_to_str:n {#3}
+ \exp_not:n { \q_dt {#4} \q_dt \q_nil \q_dt_row #7 }
+ }
+ }
+\cs_new:Npn \dt_put_add_row_aux:w #1 \q_nil {#1}
+\cs_new:Npn \dt_keys:N #1
+ { \dt_split_header:NT #1 { \dt_keys_aux:nn } }
+\cs_new:Npn \dt_keys_aux:nn #1#2
+ {
+ \int_eval:n
+ {
+ 0
+ \exp_after:wN \dt_keys_aux:wN \use_none:nn #1 \q_recursion_tail \q_dt
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \dt_keys_aux:wN #1 \q_dt
+ {
+ \if_meaning:w \q_recursion_tail #1
+ \exp_after:wN \prg_map_break:
+ \fi:
+ +1
+ \dt_keys_aux:wN
+ }
+\cs_new:Npn \dt_rows:N #1
+ { \exp_after:wN \use_i_delimit_by_q_stop:nw #1 \q_stop }
+\cs_new_protected:Npn \dt_del:Nn #1 { \dt_del:Nnn #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_gdel:Nn #1 { \dt_gdel:Nnn #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected_nopar:Npn \dt_del:Nnn { \dt_del_aux:NNnn \tl_set:Nn }
+\cs_new_protected_nopar:Npn \dt_gdel:Nnn { \dt_del_aux:NNnn \tl_gset:Nn }
+\cs_new_protected:Npn \dt_del_aux:NNnn #1#2#3#4
+ {
+ \dt_split_row:NnTF #2 {#3}
+ { \dt_del_aux:NNnnnn #1 #2 {#4} }
+ { }
+ }
+\cs_new_protected:Npn \dt_del_aux:NNnnnn #1#2#3#4#5#6
+ {
+ \dt_split_key:nnTF {#5} {#3}
+ { \dt_del_aux:NNnnnnn #1 #2 {#4} {#6} }
+ { }
+ }
+\cs_new_protected:Npn \dt_del_aux:NNnnnnn #1#2#3#4#5#6#7
+ { #1 #2 { #3 #5 #7 #4 } }
+\cs_new_protected_nopar:Npn \dt_remove_key:Nn
+ { \dt_remove_key_aux:NNn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gremove_key:Nn
+ { \dt_remove_key_aux:NNn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_remove_key_aux:NNn #1#2#3
+ {
+ \dt_split_key_list:NnTF #2 {#3}
+ { \exp_args:No \dt_remove_key_aux:nNNnn { \tl_to_str:n {#3} } #1 #2 }
+ { }
+ }
+\cs_new_protected:Npn \dt_remove_key_aux:nNNnn #1#2#3#4#5
+ {
+ \cs_set:Npn \dt_remove_key_aux:w ##1 \q_dt #1 \q_dt ##2 ##3
+ {
+ \exp_not:n {##1}
+ \if_meaning:w \q_recursion_tail ##3
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \dt_remove_key_aux:w ##3
+ }
+ #2 #3
+ {
+ \exp_not:n {#4}
+ \dt_remove_key_aux:w #5 \q_dt #1 \q_dt { } \q_recursion_tail
+ \prg_break_point:n { }
+ }
+ }
+\cs_new:Npn \dt_remove_key_aux:w { }
+\cs_new_protected_nopar:Npn \dt_remove_row:Nn
+ { \dt_remove_row_aux:NNn \tl_set:Nx }
+\cs_new_protected_nopar:Npn \dt_gremove_row:Nn
+ { \dt_remove_row_aux:NNn \tl_gset:Nx }
+\cs_new_protected:Npn \dt_remove_row_aux:NNn #1#2#3
+ {
+ \dt_split_row:NnTF #2 {#3}
+ { \dt_remove_row_aux:NNnnn #1 #2 }
+ { }
+ }
+\cs_new_protected:Npn \dt_remove_row_aux:NNnnn #1#2#3#4#5
+ {
+ #1 #2
+ {
+ { \int_eval:n { \dt_rows:N #2 - \c_one } }
+ \dt_remove_row_aux:nw #3 \q_stop
+ \dt_remove_row_loop:nw #5 \q_recursion_tail \q_dt_row
+ \prg_break_point:n { }
+ }
+ }
+\cs_new_eq:NN \dt_remove_row_aux:nw \dt_add_row_aux:nw
+\cs_new:Npn \dt_remove_row_loop:nw #1#2 \q_dt_row
+ {
+ \if_meaning:w \q_recursion_tail #1
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \int_eval:n { #1 - \c_one }
+ \exp_not:n { #2 \q_dt_row }
+ \dt_remove_row_loop:nw
+ }
+\cs_new_protected:Npn \dt_get:NnnN #1#2#3#4
+ {
+ \dt_split_row:NnTF #1 {#2}
+ { \dt_get_aux:nNnnn {#3} #4 }
+ { \tl_set:Nn #4 { \q_no_value } }
+ }
+\cs_new_protected:Npn \dt_get_aux:nNnnn #1#2#3#4#5
+ {
+ \dt_split_key:nnTF {#4} {#1}
+ { \dt_get_aux:Nnnn #2 }
+ { \tl_set:Nn #2 { \q_no_value } }
+ }
+\cs_new_protected:Npn \dt_get_aux:Nnnn #1#2#3#4 { \tl_set:Nn #1 {#3} }
+\prg_new_protected_conditional:Npnn \dt_get:NnnN #1#2#3#4 { T , F , TF }
+ {
+ \dt_split_row:NnTF #1 {#2}
+ { \dt_get_aux_true:nNnnn {#3} #4 }
+ { \prg_return_false: }
+ }
+\cs_new_protected:Npn \dt_get_aux_true:nNnnn #1#2#3#4#5
+ {
+ \dt_split_key:nnTF {#4} {#1}
+ { \dt_get_aux_true:Nnnn #2 }
+ { \prg_return_false: }
+ }
+\cs_new_protected:Npn \dt_get_aux_true:Nnnn #1#2#3#4
+ {
+ \tl_set:Nn #1 {#3}
+ \prg_return_true:
+ }
+\cs_new_protected:Npn \dt_get:NnN #1 { \dt_get:NnnN #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_get:NnNT #1 { \dt_get:NnnNF #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_get:NnNF #1 { \dt_get:NnnNF #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new_protected:Npn \dt_get:NnNTF #1 { \dt_get:NnnNTF #1
+ { \dt_rows:N #1 - \c_one } }
+\int_new:N \g_dt_map_level_int
+\cs_new_protected:Npn \dt_map_variables:Nnn #1#2#3
+ {
+ \int_gincr:N \g_dt_map_level_int
+ \seq_gclear_new:c { g_dt_map_ \int_use:N \g_dt_map_level_int _seq }
+ \keyval_parse:NNn \use_none:n \dt_map_variables_key:nn {#2}
+ \dt_split_header:NT #1 { \dt_map_variables_aux:nnn {#3} }
+ }
+\cs_new_protected:Npn \dt_map_variables_key:nn #1#2
+ {
+ \seq_gput_right:cn { g_dt_map_ \int_use:N \g_dt_map_level_int _seq }
+ { {#1} #2 }
+ }
+\cs_new_protected:Npn \dt_map_variables_aux:nnn #1#2#3
+ { \dt_map_variables_aux:nNNw {#1} #3 \q_stop }
+\cs_new_protected:Npn \dt_map_variables_aux:nNNw
+ #1 \q_dt_header \q_dt_row #2 \q_stop
+ {
+ \int_zero_new:c { l_dt_map_ \int_use:N \g_dt_map_level_int _row_int }
+ \dt_map_variables_aux:nnw {#1} #2 { } \q_recursion_tail \q_dt_row
+ \prg_break_point:n { \int_gdecr:N \g_dt_map_level_int }
+ }
+\cs_new_protected:Npn \dt_map_variables_aux:nnw #1#2#3#4 \q_dt_row
+ {
+ \if_meaning:w \q_recursion_tail #3
+ \exp_after:wN \dt_map_break:
+ \fi:
+ \seq_map_inline:cn { g_dt_map_ \int_use:N \g_dt_map_level_int _seq }
+ { \dt_get_aux:nNnnn ##1 { } {#3#4} { } }
+ #1
+ \int_incr:c { l_dt_map_ \int_use:N \g_dt_map_level_int _row_int }
+ \dt_map_variables_aux:nnw {#1}
+ }
+\cs_new_eq:NN \dt_map_break: \prg_map_break:
+\cs_new_eq:NN \dt_map_break:n \prg_map_break:n
+\prg_new_conditional:Npnn \dt_if_empty:N #1 { T , F , TF , p }
+ {
+ \if_meaning:w #1 \c_empty_dt
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ }
+\prg_new_conditional:Npnn \dt_if_in:Nn #1#2 { p , T , F , TF }
+ { \dt_split_header:NT #1 { \dt_if_in_aux:nnn {#2} } }
+\cs_new:Npn \dt_if_in_aux:nnn #1#2#3
+ {
+ \exp_last_unbraced:Nno \dt_if_in_aux:nwN {#1} { \use_none:nn #2 }
+ \q_recursion_tail \q_dt
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \dt_if_in_aux:nwN #1#2 \q_dt
+ {
+ \if_meaning:w \q_recursion_tail #2
+ \exp_after:wN \prg_map_break:n
+ \else:
+ \exp_after:wN \use_none:n
+ \fi:
+ { \prg_return_false: }
+ \str_if_eq:nnTF {#1} {#2}
+ { \prg_map_break:n { \prg_return_true: } }
+ { \dt_if_in_aux:nwN {#1} }
+ }
+\prg_new_conditional:Npnn \dt_if_in_row:Nnn #1#2#3 { p , T , F , TF }
+ {
+ \exp_last_unbraced:Nno \dt_if_in_row_aux:nw {#2} #1
+ \q_recursion_tail \q_nil
+ \prg_break_point:n { }
+ { \tl_to_str:n {#3} }
+ }
+\cs_new:Npn \dt_if_in_row_aux:nw #1#2 \q_dt_row #3#4 \q_nil
+ {
+ \if_meaning:w \q_recursion_tail #3
+ \exp_after:wN \prg_map_break:n
+ \else:
+ \exp_after:wN \use_none:n
+ \fi:
+ {
+ \use_i:nn
+ \prg_return_false:
+ }
+ \int_compare:nNnTF {#1} = {#3}
+ { \prg_map_break:n { \exp_args:Nno \dt_if_in_row_aux:nn {#4} } }
+ { \dt_if_in_row_aux:nw {#1} }
+ }
+\cs_new:Npn \dt_if_in_row_aux:nn #1#2
+ {
+ \dt_if_in_row_aux:nwn {#2} #1 {#2} \q_dt { } \q_recursion_tail
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \dt_if_in_row_aux:nwn #1 \q_dt #2 \q_dt #3
+ {
+ \str_if_eq:xxTF {#1} {#2}
+ { \dt_if_in_row_aux:N }
+ { \dt_if_in_row_aux:nwn {#1} }
+ }
+\cs_new:Npn \dt_if_in_row_aux:N #1
+ {
+ \if_meaning:w \q_dt #1
+ \prg_return_true:
+ \else:
+ \prg_return_false:
+ \fi:
+ \prg_map_break:
+ }
+\cs_new:Npn \dt_if_in_row_p:Nn #1 { \dt_if_in_row_p:Nnn #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new:Npn \dt_if_in_row:NnT #1 { \dt_if_in_row:NnnT #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new:Npn \dt_if_in_row:NnF #1 { \dt_if_in_row:NnnF #1
+ { \dt_rows:N #1 - \c_one } }
+\cs_new:Npn \dt_if_in_row:NnTF #1 { \dt_if_in_row:NnnTF #1
+ { \dt_rows:N #1 - \c_one } }
+\msg_kernel_new:nnnn { dt } { unknown-row }
+ { Data~table~#1~does~not~contain~a~row~'#2'. }
+ {
+ Data~table~#1~contains~#3~rows.~These~must~be~accessed~by~number:~row~
+ #2~is~not~present~in~the~table.
+ }
+%%
+%%
+%% End of file `l3dt.sty'.
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty
index 2bb94e7c62a..b56100853ab 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3flag.sty
@@ -50,48 +50,56 @@
{L3 Experimental Flags}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\tl_new:N \g_flag_list_tl
-\cs_new_protected:Npn \flag_new:N #1
+\cs_new_protected:Npn \flag_new:n #1
{
- \cs_new_eq:NN #1 \c_undefined:D
- \tl_if_in:NnTF \g_flag_list_tl {#1}
- {
- \msg_kernel_error:nnx
- { flag } { already-defined }
- { \token_to_str:N #1 }
- }
- { \tl_gput_right:Nn \g_flag_list_tl {#1} }
+ \cs_new:cpn { flag_trap_#1:w } ##1 ;
+ { \exp_after:wN \use_none:n \cs:w l_#1_##1_flag \cs_end: }
}
-\cs_generate_variant:Nn \flag_new:N { c }
-\prg_new_conditional:Npnn \flag_test:N #1 { p , T , F , TF }
+\cs_new_protected:Npn \flag_clear:n #1
+ { \flag_clear_aux:ww 0 ; #1 \q_stop }
+\cs_new_protected:Npn \flag_clear_aux:ww #1 ; #2 \q_stop
{
- \if_cs_exist:N #1
- \prg_return_true:
+ \if_cs_exist:w l_#2_#1_flag \cs_end:
\else:
- \prg_return_false:
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
+ \cs_set_eq:cN { l_#2_#1_flag } \c_undefined:D
+ \exp_after:wN \flag_clear_aux:ww
+ \int_use:N \int_eval:w \c_one + #1 ;
+ #2 \q_stop
+ }
+\cs_new_protected:Npn \flag_clear_new:n #1
+ { \flag_if_exist:nTF {#1} { \flag_clear:n } { \flag_new:n } {#1} }
+\cs_new_protected:Npn \flag_set_trap:nn #1#2
+ { \cs_set:cpn { flag_trap_#1:w } ##1 ; {#2} }
+\prg_new_conditional:Npnn \flag_if_exist:n #1 { p , T , F , TF }
+ {
+ \cs_if_exist:cTF { flag_trap_#1:w }
+ { \prg_return_true: } { \prg_return_false: }
}
-\prg_new_conditional:Npnn \flag_test:c #1 { p , T , F , TF }
+\prg_new_conditional:Npnn \flag_if_raised:n #1 { p , T , F , TF }
{
- \if_cs_exist:w #1 \cs_end:
+ \if_cs_exist:w l_#1_0_flag \cs_end:
\prg_return_true:
\else:
\prg_return_false:
\fi:
}
-\cs_new:Npn \flag_raise:N #1
- { \exp_after:wN \use_none:n \cs:w \cs_to_str:N #1 \cs_end: }
-\cs_new:Npn \flag_raise:c #1
- { \exp_after:wN \use_none:n \cs:w #1 \cs_end: }
-\cs_new_protected:Npn \flag_lower:N #1
- { \cs_set_eq:NN #1 \c_undefined:D }
-\cs_generate_variant:Nn \flag_lower:N { c }
-\msg_kernel_new:nnnn { flag } { already-defined }
- { The~control~sequence~#1~is~already~declared~as~a~flag. }
+\cs_new:Npn \flag_height:n #1 { \flag_height_loop:ww 0; #1 \q_stop }
+\cs_new:Npn \flag_height_loop:ww #1 ; #2 \q_stop
+ {
+ \if_cs_exist:w l_#2_#1_flag \cs_end:
+ \exp_after:wN \flag_height_loop:ww \int_use:N \int_eval:w \c_one +
+ \else:
+ \exp_after:wN \flag_height_end:ww
+ \fi:
+ #1 ; #2 \q_stop
+ }
+\cs_new:Npn \flag_height_end:ww #1 ; #2 \q_stop { #1 }
+\cs_new:Npn \flag_raise:n #1
{
- LaTeX~was~asked~to~define~the~flag~#1,~but~it~has~already~
- been~defined~as~a~flag.~The~flag~module~is~mostly~meant~
- for~kernel~use,~and~booleans~should~be~preferred.
+ \cs:w flag_trap_#1:w \exp_after:wN \cs_end:
+ \int_value:w \flag_height:n {#1} ;
}
%%
%%
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty
index 8d1f45803f9..f88c8fa4e15 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty
@@ -46,19 +46,63 @@
%%
%% -----------------------------------------------------------------------
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 3200 2012-01-16 03:50:54Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3259 2012-01-29 20:50:48Z bruno $
{L3 Experimental Regular Expressions}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
\RequirePackage{l3str, l3tl-analysis, l3flag}
\cs_new:Npn \regex_tmp:w { }
-\tl_new:N \l_regex_tmpa_tl
-\tl_new:N \l_regex_tmpb_tl
-\tl_new:N \l_regex_tmpc_tl
-\int_new:N \l_regex_tmpa_int
-\int_new:N \l_regex_tmpb_int
-\flag_new:N \l_regex_group_begin_flag
-\flag_new:N \l_regex_group_end_flag
+\tl_new:N \l_regex_internal_a_tl
+\tl_new:N \l_regex_internal_b_tl
+\int_new:N \l_regex_internal_a_int
+\int_new:N \l_regex_internal_b_int
+\int_new:N \l_regex_internal_c_int
+\tl_new:N \g_regex_internal_tl
+\tl_const:Nn \c_regex_d_tl
+ { \regex_item_range:nn \c_forty_eight { 57 } } % 0--9
+\tl_const:Nn \c_regex_h_tl
+ {
+ \regex_item_equal:n \c_thirty_two % space
+ \regex_item_equal:n \c_nine % tab
+ }
+\tl_const:Nn \c_regex_s_tl
+ {
+ \regex_item_equal:n \c_thirty_two % space
+ \regex_item_equal:n \c_nine % tab
+ \regex_item_equal:n \c_ten % lf
+ \regex_item_equal:n \c_twelve % ff
+ \regex_item_equal:n \c_thirteen % cr
+ }
+\tl_const:Nn \c_regex_v_tl
+ { \regex_item_range:nn \c_ten \c_thirteen } % lf, vtab, ff, cr
+\tl_const:Nn \c_regex_w_tl
+ {
+ \regex_item_range:nn \c_ninety_seven { 122 } % a--z
+ \regex_item_range:nn \c_sixty_five { 90 } % A--Z
+ \regex_item_range:nn \c_forty_eight { 57 } % 0--9
+ \regex_item_equal:n { 95 } % _
+ }
+\tl_const:Nn \c_regex_D_tl
+ { \c_regex_d_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_H_tl
+ { \c_regex_h_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_S_tl
+ { \c_regex_s_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_V_tl
+ { \c_regex_v_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_W_tl
+ { \c_regex_w_tl \regex_break_point:TF { } \regex_break_true:w }
+\tl_const:Nn \c_regex_N_tl
+ {
+ \regex_item_equal:n \c_ten
+ \regex_break_point:TF { } { \regex_break_true:w }
+ }
+\tl_const:cn { c_regex_._tl }
+ {
+ \if_num:w \l_regex_current_char_int > - \c_two
+ \exp_after:wN \regex_break_true:w
+ \fi:
+ }
\int_new:N \l_regex_build_mode_int
\int_new:N \l_regex_max_state_int
\int_new:N \l_regex_left_state_int
@@ -87,105 +131,56 @@
\int_const:Nn \c_regex_catcode_A_int { "4000000 }
\int_const:Nn \c_regex_catcodes_all_int { "5515155 }
\bool_new:N \l_regex_catcodes_bool
-\tl_new:N \l_regex_tmpa_regex_tl
-\bool_new:N \l_regex_class_bool
-\tl_new:N \l_regex_class_tl
-\bool_new:N \l_regex_class_saved_bool
-\tl_new:N \l_regex_class_saved_tl
-\tl_const:Nn \c_regex_d_tl
- {
- \regex_item_range:nn { \c_forty_eight } { 57 } % 0--9
- }
-\tl_const:Nn \c_regex_D_tl
- {
- \regex_item_geq:n { \c_fifty_eight } % > `9
- \regex_item_range:nn { \c_zero } { 47 } % 0
- }
-\tl_const:Nn \c_regex_h_tl
- {
- \regex_item_equal:n { \c_thirty_two } % space
- \regex_item_equal:n { \c_nine } % tab
- }
-\tl_const:Nn \c_regex_H_tl
- {
- \regex_item_geq:n { 33 } % > space
- \regex_item_range:nn { \c_ten } { 31 } % tab < ... < space
- \regex_item_range:nn { \c_zero } { \c_eight } % < tab
- }
-\tl_const:Nn \c_regex_s_tl
- {
- \regex_item_equal:n { \c_thirty_two } % space
- \regex_item_range:nn { \c_nine } { \c_ten } % tab, lf
- \regex_item_range:nn { \c_twelve } { \c_thirteen } % ff, cr
- }
-\tl_const:Nn \c_regex_S_tl
- {
- \regex_item_geq:n { 33 } % > space
- \regex_item_range:nn { \c_fourteen } { 31 } % tab < ... < space
- \regex_item_range:nn { \c_zero } { \c_eight } % < tab
- \regex_item_equal:n { \c_eleven } % vtab
- }
-\tl_const:Nn \c_regex_v_tl
- {
- \regex_item_range:nn { \c_ten } { \c_thirteen } % lf, vtab, ff, cr
- }
-\tl_const:Nn \c_regex_V_tl
- {
- \regex_item_geq:n { \c_fourteen } % >cr
- \regex_item_range:nn { \c_zero } { \c_nine } % < lf
- }
-\tl_const:Nn \c_regex_w_tl
- {
- \regex_item_range:nn { \c_ninety_seven } { 122 } % a--z
- \regex_item_range:nn { \c_sixty_five } { 90 } % A--Z
- \regex_item_range:nn { \c_forty_eight } { 57 } % 0--9
- \regex_item_equal:n { 95 } % _
- }
-\tl_const:Nn \c_regex_W_tl
- {
- \regex_item_range:nn { \c_zero } { 47 } % <`0
- \regex_item_range:nn { \c_fifty_eight } { 64 } % (`9+1)--(`A-1)
- \regex_item_range:nn { \c_ninety_one } { 94 } % (`Z+1)--(`_-1)
- \regex_item_equal:n { 96 } % `
- \regex_item_geq:n { \c_one_hundred_twenty_three } % z
- }
-\tl_const:Nn \c_regex_N_tl
- {
- \regex_item_geq:n { \c_eleven } % > lf
- \regex_item_range:nn { \c_zero } { \c_nine } % < lf
- }
+\bool_new:N \l_regex_tests_bool
+\tl_new:N \l_regex_tests_tl
+\bool_new:N \l_regex_tests_saved_bool
+\tl_new:N \l_regex_tests_saved_tl
+\int_new:N \l_regex_catcodes_saved_int
+\int_new:N \l_regex_min_index_int
+\int_new:N \l_regex_max_index_int
\int_new:N \l_regex_nesting_int
-\int_new:N \l_regex_min_step_int
-\int_new:N \l_regex_max_step_int
-\int_new:N \l_regex_current_step_int
-\int_new:N \l_regex_start_step_int
-\int_new:N \l_regex_success_step_int
-\int_new:N \l_regex_unique_id_int
+\int_new:N \l_regex_current_index_int
+\int_new:N \l_regex_start_index_int
+\int_new:N \l_regex_success_index_int
\int_new:N \l_regex_current_char_int
\int_new:N \l_regex_current_catcode_int
-\tl_new:N \l_regex_current_token_tl
+\tl_new:N \l_regex_current_token_tl
\int_new:N \l_regex_last_char_int
\int_new:N \l_regex_case_changed_char_int
\bool_new:N \l_regex_caseless_bool
\int_new:N \l_regex_current_state_int
\prop_new:N \l_regex_current_submatches_prop
\prop_new:N \l_regex_success_submatches_prop
-\int_new:N \l_regex_max_index_int
+\int_new:N \l_regex_step_int
+\int_new:N \l_regex_max_active_int
\tl_new:N \l_regex_every_match_tl
+\bool_new:N \l_regex_fresh_thread_bool
+\bool_new:N \l_regex_success_empty_bool
+\cs_new_eq:NN \regex_if_two_empty_matches:F \use:n
\bool_new:N \g_regex_success_bool
\bool_new:N \l_regex_saved_success_bool
\bool_new:N \l_regex_success_match_bool
-\cs_new_protected:Npn \regex_last_match_empty_no:F #1 {#1}
-\cs_new_protected:Npn \regex_last_match_empty_yes:F
- { \int_compare:nNnF \l_regex_start_step_int = \l_regex_current_step_int }
-\cs_new_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F
-\bool_new:N \l_regex_success_empty_bool
-\bool_new:N \l_regex_fresh_thread_bool
-\int_new:N \l_regex_replacement_int
+\tl_const:Nn \c_regex_no_match_regex
+ {
+ \regex_nfa:Nw \c_regex_no_match_regex
+ \l_regex_max_state_int = \c_one
+ \l_regex_capturing_group_int = \c_zero
+ \tex_toks:D \c_zero { \s_stop }
+ \s_stop
+ }
+\cs_new_protected:Npn \regex_new:N #1
+ {
+ \chk_if_free_cs:N #1
+ \cs_gset_eq:NN #1 \c_regex_no_match_regex
+ }
+\regex_new:N \l_regex_internal_regex
\int_new:N \l_regex_replacement_csnames_int
+\int_new:N \l_regex_replacement_int
\int_new:N \l_regex_submatch_int
\int_new:N \l_regex_submatch_start_int
\int_new:N \l_regex_match_count_int
+\flag_new:n { regex_begin }
+\flag_new:n { regex_end }
\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
{
\cs_set_nopar:Npx \regex_tmp:w \s_stop { \s_stop #2 }
@@ -198,11 +193,22 @@
\tex_toks:D #1 \exp_after:wN
{ \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w }
}
-\cs_new:Npn \regex_query_substr:nn #1#2
+\cs_new:Npn \regex_toks_range:nn #1#2
+ {
+ \exp_after:wN \regex_toks_range:ww
+ \int_use:N \int_eval:w #1 \exp_after:wN ;
+ \int_use:N \int_eval:w #2 ;
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \regex_toks_range:ww #1 ; #2 ;
{
- \if_num:w \int_eval:w #1 < \int_eval:w #2 \int_eval_end:
- \str_aux_toks_range:nn {#1} {#2}
+ \if_num:w #1 < #2 \exp_stop_f:
+ \else:
+ \exp_after:wN \prg_map_break:
\fi:
+ \tex_the:D \tex_toks:D #1 \exp_stop_f:
+ \exp_after:wN \regex_toks_range:ww
+ \int_use:N \int_eval:w #1 + \c_one ; #2 ;
}
\cs_new:Npn \regex_query_submatch:nn #1#2
{
@@ -213,41 +219,63 @@
}
\cs_new:Npn \regex_query_submatch:w #1 ;
{
- \regex_query_substr:nn
+ \regex_toks_range:nn
{ \tex_skip:D #1 }
{ \etex_gluestretch:D \tex_skip:D #1 }
}
\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2
{
- \seq_pop:NN #1 \l_regex_tmpa_tl
- \int_set:Nn #2 \l_regex_tmpa_tl
+ \seq_pop:NN #1 \l_regex_internal_a_tl
+ \int_set:Nn #2 \l_regex_internal_a_tl
}
\cs_new_protected:Npn \regex_seq_get_int:NN #1#2
{
- \seq_get:NN #1 \l_regex_tmpa_tl
- \int_set:Nn #2 \l_regex_tmpa_tl
+ \seq_get:NN #1 \l_regex_internal_a_tl
+ \int_set:Nn #2 \l_regex_internal_a_tl
}
\cs_new_protected:Npn \regex_seq_push_int:NN #1#2
{ \seq_push:No #1 { \int_use:N #2 } }
\cs_new_protected:Npn \regex_seq_pop_use:N #1
{
- \seq_pop:NN #1 \l_regex_tmpa_tl
- \l_regex_tmpa_tl
+ \seq_pop:NN #1 \l_regex_internal_a_tl
+ \l_regex_internal_a_tl
}
\cs_new_protected:Npn \regex_seq_get_use:N #1
{
- \seq_get:NN #1 \l_regex_tmpa_tl
- \l_regex_tmpa_tl
+ \seq_get:NN #1 \l_regex_internal_a_tl
+ \l_regex_internal_a_tl
}
-\cs_new_protected:Npn \regex_break_true:w
- #1 \regex_break_point:TF #2 #3 {#2}
-\cs_new_protected:Npn \regex_break_point:TF #1 #2 { #2 }
-\cs_new_protected_nopar:Npn \regex_item_dot:
+\cs_new_protected:Npn \regex_get_digits:nw #1
{
- \if_num:w \l_regex_current_char_int > - \c_two
- \exp_after:wN \regex_break_true:w
+ \tex_afterassignment:D \regex_tmp:w
+ \cs_set_nopar:Npx \regex_tmp:w
+ {
+ \exp_not:n {#1}
+ { \if_false: } } \fi:
+ \regex_get_digits_aux:NN
+ }
+\cs_new:Npn \regex_get_digits_aux:NN #1#2
+ {
+ \if_meaning:w \regex_build_raw:N #1
+ \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f:
+ #2
+ \else:
+ \regex_get_digits_end:w #1 #2
+ \fi:
+ \else:
+ \regex_get_digits_end:w #1 #2
\fi:
+ \regex_get_digits_aux:NN
+ }
+\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
+ {
+ \fi: #2
+ \if_false: { { \fi: } }
+ #1
}
+\cs_new_protected:Npn \regex_break_true:w
+ #1 \regex_break_point:TF #2 #3 {#2}
+\cs_new_protected:Npn \regex_break_point:TF #1 #2 { #2 }
\cs_new_protected:Npn \regex_item_caseful_equal:n #1
{
\if_num:w #1 = \l_regex_current_char_int
@@ -316,16 +344,19 @@
\cs_set:Npn \regex_item_range:nn { \regex_item_caseful_range:nn }
\cs_set:Npn \regex_item_geq:n { \regex_item_caseful_geq:n }
}
+\cs_new_protected:Npn \regex_item_catcode_aux:
+ {
+ "
+ \if_case:w \l_regex_current_catcode_int
+ 1 \or: 4 \or: 10 \or: 40
+ \or: 100 \or: \or: 1000 \or: 4000
+ \or: 10000 \or: \or: 100000 \or: 400000
+ \or: 1000000 \or: 4000000 \else: 1*\c_zero
+ \fi:
+ }
\cs_new_protected:Npn \regex_item_catcode:nT #1
{
- \if_int_odd:w \int_eval:w #1 / "
- \if_case:w \l_regex_current_catcode_int
- 1 \or: 4 \or: 10 \or: 40
- \or: 100 \or: \or: 1000 \or: 4000
- \or: 10000 \or: \or: 100000 \or: 400000
- \or: 1000000 \or: 4000000 \else: 1*\c_zero
- \fi:
- \int_eval_end:
+ \if_int_odd:w \int_eval:w #1 / \regex_item_catcode_aux: \int_eval_end:
\exp_after:wN \use:n
\else:
\exp_after:wN \use_none:n
@@ -335,9 +366,9 @@
{
\int_compare:nNnT \l_regex_current_catcode_int = \c_zero
{
- \tl_set:Nn \l_regex_tmpa_regex_tl {#1}
+ \tl_set:Nn \l_regex_internal_regex {#1}
\bool_set_eq:NN \l_regex_saved_success_bool \g_regex_success_bool
- \exp_args:NNx \regex_match:NnTF \l_regex_tmpa_regex_tl
+ \exp_args:NNx \regex_match:NnTF \l_regex_internal_regex
{
\exp_after:wN \exp_after:wN
\exp_after:wN \cs_to_str:N \l_regex_current_token_tl
@@ -349,43 +380,140 @@
{ \bool_gset_eq:NN \g_regex_success_bool \l_regex_saved_success_bool }
}
}
-\cs_new_protected:Npn \regex_get_digits:nw #1
+\cs_new_protected:Npn \regex_escape_use:nnnn #1#2#3#4
{
- \tex_afterassignment:D \regex_tmp:w
- \cs_set_nopar:Npx \regex_tmp:w
+ \group_begin:
+ \cs_set_nopar:Npn \regex_escape_unescaped:N ##1 { #1 }
+ \cs_set_nopar:Npn \regex_escape_escaped:N ##1 { #2 }
+ \cs_set_nopar:Npn \regex_escape_raw:N ##1 { #3 }
+ \int_set:Nn \tex_escapechar:D { 92 }
+ \str_gset_other:Nn \g_regex_internal_tl {#4}
+ \int_zero:N \l_regex_internal_a_int
+ \tl_set:Nx \l_regex_internal_a_tl
+ {
+ \exp_after:wN \regex_escape_loop:N \g_regex_internal_tl
+ { break } \prg_break_point:n { }
+ }
+ \use:x
+ {
+ \group_end:
+ \regex_toks_range:nn \c_zero \l_regex_internal_a_int
+ \exp_not:o \l_regex_internal_a_tl
+ }
+ }
+\cs_new:Npn \regex_escape_loop:N #1
+ {
+ \cs_if_exist_use:cF { regex_escape_\token_to_str:N #1:w }
+ { \regex_escape_unescaped:N #1 }
+ \regex_escape_loop:N
+ }
+\cs_new_nopar:cpn { regex_escape_ \c_backslash_str :w }
+ \regex_escape_loop:N #1
+ {
+ \cs_if_exist_use:cF { regex_escape_/\token_to_str:N #1:w }
+ { \regex_escape_escaped:N #1 }
+ \regex_escape_loop:N
+ }
+\cs_new_eq:NN \regex_escape_unescaped:N ?
+\cs_new_eq:NN \regex_escape_escaped:N ?
+\cs_new_eq:NN \regex_escape_raw:N ?
+\cs_new_eq:NN \regex_escape_break:w \prg_map_break:
+\cs_new_nopar:cpn { regex_escape_/break:w }
+ {
+ \if_false: { \fi: }
+ \msg_kernel_error:nn { regex } { trailing-backslash }
+ \exp_after:wN \use_none:n \exp_after:wN { \if_false: } \fi:
+ }
+\cs_new_nopar:cpn { regex_escape_~:w } { }
+\cs_new_nopar:cpx { regex_escape_/a:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^G }
+\cs_new_nopar:cpx { regex_escape_/t:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^I }
+\cs_new_nopar:cpx { regex_escape_/n:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^J }
+\cs_new_nopar:cpx { regex_escape_/f:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^L }
+\cs_new_nopar:cpx { regex_escape_/r:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^M }
+\cs_new_nopar:cpx { regex_escape_/e:w }
+ { \exp_not:N \regex_escape_raw:N \iow_char:N \^^[ }
+\cs_new_nopar:cpn { regex_escape_/x:w } \regex_escape_loop:N
+ {
+ \if_false: { \fi: }
+ \tex_toks:D \l_regex_internal_a_int
+ \exp_after:wN { \l_regex_internal_a_tl }
+ \int_incr:N \l_regex_internal_a_int
+ \int_zero:N \l_regex_current_char_int
+ \regex_escape_x_test:N
+ }
+\cs_new_protected:Npx \regex_escape_x_test:N #1
+ {
+ \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
+ { \exp_not:N \regex_escape_x_test:N }
{
- \exp_not:n {#1}
- { \if_false: } } \fi:
- \regex_get_digits_aux:NN
+ \l_regex_current_char_int = "0
+ \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
+ { \exp_not:N \regex_escape_x_braced_loop:N }
+ { \exp_not:N \regex_escape_x_unbraced_i:N #1 }
+ }
}
-\cs_new:Npn \regex_get_digits_aux:NN #1#2
+\cs_new:Npn \regex_escape_x_unbraced_i:N #1
{
- \if_meaning:w \regex_build_raw:N #1
- \if_charcode:w \c_space_token \exp_not:N #2
- \else:
- \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f:
- #2
- \else:
- \regex_get_digits_end:w #1 #2
- \fi:
- \fi:
- \else:
- \regex_get_digits_end:w #1 #2
- \fi:
- \regex_get_digits_aux:NN
+ \str_aux_hexadecimal_use:NTF #1
+ { \regex_escape_x_unbraced_ii:N }
+ { \exp_stop_f: \regex_escape_x_end: #1 }
}
-\cs_new:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN
+\cs_new:Npn \regex_escape_x_unbraced_ii:N #1
{
- \fi: #2
- \if_false: { { \fi: } }
- #1
+ \token_if_eq_charcode:NNTF \c_space_token #1
+ { \regex_escape_x_unbraced_ii:N }
+ {
+ \str_aux_hexadecimal_use:NTF #1
+ { \exp_stop_f: \regex_escape_x_end: }
+ { \exp_stop_f: \regex_escape_x_end: #1 }
+ }
+ }
+\cs_new:Npn \regex_escape_x_braced_loop:N #1
+ {
+ \token_if_eq_charcode:NNTF \c_space_token #1
+ { \regex_escape_x_braced_loop:N }
+ {
+ \str_aux_hexadecimal_use:NTF #1
+ { \regex_escape_x_braced_loop:N }
+ { \exp_stop_f: \regex_escape_x_braced_end:N #1 }
+ }
}
-\prg_new_conditional:Npnn \regex_token_if_other_digit:N #1 { TF }
+\cs_new_protected:Npx \regex_escape_x_braced_end:N #1
{
- \if_num:w \c_nine < 1 \exp_not:N #1 \exp_stop_f:
- \prg_return_true: \else: \prg_return_false: \fi:
+ \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
+ { \exp_not:N \regex_escape_x_end: }
+ {
+ \msg_kernel_error:nnx { regex } { x-missing-rbrace }
+ { \int_use:N \l_regex_current_char_int }
+ \exp_not:N \regex_escape_x_end: #1
+ }
}
-\prg_new_conditional:Npnn \regex_aux_char_if_special:N #1 { TF }
+\group_begin:
+ \char_set_catcode_other:N \^^@
+ \cs_new_protected_nopar:Npn \regex_escape_x_end:
+ {
+ \if_num:w \l_regex_current_char_int > \c_max_char_int
+ \msg_kernel_error:nnx { regex } { x-overflow }
+ { \int_use:N \l_regex_current_char_int }
+ \exp_after:wN \use:n
+ \else:
+ \tex_lccode:D \c_zero \l_regex_current_char_int
+ \exp_after:wN \tl_to_lowercase:n
+ \fi:
+ {
+ \tl_set:Nx \l_regex_internal_a_tl
+ { \if_false: } \fi:
+ \regex_escape_raw:N ^^@
+ \regex_escape_loop:N
+ }
+ }
+\group_end:
+\prg_new_conditional:Npnn \regex_char_if_special:N #1 { TF }
{
\if_num:w `#1 < \c_ninety_one
\if_num:w `#1 < \c_fifty_eight
@@ -407,7 +535,7 @@
\fi:
\fi:
}
-\prg_new_conditional:Npnn \regex_aux_char_if_alphanumeric:N #1 { TF }
+\prg_new_conditional:Npnn \regex_char_if_alphanumeric:N #1 { TF }
{
\if_num:w `#1 < \c_ninety_one
\if_num:w `#1 < \c_fifty_eight
@@ -434,6 +562,18 @@
\exp_after:wN \use_ii:nn
\fi:
}
+\cs_new_nopar:Npn \regex_build_if_assertions_forbidden:TF
+ {
+ \if_int_odd:w \l_regex_build_mode_int
+ \exp_after:wN \use_i:nn
+ \else:
+ \if_int_compare:w \l_regex_build_mode_int > \c_zero
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_i:nn
+ \else:
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_ii:nn
+ \fi:
+ \fi:
+ }
\cs_new_protected_nopar:Npn \regex_build_new_state:
{
\tex_toks:D \l_regex_max_state_int { \s_stop }
@@ -456,61 +596,35 @@
#3 { \int_eval:n { #4 - \l_regex_left_state_int } }
}
}
-\cs_new_protected:Npn \regex_build:n #1
- {
- \int_set_eq:NN \l_regex_build_mode_int \c_zero
- \regex_build:w
- \regex_build_new_state:
- \regex_toks_put_right:Nx \l_regex_left_state_int
- { \regex_action_start_wildcard: }
- \regex_build_open_aux:
- \str_escape_use:NNNn
- \regex_build_i_unescaped:N
- \regex_build_i_escaped:N
- \regex_build_i_raw:N
- { #1 }
- \prg_do_nothing: \prg_do_nothing:
- \prg_do_nothing: \prg_do_nothing:
- \regex_build_end:
- }
-\cs_new:Npn \regex_build_i_unescaped:N #1
- {
- \regex_aux_char_if_special:NTF #1
- { \exp_not:N \regex_build_special:N #1 }
- { \exp_not:N \regex_build_raw:N #1 }
- }
-\cs_new:Npn \regex_build_i_escaped:N #1
- {
- \regex_aux_char_if_alphanumeric:NTF #1
- { \exp_not:N \regex_build_escaped:N #1 }
- { \exp_not:N \regex_build_raw:N #1 }
- }
-\cs_new:Npn \regex_build_i_raw:N #1
- { \exp_not:N \regex_build_raw:N #1 }
-\cs_new_protected:Npn \regex_build_special:N #1
- {
- \cs_if_exist_use:cF { regex_build_#1: }
- { \regex_build_raw:N #1 }
- }
-\cs_new_protected:Npn \regex_build_escaped:N #1
- {
- \cs_if_exist_use:cF { regex_build_/#1: }
- { \regex_build_raw:N #1 }
- }
\cs_new_protected_nopar:Npn \regex_build:w
{
\int_set_eq:NN \l_regex_catcodes_default_int \c_regex_catcodes_all_int
- \int_set_eq:NN \l_regex_capturing_group_int \c_zero
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
+ \int_set_eq:NN \l_regex_capturing_group_int \c_zero
+ \tl_clear:N \l_regex_tests_tl
+ \bool_set_true:N \l_regex_tests_bool
\int_zero:N \l_regex_max_state_int
\regex_build_new_state:
- \tl_clear:N \l_regex_class_tl
- \bool_set_true:N \l_regex_class_bool
- \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
}
\cs_new_protected_nopar:Npn \regex_build_end:
{
- \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
- \regex_build_close_aux: \regex_build_group_:
+ \regex_build_exit_class:
+ \regex_build_exit_groups:
+ \regex_toks_put_right:Nx \l_regex_right_state_int
+ { \regex_action_success: }
+ \int_incr:N \l_regex_capturing_group_int
+ }
+\cs_new_protected_nopar:Npn \regex_build_exit_class:
+ {
+ \regex_build_if_in_class:TF
+ {
+ \msg_kernel_error:nn { regex } { missing-rbrack }
+ \use:c { regex_build_]: }
+ }
+ { }
+ }
+\cs_new_protected_nopar:Npn \regex_build_exit_groups:
+ {
\seq_if_empty:NF \l_regex_capturing_group_seq
{
\msg_kernel_error:nnx { regex } { missing-rparen }
@@ -519,23 +633,61 @@
{ \seq_length:N \l_regex_capturing_group_seq }
{ \regex_build_close_aux: \regex_build_group_: }
}
- \regex_toks_put_right:Nx \l_regex_right_state_int
- { \regex_action_success: }
- \int_incr:N \l_regex_capturing_group_int
+ }
+\cs_new_protected:Npn \regex_build:n #1
+ {
+ \regex_build:w
+ \int_set_eq:NN \l_regex_build_mode_int \c_zero
+ \regex_build_new_state:
+ \regex_toks_put_right:Nx \l_regex_left_state_int
+ { \regex_action_start_wildcard: }
+ \regex_build_open_aux:
+ \regex_escape_use:nnnn
+ {
+ \regex_char_if_special:NTF ##1
+ \regex_build_special:N \regex_build_raw:N ##1
+ }
+ {
+ \regex_char_if_alphanumeric:NTF ##1
+ \regex_build_escaped:N \regex_build_raw:N ##1
+ }
+ { \regex_build_raw:N ##1 }
+ { #1 }
+ \prg_do_nothing: \prg_do_nothing:
+ \prg_do_nothing: \prg_do_nothing:
+ \int_compare:nNnT \l_regex_build_mode_int < \c_zero
+ {
+ \msg_kernel_error:nn { regex } { c-missing-rbrace }
+ \exp_after:wN \regex_build_special:N \c_rbrace_str
+ }
+ \seq_put_right:Nn \l_regex_capturing_group_seq {0}
+ \regex_build_close_aux: \regex_build_group_:
+ \regex_build_end:
+ }
+\cs_new_protected:Npn \regex_build_special:N #1
+ {
+ \cs_if_exist_use:cF { regex_build_#1: }
+ { \regex_build_raw:N #1 }
+ }
+\cs_new_protected:Npn \regex_build_escaped:N #1
+ {
+ \cs_if_exist_use:cF { regex_build_/#1: }
+ { \regex_build_raw:N #1 }
}
\cs_new_protected:Npn \regex_build_one:n #1
{ \regex_build_one:x { \exp_not:n {#1} } }
\cs_new_protected:Npn \regex_build_one:x #1
{
- \tl_put_right:Nx \l_regex_class_tl
+ \tl_put_right:Nx \l_regex_tests_tl
{
- \if_num:w \l_regex_catcodes_int = \c_regex_catcodes_all_int
- \exp_after:wN \use:n
- \else:
+ \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int
\regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ \else:
+ \exp_after:wN \use:n
\fi:
{#1}
}
+ \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
\if_num:w \l_regex_build_mode_int = \c_two
\l_regex_build_mode_int = \c_zero
\else:
@@ -547,28 +699,21 @@
\exp_after:wN \regex_build_one_quantifier:
\fi:
}
-\cs_new:Npn \regex_build_tmp_class:n #1
+\cs_new:Npn \regex_tests_action_cost:n #1
{
- \exp_not:o \l_regex_class_tl
- \bool_if:NTF \l_regex_class_bool
+ \exp_not:o \l_regex_tests_tl
+ \bool_if:NTF \l_regex_tests_bool
{ \regex_break_point:TF { \regex_action_cost:n {#1} } { } }
{ \regex_break_point:TF { } { \regex_action_cost:n {#1} } }
}
-\cs_new_protected:Npn \regex_class_put:N #1
- {
- \tl_put_right:Nx \l_regex_class_tl
- { \regex_item_equal:n { \int_value:w `#1 ~ } }
- }
-\cs_new_protected:Npn \regex_class_put:NN #1#2
- {
- \tl_put_right:Nx \l_regex_class_tl
- { \regex_item_range:nn { \int_value:w `#1 ~ } { \int_value:w `#2 ~ } }
- }
-\cs_new_protected:Npn \regex_build_raw_alphanum_is_error:N #1
+\cs_new_protected:Npn \regex_build_raw_error:N #1
{
- \regex_aux_char_if_alphanumeric:NTF #1
- { \msg_kernel_error:nnx { regex } { class-bad-escape } { #1 } }
- { \regex_build_raw:N #1 }
+ \if_int_odd:w \l_regex_build_mode_int
+ \msg_kernel_error:nnx { regex } { class-bad-escape } {#1}
+ \else:
+ \msg_kernel_error:nnx { regex } { catcode-bad-escape } {#1}
+ \fi:
+ \regex_build_raw:N #1
}
\cs_new_protected:Npn \regex_build_raw:N #1#2#3
{
@@ -586,39 +731,45 @@
\token_if_eq_meaning:NNTF #2 \regex_build_escaped:N % [
{ \use_i:nn } { \str_if_eq:nnTF { #2#3 } { \regex_build_special:N ] } }
{
- \msg_kernel_warning:nnx { regex } { end-range-missing } % [
- { #1 - \if_meaning:w #3 ] \else: \c_backslash_str \fi: #3 }
- \regex_class_put:N #1
- \regex_class_put:N -
+ \msg_kernel_warning:nnxx { regex } { range-missing-end } % [
+ {#1} { \token_if_eq_charcode:NNF #3 ] { \c_backslash_str } #3 }
+ \tl_put_right:Nx \l_regex_tests_tl
+ {
+ \regex_item_equal:n { \int_value:w `#1 ~ }
+ \regex_item_equal:n { \int_value:w `- ~ }
+ }
#2#3
}
{
\if_num:w `#1 > `#3 \exp_stop_f:
- \msg_kernel_error:nnxx { regex } { backwards-range } {#1} {#3}
+ \msg_kernel_error:nnxx { regex } { range-backwards } {#1} {#3}
\else:
- \if_num:w `#1 = `#3 \exp_stop_f:
- \regex_class_put:N #3
- \else:
- \regex_class_put:NN #1#3
- \fi:
+ \tl_put_right:Nx \l_regex_tests_tl
+ {
+ \if_num:w `#1 = `#3 \exp_stop_f:
+ \regex_item_equal:n
+ \else:
+ \regex_item_range:nn { \int_value:w `#1 ~ }
+ \fi:
+ { \int_value:w `#3 ~ }
+ }
\fi:
}
}
-\cs_new_protected_nopar:cpn { regex_build_.: }
+\cs_new_protected_nopar:cpx { regex_build_.: }
{
- \regex_build_if_in_class:TF
+ \exp_not:N \regex_build_if_in_class:TF
{ \regex_build_raw:N . }
- { \regex_build_one:x \regex_item_dot: }
+ { \regex_build_one:n \exp_not:c { c_regex_._tl } }
}
\tl_map_inline:nn { dDhHsSvVwWN }
{
\cs_new_protected_nopar:cpx { regex_build_/#1: }
{ \regex_build_one:n \exp_not:c { c_regex_#1_tl } }
}
-\cs_new_protected:Npn \regex_build_simple_assertion:Nn #1#2
+\cs_new_protected:Npn \regex_build_simple_assertion:nn #1#2
{
- \regex_build_if_in_class:TF
- { \regex_build_raw_alphanum_is_error:N #1 }
+ \regex_build_if_assertions_forbidden:TF {#1}
{
\regex_build_new_state:
\regex_toks_put_right:Nx \l_regex_left_state_int
@@ -632,47 +783,50 @@
}
}
}
- %^^A \regex_assertion_quantifier:
+ %^^A todo: add \regex_assertion_quantifier:
}
}
\cs_new_protected_nopar:cpn { regex_build_^: }
{
- \regex_build_simple_assertion:Nn ^
- { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int }
+ \regex_build_simple_assertion:nn { \regex_build_raw:N ^ }
+ { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int }
}
\cs_new_protected_nopar:cpn { regex_build_/A: }
{
- \regex_build_simple_assertion:Nn A
- { \int_compare:nNnT \l_regex_min_step_int = \l_regex_current_step_int }
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N A }
+ { \int_compare:nNnT \l_regex_min_index_int = \l_regex_current_index_int }
}
\cs_new_protected_nopar:cpn { regex_build_/G: }
{
- \regex_build_simple_assertion:Nn G
- { \int_compare:nNnT \l_regex_start_step_int = \l_regex_current_step_int }
- }
-\cs_new_protected_nopar:cpn { regex_build_$: } % $
- {
- \regex_build_simple_assertion:Nn $ % $
- { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N G }
+ { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int }
}
+\group_begin:
+ \char_set_catcode_other:N \$
+ \cs_new_protected_nopar:cpn { regex_build_$: } % $
+ {
+ \regex_build_simple_assertion:nn { \regex_build_raw:N $ } % $
+ { \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
+ }
+\group_end:
\cs_new_protected_nopar:cpn { regex_build_/Z: }
{
- \regex_build_simple_assertion:Nn Z
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N Z }
{ \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
}
\cs_new_protected_nopar:cpn { regex_build_/z: }
{
- \regex_build_simple_assertion:Nn z
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N z }
{ \int_compare:nNnT \l_regex_current_char_int < \c_minus_one }
}
\cs_new_protected_nopar:cpn { regex_build_/b: }
{
- \regex_build_simple_assertion:Nn b
- { \regex_if_word_boundary:TF \use:n \use_none:n }
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N b }
+ { \regex_if_word_boundary:T }
}
\cs_new_protected_nopar:cpn { regex_build_/B: }
{
- \regex_build_simple_assertion:Nn B
+ \regex_build_simple_assertion:nn { \regex_build_raw_error:N B }
{ \regex_if_word_boundary:TF { } }
}
\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF
@@ -685,6 +839,8 @@
{ \group_end: \c_regex_w_tl }
\regex_break_point:TF
}
+\cs_new_protected_nopar:Npn \regex_if_word_boundary:T
+ { \regex_if_word_boundary:TF \use:n \use_none:n }
\cs_new_protected_nopar:cpn { regex_build_[: }
{
\regex_build_if_in_class:TF
@@ -696,21 +852,22 @@
\regex_build_if_in_class:TF
{
\if_num:w \l_regex_build_mode_int > \c_sixteen
- \tl_set:Nx \l_regex_class_tl
+ \tl_set:Nx \l_regex_tests_tl
{
- \exp_not:o \l_regex_class_saved_tl
- \if_num:w \l_regex_catcodes_int < \c_regex_catcodes_all_int
- \regex_item_catcode:nT { \int_use:N \l_regex_catcodes_int }
+ \exp_not:o \l_regex_tests_saved_tl
+ \if_num:w \l_regex_catcodes_saved_int < \c_regex_catcodes_all_int
+ \regex_item_catcode:nT
+ { \int_use:N \l_regex_catcodes_saved_int }
\else:
\exp_after:wN \use:n
\fi:
{
- \exp_not:o \l_regex_class_tl
- \bool_if:NF \l_regex_class_bool
- { \regex_break_point:TF { } { \regex_break_true:w } }
+ \exp_not:o \l_regex_tests_tl
+ \bool_if:NF \l_regex_tests_bool
+ { \regex_break_point:TF { } \regex_break_true:w }
}
}
- \bool_set_eq:NN \l_regex_class_bool \l_regex_class_saved_bool
+ \bool_set_eq:NN \l_regex_tests_bool \l_regex_tests_saved_bool
\fi:
\tex_advance:D \l_regex_build_mode_int - \c_fifteen
\tex_divide:D \l_regex_build_mode_int \c_thirteen
@@ -724,14 +881,16 @@
{
\l_regex_build_mode_int = \int_value:w \l_regex_build_mode_int 3 ~ %
\if_num:w \l_regex_build_mode_int > \c_sixteen
- \tl_set_eq:NN \l_regex_class_saved_tl \l_regex_class_tl
- \bool_set_eq:NN \l_regex_class_saved_bool \l_regex_class_bool
+ \tl_set_eq:NN \l_regex_tests_saved_tl \l_regex_tests_tl
+ \bool_set_eq:NN \l_regex_tests_saved_bool \l_regex_tests_bool
+ \int_set_eq:NN \l_regex_catcodes_saved_int \l_regex_catcodes_int
+ \int_set_eq:NN \l_regex_catcodes_int \c_regex_catcodes_all_int
\fi:
\token_if_eq_meaning:NNTF #1 \regex_build_special:N
{
\token_if_eq_charcode:NNTF #2 ^
{
- \bool_set_false:N \l_regex_class_bool % [
+ \bool_set_false:N \l_regex_tests_bool % [
\str_if_eq:nnTF {#3#4} { \regex_build_special:N ] }
{ \regex_build_raw:N }
}
@@ -766,8 +925,7 @@
}
{ \cs_if_exist_use:cF { regex_build_c_#3: } }
{
- \msg_kernel_error:nnxx { regex } { c-command }
- { regular~expression } { #3 }
+ \msg_kernel_error:nnx { regex } { c-missing-category } {#3}
#2 #3
}
}
@@ -808,8 +966,7 @@
{ \regex_build_c_lbrack_end: }
}
{
- \msg_kernel_error:nnxx { regex } { c-command }
- { regular-expression } { #2 }
+ \msg_kernel_error:nnx { regex } { c-missing-rbrack } {#2}
\regex_build_c_lbrack_end:
}
}
@@ -830,18 +987,21 @@
\fi:
}
\cs_new_protected:cpn { regex_build_ \c_rbrace_str : }
- { %^^A incorrect recovery for \c{...(...[...} with missing ) or ].
+ {
\int_compare:nNnTF \l_regex_build_mode_int < \c_zero
{
- \use:c { regex_build_/Z: }
- \regex_toks_put_right:Nx \l_regex_right_state_int
- { \regex_action_success: }
- \int_incr:N \l_regex_capturing_group_int
+ \regex_build_exit_class:
+ \regex_build_exit_groups:
+ \regex_build_escaped:N Z
+ \regex_build_end:
\use:x
{
\group_end:
\regex_build_one:n
- { \regex_item_cs:n { \regex_set_aux:N ? } }
+ {
+ \regex_item_cs:n
+ { \regex_set_aux:N \l_regex_internal_regex }
+ }
}
}
{ \exp_after:wN \regex_build_raw:N \c_rbrace_str }
@@ -913,7 +1073,7 @@
\tl_if_empty:nTF {#2}
{ \regex_build_quantifier_aux:nnNN { {#1} } { n* } }
{ \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } }
- } %^^A todo: catch the case m>n
+ }
{
\regex_build_quantifier_end:nn { } { }
\use:x
@@ -928,82 +1088,101 @@
\cs_new_protected:Npn \regex_build_quantifier_end:nn #1#2
{
\use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } #2
- \tl_clear:N \l_regex_class_tl
- \bool_set_true:N \l_regex_class_bool
- \int_set_eq:NN \l_regex_catcodes_int \l_regex_catcodes_default_int
+ \tl_clear:N \l_regex_tests_tl
+ \bool_set_true:N \l_regex_tests_bool
}
\cs_new_protected_nopar:Npn \regex_build_one_quantifier:
{
\tl_set:Nx \l_regex_one_or_group_tl { one }
\regex_build_quantifier:w
}
-\cs_new_protected_nopar:Npn \regex_build_one_:
- {
- \regex_build_transition:NN
- \regex_build_tmp_class:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_one_?: }
- {
- \regex_build_transitions:NNNN
- \regex_build_tmp_class:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_one_??: }
- {
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_tmp_class:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_one_*: }
- {
- \regex_build_transitions:NNNN
- \regex_build_tmp_class:n \l_regex_left_state_int
- \regex_action_free:n \l_regex_right_state_int
- }
-\cs_new_protected_nopar:cpn { regex_build_one_*?: }
+\cs_new_protected:Npn \regex_build_one_n: #1
{
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_tmp_class:n \l_regex_left_state_int
+ \prg_replicate:nn {#1}
+ {
+ \regex_build_transition:NN
+ \regex_tests_action_cost:n \l_regex_right_state_int
+ }
}
-\cs_new_protected_nopar:cpn { regex_build_one_+: }
+\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n:
+\cs_new_protected_nopar:Npn \regex_build_one_:
+ { \regex_build_one_n: \c_one }
+\cs_new_protected:cpn { regex_build_one_n*: } #1
{
- \regex_build_one_:
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_tmpa_int
- \regex_action_free:n \l_regex_right_state_int
+ \int_compare:nNnTF {#1} = \c_zero
+ {
+ \regex_build_transitions:NNNN
+ \regex_tests_action_cost:n \l_regex_left_state_int
+ \regex_action_free:n \l_regex_right_state_int
+ }
+ {
+ \regex_build_one_n: {#1}
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_internal_a_int
+ \regex_action_free:n \l_regex_right_state_int
+ }
}
-\cs_new_protected_nopar:cpn { regex_build_one_+?: }
+\cs_new_protected:cpn { regex_build_one_n*?: } #1
{
- \regex_build_one_:
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_tmpa_int
+ \int_compare:nNnTF {#1} = \c_zero
+ {
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_right_state_int
+ \regex_tests_action_cost:n \l_regex_left_state_int
+ }
+ {
+ \regex_build_one_n: {#1}
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_right_state_int
+ \regex_action_free:n \l_regex_internal_a_int
+ }
}
-\cs_new_protected:Npn \regex_build_one_n: #1
- { \prg_replicate:nn {#1} { \regex_build_one_: } }
-\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n:
-\cs_new_protected:cpx { regex_build_one_n*: } #1
- {
- \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: }
- \exp_not:c { regex_build_one_*: }
+\cs_new_protected_nopar:cpx { regex_build_one_*: }
+ { \exp_not:c { regex_build_one_n*: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_one_*?: }
+ { \exp_not:c { regex_build_one_n*?: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_one_+: }
+ { \exp_not:c { regex_build_one_n*: } \c_one }
+\cs_new_protected_nopar:cpx { regex_build_one_+?: }
+ { \exp_not:c { regex_build_one_n*?: } \c_one }
+\cs_new_protected_nopar:Npn \regex_build_one_nn:
+ {
+ \regex_build_one_nn_aux:nnnn { }
+ {
+ \regex_build_transitions:NNNN
+ \regex_tests_action_cost:n \l_regex_right_state_int
+ \regex_action_free:n \l_regex_internal_a_int
+ }
}
-\cs_new_protected:cpx { regex_build_one_n*?: } #1
+\cs_new_protected_nopar:cpn { regex_build_one_nn?: }
{
- \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: }
- \exp_not:c { regex_build_one_*?: }
+ \regex_build_one_nn_aux:nnnn {?}
+ {
+ \regex_build_transitions:NNNN
+ \regex_action_free:n \l_regex_internal_a_int
+ \regex_tests_action_cost:n \l_regex_right_state_int
+ }
}
-\cs_new_protected:Npn \regex_build_one_nn_aux:Nnn #1#2#3
+\cs_new_protected:Npn \regex_build_one_nn_aux:nnnn #1#2#3#4
{
- \prg_replicate:nn {#2} { \regex_build_one_: }
- \prg_replicate:nn {#3-#2} {#1}
+ \regex_build_one_n: {#3}
+ \int_compare:nNnTF {#3} > {#4}
+ {
+ \msg_kernel_error:nnxxx
+ { regex } { quantifier-backwards } {#3} {#4} {#1}
+ }
+ {
+ \int_set:Nn \l_regex_internal_a_int
+ { \l_regex_right_state_int + #4 - #3 }
+ \prg_replicate:nn { #4 - #3 } { #2 }
+ }
}
-\cs_new_protected_nopar:Npx \regex_build_one_nn:
- { \regex_build_one_nn_aux:Nnn \exp_not:c { regex_build_one_?: } }
-\cs_new_protected_nopar:cpx { regex_build_one_nn?: }
- { \regex_build_one_nn_aux:Nnn \exp_not:c { regex_build_one_??: } }
+\cs_new_protected_nopar:cpn { regex_build_one_?: }
+ { \regex_build_one_nn: \c_zero \c_one }
+\cs_new_protected_nopar:cpx { regex_build_one_??: }
+ { \exp_not:c { regex_build_one_nn?: } \c_zero \c_one }
\cs_new_protected:cpn { regex_build_(: } #1#2
{
\regex_build_if_in_class:TF
@@ -1060,7 +1239,7 @@
\regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int
\regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int
\regex_seq_pop_use:N \l_regex_end_group_seq
- \seq_pop:NN \l_regex_end_alternation_seq \l_regex_tmpa_tl
+ \seq_pop:NN \l_regex_end_alternation_seq \l_regex_internal_a_tl
}
\cs_new_protected_nopar:Npn \regex_build_begin_alternation:
{
@@ -1093,7 +1272,7 @@
{
\cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : }
{
- \msg_kernel_error:nnx { regex } { unsupported }
+ \msg_kernel_error:nnx { regex } { special-group-unknown }
{ (? \token_to_str:N #2 } %)
\regex_build_special:N ( % )
\regex_build_raw:N ?
@@ -1149,7 +1328,7 @@
\token_if_eq_meaning:NNTF \regex_build_raw:N #2
{
\cs_if_exist_use:cF { regex_build_option_#1#3: }
- { \msg_error:nnx { regex } { unknown-option } { #3 } }
+ { \msg_kernel_error:nnx { regex } { unknown-option } { #3 } }
\regex_build_options:NNN #1
}
{
@@ -1158,7 +1337,7 @@
{ ) } { }
{ - } { \regex_build_options:NNN - }
}
- { \msg_error:nnx { regex } { invalid-in-option } { #3 } }
+ { \msg_kernel_error:nnx { regex } { invalid-in-option } { #3 } }
}
}
\cs_new_protected_nopar:cpn { regex_build_option_+i: }
@@ -1176,53 +1355,52 @@
}
\cs_new_protected:Npn \regex_build_group_submatches:NN #1#2
{
- \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl
- \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero
+ \seq_pop:NN \l_regex_capturing_group_seq \l_regex_internal_a_tl
+ \int_compare:nNnF { \l_regex_internal_a_tl } < \c_zero
{
\regex_toks_put_left:Nx #1
- { \regex_action_submatch:n { \l_regex_tmpa_tl < } }
+ { \regex_action_submatch:n { \l_regex_internal_a_tl < } }
\regex_toks_put_left:Nx #2
- { \regex_action_submatch:n { \l_regex_tmpa_tl > } }
+ { \regex_action_submatch:n { \l_regex_internal_a_tl > } }
}
}
-\cs_new_protected_nopar:Npn \regex_build_group_:
- {
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \regex_build_transition:NN
- \regex_action_free:n \l_regex_right_state_int
- }
-\cs_new_protected:Npn \regex_build_group_shift:N #1
+\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:
{
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
\regex_build_new_state:
- \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_tmpa_int
+ \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_internal_a_int
\regex_toks_put_left:Nx \l_regex_right_state_int
{
- \int_set:Nn \l_regex_current_state_int
- { \int_use:N \l_regex_tmpa_int } % ^^A here we lie!
+ \int_sub:Nn \l_regex_current_state_int
+ {
+ \int_eval:n % ^^A here we lie!
+ { \l_regex_right_state_int - \l_regex_internal_a_int }
+ }
}
- \cs_set:Npx \regex_tmp:w
+ \use:x
{
- \tex_toks:D \l_regex_tmpa_int
+ \tex_toks:D \l_regex_internal_a_int
{
\s_stop
- #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_right_state_int - \l_regex_internal_a_int }
+ }
}
}
- \regex_tmp:w
\regex_build_group_submatches:NN
\l_regex_right_state_int \l_regex_left_state_int
}
\cs_new_protected:Npn \regex_build_group_qs_aux:NN #1#2
{
- \regex_build_group_shift:N \regex_action_free:n
+ \regex_build_group_qs_aux:
\int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
\regex_build_transition:NN \regex_action_free:n #1
- #2 \l_regex_tmpa_int
+ #2 \l_regex_internal_a_int
{
\regex_action_free:n
- { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
+ { \int_eval:n { \l_regex_right_state_int - \l_regex_internal_a_int } }
}
}
\cs_new_protected_nopar:cpn { regex_build_group_?: }
@@ -1235,99 +1413,241 @@
\regex_build_group_qs_aux:NN
\l_regex_right_state_int \regex_toks_put_left:Nx
}
-\cs_new_protected_nopar:cpn { regex_build_group_*: }
+\cs_new_protected:Npn \regex_build_group_transition_right_max:
{
- \regex_build_group_qs_aux:NN
- \l_regex_tmpa_int \regex_toks_put_right:Nx
+ \regex_toks_put_right:Nx \l_regex_right_state_int
+ {
+ \regex_action_free:n
+ { \int_eval:n { \l_regex_max_state_int - \l_regex_right_state_int } }
+ }
}
-\cs_new_protected_nopar:cpn { regex_build_group_*?: }
+\cs_new_protected:Npn \regex_build_group_n_aux:n #1
{
- \regex_build_group_qs_aux:NN
- \l_regex_tmpa_int \regex_toks_put_left:Nx
+ \int_set_eq:NN \l_regex_internal_a_int \l_regex_left_state_int
+ \int_set_eq:NN \l_regex_internal_b_int \l_regex_max_state_int
+ \int_add:Nn \l_regex_right_state_int
+ { ( #1 - 1 ) * ( \l_regex_max_state_int - \l_regex_left_state_int ) }
+ \int_set:Nn \l_regex_max_state_int
+ {
+ \l_regex_left_state_int
+ + (#1) * ( \l_regex_max_state_int - \l_regex_left_state_int )
+ }
+ \int_while_do:nNnn \l_regex_internal_b_int < \l_regex_max_state_int
+ {
+ \tex_toks:D \l_regex_internal_b_int
+ = \tex_toks:D \l_regex_internal_a_int
+ \int_incr:N \l_regex_internal_a_int
+ \int_incr:N \l_regex_internal_b_int
+ }
+ }
+\cs_new_protected:Npn \regex_build_group_n: #1
+ {
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n {#1}
+ \regex_build_new_state:
+ \int_compare:nNnF {#1} = \c_zero
+ {
+ \regex_build_group_submatches:NN
+ \l_regex_internal_a_int \l_regex_left_state_int
+ }
}
-\cs_new_protected_nopar:cpn { regex_build_group_+: }
+\cs_new_eq:cN { regex_build_group_n?: } \regex_build_group_n:
+\cs_new_protected_nopar:Npn \regex_build_group_:
+ { \regex_build_group_n: \c_one }
+\cs_new_protected_nopar:cpn { regex_build_group_n*: }
{
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_tmpa_int
- \regex_action_free:n \l_regex_right_state_int
+ \regex_build_group_n_star_aux:NNn
+ \regex_toks_put_right:Nx \regex_toks_put_left:Nx
}
-\cs_new_protected_nopar:cpn { regex_build_group_+?: }
+\cs_new_protected_nopar:cpn { regex_build_group_n*?: }
{
- \regex_build_group_submatches:NN
- \l_regex_left_state_int \l_regex_right_state_int
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_tmpa_int
+ \regex_build_group_n_star_aux:NNn
+ \regex_toks_put_left:Nx \regex_toks_put_right:Nx
}
-\cs_new_protected:Npn \regex_build_group_n_aux:n #1
+\cs_new_protected:Npn \regex_build_group_n_star_aux:NNn #1#2#3
{
- \regex_toks_put_right:Nx \l_regex_right_state_int
+ \int_compare:nNnTF {#3} = \c_zero
+ { \regex_build_group_qs_aux:NN \l_regex_internal_a_int #1 }
{
- \regex_action_free:n
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n {#3}
+ \regex_build_new_state:
+ #2 \l_regex_left_state_int
{
- \int_eval:n %^^A todo: document why that value.
- { \l_regex_max_state_int - \c_one - \l_regex_left_state_int }
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_internal_a_int - \l_regex_left_state_int }
+ }
}
+ \regex_build_group_submatches:NN
+ \l_regex_internal_a_int \l_regex_left_state_int
}
- \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int
- \int_set_eq:NN \l_regex_tmpb_int \l_regex_max_state_int
- \int_set:Nn \l_regex_max_state_int
+ }
+\cs_new_protected_nopar:cpx { regex_build_group_*: }
+ { \exp_not:c { regex_build_group_n*: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_group_*?: }
+ { \exp_not:c { regex_build_group_n*?: } \c_zero }
+\cs_new_protected_nopar:cpx { regex_build_group_+: }
+ { \exp_not:c { regex_build_group_n*: } \c_one }
+\cs_new_protected_nopar:cpx { regex_build_group_+?: }
+ { \exp_not:c { regex_build_group_n*?: } \c_one }
+\cs_new_protected_nopar:Npn \regex_build_group_nn:
+ {
+ \regex_build_group_nn_aux:nnnn { }
+ \regex_build_group_transition_right_max:
+ }
+\cs_new_protected_nopar:cpn { regex_build_group_nn?: }
+ {
+ \regex_build_group_nn_aux:nnnn { }
{
- \l_regex_left_state_int
- + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int )
+ \use:x
+ {
+ \tex_toks:D \l_regex_right_state_int
+ {
+ \exp_after:wN \regex_build_group_nn_aux_ii:w
+ \tex_the:D \tex_toks:D \l_regex_right_state_int
+ }
+ }
}
- \int_while_do:nNnn \l_regex_tmpb_int < \l_regex_max_state_int
+ }
+\cs_new_protected:Npn \regex_build_group_nn_aux:nnnn #1#2#3#4
+ {
+ \int_compare:nNnTF {#3} > {#4}
{
- \tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int
- \int_incr:N \l_regex_tmpa_int
- \int_incr:N \l_regex_tmpb_int
+ \msg_kernel_error:nnxxx
+ { regex } { quantifier-backwards } {#3} {#4} {#1}
+ }
+ {
+ \int_compare:nNnTF {#3} = \c_zero
+ {
+ \regex_build_group_qs_aux:
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n { #4 - #3 }
+ \int_set_eq:NN \l_regex_internal_c_int \l_regex_right_state_int
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int
+ \prg_replicate:nn { #4 - #3 }
+ {
+ #2
+ \int_add:Nn \l_regex_right_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ }
+ \int_set_eq:NN \l_regex_right_state_int \l_regex_internal_c_int
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_new_state:
+ }
+ {
+ \regex_build_group_transition_right_max:
+ \regex_build_group_n_aux:n {#3}
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_group_submatches:NN
+ \l_regex_left_state_int \l_regex_right_state_int
+ \regex_build_group_n_aux:n { #4 - #3 + \c_one }
+ \int_sub:Nn \l_regex_right_state_int
+ { \l_regex_internal_a_int - \l_regex_left_state_int }
+ \prg_replicate:nn { #4 - #3 }
+ {
+ #2
+ \int_add:Nn \l_regex_right_state_int
+ { \l_regex_internal_b_int - \l_regex_internal_a_int }
+ }
+ \int_set_eq:NN \l_regex_left_state_int \l_regex_internal_a_int
+ \regex_build_new_state:
+ }
}
}
-\cs_new_protected:Npn \regex_build_group_n: #1
- { % ^^A todo: catch case #1 <= 0.
- \regex_build_group_n_aux:n {#1}
- \regex_build_transition:NN
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_group_submatches:NN
- \l_regex_tmpa_int \l_regex_left_state_int
+\cs_new_protected:Npn \regex_build_group_nn_aux_ii:w #1 \regex_action_free:n
+ {
+ #1
+ \regex_action_free:n
+ {
+ \int_eval:n
+ { \l_regex_max_state_int - \l_regex_right_state_int }
+ }
+ \regex_action_free:n
}
-\cs_new_eq:cN { regex_build_group_n?: } \regex_build_group_n:
-\cs_new_protected:cpn { regex_build_group_n*: } #1
- { % ^^A todo: catch case #1 <= 0.
- \regex_build_group_n_aux:n {#1}
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_tmpa_int
- \regex_action_free:n \l_regex_right_state_int
- \regex_build_group_submatches:NN
- \l_regex_tmpa_int \l_regex_left_state_int
+\cs_new_protected:cpn { regex_build_/u: } #1#2
+ {
+ \regex_build_if_assertions_forbidden:TF
+ { \regex_build_raw_error:N u #1#2 }
+ {
+ \str_if_eq:xxTF {#1#2} { \regex_build_special:N \c_lbrace_str }
+ {
+ \tl_set:Nx \l_regex_internal_a_tl { \if_false: } \fi:
+ \regex_build_u_loop:NN
+ }
+ {
+ \msg_kernel_error:nn { regex } { u-missing-lbrace }
+ #1#2
+ }
+ }
}
-\cs_new_protected:cpn { regex_build_group_n*?: } #1
- { % ^^A todo: catch case #1 <= 0.
- \regex_build_group_n_aux:n {#1}
- \regex_build_transitions:NNNN
- \regex_action_free:n \l_regex_right_state_int
- \regex_action_free:n \l_regex_tmpa_int
- \regex_build_group_submatches:NN
- \l_regex_tmpa_int \l_regex_left_state_int
+\cs_new:Npn \regex_build_u_loop:NN #1#2
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_build_raw:N
+ { #2 \regex_build_u_loop:NN }
+ {
+ \token_if_eq_meaning:NNTF #1 \regex_build_special:N
+ {
+ \exp_after:wN \token_if_eq_charcode:NNTF \c_rbrace_str #2
+ { \if_false: { \fi: } \regex_build_u_end: }
+ { #2 \regex_build_u_loop:NN }
+ }
+ {
+ \if_false: { \fi: }
+ \msg_kernel_error:nnx { regex } { u-missing-rbrace } {#2}
+ \regex_build_u_end:
+ #1#2
+ }
+ }
+ }
+\cs_new_protected:Npn \regex_build_u_end:
+ {
+ \tl_set:Nv \l_regex_internal_a_tl \l_regex_internal_a_tl
+ \if_num:w \l_regex_build_mode_int = \c_zero
+ \regex_build_u_not_cs:
+ \else:
+ \regex_build_u_in_cs:
+ \fi:
+ \tl_clear:N \l_regex_tests_tl
}
-\cs_new_protected:Npn \regex_build_group_nn: #1#2
- { % ^^A Not Implemented Yet!
- \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet }
- \use:c { regex_build_group_n*: } {#1}
+\cs_new_protected:Npn \regex_build_u_in_cs:
+ {
+ \exp_args:NNo \str_gset_other:Nn \g_regex_internal_tl
+ { \l_regex_internal_a_tl }
+ \tl_map_inline:Nn \g_regex_internal_tl
+ {
+ \tl_set:Nx \l_regex_tests_tl { \regex_item_equal:n { `##1 } }
+ \regex_build_one_:
+ }
}
-\cs_new_protected:cpn { regex_build_group_nn?: } #1#2
- { % ^^A Not Implemented Yet!
- \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet }
- \use:c { regex_build_group_n*?: } {#1}
+\cs_new_protected:Npn \regex_build_u_not_cs:
+ {
+ \exp_args:No \tl_analysis_map_inline:nn { \l_regex_internal_a_tl }
+ {
+ \tl_set:Nx \l_regex_tests_tl
+ {
+ \exp_not:N \int_compare:nNnT
+ \l_regex_current_catcode_int = { \int_value:w "##2 }
+ {
+ \int_compare:nNnTF { "##2 } = \c_zero
+ {
+ \exp_not:n
+ { \str_if_eq:xxT { \l_regex_current_token_tl } {##1} }
+ { \regex_break_true:w }
+ }
+ { \regex_item_equal:n {##3} }
+ }
+ }
+ \regex_build_one_:
+ }
}
\cs_new_protected:Npn \regex_store_state:n #1
{
- \int_incr:N \l_regex_max_index_int
- \tex_skip:D \l_regex_max_index_int #1 sp \scan_stop:
+ \int_incr:N \l_regex_max_active_int
+ \tex_skip:D \l_regex_max_active_int #1 sp \scan_stop:
\regex_store_submatches:n {#1}
}
\cs_new_protected_nopar:Npn \regex_state_use_with_submatches:
@@ -1336,10 +1656,9 @@
{ \regex_state_use_aux:n { \exp_after:wN \use_none_delimit_by_s_stop:w } }
\cs_new_protected:Npn \regex_state_use_aux:n #1
{
- \if_num:w \tex_dimen:D \l_regex_current_state_int
- < \l_regex_unique_id_int
+ \if_num:w \tex_dimen:D \l_regex_current_state_int < \l_regex_step_int
\tex_dimen:D \l_regex_current_state_int
- = \l_regex_unique_id_int sp \scan_stop:
+ = \l_regex_step_int sp \scan_stop:
#1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int
\fi:
\scan_stop:
@@ -1374,68 +1693,55 @@
{
\exp_after:wN \c_zero
\exp_after:wN \regex_state_submatches:nn \exp_after:wN
- {
- \int_value:w \int_eval:w
- \l_regex_unique_id_int + \c_one
- \exp_after:wN
- }
+ { \int_value:w \int_eval:w \l_regex_step_int + \c_one \exp_after:wN }
\exp_after:wN { \l_regex_current_submatches_prop }
\regex_state_submatches:nn {#1} {#2}
\s_stop
}
\cs_new_protected:Npn \regex_state_submatches:nn #1#2
{
- \if_num:w #1 = \l_regex_unique_id_int
+ \if_num:w #1 = \l_regex_step_int
\tl_set:Nn \l_regex_current_submatches_prop { #2 }
\fi:
}
\cs_new_protected:Npn \regex_match:n #1
{
- \tl_set_analysis:Nn \l_regex_tmpa_tl {#1}
-
\int_zero:N \l_regex_nesting_int
- \int_set_eq:NN \l_regex_current_step_int \l_regex_max_state_int
+ \int_set_eq:NN \l_regex_current_index_int \l_regex_max_state_int
\regex_query_set:nnn { } { -1 } { -2 }
- \int_set_eq:NN \l_regex_min_step_int \l_regex_current_step_int
- \exp_after:wN \regex_query_set_loop:ww
- \l_regex_tmpa_tl
- \s_tl { ? \prg_map_break: } \s_tl
- \prg_break_point:n { }
- \int_set_eq:NN \l_regex_max_step_int \l_regex_current_step_int
+ \int_set_eq:NN \l_regex_min_index_int \l_regex_current_index_int
+ \tl_analysis_map_inline:nn {#1}
+ {
+ \regex_query_set:nnn {##1} {"##2} {##3}
+ \if_case:w "##2 \exp_stop_f:
+ \or: \int_incr:N \l_regex_nesting_int
+ \or: \int_decr:N \l_regex_nesting_int
+ \fi:
+ }
+ \int_set_eq:NN \l_regex_max_index_int \l_regex_current_index_int
\regex_query_set:nnn { } { -1 } { -2 }
-
\regex_match_initial_setup:
\regex_match_once:
}
-\cs_new:Npn \regex_query_set_loop:ww #1 \s_tl #2#3 \s_tl
- {
- \use_none:n #2
- \regex_query_set:nnn {#1} {"#2} {#3}
- \if_case:w "#2 \exp_stop_f:
- \or: \int_incr:N \l_regex_nesting_int
- \or: \int_decr:N \l_regex_nesting_int
- \fi:
- \regex_query_set_loop:ww
- }
\cs_new_protected:Npn \regex_query_set:nnn #1#2#3
{
- \tex_muskip:D \l_regex_current_step_int
+ \tex_muskip:D \l_regex_current_index_int
= \etex_gluetomu:D
#3 sp
plus #2 sp
minus \l_regex_nesting_int sp
\scan_stop:
- \tex_toks:D \l_regex_current_step_int {#1}
- \int_incr:N \l_regex_current_step_int
+ \tex_toks:D \l_regex_current_index_int {#1}
+ \int_incr:N \l_regex_current_index_int
}
\cs_new_protected_nopar:Npn \regex_query_get:
{
\tl_set:Nx \l_regex_current_token_tl
- { \tex_the:D \tex_toks:D \l_regex_current_step_int }
+ { \tex_the:D \tex_toks:D \l_regex_current_index_int }
\l_regex_current_char_int
- = \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ = \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int
\l_regex_current_catcode_int = \etex_gluestretch:D
- \etex_mutoglue:D \tex_muskip:D \l_regex_current_step_int
+ \etex_mutoglue:D \tex_muskip:D \l_regex_current_index_int
}
\cs_new_protected_nopar:Npn \regex_match_once:
{
@@ -1451,13 +1757,14 @@
}
\cs_new_protected_nopar:Npn \regex_match_initial_setup:
{
- \prg_stepwise_inline:nnnn {0} {1} { \l_regex_max_state_int - \c_one }
- { \tex_dimen:D ##1 \c_minus_one sp \scan_stop: }
- \int_set_eq:NN \l_regex_unique_id_int \c_minus_one
- \int_set:Nn \l_regex_start_step_int
- { \l_regex_min_step_int - \c_one }
- \int_set_eq:NN \l_regex_current_step_int \l_regex_min_step_int
- \int_set_eq:NN \l_regex_success_step_int \l_regex_min_step_int
+ \int_set_eq:NN \l_regex_step_int \c_minus_one
+ \prg_stepwise_inline:nnnn
+ \c_zero \c_one { \l_regex_max_state_int - \c_one }
+ { \tex_dimen:D ##1 \l_regex_step_int sp \scan_stop: }
+ \int_set:Nn \l_regex_start_index_int
+ { \l_regex_min_index_int - \c_one }
+ \int_set_eq:NN \l_regex_current_index_int \l_regex_min_index_int
+ \int_set_eq:NN \l_regex_success_index_int \l_regex_min_index_int
\int_set_eq:NN \l_regex_submatch_int \l_regex_max_state_int
\bool_set_false:N \l_regex_success_empty_bool
\bool_gset_false:N \g_regex_success_bool
@@ -1466,49 +1773,48 @@
{
\prop_clear:N \l_regex_current_submatches_prop
\bool_if:NTF \l_regex_success_empty_bool
- { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F }
- { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F }
- \int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int
- \int_set:Nn \l_regex_current_step_int
- { \l_regex_start_step_int - \c_one }
+ { \cs_set_eq:NN \regex_if_two_empty_matches:F \regex_if_match_empty:F }
+ { \cs_set_eq:NN \regex_if_two_empty_matches:F \use:n }
+ \int_set_eq:NN \l_regex_start_index_int \l_regex_success_index_int
+ \int_set:Nn \l_regex_current_index_int
+ { \l_regex_start_index_int - \c_one }
\bool_set_false:N \l_regex_success_match_bool
- \int_zero:N \l_regex_max_index_int
+ \int_zero:N \l_regex_max_active_int
\regex_store_state:n {0} %^^A _state_int!
}
\cs_new_protected_nopar:Npn \regex_match_loop:
{
- \int_incr:N \l_regex_current_step_int
- \int_incr:N \l_regex_unique_id_int
+ \int_incr:N \l_regex_current_index_int
+ \int_incr:N \l_regex_step_int
\bool_set_false:N \l_regex_fresh_thread_bool
\int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int
\regex_query_get:
\regex_match_loop_case_hook:
- \cs_set_nopar:Npx \regex_tmp:w
+ \use:x
{
- \int_zero:N \l_regex_max_index_int
- \regex_match_one_index:w 1 ; \prg_break_point:n { }
+ \int_zero:N \l_regex_max_active_int
+ \regex_match_one_active:w 1 ; \prg_break_point:n { }
\exp_not:N \prg_break_point:n { }
}
- \regex_tmp:w
\if_num:w \l_regex_current_char_int < \c_minus_one
\exp_after:wN \prg_map_break:
\fi:
- \if_num:w \l_regex_max_index_int = \c_zero
+ \if_num:w \l_regex_max_active_int = \c_zero
\exp_after:wN \prg_map_break:
\fi:
\regex_match_loop:
}
-\cs_new:Npn \regex_match_one_index:w #1;
+\cs_new:Npn \regex_match_one_active:w #1;
{
- \if_num:w #1 > \l_regex_max_index_int
+ \if_num:w #1 > \l_regex_max_active_int
\exp_after:wN \prg_map_break:
\fi:
- \regex_match_one_index_aux:n
+ \regex_match_one_active_aux:n
{ \int_value:w \tex_skip:D #1 }
- \exp_after:wN \regex_match_one_index:w
+ \exp_after:wN \regex_match_one_active:w
\int_use:N \int_eval:w #1 + \c_one ;
}
-\cs_new_protected:Npn \regex_match_one_index_aux:n #1
+\cs_new_protected:Npn \regex_match_one_active_aux:n #1
{
\int_set:Nn \l_regex_current_state_int {#1}
\prop_clear:N \l_regex_current_submatches_prop
@@ -1546,12 +1852,12 @@
}
\cs_new_protected_nopar:Npn \regex_action_success:
{
- \regex_last_match_empty:F
+ \regex_if_two_empty_matches:F
{
\bool_set_true:N \l_regex_success_match_bool
\bool_set_eq:NN \l_regex_success_empty_bool
\l_regex_fresh_thread_bool
- \int_set_eq:NN \l_regex_success_step_int \l_regex_current_step_int
+ \int_set_eq:NN \l_regex_success_index_int \l_regex_current_index_int
\prop_set_eq:NN \l_regex_success_submatches_prop
\l_regex_current_submatches_prop
\prg_map_break:
@@ -1559,7 +1865,7 @@
}
\cs_new_protected:Npn \regex_action_free:n #1
{
- \cs_set_nopar:Npx \regex_tmp:w
+ \use:x
{
\int_add:Nn \l_regex_current_state_int {#1}
\regex_state_use:
@@ -1568,13 +1874,16 @@
\tl_set:Nn \exp_not:N \l_regex_current_submatches_prop
{ \exp_not:o \l_regex_current_submatches_prop }
}
- \regex_tmp:w
}
\cs_new_protected:Npn \regex_action_submatch:n #1
{
\prop_put:Nno \l_regex_current_submatches_prop {#1}
- { \int_use:N \l_regex_current_step_int }
+ { \int_use:N \l_regex_current_index_int }
}
+\cs_new:Npn \regex_if_match_empty:T
+ { \int_compare:nNnT \l_regex_start_index_int = \l_regex_current_index_int }
+\cs_new:Npn \regex_if_match_empty:F
+ { \int_compare:nNnF \l_regex_start_index_int = \l_regex_current_index_int }
\cs_new_protected:Npn \regex_submatch_nesting_aux:n #1
{
+ \etex_glueshrink:D \etex_mutoglue:D \etex_muexpr:D
@@ -1589,10 +1898,10 @@
\int_zero:N \l_regex_replacement_int
\int_zero:N \l_regex_nesting_int
\tl_clear:N \l_regex_nesting_tl
- \str_escape_use:NNNn
- \regex_replacement_unescaped:N
- \regex_replacement_escaped:N
- \regex_replacement_raw:N
+ \regex_escape_use:nnnn
+ { \regex_replacement_unescaped:N ##1 }
+ { \regex_replacement_escaped:N ##1 }
+ { \regex_replacement_raw:N ##1 }
{#1}
\prg_do_nothing: \prg_do_nothing:
\cs_set:Npx \regex_nesting:n ##1
@@ -1604,9 +1913,9 @@
\use:x
{
\exp_not:n { \cs_set:Npn \regex_replacement_tl:n ##1 }
- { \str_aux_toks_range:nn \c_zero \l_regex_replacement_int }
+ { \regex_toks_range:nn \c_zero \l_regex_replacement_int }
}
- % ^^A rename! Careful with \cP\#.
+ % ^^A rename!
}
\cs_new_protected:Npn \regex_replacement_raw:N #1
{ \regex_replacement_put:n {#1} }
@@ -1629,19 +1938,14 @@
}
\cs_new_protected:Npn \regex_replacement_escaped:N #1
{
- \if_charcode:w c #1
- \exp_after:wN \regex_replacement_c:w
- \else:
- \if_charcode:w g #1
- \exp_after:wN \exp_after:wN \exp_after:wN \regex_replacement_g:w
- \else:
+ \cs_if_exist_use:cF { regex_replacement_#1:w }
+ {
\if_num:w \c_one < 1#1 \exp_stop_f:
\regex_replacement_put_submatch:n {#1}
\else:
\regex_replacement_put:n #1
\fi:
- \fi:
- \fi:
+ }
}
\cs_new_protected:Npn \regex_replacement_put_submatch:n #1
{
@@ -1668,7 +1972,7 @@
{ \exp_not:n { #1#2 } }
{ \regex_replacement_unescaped:N \c_lbrace_str }
{
- \int_zero:N \l_regex_tmpa_int
+ \int_zero:N \l_regex_internal_a_int
\regex_replacement_g_digits:NN
}
{ \regex_replacement_error:NNN g #1 #2 }
@@ -1678,13 +1982,13 @@
\token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
{
\if_num:w \c_one < 1#2 \exp_stop_f:
- \int_set:Nn \l_regex_tmpa_int { \c_ten * \l_regex_tmpa_int + #2 }
+ \int_set:Nn \l_regex_internal_a_int { \c_ten * \l_regex_internal_a_int + #2 }
\exp_after:wN \use_i:nnn
\exp_after:wN \regex_replacement_g_digits:NN
\else:
\if_charcode:w \c_rbrace_str #2
\exp_args:No \regex_replacement_put_submatch:n
- { \int_use:N \l_regex_tmpa_int }
+ { \int_use:N \l_regex_internal_a_int }
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
\else:
\exp_after:wN \exp_after:wN
@@ -1696,6 +2000,16 @@
{ \regex_replacement_error:NNN g }
#1 #2
}
+\cs_new_protected:Npn \regex_replacement_u:w #1#2
+ {
+ \str_if_eq:xxTF { #1#2 } { \regex_replacement_unescaped:N \c_lbrace_str }
+ {
+ \int_incr:N \l_regex_replacement_csnames_int
+ \regex_replacement_put:n
+ { \exp_not:n { \exp_after:wN \exp_not:V \cs:w } }
+ }
+ { \regex_replacement_error:NNN u #1#2 }
+ }
\cs_new_protected:Npn \regex_replacement_c:w #1#2
{
\token_if_eq_meaning:NNTF #1 \regex_replacement_unescaped:N
@@ -1711,8 +2025,6 @@
\regex_replacement_put:n
{ \exp_not:n { \exp_after:wN \regex_replacement_exp_not:N \cs:w } }
}
-\cs_new_eq:cc { regex_replacement_c_C:w }
- { regex_replacement_c_ \c_lbrace_str :w }
\cs_new:Npn \regex_replacement_exp_not:N #1 { \exp_not:n {#1} }
\group_begin:
@@ -1790,11 +2102,11 @@
{
\group_begin:
\regex_build:n {#3}
- \cs_set_nopar:Npx \regex_tmp:w
- { #1 \exp_not:N #2 { \regex_set_aux:N #2 } }
- \exp_after:wN
- \group_end:
- \regex_tmp:w
+ \use:x
+ {
+ \group_end:
+ #1 \exp_not:N #2 { \regex_set_aux:N #2 }
+ }
}
\cs_new:Npn \regex_set_aux:N #1
{
@@ -1811,7 +2123,7 @@
\regex_match_loop_caseless_hook:
}
\prg_stepwise_function:nnnN
- {0} {1} {\l_regex_max_state_int - \c_one }
+ \c_zero \c_one {\l_regex_max_state_int - \c_one }
\regex_set_aux:n
\s_stop
}
@@ -1824,18 +2136,6 @@
}
\cs_new_protected_nopar:Npn \regex_use:N
{ \exp_last_unbraced:No \use_none:nn }
-\cs_new:Npn \regex_to_str:N #1
- {
- \exp_after:wN \regex_to_str_aux:Nw #1
- \tex_toks:D \q_stop \prg_map_break: \tex_toks:D
- \prg_break_point:n { }
- }
-\cs_new:Npn \regex_to_str_aux:Nw #1#2\tex_toks:D
- {
- \use_none_delimit_by_q_stop:w #2 \q_stop
- \tl_to_str:n {#1#2} \iow_newline:
- \regex_to_str_aux:Nw \tex_toks:D
- }
\cs_new_protected_nopar:Npn \regex_aux_return:
{
\if_meaning:w \c_true_bool \g_regex_success_bool
@@ -1921,9 +2221,9 @@
\cs_new_protected:Npn \regex_group_end_extract_seq:N #1
{
\cs_set_eq:NN \seq_item:n \scan_stop:
- \flag_lower:N \l_regex_group_begin_flag
- \flag_lower:N \l_regex_group_end_flag
- \tl_set:Nx \l_regex_tmpa_tl
+ \flag_clear:n { regex_begin }
+ \flag_clear:n { regex_end }
+ \tl_set:Nx \l_regex_internal_a_tl
{
\prg_stepwise_function:nnnN
\l_regex_max_state_int
@@ -1931,12 +2231,13 @@
{ \l_regex_submatch_int - \c_one }
\regex_extract_seq_aux:n
}
- \flag_test:NF \l_regex_group_begin_flag
- { \flag_test:NF \l_regex_group_end_flag { \use_none:nnn } }
- \msg_kernel_error:nn { regex } { sequence-unbalanced }
- \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \int_compare:nNnF
+ { \flag_height:n { regex_begin } + \flag_height:n { regex_end } }
+ = \c_zero
+ { \msg_kernel_error:nn { regex } { sequence-unbalanced } }
+ \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
\exp_args:NNNo \group_end:
- \tl_set:Nn #1 \l_regex_tmpa_tl
+ \tl_set:Nn #1 \l_regex_internal_a_tl
}
\cs_new:Npn \regex_extract_seq_aux:n #1
{
@@ -1947,14 +2248,14 @@
}
}
\cs_new:Npn \regex_extract_seq_aux:ww #1; #2;
- { %^^A todo: use flags
+ {
\if_num:w #1 < \c_zero
- \flag_raise:N \l_regex_group_end_flag
+ \flag_raise:n { regex_end }
\prg_replicate:nn {-#1} { \exp_not:n { { \if_false: } \fi: } }
\fi:
\regex_query_submatch:w #2;
\if_num:w #1 > \c_zero
- \flag_raise:N \l_regex_group_begin_flag
+ \flag_raise:n { regex_begin }
\prg_replicate:nn {#1} { \exp_not:n { \if_false: { \fi: } } }
\fi:
}
@@ -2016,20 +2317,20 @@
\group_begin:
\tl_set:Nn \l_regex_every_match_tl
{
- \if_num:w \l_regex_start_step_int < \l_regex_success_step_int
+ \if_num:w \l_regex_start_index_int < \l_regex_success_index_int
\regex_extract:
\tex_skip:D \l_regex_submatch_start_int
- = \l_regex_start_step_int sp
+ = \l_regex_start_index_int sp
plus \tex_skip:D \l_regex_submatch_start_int \scan_stop:
\fi:
\regex_match_once:
}
#1 {#2} {#3}
\tex_skip:D \l_regex_submatch_int
- = \l_regex_start_step_int sp
- plus \l_regex_current_step_int sp \scan_stop:
+ = \l_regex_start_index_int sp
+ plus \l_regex_current_index_int sp \scan_stop:
\int_incr:N \l_regex_submatch_int
- \if_num:w \l_regex_start_step_int = \l_regex_current_step_int
+ \if_num:w \l_regex_start_index_int = \l_regex_current_index_int
\if_meaning:w \c_true_bool \l_regex_success_empty_bool
\int_decr:N \l_regex_submatch_int
\fi:
@@ -2044,34 +2345,34 @@
\exp_args:Nno #1 {#2} #4
\if_meaning:w \c_true_bool \g_regex_success_bool
\regex_extract:
- \int_set:Nn \l_regex_tmpa_int
+ \int_set:Nn \l_regex_internal_a_int
{ \regex_nesting:n { \l_regex_submatch_start_int } }
- \if_num:w \l_regex_tmpa_int = \c_zero
+ \if_num:w \l_regex_internal_a_int = \c_zero
\else:
\msg_kernel_error:nnx { regex } { replace-unbalanced }
- { \l_regex_tmpa_int }
+ { \l_regex_internal_a_int }
\fi:
- \tl_set:Nx \l_regex_tmpa_tl
+ \tl_set:Nx \l_regex_internal_a_tl
{
- \if_num:w \l_regex_tmpa_int < \c_zero
- \prg_replicate:nn { - \l_regex_tmpa_int }
+ \if_num:w \l_regex_internal_a_int < \c_zero
+ \prg_replicate:nn { - \l_regex_internal_a_int }
{ \exp_not:n { { \if_false: } \fi: } }
\fi:
- \regex_query_substr:nn
- { \l_regex_min_step_int }
+ \regex_toks_range:nn
+ { \l_regex_min_index_int }
{ \tex_skip:D \l_regex_submatch_start_int }
\regex_replacement_tl:n { \l_regex_submatch_start_int }
- \regex_query_substr:nn
+ \regex_toks_range:nn
{ \etex_gluestretch:D \tex_skip:D \l_regex_submatch_start_int }
- { \l_regex_max_step_int }
- \if_num:w \l_regex_tmpa_int > \c_zero
- \prg_replicate:nn { \l_regex_tmpa_int }
+ { \l_regex_max_index_int }
+ \if_num:w \l_regex_internal_a_int > \c_zero
+ \prg_replicate:nn { \l_regex_internal_a_int }
{ \exp_not:n { \if_false: { \fi: } } }
\fi:
}
- \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
\exp_args:NNNo \group_end:
- \tl_set:Nn #4 \l_regex_tmpa_tl
+ \tl_set:Nn #4 \l_regex_internal_a_tl
\else:
\group_end:
\fi:
@@ -2087,12 +2388,12 @@
\regex_extract:
\tex_skip:D \l_regex_submatch_start_int
= \tex_the:D \tex_skip:D \l_regex_submatch_start_int
- minus \l_regex_start_step_int sp \scan_stop:
+ minus \l_regex_start_index_int sp \scan_stop:
\regex_match_once:
}
\regex_replacement:n {#3}
\exp_args:Nno #1 {#2} #4
- \int_set:Nn \l_regex_tmpa_int
+ \int_set:Nn \l_regex_internal_a_int
{
0
\prg_stepwise_function:nnnN
@@ -2101,15 +2402,15 @@
{ \l_regex_submatch_int - \c_one }
\regex_nesting:n
}
- \if_num:w \l_regex_tmpa_int = \c_zero
+ \if_num:w \l_regex_internal_a_int = \c_zero
\else:
\msg_kernel_error:nnx { regex } { replace-unbalanced }
- { \l_regex_tmpa_int }
+ { \l_regex_internal_a_int }
\fi:
- \tl_set:Nx \l_regex_tmpa_tl
+ \tl_set:Nx \l_regex_internal_a_tl
{
- \if_num:w \l_regex_tmpa_int < \c_zero
- \prg_replicate:nn { - \l_regex_tmpa_int }
+ \if_num:w \l_regex_internal_a_int < \c_zero
+ \prg_replicate:nn { - \l_regex_internal_a_int }
{ \exp_not:n { { \if_false: } \fi: } }
\fi:
\prg_stepwise_function:nnnN
@@ -2117,30 +2418,144 @@
\l_regex_capturing_group_int
{ \l_regex_submatch_int - \c_one }
\regex_replace_all_aux:n
- \regex_query_substr:nn
- \l_regex_start_step_int \l_regex_max_step_int
- \if_num:w \l_regex_tmpa_int > \c_zero
- \prg_replicate:nn { \l_regex_tmpa_int }
+ \regex_toks_range:nn
+ \l_regex_start_index_int \l_regex_max_index_int
+ \if_num:w \l_regex_internal_a_int > \c_zero
+ \prg_replicate:nn { \l_regex_internal_a_int }
{ \exp_not:n { \if_false: { \fi: } } }
\fi:
}
- \tl_set:Nx \l_regex_tmpa_tl { \l_regex_tmpa_tl }
+ \tl_set:Nx \l_regex_internal_a_tl { \l_regex_internal_a_tl }
\exp_args:NNNo \group_end:
- \tl_set:Nn #4 \l_regex_tmpa_tl
+ \tl_set:Nn #4 \l_regex_internal_a_tl
}
\cs_new:Npn \regex_replace_all_aux:n #1
{
- \regex_query_substr:nn
+ \regex_toks_range:nn
{ \etex_glueshrink:D \tex_skip:D #1 } { \tex_skip:D #1 }
\regex_replacement_tl:n {#1}
}
+\msg_kernel_new:nnnn { regex } { trailing-backslash }
+ { Trailing~escape~character~\iow_char:N\\. }
+ {
+ A~regular~expression~or~its~replacement~text~ends~with~
+ the~escape~character~\iow_char:N\\.~It~will~be~ignored.
+ }
+\msg_kernel_new:nnnn { regex } { x-missing-rbrace }
+ { Missing~closing~brace~in~\iow_char:N\\x~hexadecimal~sequence. }
+ {
+ You~wrote~something~like~
+ `\iow_char:N\\x\{\int_to_hexadecimal:n{#1}'.~
+ The~closing~brace~is~missing.
+ }
+\msg_kernel_new:nnnn { regex } { x-overflow }
+ { Character~code~'#1'~too~large~in~\iow_char:N\\x~hexadecimal~sequence. }
+ {
+ You~wrote~something~like~
+ `\iow_char:N\\x\{\int_to_hexadecimal:n{#1}\}'.~
+ The~character~code~'#1'~is~larger~than~\int_use:N \c_max_char_int.
+ }
+\msg_kernel_new:nnnn { regex } { missing-rbrack }
+ { Missing~right~bracket~inserted~in~regular~expression. }
+ {
+ LaTeX~was~given~a~regular~expression~where~a~character~class~
+ was~started~with~'[',~but~the~matching~']'~is~missing.
+ }
+\msg_kernel_new:nnnn { regex } { missing-rparen }
+ {
+ Missing~right~parenthes\int_compare:nTF{#1=1}{i}{e}s~
+ inserted~in~regular~expression.
+ }
+ {
+ LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~
+ more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~
+ parenthes\int_compare:nTF{#1=1}{i}{e}s.
+ }
+\msg_kernel_new:nnnn { regex } { extra-rparen }
+ { Extra~right~parenthesis~ignored~in~regular~expression. }
+ {
+ LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~
+ was~open.~The~parenthesis~will~be~ignored.
+ }
+\msg_kernel_new:nnnn { regex } { class-bad-escape }
+ { Invalid~escape~\c_backslash_str #1~in~character~class. }
+ {
+ The~escape~sequence~\iow_char:N\\#1~may~not~appear~within~
+ a~character~class.~For~instance,~assertions~(which~match~zero~
+ characters)~would~not~make~sense~there.
+ }
+\msg_kernel_new:nnnn { regex } { catcode-bad-escape }
+ { Invalid~escape~\c_backslash_str #1~following~category~test. }
+ {
+ The~escape~sequence~\iow_char:N\\#1~may~not~appear~following~
+ a~category~test~such~as~\iow_char:N\\cL~or~
+ \iow_char:N\\c[\iow_char:N\^BE].
+ }
+\msg_kernel_new:nnnn { regex } { range-missing-end }
+ { Invalid~end-point~for~range~'#1-#2'~in~character~class. }
+ {
+ The~end-point~'#2'~of~the~range~'#1-#2'~may~not~serve~as~an~
+ end-point~for~a~range:~alphanumeric~characters~should~not~be~
+ escaped,~and~non-alphanumeric~characters~should~be~escaped.
+ }
+\msg_kernel_new:nnnn { regex } { range-backwards }
+ { Range~[#1-#2]~out~of~order~in~character~class. }
+ {
+ In~ranges~of~characters~[x-y]~appearing~in~character~classes,~
+ the~first~character~code~must~not~be~larger~than~the~second.~
+ Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~
+ character~code~\int_eval:n {`#2}.
+ }
+\msg_kernel_new:nnnn { regex } { c-bad-mode }
+ { Invalid~nested~\iow_char:N\\c~escape~in~regular~expression. }
+ {
+ The~\iow_char:N\\c~escape~cannot~be~used~within~
+ a~control~sequence~test~'\iow_char:N\\c{...}'.~
+ To~combine~several~category~tests,~use~'\iow_char:N\\c[...]'.
+ }
+\msg_kernel_new:nnnn { regex } { c-missing-rbrace }
+ { Missing~right~brace~inserted~for~\iow_char:N\\c~escape. }
+ {
+ LaTeX~was~given~a~regular~expression~where~a~
+ '\iow_char:N\\c\iow_char:N\{...'~construction~was~not~ended~
+ with~a~closing~brace~'\iow_char:N\}'.
+ }
+\msg_kernel_new:nnnn { regex } { c-missing-rbrack }
+ { Missing~right~bracket~inserted~for~\iow_char:N\\c~escape. }
+ {
+ A~construction~'\iow_char:N\\c[...'~appears~in~a~
+ regular~expression,~but~the~closing~']'~is~not~present.
+ }
+\msg_kernel_new:nnnn { regex } { c-missing-category }
+ { Invalid~character~'#1'~following~\iow_char\\c~escape. }
+ {
+ In~regular~expressions,~the~\iow_char:N\\c~escape~sequence~
+ may~only~be~followed~by~a~left~brace,~a~left~bracket,~or~a~
+ capital~letter~representing~a~character~category,~namely~
+ one~of~ABCDELMOPSTU.
+ }
+\msg_kernel_new:nnnn { regex } { u-missing-lbrace }
+ { Missing~left~brace~following~\iow_char:N\\u~escape. }
+ {
+ The~\iow_char:N\\u~escape~sequence~must~be~followed~by~
+ a~brace~group~with~the~name~of~the~variable~to~use.
+ }
+\msg_kernel_new:nnnn { regex } { u-missing-rbrace }
+ { Missing~right~brace~inserted~for~\iow_char:N\\u~escape. }
+ {
+ LaTeX~
+ \tl_if_empty:xTF {#2}
+ { reached~the~end~of~the~string~ }
+ { encountered~an~escaped~alphanumeric~character '\iow_char:N\\#2'~ }
+ when~parsing~the~argument~of~an~'\iow_char:N\\u\iow_char:N\{...\}'~escape.
+ }
\msg_kernel_new:nnnn { regex } { sequence-unbalanced }
{
Missing~
- \flag_test:NTF \l_regex_group_end_flag
+ \flag_if_raised:nTF { regex_end }
{
left~
- \flag_test:NTF \l_regex_group_begin_flag
+ \flag_if_raised:nTF { regex_begin }
{ and~right~braces } { brace }
}
{ right~brace }
@@ -2170,34 +2585,6 @@
}
\ inserted.
}
-\msg_kernel_new:nnnn { regex } { missing-rparen }
- { Missing~right~parenthesis~added~in~regular~expression. }
- {
- LaTeX~was~given~a~regular~expression~with~\int_eval:n{#1}~
- more~left~parenthes\int_compare:nTF{#1=1}{i}{e}s~than~right~
- parenthes\int_compare:nTF{#1=1}{i}{e}s.
- }
-\msg_kernel_new:nnnn { regex } { extra-rparen }
- { Extra~right~parenthesis~ignored~in~regular~expression. }
- {
- LaTeX~came~across~a~closing~parenthesis~when~no~submatch~group~
- was~open.~The~parenthesis~will~be~ignored.
- }
-\msg_kernel_new:nnnn { regex } { backwards-range }
- { Range~[#1-#2]~out~of~order~in~character~class. }
- {
- In~ranges~of~characters~[x-y]~appearing~in~character~classes,~
- the~first~character~code~must~not~be~larger~than~the~second.~
- Here,~#1~has~character~code~\int_eval:n {`#1},~while~#2~has~
- character~code~\int_eval:n {`#2}.
- }
-\msg_kernel_new:nnnn { regex } { unsupported }
- { Unsupported~construction~`#1'. }
- {
- The~construction~`#1'~is~not~supported~by~the~LaTeX~
- regular~expression~module.~Perhaps~some~character~should~
- have~been~escaped?
- }
\msg_kernel_new:nnnn { regex } { not-nfa }
{ This~is~not~a~regular~expression~variable. }
{
@@ -2206,6 +2593,8 @@
is~\\\\
\ \ \ \ \token_to_str:N #1 = \token_to_meaning:N #1 .
}
+\msg_kernel_new:nnn { regex } { nfa-misused }
+ { Automaton~#1 used~incorrectly. }
\msg_kernel_new:nnnn { regex } { c-command }
{ Misused~\iow_char:N\\c~or\iow_char:N\\C~command~in~a~#1. }
{
@@ -2232,8 +2621,6 @@
submatches~are~represented~either~as~\iow_char:N \\g{dd..d},~
or~\\d,~where~`d'~are~single~digits.~Here,~a~brace~is~missing.
}
-\msg_kernel_new:nnn { regex } { nfa-misused }
- { Automaton~#1 used~incorrectly. }
%%
%%
%% End of file `l3regex.sty'.
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty
index cc4d6716ed5..d9ef6271899 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty
@@ -46,31 +46,32 @@
%%
%% -----------------------------------------------------------------------
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 3159 2012-01-09 00:51:58Z bruno $
+\GetIdInfo$Id: l3str.dtx 3257 2012-01-29 13:37:03Z bruno $
{L3 Experimental Strings}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
-\RequirePackage{l3tl-analysis}
-\group_begin:
- \cs_set_protected_nopar:Npn \str_tmp:w #1
- {
- \cs_new_protected:cpx { str_ #1 :Nn } ##1##2
- { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } }
- \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx }
- }
- \str_tmp:w { set }
- \str_tmp:w { gset }
- \str_tmp:w { const }
- \str_tmp:w { put_left }
- \str_tmp:w { gput_left }
- \str_tmp:w { put_right }
- \str_tmp:w { gput_right }
-\group_end:
+\RequirePackage{l3tl-analysis,l3flag}
+\cs_if_exist:NF \use_ii_i:nn
+ { \cs_new:Npn \use_ii_i:nn #1#2 { #2 #1 } }
+\tl_map_inline:nn
+ {
+ { set }
+ { gset }
+ { const }
+ { put_left }
+ { gput_left }
+ { put_right }
+ { gput_right }
+ }
+ {
+ \cs_new_protected:cpx { str_ #1 :Nn } ##1##2
+ { \exp_not:c { tl_ #1 :Nx } ##1 { \exp_not:N \tl_to_str:n {##2} } }
+ \exp_args:Nc \cs_generate_variant:Nn { str_ #1 :Nn } { Nx , cn , cx }
+ }
\cs_new_protected_nopar:Npn \str_tmp:w { }
-\tl_new:N \l_str_tmpa_tl
-\int_new:N \l_str_tmpa_int
+\tl_new:N \l_str_internal_tl
+\int_new:N \l_str_internal_int
\tl_new:N \g_str_result_tl
-\int_new:N \l_str_char_int
\int_const:Nn \c_forty_eight { 48 }
\int_const:Nn \c_fifty_eight { 58 }
\int_const:Nn \c_sixty_five { 65 }
@@ -79,7 +80,7 @@
\int_const:Nn \c_one_hundred_twenty_three { 123 }
\int_const:Nn \c_one_hundred_twenty_seven { 127 }
\int_const:Nn \c_max_char_int
- { \pdftex_if_engine:TF { 255 } { 1114111 } }
+ { \pdftex_if_engine:TF { "FF } { "10FFFF } }
\int_const:Nn \c_str_replacement_char_int { "FFFD }
\tl_const:Nx \c_backslash_str { \cs_to_str:N \\ }
\tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ }
@@ -87,18 +88,25 @@
\tl_const:Nx \c_hash_str { \cs_to_str:N \# }
\tl_const:Nx \c_tilde_str { \cs_to_str:N \~ }
\tl_const:Nx \c_percent_str { \cs_to_str:N \% }
-\prop_new:N \g_str_aliases_prop
-\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 }
-\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 }
-\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 }
-\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 }
-\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 }
-\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 }
-\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 }
-\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 }
-\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 }
-\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 }
-\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex }
+\prop_new:N \g_str_file_prop
+\prop_gput:Nnn \g_str_file_prop { latin1 } { iso88591 }
+\prop_gput:Nnn \g_str_file_prop { latin2 } { iso88592 }
+\prop_gput:Nnn \g_str_file_prop { latin3 } { iso88593 }
+\prop_gput:Nnn \g_str_file_prop { latin4 } { iso88594 }
+\prop_gput:Nnn \g_str_file_prop { latin5 } { iso88599 }
+\prop_gput:Nnn \g_str_file_prop { latin6 } { iso885910 }
+\prop_gput:Nnn \g_str_file_prop { latin7 } { iso885913 }
+\prop_gput:Nnn \g_str_file_prop { latin8 } { iso885914 }
+\prop_gput:Nnn \g_str_file_prop { latin9 } { iso885915 }
+\prop_gput:Nnn \g_str_file_prop { latin10 } { iso885916 }
+\prop_gput:Nnn \g_str_file_prop { utf16le } { utf16 }
+\prop_gput:Nnn \g_str_file_prop { utf16be } { utf16 }
+\prop_gput:Nnn \g_str_file_prop { utf32le } { utf32 }
+\prop_gput:Nnn \g_str_file_prop { utf32be } { utf32 }
+\prop_gput:Nnn \g_str_file_prop { hexadecimal } { hex }
+\bool_new:N \g_str_error_bool
+\flag_new:n { str_byte }
+\flag_new:n { str_error }
\group_begin:
\char_set_lccode:nn { `\* } { `\ }
\char_set_lccode:nn { `\A } { `\A }
@@ -110,7 +118,7 @@
\exp_after:wN \tl_to_other_str_loop:w \tl_to_str:n {#1} ~ %
A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
}
- \cs_new_nopar:Npn \tl_to_other_str_loop:w
+ \cs_new:Npn \tl_to_other_str_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 \q_stop
{
\if_meaning:w A #8
@@ -119,7 +127,7 @@
\tl_to_other_str_loop:w
#9 #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * \q_stop
}
- \cs_new_nopar:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
+ \cs_new:Npn \tl_to_other_str_end:w \fi: #1 \q_mark #2 * A #3 \q_stop
{ \fi: #2 }
}
\group_begin:
@@ -133,10 +141,10 @@
\tl_gset:Nx #1
{
\exp_after:wN \str_gset_other_loop:w \tl_to_str:n {#2} ~ %
- A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
+ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_stop
}
}
- \cs_new_nopar:Npn \str_gset_other_loop:w
+ \cs_new:Npn \str_gset_other_loop:w
#1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~
{
\if_meaning:w A #9
@@ -145,32 +153,42 @@
#1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9
\str_gset_other_loop:w *
}
- \cs_new_nopar:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
+ \cs_new:Npn \str_gset_other_end:w \fi: #1 * A #2 \q_stop
{ \fi: #1 }
}
-\cs_if_exist:NF \use_i:nnnnnnnn
- { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} }
-\cs_new:Npn \str_aux_toks_range:nn #1#2
+\cs_new:Npn \str_count_spaces:N
+ { \exp_args:No \str_count_spaces:n }
+\cs_new:Npn \str_count_spaces:n #1
{
- \exp_after:wN \str_aux_toks_range:ww
- \int_use:N \int_eval:w #1 \exp_after:wN ;
- \int_use:N \int_eval:w #2 ;
- \prg_break_point:n { }
+ \int_eval:n
+ {
+ \exp_after:wN \str_count_spaces_loop:wwwwwwwww
+ \tl_to_str:n {#1} ~
+ X 7 ~ X 6 ~ X 5 ~ X 4 ~ X 3 ~ X 2 ~ X 1 ~ X 0 ~ X -1 ~
+ \q_stop
+ }
}
-\cs_new:Npn \str_aux_toks_range:ww #1 ; #2 ;
+\cs_new:Npn \str_count_spaces_loop:wwwwwwwww #1~#2~#3~#4~#5~#6~#7~#8~#9~
{
- \if_num:w #1 = #2 \exp_stop_f:
- \exp_after:wN \prg_map_break:
+ \if_meaning:w X #9
+ \exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
- \tex_the:D \tex_toks:D #1 \exp_stop_f:
- \exp_after:wN \str_aux_toks_range:ww
- \int_use:N \int_eval:w #1 + \c_one ; #2 ;
+ \c_nine + \str_count_spaces_loop:wwwwwwwww
}
\cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n }
\cs_new:Npn \str_length:n #1
- { \exp_args:Nf \str_length_unsafe:n { \tl_to_other_str:n {#1} } }
-\cs_new_nopar:Npn \str_length_unsafe:n #1
- { \str_length_aux:n { \str_length_loop:NNNNNNNNN #1 } }
+ {
+ \str_length_aux:n
+ {
+ \str_count_spaces:n {#1}
+ + \exp_after:wN \str_length_loop:NNNNNNNNN \tl_to_str:n {#1}
+ }
+ }
+\cs_new:Npn \str_length_unsafe:n #1
+ {
+ \str_length_aux:n
+ { \str_length_loop:NNNNNNNNN #1 }
+ }
\cs_new:Npn \str_length_ignore_spaces:n #1
{
\str_length_aux:n
@@ -189,7 +207,7 @@
}
\cs_set:Npn \str_length_loop:NNNNNNNNN #1#2#3#4#5#6#7#8#9
{
- \if_catcode:w X #9
+ \if_meaning:w X #9
\exp_after:wN \use_none_delimit_by_q_stop:w
\fi:
\c_nine + \str_length_loop:NNNNNNNNN
@@ -201,12 +219,13 @@
\tl_to_str:n {#1}
{ { } } ~ \q_stop
}
-\cs_set_nopar:Npn \str_head_aux:w #1 ~ %
+\cs_set:Npn \str_head_aux:w #1 ~ %
{ \use_i_delimit_by_q_stop:nw #1 { ~ } }
\cs_new:Npn \str_head_ignore_spaces:n #1
- { \exp_after:wN \use_i_delimit_by_q_stop:nw \tl_to_str:n {#1} { } \q_stop }
-\cs_new_nopar:Npn \str_head_unsafe:n #1
- { \use_i_delimit_by_q_stop:nw #1 { } \q_stop }
+ {
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tl_to_str:n {#1} { } \q_stop
+ }
\cs_new_nopar:Npn \str_tail:N { \exp_args:No \str_tail:n }
\cs_set:Npn \str_tail:n #1
{
@@ -214,107 +233,93 @@
\reverse_if:N \if_charcode:w
\scan_stop: \tl_to_str:n {#1} X X \q_stop
}
-\cs_set_nopar:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 }
+\cs_set:Npn \str_tail_aux:w #1 X #2 \q_stop { \fi: #1 }
\cs_new:Npn \str_tail_ignore_spaces:n #1
{
\exp_after:wN \str_tail_aux_ii:w
- \tl_to_str:n {#1} X { } X \q_stop
+ \tl_to_str:n {#1} X X \q_stop
}
-\cs_new_nopar:Npn \str_tail_unsafe:n #1
- { \str_tail_aux_ii:w #1 X { } X \q_stop }
-\cs_new_nopar:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 }
-\cs_new:Npn \str_skip_do:nn #1
+\cs_new:Npn \str_tail_aux_ii:w #1 #2 X #3 \q_stop { #2 }
+\cs_new:Npn \str_skip_c_zero:w #1;
{
- \if_num:w \int_eval:w #1 > \c_seven
- \exp_after:wN \str_skip_aux:nnnnnnnnn
+ \if_num:w \int_eval:w #1 > \c_eight
+ \exp_after:wN \str_skip_aux:wNNNNNNNN
\else:
- \exp_after:wN \str_skip_end:n
- \fi:
- {#1}
- }
-\cs_new:Npn \str_skip_aux:nnnnnnnnn #1#2#3#4#5#6#7#8#9
- { \exp_args:Nf \str_skip_do:nn { \int_eval:n { #1 - \c_seven } } {#2} }
-\cs_new:Npn \str_skip_end:n #1
- {
- \if_case:w \int_eval:w #1 \int_eval_end:
- \str_skip_end_ii:nwn { XXXXXXX }
- \or: \str_skip_end_ii:nwn { XXXXXX }
- \or: \str_skip_end_ii:nwn { XXXXX }
- \or: \str_skip_end_ii:nwn { XXXX }
- \or: \str_skip_end_ii:nwn { XXX }
- \or: \str_skip_end_ii:nwn { XX }
- \or: \str_skip_end_ii:nwn { X }
- \or: \str_skip_end_ii:nwn { }
- \else: \str_skip_end_ii:nwn { XXXXXXX }
+ \exp_after:wN \str_skip_end:w
+ \int_use:N \int_eval:w
\fi:
+ #1 ;
+ }
+\cs_new:Npn \str_skip_aux:wNNNNNNNN #1; #2#3#4#5#6#7#8#9
+ { \exp_after:wN \str_skip_c_zero:w \int_use:N \int_eval:w #1 - \c_eight ; }
+\cs_new:Npn \str_skip_end:w #1 ;
+ {
+ \exp_after:wN \str_skip_end_ii:NNNNNNNN
+ \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f:
+ \or: \or: \or: \or: \or: \or: \or: \or:
+ }
+\cs_new:Npn \str_skip_end_ii:NNNNNNNN #1#2#3#4#5#6#7#8 { \fi: \c_zero }
+\cs_new:Npn \str_collect_delimit_by_q_stop:w #1;
+ {
+ \exp_after:wN \str_collect_aux:wn
+ \int_use:N \int_eval:w #1 ;
+ { }
}
-\cs_new:Npn \str_skip_end_ii:nwn #1#2 \fi: #3
- { \fi: \use_i:nnnnnnnn {#3} #1 }
-\cs_new:Npn \str_collect_do:nn #1#2
- { \str_collect_aux:n {#1} { \str_collect_end_iii:nwNNNNNNN {#2} } }
-\cs_new:Npn \str_collect_aux:n #1
- {
- \int_compare:nNnTF {#1} > \c_seven
- { \str_collect_aux:nnNNNNNNN }
- { \str_collect_end:n }
- {#1}
- }
-\cs_new:Npn \str_collect_aux:nnNNNNNNN #1#2 #3#4#5#6#7#8#9
- {
- \exp_args:Nf \str_collect_aux:n
- { \int_eval:n { #1 - \c_seven } }
- { #2 #3#4#5#6#7#8#9 }
- }
-\cs_new:Npn \str_collect_end:n #1
- {
- \if_case:w \int_eval:w #1 \int_eval_end:
- \str_collect_end_ii:nwn { { } { } { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } { } }
- \or: \str_collect_end_ii:nwn { { } { } }
- \or: \str_collect_end_ii:nwn { { } }
- \or: \str_collect_end_ii:nwn { }
- \else: \str_collect_end_ii:nwn { { } { } { } { } { } { } { } }
+\cs_new:Npn \str_collect_aux:wn #1 ;
+ {
+ \if_num:w #1 > \c_seven
+ \exp_after:wN \str_collect_aux:wnNNNNNNN
+ \else:
+ \exp_after:wN \str_collect_end:wn
\fi:
+ #1 ;
+ }
+\cs_new:Npn \str_collect_aux:wnNNNNNNN #1; #2 #3#4#5#6#7#8#9
+ {
+ \exp_after:wN \str_collect_aux:wn
+ \int_use:N \int_eval:w #1 - \c_seven ;
+ { #2 #3#4#5#6#7#8#9 }
+ }
+\cs_new:Npn \str_collect_end:wn #1 ;
+ {
+ \exp_after:wN \str_collect_end_ii:nnnnnnnnw
+ \if_case:w \if_num:w #1 > \c_zero #1 \else: 0 \fi: \exp_stop_f:
+ \or: \or: \or: \or: \or: \or: \fi:
}
-\cs_new:Npn \str_collect_end_ii:nwn #1#2 \fi: #3
- { \fi: #3 \q_stop #1 }
-\cs_new:Npn \str_collect_end_iii:nwNNNNNNN #1 #2 \q_stop #3#4#5#6#7#8#9
- { #1 {#2#3#4#5#6#7#8#9} }
+\cs_new:Npn \str_collect_end_ii:nnnnnnnnw #1#2#3#4#5#6#7#8 #9 \q_stop
+ { #1#2#3#4#5#6#7#8 }
\cs_new_nopar:Npn \str_item:Nn { \exp_args:No \str_item:nn }
-\cs_new:Npn \str_item:nn #1#2
+\cs_new:Npn \str_item:nn #1
{
- \exp_args:Nf \tl_to_str:n
- { \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} } {#2} }
+ \exp_last_unbraced:Nf \token_to_str:N
+ \exp_args:Nf \str_item_unsafe:nn { \tl_to_other_str:n {#1} }
}
\cs_new:Npn \str_item_ignore_spaces:nn #1
{ \exp_args:No \str_item_unsafe:nn { \tl_to_str:n {#1} } }
-\cs_new_nopar:Npn \str_item_unsafe:nn #1#2
+\cs_new:Npn \str_item_unsafe:nn #1#2
{
- \exp_args:Nff \str_item_aux:nn
- { \int_eval:n {#2} }
- { \str_length_unsafe:n {#1} }
+ \exp_after:wN \str_item_aux:ww
+ \int_use:N \int_eval:w #2 \exp_after:wN ;
+ \int_value:w \str_length_unsafe:n {#1} ;
#1
\q_stop
}
-\cs_new_nopar:Npn \str_item_aux:nn #1#2
+\cs_new:Npn \str_item_aux:ww #1; #2;
{
\int_compare:nNnTF {#1} < \c_zero
{
\int_compare:nNnTF {#1} < {-#2}
{ \use_none_delimit_by_q_stop:w }
{
- \str_skip_do:nn { #1 + #2 }
- { \use_i_delimit_by_q_stop:nw }
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tex_romannumeral:D \str_skip_c_zero:w #1 + #2 ;
}
}
{
\int_compare:nNnTF {#1} < {#2}
{
- \str_skip_do:nn {#1}
- { \use_i_delimit_by_q_stop:nw }
+ \exp_after:wN \use_i_delimit_by_q_stop:nw
+ \tex_romannumeral:D \str_skip_c_zero:w #1 ;
}
{ \use_none_delimit_by_q_stop:w }
}
@@ -335,28 +340,26 @@
\exp_after:wN \str_substr_aux:www
\int_value:w \str_length_unsafe:n {#1} \exp_after:wN ;
\int_use:N \int_eval:w #2 + \c_zero \exp_after:wN ;
- \int_use:N \int_eval:w \exp_args:Nf \str_substr_unsafe:nN {#3} \c_max_int ;
+ \int_use:N \int_eval:w
+ \exp_args:Nf \str_substr_aux:nN {#3} \c_max_int ;
#1
\q_stop
}
-\cs_new:Npn \str_substr_unsafe:nN #1 #2
+\cs_new:Npn \str_substr_aux:nN #1 #2
{ \tl_if_empty:nTF {#1} {#2} {#1} }
\cs_new:Npn \str_substr_aux:www #1; #2; #3;
{
\exp_args:Nf \str_substr_aux:nnw
- { \str_aux_normalize_range:nn {#2} {#1} }
- { \str_aux_normalize_range:nn {#3} {#1} }
+ { \str_substr_normalize_range:nn {#2} {#1} }
+ { \str_substr_normalize_range:nn {#3} {#1} }
}
\cs_new:Npn \str_substr_aux:nnw #1#2
{
- \str_skip_do:nn {#1}
- {
- \exp_args:Nf \str_collect_do:nn
- { \int_eval:n { #2 - #1 } }
- { \use_i_delimit_by_q_stop:nw }
- }
+ \exp_after:wN \str_collect_delimit_by_q_stop:w
+ \int_use:N \int_eval:w #2 - #1 \exp_after:wN ;
+ \tex_romannumeral:D \str_skip_c_zero:w #1 ;
}
-\cs_new:Npn \str_aux_normalize_range:nn #1#2
+\cs_new:Npn \str_substr_normalize_range:nn #1#2
{
\int_eval:n
{
@@ -383,7 +386,7 @@
\prg_new_conditional:Npnn \str_if_contains_char:NN #1#2 { T , TF }
{
\exp_after:wN \str_if_contains_char_aux:NN \exp_after:wN #2
- #1 { \prg_map_break:n { ? \fi: } }
+ \tl_to_str:n \exp_after:wN {#1} { \prg_map_break:n { ? \fi: } }
\prg_break_point:n { }
\prg_return_false:
}
@@ -394,14 +397,14 @@
\prg_break_point:n { }
\prg_return_false:
}
-\cs_new_nopar:Npn \str_if_contains_char_aux:NN #1#2
+\cs_new:Npn \str_if_contains_char_aux:NN #1#2
{
\if_charcode:w #1 #2
- \exp_after:wN \str_if_contains_char_end:
+ \exp_after:wN \str_if_contains_char_true:
\fi:
\str_if_contains_char_aux:NN #1
}
-\cs_new_nopar:Npn \str_if_contains_char_end:
+\cs_new_nopar:Npn \str_if_contains_char_true:
{ \prg_map_break:n { \prg_return_true: \use_none:n } }
\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF }
{
@@ -433,11 +436,11 @@
\fi:
}
\group_begin:
- \tl_clear:N \l_str_tmpa_tl
+ \tl_clear:N \l_str_internal_tl
\char_set_catcode_other:n { \c_zero }
\tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
- \tl_map_inline:Nn \g_str_result_tl
- { \char_set_lccode:nn {`#1} {`#1} }
+ \exp_args:No \tl_map_inline:nn { \g_str_result_tl " }
+ { \char_set_lccode:nn {`#1} { \c_zero } }
\tl_map_inline:Nn \g_str_result_tl
{
\tl_map_inline:Nn \g_str_result_tl
@@ -448,12 +451,12 @@
\tl_const:cx
{ c_str_byte_ \int_eval:n {"#1##1} _tl }
{ ^^@ #1 ##1 }
- \tl_put_right:Nn \l_str_tmpa_tl { ^^@ }
+ \tl_put_right:Nn \l_str_internal_tl { ^^@ }
}
}
}
\tl_const:Nx \c_str_positive_bytes_tl
- { \exp_after:wN \use_none:n \l_str_tmpa_tl }
+ { \exp_after:wN \use_none:n \l_str_internal_tl }
\group_end:
\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
\cs_new:Npn \str_output_byte:n #1
@@ -474,244 +477,166 @@
}
\cs_new_nopar:Npn \str_output_end:
{ \int_eval_end: _tl \cs_end: }
-\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1
+\cs_new:Npn \str_output_byte_pair_be:n #1
+ {
+ \exp_args:Nf \str_output_byte_pair_aux:nnN
+ { \int_div_truncate:nn { #1 } { "100 } } {#1} \use:nn
+ }
+\cs_new:Npn \str_output_byte_pair_le:n #1
+ {
+ \exp_args:Nf \str_output_byte_pair_aux:nnN
+ { \int_div_truncate:nn { #1 } { "100 } } {#1} \use_ii_i:nn
+ }
+\cs_new:Npn \str_output_byte_pair_aux:nnN #1#2#3
+ {
+ #3
+ { \str_output_byte:n { #1 } }
+ { \str_output_byte:n { #2 - #1 * "100 } }
+ }
+\cs_new_protected:Npn \str_convert_gmap:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_result_loop:NN
+ \exp_after:wN \str_convert_gmap_loop:NN
\exp_after:wN #1
\g_str_result_tl { ? \prg_map_break: }
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2
+\cs_new:Npn \str_convert_gmap_loop:NN #1#2
{
\use_none:n #2
#1#2
- \str_aux_gmap_result_loop:NN #1
+ \str_convert_gmap_loop:NN #1
}
-\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1
+\cs_new_protected:Npn \str_convert_gmap_internal:N #1
{
\tl_gset:Nx \g_str_result_tl
{
- \exp_after:wN \str_aux_gmap_internal_result_loop:Nww
+ \exp_after:wN \str_convert_gmap_internal_loop:Nww
\exp_after:wN #1
\g_str_result_tl \s_tl \q_stop \prg_map_break: \s_tl
\prg_break_point:n { }
}
}
-\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nww #1 #2 \s_tl #3 \s_tl
+\cs_new:Npn \str_convert_gmap_internal_loop:Nww #1 #2 \s_tl #3 \s_tl
{
\use_none_delimit_by_q_stop:w #3 \q_stop
#1 {#3}
- \str_aux_gmap_internal_result_loop:Nww #1
- }
-\cs_new_protected:Npn \str_input:Nn #1#2
- {
- \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
- \tl_set_eq:NN #1 \g_str_result_tl
- }
-\cs_new_protected:Npn \str_ginput:Nn #1#2
- {
- \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2}
- \tl_gset_eq:NN #1 \g_str_result_tl
- }
-\cs_new_protected:Npn \str_escape_use:NNNn #1#2#3#4
- {
- \str_aux_escape:NNNn #1#2#3 {#4}
- \g_str_result_tl
- }
-\cs_new_protected:Npn \str_aux_escape:NNNn #1#2#3#4
+ \str_convert_gmap_internal_loop:Nww #1
+ }
+\cs_new_protected:Npn \str_if_flag_error:nnx #1
+ {
+ \flag_if_raised:nTF {#1}
+ { \msg_kernel_error:nnx { str } }
+ { \use_none:nn }
+ }
+\cs_new_protected:Npn \str_if_flag_no_error:nnx #1#2#3
+ { \flag_if_raised:nT {#1} { \bool_gset_true:N \g_str_error_bool } }
+\cs_new_protected:Npn \str_if_flag_times:nT #1#2
+ { \flag_if_raised:nT {#1} { #2~(x \flag_height:n {#1} ) } }
+\cs_new_protected_nopar:Npn \str_set_convert:Nnnn
+ { \str_convert_aux_i:nNNnnn { } \tl_set_eq:NN }
+\cs_new_protected_nopar:Npn \str_gset_convert:Nnnn
+ { \str_convert_aux_i:nNNnnn { } \tl_gset_eq:NN }
+\prg_new_protected_conditional:Npnn
+ \str_set_convert:Nnnn #1#2#3#4 { T , F , TF }
+ {
+ \bool_gset_false:N \g_str_error_bool
+ \str_convert_aux_i:nNNnnn
+ { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx }
+ \tl_set_eq:NN #1 {#2} {#3} {#4}
+ \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true:
+ }
+\prg_new_protected_conditional:Npnn
+ \str_gset_convert:Nnnn #1#2#3#4 { T , F , TF }
+ {
+ \bool_gset_false:N \g_str_error_bool
+ \str_convert_aux_i:nNNnnn
+ { \cs_set_eq:NN \str_if_flag_error:nnx \str_if_flag_no_error:nnx }
+ \tl_gset_eq:NN #1 {#2} {#3} {#4}
+ \bool_if:NTF \g_str_error_bool \prg_return_false: \prg_return_true:
+ }
+\cs_new_protected:Npn \str_convert_aux_i:nNNnnn #1#2#3#4#5#6
{
\group_begin:
- \cs_set_nopar:Npn \str_aux_escape_unescaped:N { #1 }
- \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 }
- \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 }
- \int_set:Nn \tex_escapechar:D { 92 }
+ #1
\str_gset_other:Nn \g_str_result_tl {#4}
- \int_zero:N \l_str_tmpa_int
- \tl_gset:Nx \g_str_result_tl
- {
- \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl
- \q_recursion_tail \q_recursion_stop
- }
- \tl_gput_left:Nx \g_str_result_tl
- { \str_aux_toks_range:nn \c_zero \l_str_tmpa_int }
- \group_end:
- }
-\cs_new_nopar:Npn \str_aux_escape_loop:N #1
- {
- \cs_if_exist_use:cF { str_aux_escape_\token_to_str:N #1:w }
- { \str_aux_escape_unescaped:N #1 }
- \str_aux_escape_loop:N
- }
-\cs_new_nopar:cpn { str_aux_escape_ \c_backslash_str :w }
- \str_aux_escape_loop:N #1
- {
- \cs_if_exist_use:cF { str_aux_escape_/\token_to_str:N #1:w }
- { \str_aux_escape_escaped:N #1 }
- \str_aux_escape_loop:N
- }
-\cs_new_eq:NN \str_aux_escape_unescaped:N \prg_do_nothing:
-\cs_new_eq:NN \str_aux_escape_escaped:N \prg_do_nothing:
-\cs_new_eq:NN \str_aux_escape_raw:N \prg_do_nothing:
-\cs_new_eq:cN
- { str_aux_escape_ \c_backslash_str q_recursion_tail :w }
- \use_none_delimit_by_q_recursion_stop:w
-\cs_new_eq:cN
- { str_aux_escape_ / \c_backslash_str q_recursion_tail :w }
- \use_none_delimit_by_q_recursion_stop:w
-\cs_new_nopar:cpn { str_aux_escape_~:w } { }
-\cs_new_nopar:cpx { str_aux_escape_/a:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^G }
-\cs_new_nopar:cpx { str_aux_escape_/t:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^I }
-\cs_new_nopar:cpx { str_aux_escape_/n:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^J }
-\cs_new_nopar:cpx { str_aux_escape_/f:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^L }
-\cs_new_nopar:cpx { str_aux_escape_/r:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^M }
-\cs_new_nopar:cpx { str_aux_escape_/e:w }
- { \exp_not:N \str_aux_escape_raw:N \iow_char:N \^^[ }
-\cs_new_nopar:cpn { str_aux_escape_/x:w } \str_aux_escape_loop:N
- {
- \if_false: { \fi: }
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \g_str_result_tl }
- \int_incr:N \l_str_tmpa_int
- \int_zero:N \l_str_char_int
- \str_aux_escape_x_test:N
- }
-\cs_new_protected_nopar:Npx \str_aux_escape_x_test:N #1
- {
- \exp_not:N \token_if_eq_charcode:NNTF \c_space_token #1
- { \exp_not:N \str_aux_escape_x_test:N }
- {
- \l_str_char_int = "0
- \exp_not:N \token_if_eq_charcode:NNTF \c_lbrace_str #1
- { \exp_not:N \str_aux_escape_x_braced_loop:N }
- { \exp_not:N \str_aux_escape_x_unbraced_i:N #1 }
- }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_unbraced_i:N #1
- {
- \str_aux_hexadecimal_use:NTF #1
- { \str_aux_escape_x_unbraced_ii:N }
- { \exp_stop_f: \str_aux_escape_x_end: #1 }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_unbraced_ii:N #1
- {
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \str_aux_escape_x_unbraced_ii:N }
- {
- \str_aux_hexadecimal_use:NTF #1
- { \exp_stop_f: \str_aux_escape_x_end: }
- { \exp_stop_f: \str_aux_escape_x_end: #1 }
- }
- }
-\cs_new_nopar:Npn \str_aux_escape_x_braced_loop:N #1
- {
- \token_if_eq_charcode:NNTF \c_space_token #1
- { \str_aux_escape_x_braced_loop:N }
- {
- \str_aux_hexadecimal_use:NTF #1
- { \str_aux_escape_x_braced_loop:N }
- { \exp_stop_f: \str_aux_escape_x_braced_end:N #1 }
- }
- }
-\cs_new_protected_nopar:Npx \str_aux_escape_x_braced_end:N #1
- {
- \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
- { \exp_not:N \str_aux_escape_x_end: }
- {
- \msg_kernel_error:nn { str } { x-missing-brace }
- \exp_not:N \str_aux_escape_x_end: #1
- }
- }
-\group_begin:
- \char_set_catcode_other:N \^^@
- \cs_new_protected_nopar:Npn \str_aux_escape_x_end:
- {
- \tex_lccode:D \c_zero \l_str_char_int
- \tl_to_lowercase:n
- {
- \tl_gset:Nx \g_str_result_tl
- { \if_false: } \fi:
- \str_aux_escape_raw:N ^^@
- \str_aux_escape_loop:N
- }
- }
-\group_end:
-\cs_new_protected:Npn \str_set_convert:Nnnn
- { \str_convert_aux_i:NNnnn \tl_set_eq:NN }
-\cs_new_protected:Npn \str_gset_convert:Nnnn
- { \str_convert_aux_i:NNnnn \tl_gset_eq:NN }
-\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5
- {
- \group_begin:
- \str_gset_other:Nn \g_str_result_tl {#5}
-
\exp_after:wN \str_convert_aux_ii:wwwnn
- \tl_to_str:n {#3} /// \q_stop
- { from } { input } \str_convert_from_native: \use_i:nn
-
+ \tl_to_str:n {#5} /// \q_stop
+ { decode } { unescape }
+ \prg_do_nothing:
+ \str_convert_decode_:
\exp_after:wN \str_convert_aux_ii:wwwnn
- \tl_to_str:n {#4} /// \q_stop
- { to } { output } \str_convert_to_native: \use_ii:nn
-
+ \tl_to_str:n {#6} /// \q_stop
+ { encode } { escape }
+ \use_ii_i:nn
+ \str_convert_encode_:
\group_end:
- #1 #2 \g_str_result_tl
+ #2 #3 \g_str_result_tl
}
-\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5
+\cs_new_protected:Npn \str_convert_aux_ii:wwwnn
+ #1 / #2 // #3 \q_stop #4#5
{
- \str_convert_aux_iii:nnF {#4} {#1} {native}
- \str_convert_aux_iii:nnF {#5} {#2} { }
- \exp_args:Ncc \str_convert_aux_v:NNnNN
+ \str_convert_aux_iv:nnn {enc} {#4} {#1}
+ \str_convert_aux_iv:nnn {esc} {#5} {#2}
+ \exp_args:Ncc \str_convert_aux_iii:NNnNN
{ str_convert_#4_#1: } { str_convert_#5_#2: } {#2}
}
-\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3
+\cs_new_protected:Npn \str_convert_aux_iii:NNnNN #1#2#3#4#5
{
- \cs_if_exist:cF { str_convert_#1_#2: }
- {
- \exp_args:Nx \str_convert_aux_iv:nnnF
- { \str_aux_lowercase_alphanum:n {#2} }
- {#1} {#2} {#3}
- }
+ \if_meaning:w #1 #5
+ \tl_if_empty:nF {#3}
+ { \msg_kernel_error:nnx { str } { native-escaping } {#3} }
+ #1
+ \else:
+ #4 #2 #1
+ \fi:
}
-\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4
+\cs_new_protected:Npn \str_convert_aux_iv:nnn #1#2#3
{
- \cs_if_exist:cF { str_convert_#2_#1: }
+ \cs_if_exist:cF { str_convert_#2_#3: }
{
- \group_begin:
- \str_load_catcodes:
- \str_load_one:n {#1}
- \group_end:
- }
- \cs_if_exist:cTF { str_convert_#2_#1: }
- { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } }
- {
- \msg_kernel_error:nnx { str } { unknown-#2 } {#3}
- \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: }
+ \exp_args:Nx \str_convert_aux_v:nnnn
+ { \str_convert_lowercase_alphanum:n {#3} }
+ {#1} {#2} {#3}
}
}
-\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5
+\cs_new_protected:Npn \str_convert_aux_v:nnnn #1#2#3#4
{
- #5 { } {#1}
- \cs_if_eq:NNTF #1 #4
+ \cs_if_exist:cF { str_convert_#3_#1: }
{
- \tl_if_empty:nF {#3}
- { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} }
+ \prop_get:NnNF \g_str_file_prop {#1} \l_str_internal_tl
+ { \tl_set:Nn \l_str_internal_tl {#1} }
+ \cs_if_exist:cF { str_convert_#3_ \l_str_internal_tl : }
+ {
+ \file_if_exist:nTF { l3str-#2- \l_str_internal_tl .def }
+ {
+ \group_begin:
+ \str_load_catcodes:
+ \file_input:n { l3str-#2- \l_str_internal_tl .def }
+ \group_end:
+ }
+ {
+ \tl_clear:N \l_str_internal_tl
+ \msg_kernel_error:nnxx { str } { unknown-#2 } {#4} {#1}
+ }
+ }
+ \cs_if_exist:cF { str_convert_#3_#1: }
+ {
+ \cs_gset_eq:cc { str_convert_#3_#1: }
+ { str_convert_#3_ \l_str_internal_tl : }
+ }
}
- { #2 }
- #5 {#1} { }
+ \cs_gset_eq:cc { str_convert_#3_#4: } { str_convert_#3_#1: }
}
-\cs_new:Npn \str_aux_lowercase_alphanum:n #1
+\cs_new:Npn \str_convert_lowercase_alphanum:n #1
{
- \exp_after:wN \str_aux_lowercase_alphanum_loop:N
+ \exp_after:wN \str_convert_lowercase_alphanum_loop:N
\tl_to_str:n {#1} { ? \prg_map_break: }
\prg_break_point:n { }
}
-\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1
+\cs_new:Npn \str_convert_lowercase_alphanum_loop:N #1
{
\use_none:n #1
\if_num:w `#1 < \c_ninety_one
@@ -730,312 +655,287 @@
\fi:
\fi:
\fi:
- \str_aux_lowercase_alphanum_loop:N
- }
-\cs_new_protected_nopar:Npn \str_convert_from_internal: { }
-\cs_new_protected_nopar:Npn \str_convert_to_internal: { }
-\cs_new_protected:Npn \str_load:n #1
- {
- \group_begin:
- \str_load_catcodes:
- \clist_map_inline:nn {#1}
- {
- \exp_args:Nx \str_load_one:n
- { \str_aux_lowercase_alphanum:n {#1} }
- }
- \group_end:
- }
-\cs_new_protected:Npn \str_load_one:n #1
- {
- \cs_if_exist:cF { str_convert_from_ #1 : }
- {
- \cs_if_exist:cF { str_convert_input_#1: }
- {
- \file_if_exist:nTF { l3str- #1 .def }
- { \file_input:n { l3str- #1 .def } }
- { \str_load_alias:n {#1} }
- }
- }
+ \str_convert_lowercase_alphanum_loop:N
}
-\cs_new_protected:Npn \str_load_alias:n #1
- {
- \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl
- {
- \str_load_one:n { \l_str_tmpa_tl }
- \cs_if_exist:cTF
- { str_convert_from_ \l_str_tmpa_tl : }
- {
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from }
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to }
- }
- {
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input }
- \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output }
- }
- }
- { \msg_kernel_error:nnx { str } { file-not-found } {#1} }
- }
-\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3
- { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } }
\cs_new_protected:Npn \str_load_catcodes:
{
\char_set_catcode_escape:N \\
\char_set_catcode_group_begin:N \{
\char_set_catcode_group_end:N \}
+ \char_set_catcode_math_toggle:N \$
+ \char_set_catcode_alignment:N \&
+ \char_set_catcode_parameter:N \#
\char_set_catcode_math_superscript:N \^
\char_set_catcode_ignore:N \ %
- \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN }
+ \char_set_catcode_space:N \~
+ \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFILNPSTUX }
\char_set_catcode_letter:N
- \tl_map_function:nN { 0123456789"'?*+ }
+ \tl_map_function:nN { 0123456789"'?*+-.(),`!/<>[];= }
\char_set_catcode_other:N
\char_set_catcode_comment:N \%
\int_set:Nn \tex_endlinechar:D {32}
}
-\cs_new_protected:Npn \str_convert_from_eight_bit:n #1
+\pdftex_if_engine:TF
+ { \cs_new_eq:NN \str_filter_bytes:n \use:n }
+ {
+ \cs_new:Npn \str_filter_bytes:n #1
+ {
+ \str_filter_bytes_aux:N #1
+ { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \cs_new:Npn \str_filter_bytes_aux:N #1
+ {
+ \use_none:n #1
+ \if_num:w `#1 < 256 \exp_stop_f:
+ #1
+ \else:
+ \flag_raise:n { str_byte }
+ \fi:
+ \str_filter_bytes_aux:N
+ }
+ }
+\pdftex_if_engine:TF
+ { \cs_new_protected_nopar:Npn \str_convert_unescape_: { } }
+ {
+ \cs_new_protected_nopar:Npn \str_convert_unescape_:
+ {
+ \flag_clear:n { str_byte }
+ \tl_gset:Nx \g_str_result_tl
+ { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
+ \str_if_flag_error:nnx { str_byte } { non-byte } { bytes }
+ }
+ }
+\cs_new_eq:NN \str_convert_unescape_bytes: \str_convert_unescape_:
+\cs_new_protected_nopar:Npn \str_convert_escape_: { }
+\cs_new_eq:NN \str_convert_escape_bytes: \str_convert_escape_:
+\cs_new_protected_nopar:Npn \str_convert_decode_:
+ { \str_convert_gmap:N \str_decode_native_char:N }
+\cs_new:Npn \str_decode_native_char:N #1
+ { #1 \s_tl \int_value:w `#1 \s_tl }
+\group_begin:
+ \char_set_catcode_other:n { 0 }
+ \pdftex_if_engine:TF
+ {
+ \cs_new_protected_nopar:Npn \str_convert_encode_:
+ {
+ \flag_clear:n { str_error }
+ \str_convert_gmap_internal:N \str_encode_native_char:n
+ \str_if_flag_error:nnx { str_error }
+ { pdfTeX-native-overflow } { }
+ }
+ \cs_new:Npn \str_encode_native_char:n #1
+ {
+ \if_num:w #1 < \c_two_hundred_fifty_six
+ \str_output_byte:n {#1}
+ \else:
+ \flag_raise:n { str_error }
+ ?
+ \fi:
+ }
+ \msg_kernel_new:nnnn { str } { pdfTeX-native-overflow }
+ { Character~code~too~large~for~pdfTeX. }
+ {
+ The~pdfTeX~engine~only~supports~8-bit~characters:~
+ valid~character~codes~are~in~the~range~[0,255].~
+ To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
+ }
+ }
+ {
+ \cs_new_protected_nopar:Npn \str_convert_encode_:
+ {
+ \tex_lccode:D \c_zero \c_zero
+ \int_zero:N \l_str_internal_int
+ \tl_set_eq:NN \l_str_internal_tl \g_str_result_tl
+ \tl_gclear:N \g_str_result_tl
+ \exp_after:wN \str_encode_native_loop:w
+ \l_str_internal_tl \s_tl { \q_stop \prg_map_break: } \s_tl
+ \prg_break_point:n { \str_encode_native_flush: }
+ }
+ \cs_new_protected:Npn \str_encode_native_loop:w #1 \s_tl #2 \s_tl
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ \if_num:w \l_str_internal_int = \c_two_hundred_fifty_five
+ \str_encode_native_flush:
+ \l_str_internal_int \c_zero
+ \fi:
+ \tex_advance:D \l_str_internal_int \c_one
+ \tex_lccode:D \l_str_internal_int \int_eval:w #2 \int_eval_end:
+ \str_encode_native_loop:w
+ }
+ \cs_new_protected_nopar:Npn \str_encode_native_flush:
+ {
+ \use:x
+ {
+ \tl_to_lowercase:n
+ {
+ \tl_gput_right:Nx \exp_not:N \g_str_result_tl
+ {
+ \exp_after:wN \str_encode_native_filter:N
+ \c_str_positive_bytes_tl
+ { ? = \c_zero \fi: \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+ }
+ }
+ \cs_new:Npn \str_encode_native_filter:N #1
+ {
+ \if_num:w `#1 > \l_str_internal_int
+ \exp_after:wN \prg_map_break:
+ \fi:
+ \if_num:w \tex_lccode:D `#1 = \c_zero
+ ^^@
+ \else:
+ #1
+ \fi:
+ \str_encode_native_filter:N
+ }
+ }
+\group_end:
+\cs_new_protected:Npn \str_declare_eight_bit_encoding:nnn #1#2#3
+ {
+ \tl_set:Nn \l_str_internal_tl {#1}
+ \cs_new_protected_nopar:cpn { str_convert_decode_#1: }
+ { \str_convert_decode_eight_bit:n {#1} }
+ \cs_new_protected_nopar:cpn { str_convert_encode_#1: }
+ { \str_convert_encode_eight_bit:n {#1} }
+ \tl_const:cn { c_str_encoding_#1_tl } {#2}
+ \tl_const:cn { c_str_encoding_#1_missing_tl } {#3}
+ }
+\cs_new_protected:Npn \str_convert_decode_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_tmpa_int
- \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn
+ \int_zero:N \l_str_internal_int
+ \exp_last_unbraced:Nx \str_decode_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
\prg_break_point:n { }
- \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n
+ \exp_last_unbraced:Nx \str_decode_eight_bit_load_missing:n
{ \tl_use:c { c_str_encoding_#1_missing_tl } }
{ \q_stop \prg_map_break: }
\prg_break_point:n { }
- \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N
+ \flag_clear:n { str_error }
+ \str_convert_gmap:N \str_decode_eight_bit_char:N
+ \str_if_flag_error:nnx { str_error } { decode-8-bit } {#1}
\group_end:
}
-\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2
+\cs_new_protected:Npn \str_decode_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN { \int_value:w "#2 }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_from_eight_bit_load:nn
+ \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_internal_int \exp_after:wN { \int_value:w "#2 }
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_decode_eight_bit_load:nn
}
-\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1
+\cs_new_protected:Npn \str_decode_eight_bit_load_missing:n #1
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#1 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#1 sp \scan_stop:
- \tex_toks:D \l_str_tmpa_int \exp_after:wN
+ \tex_dimen:D "#1 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_internal_int \exp_after:wN
{ \int_use:N \c_str_replacement_char_int }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_from_eight_bit_load_missing:n
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_decode_eight_bit_load_missing:n
}
-\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
+\cs_new:Npn \str_decode_eight_bit_char:N #1
{
#1 \s_tl
- \if_num:w \tex_dimen:D `#1 < \l_str_tmpa_int
+ \if_num:w \tex_dimen:D `#1 < \l_str_internal_int
\if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D
\fi:
\fi:
\int_value:w `#1 \s_tl
}
-\cs_new_protected:Npn \str_convert_to_eight_bit:n #1
+\cs_new_protected:Npn \str_convert_encode_eight_bit:n #1
{
\group_begin:
- \int_zero:N \l_str_tmpa_int
- \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn
+ \int_zero:N \l_str_internal_int
+ \exp_last_unbraced:Nx \str_encode_eight_bit_load:nn
{ \tl_use:c { c_str_encoding_#1_tl } }
{ \q_stop \prg_map_break: } { }
\prg_break_point:n { }
- \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n
+ \flag_clear:n { str_error }
+ \str_convert_gmap_internal:N \str_encode_eight_bit_char:n
+ \str_if_flag_error:nnx { str_error } { encode-8-bit } {#1}
\group_end:
}
-\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2
+\cs_new_protected:Npn \str_encode_eight_bit_load:nn #1#2
{
\use_none_delimit_by_q_stop:w #1 \q_stop
- \tex_dimen:D "#2 = \l_str_tmpa_int sp \scan_stop:
- \tex_skip:D \l_str_tmpa_int = "#2 sp \scan_stop:
- \exp_args:NNf \tex_toks:D \l_str_tmpa_int
+ \tex_dimen:D "#2 = \l_str_internal_int sp \scan_stop:
+ \tex_skip:D \l_str_internal_int = "#2 sp \scan_stop:
+ \exp_args:NNf \tex_toks:D \l_str_internal_int
{ \str_output_byte:n { "#1 } }
- \tex_advance:D \l_str_tmpa_int \c_one
- \str_convert_to_eight_bit_load:nn
+ \tex_advance:D \l_str_internal_int \c_one
+ \str_encode_eight_bit_load:nn
}
-\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
+\cs_new:Npn \str_encode_eight_bit_char:n #1
{
\if_num:w #1 > \c_max_register_int
- \msg_expandable_kernel_error:nnn
- { str } { eight-bit-to-byte-overflow } {#1}
+ \flag_raise:n { str_error }
\else:
- \if_num:w \tex_dimen:D #1 < \l_str_tmpa_int
+ \if_num:w \tex_dimen:D #1 < \l_str_internal_int
\if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
\tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
\exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
\fi:
\fi:
- \str_convert_to_eight_bit_aux_ii:n {#1}
+ \str_encode_eight_bit_char_aux:n {#1}
\fi:
}
-\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
+\cs_new:Npn \str_encode_eight_bit_char_aux:n #1
{
\if_num:w #1 < \c_two_hundred_fifty_six
\str_output_byte:n {#1}
\else:
- \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte }
+ \flag_raise:n { str_error }
\fi:
}
-\cs_new_protected:Npn \str_encoding_eight_bit:n #1
+\msg_kernel_new:nnn { str } { unknown-esc }
+ { Escaping~scheme~'#1'~(filtered:~'#2')~unknown. }
+\msg_kernel_new:nnn { str } { unknown-enc }
+ { Encoding~scheme~'#1'~(filtered:~'#2')~unknown. }
+\msg_kernel_new:nnnn { str } { native-escaping }
+ { The~'native'~encoding~scheme~does~not~support~any~escaping. }
{
- \cs_new_protected:cpn { str_convert_from_#1: }
- { \str_convert_from_eight_bit:n {#1} }
- \cs_new_protected:cpn { str_convert_to_#1: }
- { \str_convert_to_eight_bit:n {#1} }
- }
-\pdftex_if_engine:TF
- { \cs_new_eq:NN \str_filter_bytes:n \use:n }
- {
- \cs_new_nopar:Npn \str_filter_bytes:n #1
- {
- \str_filter_bytes_aux:N #1
- { ? \prg_map_break: }
- \prg_break_point:n { }
- }
- \cs_new_nopar:Npn \str_filter_bytes_aux:N #1
- {
- \use_none:n #1
- \if_num:w `#1 < 256 \exp_stop_f: #1 \fi:
- \str_filter_bytes_aux:N
- }
+ Since~native~strings~do~not~consist~in~bytes,~
+ none~of~the~escaping~methods~make~sense.~
+ The~specified~escaping,~'#1',~will be ignored.
}
-\pdftex_if_engine:TF
- { \cs_new_protected_nopar:Npn \str_convert_input_: { } }
+\msg_kernel_new:nnn { str } { file-not-found }
+ { File~'l3str-#1.def'~not~found. }
+\pdftex_if_engine:F
{
- \cs_new_protected_nopar:Npn \str_convert_input_:
+ \msg_kernel_new:nnnn { str } { non-byte }
+ { String~invalid~in~escaping~'#1':~it~may~only~contain~bytes. }
{
- \tl_gset:Nx \g_str_result_tl
- { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
- }
- }
-\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_:
-\cs_new_protected_nopar:Npn \str_convert_output_: { }
-\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_:
-\cs_new_protected_nopar:Npn \str_convert_from_native:
- { \str_aux_gmap_result:N \str_convert_from_native_aux:N }
-\cs_new:Npn \str_convert_from_native_aux:N #1
- { #1 \s_tl \int_value:w `#1 \s_tl }
-\cs_new_protected_nopar:Npn \str_convert_to_native:
- {
- \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl
- \str_aux_convert_to_native:n
- {
- \exp_after:wN \str_convert_to_native_aux:w
- \l_str_tmpa_tl \s_tl { \q_stop \prg_map_break: } \s_tl
- }
- }
-\cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 \s_tl #2 \s_tl
- {
- \use_none_delimit_by_q_stop:w #2 \q_stop
- \str_aux_convert_to_native_step:n {#2}
- \str_convert_to_native_aux:w
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native:n #1
- {
- \tex_lccode:D \c_zero \c_zero
- \int_zero:N \l_str_tmpa_int
- \tl_gclear:N \g_str_result_tl
- #1
- \prg_break_point:n { \str_aux_convert_to_native_flush: }
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native_step:n #1
- {
- \if_num:w \l_str_tmpa_int = \c_two_hundred_fifty_five
- \str_aux_convert_to_native_flush:
- \l_str_tmpa_int \c_zero
- \fi:
- \tex_advance:D \l_str_tmpa_int \c_one
- \if_int_compare:w #1 > \c_max_char_int
- \msg_kernel_error:nnx { str } { overflow } {#1}
- \tex_lccode:D \l_str_tmpa_int \c_str_replacement_char_int
- \else:
- \tex_lccode:D \l_str_tmpa_int \int_eval:w #1 \int_eval_end:
- \fi:
- }
-\cs_new_protected_nopar:Npn \str_aux_convert_to_native_flush:
- {
- \use:x
- {
- \tl_to_lowercase:n
+ Some~characters~in~the~string~you~asked~to~convert~are~not~
+ 8-bit~characters.~Perhaps~the~string~is~a~'native'~Unicode~string?~
+ If~it~is,~try~using\\
+ \\
+ \iow_indent:n
{
- \tl_gput_right:Nx \exp_not:N \g_str_result_tl
- {
- \exp_after:wN \str_aux_convert_to_native_filter:N
- \c_str_positive_bytes_tl
- { ? = \c_zero \fi: \prg_map_break: }
- \prg_break_point:n { }
- }
+ \iow_char:N\\str_set_convert:Nnnn \\
+ \ \ <str~var>~\{~<string>~\}~\{~native~\}~\{~<target~encoding>~\}
}
}
}
-\group_begin:
- \char_set_catcode_other:n { 0 }
- \cs_new:Npn \str_aux_convert_to_native_filter:N #1
- {
- \if_num:w `#1 > \l_str_tmpa_int
- \exp_after:wN \prg_map_break:
- \fi:
- \if_num:w \tex_lccode:D `#1 = \c_zero
- ^^@
- \else:
- #1
- \fi:
- \str_aux_convert_to_native_filter:N
- }
-\group_end:
-\msg_kernel_new:nnnn { str } { x-missing-brace }
- { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. }
+\msg_kernel_new:nnnn { str } { decode-8-bit }
+ { Invalid~string~in~encoding~'#1'. }
{
- You~wrote~something~like~
- `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~
- The~closing~brace~is~missing.
+ LaTeX~came~across~a~byte~which~is~not~defined~to~represent~
+ any~character~in~the~encoding~'#1'.
}
-\msg_kernel_new:nnnn { str } { overflow }
- { Character~code~#1~too~big. }
+\msg_kernel_new:nnnn { str } { encode-8-bit }
+ { Unicode~string~cannot~be~converted~to~encoding~'#1'. }
{
- \int_compare:nNnTF {#1} > { 1114111 }
- { The~Unicode~standard~limits~code~points~to~1114111. }
- {
- The~pdfTeX~engine~only~supports~8-bit~characters:~
- valid~character~codes~are~in~the~range~[0,255].~
- To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
- }
+ The~encoding~'#1'~only~contains~a~subset~of~all~Unicode~characters.~
+ LaTeX~was~asked~to~convert~a~string~to~that~encoding,~but~that~
+ string~contains~a~character~that~'#1'~does~not~support.
}
-\msg_kernel_new:nnn { str } { convert-input }
- { Input~scheme~`#1'~unknown. }
-\msg_kernel_new:nnn { str } { convert-from }
- { Encoding~`#1'~unknown. }
-\msg_kernel_new:nnn { str } { convert-to }
- { Encoding~`#1'~unknown.~Using~UTF-8. }
-\msg_kernel_new:nnn { str } { convert-output }
- { Output~scheme~`#1~unknown. }
-\msg_kernel_new:nnn { str } { to-native }
- { Use~\str_set_convert:Nnn to~convert~to~native~strings. }
-\msg_kernel_new:nnn { str } { unicode-surrogate }
- { Code~point~#1~is~an~unpaired~surrogate. }
-\msg_kernel_new:nnn { str } { utf16-surrogate }
- { UTF-16~unpaired~surrogate~#1. }
-\msg_kernel_new:nnn { str } { utf16-odd }
- { UTF-16~dangling~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-extra-conti }
- { UTF-8~extra~continuation~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-missing-conti }
- { UTF-8~missing~continuation~byte~#1. }
-\msg_kernel_new:nnn { str } { utf8-invalid-byte }
- { Byte~#1~cannot~appear~in~UTF-8. }
-\msg_kernel_new:nnn { str } { utf8-overlong }
- { Overlong~UTF-8~byte~sequence~for~code~point~#1. }
-\msg_kernel_new:nnn { str } { utf8-premature-end }
- { Incomplete~last~UTF-8~character. }
-\msg_kernel_new:nnn { str } { utf8-overflow }
- { Code~point~#1~too~large~(UTF-8). }
-\msg_kernel_new:nnn { str } { utf32-overflow }
- { Code~point~too~large~(UTF-32~`#1'). }
-\msg_kernel_new:nnn { str } { utf32-truncated }
- { Truncated~UTF-32~string~`...#1'. }
\cs_set:Npn \str_length_skip_spaces:N
{ \exp_args:No \str_length_skip_spaces:n }
\cs_set_eq:NN \str_length_skip_spaces:n \str_length_ignore_spaces:n
diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty
index d45f996bdc4..6f82415d186 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3tl-analysis.sty
@@ -91,32 +91,6 @@
\group_align_safe_end:
\group_end:
}
-\cs_new_protected:Npn \tl_set_analysis:Nn #1#2
- {
- \tl_analysis:n {#2}
- \tl_set_eq:NN #1 \g_tl_analysis_result_tl
- }
-\cs_new_protected:Npn \tl_set_analysis_from_str:Nn #1#2
- {
- \str_gset_other:Nn \g_tl_analysis_result_tl {#2}
- \tl_set:Nx #1
- {
- \exp_after:wN \tl_analysis_from_str_loop:N
- \g_tl_analysis_result_tl \prg_map_break:
- \prg_break_point:n { }
- }
- }
-\group_begin:
- \char_set_catcode_other:N A
- \char_set_catcode_other:N C
- \cs_new_protected:Npn \tl_analysis_from_str_loop:N #1
- {
- #1 \s_tl
- \if_charcode:w #1 ~ A \else: C \fi:
- \int_value:w `#1 \s_tl
- \tl_analysis_from_str_loop:N
- }
-\group_end:
\cs_new_protected:Npn \tl_analysis_setup:n #1
{
\int_set_eq:NN \tex_escapechar:D \c_minus_one
@@ -363,6 +337,26 @@
32 \s_tl
\tl_analysis_ii_loop:w #1 ;
}
+\cs_new_protected:Npn \tl_analysis_map_inline:nn #1
+ {
+ \tl_analysis:n {#1}
+ \int_gincr:N \g_prg_map_int
+ \exp_args:Nc \tl_analysis_map_inline_aux:Nn
+ { tl_analysis_map_inline_ \int_use:N \g_prg_map_int :wNw }
+ }
+\cs_new_protected:Npn \tl_analysis_map_inline_aux:Nn #1#2
+ {
+ \cs_gset_protected:Npn #1 ##1 \s_tl ##2 ##3 \s_tl
+ {
+ \use_none:n ##2
+ #2
+ #1
+ }
+ \exp_after:wN #1
+ \g_tl_analysis_result_tl
+ \s_tl { ? \prg_map_break: } \s_tl
+ \prg_break_point:n { \int_gdecr:N \g_prg_map_int }
+ }
\cs_new_protected:Npn \tl_show_analysis:N #1
{
\exp_args:No \tl_analysis:n {#1}
diff --git a/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty b/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty
index d95d1b691d5..590062f47ae 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/xgalley/l3galley.sty
@@ -14,7 +14,7 @@
%% Do not distribute a modified version of this file.
%%
%% File: l3galley.dtx Copyright (C) 1999-2001, 2004-2009 Frank Mittelbach
-%% (C) 2010,2011 The LaTeX3 Project
+%% (C) 2010-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -47,8 +47,8 @@
%%
%% -----------------------------------------------------------------------
%%
-\RequirePackage{l3names}
-\GetIdInfo$Id: l3galley.dtx 3205 2012-01-18 22:09:05Z joseph $
+\RequirePackage{xparse}
+\GetIdInfo$Id: l3galley.dtx 3253 2012-01-28 14:59:55Z joseph $
{L3 Experimental galley code}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
@@ -252,7 +252,7 @@
}
\par
}
-\cs_new_protected_nopar:Npn \galley_display_penalty:N #1
+\cs_new_protected:Npn \galley_display_penalty:N #1
{
\tl_if_empty:NF #1
{
@@ -264,7 +264,7 @@
\int_gset:Nn \g_galley_penalty_int {#1}
}
}
-\cs_new_protected_nopar:Npn \galley_display_vspace:N #1
+\cs_new_protected:Npn \galley_display_vspace:N #1
{
\tl_if_empty:NF #1
{
@@ -397,7 +397,7 @@
\g_galley_restore_running_tl
\tl_gclear:N \g_galley_restore_running_tl
}
-\cs_new_protected_nopar:Npn \galley_margins_set_absolute:nn #1#2
+\cs_new_protected:Npn \galley_margins_set_absolute:nn #1#2
{
\dim_set:Nn \l_galley_total_left_margin_dim {#1}
\dim_set:Nn \l_galley_total_right_margin_dim {#2}
@@ -408,7 +408,7 @@
- \l_galley_total_right_margin_dim
}
}
-\cs_new_protected_nopar:Npn \galley_margins_set_relative:nn #1#2
+\cs_new_protected:Npn \galley_margins_set_relative:nn #1#2
{
\dim_add:Nn \l_galley_total_left_margin_dim {#1}
\dim_add:Nn \l_galley_total_right_margin_dim {#2}
@@ -419,7 +419,7 @@
- \l_galley_total_right_margin_dim
}
}
-\cs_new_protected_nopar:Npn \galley_parshape_fixed_lines:nnn #1#2#3
+\cs_new_protected:Npn \galley_parshape_fixed_lines:nnn #1#2#3
{
\bool_gset_true:N \g_galley_parshape_set_bool
\bool_set_true:N \l_galley_parshape_fixed_lines_bool
@@ -428,7 +428,7 @@
\clist_set:Nn \l_galley_parshape_right_indent_clist {#3}
\bool_set_true:N \l_galley_parshape_resume_std_bool
}
-\cs_new_protected_nopar:Npn \galley_parshape_multi_par:nnnN #1#2#3#4
+\cs_new_protected:Npn \galley_parshape_multi_par:nnnN #1#2#3#4
{
\bool_gset_true:N \g_galley_parshape_set_bool
\bool_set_true:N \l_galley_parshape_multipar_bool
@@ -438,7 +438,7 @@
\clist_set:Nn \l_galley_parshape_right_indent_clist {#3}
\bool_set_eq:NN \l_galley_parshape_resume_std_bool #4
}
-\cs_new_protected_nopar:Npn \galley_parshape_single_par:nnnN #1#2#3#4
+\cs_new_protected:Npn \galley_parshape_single_par:nnnN #1#2#3#4
{
\bool_gset_true:N \g_galley_parshape_set_bool
\bool_set_false:N \l_galley_parshape_multipar_bool
@@ -503,13 +503,12 @@
\dim_use:N \l_galley_text_width_dim
}
}
-\cs_new_nopar:Npn \galley_set_parshape_map:nn #1#2
+\cs_new:Npn \galley_set_parshape_map:nn #1#2
{ \galley_set_parshape_map_aux:nw { } #1 , \q_mark #2 , \q_stop }
\cs_generate_variant:Nn \galley_set_parshape_map:nn { oo }
-\cs_new_nopar:Npn \galley_set_parshape_map_aux:nw
- #1#2 , #3 \q_mark #4 , #5 \q_stop
+\cs_new:Npn \galley_set_parshape_map_aux:nw #1#2 , #3 \q_mark #4 , #5 \q_stop
{
- \bool_if:nTF { \tl_if_empty_p:n {#3} || \tl_if_empty_p:n {#4} }
+ \bool_if:nTF { \tl_if_empty_p:n {#3} || \tl_if_empty_p:n {#5} }
{
#1
\dim_eval:n { \l_galley_total_left_margin_dim + ( #2 ) }
@@ -548,15 +547,15 @@
\dim_use:N \l_galley_text_width_dim
}
}
-\cs_new_nopar:Npn \galley_generate_parshape_lines_aux:n #1
+\cs_new:Npn \galley_generate_parshape_lines_aux:n #1
{
\etex_parshapeindent:D #1
~
\etex_parshapelength:D #1
}
-\cs_new_protected_nopar:Npn \galley_set_user_penalty:n #1
+\cs_new_protected:Npn \galley_set_user_penalty:n #1
{ \tl_gset:Nx \g_galley_interpar_penalty_user_tl { \int_eval:n {#1} } }
-\cs_new_protected_nopar:Npn \galley_set_user_vspace:n #1
+\cs_new_protected:Npn \galley_set_user_vspace:n #1
{ \tl_gset:Nx \g_galley_interpar_vspace_user_tl { \skip_eval:n {#1} } }
\dim_set:Nn \parskip \c_zero_dim
\cs_undefine:N \parskip
@@ -567,7 +566,7 @@
\cs_new_eq:NN \l_galley_par_end_skip \tex_parfillskip:D
\cs_new_eq:NN \l_galley_par_indent_dim \tex_parindent:D
\cs_new_eq:NN \l_galley_last_line_fit_int \etex_lastlinefit:D
-\cs_new_protected_nopar:Npn \galley_set_interword_spacing:N #1
+\cs_new_protected:Npn \galley_set_interword_spacing:N #1
{
\bool_if:NTF #1
{ % TODO Hook for font changes required!
@@ -578,6 +577,8 @@
}
{ \tex_spaceskip:D \c_zero_dim }
}
+\int_new:N \l_galley_hyphen_left_int
+\int_set:Nn \l_galley_hyphen_left_int { \tex_lefthyphenmin:D }
\cs_new_eq:NN \l_galley_binop_penalty_int \tex_binoppenalty:D
\cs_new_eq:NN \l_galley_double_hyphen_demerits_int \tex_doublehyphendemerits:D
\cs_new_eq:NN \l_galley_emergency_stretch_skip \tex_emergencystretch:D
@@ -589,6 +590,21 @@
\cs_new_eq:NN \l_galley_mismatch_demerits_int \tex_adjdemerits:D
\cs_new_eq:NN \l_galley_relation_penalty_int \tex_relpenalty:D
\cs_new_eq:NN \l_galley_linebreak_tolerance_int \tex_tolerance:D
+\cs_new_protected:Npn \galley_break_line:Nn #1#2
+ {
+ \mode_if_vertical:TF
+ { \msg_kernel_error:nn { galley } { no-line-to-end } }
+ {
+ \tex_unskip:D
+ \bool_if:NF #1
+ { \tex_vadjust:D { \tex_penalty:D \c_ten_thousand } }
+ \dim_compare:nNnF {#2} = \c_zero_dim
+ { \tex_vadjust:D { \skip_vertical:n {#2} } }
+ \tex_penalty:D \c_ten_thousand
+ \tex_hfil:D
+ \tex_penalty:D -\c_ten_thousand
+ }
+ }
\cs_new_eq:NN \l_galley_broken_penalty_int \tex_brokenpenalty:D
\cs_new_eq:NN \l_galley_interline_penalty_int \tex_interlinepenalty:D
\cs_new_eq:NN \l_galley_parbreak_badness_int \tex_vbadness:D
@@ -597,7 +613,7 @@
\cs_new_eq:NN \l_galley_pre_display_penalty_int \tex_predisplaypenalty:D
\clist_new:N \l_galley_club_penalties_clist
\clist_new:N \l_galley_line_penalties_clist
-\cs_new_protected_nopar:Npn \galley_set_display_widow_penalties:n #1
+\cs_new_protected:Npn \galley_set_display_widow_penalties:n #1
{
\etex_displaywidowpenalties:D
\int_eval:w \clist_length:n {#1} + \c_one \int_eval_end:
@@ -605,7 +621,7 @@
\c_zero
}
\cs_generate_variant:Nn \galley_set_display_widow_penalties:n { V , v }
-\cs_new_protected_nopar:Npn \galley_set_widow_penalties:n #1
+\cs_new_protected:Npn \galley_set_widow_penalties:n #1
{
\etex_widowpenalties:D
\int_eval:w \clist_length:n {#1} + \c_one \int_eval_end:
@@ -613,20 +629,20 @@
\c_zero
}
\cs_generate_variant:Nn \galley_set_widow_penalties:n { V , v }
-\cs_new_nopar:Npn \galley_set_aux:n #1 { #1 ~ }
-\cs_new_protected_nopar:Npn \galley_set_club_penalties:n #1
+\cs_new:Npn \galley_set_aux:n #1 { #1 ~ }
+\cs_new_protected:Npn \galley_set_club_penalties:n #1
{
\clist_set:Nn \l_galley_club_penalties_clist {#1}
\galley_calc_interline_penalties:
}
\cs_generate_variant:Nn \galley_set_club_penalties:n { V , v }
-\cs_new_protected_nopar:Npn \galley_set_interline_penalties:n #1
+\cs_new_protected:Npn \galley_set_interline_penalties:n #1
{
\clist_set:Nn \l_galley_line_penalties_clist {#1}
\galley_calc_interline_penalties:
}
\cs_generate_variant:Nn \galley_set_interline_penalties:n { V , v }
-\cs_new_protected_nopar:Npn \galley_set_display_club_penalties:n #1
+\cs_new_protected:Npn \galley_set_display_club_penalties:n #1
{
\etex_clubpenalties:D
\int_eval:w \clist_length:n {#1} + \c_one \int_eval_end:
@@ -635,7 +651,7 @@
\galley_calc_interline_penalties:
}
\cs_generate_variant:Nn \galley_set_display_club_penalties:n { V , v }
-\cs_new_protected_nopar:Npn \galley_set_interline_penalty:n #1
+\cs_new_protected:Npn \galley_set_interline_penalty:n #1
{
\int_compare:nNnTF { \etex_interlinepenalties:D \c_zero } = \c_zero
{
@@ -643,7 +659,7 @@
\galley_calc_interline_penalties:
}
{
- \cs_set_nopar:Npn \galley_set_interline_penalty_aux_ii:n ##1
+ \cs_set:Npn \galley_set_interline_penalty_aux_ii:n ##1
{
\int_eval:w
\etex_interlinepenalties:D ##1
@@ -655,7 +671,7 @@
{ \clist_length:N \l_galley_line_penalties_clist } {#1}
}
}
-\cs_new_protected_nopar:Npn \galley_set_interline_penalty_aux:nn #1#2
+\cs_new_protected:Npn \galley_set_interline_penalty_aux:nn #1#2
{
\etex_interlinepenalties:D
\etex_interlinepenalties:D \c_zero
@@ -666,9 +682,9 @@
\galley_set_interline_penalty_aux_ii:n
\int_eval:w #2 \int_eval_end:
}
-\cs_new_nopar:Npn \galley_set_interline_penalty_aux_i:n #1
+\cs_new:Npn \galley_set_interline_penalty_aux_i:n #1
{ \etex_interlinepenalties:D \int_eval:w #1 \int_eval_end: }
-\cs_new_nopar:Npn \galley_set_interline_penalty_aux_ii:n #1 { }
+\cs_new:Npn \galley_set_interline_penalty_aux_ii:n #1 { }
\cs_new_protected_nopar:Npn \galley_calc_interline_penalties:
{
\exp_args:Nff \galley_calc_interline_penalties_aux:nn
@@ -692,8 +708,7 @@
}
{ \clist_length:N \l_galley_line_penalties_clist }
}
-\cs_new_protected_nopar:Npn \galley_calc_interline_penalties_aux:nn
- #1#2
+\cs_new_protected:Npn \galley_calc_interline_penalties_aux:nn #1#2
{
\etex_interlinepenalties:D #1 ~
\prg_stepwise_function:nnnN \c_one \c_one {#2}
@@ -702,8 +717,7 @@
\galley_calc_interline_penalties_aux_ii:n
\etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
}
-\cs_new_nopar:Npn \galley_calc_interline_penalties_aux_i:n
- #1
+\cs_new:Npn \galley_calc_interline_penalties_aux_i:n #1
{
\int_eval:w
\clist_item:Nn \l_galley_line_penalties_clist { #1 - \c_one }
@@ -712,8 +726,7 @@
- \etex_clubpenalties:D #1 ~
\int_eval_end:
}
-\cs_new_nopar:Npn \galley_calc_interline_penalties_aux_ii:n
- #1
+\cs_new:Npn \galley_calc_interline_penalties_aux_ii:n #1
{
\int_eval:w
\etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero
@@ -722,11 +735,11 @@
- \etex_clubpenalties:D #1 ~
\int_eval_end:
}
-\cs_new_protected_nopar:Npn \galley_save_club_penalties:N #1
+\cs_new_protected:Npn \galley_save_club_penalties:N #1
{ \clist_set_eq:NN #1 \l_galley_club_penalties_clist }
-\cs_new_protected_nopar:Npn \galley_save_interline_penalties:N #1
+\cs_new_protected:Npn \galley_save_interline_penalties:N #1
{ \clist_set_eq:NN #1 \l_galley_line_penalties_clist }
-\cs_new_protected_nopar:Npn \galley_save_display_club_penalties:N #1
+\cs_new_protected:Npn \galley_save_display_club_penalties:N #1
{
\clist_set:Nx #1
{
@@ -735,9 +748,9 @@
\galley_save_display_club_penalties:_aux:n
}
}
-\cs_new_nopar:Npn \galley_save_display_club_penalties:_aux:n #1
+\cs_new:Npn \galley_save_display_club_penalties:_aux:n #1
{ \int_use:N \etex_clubpenaltes:D \int_eval:w #1 \int_eval_end: , }
-\cs_new_protected_nopar:Npn \galley_save_display_widow_penalties:N #1
+\cs_new_protected:Npn \galley_save_display_widow_penalties:N #1
{
\clist_set:Nx #1
{
@@ -746,9 +759,9 @@
\galley_save_display_widow_penalties:_aux:n
}
}
-\cs_new_nopar:Npn \galley_save_display_widow_penalties:_aux:n #1
+\cs_new:Npn \galley_save_display_widow_penalties:_aux:n #1
{ \int_use:N \etex_displaywidowpenalties:D \int_eval:w #1 \int_eval_end: , }
-\cs_new_protected_nopar:Npn \galley_save_widow_penalties:N #1
+\cs_new_protected:Npn \galley_save_widow_penalties:N #1
{
\clist_set:Nx #1
{
@@ -757,23 +770,29 @@
\galley_save_widow_penalties:_aux:n
}
}
-\cs_new_nopar:Npn \galley_save_widow_penalties:_aux:n #1
+\cs_new:Npn \galley_save_widow_penalties:_aux:n #1
{ \int_use:N \etex_widowpenalties:D \int_eval:w #1 \int_eval_end: , }
\cs_new_protected_nopar:Npn \galley_interline_penalty:
{ \int_use:N \etex_interlinepenalties:D \etex_interlinepenalties:D \c_zero }
-\cs_set_protected_nopar:Npn \nobreak
+\msg_kernel_new:nnn { galley } { no-line-to-end }
+ { There's~no~line~here~to~end. }
+\RenewDocumentCommand \nobreak { }
{ \bool_gset_true:N \g_galley_no_break_next_bool }
-\cs_set_protected_nopar:Npn \noindent
+\RenewDocumentCommand \noindent { }
{
\tex_noindent:D
\bool_gset_false:N \g_galley_omit_next_indent_bool
}
-\cs_set_protected_nopar:Npn \vspace #1
+\RenewDocumentCommand \vspace { m }
{
\@ifstar
{ \galley_set_user_vspace:n {#1} }
{ \galley_set_user_vspace:n {#1} }
}
+\RenewDocumentCommand \\ { s O { 0 pt } }
+ { \galley_break_line:Nn #1 {#2} }
+\RenewDocumentCommand \newline { }
+ { \galley_break_line:Nn \c_true_bool { 0 pt } }
\cs_set_eq:NN \@@par \galley_std_par:
\cs_set_protected_nopar:Npn \@afterheading
{
@@ -797,6 +816,7 @@
\bool_gset_true:N \g_galley_omit_next_indent_bool
\box \@tempboxa
}
+\cs_set_eq:Nc \@normalcr { \token_to_str:N \\ }
%%
%%
%% End of file `l3galley.sty'.
diff --git a/Master/texmf-dist/tex/latex/l3experimental/xgalley/xgalley.sty b/Master/texmf-dist/tex/latex/l3experimental/xgalley/xgalley.sty
index 5ebe766a12b..3e3d251791a 100644
--- a/Master/texmf-dist/tex/latex/l3experimental/xgalley/xgalley.sty
+++ b/Master/texmf-dist/tex/latex/l3experimental/xgalley/xgalley.sty
@@ -14,7 +14,7 @@
%% Do not distribute a modified version of this file.
%%
%% File: xgalley.dtx Copyright (C) 1999-2001, 2004-2009 Frank Mittelbach
-%% (C) 2010,2011 The LaTeX3 Project
+%% (C) 2010-2012 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
@@ -48,7 +48,7 @@
%% -----------------------------------------------------------------------
%%
\RequirePackage{l3names}
-\GetIdInfo$Id: xgalley.dtx 2922 2011-10-17 19:01:04Z joseph $
+\GetIdInfo$Id: xgalley.dtx 3253 2012-01-28 14:59:55Z joseph $
{L3 Experimental galley}
\ProvidesExplPackage
{\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
@@ -195,11 +195,17 @@
\int_set:Nn \tex_lefthyphenmin:D
{
\bool_if:NTF \l_galley_hyphen_enable_bool
- { \l_galley_hyphen_left_int } { 63 }
+ { \l_galley_hyphen_left_int }
+ { 63 }
}
\int_set:Nn \tex_uchyph:D
- { \bool_if:NTF \l_galley_hyphen_uppercase_bool { 1 } { 0 } }
+ {
+ \bool_if:NTF \l_galley_hyphen_uppercase_bool
+ { 1 }
+ { 0 }
+ }
}
+\UseTemplate { hyphenation } { std } { }
\tex_defaulthyphenchar:D 45 \scan_stop:
\tl_new:N \l_galley_justification_other_tl
\DeclareObjectType { justification } { 0 }
@@ -260,7 +266,7 @@
\bool_if:NTF \l_galley_fixed_spacing_bool
{ \bool_set_true:N \exp_not:N \l_galley_fixed_spacing_bool }
{ \bool_set_false:N \exp_not:N \l_galley_fixed_spacing_bool }
- \galley_set_interword_space:N
+ \galley_set_interword_spacing:N
\exp_not:N \l_galley_fixed_spacing_bool
\dim_set:Nn \exp_not:N \l_galley_par_indent_dim
{ \dim_use:N \l_galley_par_indent_dim }
@@ -268,8 +274,8 @@
{ \skip_use:N \l_galley_line_left_skip }
\skip_set:Nn \exp_not:N \l_galley_line_right_skip
{ \skip_use:N \l_galley_line_right_skip }
- \skip_set:Nn \exp_not:N \l_galley_par_start_skip
- { \skip_use:N \l_galley_par_start_skip }
+ \skip_set:Nn \exp_not:N \l_galley_par_begin_skip
+ { \skip_use:N \l_galley_par_begin_skip }
\int_set:Nn \exp_not:N \l_galley_last_line_fit_int
{ \int_use:N \l_galley_last_line_fit_int }
\skip_set:Nn \exp_not:N \@rightskip