summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/l3experimental
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2011-12-11 22:31:32 +0000
committerKarl Berry <karl@freefriends.org>2011-12-11 22:31:32 +0000
commita57ec5a6dea711c791fc567803079341c6fbfae3 (patch)
tree2d68d67d8356cba9a3f6063847e6a96cb913ad37 /Master/texmf-dist/source/latex/l3experimental
parentbca3b6017c5eda5d6c5d67bb266062fb7f363b5b (diff)
l3experimental 3036 (11dec11)
git-svn-id: svn://tug.org/texlive/trunk@24820 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/source/latex/l3experimental')
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx284
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx3415
-rw-r--r--Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins37
3 files changed, 2970 insertions, 766 deletions
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
index cb0476ac898..94bd054f0ce 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3regex.dtx 2943 2011-10-30 15:59:43Z bruno $
+\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $
{L3 Experimental Regular Expressions}
%</driver|package>
%<*driver>
@@ -210,6 +210,10 @@
% or \texttt{C}.
% \item[(\ldots{})] Capturing group.
% \item[(?:\ldots{})] Non-capturing group.
+% \item[(?\char`|\ldots{})] Non-capturing group which resets
+% the group number for capturing groups in each alternative.
+% The following group will be numbered with the first unused
+% group number.
% \end{l3regex-syntax}
%
% Options can be set with |(?|\meta{option}|)| and
@@ -267,7 +271,7 @@
% regular expression argument either as an explicit string
% or as a precompiled regular expression.
%
-% \begin{function}{\regex_set:Nn, \regex_gset:Nn}
+% \begin{function}{\regex_set:Nn, \regex_gset:Nn, \regex_const:Nn}
% \begin{syntax}
% \cs{regex_set:Nn} \meta{tl var} \Arg{regex}
% \end{syntax}
@@ -279,7 +283,8 @@
% \regex_set:Nn \l_my_regex_tl { my\ (simple\ )? reg(ex|ular\ expression) }
% \end{verbatim}
% The assignment is local for \cs{regex_set:Nn} and global for
-% \cs{regex_gset:Nn}.
+% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions
+% which will never change.
% \begin{texnote}
% Precompiled regular expressions can be safely written to a file
% and read when the \LaTeX3 syntax is active (as triggered by
@@ -371,8 +376,6 @@
% sequence contains the two items \texttt{Hello} and \texttt{world}.
% \end{function}
%
-% \subsection{String splitting}
-%
% \begin{function}{\regex_split:nnN, \regex_split:NnN}
% \begin{syntax}
% \cs{regex_split:nnN} \Arg{regular expression} \meta{string} \meta{seq~var}
@@ -424,9 +427,9 @@
% The following need to be done now.
% \begin{itemize}
% \item The \texttt{\{\}} quantifiers are only partially implemented.
-% \item Space's catcode wrong in the result of a replacement!
+% \item Check the catcode of spaces everywhere.
% \item Newline conventions are not done.
-% In particular, |.| should not match newlines.
+% In particular, we should have an option for |.| not to match newlines.
% Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should
% differ.
% \end{itemize}
@@ -438,8 +441,6 @@
% \item Regex matching on external files.
% \item Conditional subpatterns with look ahead/behind: \enquote{if
% what follows is [\ldots{}], then [\ldots{}]}.
-% \item \verb"(?|..|..)" to reset the capturing group number at the start
-% of each alternative.
% \item |(*..)| and |(?..)| sequences to set some options.
% \item Reduce the number of epsilon-transitions in alternatives.
% \item Optimize simple strings: use less states
@@ -452,12 +453,12 @@
%
% The following features of PCRE or Perl will probably not be implemented.
% \begin{itemize}
-% \item Other forms of escapes, and character classes.
-% |\cx|$\simeq$|\^^x|, |\ddd| (octal \texttt{ddd}).
-% POSIX character classes.
-% |\p{..}| and |\P{..}| for having/not having a Unicode property.
-% |\X| for \enquote{extended} Unicode sequence.
-% \item Callout with |(?C...)|.
+% \item |\cx|, similar to \TeX{}'s own |\^^x|;
+% \item |\ddd|, matching the character with code \texttt{ddd} in octal;
+% \item POSIX character classes |[:alpha:]| \emph{etc.};
+% \item Unicode properties: |\p{..}| and |\P{..}|;
+% \item |\X| which should match any \enquote{extended} Unicode sequence;
+% \item Callout with |(?C...)|;
% \item Conditional subpatterns (other than with a look-ahead
% or look-behind condition): this is non-regular, isn't it?
% \end{itemize}
@@ -518,25 +519,6 @@
%
% \subsection{Constants and variables}
%
-% \begin{variable}
-% {
-% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one,
-% \c_ninety_seven, \c_one_hundred_twenty_three,
-% \c_one_hundred_twenty_seven
-% }
-% We declare here some integer values which delimit ranges of
-% ASCII characters of various types.
-% \begin{macrocode}
-\int_const:Nn \c_forty_eight { 48 }
-\int_const:Nn \c_fifty_eight { 58 }
-\int_const:Nn \c_sixty_five { 65 }
-\int_const:Nn \c_ninety_one { 91 }
-\int_const:Nn \c_ninety_seven { 97 }
-\int_const:Nn \c_one_hundred_twenty_three { 123 }
-\int_const:Nn \c_one_hundred_twenty_seven { 127 }
-% \end{macrocode}
-% \end{variable}
-%
% \begin{macro}{\regex_tmp:w}
% \begin{variable}{\g_regex_tmpa_tl, \l_regex_tmpa_tl, \l_regex_tmpb_tl}
% \begin{variable}{\l_regex_tmpa_int, \l_regex_tmpb_int}
@@ -569,19 +551,33 @@
% \end{variable}
% \end{variable}
%
-% \begin{variable}{\l_regex_group_begin_seq, \l_regex_group_end_seq}
+% \begin{variable}{\l_regex_left_state_seq, \l_regex_right_state_seq}
% Alternatives are implemented by branching from a state into the
% various choices, then merging those into another state. We store
-% information about those states in two sequences (several can be
-% active at the same time when nesting groups).
+% information about those states in two sequences.
% \begin{macrocode}
-\seq_new:N \l_regex_group_begin_seq
-\seq_new:N \l_regex_group_end_seq
+\seq_new:N \l_regex_left_state_seq
+\seq_new:N \l_regex_right_state_seq
% \end{macrocode}
% \end{variable}
%
+% \begin{variable}{\l_regex_end_group_seq}
+% \begin{variable}{\l_regex_end_alternation_seq}
+% These sequences hold actions to be performed at the end of a group,
+% and at the end of each branch of the alternation, respectively.
+% Currently, \cs{l_regex_end_group_seq} is used to keep track of
+% letter case, and \cs{l_regex_end_alternation_seq} is used
+% for \verb"(?|...)" groups.
+% \begin{macrocode}
+\seq_new:N \l_regex_end_group_seq
+\seq_new:N \l_regex_end_alternation_seq
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
% \begin{variable}{\l_regex_capturing_group_int}
% \begin{variable}{\l_regex_capturing_group_seq}
+% \begin{variable}{\l_regex_capturing_group_max_int}
% \cs{l_regex_capturing_group_int} is the ID number of the current
% capturing group, starting at $0$ for a group enclosing the full
% regular expression, and counting in the order of their left parenthesis.
@@ -591,9 +587,11 @@
% \begin{macrocode}
\int_new:N \l_regex_capturing_group_int
\seq_new:N \l_regex_capturing_group_seq
+\int_new:N \l_regex_capturing_group_max_int
% \end{macrocode}
% \end{variable}
% \end{variable}
+% \end{variable}
%
% \begin{variable}{\l_regex_one_or_group_tl}
% When looking for quantifiers, this variable holds either
@@ -930,30 +928,34 @@
%
% \subsection{Helpers}
%
-% \subsubsection{Toks}
+% \subsubsection{Scan markers}
%
-% \begin{macro}[int]{\s_regex_toks}
+% \begin{macro}[int]{\s_regex_stop}
+% Unexpandable markers, which does nothing if it reaches \TeX{}'s stomach.
% When performing the matching, the \tn{toks} registers hold submatch
% information, followed by the instruction for a given state of the NFA.
-% The two parts are separated by \cs{s_regex_toks}.
+% The two parts are separated by \cs{s_regex_stop}.
+% See \cs{regex_nfa:Nw} for another use.
% \begin{macrocode}
-\cs_new_eq:NN \s_regex_toks \scan_stop:
+\cs_new_eq:NN \s_regex_stop \scan_stop:
% \end{macrocode}
% \end{macro}
%
+% \subsubsection{Toks}
+%
% \begin{macro}[int]{\regex_toks_put_left:Nx}
% \begin{macro}[int]{\regex_toks_put_right:Nx}
% During the building phase, every \tn{toks} register starts with
-% \cs{s_regex_toks}, and we wish to add \texttt{x}-expanded material
+% \cs{s_regex_stop}, and we wish to add \texttt{x}-expanded material
% to those registers. The expansion is done \enquote{by hand} for
% optimization (these operations are used quite a lot). When adding
% material to the left, we define \cs{regex_tmp:w} to remove the
-% \cs{s_regex_toks} marker and put it back to the left of the new
+% \cs{s_regex_stop} marker and put it back to the left of the new
% material.
% \begin{macrocode}
\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2
{
- \cs_set_nopar:Npx \regex_tmp:w \s_regex_toks { \s_regex_toks #2 }
+ \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 }
\tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN
{ \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 }
}
@@ -994,20 +996,44 @@
%
% \subsubsection{Sequences}
%
-% \begin{macro}[int]{\regex_seq_pop:N}
-% \begin{macro}[int]{\regex_seq_get:N}
+% \begin{macro}[int]{\regex_seq_pop_int:NN}
+% \begin{macro}[int]{\regex_seq_get_int:NN}
+% \begin{macro}[int]{\regex_seq_push_int:NN}
+% When building the regular expression, we keep track of some integers
+% (pointers to various states) without help from \TeX{}'s grouping.
+% Here are variants of \cs{seq_pop:NN} and \cs{seq_get:NN} which
+% assign using \cs{int_set:Nn} rather than \cs{tl_set:Nn}.
+% \begin{macrocode}
+\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2
+ {
+ \seq_pop:NN #1 \l_regex_tmpa_tl
+ \int_set:Nn #2 \l_regex_tmpa_tl
+ }
+\cs_new_protected:Npn \regex_seq_get_int:NN #1#2
+ {
+ \seq_get:NN #1 \l_regex_tmpa_tl
+ \int_set:Nn #2 \l_regex_tmpa_tl
+ }
+\cs_new_protected:Npn \regex_seq_push_int:NN #1#2
+ { \seq_push:No #1 { \int_use:N #2 } }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\regex_seq_pop_use:N}
+% \begin{macro}[int]{\regex_seq_get_use:N}
% When building the regular expression, some settings are kept
% local to capturing groups without any help from \TeX{}'s grouping.
-% This is done \enquote{by hand}, in sequences whose items contain
-% instructions such as |\int_set:Nn| \meta{int~var} \Arg{value},
-% which should be run immediately.
+% This is done \enquote{by hand}, in sequences whose items should
+% be run immediately.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \regex_seq_pop:N #1
+\cs_new_protected_nopar:Npn \regex_seq_pop_use:N #1
{
\seq_pop:NN #1 \l_regex_tmpa_tl
\l_regex_tmpa_tl
}
-\cs_new_protected_nopar:Npn \regex_seq_get:N #1
+\cs_new_protected_nopar:Npn \regex_seq_get_use:N #1
{
\seq_get:NN #1 \l_regex_tmpa_tl
\l_regex_tmpa_tl
@@ -1297,7 +1323,7 @@
% \begin{macro}[int]{\regex_build_new_state:}
% Here, we add a new state to the NFA. At the end of the building
% phase, we want every \tn{toks} register to start with
-% \cs{s_regex_toks}, hence initialize the new register appropriately.
+% \cs{s_regex_stop}, hence initialize the new register appropriately.
% Then set \cs{l_regex_left/right_state_int} to their new values.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \regex_build_new_state:
@@ -1306,7 +1332,7 @@
{ \msg_error:nn { regex } { -997 } }
{
\int_incr:N \l_regex_max_state_int
- \tex_toks:D \l_regex_max_state_int { \s_regex_toks }
+ \tex_toks:D \l_regex_max_state_int { \s_regex_stop }
}
\int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
\int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int
@@ -1367,7 +1393,7 @@
{ #1 }
\regex_build_open_aux:
\g_str_result_tl \prg_do_nothing: \prg_do_nothing:
- \seq_push:Nn \l_regex_capturing_group_seq { 0 }
+ \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero
\regex_build_close_aux: \regex_build_group_:
\regex_build_end:
}
@@ -1595,7 +1621,7 @@
{
\cs_set:Npx \regex_build_tmp_class:n ##1
{
- \regex_item_equal:n { \int_value:w `#1 }
+ \regex_item_equal:n { \int_value:w `#1 ~ }
\regex_break_point:TF { \regex_action_cost:n { ##1 } } { }
}
\regex_build_one_quantifier:
@@ -2139,8 +2165,8 @@
{ \regex_build_special_group:NN }
{
\int_incr:N \l_regex_capturing_group_int
- \seq_push:Nx \l_regex_capturing_group_seq
- { \int_use:N \l_regex_capturing_group_int }
+ \regex_seq_push_int:NN
+ \l_regex_capturing_group_seq \l_regex_capturing_group_int
\regex_build_open_aux:
#1 #2
}
@@ -2148,19 +2174,12 @@
\cs_new_protected_nopar:Npn \regex_build_open_aux:
{
\regex_build_new_state:
- \seq_push:Nx \l_regex_group_begin_seq
- {
- \int_set:Nn \l_regex_left_state_int
- { \int_use:N \l_regex_left_state_int }
- }
+ \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int
+ \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int
\bool_if:NTF \l_regex_caseless_bool
- { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseless: } }
- { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseful: } }
- \seq_push:Nx \l_regex_group_end_seq
- {
- \int_set:Nn \l_regex_right_state_int
- { \int_use:N \l_regex_right_state_int }
- }
+ { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseless: }
+ { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseful: }
+ \seq_push:Nn \l_regex_end_alternation_seq { }
\regex_build_begin_alternation:
}
\cs_new_protected_nopar:cpn { regex_build_|: }
@@ -2180,9 +2199,10 @@
\cs_new_protected_nopar:Npn \regex_build_close_aux:
{
\regex_build_end_alternation:
- \regex_seq_pop:N \l_regex_group_begin_seq
- \regex_seq_pop:N \l_regex_group_end_seq
- \regex_seq_pop:N \l_regex_group_end_seq
+ \regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int
+ \regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int
+ \regex_seq_pop_use:N \l_regex_end_group_seq
+ \seq_pop:NN \l_regex_end_alternation_seq \l_regex_tmpa_tl
}
% \end{macrocode}
% Building each branch.
@@ -2190,7 +2210,7 @@
\cs_new_protected_nopar:Npn \regex_build_begin_alternation:
{
\regex_build_new_state:
- \regex_seq_get:N \l_regex_group_begin_seq
+ \regex_seq_get_int:NN \l_regex_left_state_seq \l_regex_left_state_int
\regex_toks_put_right:Nx \l_regex_left_state_int
{
\regex_action_free:n
@@ -2203,7 +2223,7 @@
\cs_new_protected_nopar:Npn \regex_build_end_alternation:
{
\int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int
- \regex_seq_get:N \l_regex_group_end_seq
+ \regex_seq_get_int:NN \l_regex_right_state_seq \l_regex_right_state_int
\regex_toks_put_right:Nx \l_regex_left_state_int
{
\regex_action_free:n
@@ -2212,6 +2232,7 @@
{ \l_regex_right_state_int - \l_regex_left_state_int }
}
}
+ \regex_seq_get_use:N \l_regex_end_alternation_seq
}
% \end{macrocode}
% \end{macro}
@@ -2239,18 +2260,62 @@
%
% \begin{macro}{\regex_build_special_group_::}
% Non-capturing groups are like capturing groups, except that
-% we set the group id to \texttt{*}, which will then inhibit
+% we set the group id to \texttt{-1}, which will then inhibit
% submatching in \cs{regex_build_group_submatches:NN}.
% The group number is not increased.
% \begin{macrocode}
\cs_new_protected_nopar:cpn { regex_build_special_group_:: }
{
- \seq_push:Nx \l_regex_capturing_group_seq { * }
+ \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one
\regex_build_open_aux:
}
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}+\regex_build_special_group_|:+
+% The special group \verb"(?|..|..)" is non-capturing
+% (hence we set the |capturing_group| to $-1$), and resets
+% the group number in each branch of the alternation.
+% We use a variant of \cs{regex_build_open_aux:}, adding
+% some code to be performed at every alternation, and at
+% the end of the group. Namely, we keep track of
+% the maximal value that \cs{l_regex_capturing_group_int}
+% takes, and restore that value when the group end,
+% and in every branch, we reset the capturing group number.
+% \begin{macrocode}
+\cs_new_protected_nopar:cpn { regex_build_special_group_|: }
+ {
+ \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one
+ \regex_build_new_state:
+ \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int
+ \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int
+ \seq_push:Nx \l_regex_end_alternation_seq
+ {
+ \exp_not:N \int_compare:nNnT
+ \l_regex_capturing_group_int
+ > \l_regex_capturing_group_max_int
+ {
+ \int_set_eq:NN
+ \l_regex_capturing_group_max_int
+ \l_regex_capturing_group_int
+ }
+ \int_set:Nn \l_regex_capturing_group_int
+ { \int_use:N \l_regex_capturing_group_int }
+ }
+ \seq_push:Nx \l_regex_end_group_seq
+ {
+ \bool_if:NTF \l_regex_caseless_bool
+ \regex_build_caseless:
+ \regex_build_caseful:
+ \int_set_eq:NN
+ \l_regex_capturing_group_int
+ \l_regex_capturing_group_max_int
+ }
+ \regex_build_begin_alternation:
+ }
+% \end{macrocode}
+% \end{macro}
+%
% \begin{macro}{\regex_build_special_group_i:}
% \begin{macro}{\regex_build_special_group_-:}
% \begin{macro}[aux]{\regex_build_options:NNN}
@@ -2324,7 +2389,7 @@
\cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2
{
\seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl
- \str_if_eq:xxF { \l_regex_tmpa_tl } { * }
+ \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero
{
\regex_toks_put_left:Nx #1
{ \regex_action_submatch:n { \l_regex_tmpa_tl < } }
@@ -2368,7 +2433,7 @@
{
\tex_toks:D \l_regex_tmpa_int
{
- \s_regex_toks
+ \s_regex_stop
#1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } }
}
}
@@ -2680,7 +2745,7 @@
{ \regex_state_use_aux:n { } }
\cs_new_protected_nopar:Npn \regex_state_use:
{ \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } }
-\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_toks { }
+\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { }
\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1
{
\if_num:w \tex_dimen:D \l_regex_current_state_int
@@ -2734,15 +2799,15 @@
\tex_the:D \tex_toks:D #1
}
}
-\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_toks
+\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop
{
\regex_store_submatches_aux_ii:Nnnw
#1
\regex_state_submatches:nn \c_minus_one \q_prop
- \s_regex_toks
+ \s_regex_stop
}
\cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw
- \regex_state_submatches:nn #1 #2 #3 \s_regex_toks
+ \regex_state_submatches:nn #1 #2 #3 \s_regex_stop
{
\exp_after:wN \c_zero
\exp_after:wN \regex_state_submatches:nn \exp_after:wN
@@ -2753,7 +2818,7 @@
}
\exp_after:wN { \l_regex_current_submatches_prop }
\regex_state_submatches:nn {#1} {#2}
- \s_regex_toks
+ \s_regex_stop
}
% \end{macrocode}
% \end{macro}
@@ -2873,19 +2938,6 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[int]{\s_regex_step}
-% \begin{macro}[aux]{\regex_break_step:w}
-% When a thread succeeds, all threads with lower precedence
-% can be ignored, and the next character should be read.
-% This is done by skipping to \cs{s_regex_step}. This marker
-% does nothing if no thread succeeded at this step.
-% \begin{macrocode}
-\cs_new_eq:NN \s_regex_step \scan_stop:
-\cs_new:Npn \regex_break_step:w #1 \s_regex_step { }
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-%
% \begin{macro}[aux]{\regex_match_loop:N}
% \begin{macro}[aux]{\regex_match_one_index:n}
% \begin{macro}[aux]{\regex_match_one_index_aux:n}
@@ -2907,8 +2959,7 @@
{1} {1} { \l_regex_max_index_int }
\regex_match_one_index:n
}
- \regex_tmp:w
- \s_regex_step
+ \regex_tmp:w \prg_break_point:n { }
\if_num:w \l_regex_max_index_int = \c_zero
\exp_after:wN \use_none_delimit_by_q_recursion_stop:w
\fi:
@@ -3021,7 +3072,8 @@
% \begin{macro}[aux]{\regex_action_success:}
% There is a successful match when an execution path reaches
% the end of the regular expression. Then store the current
-% step and submatches. The current step is then interrupted,
+% step and submatches. The current step is then interrupted
+% with \cs{prg_map_break:},
% and only paths with higher precedence are pursued further.
% The values stored here may be overwritten by a later success
% of a path with higher precedence.
@@ -3037,7 +3089,7 @@
\l_regex_current_step_int
\prop_set_eq:NN \l_regex_success_submatches_prop
\l_regex_current_submatches_prop
- \regex_break_step:w
+ \prg_map_break:
}
}
% \end{macrocode}
@@ -3127,7 +3179,7 @@
% \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\
% \ldots{} \\
% \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\
-% \cs{regex_nfa_end:}
+% \cs{s_regex_stop}
% \end{quote}
% where $n$ is the number of states in the NFA,
% and the various \meta{instruction$\sb{i}$} control
@@ -3144,7 +3196,6 @@
% \begin{macro}[aux]{\regex_set_aux:NNn}
% \begin{macro}[aux]{\regex_set_aux:n}
% \begin{macro}[aux]{\regex_nfa:Nw}
-% \begin{macro}[aux]{\regex_nfa_end:}
% Within a group, build the NFA corresponding to the given regular
% expression, with submatch tracking. Then save the contents of all
% relevant \tn{toks} registers into \cs{g_regex_tmpa_tl}, then
@@ -3178,22 +3229,28 @@
\prg_stepwise_function:nnnN
{1} {1} {\l_regex_max_state_int}
\regex_set_aux:n
- \regex_nfa_end:
+ \s_regex_stop
}
\group_end:
#1 #2 \g_regex_tmpa_tl
}
\cs_new_nopar:Npn \regex_set_aux:n #1
{ \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } }
-\cs_new:Npn \regex_nfa:Nw #1 #2 \regex_nfa_end:
- { \msg_expandable_error:n { Automaton~#1 used~incorrectly. } }
-\cs_new_eq:NN \regex_nfa_end: \scan_stop:
+\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop
+ { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
+%
+% \begin{macro}{\regex_const:Nn}
+% The same idea as for setting, but using the new function.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \regex_const:Nn
+ { \regex_set_aux:NNn \cs_new_eq:NN }
+% \end{macrocode}
% \end{macro}
%
% \begin{macro}[int,TF]{\regex_check_nfa:N}
@@ -3611,7 +3668,7 @@
{
\regex_token_if_other_digit:NTF #1
{ \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } }
- { \msg_expandable_error:n { \g misused~in~replacement~text. } }
+ { \msg_expandable_kernel_error:nn { regex } { g-misused } }
#1
}
}
@@ -3623,7 +3680,7 @@
\exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1}
\exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2
{
- \msg_expandable_error:n { \g misused~in~replacement~text. }
+ \msg_expandable_kernel_error:nn { regex } { g-misused }
#2
}
}
@@ -3818,6 +3875,13 @@
% \end{macrocode}
%
% \begin{macrocode}
+\msg_kernel_new:nnn { regex } { nfa-misused }
+ { Automaton~#1 used~incorrectly. }
+\msg_kernel_new:nnn { regex } { g-misused }
+ { \g misused~in~replacement~text. }
+% \end{macrocode}
+%
+% \begin{macrocode}
%</package>
% \end{macrocode}
%
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
index b3648c57ede..db73291a70b 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx
@@ -35,7 +35,7 @@
%
%<*driver|package>
\RequirePackage{expl3}
-\GetIdInfo$Id: l3str.dtx 2921 2011-10-17 13:23:36Z bruno $
+\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $
{L3 Experimental Strings}
%</driver|package>
%<*driver>
@@ -72,7 +72,7 @@
% \LaTeX3 provides a set of functions to manipulate token lists
% as strings of characters, ignoring the category codes of those
% characters.
-%
+%
% String variables are simply specialised token lists, but by convention
% should be named with the suffix \texttt{\ldots str}. Such variables should
% contain characters with category code $12$ (other), except spaces, which
@@ -87,14 +87,14 @@
% variable as their argument;
% \item \cs{str_...:n...}, taking any token list (or string) as an argument;
% \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered
-% during the operation: these functions are slightly faster than those
+% during the operation: these functions are faster than those
% which take care of escaping spaces appropriately;
% \end{itemize}
% When performance is important, the internal \cs{str_..._unsafe:n...}
% functions, which expect a \enquote{safe} string in which spaces
% have category code $12$ instead of $10$, might be useful.
%
-% \subsection{Conversion and input of strings}
+% \section{Conversion and input of strings}
%
% \begin{variable}
% {
@@ -102,6 +102,7 @@
% \c_lbrace_str,
% \c_rbrace_str,
% \c_hash_str,
+% \c_tilde_str,
% \c_percent_str
% }
% Constant strings, containing a single character, with category code $12$.
@@ -116,6 +117,16 @@
% tokens in the input stream.
% \end{function}
%
+% \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx}
+% \begin{syntax}
+% \cs{str_const:Nn} \meta{str~var} \Arg{token list}
+% \end{syntax}
+% Creates a new constant \meta{str~var} or raises an error
+% if the name is already taken. The value of the
+% \meta{str~var} will be set globally to the
+% \meta{token list}, after being converted to a string.
+% \end{function}
+%
% \begin{function}{\str_set:Nn, \str_set:Nx, \str_set:cn, \str_set:cx}
% \begin{syntax}
% \cs{str_set:Nn} \meta{str var} \Arg{token list}
@@ -126,10 +137,10 @@
%
% \begin{function}{\str_gset:Nn, \str_gset:Nx, \str_gset:cn, \str_gset:cx}
% \begin{syntax}
-% \cs{str_gset:Nn} \meta{str var} \Arg{token list}
+% \cs{str_gset:Nn} \meta{str~var} \Arg{token list}
% \end{syntax}
% Converts the \meta{token list} to a \meta{string},
-% and saves the result in \meta{str var} globally.
+% and saves the result in \meta{str~var} globally.
% \end{function}
%
% \begin{function}
@@ -230,7 +241,7 @@
% since |<| has \textsc{ascii} code \texttt{3C} (in hexadecimal).
% \end{function}
%
-% \subsection{Characters given by their position}
+% \section{Characters given by their position}
%
% \begin{function}[EXP]
% {\str_length:N, \str_length:n, \str_length_ignore_spaces:n}
@@ -310,11 +321,11 @@
% and leaves in the input stream the characters
% between \meta{start index} (inclusive) and \meta{end index} (exclusive).
% If either of \meta{start index} or \meta{end index} is negative,
-% the it is incremented by the length of the list. Both \meta{start index}
+% then it is incremented by the length of the list. Both \meta{start index}
% and \meta{end index} count from $0$ for the first (left most) character.
% \end{function}
%
-% \subsection{String conditionals}
+% \section{String conditionals}
%
% \begin{function}[EXP, pTF]
% {
@@ -337,175 +348,133 @@
% expansion).
% \end{function}
%
-% \begin{function}[EXP, pTF]
-% {\str_if_contains_char:NN, \str_if_contains_char:nN}
-% \begin{syntax}
-% \cs{str_if_contains_char:nN} \Arg{token list} \meta{char}
-% \end{syntax}
-% Converts the \meta{token list} to a \meta{string}
-% and tests whether the \meta{char} is present in the \meta{string}.
-% The \meta{char} can be given either directly, or as a one
-% letter control sequence.
-% \end{function}
-%
-% \subsection{Byte conditionals}
-%
% \begin{function}[EXP, pTF]{\str_if_bytes:N}
% \begin{syntax}
% \cs{str_if_bytes:NTF} \meta{str var} \Arg{true code} \Arg{false code}
% \end{syntax}
% Tests whether the \meta{str var} only contains characters
-% in the range $0$ to $255$.
+% in the range $0$ to $255$. This is automatically true when
+% the pdf\TeX{} engine is in use.
% \end{function}
%
-% ^^A\begin{function}[EXP, pTF]{\str_if_UTF_viii:N}
-% ^^A \begin{syntax}
-% ^^A \cs{str_if_UTF_viii:N} \meta{str var} \Arg{true code} \Arg{false code}
-% ^^A \end{syntax}
-% ^^A Tests whether the \meta{str var} only contains characters
-% ^^A in the range $0$--$255$, and forms a valid \texttt{UTF8} string.
-% ^^A \textbf{Missing!} ^^A Missing.
-% ^^A\end{function}
-%
-% \subsection{Byte conversion}
-%
-% Byte conversions are necessary where strings are to be used in PDF
-% string objects or within URLs. In both cases, only a restricted subset
-% of characters are permitted.
-%
-% \begin{function}{\str_native_from_UTF_viii:NN}
-% \begin{syntax}
-% \cs{str_native_from_UTF_viii:NN} \meta{str~var1} \meta{str~var2}
-% \end{syntax}
-% Reads the contents of the \meta{str~var2} as an \textsc{utf8}-encoded
-% sequence of bytes, and stores the resulting characters (which can
-% now have any character code) into \meta{str~var1}.
-% This function raises an error if the \meta{str~var2} is not
-% a valid sequence of bytes in the \textsc{utf8} encoding.
-% In the \pdfTeX{} engine, this function raises an error if any of the
-% resulting characters is outside the range $0$ to $255$.
-% \end{function}
+% \section{Encoding functions}
%
-% \begin{function}{\str_UTF_viii_from_native:NN}
-% \begin{syntax}
-% \cs{str_UTF_viii_from_native:NN} \meta{str~var1} \meta{str~var2}
-% \end{syntax}
-% Converts each character of the \meta{str~var2} into
-% a sequence of bytes, as defined by the \textsc{utf8}
-% encoding, and stores the result in \meta{str~var1}.
-% In the \pdfTeX{} engine, this function is of course
-% of very little use, but can be used without harm:
-% characters in the range $0$ to $127$ are left unchanged,
-% and characters in the range $128$ to $255$ become two-byte
-% sequences, as per the definition of \texttt{UTF8}.
-% \end{function}
+% Traditionally, string encodings only specify how strings of characters
+% should be stored as bytes. However, the resulting lists of bytes are
+% often to be used in contexts where only a restricted subset of bytes
+% are permitted (\textsc{pdf} string objects, \textsc{url}s).
+% Hence, storing strings of characters has two components.
+% \begin{itemize}
+% \item The code points (\enquote{character codes})
+% are expressed as bytes following a given
+% \enquote{encoding}. This can be \textsc{utf-16},
+% \textsc{iso 8859-1}, \emph{etc.}
+% See Table~\ref{tab:encodings} for a list
+% of encodings supported.
+% \item Bytes are translated to \TeX{} tokens through
+% a given \enquote{escaping}. Those are defined for
+% the most part by the \texttt{pdf} file format.
+% See Table~\ref{tab:escapings} for a list
+% of escaping methods supported.
+% \end{itemize}
%
-% \begin{function}
-% {\str_bytes_escape_hexadecimal:NN, \str_bytes_unescape_hexadecimal:NN}
-% \begin{syntax}
-% \cs{str_bytes_escape_hexadecimal:NN} \meta{str var1} \meta{str var2}
-% \cs{str_bytes_unescape_hexadecimal:NN} \meta{str var1} \meta{str var2}
-% \end{syntax}
-% The \texttt{escape} variant takes each character in \meta{str var1}
-% and converts it into the hexadecimal representation. The resulting
-% string is stored in \meta{str var2} within the current \TeX{} group
-% level. The \texttt{unescape} variant reverses this process.
-% Thus for example
-% \begin{verbatim}
-% \str_set:Nn \l_mya_str { Hello ( ) }
-% \str_bytes_escape_hexadecimal:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |48656C6C6F2829|,
-% while
-% \begin{verbatim}
-% \str_set:Nn \l_mya_str { 48656C6C6F2829 }
-% \str_bytes_unescape_hexadecimal:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello()|.
-% See also \tn{pdfescapehex}.
-% \end{function}
+% \begin{table}\centering
+% \caption{\label{tab:encodings}Supported encodings.
+% Non-alphanumeric characters are ignored,
+% and capital letters are lower-cased.}
+% \begin{tabular}{cc}
+% \toprule
+% \texttt{utf8} & \textsc{utf-8} \\
+% \texttt{utf16} & \textsc{utf-16}, with byte-order mark \\
+% \texttt{utf16be} & \textsc{utf-16}, big-endian \\
+% \texttt{utf16le} & \textsc{utf-16}, little-endian \\
+% \texttt{utf32} & \textsc{utf-32}, with byte-order mark \\
+% \texttt{utf32be} & \textsc{utf-32}, big-endian \\
+% \texttt{utf32le} & \textsc{utf-32}, little-endian \\
+% \texttt{iso88591}, \texttt{latin1} & \textsc{iso 8859-1} \\
+% \texttt{iso88592}, \texttt{latin2} & \textsc{iso 8859-2} \\
+% \texttt{iso88593}, \texttt{latin3} & \textsc{iso 8859-3} \\
+% \texttt{iso88594}, \texttt{latin4} & \textsc{iso 8859-4} \\
+% \texttt{iso88595} & \textsc{iso 8859-5} \\
+% \texttt{iso88596} & \textsc{iso 8859-6} \\
+% \texttt{iso88597} & \textsc{iso 8859-7} \\
+% \texttt{iso88598} & \textsc{iso 8859-8} \\
+% \texttt{iso88599}, \texttt{latin5} & \textsc{iso 8859-9} \\
+% \texttt{iso885910}, \texttt{latin6} & \textsc{iso 8859-10} \\
+% \texttt{iso885911} & \textsc{iso 8859-11} \\
+% \texttt{iso885913}, \texttt{latin7} & \textsc{iso 8859-13} \\
+% \texttt{iso885914}, \texttt{latin8} & \textsc{iso 8859-14} \\
+% \texttt{iso885915}, \texttt{latin9} & \textsc{iso 8859-15} \\
+% \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\
+% \midrule
+% \texttt{native} & Native Unicode string. \\
+% \texttt{internal} & Comma-list of integers. \\
+% \bottomrule
+% \end{tabular}
+% \end{table}
%
-% \begin{function}
-% {\str_bytes_escape_name:NN, \str_bytes_unescape_name:NN}
-% \begin{syntax}
-% \cs{str_bytes_escape_name:NN} \meta{str var1} \meta{str var2}
-% \cs{str_bytes_unescape_name:NN} \meta{str var1} \meta{str var2}
-% \end{syntax}
-% The \texttt{escape} variant takes each character in \meta{str var1}
-% and replaces any which cannot be used directly in a PDF string object
-% with the appropriate hexadecimal representation preceded by the |#|
-% character. The resulting
-% string is stored in \meta{str var2} within the current \TeX{} group
-% level. The \texttt{unescape} variant reverses this process. Thus for
-% example
-% \begin{verbatim}
-% \str_set:Nn \l_mya_str { Hello ( ) }
-% \str_bytes_escape_name:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello#28#29|,
-% while
-% \begin{verbatim}
-% \char_set_catcode_other:N \#
-% \str_set:Nn \l_mya_str { Hello#28#29 }
-% \str_bytes_unescape_name:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello()|.
-% See also \tn{pdfescapename}.
-% \end{function}
+% \begin{table}\centering
+% \caption{\label{tab:escapings}Supported escapings.
+% Non-alphanumeric characters are ignored,
+% and capital letters are lower-cased.}
+% \begin{tabular}{cc}
+% \toprule
+% \texttt{bytes}, or empty
+% & arbitrary bytes \\
+% \texttt{hex}, \texttt{hexadecimal}
+% & byte $=$ two hexadecimal digits \\
+% \texttt{name}
+% & see \tn{pdfescapename} \\
+% \texttt{string}
+% & see \tn{pdfescapestring} \\
+% \texttt{url}, \texttt{percent}
+% & encoding used in \textsc{url}s \\
+% \bottomrule
+% \end{tabular}
+% \end{table}
%
-% \begin{function}{\str_bytes_escape_string:NN, \str_bytes_unescape_string:NN}
+% \begin{function}{\str_set_convert:Nnnn}
% \begin{syntax}
-% \cs{str_bytes_escape_string:NN} \meta{str var1} \meta{str var2}
-% \cs{str_bytes_unescape_string:NN} \meta{str var1} \meta{str var2}
+% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string}
+% ~~\Arg{name 1} \Arg{name 2}
% \end{syntax}
-% The \texttt{escape} variant takes each character in \meta{str var1}
-% and prepends a |\| to any which cannot be used directly in a PDF string
-% object. The resulting string is stored in \meta{str var2} within the
-% current \TeX{} group level. The \texttt{unescape} variant reverses this
-% process. Thus for example
-% \begin{verbatim}
-% \str_set:Nn \l_mya_str { Hello [ ] }
-% \str_bytes_escape_string:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello\(\)|,
-% while
-% \begin{verbatim}
-% \str_set:Nn \l_mya_str { Hello\(\) }
-% \str_bytes_unescape_string:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello()|.
-% Some characters (for example a space) are converted to a three-digit
-% octal representation, so for example a space gives |\040|.
-% See also \tn{pdfescapename}.
-% \end{function}
+% This function converts the \meta{string} from the encoding
+% given by \meta{name 1} to the encoding given by \meta{name 2},
+% and stores the result in the \meta{str~var}.
+% Each \meta{name} can have the form \meta{encoding} or
+% \meta{encoding}\texttt{/}\meta{escaping}, where the possible values
+% of \meta{encoding} and \meta{escaping} are given in
+% Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively.
+% The default escaping is to input and output bytes directly.
+% The special encodings \texttt{native} and \texttt{internal}
+% do not support a \meta{escaping}, since those formats do not
+% correspond to strings of bytes.
%
-% \begin{function}{\str_bytes_percent_encode:NN, \str_bytes_percent_decode:NN}
-% \begin{syntax}
-% \cs{str_bytes_percent_encode:NN} meta{str var1} \meta{str var2}
-% \cs{str_bytes_percent_decode:NN} meta{str var1} \meta{str var2}
-% \end{syntax}
-% The \texttt{escape} variant takes each character in \meta{str var1}
-% and replaces any which cannot be used directly in a URL
-% with the appropriate hexadecimal representation preceded by the |%|
-% character. The resulting
-% string is stored in \meta{str var2} within the current \TeX{} group
-% level. The \texttt{unescape} variant reverses this process. Thus for
-% example
+% For example,
% \begin{verbatim}
-% \str_set:Nn \l_mya_str { Hello ( ) }
-% \str_bytes_precent_encode:NN \l_mya_str \l_myb_str
+% \str_set_convert:Nnnn \l_foo_str { Hello! }
+% { native } { utf16/hex }
% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello%28%29|,
-% while
-% \begin{verbatim}
-% \char_set_catcode_other:N \%
-% \str_set:Nn \l_mya_str { Hello%28%29 }
-% \str_bytes_unescape_name:NN \l_mya_str \l_myb_str
-% \end{verbatim}
-% will result in \cs{l_myb_str} containing the string |Hello()|.
+% results in the variable \cs{l_foo_str} holding the string
+% \texttt{FEFF00480065006C006C006F0021}. This is obtained
+% by converting each character in the string \texttt{Hello!}
+% to the \textsc{utf-16} encoding, and expressing each byte
+% as a pair of hexadecimal digits. Note the presence of a
+% (big-endian) byte order mark \texttt{FEFF}, which can be
+% avoided by specifying the encoding \texttt{utf16be/hex}.
+%
+% Parts of the \meta{string} which cannot be converted to bytes
+% using the \meta{escaping 1} are silently ignored (including non-byte
+% characters which could be present in the string).
+% Sequences of bytes which are not valid in the \meta{encoding 1}
+% are converted to the replacement character |"FFFD|, and raise
+% an error.
+% Characters which cannot be encoded in the \meta{encoding 2}
+% are silently ignored.
+%^^A todo: add doc and examples about each encoding and escaping method.
% \end{function}
%
-% \subsection{Internal string functions}
+% \section{Internal string functions}
%
% \begin{function}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n}
% \begin{syntax}
@@ -531,13 +500,33 @@
% and hands-in the result as arguments to \meta{function}.
% \end{function}
%
-% \begin{function}[rEXP]{\str_map_tokens:Nn}
+% \begin{function}{\str_aux_gset_other:Nn}
+% \begin{syntax}
+% \cs{str_aux_gset_other:Nn} \meta{tl~var} \Arg{token list}
+% \end{syntax}
+% Converts the \meta{token list} to an \meta{other string}, where
+% spaces have category code \enquote{other}, and assigns the result
+% to the \meta{tl~var}, globally.
+% \end{function}
+%
+% \begin{function}[rEXP]{\str_map_tokens:Nn, \str_map_tokens:nn}
% \begin{syntax}
% \cs{str_map_tokens:Nn} \meta{str var} \meta{tokens}
% \end{syntax}
% Maps the \meta{tokens} over every character in the \meta{str var}.
% \end{function}
%
+% \begin{function}[EXP, pTF]
+% {\str_if_contains_char:NN, \str_if_contains_char:nN}
+% \begin{syntax}
+% \cs{str_if_contains_char:nN} \Arg{token list} \meta{char}
+% \end{syntax}
+% Converts the \meta{token list} to a \meta{string}
+% and tests whether the \meta{char} is present in the \meta{string}.
+% The \meta{char} can be given either directly, or as a one
+% letter control sequence.
+% \end{function}
+%
% \begin{function}{\str_aux_escape:NNNn}
% \begin{syntax}
% \cs{str_aux_escape:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list}
@@ -554,26 +543,18 @@
% \meta{fn3}. The result is assigned globally to \cs{g_str_result_tl}.
% \end{function}
%
-% \subsection{Possibilities}
+% \section{Possibilities}
%
% \begin{itemize}
% \item More encodings (see Heiko's \pkg{stringenc}).
-% In particular, what is needed for pdf: \texttt{UTF-16}?
-% \item \cs{str_between:nnn} \Arg{str}
-% \Arg{begin delimiter} \Arg{end delimiter} giving the piece of
-% \meta{str} between \meta{begin} and \meta{end}; could be used
-% with empty \meta{begin} or \meta{end} to indicate that we
-% want everything until \meta{end} or starting from \meta{begin},
-% respectively;
-% \item \cs{str_count_in:nn} \Arg{str} \Arg{substr} giving the number
-% of occurrences of \meta{substr} in \meta{str};
+% In particular, what is needed for pdf: \textsc{utf-16}?
+% \item CESU
% \item \cs{str_if_head_eq:nN} alias of \cs{tl_if_head_eq_charcode:nN}
% \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}?
+% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}?
% \end{itemize}
% Some functionalities of \pkg{stringstrings} and \pkg{xstring} as well.
%
-%^^A Should "str_item" be "str_char"?
-%
% \end{documentation}
%
% \begin{implementation}
@@ -611,12 +592,51 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{variable}
+% {
+% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one,
+% \c_ninety_seven, \c_one_hundred_twenty_three,
+% \c_one_hundred_twenty_seven
+% }
+% We declare here some integer values which delimit ranges of
+% ASCII characters of various types. This is mostly used in
+% \pkg{l3regex}.
+% \begin{macrocode}
+\int_const:Nn \c_forty_eight { 48 }
+\int_const:Nn \c_fifty_eight { 58 }
+\int_const:Nn \c_sixty_five { 65 }
+\int_const:Nn \c_ninety_one { 91 }
+\int_const:Nn \c_ninety_seven { 97 }
+\int_const:Nn \c_one_hundred_twenty_three { 123 }
+\int_const:Nn \c_one_hundred_twenty_seven { 127 }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\c_max_char_int}
+% The maximum character code depends on the engine.
+% \begin{macrocode}
+\pdftex_if_engine:TF
+ { \int_const:Nn \c_max_char_int { 255 } }
+ { \int_const:Nn \c_max_char_int { 1114111 } }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}{\c_str_replacement_char_int}
+% When converting, invalid bytes are replaced by the Unicode
+% replacement character \textsc{u-fffd}.
+% \begin{macrocode}
+\int_const:Nn \c_str_replacement_char_int { "FFFD }
+% \end{macrocode}
+% \end{variable}
+%
% \begin{macro}
% {
% \str_set:Nn, \str_set:Nx,
% \str_set:cn, \str_set:cx,
% \str_gset:Nn, \str_gset:Nx,
% \str_gset:cn, \str_gset:cx,
+% \str_const:Nn, \str_const:Nx,
+% \str_const:cn, \str_const:cx,
% \str_put_left:Nn, \str_put_left:Nx,
% \str_put_left:cn, \str_put_left:cx,
% \str_gput_left:Nn, \str_gput_left:Nx,
@@ -637,6 +657,7 @@
}
\str_tmp:w { set }
\str_tmp:w { gset }
+ \str_tmp:w { const }
\str_tmp:w { put_left }
\str_tmp:w { gput_left }
\str_tmp:w { put_right }
@@ -648,11 +669,11 @@
% \subsection{Variables and constants}
%
% \begin{macro}{\str_tmp:w}
-% \begin{variable}{\g_str_tmpa_tl}
+% \begin{variable}{\l_str_tmpa_tl}
% Internal scratch space for some functions.
% \begin{macrocode}
\cs_new_protected_nopar:Npn \str_tmp:w { }
-\tl_new:N \g_str_tmpa_tl
+\tl_new:N \l_str_tmpa_tl
% \end{macrocode}
% \end{variable}
% \end{macro}
@@ -667,15 +688,12 @@
% \end{macrocode}
% \end{variable}
%
-% \begin{variable}{\l_str_char_int, \l_str_bytes_int}
-% When converting from various escaped forms to raw characters,
-% we often need to read several digits (hexadecimal or octal depending
-% on the case) and keep track of the corresponding character code
-% in \cs{l_str_char_int}. For \textsc{utf8} support, the number of bytes
-% of for the current character is stored in \cs{l_str_bytes_int}.
+% \begin{variable}{\l_str_char_int}
+% When converting from various forms to characters,
+% \cs{l_str_char_int} is the character code of the
+% character which should be created.
% \begin{macrocode}
\int_new:N \l_str_char_int
-\int_new:N \l_str_bytes_int
% \end{macrocode}
% \end{variable}
%
@@ -685,6 +703,7 @@
% \c_lbrace_str,
% \c_rbrace_str,
% \c_hash_str,
+% \c_tilde_str,
% \c_percent_str
% }
% For all of those strings, \cs{cs_to_str:N} produce characters
@@ -694,11 +713,34 @@
\tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ }
\tl_const:Nx \c_rbrace_str { \cs_to_str:N \} }
\tl_const:Nx \c_hash_str { \cs_to_str:N \# }
+\tl_const:Nx \c_tilde_str { \cs_to_str:N \~ }
\tl_const:Nx \c_percent_str { \cs_to_str:N \% }
% \end{macrocode}
% \end{variable}
%
-% \subsection{Escaping spaces}
+% \begin{variable}{\g_str_aliases_prop}
+% To avoid needing one file per encoding/escaping alias,
+% we keep track of those in a property list.
+% \begin{macrocode}
+\prop_new:N \g_str_aliases_prop
+\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 }
+\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 }
+\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 }
+\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 }
+\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 }
+\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 }
+\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 }
+\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 }
+\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 }
+\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 }
+\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex }
+\prop_gput:Nnn \g_str_aliases_prop { percent } { url }
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Internal functions}
+%
+% \subsubsection{Escaping spaces}
%
% \begin{macro}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n}
% \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w}
@@ -748,6 +790,90 @@
% \end{macrocode}
% \end{macro}
%
+% \begin{macro}[int]{\str_aux_gset_other:Nn}
+% \begin{macro}[aux,EXP]{\str_aux_gset_other_loop:w}
+% \begin{macro}[aux,EXP]{\str_aux_gset_other_end:w}
+% \begin{macrocode}
+\group_begin:
+\char_set_lccode:nn { `\* } { `\ }
+\char_set_lccode:nn { `\A } { `\A }
+\tl_to_lowercase:n
+ {
+ \group_end:
+ \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2
+ {
+ \tl_gset:Nx #1
+ {
+ \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ %
+ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop
+ }
+ }
+ \cs_new_nopar:Npn \str_aux_gset_other_loop:w
+ #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~
+ {
+ \if_meaning:w A #9
+ \str_aux_gset_other_end:w
+ \fi:
+ #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9
+ \str_aux_gset_other_loop:w *
+ }
+ \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop
+ { \fi: #1 }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Mapping}
+%
+% \begin{macro}[rEXP,int]{\str_map_tokens:nn}
+% \begin{macro}[rEXP,int]{\str_map_tokens:Nn}
+% \begin{macro}[rEXP,aux]{\str_map_tokens_aux:nn}
+% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nw}
+% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nN}
+% \begin{macro}[rEXP,int]{\str_map_break_do:n}
+% We simply need to be careful with spaces.
+% Spaces are fed to the mapped tokens with
+% category code $12$, not $10$.
+% \begin{macrocode}
+\cs_new_nopar:Npn \str_map_tokens:Nn
+ { \exp_args:No \str_map_tokens:nn }
+\cs_new:Npn \str_map_tokens:nn #1
+ { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } }
+\cs_new:Npn \str_map_tokens_aux:nn #1#2
+ {
+ \str_map_tokens_loop:nw {#2} #1
+ { ? \use_none_delimit_by_q_recursion_stop:w }
+ ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop
+ }
+\group_begin:
+ \char_set_lccode:nn { `* } { `\ }
+ \tl_to_lowercase:n
+ {
+ \group_end:
+ \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~
+ {
+ \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9*
+ { ? \use_none_delimit_by_q_stop:w } \q_stop
+ \str_map_tokens_loop:nw {#1}
+ }
+ }
+\cs_new:Npn \str_map_tokens_loop:nN #1#2
+ {
+ \use_none:n #2
+ #1 #2
+ \str_map_tokens_loop:nN {#1}
+ }
+\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
% \subsection{Characters given by their position}
%
% \begin{macro}[EXP]{\str_length:N}
@@ -1113,40 +1239,6 @@
% \end{macro}
% \end{macro}
%
-% \subsection{Internal mapping function}
-%
-% \begin{macro}[rEXP,int]{\str_map_tokens:nn}
-% \begin{macro}[rEXP,int]{\str_map_tokens:Nn}
-% \begin{macro}[rEXP,aux]{\str_map_tokens_aux:nn}
-% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nN}
-% \begin{macro}[rEXP,int]{\str_map_break_do:n}
-% We simply need to be careful with spaces.
-% Spaces are fed to the mapped tokens with
-% category code $12$, not $10$.
-% \begin{macrocode}
-\cs_new_nopar:Npn \str_map_tokens:Nn
- { \exp_args:No \str_map_tokens:nn }
-\cs_new_nopar:Npn \str_map_tokens:nn
- { \str_sanitize_args:Nn \str_map_tokens_aux:nn }
-\cs_new:Npn \str_map_tokens_aux:nn #1#2
- {
- \str_map_tokens_loop:nN {#2} #1
- { ? \use_none_delimit_by_q_recursion_stop:w } \q_recursion_stop
- }
-\cs_new_nopar:Npn \str_map_tokens_loop:nN #1#2
- {
- \use_none:n #2
- #1 #2
- \str_map_tokens_loop:nN {#1}
- }
-\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-% \end{macro}
-%
% \subsection{String conditionals}
%
% \begin{macro}[EXP,pTF]{\str_if_eq:NN}
@@ -1163,6 +1255,7 @@
% \end{macro}
% \end{macro}
%
+%^^A treat spaces separately, or make that internal and faster.
% \begin{macro}[EXP,pTF]{\str_if_contains_char:NN}
% \begin{macro}[EXP,pTF]{\str_if_contains_char:nN}
% \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN}
@@ -1199,34 +1292,53 @@
%
% \begin{macro}[EXP,pTF]{\str_if_bytes:N}
% \begin{macro}[EXP,aux]{\str_if_bytes_aux:N}
-% Loop over the string, checking if every character code is less than $256$.
+% \begin{macro}[EXP,aux]{\str_if_bytes_aux:}
+% In the pdf\TeX{} engine, every character is a byte,
+% so there is nothing to check. In other engines, we
+% loop over the string, checking if every character
+% code is less than $256$.
% \begin{macrocode}
-\prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF }
+\pdftex_if_engine:TF
{
- \str_map_tokens:Nn #1 { \str_if_bytes_aux:N }
- \prg_return_true:
+ \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool
+ \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn
+ \cs_new_eq:NN \str_if_bytes:NF \use_none:nn
+ \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn
}
-\cs_new_nopar:Npn \str_if_bytes_aux:N #1
{
- \int_compare:nNnF {`#1} < \c_two_hundred_fifty_six
+ \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF }
{
- \use_i_delimit_by_q_recursion_stop:nw
- { \prg_return_false: \use_none:n }
+ \exp_args:Nf \tl_map_function:nN
+ { \tl_to_str:N #1 }
+ \str_if_bytes_aux:N
+ \prg_return_true:
+ }
+ \cs_new_nopar:Npn \str_if_bytes_aux:N #1
+ {
+ \if_num:w `#1 > \c_two_hundred_fifty_five
+ \exp_after:wN \str_if_bytes_aux:
+ \fi:
}
+ \cs_new_nopar:Npn \str_if_bytes_aux:
+ { \tl_map_break:n { \prg_return_false: \use_none:n } }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
+%
+% \subsection{Conversions}
%
-% \subsection{Internal conditionals}
+% \subsubsection{Internal conditionals}
%
% \begin{macro}[aux]{\str_aux_hexadecimal_test:NTF}
% This test is used when reading hexadecimal digits, for the |\x| escape
-% sequence, and some conversion functions. It returns \meta{true} if the
+% sequence. It returns \meta{true} if the
% token is a hexadecimal digit, and \meta{false} otherwise. It has the
% additional side-effect of updating the value of
% \cs{l_str_char_int} (number formed in base $16$ from the digits
% read so far).
+%^^A todo: speed tweaks?
% \begin{macrocode}
\prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF }
{
@@ -1249,20 +1361,17 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}[aux]{\str_aux_octal_test:NTF}
-% This test is used when reading octal digits, for
-% some conversion functions. It returns \meta{true} if the
-% token is an octal digit, and \meta{false} otherwise.
-% It has the additional side-effect of updating the value of
-% \cs{l_str_char_int} (number formed in base $8$ from the digits
-% read so far).
+% \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF}
+% \TeX{} dutifully detects octal digits for us: if |#1|
+% is an octal digit, then the right-hand side of the
+% comparison is |'1#1|, greater than $1$. Otherwise,
+% the right-hand side stops as |'1|, and the conditional
+% takes the \texttt{false} branch.
% \begin{macrocode}
-\prg_new_protected_conditional:Npnn \str_aux_octal_test:N #1 { TF }
+\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF }
{
- \if_num:w \c_seven < '1 \exp_not:N #1 \exp_stop_f:
- \int_set:Nn \l_str_char_int
- { \c_eight * \l_str_char_int + '#1 }
- \prg_return_true:
+ \if_num:w \c_one < '1 #1 \exp_stop_f:
+ #1 \prg_return_true:
\else:
\prg_return_false:
\fi:
@@ -1270,17 +1379,154 @@
% \end{macrocode}
% \end{macro}
%
-% \begin{macro}{\str_aux_char_if_octal_digit:NTF}
-% ^^A to be removed.
+% \begin{macro}[aux, rEXP]{\str_aux_hexadecimal_use:NTF}
+% \TeX{} detects uppercase hexadecimal digits for us
+% (see \cs{str_aux_octal_use:NTF}, but not the
+% lowercase letters, which we need to detect and replace
+% by their uppercase counterpart.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_aux_char_if_octal_digit:NTF #1
- { \tl_if_in:nnTF { 01234567 } {#1} }
+\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF }
+ {
+ \if_num:w \c_two < "1 #1 \exp_stop_f:
+ #1 \prg_return_true:
+ \else:
+ \if_case:w \int_eval:w `#1 - `a \int_eval_end:
+ A \prg_return_true:
+ \or: B \prg_return_true:
+ \or: C \prg_return_true:
+ \or: D \prg_return_true:
+ \or: E \prg_return_true:
+ \or: F \prg_return_true:
+ \else: \prg_return_false:
+ \fi:
+ \fi:
+ }
% \end{macrocode}
% \end{macro}
%
-% \subsection{Conversions}
+% \subsubsection{Producing one byte or character}
%
-% \subsubsection{Simple unescaping}
+% \begin{variable}{\c_str_byte_0_tl}
+% \begin{variable}{\c_str_byte_-1_tl}
+% For each integer $N$ in the range $[0,255]$, we create
+% a constant token list which holds three character tokens
+% with category code other: the character with character
+% code $N$, followed by the representation of $N$ as
+% two hexadecimal digits.
+% The value $-1$ is given a default token list which ensures
+% that later functions give an empty result for the input $-1$.
+% \begin{macrocode}
+\group_begin:
+ \char_set_catcode_other:n { \c_zero }
+ \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } }
+ \tl_map_inline:Nn \g_str_result_tl
+ { \char_set_lccode:nn {`#1} {`#1} }
+ \tl_map_inline:Nn \g_str_result_tl
+ {
+ \tl_map_inline:Nn \g_str_result_tl
+ {
+ \char_set_lccode:nn { \c_zero } {"##1#1}
+ \tl_to_lowercase:n
+ {
+ \tl_const:cx
+ { c_str_byte_ \int_eval:n {"##1#1} _tl }
+ { ^^@ ##1 #1 }
+ }
+ }
+ }
+\group_end:
+\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } }
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+%
+% \begin{macro}[int, EXP]{\str_output_byte:n}
+% \begin{macro}[aux, EXP]{\str_output_byte:w}
+% \begin{macro}[int, EXP]{\str_output_hexadecimal:n}
+% \begin{macro}[aux, EXP]{\str_output_hexadecimal:w}
+% \begin{macro}[aux, EXP]{\str_output_end:}
+% For now, no error detection, we simply pick either
+% the byte or the hexadecimal representation of it
+% from the three-character token list corresponding
+% to the argument.
+% The value $-1$ produces an empty result.
+% \begin{macrocode}
+\cs_new:Npn \str_output_byte:n #1
+ { \str_output_byte:w #1 \str_output_end: }
+\cs_new_nopar:Npn \str_output_byte:w
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \use_i:nnn
+ \cs:w c_str_byte_ \int_use:N \int_eval:w
+ }
+\cs_new:Npn \str_output_hexadecimal:n #1
+ { \str_output_hexadecimal:w #1 \str_output_end: }
+\cs_new_nopar:Npn \str_output_hexadecimal:w
+ {
+ \exp_after:wN \exp_after:wN
+ \exp_after:wN \use_none:n
+ \cs:w c_str_byte_ \int_use:N \int_eval:w
+ }
+\cs_new_nopar:Npn \str_output_end:
+ { \int_eval_end: _tl \cs_end: }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{}%^^A here
+%
+% \begin{macro}{\str_aux_gmap_result:N}
+% \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN}
+%
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1
+ {
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_aux_gmap_result_loop:NN
+ \exp_after:wN #1
+ \g_str_result_tl { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2
+ {
+ \use_none:n #2
+ #1#2
+ \str_aux_gmap_result_loop:NN #1
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}{\str_aux_gmap_internal_result:N}
+% \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw}
+%
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1
+ {
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_aux_gmap_internal_result_loop:Nw
+ \exp_after:wN #1
+ \g_str_result_tl \q_stop \prg_map_break: ,
+ \prg_break_point:n { }
+ }
+ }
+\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2,
+ {
+ \use_none_delimit_by_q_stop:w #2 \q_stop
+ #1 {#2}
+ \str_aux_gmap_internal_result_loop:Nw #1
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Unescape user input}
%
% The code of this section is used both here for \cs{str_(g)input:Nn},
% and in the regular expression module to go through the regular expression
@@ -1340,7 +1586,7 @@
\cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 }
\cs_set_nopar:Npn \str_aux_escape_raw:N { #3 }
\int_set:Nn \tex_escapechar:D { 92 }
- \tl_gset:Nx \g_str_result_tl { \tl_to_other_str:n {#4} }
+ \str_aux_gset_other:Nn \g_str_result_tl {#4}
\tl_gset:Nx \g_str_result_tl
{
\exp_after:wN \str_aux_escape_loop:N \g_str_result_tl
@@ -1472,7 +1718,7 @@
\exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1
{ \exp_not:N \str_aux_escape_x_end: }
{
- \msg_error:nn { str } { x-missing-brace }
+ \msg_kernel_error:nn { str } { x-missing-brace }
\exp_not:N \str_aux_escape_x_end: #1
}
}
@@ -1480,11 +1726,9 @@
\char_set_catcode_other:N \^^@
\cs_new_protected_nopar:Npn \str_aux_escape_x_end:
{
- \group_begin:
- \char_set_lccode:nn { \c_zero } { \l_str_char_int }
+ \tex_lccode:D \c_zero \l_str_char_int
\tl_to_lowercase:n
{
- \group_end:
\tl_gput_right:Nx \g_str_result_tl
{ \if_false: } \fi:
\str_aux_escape_raw:N ^^@
@@ -1501,577 +1745,2440 @@
% \end{macro}
% \end{macro}
%
-% \subsubsection{Escape and unescape strings for PDF use}
+% \subsubsection{Framework for encoding conversions}
%
-% \begin{macro}{\str_aux_convert_store:}
-% \begin{macro}{\str_aux_convert_store:NNn}
-% \begin{macrocode}
-\group_begin:
- \char_set_catcode_other:n { `\^^@ }
- \cs_new_protected_nopar:Npn \str_aux_convert_store:
- {
- \char_set_lccode:nn { \c_zero } { \l_str_char_int }
- \tl_to_lowercase:n { \tl_gput_right:Nx \g_str_result_tl {^^@} }
- }
- \cs_new_protected_nopar:Npn \str_aux_convert_store:NNn #1#2#3
- {
- \char_set_lccode:nn { \c_zero } {#3}
- \tl_to_lowercase:n { #1 #2 {^^@} }
- }
-\group_end:
-% \end{macrocode}
-% \end{macro}
-% \end{macro}
+% Most functions in this module expect to be working with
+% \enquote{native} strings. Strings can also be stored as
+% bytes, in one of many encodings, for instance \textsc{utf8}.
+% The bytes themselves can be expressed as character
+% tokens with the relevant character code, or as pairs
+% of hexadecimal digits, \emph{etc.} The questions of going
+% from arbitrary characters to bytes, and from bytes to \TeX{}
+% tokens are mostly independent.
%
-% \begin{macro}{\str_aux_byte_to_hexadecimal:N}
-% \begin{macro}{\str_aux_byte_to_octal:N}
+% Conversions are done in four steps:
+% \begin{itemize}
+% \item \enquote{input} produces a string of bytes;
+% \item \enquote{from} interprets the byte string in the old encoding,
+% and converts it to a comma-list, each character becoming
+% \enquote{\ttfamily \meta{character code},};
+% \item \enquote{to} encodes the internal comma list as a byte string
+% in the new encoding;
+% \item \enquote{output} escapes bytes as requested.
+% \end{itemize}
+% The process is modified in case one of the encoding is
+% \texttt{internal} or \texttt{native}: then the input or
+% output step is ignored.
+%
+% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn}
+% \begin{macro}[aux]{\str_convert_aux_i:NNnnn}
+% \begin{macro}[aux]{\str_convert_aux_ii:wwwnn}
+% \begin{macro}[aux]{\str_convert_aux_iii:nnF}
+% \begin{macro}[aux]{\str_convert_aux_iv:nnnF}
+% \begin{macro}[aux]{\str_convert_aux_v:NNnNN}
+% The input string is stored in \cs{g_str_result_tl},
+% then we call the various conversion functions, which all
+% act on \cs{g_str_result_tl}. The arguments |#3| and |#4|
+% of the \texttt{aux_i} function are split into their
+% \meta{encoding} and \meta{escaping} parts by \texttt{aux_ii}.
+% The conversion function names are built by \texttt{aux_iii},
+% which first tries to use the name as supplied by the user,
+% then filters it with \cs{str_aux_lowercase_alphanum:n},
+% and calls \texttt{aux_iv} to load the relevant definition file.
+% In all cases, \texttt{aux_iii} ensures that the two
+% required conversion functions are defined, with
+% a fall-back to the native encoding with no escaping.
+% Once this is done, \texttt{aux_v} does the main work:
+% \begin{itemize}
+% \item |#1| is the encoding conversion function;
+% \item |#2| is the escaping function;
+% \item |#3| is the escaping name;
+% \item |#4| is the \enquote{native} encoding function
+% (either \enquote{from} or \enquote{to});
+% \item |#5| is \cs{use_i:nn} or \cs{use_ii:nn}.
+% \end{itemize}
+% When converting to the internal format, the escaping must be done
+% first, then the encoding (this is controlled by |#5|).
+% If the encoding is native, then the escaping is ignored and
+% must be empty. Otherwise, the escaping |#2| is performed.
% \begin{macrocode}
-\cs_new_nopar:Npn \str_aux_byte_to_hexadecimal:N #1
+\cs_new_protected:Npn \str_set_convert:Nnnn
+ { \str_convert_aux_i:NNnnn \str_set:Nx }
+\cs_new_protected:Npn \str_gset_convert:Nnnn
+ { \str_convert_aux_i:NNnnn \str_gset:Nx }
+\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5
{
- \int_compare:nNnTF {`#1} < { 256 }
+ \group_begin:
+ \str_aux_gset_other:Nn \g_str_result_tl {#5}
+
+ \exp_after:wN \str_convert_aux_ii:wwwnn
+ \tl_to_str:n {#3} /// \q_stop
+ { from } { input } \str_convert_from_native: \use_i:nn
+
+ \exp_after:wN \str_convert_aux_ii:wwwnn
+ \tl_to_str:n {#4} /// \q_stop
+ { to } { output } \str_convert_to_native: \use_ii:nn
+
+ \group_end:
+ #1 #2 \g_str_result_tl
+ }
+\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5
+ {
+ \str_convert_aux_iii:nnF {#4} {#1} {native}
+ \str_convert_aux_iii:nnF {#5} {#2} { }
+ \exp_args:Ncc \str_convert_aux_v:NNnNN
+ { str_convert_#4_#1: } { str_convert_#5_#2: } {#2}
+ }
+\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3
+ {
+ \cs_if_exist:cF { str_convert_#1_#2: }
{
- \int_to_letter:n { \int_div_truncate:nn {`#1} \c_sixteen }
- \int_to_letter:n { \int_mod:nn {`#1} \c_sixteen }
+ \exp_args:Nx \str_convert_aux_iv:nnnF
+ { \str_aux_lowercase_alphanum:n {#2} }
+ {#1} {#2} {#3}
}
- { \msg_expandable_error:n { Invalid~byte~`#1'. } }
}
-\cs_new_nopar:Npn \str_aux_byte_to_octal:N #1
+\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4
{
- \int_compare:nNnTF {`#1} < { 64 }
+ \cs_if_exist:cF { str_convert_#2_#1: }
{
- 0
- \int_to_letter:n { \int_div_truncate:nn {`#1} \c_eight }
- \int_to_letter:n { \int_mod:nn {`#1} \c_eight }
+ \group_begin:
+ \str_load_catcodes:
+ \str_load_one:n {#1}
+ \group_end:
}
+ \cs_if_exist:cTF { str_convert_#2_#1: }
+ { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } }
{
- \int_compare:nNnTF {`#1} < { 256 }
- { \int_to_octal:n {`#1} }
- { \msg_expandable_error:n { Invalid~byte~`#1'. } }
+ \msg_kernel_error:nnx { str } { unknown-#2 } {#3}
+ \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: }
}
}
+\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5
+ {
+ #5 { } {#1}
+ \cs_if_eq:NNTF #1 #4
+ {
+ \tl_if_empty:nF {#3}
+ { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} }
+ }
+ { #2 }
+ #5 {#1} { }
+ }
% \end{macrocode}
% \end{macro}
% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}{\str_bytes_escape_hexadecimal:NN}
+% \begin{macro}[int, rEXP]{\str_aux_lowercase_alphanum:n}
+% \begin{macro}[aux, rEXP]{\str_aux_lowercase_alphanum_loop:N}
+% This function keeps only letters and digits, with upper case
+% letters converted to lower case.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_bytes_escape_hexadecimal:NN #1#2
+\cs_new:Npn \str_aux_lowercase_alphanum:n #1
{
- \str_set:Nx #1
- { \str_map_tokens:Nn #2 \str_aux_byte_to_hexadecimal:N }
+ \exp_after:wN \str_aux_lowercase_alphanum_loop:N
+ \tl_to_str:n {#1} { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1
+ {
+ \use_none:n #1
+ \if_num:w `#1 < \c_ninety_one
+ \if_num:w `#1 < \c_sixty_five
+ \if_num:w \c_one < 1#1 \exp_stop_f:
+ #1
+ \fi:
+ \else:
+ \str_output_byte:n { `#1 + \c_thirty_two }
+ \fi:
+ \else:
+ \if_num:w `#1 < \c_one_hundred_twenty_three
+ \if_num:w `#1 < \c_ninety_seven
+ \else:
+ #1
+ \fi:
+ \fi:
+ \fi:
+ \str_aux_lowercase_alphanum_loop:N
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}{\str_bytes_escape_name:NN}
+% \begin{macro}[int]{\str_convert_from_internal:}
+% \begin{macro}[int]{\str_convert_to_internal:}
+% Converting from the internal format to itself is
+% of course trivial.
% \begin{macrocode}
-\tl_const:Nx \c_str_bytes_escape_name_str
- { \c_hash_str \c_percent_str \c_lbrace_str \c_rbrace_str ()/<>[] }
-\cs_new_protected_nopar:Npn \str_bytes_escape_name:NN #1#2
+\cs_new_protected_nopar:Npn \str_convert_from_internal: { }
+\cs_new_protected_nopar:Npn \str_convert_to_internal: { }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Loading encoding files}
+%
+% \begin{macro}{\str_load:n}
+% \begin{macro}[int]{\str_load_one:n}
+% \begin{macro}[aux]{\str_load_alias:n, \str_load_alias_aux:nnn}
+% Loading encoding or escaping files can be done in bulk by
+% specifying a comma-list of encodings and escapings. Each
+% name is normalized: only alphanumeric characters are kept,
+% lowercased. The relevant file is loaded if it was not loaded
+% already.
+% \begin{macrocode}
+\cs_new_protected:Npn \str_load:n #1
{
- \str_set:Nx #1
- { \str_map_tokens:Nn #2 \str_bytes_escape_name_aux:N }
+ \group_begin:
+ \str_load_catcodes:
+ \clist_map_inline:nn {#1}
+ {
+ \exp_args:Nx \str_load_one:n
+ { \str_aux_lowercase_alphanum:n {#1} }
+ }
+ \group_end:
}
-\cs_new_nopar:Npn \str_bytes_escape_name_aux:N #1
+\cs_new_protected:Npn \str_load_one:n #1
{
- \int_compare:nNnTF {`#1} < { "21 }
- { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 }
+ \cs_if_exist:cF { str_convert_from_ #1 : }
{
- \int_compare:nNnTF {`#1} > { "7E }
- { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 }
+ \cs_if_exist:cF { str_convert_input_#1: }
{
- \str_if_contains_char:NNTF \c_str_bytes_escape_name_str #1
- { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 }
- {#1}
+ \file_if_exist:nTF { l3str- #1 .def }
+ { \file_input:n { l3str- #1 .def } }
+ { \str_load_alias:n {#1} }
}
}
}
+\cs_new_protected:Npn \str_load_alias:n #1
+ {
+ \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl
+ {
+ \str_load_one:n { \l_str_tmpa_tl }
+ \cs_if_exist:cTF
+ { str_convert_from_ \l_str_tmpa_tl : }
+ {
+ \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from }
+ \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to }
+ }
+ {
+ \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input }
+ \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output }
+ }
+ }
+ { \msg_kernel_error:nnx { str } { file-not-found } {#1} }
+ }
+\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3
+ { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } }
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}{\str_bytes_escape_string:NN}
-% Any character below (and including) space, and any character
-% above (and including) \texttt{del}, are converted to octal.
-% One backslash is added before each parenthesis and backslash.
+% \begin{macro}[int]{\str_load_catcodes:}
+% Since encoding files may be loaded at arbitrary places
+% in a \TeX{} document, including within verbatim mode,
+% we set the catcodes of all characters appearing in any
+% encoding definition file.
% \begin{macrocode}
-\tl_const:Nx \c_str_bytes_escape_string_str { \c_backslash_str ( ) }
-\cs_new_protected_nopar:Npn \str_bytes_escape_string:NN #1#2
+\cs_new_protected:Npn \str_load_catcodes:
{
- \str_set:Nx #1
- { \str_map_tokens:Nn #2 \str_bytes_escape_string_aux:N }
+ \char_set_catcode_escape:N \\
+ \char_set_catcode_group_begin:N \{
+ \char_set_catcode_group_end:N \}
+ \char_set_catcode_math_superscript:N \^
+ \char_set_catcode_ignore:N \ %
+ \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN }
+ \char_set_catcode_letter:N
+ \tl_map_function:nN { 0123456789"'?*+ }
+ \char_set_catcode_other:N
+ \char_set_catcode_comment:N \%
+ \int_set:Nn \tex_endlinechar:D {32}
}
-\cs_new_nopar:Npn \str_bytes_escape_string_aux:N #1
+% \end{macrocode}
+% \end{macro}
+%
+%^^A todo: document, move.
+% \begin{macrocode}
+\int_new:N \l_str_encoding_int
+% \end{macrocode}
+%
+% \begin{macro}[int]{\str_convert_from_eight_bit:n}
+% \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn}
+% \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n}
+% \begin{macro}[aux, EXP]{\str_convert_from_eight_bit_aux:N}
+%^^A todo: document
+% \begin{macrocode}
+\cs_new_protected:Npn \str_convert_from_eight_bit:n #1
{
- \int_compare:nNnTF {`#1} < { "21 }
- { \c_backslash_str \str_aux_byte_to_octal:N #1 }
- {
- \int_compare:nNnTF {`#1} > { "7E }
- { \c_backslash_str \str_aux_byte_to_octal:N #1 }
- {
- \str_if_contains_char:NNT \c_str_bytes_escape_string_str #1
- { \c_backslash_str }
- #1
- }
- }
+ \group_begin:
+ \int_zero:N \l_str_encoding_int
+ \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn
+ { \tl_use:c { c_str_encoding_#1_tl } }
+ { \q_stop \prg_map_break: } { }
+ \prg_break_point:n { }
+ \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n
+ { \tl_use:c { c_str_encoding_#1_missing_tl } }
+ { \q_stop \prg_map_break: }
+ \prg_break_point:n { }
+ \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N
+ \group_end:
+ }
+\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2
+ {
+ \use_none_delimit_by_q_stop:w #1 \q_stop
+ \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop:
+ \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 }
+ \tex_advance:D \l_str_encoding_int \c_one
+ \str_convert_from_eight_bit_load:nn
+ }
+\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1
+ {
+ \use_none_delimit_by_q_stop:w #1 \q_stop
+ \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop:
+ \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop:
+ \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD }
+ \tex_advance:D \l_str_encoding_int \c_one
+ \str_convert_from_eight_bit_load_missing:n
+ }
+\cs_new:Npn \str_convert_from_eight_bit_aux:N #1
+ {
+ \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int
+ \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f:
+ \tex_the:D \tex_toks:D \tex_dimen:D
+ \fi:
+ \fi:
+ \int_value:w `#1 ,
}
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}{\str_bytes_unescape_hexadecimal:NN}
-% \begin{macro}[aux]{\str_bytes_unescape_hexadecimal_aux:N}
-% \begin{macro}[aux]{\str_bytes_unescape_hexadecimal_aux_ii:N}
-% Takes chars two by two, and interprets each pair as a hexadecimal code
-% for a character. Any non-hexadecimal-digit is ignored. An odd-length
-% string gets a |0| appended to it (this is equivalent to appending a |0|
-% in all cases, and dropping it if it is alone).
+% \begin{macro}[int]{\str_convert_to_eight_bit:n}
+% \begin{macro}[aux]{\str_convert_to_eight_bit_load:nn}
+% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux:n}
+% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux_ii:n}
+%^^A todo: document
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal:NN #1#2
+\cs_new_protected:Npn \str_convert_to_eight_bit:n #1
{
\group_begin:
- \tl_gclear:N \g_str_result_tl
- \int_zero:N \l_str_char_int
- \exp_last_unbraced:Nf \str_bytes_unescape_hexadecimal_aux:N
- { \tl_to_other_str:N #2 } 0
- \q_recursion_tail \q_recursion_stop
+ \int_zero:N \l_str_encoding_int
+ \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn
+ { \tl_use:c { c_str_encoding_#1_tl } }
+ { \q_stop \prg_map_break: } { }
+ \prg_break_point:n { }
+ \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n
\group_end:
- \tl_set_eq:NN #1 \g_str_result_tl
}
-\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal_aux:N #1
+\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2
{
- \quark_if_recursion_tail_stop:N #1
- \str_aux_hexadecimal_test:NTF #1
- { \str_bytes_unescape_hexadecimal_aux_ii:N }
- { \str_bytes_unescape_hexadecimal_aux:N }
+ \use_none_delimit_by_q_stop:w #1 \q_stop
+ \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop:
+ \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop:
+ \exp_args:NNf \tex_toks:D \l_str_encoding_int
+ { \str_output_byte:n { "#1 } }
+ \tex_advance:D \l_str_encoding_int \c_one
+ \str_convert_to_eight_bit_load:nn
}
-\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal_aux_ii:N #1
+\cs_new:Npn \str_convert_to_eight_bit_aux:n #1
{
- \quark_if_recursion_tail_stop:N #1
- \str_aux_hexadecimal_test:NTF #1
+ \if_num:w #1 < "8000 \exp_stop_f:
+ \if_num:w \tex_dimen:D #1 < \l_str_encoding_int
+ \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f:
+ \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f:
+ \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn
+ \fi:
+ \fi:
+ \str_convert_to_eight_bit_aux_ii:n {#1}
+ \else:
+ \msg_expandable_kernel_error:nnn
+ { str } { eight_bit-to-byte-overflow } {#1}
+ \fi:
+ }
+\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1
+ {
+ \if_num:w #1 < \c_two_hundred_fifty_six
+ \str_output_byte:n {#1}
+ \else:
+ \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte }
+ \fi:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_encoding_eight_bit:n}
+%^^A todo: document
+% \begin{macrocode}
+\cs_new_protected:Npn \str_encoding_eight_bit:n #1
+ {
+ \cs_new_protected:cpn { str_convert_from_#1: }
+ { \str_convert_from_eight_bit:n {#1} }
+ \cs_new_protected:cpn { str_convert_to_#1: }
+ { \str_convert_to_eight_bit:n {#1} }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsubsection{Byte input and output}
+%
+% Strings of bytes may need to be stored in auxiliary files
+% in safe \enquote{escaping} formats. Each such escaping
+% is only loaded as needed. By default, on input any non-byte
+% is filtered out, while the output simply consists in letting
+% bytes through.
+%
+% \begin{macro}[int, rEXP]{\str_filter_bytes:n}
+% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N}
+% In the case of pdf\TeX{}, every character is a byte.
+% For Unicode-aware engines, test the character code.
+% Spaces have already been given the correct category
+% code when this function is called.
+% \begin{macrocode}
+\pdftex_if_engine:TF
+ { \cs_new_eq:NN \str_filter_bytes:n \use:n }
+ {
+ \cs_new_nopar:Npn \str_filter_bytes:n #1
+ {
+ \str_filter_bytes_aux:N #1
+ { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ \cs_new_nopar:Npn \str_filter_bytes_aux:N #1
{
- \str_aux_convert_store:
- \int_zero:N \l_str_char_int
- \str_bytes_unescape_hexadecimal_aux:N
+ \use_none:n #1
+ \if_num:w `#1 < 256 \exp_stop_f: #1 \fi:
+ \str_filter_bytes_aux:N
}
- { \str_bytes_unescape_hexadecimal_aux_ii:N }
}
% \end{macrocode}
% \end{macro}
% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_input_:}
+% \begin{macro}[int]{\str_convert_input_bytes:}
+% The simplest input method removes non-bytes
+% from \cs{g_str_result_tl}.
+% \begin{macrocode}
+\pdftex_if_engine:TF
+ { \cs_new_protected_nopar:Npn \str_convert_input_: { } }
+ {
+ \cs_new_protected_nopar:Npn \str_convert_input_:
+ {
+ \tl_gset:Nx \g_str_result_tl
+ { \exp_args:No \str_filter_bytes:n \g_str_result_tl }
+ }
+ }
+\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_:
+% \end{macrocode}
+% \end{macro}
% \end{macro}
%
-% \begin{macro}{\str_bytes_unescape_name:NN}
-% \begin{macro}[aux]{\str_bytes_unescape_name_aux:wN}
-% \begin{macro}[aux]{\str_bytes_unescape_name_aux_ii:N}
-% This changes all occurrences of |#| followed by two upper- or lowercase
-% hexadecimal digits by the corresponding unescaped character.
+% \begin{macro}[int]{\str_convert_output_:}
+% \begin{macro}[int]{\str_convert_output_bytes:}
+% The simplest form of output leaves the bytes from the previous
+% step of the conversion unchanged.
% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_convert_output_: { }
+\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_:
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Convert to and from native strings}
+%
+% \begin{macro}[int]{\str_convert_from_native:}
+% \begin{macro}[aux, rEXP]{\str_convert_from_native_aux:N}
+% Convert each character to its character code, one at a time.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_convert_from_native:
+ { \str_aux_gmap_result:N \str_convert_from_native_aux:N }
+\cs_new:Npn \str_convert_from_native_aux:N #1
+ { \int_value:w `#1 , }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_to_native:}
+% \begin{macro}[aux]{\str_convert_to_native_aux:w}
+% Convert the string from the given encoding
+% to the internal format. We grab integers
+% from the internal string using the low-level
+% method to assign integers.
+% Except at the end of the loop, the function
+% \cs{str_convert_to_native_aux:w} is followed by a
+% comma, hence its argument is empty.
+% We are making heavy use of \TeX{} primitives here,
+% because the function can become very slow for large
+% strings.
+% \begin{macrocode}
+\cs_new_protected_nopar:Npn \str_convert_to_native:
+ {
+ \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl
+ \tl_gclear:N \g_str_result_tl
+ \tex_afterassignment:D \str_convert_to_native_aux:w
+ \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: ,
+ \prg_break_point:n { }
+ }
\group_begin:
- \char_set_lccode:nn {`\*} {`\#}
- \tl_to_lowercase:n
+ \char_set_catcode_other:n { `\^^@ }
+ \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 ,
{
- \group_end:
- \cs_new_protected_nopar:Npn \str_bytes_unescape_name:NN #1#2
- {
- \group_begin:
- \tl_gclear:N \g_str_result_tl
- \int_zero:N \l_str_char_int
- \exp_last_unbraced:Nf \str_bytes_unescape_name_aux:wN
- { \tl_to_other_str:N #2 }
- * \q_recursion_tail \q_recursion_stop
- \group_end:
- \tl_set_eq:NN #1 \g_str_result_tl
- }
- \cs_new_protected_nopar:Npn \str_bytes_unescape_name_aux:wN #1 * #2
- {
- \tl_gput_right:Nx \g_str_result_tl {#1}
- \quark_if_recursion_tail_stop:N #2
- \str_aux_hexadecimal_test:NTF #2
- { \str_bytes_unescape_name_aux_ii:NN #2 }
- {
- \tl_gput_right:Nx \g_str_result_tl { \c_hash_str }
- \str_bytes_unescape_name_aux:wN #2
- }
- }
+ #1
+ \if_int_compare:w \l_str_char_int > \c_max_char_int
+ \msg_kernel_error:nnx { str } { overflow }
+ { \int_use:N \l_str_char_int }
+ \else:
+ \tex_lccode:D \c_zero \l_str_char_int
+ \tl_to_lowercase:n
+ { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } }
+ \fi:
+ \tex_afterassignment:D \str_convert_to_native_aux:w
+ \l_str_char_int =
}
-\cs_new_protected_nopar:Npn \str_bytes_unescape_name_aux_ii:NN #1#2
+\group_end:
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Messages}
+%
+% \begin{macrocode}
+\msg_kernel_new:nnnn { str } { x-missing-brace }
+ { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. }
+ {
+ You~wrote~something~like~
+ `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~
+ The~closing~brace~is~missing.
+ }
+\msg_kernel_new:nnnn { str } { overflow }
+ { Character~code~#1~too~big. }
{
- \str_aux_hexadecimal_test:NTF #2
+ \int_compare:nNnTF {#1} > { 1114111 }
+ { The~Unicode~standard~limits~code~points~to~1114111. }
{
- \str_aux_convert_store:
- \int_zero:N \l_str_char_int
- \str_bytes_unescape_name_aux:wN
+ The~pdfTeX~engine~only~supports~8-bit~characters:~
+ valid~character~codes~are~in~the~range~[0,255].~
+ To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX.
}
+ }
+\msg_kernel_new:nnn { str } { convert-input }
+ { Input~scheme~`#1'~unknown. }
+\msg_kernel_new:nnn { str } { convert-from }
+ { Encoding~`#1'~unknown. }
+\msg_kernel_new:nnn { str } { convert-to }
+ { Encoding~`#1'~unknown.~Using~UTF-8. }
+\msg_kernel_new:nnn { str } { convert-output }
+ { Output~scheme~`#1~unknown. }
+\msg_kernel_new:nnn { str } { to-native }
+ { Use~\str_set_convert:Nnn to~convert~to~native~strings. }
+\msg_kernel_new:nnn { str } { unicode-surrogate }
+ { Code~point~#1~is~an~unpaired~surrogate. }
+\msg_kernel_new:nnn { str } { utf16-surrogate }
+ { UTF-16~unpaired~surrogate~#1. }
+\msg_kernel_new:nnn { str } { utf16-odd }
+ { UTF-16~dangling~byte~#1. }
+\msg_kernel_new:nnn { str } { utf8-extra-conti }
+ { UTF-8~extra~continuation~byte~#1. }
+\msg_kernel_new:nnn { str } { utf8-missing-conti }
+ { UTF-8~missing~continuation~byte~#1. }
+\msg_kernel_new:nnn { str } { utf8-invalid-byte }
+ { Byte~#1~cannot~appear~in~UTF-8. }
+\msg_kernel_new:nnn { str } { utf8-overlong }
+ { Overlong~UTF-8~byte~sequence~for~code~point~#1. }
+\msg_kernel_new:nnn { str } { utf8-premature-end }
+ { Incomplete~last~UTF-8~character. }
+\msg_kernel_new:nnn { str } { utf32-overflow }
+ { Code~point~too~large~(UTF-32~`#1'). }
+\msg_kernel_new:nnn { str } { utf32-truncated }
+ { Truncated~UTF-32~string~`...#1'. }
+% \end{macrocode}
+%
+% \subsection{Deprecated string functions}
+%
+% \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n}
+% The naming scheme is a little bit more consistent
+% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}.
+% \begin{macrocode}
+\cs_set:Npn \str_length_skip_spaces:N
+ { \exp_args:No \str_length_skip_spaces:n }
+\cs_set_eq:NN \str_length_skip_spaces:n \str_length_ignore_spaces:n
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macrocode}
+%</initex|package>
+% \end{macrocode}
+%
+% \subsection{Escaping definition files}
+%
+% Several of those encodings are defined by the pdf file format.
+% The following byte storage methods are defined:
+% \begin{itemize}
+% \item \texttt{bytes} (default), non-bytes are filtered out,
+% and bytes are left untouched;
+% \item \texttt{hex} or \texttt{hexadecimal},
+% as per the pdf\TeX{} primitive \tn{pdfescapehex}
+% \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename}
+% \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring}
+% \item \texttt{url} or \texttt{percent}, as per the percent encoding of urls.
+% \end{itemize}
+%^^A todo: in pdfTeX, use the \tn{pdfescape...}
+%^^A todo: add various shell escapes, lua escapes etc?
+%
+% \subsubsection{Input methods}
+%
+% \begin{macro}[int]{\str_convert_input_hex:}
+% \begin{macro}[int]{\str_convert_input_hexadecimal:}
+% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux:N}
+% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux_ii:N}
+% Take chars two by two, and interpret each pair as
+% the hexadecimal code for a byte. Anything else than
+% hexadecimal digits is ignored.
+% A string which contains an odd number of hexadecimal
+% digits gets |0| appended to it: this is equivalent
+% to appending a |0| in all cases, and dropping it if
+% it is alone.
+% \begin{macrocode}
+%<*hex>
+\cs_new_protected_nopar:Npn \str_convert_input_hex:
+ {
+ \tl_gset:Nx \g_str_result_tl
{
- \tl_gput_right:Nx \g_str_result_tl { \c_hash_str #1 }
- \int_zero:N \l_str_char_int
- \str_bytes_unescape_name_aux:wN #2
+ \str_output_byte:w "
+ \exp_after:wN \str_convert_input_hex_aux:N
+ \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: }
+ \prg_break_point:n { \str_output_end: }
}
}
+\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1
+ {
+ \use_none:n #1
+ \str_aux_hexadecimal_use:NTF #1
+ \str_convert_input_hex_aux_ii:N
+ \str_convert_input_hex_aux:N
+ }
+\cs_new_nopar:Npn \str_convert_input_hex_aux_ii:N #1
+ {
+ \use_none:n #1
+ \str_aux_hexadecimal_use:NTF #1
+ {
+ \str_output_end:
+ \str_output_byte:w " \str_convert_input_hex_aux:N
+ }
+ \str_convert_input_hex_aux_ii:N
+ }
+%</hex>
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
+% \end{macro}
%
-% \begin{macro}{\str_bytes_unescape_string:NN}
-% Here, we need to detect backslashes, which escape characters
-% as follows.
-% \begin{itemize}
-% \item[\texttt{\string\n}] Line feed ($10$)
-% \item[\texttt{\string\r}] Carriage return ($13$)
-% \item[\texttt{\string\t}] Horizontal tab ($9$)
-% \item[\texttt{\string\b}] Backspace ($8$)
-% \item[\texttt{\string\f}] Form feed ($12$)
-% \item[\texttt{(}] Left parenthesis
-% \item[\texttt{)}] Right parenthesis
-% \item[\texttt{\string\\}] Backslash
-% \item[\texttt{\string\ddd}] Character code \texttt{ddd} (octal)
+% \begin{macro}[int]{\str_convert_input_name:}
+% \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN}
+% \begin{macro}[int]{\str_convert_input_url:}
+% \begin{macro}[int]{\str_convert_input_percent:}
+% \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN}
+% The \cs{str_convert_input_name:} function replaces each
+% occurrence of |#| followed by two hexadecimal digits
+% in \cs{g_str_result_tl} by the corresponding byte.
+% The \texttt{url} function is identical, with escape
+% character |%| instead of |#|. Thus we define the two
+% together. The arguments of \cs{str_tmp:w} are the character
+% code of |#| or |%| in hexadecimal, the name of the main
+% function to define, and the name of the auxiliary which
+% performs the loop.
+%
+% The looping auxiliary |#3| finds the next escape character,
+% reads the following two characters, and tests them. The test
+% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit
+% in the input stream, hence we surround the test with
+% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}.
+% If both characters are hexadecimal digits, they should be
+% removed before looping: this is done by \cs{use_i:nnn}.
+% If one of the characters is not a hexadecimal digit,
+% then feed |"#1| to \cs{str_output_byte:w} to produce the
+% escape character, and call the looping function followed
+% by the two characters (remove \cs{use_i:nnn}).
+% \begin{macrocode}
+%<*name|url>
+\cs_set_protected:Npn \str_tmp:w #1#2#3
+ {
+ \cs_new_protected_nopar:Npn #2
+ {
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN #3 \g_str_result_tl
+ #1 ? { ? \prg_map_break: }
+ \prg_break_point:n { }
+ }
+ }
+ \cs_new_nopar:Npn #3 ##1#1##2##3
+ {
+ \str_filter_bytes:n {##1}
+ \use_none:n ##3
+ \str_output_byte:w "
+ \str_aux_hexadecimal_use:NTF ##2
+ {
+ \str_aux_hexadecimal_use:NTF ##3
+ { }
+ { * \c_zero + `#1 \use_i:nn }
+ }
+ { 0 + `#1 \use_i:nn }
+ \str_output_end:
+ \use_i:nnn #3 ##2##3
+ }
+ }
+%</name|url>
+%<name>\exp_after:wN \str_tmp:w \c_hash_str
+%<name> \str_convert_input_name: \str_convert_input_name_aux:wNN
+%<url>\exp_after:wN \str_tmp:w \c_percent_str
+%<url> \str_convert_input_url: \str_convert_input_url_aux:wNN
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_input_string:}
+% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN}
+% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wNNN}
+% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:NNNNNN}
+% The \texttt{string} escaping is somewhat similar to
+% the \texttt{name} and \texttt{url} escapings, with
+% escape character |\|.
+% The first step is to convert all three line endings,
+% |^^J|, |^^M|, and |^^M^^J| to the common |^^J|, as per
+% the \textsc{pdf} specification.
+% Then the following escape sequences are decoded.
+% \begin{itemize}\def\makelabel#1{\hss\llap{\ttfamily\string#1}}
+% \item[\n] Line feed ($10$)
+% \item[\r] Carriage return ($13$)
+% \item[\t] Horizontal tab ($9$)
+% \item[\b] Backspace ($8$)
+% \item[\f] Form feed ($12$)
+% \item[\(] Left parenthesis
+% \item[\)] Right parenthesis
+% \item[\\] Backslash
+% \item[\ddd] (backslash followed by $1$ to $3$ octal digits)
+% Byte \texttt{ddd} (octal), subtracting $256$ in case of overflow.
% \end{itemize}
% If followed by an end-of-line character, the backslash and
% the end-of-line are ignored. If followed by anything else,
% the backslash is ignored.
-% The PDF specification indicates that LF, CR, and CRLF should be
-% converted to LF: \emph{this is not implemented here}.
% \begin{macrocode}
+%<*string>
\group_begin:
\char_set_lccode:nn {`\*} {`\\}
\char_set_catcode_other:N \^^J
\char_set_catcode_other:N \^^M
\tl_to_lowercase:n
{
- \group_end:
- \cs_new_protected_nopar:Npn \str_bytes_unescape_string:NN #1#2
+ \cs_new_protected_nopar:Npn \str_convert_input_string:
{
- \group_begin:
- \tl_gclear:N \g_str_result_tl
- \exp_last_unbraced:Nf \str_bytes_unescape_string_aux:wN
- { \tl_to_other_str:N #2 }
- * \q_recursion_tail \q_recursion_stop
- \group_end:
- \tl_set_eq:NN #1 \g_str_result_tl
- }
- \cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux:wN #1 * #2
- {
- \tl_gput_right:Nx \g_str_result_tl {#1}
- \quark_if_recursion_tail_stop:N #2
- \int_zero:N \l_str_char_int
- \str_aux_octal_test:NTF #2
- { \str_bytes_unescape_string_aux_d:N }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_convert_input_string_aux:wN
+ \g_str_result_tl \prg_map_break: ^^M ?
+ \prg_break_point:n { }
+ }
+ \tl_gset:Nx \g_str_result_tl
{
- \int_set:Nn \l_str_char_int
- {
- \prg_case_str:xxn {#2}
- {
- { n } { 10 }
- { r } { 13 }
- { t } { 9 }
- { b } { 8 }
- { f } { 12 }
- { ( } { 40 }
- { ) } { 41 }
- { \c_backslash_str } { 92 }
- { ^^J } { -1 }
- { ^^M } { -1 }
- }
- {`#2}
- }
- \int_compare:nNnF \l_str_char_int = \c_minus_one
- { \str_aux_convert_store: }
- \str_bytes_unescape_string_aux:wN
+ \exp_after:wN \str_convert_input_string_aux:wNNN
+ \g_str_result_tl * ?? { ? \prg_map_break: }
+ \prg_break_point:n { }
}
}
+ \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4
}
-\cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux_d:N #1
+ {
+ \str_filter_bytes:n {#1}
+ \use_none:n #4
+ \str_output_byte:w '
+ \str_aux_octal_use:NTF #2
+ {
+ \str_aux_octal_use:NTF #3
+ {
+ \str_aux_octal_use:NTF #4
+ {
+ \if_int_compare:w #2 > \c_three
+ - 256
+ \fi:
+ \str_convert_input_string_aux:NNNNNN
+ }
+ { \str_convert_input_string_aux:NNNNNN ? }
+ }
+ { \str_convert_input_string_aux:NNNNNN ?? }
+ }
+ {
+ \prg_case_str:xxn {#2}
+ {
+ { \c_backslash_str } { 134 }
+ { ( } { 50 }
+ { ) } { 51 }
+ { r } { 15 }
+ { f } { 14 }
+ { n } { 12 }
+ { t } { 11 }
+ { b } { 10 }
+ { ^^J } { 0 - \c_one }
+ }
+ { 0 - \c_one \use_i:nn }
+ }
+ \str_output_end:
+ \use_i:nn \str_convert_input_string_aux:wNNN #2#3#4
+ }
+ \cs_new_nopar:Npn \str_convert_input_string_aux:NNNNNN #1#2#3#4#5#6
+ { \str_output_end: \str_convert_input_string_aux:wNNN }
+ \cs_new_nopar:Npn \str_convert_input_string_aux:wN #1 ^^M #2
+ {
+ #1 ^^J
+ \if_charcode:w ^^J #2
+ \exp_after:wN \use_i:nn
+ \fi:
+ \str_convert_input_string_aux:wN #2
+ }
+\group_end:
+%</string>
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsubsection{Output methods}
+%
+% \begin{macro}[int]{\str_convert_output_hex:}
+% \begin{macro}[int]{\str_convert_output_hexadecimal:}
+% \begin{macro}[aux, rEXP]{\str_convert_output_hex_aux:N}
+% Loop and convert each byte to hexadecimal.
+% \begin{macrocode}
+%<*hex>
+\cs_new_protected_nopar:Npn \str_convert_output_hex:
+ { \str_aux_gmap_result:N \str_convert_output_hex_aux:N }
+\cs_new_nopar:Npn \str_convert_output_hex_aux:N #1
+ { \str_output_hexadecimal:n { `#1 } }
+%</hex>
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_output_name:}
+% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:N}
+% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:NTF}
+% \begin{variable}{\c_str_convert_output_name_str}
+% \begin{variable}{\c_str_convert_output_name_not_str}
+% For each byte, test whether it should
+% be output as is, or be \enquote{hash-encoded}.
+% Roughly, bytes outside the range from |"2A| to |"7E|
+% are hash-encoded. We keep two lists of exceptions:
+% characters in \cs{c_str_convert_output_name_not_str}
+% are not hash-encoded, and characters in the
+% \cs{c_str_convert_output_name_str} are encoded.
+% \begin{macrocode}
+%<*name>
+\str_const:Nn \c_str_convert_output_name_not_str { ! " $ & ' } %$
+\str_const:Nn \c_str_convert_output_name_str { {}/<>[] }
+\cs_new_protected_nopar:Npn \str_convert_output_name:
+ { \str_aux_gmap_result:N \str_convert_output_name_aux:N }
+\cs_new_nopar:Npn \str_convert_output_name_aux:N #1
{
- \str_aux_octal_test:NTF #1
- { \str_bytes_unescape_string_aux_dd:N }
- {
- \str_aux_convert_store:
- \str_bytes_unescape_string_aux:wN #1
- }
+ \str_convert_output_name_aux:NTF #1 {#1}
+ { \c_hash_str \str_output_hexadecimal:n {`#1} }
}
-\cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux_dd:N #1
+\prg_new_conditional:Npnn \str_convert_output_name_aux:N #1 { TF }
{
- \str_aux_octal_test:NTF #1
+ \if_num:w `#1 < "2A \exp_stop_f:
+ \str_if_contains_char:NNTF \c_str_convert_output_name_not_str #1
+ \prg_return_true: \prg_return_false:
+ \else:
+ \if_num:w `#1 > "7E \exp_stop_f:
+ \prg_return_false:
+ \else:
+ \str_if_contains_char:NNTF \c_str_convert_output_name_str #1
+ \prg_return_false: \prg_return_true:
+ \fi:
+ \fi:
+ }
+%</name>
+% \end{macrocode}
+% \end{variable}
+% \end{variable}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_output_string:}
+% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:N}
+% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:NTF}
+% \begin{variable}{\c_str_convert_output_string_str}
+% Any character below (and including) space, and any character
+% above (and including) \texttt{del}, are converted to octal.
+% One backslash is added before each parenthesis and backslash.
+% \begin{macrocode}
+%<*string>
+\str_const:Nx \c_str_convert_output_string_str
+ { \c_backslash_str ( ) }
+\cs_new_protected_nopar:Npn \str_convert_output_string:
+ { \str_aux_gmap_result:N \str_convert_output_string_aux:N }
+\cs_new_nopar:Npn \str_convert_output_string_aux:N #1
+ {
+ \str_convert_output_string_aux:NTF #1
{
- \str_aux_convert_store:
- \str_bytes_unescape_string_aux:wN
+ \str_if_contains_char:NNT
+ \c_str_convert_output_string_str #1
+ { \c_backslash_str }
+ #1
}
{
- \str_aux_convert_store:
- \str_bytes_unescape_string_aux:wN #1
+ \c_backslash_str
+ \int_div_truncate:nn {`#1} {64}
+ \int_mod:nn { \int_div_truncate:nn {`#1} \c_eight } \c_eight
+ \int_mod:nn {`#1} \c_eight
}
}
+\prg_new_conditional:Npnn \str_convert_output_string_aux:N #1 { TF }
+ {
+ \if_num:w `#1 < "21 \exp_stop_f:
+ \prg_return_false:
+ \else:
+ \if_num:w `#1 > "7E \exp_stop_f:
+ \prg_return_false:
+ \else:
+ \prg_return_true:
+ \fi:
+ \fi:
+ }
+%</string>
% \end{macrocode}
+% \end{variable}
+% \end{macro}
+% \end{macro}
% \end{macro}
%
-% \subsubsection{Percent encoding}
+% \begin{macro}[int]{\str_convert_output_url:}
+% \begin{macro}[int]{\str_convert_output_percent:}
+% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N}
+% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF}
+%^^A todo: Figure out what bytes should be percent encoded! Safe could be
+%^^A !'()*-./0123456789_
+% This function is similar to \cs{str_convert_output_name:},
+% escaping different characters.
+% \begin{macrocode}
+%<*url>
+\cs_new_protected_nopar:Npn \str_convert_output_url:
+ { \str_aux_gmap_result:N \str_convert_output_url_aux:N }
+\cs_new_nopar:Npn \str_convert_output_url_aux:N #1
+ {
+ \str_convert_output_url_aux:NTF #1 {#1}
+ { \c_percent_str \str_output_hexadecimal:n { `#1 } }
+ }
+\prg_new_conditional:Npnn \str_convert_output_url_aux:N #1 { TF }
+ {
+ \if_num:w `#1 < "41 \exp_stop_f:
+ \str_if_contains_char:nNTF { "-.<> } #1
+ \prg_return_true: \prg_return_false:
+ \else:
+ \if_num:w `#1 > "7E \exp_stop_f:
+ \prg_return_false:
+ \else:
+ \str_if_contains_char:nNTF { [ ] } #1
+ \prg_return_false: \prg_return_true:
+ \fi:
+ \fi:
+ }
+%</url>
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Encoding definition files}
+%
+% The \texttt{native} and \texttt{internal} encodings are automatically
+% defined. Other encodings are loaded as needed. The following encodings
+% are supported:
+% \begin{itemize}
+% \item \textsc{utf-8};
+% \item \textsc{utf-16}, big-endian, little-endian, or with byte order mark;
+% \item \textsc{utf-32}, big-endian, little-endian, or with byte order mark;
+% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$,
+% skipping the inexistent \textsc{iso 8859-12}.
+% \end{itemize}
%
-% \begin{macro}{\str_bytes_percent_encode:NN}
+% \subsubsection{\textsc{utf-8} support}
+%
+% \begin{macrocode}
+%<*utf8>
+% \end{macrocode}
+%
+% \begin{macro}[int]{\str_convert_to_utf8:}
+% \begin{macro}[aux, rEXP]
+% {
+% \str_aux_to_utf_viii:n,
+% \str_aux_to_utf_viii:nnnnw
+% }
+% Loop through the internal string, and convert each character
+% to the \textsc{utf-8} representation. The details are messy.
+%^^A todo: document.
% \begin{macrocode}
-\tl_const:Nx \c_str_bytes_percent_encode_str { \tl_to_str:n { [] } }
-\tl_const:Nx \c_str_bytes_percent_encode_not_str { \tl_to_str:n { "-.<> } }
-\cs_new_protected_nopar:Npn \str_bytes_percent_encode:NN #1#2
+\cs_new_protected_nopar:cpn { str_convert_to_utf8: }
+ { \str_aux_gmap_internal_result:N \str_aux_to_utf_viii:n }
+\cs_new:Npn \str_aux_to_utf_viii:n #1
{
- \str_set:Nx #1
- { \str_map_tokens:Nn #2 \str_bytes_percent_encode_aux:N }
+ \str_aux_to_utf_viii:nnnnw
+ {#1} { \c_minus_one + \c_zero * \use_none:n }
+ { 128 } { \c_zero }
+ { 32 } { 192 }
+ { 16 } { 224 }
+ { 8 } { 240 }
+ \q_stop
}
-\cs_new_nopar:Npn \str_bytes_percent_encode_aux:N #1
+\cs_new_nopar:Npn \str_aux_to_utf_viii:nnnnw #1#2#3#4 #5 \q_stop
{
- \int_compare:nNnTF {`#1} < { "41 }
- {
- \str_if_contains_char:NNTF \c_str_bytes_percent_encode_not_str #1
- { #1 }
- { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 }
- }
+ \if_num:w #1 < #3 \exp_stop_f:
+ \str_output_byte:n { #1 + #4 }
+ \else:
+ \exp_args:Nf \str_aux_to_utf_viii:nnnnw
+ { \int_div_truncate:nn {#1} {64} }
+ {#1}
+ #5 \q_stop
+ \fi:
+ \str_output_byte:n { #2 - 64 * ( #1 - \c_two ) }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]{\str_convert_from_utf8:}
+% \begin{macro}[aux, rEXP]
+% {
+% \str_aux_from_utf_viii:N,
+% \str_aux_from_utf_viii:wwNN,
+% \str_aux_from_utf_viii:wNNww
+% }
+% \begin{macro}[aux, rEXP]
+% {
+% \str_aux_from_utf_viii_error:w,
+% \str_aux_from_utf_viii_error:NwNN,
+% \str_aux_from_utf_viii_error:
+% }
+% In the \textsc{utf-8} encoding, bytes in the range $[0,127]$ stand
+% for themselves, bytes in the range $[128,191]$ are continuation bytes,
+% and larger bytes must be followed by a number of continuation bytes.
+% Documentation to come.
+% ^^A todo: refuse "D800 -- "DFFF?
+% ^^A todo: document
+% ^^A todo: what error recovery for code points > "10FFFF?
+% \begin{macrocode}
+\cs_new_protected_nopar:cpn { str_convert_from_utf8: }
+ {
+ \tl_gset:Nx \g_str_result_tl
{
- \int_compare:nNnTF {`#1} > { "7E }
- { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 }
- {
- \str_if_contains_char:NNTF \c_str_bytes_percent_encode_str #1
- { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 }
- { #1 }
- }
+ \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl
+ { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: }
+ \prg_break_point:n { }
}
}
+\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1
+ {
+ \use_none:n #1
+ \int_use:N \int_eval:w
+ \if_num:w `#1 < "C0 \exp_stop_f:
+ \if_num:w `#1 < "80 \exp_stop_f:
+ `#1
+ \else:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-extra-conti } {#1}
+ \c_str_replacement_char_int
+ \fi:
+ \else:
+ \if_num:w `#1 < "F5 \exp_stop_f:
+ \exp_after:wN \str_aux_from_utf_viii:wwNN
+ \int_use:N \int_eval:w `#1 - "C0
+ \else:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-invalid-byte } {#1}
+ \c_str_replacement_char_int
+ \fi:
+ \fi:
+ ,
+ \use_none_delimit_by_q_stop:w
+ "80 , "800 , "10000 , "10FFFF ,
+ \q_stop
+ \str_aux_from_utf_viii:N
+ }
+\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4
+ {
+ \use_none_delimit_by_q_stop:w #4 \q_stop
+ \exp_after:wN \str_aux_from_utf_viii:wNNww
+ \int_use:N \int_eval:w #1 * "40 + `#4 - "80 ,
+ #4
+ #2
+ \q_stop #3
+ }
+\cs_new_nopar:Npn \str_aux_from_utf_viii:wNNww #1, #2 #3#4, #5,
+ {
+ \if_num:w \int_eval:w ( `#2 + \c_thirty_two ) / "40 = \c_three
+ \if_num:w #1 < #4 \exp_stop_f:
+ \str_aux_from_utf_viii_error:w #1 ,
+ \else:
+ \if_num:w #1 < #5 \exp_stop_f:
+ #1
+ \else:
+ \exp_after:wN \str_aux_from_utf_viii:wwNN
+ \int_use:N \int_eval:w #1 - #5
+ \fi:
+ \fi:
+ \else:
+ \exp_after:wN \str_aux_from_utf_viii_error:NwN
+ \exp_after:wN #2
+ \fi:
+ , #3#5,
+ }
+\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 ,
+ {
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-overlong } {#1}
+ \c_str_replacement_char_int
+ }
+\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3
+ {
+ \msg_expandable_kernel_error:nnn
+ { str } { utf8-missing-conti } {#1}
+ \c_str_replacement_char_int ,
+ #3#1
+ }
+\cs_new_nopar:Npn \str_aux_from_utf_viii_error:
+ {
+ \msg_expandable_kernel_error:nn
+ { str } { utf8-premature-end }
+ \c_str_replacement_char_int ,
+ \prg_map_break:
+ }
% \end{macrocode}
% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macrocode}
+%</utf8>
+% \end{macrocode}
%
-% \begin{macro}{\str_bytes_percent_decode:NN}
-% \begin{macro}[aux]{\str_bytes_percent_decode_aux:wN}
-% \begin{macro}[aux]{\str_bytes_percent_decode_aux_ii:NN}
+% \subsubsection{\textsc{utf-16} support}
+%
+% The definitions are done in a category code regime where
+% the bytes $254$ and $255$ used by the byte order mark
+% have catcode \enquote{other}.
% \begin{macrocode}
+%<*utf16>
\group_begin:
- \char_set_lccode:nn {`\*} {`\%}
- \tl_to_lowercase:n
+ \char_set_catcode_other:N \^^fe
+ \char_set_catcode_other:N \^^ff
+% \end{macrocode}
+%
+% \begin{macro}[int]{\str_convert_to_utf16:}
+% \begin{macro}[int]{\str_convert_to_utf16be:}
+% \begin{macro}[int]{\str_convert_to_utf16le:}
+% \begin{macro}[aux, rEXP]
+% {
+% \str_aux_to_utf_xvi:n,
+% \str_aux_to_utf_xvi_be:n,
+% \str_aux_to_utf_xvi_le:n
+% }
+% Convert characters one by one in a loop, with different
+% behaviours depending on the character code:
+% \begin{itemize}
+% \item $[0, 55295]$ converted to two bytes;
+% \item $[55296, 57343]$ cannot be converted, replaced by
+% the replacement character;
+% \item $[57344, 65535]$ converted to two bytes;
+% \item $[65536, 1114111]$ converted to a pair of surrogates,
+% each two bytes.
+% \end{itemize}
+% For the duration of this operation, \cs{str_tmp:w} is defined
+% as a function to convert a number in the range $[0, 65536]$
+% to a pair of bytes (either big endian or little endian).
+% \begin{macrocode}
+ \cs_new_protected_nopar:cpx { str_convert_to_utf16: }
{
- \group_end:
- \cs_new_protected_nopar:Npn \str_bytes_percent_decode:NN #1#2
+ \exp_not:c { str_convert_to_utf16be: }
+ \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } }
+ }
+ \cs_new_protected_nopar:cpn { str_convert_to_utf16be: }
+ {
+ \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_be:n
+ \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n
+ }
+ \cs_new_protected_nopar:cpn { str_convert_to_utf16le: }
+ {
+ \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_le:n
+ \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n
+ }
+ \cs_new_nopar:Npn \str_aux_to_utf_xvi:n #1
+ {
+ \if_int_compare:w #1 < "D800 \exp_stop_f:
+ \str_tmp:w {#1}
+ \else:
+ \if_int_compare:w #1 < "10000 \exp_stop_f:
+ \if_int_compare:w #1 < "E000 \exp_stop_f:
+ \msg_expandable_kernel_error:nnn
+ { str } { unicode-surrogate } {#1}
+ \str_tmp:w { \c_str_replacement_char_int }
+ \else:
+ \str_tmp:w {#1}
+ \fi:
+ \else:
+ \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D800 }
+ \exp_args:Nf \str_tmp:w { \int_mod:nn {#1} {"400} + "DC00 }
+ \fi:
+ \fi:
+ }
+ \cs_new_nopar:Npn \str_aux_to_utf_xvi_be:n #1
+ {
+ \str_output_byte:n { \int_div_truncate:nn {#1} {"100} }
+ \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ }
+ \cs_new_nopar:Npn \str_aux_to_utf_xvi_le:n #1
+ {
+ \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ \str_output_byte:n { \int_div_truncate:nn {#1} {"100} }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% \begin{macro}[int]
+% {
+% \str_convert_from_utf16:,
+% \str_convert_from_utf16be:,
+% \str_convert_from_utf16le:
+% }
+% \begin{macro}[aux]{\str_aux_from_utf_xvi_bom:NNw}
+% \begin{macro}[aux]{\str_aux_from_utf_xvi:No}
+% Define \cs{str_tmp:w} to take two arguments and return the
+% character code of the first one if the string is big-endian,
+% and the second one if the string is little-endian.
+% If the endianness is not known, look for a byte order mark
+% to decide.
+% \begin{macrocode}
+ \cs_new_protected_nopar:cpn { str_convert_from_utf16be: }
+ { \str_aux_from_utf_xvi:No 1 { \g_str_result_tl } }
+ \cs_new_protected_nopar:cpn { str_convert_from_utf16le: }
+ { \str_aux_from_utf_xvi:No 2 { \g_str_result_tl } }
+ \cs_new_protected_nopar:cpn { str_convert_from_utf16: }
+ {
+ \exp_after:wN \str_aux_from_utf_xvi_bom:NNw
+ \g_str_result_tl \scan_stop: \scan_stop: \scan_stop:
+ }
+ \cs_new_protected_nopar:Npn
+ \str_aux_from_utf_xvi_bom:NNw #1#2#3\scan_stop:
+ {
+ \str_if_eq:nnTF { #1#2 } { ^^ff ^^fe }
+ { \str_aux_from_utf_xvi:No 2 {#3} }
{
- \group_begin:
- \tl_gclear:N \g_str_result_tl
- \exp_last_unbraced:Nf \str_bytes_percent_decode_aux:wN
- { \tl_to_other_str:N #2 }
- * \q_recursion_tail \q_recursion_stop
- \group_end:
- \tl_set_eq:NN #1 \g_str_result_tl
+ \str_if_eq:nnTF { #1#2 } { ^^fe ^^ff }
+ { \str_aux_from_utf_xvi:No 1 {#3} }
+ { \str_aux_from_utf_xvi:No 1 {#1#2#3} }
}
- \cs_new_protected_nopar:Npn \str_bytes_percent_decode_aux:wN #1 * #2
+ }
+ \cs_new_protected_nopar:Npn \str_aux_from_utf_xvi:No #1#2
+ {
+ \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
+ \tl_gset:Nx \g_str_result_tl
{
- \tl_gput_right:Nx \g_str_result_tl {#1}
- \quark_if_recursion_tail_stop:N #2
- \int_zero:N \l_str_char_int
- \str_aux_hexadecimal_test:NTF #2
- { \str_bytes_percent_decode_aux_ii:NN #2 }
- {
- \tl_gput_right:Nx \g_str_result_tl { \c_percent_str }
- \str_bytes_percent_decode_aux:wN #2
- }
+ \exp_after:wN \str_aux_from_utf_xvi:NN
+ #2 \scan_stop: \scan_stop:
+ \prg_break_point:n { }
}
}
-\cs_new_protected_nopar:Npn \str_bytes_percent_decode_aux_ii:NN #1#2
- {
- \str_aux_hexadecimal_test:NTF #2
- {
- \str_aux_convert_store:
- \str_bytes_percent_decode_aux:wN
- }
- {
- \tl_gput_right:Nx \g_str_result_tl {#1}
- \str_bytes_percent_decode_aux:wN #2
- }
- }
% \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
-% \subsubsection{UTF-8 support}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:wNNN}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:wNNw}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw}
+% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:wNNww}
+% When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w}
+% is such that |\str_tmp:w #1#2| expands to |`#1| if the string
+% is big-endian, and |`#2| if the string is little-endian.
+% If the most significant byte is between |"D8| and |"DB|
+% inclusive, then the pair of bytes is a high surrogate,
+% and we fetch the following pair, which must be a low surrogate,
+% \emph{i.e.}, between |"DC| and |"DF| inclusive.
+% Unpaired surrogates are errors, handled with
+% \cs{str_aux_from_utf_xvi_error:wNNw}.
+% \begin{macrocode}
+ \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2
+ {
+ \if_meaning:w \scan_stop: #2
+ \str_aux_from_utf_xvi_end:Nw #1
+ \fi:
+ \int_use:N \int_eval:w
+ \if_case:w
+ \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end:
+ \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN
+ \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw
+ \fi:
+ "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 ,
+ \str_aux_from_utf_xvi:NN
+ }
+ \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4
+ {
+ \if_meaning:w \scan_stop: #4
+ \str_aux_from_utf_xvi_end:wNNww #1 , #3
+ \fi:
+ \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one
+ ( #1 - "D800 ) * "400
+ + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 ,
+ \exp_after:wN \use_i:nnn
+ \else:
+ \str_aux_from_utf_xvi_error:wNNw #1 ,
+ \fi:
+ #2#3#4
+ }
+ \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4,
+ {
+ \msg_expandable_kernel_error:nnn
+ { str } { utf16-surrogate } { #2#3 }
+ \c_str_replacement_char_int ,
+ }
+ \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi:
+ {
+ \fi:
+ \if_meaning:w \scan_stop: #1
+ \else:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf16-odd } {#1}
+ \c_str_replacement_char_int ,
+ \fi:
+ \prg_map_break:
+ }
+ \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi:
+ {
+ \fi:
+ \if_meaning:w \scan_stop: #5
+ \msg_expandable_kernel_error:nnn
+ { str } { utf16-surrogate } { #2#3 }
+ \else:
+ \msg_expandable_kernel_error:nnn
+ { str } { utf16-odd } { #2#3#5 }
+ \fi:
+ \c_str_replacement_char_int ,
+ \prg_map_break:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% Restore the original catcodes of bytes $254$ and $255$.
+% \begin{macrocode}
+\group_end:
+%</utf16>
+% \end{macrocode}
+%
+% \subsubsection{\textsc{utf-32} support}
+%
+% The definitions are done in a category code regime where
+% the bytes $0$, $254$ and $255$ used by the byte order mark
+% have catcode \enquote{other}.
+% \begin{macrocode}
+%<*utf32>
+\group_begin:
+ \char_set_catcode_other:N \^^00
+ \char_set_catcode_other:N \^^fe
+ \char_set_catcode_other:N \^^ff
+% \end{macrocode}
+%
+% \begin{macro}[int]
+% {
+% \str_convert_to_utf32:,
+% \str_convert_to_utf32be:,
+% \str_convert_to_utf32le:
+% }
+% \begin{macro}[aux, rEXP]
+% {
+% \str_aux_to_utf_xxxii_be:n,
+% \str_aux_to_utf_xxxii_le:n
+% }
+% Convert each integer in the comma-list \cs{g_str_result_tl}
+% to a sequence of four bytes. The functions for big-endian
+% and little-endian encodings are very similar, but the
+% \cs{str_output_byte:n} instructions are reversed.
+% \begin{macrocode}
+ \cs_new_protected_nopar:cpx { str_convert_to_utf32: }
+ {
+ \exp_not:c { str_convert_to_utf32be: }
+ \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } }
+ }
+ \cs_new_protected_nopar:cpn { str_convert_to_utf32be: }
+ { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_be:n }
+ \cs_new_protected_nopar:cpn { str_convert_to_utf32le: }
+ { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_le:n }
+ \cs_new:Npn \str_aux_to_utf_xxxii_be:n #1
+ {
+ ^^00
+ \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } }
+ \str_output_byte:n
+ { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} }
+ \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ }
+ \cs_new:Npn \str_aux_to_utf_xxxii_le:n #1
+ {
+ \str_output_byte:n { \int_mod:nn {#1} {"100} }
+ \str_output_byte:n
+ { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} }
+ \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } }
+ ^^00
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
%
-% \begin{macro}{\str_native_from_UTF_viii:NN}
+% \begin{macro}[int]
+% {
+% \str_convert_from_utf32:,
+% \str_convert_from_utf32be:,
+% \str_convert_from_utf32le:
+% }
% \begin{macro}[aux]
% {
-% \str_native_from_UTF_viii_aux_i:N,
-% \str_native_from_UTF_viii_aux_ii:nN,
-% \str_native_from_UTF_viii_aux_iii:N,
-% \str_native_from_UTF_viii_aux_iv:,
-% \str_native_from_UTF_viii_aux_v:
+% \str_aux_from_utf_xxxii_bom:NNNNw,
+% \str_aux_from_utf_xxxii:No
+% }
+% \begin{macro}[aux, rEXP]
+% {
+% \str_aux_from_utf_xxxii:NNNN,
+% \str_aux_from_utf_xxxii_end:ww
% }
+% See the conversion functions from \textsc{utf-16} encodings.
+% The \cs{str_aux_from_utf_xxxii:No} auxiliary defines
+% \cs{str_tmp:w} to take two arguments and give the character
+% code of either the first argument (big-endian) or the second
+% argument (little-endian). Then we loop over the string,
+% four bytes at a time, check for overflow, and otherwise
+% produce the number corresponding to the four bytes with
+% the given endianness.
+% \begin{macrocode}
+ \cs_new_protected_nopar:cpn { str_convert_from_utf32be: }
+ { \str_aux_from_utf_xxxii:No 1 { \g_str_result_tl } }
+ \cs_new_protected_nopar:cpn { str_convert_from_utf32le: }
+ { \str_aux_from_utf_xxxii:No 2 { \g_str_result_tl } }
+ \cs_new_protected_nopar:cpn { str_convert_from_utf32: }
+ {
+ \exp_after:wN \str_aux_from_utf_xxxii_bom:NNNNw \g_str_result_tl
+ \scan_stop: \scan_stop: \scan_stop: \scan_stop: \scan_stop:
+ }
+ \cs_new_protected_nopar:Npn
+ \str_aux_from_utf_xxxii_bom:NNNNw #1#2#3#4#5\scan_stop:
+ {
+ \str_if_eq:nnTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 }
+ { \str_aux_from_utf_xxxii:No 2 {#5} }
+ {
+ \str_if_eq:nnTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff }
+ { \str_aux_from_utf_xxxii:No 1 {#5} }
+ { \str_aux_from_utf_xxxii:No 1 {#1#2#3#4#5} }
+ }
+ }
+ \cs_new_protected_nopar:Npn \str_aux_from_utf_xxxii:No #1#2
+ {
+ \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 }
+ \tl_gset:Nx \g_str_result_tl
+ {
+ \exp_after:wN \str_aux_from_utf_xxxii:NNNN
+ #2 \scan_stop: \scan_stop: \scan_stop: \scan_stop:
+ \prg_break_point:n { }
+ }
+ }
+ \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4
+ {
+ \if_meaning:w \scan_stop: #4
+ \str_aux_from_utf_xxxii_end:ww #1#2#3#4
+ \fi:
+ \int_use:N \int_eval:w
+ \if_num:w \int_eval:w
+ \str_tmp:w #1#4 * \c_two_hundred_fifty_six
+ + \str_tmp:w #2#3
+ > \c_sixteen
+ \msg_expandable_kernel_error:nnn
+ { str } { utf32-overflow } { #1#2#3#4 }
+ \c_str_replacement_char_int
+ \else:
+ \str_tmp:w #2#3*"10000
+ + \str_tmp:w #3#2 * \c_two_hundred_fifty_six
+ + \str_tmp:w #4#1
+ \fi:
+ ,
+ \str_aux_from_utf_xxxii:NNNN
+ }
+ \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi:
+ {
+ \fi:
+ \tl_if_empty:nF {#1}
+ {
+ \msg_expandable_kernel_error:nnn
+ { str } { utf32-truncated } { #1 }
+ }
+ \prg_map_break:
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+% \end{macro}
+%
+% Restore the original catcodes of bytes $0$, $254$ and $255$.
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_native_from_UTF_viii:NN #1#2
+\group_end:
+%</utf32>
+% \end{macrocode}
+%
+% \subsubsection{\textsc{iso 8859} support}
+%^^A todo: document
+%
+% The \textsc{iso-8859-1} encoding exactly matches with the $256$
+% first Unicode characters. For other 8-bit encodings
+% of the \textsc{iso-8859} family, we keep track only of
+% differences, and of unassigned bytes.
+% \begin{macrocode}
+%<*iso88591>
+\str_encoding_eight_bit:n { iso88591 }
+\tl_const:cn { c_str_encoding_iso88591_tl }
{
- \group_begin:
- \tl_gclear:N \g_str_result_tl
- \exp_last_unbraced:Nf \str_native_from_UTF_viii_aux_i:N
- { \tl_to_other_str:N #2 }
- \q_recursion_tail \q_recursion_stop
- \group_end:
- \tl_set_eq:NN #1 \g_str_result_tl
}
-\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_i:N #1
- {
- \quark_if_recursion_tail_stop:N #1
- \int_set:Nn \l_str_char_int {`#1}
- \str_native_from_UTF_viii_aux_ii:nN { 128 } \c_one
- \str_native_from_UTF_viii_aux_ii:nN { 64 } \c_zero
- \str_native_from_UTF_viii_aux_ii:nN { 32 } \c_two
- \str_native_from_UTF_viii_aux_ii:nN { 16 } \c_three
- \str_native_from_UTF_viii_aux_ii:nN { 8 } \c_four
- \msg_error:nnx { str } { utf8-invalid-byte } {#1}
- \use_i:nnn \str_native_from_UTF_viii_aux_i:N
- \q_stop
- \str_native_from_UTF_viii_aux_iii:N
+\tl_const:cn { c_str_encoding_iso88591_missing_tl }
+ {
+ }
+%</iso88591>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso88592>
+\str_encoding_eight_bit:n { iso88592 }
+\tl_const:cn { c_str_encoding_iso88592_tl }
+ {
+ { A1 } { 0104 }
+ { A2 } { 02D8 }
+ { A3 } { 0141 }
+ { A5 } { 013D }
+ { A6 } { 015A }
+ { A9 } { 0160 }
+ { AA } { 015E }
+ { AB } { 0164 }
+ { AC } { 0179 }
+ { AE } { 017D }
+ { AF } { 017B }
+ { B1 } { 0105 }
+ { B2 } { 02DB }
+ { B3 } { 0142 }
+ { B5 } { 013E }
+ { B6 } { 015B }
+ { B7 } { 02C7 }
+ { B9 } { 0161 }
+ { BA } { 015F }
+ { BB } { 0165 }
+ { BC } { 017A }
+ { BD } { 02DD }
+ { BE } { 017E }
+ { BF } { 017C }
+ { C0 } { 0154 }
+ { C3 } { 0102 }
+ { C5 } { 0139 }
+ { C6 } { 0106 }
+ { C8 } { 010C }
+ { CA } { 0118 }
+ { CC } { 011A }
+ { CF } { 010E }
+ { D0 } { 0110 }
+ { D1 } { 0143 }
+ { D2 } { 0147 }
+ { D5 } { 0150 }
+ { D8 } { 0158 }
+ { D9 } { 016E }
+ { DB } { 0170 }
+ { DE } { 0162 }
+ { E0 } { 0155 }
+ { E3 } { 0103 }
+ { E5 } { 013A }
+ { E6 } { 0107 }
+ { E8 } { 010D }
+ { EA } { 0119 }
+ { EC } { 011B }
+ { EF } { 010F }
+ { F0 } { 0111 }
+ { F1 } { 0144 }
+ { F2 } { 0148 }
+ { F5 } { 0151 }
+ { F8 } { 0159 }
+ { F9 } { 016F }
+ { FB } { 0171 }
+ { FE } { 0163 }
+ { FF } { 02D9 }
}
-\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_ii:nN #1#2
+\tl_const:cn { c_str_encoding_iso88592_missing_tl }
{
- \int_compare:nNnTF \l_str_char_int < {#1}
- {
- \int_set_eq:NN \l_str_bytes_int #2
- \use_none_delimit_by_q_stop:w
- }
- { \int_sub:Nn \l_str_char_int {#1} }
}
-\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_iii:N #1
+%</iso88592>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso88593>
+\str_encoding_eight_bit:n { iso88593 }
+\tl_const:cn { c_str_encoding_iso88593_tl }
{
- \int_compare:nNnTF \l_str_bytes_int < \c_two
- { \str_native_from_UTF_viii_aux_iv: #1 }
- {
- \if_meaning:w \q_recursion_tail #1
- \msg_error:nn { str } { utf8-premature-end }
- \exp_after:wN \use_none_delimit_by_q_recursion_stop:w
- \fi:
- \quark_if_recursion_tail_stop:N #1
- \tex_multiply:D \l_str_char_int by 64 \scan_stop:
- \int_decr:N \l_str_bytes_int
- \int_compare:nNnTF {`#1} < { 128 }
- { \use_ii:nn }
- { \int_compare:nNnTF {`#1} < { 192 } }
- {
- \int_add:Nn \l_str_char_int { `#1 - 128 }
- \str_native_from_UTF_viii_aux_iii:N
- }
- {
- \msg_error:nnx { str } { utf8-missing-byte } {#1}
- \str_native_from_UTF_viii_aux_i:N #1
- }
- }
+ { A1 } { 0126 }
+ { A2 } { 02D8 }
+ { A6 } { 0124 }
+ { A9 } { 0130 }
+ { AA } { 015E }
+ { AB } { 011E }
+ { AC } { 0134 }
+ { AF } { 017B }
+ { B1 } { 0127 }
+ { B6 } { 0125 }
+ { B9 } { 0131 }
+ { BA } { 015F }
+ { BB } { 011F }
+ { BC } { 0135 }
+ { BF } { 017C }
+ { C5 } { 010A }
+ { C6 } { 0108 }
+ { D5 } { 0120 }
+ { D8 } { 011C }
+ { DD } { 016C }
+ { DE } { 015C }
+ { E5 } { 010B }
+ { E6 } { 0109 }
+ { F5 } { 0121 }
+ { F8 } { 011D }
+ { FD } { 016D }
+ { FE } { 015D }
+ { FF } { 02D9 }
}
-\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_iv:
+\tl_const:cn { c_str_encoding_iso88593_missing_tl }
{
- \int_compare:nNnTF \l_str_bytes_int = \c_zero
- {
- \msg_error:nnx { str } { utf8-extra-byte }
- { \int_eval:n { \l_str_char_int + 128 } }
- }
- { \str_native_from_UTF_viii_aux_v: }
- \str_native_from_UTF_viii_aux_i:N
+ { A5 }
+ { AE }
+ { BE }
+ { C3 }
+ { D0 }
+ { E3 }
+ { F0 }
}
-\pdftex_if_engine:TF
+%</iso88593>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso88594>
+\str_encoding_eight_bit:n { iso88594 }
+\tl_const:cn { c_str_encoding_iso88594_tl }
{
- \cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_v:
- {
- \int_compare:nNnTF { \l_str_char_int } < { 256 }
- { \str_aux_convert_store: }
- {
- \msg_error:nnx { str } { utf8-pdftex-overflow }
- { \int_use:N \l_str_char_int }
- }
- }
+ { A1 } { 0104 }
+ { A2 } { 0138 }
+ { A3 } { 0156 }
+ { A5 } { 0128 }
+ { A6 } { 013B }
+ { A9 } { 0160 }
+ { AA } { 0112 }
+ { AB } { 0122 }
+ { AC } { 0166 }
+ { AE } { 017D }
+ { B1 } { 0105 }
+ { B2 } { 02DB }
+ { B3 } { 0157 }
+ { B5 } { 0129 }
+ { B6 } { 013C }
+ { B7 } { 02C7 }
+ { B9 } { 0161 }
+ { BA } { 0113 }
+ { BB } { 0123 }
+ { BC } { 0167 }
+ { BD } { 014A }
+ { BE } { 017E }
+ { BF } { 014B }
+ { C0 } { 0100 }
+ { C7 } { 012E }
+ { C8 } { 010C }
+ { CA } { 0118 }
+ { CC } { 0116 }
+ { CF } { 012A }
+ { D0 } { 0110 }
+ { D1 } { 0145 }
+ { D2 } { 014C }
+ { D3 } { 0136 }
+ { D9 } { 0172 }
+ { DD } { 0168 }
+ { DE } { 016A }
+ { E0 } { 0101 }
+ { E7 } { 012F }
+ { E8 } { 010D }
+ { EA } { 0119 }
+ { EC } { 0117 }
+ { EF } { 012B }
+ { F0 } { 0111 }
+ { F1 } { 0146 }
+ { F2 } { 014D }
+ { F3 } { 0137 }
+ { F9 } { 0173 }
+ { FD } { 0169 }
+ { FE } { 016B }
+ { FF } { 02D9 }
}
+\tl_const:cn { c_str_encoding_iso88594_missing_tl }
{
- \cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_v:
- { \str_aux_convert_store: }
}
+%</iso88594>
% \end{macrocode}
-% \end{macro}
-% \end{macro}
%
-% \begin{macro}{\str_UTF_viii_from_native:NN}
% \begin{macrocode}
-\cs_new_protected_nopar:Npn \str_UTF_viii_from_native:NN #1#2
+%<*iso88595>
+\str_encoding_eight_bit:n { iso88595 }
+\tl_const:cn { c_str_encoding_iso88595_tl }
{
- \group_begin:
- \tl_gclear:N \g_str_result_tl
- \str_map_tokens:Nn #2 \str_UTF_viii_from_native_aux_i:N
- \group_end:
- \tl_set_eq:NN #1 \g_str_result_tl
+ { A1 } { 0401 }
+ { A2 } { 0402 }
+ { A3 } { 0403 }
+ { A4 } { 0404 }
+ { A5 } { 0405 }
+ { A6 } { 0406 }
+ { A7 } { 0407 }
+ { A8 } { 0408 }
+ { A9 } { 0409 }
+ { AA } { 040A }
+ { AB } { 040B }
+ { AC } { 040C }
+ { AE } { 040E }
+ { AF } { 040F }
+ { B0 } { 0410 }
+ { B1 } { 0411 }
+ { B2 } { 0412 }
+ { B3 } { 0413 }
+ { B4 } { 0414 }
+ { B5 } { 0415 }
+ { B6 } { 0416 }
+ { B7 } { 0417 }
+ { B8 } { 0418 }
+ { B9 } { 0419 }
+ { BA } { 041A }
+ { BB } { 041B }
+ { BC } { 041C }
+ { BD } { 041D }
+ { BE } { 041E }
+ { BF } { 041F }
+ { C0 } { 0420 }
+ { C1 } { 0421 }
+ { C2 } { 0422 }
+ { C3 } { 0423 }
+ { C4 } { 0424 }
+ { C5 } { 0425 }
+ { C6 } { 0426 }
+ { C7 } { 0427 }
+ { C8 } { 0428 }
+ { C9 } { 0429 }
+ { CA } { 042A }
+ { CB } { 042B }
+ { CC } { 042C }
+ { CD } { 042D }
+ { CE } { 042E }
+ { CF } { 042F }
+ { D0 } { 0430 }
+ { D1 } { 0431 }
+ { D2 } { 0432 }
+ { D3 } { 0433 }
+ { D4 } { 0434 }
+ { D5 } { 0435 }
+ { D6 } { 0436 }
+ { D7 } { 0437 }
+ { D8 } { 0438 }
+ { D9 } { 0439 }
+ { DA } { 043A }
+ { DB } { 043B }
+ { DC } { 043C }
+ { DD } { 043D }
+ { DE } { 043E }
+ { DF } { 043F }
+ { E0 } { 0440 }
+ { E1 } { 0441 }
+ { E2 } { 0442 }
+ { E3 } { 0443 }
+ { E4 } { 0444 }
+ { E5 } { 0445 }
+ { E6 } { 0446 }
+ { E7 } { 0447 }
+ { E8 } { 0448 }
+ { E9 } { 0449 }
+ { EA } { 044A }
+ { EB } { 044B }
+ { EC } { 044C }
+ { ED } { 044D }
+ { EE } { 044E }
+ { EF } { 044F }
+ { F0 } { 2116 }
+ { F1 } { 0451 }
+ { F2 } { 0452 }
+ { F3 } { 0453 }
+ { F4 } { 0454 }
+ { F5 } { 0455 }
+ { F6 } { 0456 }
+ { F7 } { 0457 }
+ { F8 } { 0458 }
+ { F9 } { 0459 }
+ { FA } { 045A }
+ { FB } { 045B }
+ { FC } { 045C }
+ { FD } { 00A7 }
+ { FE } { 045E }
+ { FF } { 045F }
}
-\cs_new_protected_nopar:Npn \str_UTF_viii_from_native_aux_i:N #1
+\tl_const:cn { c_str_encoding_iso88595_missing_tl }
{
- \int_set:Nn \l_str_char_int {`#1}
- \int_compare:nNnTF \l_str_char_int < { 128 }
- { \tl_gput_right:Nx \g_str_result_tl {#1} }
- {
- \tl_gclear:N \g_str_tmpa_tl
- \int_set:Nn \l_str_bytes_int { 64 }
- \str_UTF_viii_from_native_aux_ii:n { 32 }
- \str_UTF_viii_from_native_aux_ii:n { 16 }
- \str_UTF_viii_from_native_aux_ii:n { 8 }
- \ERROR % somehow the unicode char was > "1FFFFF > "10FFFF
- \tl_gclear:N \g_str_tmpa_tl
- \use_none:n \q_stop
- \int_add:Nn \l_str_char_int { 2 * \l_str_bytes_int }
- \str_aux_convert_store:
- \tl_gput_right:Nx \g_str_result_tl { \g_str_tmpa_tl }
- }
}
-\cs_new_protected_nopar:Npn \str_UTF_viii_from_native_aux_ii:n #1
+%</iso88595>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso88596>
+\str_encoding_eight_bit:n { iso88596 }
+\tl_const:cn { c_str_encoding_iso88596_tl }
{
- \str_aux_convert_store:NNn
- \tl_gput_left:Nx \g_str_tmpa_tl
- { 128 + \int_mod:nn { \l_str_char_int } { 64 } }
- \tex_divide:D \l_str_char_int by 64 \scan_stop:
- \int_add:Nn \l_str_bytes_int {#1}
- \int_compare:nNnT \l_str_char_int < {#1}
- { \use_none_delimit_by_q_stop:w }
+ { AC } { 060C }
+ { BB } { 061B }
+ { BF } { 061F }
+ { C1 } { 0621 }
+ { C2 } { 0622 }
+ { C3 } { 0623 }
+ { C4 } { 0624 }
+ { C5 } { 0625 }
+ { C6 } { 0626 }
+ { C7 } { 0627 }
+ { C8 } { 0628 }
+ { C9 } { 0629 }
+ { CA } { 062A }
+ { CB } { 062B }
+ { CC } { 062C }
+ { CD } { 062D }
+ { CE } { 062E }
+ { CF } { 062F }
+ { D0 } { 0630 }
+ { D1 } { 0631 }
+ { D2 } { 0632 }
+ { D3 } { 0633 }
+ { D4 } { 0634 }
+ { D5 } { 0635 }
+ { D6 } { 0636 }
+ { D7 } { 0637 }
+ { D8 } { 0638 }
+ { D9 } { 0639 }
+ { DA } { 063A }
+ { E0 } { 0640 }
+ { E1 } { 0641 }
+ { E2 } { 0642 }
+ { E3 } { 0643 }
+ { E4 } { 0644 }
+ { E5 } { 0645 }
+ { E6 } { 0646 }
+ { E7 } { 0647 }
+ { E8 } { 0648 }
+ { E9 } { 0649 }
+ { EA } { 064A }
+ { EB } { 064B }
+ { EC } { 064C }
+ { ED } { 064D }
+ { EE } { 064E }
+ { EF } { 064F }
+ { F0 } { 0650 }
+ { F1 } { 0651 }
+ { F2 } { 0652 }
}
+\tl_const:cn { c_str_encoding_iso88596_missing_tl }
+ {
+ { A1 }
+ { A2 }
+ { A3 }
+ { A5 }
+ { A6 }
+ { A7 }
+ { A8 }
+ { A9 }
+ { AA }
+ { AB }
+ { AE }
+ { AF }
+ { B0 }
+ { B1 }
+ { B2 }
+ { B3 }
+ { B4 }
+ { B5 }
+ { B6 }
+ { B7 }
+ { B8 }
+ { B9 }
+ { BA }
+ { BC }
+ { BD }
+ { BE }
+ { C0 }
+ { DB }
+ { DC }
+ { DD }
+ { DE }
+ { DF }
+ }
+%</iso88596>
% \end{macrocode}
-% \end{macro}
-%
-% \subsection{Messages}
-%
+%
% \begin{macrocode}
-\msg_new:nnnn { str } { x-missing-brace }
- { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. }
+%<*iso88597>
+\str_encoding_eight_bit:n { iso88597 }
+\tl_const:cn { c_str_encoding_iso88597_tl }
{
- You~wrote~something~like~
- `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~
- The~closing~brace~is~missing.
+ { A1 } { 2018 }
+ { A2 } { 2019 }
+ { A4 } { 20AC }
+ { A5 } { 20AF }
+ { AA } { 037A }
+ { AF } { 2015 }
+ { B4 } { 0384 }
+ { B5 } { 0385 }
+ { B6 } { 0386 }
+ { B8 } { 0388 }
+ { B9 } { 0389 }
+ { BA } { 038A }
+ { BC } { 038C }
+ { BE } { 038E }
+ { BF } { 038F }
+ { C0 } { 0390 }
+ { C1 } { 0391 }
+ { C2 } { 0392 }
+ { C3 } { 0393 }
+ { C4 } { 0394 }
+ { C5 } { 0395 }
+ { C6 } { 0396 }
+ { C7 } { 0397 }
+ { C8 } { 0398 }
+ { C9 } { 0399 }
+ { CA } { 039A }
+ { CB } { 039B }
+ { CC } { 039C }
+ { CD } { 039D }
+ { CE } { 039E }
+ { CF } { 039F }
+ { D0 } { 03A0 }
+ { D1 } { 03A1 }
+ { D3 } { 03A3 }
+ { D4 } { 03A4 }
+ { D5 } { 03A5 }
+ { D6 } { 03A6 }
+ { D7 } { 03A7 }
+ { D8 } { 03A8 }
+ { D9 } { 03A9 }
+ { DA } { 03AA }
+ { DB } { 03AB }
+ { DC } { 03AC }
+ { DD } { 03AD }
+ { DE } { 03AE }
+ { DF } { 03AF }
+ { E0 } { 03B0 }
+ { E1 } { 03B1 }
+ { E2 } { 03B2 }
+ { E3 } { 03B3 }
+ { E4 } { 03B4 }
+ { E5 } { 03B5 }
+ { E6 } { 03B6 }
+ { E7 } { 03B7 }
+ { E8 } { 03B8 }
+ { E9 } { 03B9 }
+ { EA } { 03BA }
+ { EB } { 03BB }
+ { EC } { 03BC }
+ { ED } { 03BD }
+ { EE } { 03BE }
+ { EF } { 03BF }
+ { F0 } { 03C0 }
+ { F1 } { 03C1 }
+ { F2 } { 03C2 }
+ { F3 } { 03C3 }
+ { F4 } { 03C4 }
+ { F5 } { 03C5 }
+ { F6 } { 03C6 }
+ { F7 } { 03C7 }
+ { F8 } { 03C8 }
+ { F9 } { 03C9 }
+ { FA } { 03CA }
+ { FB } { 03CB }
+ { FC } { 03CC }
+ { FD } { 03CD }
+ { FE } { 03CE }
}
-\msg_new:nnn { str } { utf8-invalid-byte }
+\tl_const:cn { c_str_encoding_iso88597_missing_tl }
{
- \int_compare:nNnTF {`#1} < { 256 }
- { Byte~number~ \int_eval:n {`#1} ~invalid~in~utf-8~encoding. }
- { The~character~number~ \int_eval:n {`#1} ~is~not~a~byte. }
+ { AE }
+ { D2 }
}
-\msg_new:nnn { str } { utf8-missing-byte }
- { The~byte~number~ \int_eval:n {`#1} ~is~not~a~valid~continuation~byte. }
-\msg_new:nnn { str } { utf8-extra-byte }
- { The~byte~number~ \int_eval:n {`#1} ~is~only~valid~as~a~continuation~byte. }
-\msg_new:nnnn { str } { utf8-premature-end }
- { Incomplete~last~UTF8~character. }
+%</iso88597>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso88598>
+\str_encoding_eight_bit:n { iso88598 }
+\tl_const:cn { c_str_encoding_iso88598_tl }
{
- The~sequence~of~byte~that~to~be~converted~to~UTF8~
- ended~before~the~last~character~was~complete.~Perhaps~
- it~was~mistakenly~truncated?
+ { AA } { 00D7 }
+ { BA } { 00F7 }
+ { DF } { 2017 }
+ { E0 } { 05D0 }
+ { E1 } { 05D1 }
+ { E2 } { 05D2 }
+ { E3 } { 05D3 }
+ { E4 } { 05D4 }
+ { E5 } { 05D5 }
+ { E6 } { 05D6 }
+ { E7 } { 05D7 }
+ { E8 } { 05D8 }
+ { E9 } { 05D9 }
+ { EA } { 05DA }
+ { EB } { 05DB }
+ { EC } { 05DC }
+ { ED } { 05DD }
+ { EE } { 05DE }
+ { EF } { 05DF }
+ { F0 } { 05E0 }
+ { F1 } { 05E1 }
+ { F2 } { 05E2 }
+ { F3 } { 05E3 }
+ { F4 } { 05E4 }
+ { F5 } { 05E5 }
+ { F6 } { 05E6 }
+ { F7 } { 05E7 }
+ { F8 } { 05E8 }
+ { F9 } { 05E9 }
+ { FA } { 05EA }
+ { FD } { 200E }
+ { FE } { 200F }
}
-\msg_new:nnn { str } { utf8-pdftex-overflow }
- { The~character~number~#1~is~too~big~for~pdfTeX. }
+\tl_const:cn { c_str_encoding_iso88598_missing_tl }
+ {
+ { A1 }
+ { BF }
+ { C0 }
+ { C1 }
+ { C2 }
+ { C3 }
+ { C4 }
+ { C5 }
+ { C6 }
+ { C7 }
+ { C8 }
+ { C9 }
+ { CA }
+ { CB }
+ { CC }
+ { CD }
+ { CE }
+ { CF }
+ { D0 }
+ { D1 }
+ { D2 }
+ { D3 }
+ { D4 }
+ { D5 }
+ { D6 }
+ { D7 }
+ { D8 }
+ { D9 }
+ { DA }
+ { DB }
+ { DC }
+ { DD }
+ { DE }
+ { FB }
+ { FC }
+ }
+%</iso88598>
% \end{macrocode}
%
-% \subsection{Deprecated string functions}
+% \begin{macrocode}
+%<*iso88599>
+\str_encoding_eight_bit:n { iso88599 }
+\tl_const:cn { c_str_encoding_iso88599_tl }
+ {
+ { D0 } { 011E }
+ { DD } { 0130 }
+ { DE } { 015E }
+ { F0 } { 011F }
+ { FD } { 0131 }
+ { FE } { 015F }
+ }
+\tl_const:cn { c_str_encoding_iso88599_missing_tl }
+ {
+ }
+%</iso88599>
+% \end{macrocode}
%
-% \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n}
-% The naming scheme is a little bit more consistent
-% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}.
% \begin{macrocode}
-\cs_set:Npn \str_length_skip_spaces:N
- { \exp_args:No \str_length_skip_spaces:n }
-\cs_set_eq:NN \str_length_skip_spaces:n \str_length_ignore_spaces:n
+%<*iso885910>
+\str_encoding_eight_bit:n { iso885910 }
+\tl_const:cn { c_str_encoding_iso885910_tl }
+ {
+ { A1 } { 0104 }
+ { A2 } { 0112 }
+ { A3 } { 0122 }
+ { A4 } { 012A }
+ { A5 } { 0128 }
+ { A6 } { 0136 }
+ { A8 } { 013B }
+ { A9 } { 0110 }
+ { AA } { 0160 }
+ { AB } { 0166 }
+ { AC } { 017D }
+ { AE } { 016A }
+ { AF } { 014A }
+ { B1 } { 0105 }
+ { B2 } { 0113 }
+ { B3 } { 0123 }
+ { B4 } { 012B }
+ { B5 } { 0129 }
+ { B6 } { 0137 }
+ { B8 } { 013C }
+ { B9 } { 0111 }
+ { BA } { 0161 }
+ { BB } { 0167 }
+ { BC } { 017E }
+ { BD } { 2015 }
+ { BE } { 016B }
+ { BF } { 014B }
+ { C0 } { 0100 }
+ { C7 } { 012E }
+ { C8 } { 010C }
+ { CA } { 0118 }
+ { CC } { 0116 }
+ { D1 } { 0145 }
+ { D2 } { 014C }
+ { D7 } { 0168 }
+ { D9 } { 0172 }
+ { E0 } { 0101 }
+ { E7 } { 012F }
+ { E8 } { 010D }
+ { EA } { 0119 }
+ { EC } { 0117 }
+ { F1 } { 0146 }
+ { F2 } { 014D }
+ { F7 } { 0169 }
+ { F9 } { 0173 }
+ { FF } { 0138 }
+ }
+\tl_const:cn { c_str_encoding_iso885910_missing_tl }
+ {
+ }
+%</iso885910>
% \end{macrocode}
-% \end{macro}
%
% \begin{macrocode}
-%</initex|package>
+%<*iso885911>
+\str_encoding_eight_bit:n { iso885911 }
+\tl_const:cn { c_str_encoding_iso885911_tl }
+ {
+ { A1 } { 0E01 }
+ { A2 } { 0E02 }
+ { A3 } { 0E03 }
+ { A4 } { 0E04 }
+ { A5 } { 0E05 }
+ { A6 } { 0E06 }
+ { A7 } { 0E07 }
+ { A8 } { 0E08 }
+ { A9 } { 0E09 }
+ { AA } { 0E0A }
+ { AB } { 0E0B }
+ { AC } { 0E0C }
+ { AD } { 0E0D }
+ { AE } { 0E0E }
+ { AF } { 0E0F }
+ { B0 } { 0E10 }
+ { B1 } { 0E11 }
+ { B2 } { 0E12 }
+ { B3 } { 0E13 }
+ { B4 } { 0E14 }
+ { B5 } { 0E15 }
+ { B6 } { 0E16 }
+ { B7 } { 0E17 }
+ { B8 } { 0E18 }
+ { B9 } { 0E19 }
+ { BA } { 0E1A }
+ { BB } { 0E1B }
+ { BC } { 0E1C }
+ { BD } { 0E1D }
+ { BE } { 0E1E }
+ { BF } { 0E1F }
+ { C0 } { 0E20 }
+ { C1 } { 0E21 }
+ { C2 } { 0E22 }
+ { C3 } { 0E23 }
+ { C4 } { 0E24 }
+ { C5 } { 0E25 }
+ { C6 } { 0E26 }
+ { C7 } { 0E27 }
+ { C8 } { 0E28 }
+ { C9 } { 0E29 }
+ { CA } { 0E2A }
+ { CB } { 0E2B }
+ { CC } { 0E2C }
+ { CD } { 0E2D }
+ { CE } { 0E2E }
+ { CF } { 0E2F }
+ { D0 } { 0E30 }
+ { D1 } { 0E31 }
+ { D2 } { 0E32 }
+ { D3 } { 0E33 }
+ { D4 } { 0E34 }
+ { D5 } { 0E35 }
+ { D6 } { 0E36 }
+ { D7 } { 0E37 }
+ { D8 } { 0E38 }
+ { D9 } { 0E39 }
+ { DA } { 0E3A }
+ { DF } { 0E3F }
+ { E0 } { 0E40 }
+ { E1 } { 0E41 }
+ { E2 } { 0E42 }
+ { E3 } { 0E43 }
+ { E4 } { 0E44 }
+ { E5 } { 0E45 }
+ { E6 } { 0E46 }
+ { E7 } { 0E47 }
+ { E8 } { 0E48 }
+ { E9 } { 0E49 }
+ { EA } { 0E4A }
+ { EB } { 0E4B }
+ { EC } { 0E4C }
+ { ED } { 0E4D }
+ { EE } { 0E4E }
+ { EF } { 0E4F }
+ { F0 } { 0E50 }
+ { F1 } { 0E51 }
+ { F2 } { 0E52 }
+ { F3 } { 0E53 }
+ { F4 } { 0E54 }
+ { F5 } { 0E55 }
+ { F6 } { 0E56 }
+ { F7 } { 0E57 }
+ { F8 } { 0E58 }
+ { F9 } { 0E59 }
+ { FA } { 0E5A }
+ { FB } { 0E5B }
+ }
+\tl_const:cn { c_str_encoding_iso885911_missing_tl }
+ {
+ { DB }
+ { DC }
+ { DD }
+ { DE }
+ }
+%</iso885911>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso885913>
+\str_encoding_eight_bit:n { iso885913 }
+\tl_const:cn { c_str_encoding_iso885913_tl }
+ {
+ { A1 } { 201D }
+ { A5 } { 201E }
+ { A8 } { 00D8 }
+ { AA } { 0156 }
+ { AF } { 00C6 }
+ { B4 } { 201C }
+ { B8 } { 00F8 }
+ { BA } { 0157 }
+ { BF } { 00E6 }
+ { C0 } { 0104 }
+ { C1 } { 012E }
+ { C2 } { 0100 }
+ { C3 } { 0106 }
+ { C6 } { 0118 }
+ { C7 } { 0112 }
+ { C8 } { 010C }
+ { CA } { 0179 }
+ { CB } { 0116 }
+ { CC } { 0122 }
+ { CD } { 0136 }
+ { CE } { 012A }
+ { CF } { 013B }
+ { D0 } { 0160 }
+ { D1 } { 0143 }
+ { D2 } { 0145 }
+ { D4 } { 014C }
+ { D8 } { 0172 }
+ { D9 } { 0141 }
+ { DA } { 015A }
+ { DB } { 016A }
+ { DD } { 017B }
+ { DE } { 017D }
+ { E0 } { 0105 }
+ { E1 } { 012F }
+ { E2 } { 0101 }
+ { E3 } { 0107 }
+ { E6 } { 0119 }
+ { E7 } { 0113 }
+ { E8 } { 010D }
+ { EA } { 017A }
+ { EB } { 0117 }
+ { EC } { 0123 }
+ { ED } { 0137 }
+ { EE } { 012B }
+ { EF } { 013C }
+ { F0 } { 0161 }
+ { F1 } { 0144 }
+ { F2 } { 0146 }
+ { F4 } { 014D }
+ { F8 } { 0173 }
+ { F9 } { 0142 }
+ { FA } { 015B }
+ { FB } { 016B }
+ { FD } { 017C }
+ { FE } { 017E }
+ { FF } { 2019 }
+ }
+\tl_const:cn { c_str_encoding_iso885913_missing_tl }
+ {
+ }
+%</iso885913>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso885914>
+\str_encoding_eight_bit:n { iso885914 }
+\tl_const:cn { c_str_encoding_iso885914_tl }
+ {
+ { A1 } { 1E02 }
+ { A2 } { 1E03 }
+ { A4 } { 010A }
+ { A5 } { 010B }
+ { A6 } { 1E0A }
+ { A8 } { 1E80 }
+ { AA } { 1E82 }
+ { AB } { 1E0B }
+ { AC } { 1EF2 }
+ { AF } { 0178 }
+ { B0 } { 1E1E }
+ { B1 } { 1E1F }
+ { B2 } { 0120 }
+ { B3 } { 0121 }
+ { B4 } { 1E40 }
+ { B5 } { 1E41 }
+ { B7 } { 1E56 }
+ { B8 } { 1E81 }
+ { B9 } { 1E57 }
+ { BA } { 1E83 }
+ { BB } { 1E60 }
+ { BC } { 1EF3 }
+ { BD } { 1E84 }
+ { BE } { 1E85 }
+ { BF } { 1E61 }
+ { D0 } { 0174 }
+ { D7 } { 1E6A }
+ { DE } { 0176 }
+ { F0 } { 0175 }
+ { F7 } { 1E6B }
+ { FE } { 0177 }
+ }
+\tl_const:cn { c_str_encoding_iso885914_missing_tl }
+ {
+ }
+%</iso885914>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso885915>
+\str_encoding_eight_bit:n { iso885915 }
+\tl_const:cn { c_str_encoding_iso885915_tl }
+ {
+ { A4 } { 20AC }
+ { A6 } { 0160 }
+ { A8 } { 0161 }
+ { B4 } { 017D }
+ { B8 } { 017E }
+ { BC } { 0152 }
+ { BD } { 0153 }
+ { BE } { 0178 }
+ }
+\tl_const:cn { c_str_encoding_iso885915_missing_tl }
+ {
+ }
+%</iso885915>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<*iso885916>
+\str_encoding_eight_bit:n { iso885916 }
+\tl_const:cn { c_str_encoding_iso885916_tl }
+ {
+ { A1 } { 0104 }
+ { A2 } { 0105 }
+ { A3 } { 0141 }
+ { A4 } { 20AC }
+ { A5 } { 201E }
+ { A6 } { 0160 }
+ { A8 } { 0161 }
+ { AA } { 0218 }
+ { AC } { 0179 }
+ { AE } { 017A }
+ { AF } { 017B }
+ { B2 } { 010C }
+ { B3 } { 0142 }
+ { B4 } { 017D }
+ { B5 } { 201D }
+ { B8 } { 017E }
+ { B9 } { 010D }
+ { BA } { 0219 }
+ { BC } { 0152 }
+ { BD } { 0153 }
+ { BE } { 0178 }
+ { BF } { 017C }
+ { C3 } { 0102 }
+ { C5 } { 0106 }
+ { D0 } { 0110 }
+ { D1 } { 0143 }
+ { D5 } { 0150 }
+ { D7 } { 015A }
+ { D8 } { 0170 }
+ { DD } { 0118 }
+ { DE } { 021A }
+ { E3 } { 0103 }
+ { E5 } { 0107 }
+ { F0 } { 0111 }
+ { F1 } { 0144 }
+ { F5 } { 0151 }
+ { F7 } { 015B }
+ { F8 } { 0171 }
+ { FD } { 0119 }
+ { FE } { 021B }
+ }
+\tl_const:cn { c_str_encoding_iso885916_missing_tl }
+ {
+ }
+%</iso885916>
% \end{macrocode}
%
% \end{implementation}
diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
index 5aef8b5f0fe..cb3e893d52c 100644
--- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
+++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins
@@ -37,7 +37,40 @@ Do not distribute a modified version of this file.
\generate{\file{l3str.sty} {\from{l3str.dtx} {package}}}
\generate{\file{l3regex.sty} {\from{l3regex.dtx} {package}}}
-
-\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}}
+%\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}}
+
+% Escapings.
+\generate{%
+ \file{l3str-hex.def} {\from{l3str.dtx}{hex}}%
+ \file{l3str-name.def} {\from{l3str.dtx}{name}}%
+ \file{l3str-string.def} {\from{l3str.dtx}{string}}%
+ \file{l3str-url.def} {\from{l3str.dtx}{url}}%
+}
+
+% UTF encodings.
+\generate{%
+ \file{l3str-utf8.def} {\from{l3str.dtx}{utf8}}%
+ \file{l3str-utf16.def} {\from{l3str.dtx}{utf16}}%
+ \file{l3str-utf32.def} {\from{l3str.dtx}{utf32}}%
+}
+
+% ISO-8859 encodings.
+\generate{%
+ \file{l3str-iso88591.def} {\from{l3str.dtx}{iso88591}}%
+ \file{l3str-iso88592.def} {\from{l3str.dtx}{iso88592}}%
+ \file{l3str-iso88593.def} {\from{l3str.dtx}{iso88593}}%
+ \file{l3str-iso88594.def} {\from{l3str.dtx}{iso88594}}%
+ \file{l3str-iso88595.def} {\from{l3str.dtx}{iso88595}}%
+ \file{l3str-iso88596.def} {\from{l3str.dtx}{iso88596}}%
+ \file{l3str-iso88597.def} {\from{l3str.dtx}{iso88597}}%
+ \file{l3str-iso88598.def} {\from{l3str.dtx}{iso88598}}%
+ \file{l3str-iso88599.def} {\from{l3str.dtx}{iso88599}}%
+ \file{l3str-iso885910.def} {\from{l3str.dtx}{iso885910}}%
+ \file{l3str-iso885911.def} {\from{l3str.dtx}{iso885911}}%
+ \file{l3str-iso885913.def} {\from{l3str.dtx}{iso885913}}%
+ \file{l3str-iso885914.def} {\from{l3str.dtx}{iso885914}}%
+ \file{l3str-iso885915.def} {\from{l3str.dtx}{iso885915}}%
+ \file{l3str-iso885916.def} {\from{l3str.dtx}{iso885916}}%
+}
\endbatchfile