diff options
author | Karl Berry <karl@freefriends.org> | 2011-12-11 22:31:32 +0000 |
---|---|---|
committer | Karl Berry <karl@freefriends.org> | 2011-12-11 22:31:32 +0000 |
commit | a57ec5a6dea711c791fc567803079341c6fbfae3 (patch) | |
tree | 2d68d67d8356cba9a3f6063847e6a96cb913ad37 /Master/texmf-dist | |
parent | bca3b6017c5eda5d6c5d67bb266062fb7f363b5b (diff) |
l3experimental 3036 (11dec11)
git-svn-id: svn://tug.org/texlive/trunk@24820 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist')
33 files changed, 6052 insertions, 3023 deletions
diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf Binary files differindex 0ddd7169a49..04338df8609 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3regex.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf b/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf Binary files differdeleted file mode 100644 index d82842dca5f..00000000000 --- a/Master/texmf-dist/doc/latex/l3experimental/l3str/l3str.pdf +++ /dev/null diff --git a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf Binary files differindex 796242d76b6..7035f06a7e1 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/xcoffins/xcoffins.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf Binary files differindex c238f89d86d..7864fb8a764 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/l3galley.pdf diff --git a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf Binary files differindex 8e77fd9b96f..a2bd04b0349 100644 --- a/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf +++ b/Master/texmf-dist/doc/latex/l3experimental/xgalley/xgalley.pdf diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx index cb0476ac898..94bd054f0ce 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3regex.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 2943 2011-10-30 15:59:43Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $ {L3 Experimental Regular Expressions} %</driver|package> %<*driver> @@ -210,6 +210,10 @@ % or \texttt{C}. % \item[(\ldots{})] Capturing group. % \item[(?:\ldots{})] Non-capturing group. +% \item[(?\char`|\ldots{})] Non-capturing group which resets +% the group number for capturing groups in each alternative. +% The following group will be numbered with the first unused +% group number. % \end{l3regex-syntax} % % Options can be set with |(?|\meta{option}|)| and @@ -267,7 +271,7 @@ % regular expression argument either as an explicit string % or as a precompiled regular expression. % -% \begin{function}{\regex_set:Nn, \regex_gset:Nn} +% \begin{function}{\regex_set:Nn, \regex_gset:Nn, \regex_const:Nn} % \begin{syntax} % \cs{regex_set:Nn} \meta{tl var} \Arg{regex} % \end{syntax} @@ -279,7 +283,8 @@ % \regex_set:Nn \l_my_regex_tl { my\ (simple\ )? reg(ex|ular\ expression) } % \end{verbatim} % The assignment is local for \cs{regex_set:Nn} and global for -% \cs{regex_gset:Nn}. +% \cs{regex_gset:Nn}. Use \cs{regex_const:Nn} for precompiled expressions +% which will never change. % \begin{texnote} % Precompiled regular expressions can be safely written to a file % and read when the \LaTeX3 syntax is active (as triggered by @@ -371,8 +376,6 @@ % sequence contains the two items \texttt{Hello} and \texttt{world}. % \end{function} % -% \subsection{String splitting} -% % \begin{function}{\regex_split:nnN, \regex_split:NnN} % \begin{syntax} % \cs{regex_split:nnN} \Arg{regular expression} \meta{string} \meta{seq~var} @@ -424,9 +427,9 @@ % The following need to be done now. % \begin{itemize} % \item The \texttt{\{\}} quantifiers are only partially implemented. -% \item Space's catcode wrong in the result of a replacement! +% \item Check the catcode of spaces everywhere. % \item Newline conventions are not done. -% In particular, |.| should not match newlines. +% In particular, we should have an option for |.| not to match newlines. % Also, |\A| should differ from |^|, and |\Z|, |\z| and |$| should % differ. % \end{itemize} @@ -438,8 +441,6 @@ % \item Regex matching on external files. % \item Conditional subpatterns with look ahead/behind: \enquote{if % what follows is [\ldots{}], then [\ldots{}]}. -% \item \verb"(?|..|..)" to reset the capturing group number at the start -% of each alternative. % \item |(*..)| and |(?..)| sequences to set some options. % \item Reduce the number of epsilon-transitions in alternatives. % \item Optimize simple strings: use less states @@ -452,12 +453,12 @@ % % The following features of PCRE or Perl will probably not be implemented. % \begin{itemize} -% \item Other forms of escapes, and character classes. -% |\cx|$\simeq$|\^^x|, |\ddd| (octal \texttt{ddd}). -% POSIX character classes. -% |\p{..}| and |\P{..}| for having/not having a Unicode property. -% |\X| for \enquote{extended} Unicode sequence. -% \item Callout with |(?C...)|. +% \item |\cx|, similar to \TeX{}'s own |\^^x|; +% \item |\ddd|, matching the character with code \texttt{ddd} in octal; +% \item POSIX character classes |[:alpha:]| \emph{etc.}; +% \item Unicode properties: |\p{..}| and |\P{..}|; +% \item |\X| which should match any \enquote{extended} Unicode sequence; +% \item Callout with |(?C...)|; % \item Conditional subpatterns (other than with a look-ahead % or look-behind condition): this is non-regular, isn't it? % \end{itemize} @@ -518,25 +519,6 @@ % % \subsection{Constants and variables} % -% \begin{variable} -% { -% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one, -% \c_ninety_seven, \c_one_hundred_twenty_three, -% \c_one_hundred_twenty_seven -% } -% We declare here some integer values which delimit ranges of -% ASCII characters of various types. -% \begin{macrocode} -\int_const:Nn \c_forty_eight { 48 } -\int_const:Nn \c_fifty_eight { 58 } -\int_const:Nn \c_sixty_five { 65 } -\int_const:Nn \c_ninety_one { 91 } -\int_const:Nn \c_ninety_seven { 97 } -\int_const:Nn \c_one_hundred_twenty_three { 123 } -\int_const:Nn \c_one_hundred_twenty_seven { 127 } -% \end{macrocode} -% \end{variable} -% % \begin{macro}{\regex_tmp:w} % \begin{variable}{\g_regex_tmpa_tl, \l_regex_tmpa_tl, \l_regex_tmpb_tl} % \begin{variable}{\l_regex_tmpa_int, \l_regex_tmpb_int} @@ -569,19 +551,33 @@ % \end{variable} % \end{variable} % -% \begin{variable}{\l_regex_group_begin_seq, \l_regex_group_end_seq} +% \begin{variable}{\l_regex_left_state_seq, \l_regex_right_state_seq} % Alternatives are implemented by branching from a state into the % various choices, then merging those into another state. We store -% information about those states in two sequences (several can be -% active at the same time when nesting groups). +% information about those states in two sequences. % \begin{macrocode} -\seq_new:N \l_regex_group_begin_seq -\seq_new:N \l_regex_group_end_seq +\seq_new:N \l_regex_left_state_seq +\seq_new:N \l_regex_right_state_seq % \end{macrocode} % \end{variable} % +% \begin{variable}{\l_regex_end_group_seq} +% \begin{variable}{\l_regex_end_alternation_seq} +% These sequences hold actions to be performed at the end of a group, +% and at the end of each branch of the alternation, respectively. +% Currently, \cs{l_regex_end_group_seq} is used to keep track of +% letter case, and \cs{l_regex_end_alternation_seq} is used +% for \verb"(?|...)" groups. +% \begin{macrocode} +\seq_new:N \l_regex_end_group_seq +\seq_new:N \l_regex_end_alternation_seq +% \end{macrocode} +% \end{variable} +% \end{variable} +% % \begin{variable}{\l_regex_capturing_group_int} % \begin{variable}{\l_regex_capturing_group_seq} +% \begin{variable}{\l_regex_capturing_group_max_int} % \cs{l_regex_capturing_group_int} is the ID number of the current % capturing group, starting at $0$ for a group enclosing the full % regular expression, and counting in the order of their left parenthesis. @@ -591,9 +587,11 @@ % \begin{macrocode} \int_new:N \l_regex_capturing_group_int \seq_new:N \l_regex_capturing_group_seq +\int_new:N \l_regex_capturing_group_max_int % \end{macrocode} % \end{variable} % \end{variable} +% \end{variable} % % \begin{variable}{\l_regex_one_or_group_tl} % When looking for quantifiers, this variable holds either @@ -930,30 +928,34 @@ % % \subsection{Helpers} % -% \subsubsection{Toks} +% \subsubsection{Scan markers} % -% \begin{macro}[int]{\s_regex_toks} +% \begin{macro}[int]{\s_regex_stop} +% Unexpandable markers, which does nothing if it reaches \TeX{}'s stomach. % When performing the matching, the \tn{toks} registers hold submatch % information, followed by the instruction for a given state of the NFA. -% The two parts are separated by \cs{s_regex_toks}. +% The two parts are separated by \cs{s_regex_stop}. +% See \cs{regex_nfa:Nw} for another use. % \begin{macrocode} -\cs_new_eq:NN \s_regex_toks \scan_stop: +\cs_new_eq:NN \s_regex_stop \scan_stop: % \end{macrocode} % \end{macro} % +% \subsubsection{Toks} +% % \begin{macro}[int]{\regex_toks_put_left:Nx} % \begin{macro}[int]{\regex_toks_put_right:Nx} % During the building phase, every \tn{toks} register starts with -% \cs{s_regex_toks}, and we wish to add \texttt{x}-expanded material +% \cs{s_regex_stop}, and we wish to add \texttt{x}-expanded material % to those registers. The expansion is done \enquote{by hand} for % optimization (these operations are used quite a lot). When adding % material to the left, we define \cs{regex_tmp:w} to remove the -% \cs{s_regex_toks} marker and put it back to the left of the new +% \cs{s_regex_stop} marker and put it back to the left of the new % material. % \begin{macrocode} \cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 { - \cs_set_nopar:Npx \regex_tmp:w \s_regex_toks { \s_regex_toks #2 } + \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } } @@ -994,20 +996,44 @@ % % \subsubsection{Sequences} % -% \begin{macro}[int]{\regex_seq_pop:N} -% \begin{macro}[int]{\regex_seq_get:N} +% \begin{macro}[int]{\regex_seq_pop_int:NN} +% \begin{macro}[int]{\regex_seq_get_int:NN} +% \begin{macro}[int]{\regex_seq_push_int:NN} +% When building the regular expression, we keep track of some integers +% (pointers to various states) without help from \TeX{}'s grouping. +% Here are variants of \cs{seq_pop:NN} and \cs{seq_get:NN} which +% assign using \cs{int_set:Nn} rather than \cs{tl_set:Nn}. +% \begin{macrocode} +\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2 + { + \seq_pop:NN #1 \l_regex_tmpa_tl + \int_set:Nn #2 \l_regex_tmpa_tl + } +\cs_new_protected:Npn \regex_seq_get_int:NN #1#2 + { + \seq_get:NN #1 \l_regex_tmpa_tl + \int_set:Nn #2 \l_regex_tmpa_tl + } +\cs_new_protected:Npn \regex_seq_push_int:NN #1#2 + { \seq_push:No #1 { \int_use:N #2 } } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\regex_seq_pop_use:N} +% \begin{macro}[int]{\regex_seq_get_use:N} % When building the regular expression, some settings are kept % local to capturing groups without any help from \TeX{}'s grouping. -% This is done \enquote{by hand}, in sequences whose items contain -% instructions such as |\int_set:Nn| \meta{int~var} \Arg{value}, -% which should be run immediately. +% This is done \enquote{by hand}, in sequences whose items should +% be run immediately. % \begin{macrocode} -\cs_new_protected_nopar:Npn \regex_seq_pop:N #1 +\cs_new_protected_nopar:Npn \regex_seq_pop_use:N #1 { \seq_pop:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl } -\cs_new_protected_nopar:Npn \regex_seq_get:N #1 +\cs_new_protected_nopar:Npn \regex_seq_get_use:N #1 { \seq_get:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl @@ -1297,7 +1323,7 @@ % \begin{macro}[int]{\regex_build_new_state:} % Here, we add a new state to the NFA. At the end of the building % phase, we want every \tn{toks} register to start with -% \cs{s_regex_toks}, hence initialize the new register appropriately. +% \cs{s_regex_stop}, hence initialize the new register appropriately. % Then set \cs{l_regex_left/right_state_int} to their new values. % \begin{macrocode} \cs_new_protected_nopar:Npn \regex_build_new_state: @@ -1306,7 +1332,7 @@ { \msg_error:nn { regex } { -997 } } { \int_incr:N \l_regex_max_state_int - \tex_toks:D \l_regex_max_state_int { \s_regex_toks } + \tex_toks:D \l_regex_max_state_int { \s_regex_stop } } \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int @@ -1367,7 +1393,7 @@ { #1 } \regex_build_open_aux: \g_str_result_tl \prg_do_nothing: \prg_do_nothing: - \seq_push:Nn \l_regex_capturing_group_seq { 0 } + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero \regex_build_close_aux: \regex_build_group_: \regex_build_end: } @@ -1595,7 +1621,7 @@ { \cs_set:Npx \regex_build_tmp_class:n ##1 { - \regex_item_equal:n { \int_value:w `#1 } + \regex_item_equal:n { \int_value:w `#1 ~ } \regex_break_point:TF { \regex_action_cost:n { ##1 } } { } } \regex_build_one_quantifier: @@ -2139,8 +2165,8 @@ { \regex_build_special_group:NN } { \int_incr:N \l_regex_capturing_group_int - \seq_push:Nx \l_regex_capturing_group_seq - { \int_use:N \l_regex_capturing_group_int } + \regex_seq_push_int:NN + \l_regex_capturing_group_seq \l_regex_capturing_group_int \regex_build_open_aux: #1 #2 } @@ -2148,19 +2174,12 @@ \cs_new_protected_nopar:Npn \regex_build_open_aux: { \regex_build_new_state: - \seq_push:Nx \l_regex_group_begin_seq - { - \int_set:Nn \l_regex_left_state_int - { \int_use:N \l_regex_left_state_int } - } + \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int + \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int \bool_if:NTF \l_regex_caseless_bool - { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseless: } } - { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseful: } } - \seq_push:Nx \l_regex_group_end_seq - { - \int_set:Nn \l_regex_right_state_int - { \int_use:N \l_regex_right_state_int } - } + { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseless: } + { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseful: } + \seq_push:Nn \l_regex_end_alternation_seq { } \regex_build_begin_alternation: } \cs_new_protected_nopar:cpn { regex_build_|: } @@ -2180,9 +2199,10 @@ \cs_new_protected_nopar:Npn \regex_build_close_aux: { \regex_build_end_alternation: - \regex_seq_pop:N \l_regex_group_begin_seq - \regex_seq_pop:N \l_regex_group_end_seq - \regex_seq_pop:N \l_regex_group_end_seq + \regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int + \regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int + \regex_seq_pop_use:N \l_regex_end_group_seq + \seq_pop:NN \l_regex_end_alternation_seq \l_regex_tmpa_tl } % \end{macrocode} % Building each branch. @@ -2190,7 +2210,7 @@ \cs_new_protected_nopar:Npn \regex_build_begin_alternation: { \regex_build_new_state: - \regex_seq_get:N \l_regex_group_begin_seq + \regex_seq_get_int:NN \l_regex_left_state_seq \l_regex_left_state_int \regex_toks_put_right:Nx \l_regex_left_state_int { \regex_action_free:n @@ -2203,7 +2223,7 @@ \cs_new_protected_nopar:Npn \regex_build_end_alternation: { \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int - \regex_seq_get:N \l_regex_group_end_seq + \regex_seq_get_int:NN \l_regex_right_state_seq \l_regex_right_state_int \regex_toks_put_right:Nx \l_regex_left_state_int { \regex_action_free:n @@ -2212,6 +2232,7 @@ { \l_regex_right_state_int - \l_regex_left_state_int } } } + \regex_seq_get_use:N \l_regex_end_alternation_seq } % \end{macrocode} % \end{macro} @@ -2239,18 +2260,62 @@ % % \begin{macro}{\regex_build_special_group_::} % Non-capturing groups are like capturing groups, except that -% we set the group id to \texttt{*}, which will then inhibit +% we set the group id to \texttt{-1}, which will then inhibit % submatching in \cs{regex_build_group_submatches:NN}. % The group number is not increased. % \begin{macrocode} \cs_new_protected_nopar:cpn { regex_build_special_group_:: } { - \seq_push:Nx \l_regex_capturing_group_seq { * } + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one \regex_build_open_aux: } % \end{macrocode} % \end{macro} % +% \begin{macro}+\regex_build_special_group_|:+ +% The special group \verb"(?|..|..)" is non-capturing +% (hence we set the |capturing_group| to $-1$), and resets +% the group number in each branch of the alternation. +% We use a variant of \cs{regex_build_open_aux:}, adding +% some code to be performed at every alternation, and at +% the end of the group. Namely, we keep track of +% the maximal value that \cs{l_regex_capturing_group_int} +% takes, and restore that value when the group end, +% and in every branch, we reset the capturing group number. +% \begin{macrocode} +\cs_new_protected_nopar:cpn { regex_build_special_group_|: } + { + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one + \regex_build_new_state: + \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int + \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int + \seq_push:Nx \l_regex_end_alternation_seq + { + \exp_not:N \int_compare:nNnT + \l_regex_capturing_group_int + > \l_regex_capturing_group_max_int + { + \int_set_eq:NN + \l_regex_capturing_group_max_int + \l_regex_capturing_group_int + } + \int_set:Nn \l_regex_capturing_group_int + { \int_use:N \l_regex_capturing_group_int } + } + \seq_push:Nx \l_regex_end_group_seq + { + \bool_if:NTF \l_regex_caseless_bool + \regex_build_caseless: + \regex_build_caseful: + \int_set_eq:NN + \l_regex_capturing_group_int + \l_regex_capturing_group_max_int + } + \regex_build_begin_alternation: + } +% \end{macrocode} +% \end{macro} +% % \begin{macro}{\regex_build_special_group_i:} % \begin{macro}{\regex_build_special_group_-:} % \begin{macro}[aux]{\regex_build_options:NNN} @@ -2324,7 +2389,7 @@ \cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2 { \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl - \str_if_eq:xxF { \l_regex_tmpa_tl } { * } + \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero { \regex_toks_put_left:Nx #1 { \regex_action_submatch:n { \l_regex_tmpa_tl < } } @@ -2368,7 +2433,7 @@ { \tex_toks:D \l_regex_tmpa_int { - \s_regex_toks + \s_regex_stop #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } } } @@ -2680,7 +2745,7 @@ { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } } -\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_toks { } +\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { } \cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int @@ -2734,15 +2799,15 @@ \tex_the:D \tex_toks:D #1 } } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_toks +\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop { \regex_store_submatches_aux_ii:Nnnw #1 \regex_state_submatches:nn \c_minus_one \q_prop - \s_regex_toks + \s_regex_stop } \cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_regex_toks + \regex_state_submatches:nn #1 #2 #3 \s_regex_stop { \exp_after:wN \c_zero \exp_after:wN \regex_state_submatches:nn \exp_after:wN @@ -2753,7 +2818,7 @@ } \exp_after:wN { \l_regex_current_submatches_prop } \regex_state_submatches:nn {#1} {#2} - \s_regex_toks + \s_regex_stop } % \end{macrocode} % \end{macro} @@ -2873,19 +2938,6 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[int]{\s_regex_step} -% \begin{macro}[aux]{\regex_break_step:w} -% When a thread succeeds, all threads with lower precedence -% can be ignored, and the next character should be read. -% This is done by skipping to \cs{s_regex_step}. This marker -% does nothing if no thread succeeded at this step. -% \begin{macrocode} -\cs_new_eq:NN \s_regex_step \scan_stop: -\cs_new:Npn \regex_break_step:w #1 \s_regex_step { } -% \end{macrocode} -% \end{macro} -% \end{macro} -% % \begin{macro}[aux]{\regex_match_loop:N} % \begin{macro}[aux]{\regex_match_one_index:n} % \begin{macro}[aux]{\regex_match_one_index_aux:n} @@ -2907,8 +2959,7 @@ {1} {1} { \l_regex_max_index_int } \regex_match_one_index:n } - \regex_tmp:w - \s_regex_step + \regex_tmp:w \prg_break_point:n { } \if_num:w \l_regex_max_index_int = \c_zero \exp_after:wN \use_none_delimit_by_q_recursion_stop:w \fi: @@ -3021,7 +3072,8 @@ % \begin{macro}[aux]{\regex_action_success:} % There is a successful match when an execution path reaches % the end of the regular expression. Then store the current -% step and submatches. The current step is then interrupted, +% step and submatches. The current step is then interrupted +% with \cs{prg_map_break:}, % and only paths with higher precedence are pursued further. % The values stored here may be overwritten by a later success % of a path with higher precedence. @@ -3037,7 +3089,7 @@ \l_regex_current_step_int \prop_set_eq:NN \l_regex_success_submatches_prop \l_regex_current_submatches_prop - \regex_break_step:w + \prg_map_break: } } % \end{macrocode} @@ -3127,7 +3179,7 @@ % \cs{tex_toks:D} 0 \{ \meta{instruction0} \} \\ % \ldots{} \\ % \cs{tex_toks:D} $n$ \{ \meta{instruction$\sb{n}$} \} \\ -% \cs{regex_nfa_end:} +% \cs{s_regex_stop} % \end{quote} % where $n$ is the number of states in the NFA, % and the various \meta{instruction$\sb{i}$} control @@ -3144,7 +3196,6 @@ % \begin{macro}[aux]{\regex_set_aux:NNn} % \begin{macro}[aux]{\regex_set_aux:n} % \begin{macro}[aux]{\regex_nfa:Nw} -% \begin{macro}[aux]{\regex_nfa_end:} % Within a group, build the NFA corresponding to the given regular % expression, with submatch tracking. Then save the contents of all % relevant \tn{toks} registers into \cs{g_regex_tmpa_tl}, then @@ -3178,22 +3229,28 @@ \prg_stepwise_function:nnnN {1} {1} {\l_regex_max_state_int} \regex_set_aux:n - \regex_nfa_end: + \s_regex_stop } \group_end: #1 #2 \g_regex_tmpa_tl } \cs_new_nopar:Npn \regex_set_aux:n #1 { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 #2 \regex_nfa_end: - { \msg_expandable_error:n { Automaton~#1 used~incorrectly. } } -\cs_new_eq:NN \regex_nfa_end: \scan_stop: +\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop + { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % \end{macro} +% +% \begin{macro}{\regex_const:Nn} +% The same idea as for setting, but using the new function. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \regex_const:Nn + { \regex_set_aux:NNn \cs_new_eq:NN } +% \end{macrocode} % \end{macro} % % \begin{macro}[int,TF]{\regex_check_nfa:N} @@ -3611,7 +3668,7 @@ { \regex_token_if_other_digit:NTF #1 { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } - { \msg_expandable_error:n { \g misused~in~replacement~text. } } + { \msg_expandable_kernel_error:nn { regex } { g-misused } } #1 } } @@ -3623,7 +3680,7 @@ \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1} \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2 { - \msg_expandable_error:n { \g misused~in~replacement~text. } + \msg_expandable_kernel_error:nn { regex } { g-misused } #2 } } @@ -3818,6 +3875,13 @@ % \end{macrocode} % % \begin{macrocode} +\msg_kernel_new:nnn { regex } { nfa-misused } + { Automaton~#1 used~incorrectly. } +\msg_kernel_new:nnn { regex } { g-misused } + { \g misused~in~replacement~text. } +% \end{macrocode} +% +% \begin{macrocode} %</package> % \end{macrocode} % diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx index b3648c57ede..db73291a70b 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.dtx @@ -35,7 +35,7 @@ % %<*driver|package> \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 2921 2011-10-17 13:23:36Z bruno $ +\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $ {L3 Experimental Strings} %</driver|package> %<*driver> @@ -72,7 +72,7 @@ % \LaTeX3 provides a set of functions to manipulate token lists % as strings of characters, ignoring the category codes of those % characters. -% +% % String variables are simply specialised token lists, but by convention % should be named with the suffix \texttt{\ldots str}. Such variables should % contain characters with category code $12$ (other), except spaces, which @@ -87,14 +87,14 @@ % variable as their argument; % \item \cs{str_...:n...}, taking any token list (or string) as an argument; % \item \cs{str_..._ignore_spaces:n...}, which ignores any space encountered -% during the operation: these functions are slightly faster than those +% during the operation: these functions are faster than those % which take care of escaping spaces appropriately; % \end{itemize} % When performance is important, the internal \cs{str_..._unsafe:n...} % functions, which expect a \enquote{safe} string in which spaces % have category code $12$ instead of $10$, might be useful. % -% \subsection{Conversion and input of strings} +% \section{Conversion and input of strings} % % \begin{variable} % { @@ -102,6 +102,7 @@ % \c_lbrace_str, % \c_rbrace_str, % \c_hash_str, +% \c_tilde_str, % \c_percent_str % } % Constant strings, containing a single character, with category code $12$. @@ -116,6 +117,16 @@ % tokens in the input stream. % \end{function} % +% \begin{function}{\str_const:Nn, \str_const:Nx, \str_const:cn, \str_const:cx} +% \begin{syntax} +% \cs{str_const:Nn} \meta{str~var} \Arg{token list} +% \end{syntax} +% Creates a new constant \meta{str~var} or raises an error +% if the name is already taken. The value of the +% \meta{str~var} will be set globally to the +% \meta{token list}, after being converted to a string. +% \end{function} +% % \begin{function}{\str_set:Nn, \str_set:Nx, \str_set:cn, \str_set:cx} % \begin{syntax} % \cs{str_set:Nn} \meta{str var} \Arg{token list} @@ -126,10 +137,10 @@ % % \begin{function}{\str_gset:Nn, \str_gset:Nx, \str_gset:cn, \str_gset:cx} % \begin{syntax} -% \cs{str_gset:Nn} \meta{str var} \Arg{token list} +% \cs{str_gset:Nn} \meta{str~var} \Arg{token list} % \end{syntax} % Converts the \meta{token list} to a \meta{string}, -% and saves the result in \meta{str var} globally. +% and saves the result in \meta{str~var} globally. % \end{function} % % \begin{function} @@ -230,7 +241,7 @@ % since |<| has \textsc{ascii} code \texttt{3C} (in hexadecimal). % \end{function} % -% \subsection{Characters given by their position} +% \section{Characters given by their position} % % \begin{function}[EXP] % {\str_length:N, \str_length:n, \str_length_ignore_spaces:n} @@ -310,11 +321,11 @@ % and leaves in the input stream the characters % between \meta{start index} (inclusive) and \meta{end index} (exclusive). % If either of \meta{start index} or \meta{end index} is negative, -% the it is incremented by the length of the list. Both \meta{start index} +% then it is incremented by the length of the list. Both \meta{start index} % and \meta{end index} count from $0$ for the first (left most) character. % \end{function} % -% \subsection{String conditionals} +% \section{String conditionals} % % \begin{function}[EXP, pTF] % { @@ -337,175 +348,133 @@ % expansion). % \end{function} % -% \begin{function}[EXP, pTF] -% {\str_if_contains_char:NN, \str_if_contains_char:nN} -% \begin{syntax} -% \cs{str_if_contains_char:nN} \Arg{token list} \meta{char} -% \end{syntax} -% Converts the \meta{token list} to a \meta{string} -% and tests whether the \meta{char} is present in the \meta{string}. -% The \meta{char} can be given either directly, or as a one -% letter control sequence. -% \end{function} -% -% \subsection{Byte conditionals} -% % \begin{function}[EXP, pTF]{\str_if_bytes:N} % \begin{syntax} % \cs{str_if_bytes:NTF} \meta{str var} \Arg{true code} \Arg{false code} % \end{syntax} % Tests whether the \meta{str var} only contains characters -% in the range $0$ to $255$. +% in the range $0$ to $255$. This is automatically true when +% the pdf\TeX{} engine is in use. % \end{function} % -% ^^A\begin{function}[EXP, pTF]{\str_if_UTF_viii:N} -% ^^A \begin{syntax} -% ^^A \cs{str_if_UTF_viii:N} \meta{str var} \Arg{true code} \Arg{false code} -% ^^A \end{syntax} -% ^^A Tests whether the \meta{str var} only contains characters -% ^^A in the range $0$--$255$, and forms a valid \texttt{UTF8} string. -% ^^A \textbf{Missing!} ^^A Missing. -% ^^A\end{function} -% -% \subsection{Byte conversion} -% -% Byte conversions are necessary where strings are to be used in PDF -% string objects or within URLs. In both cases, only a restricted subset -% of characters are permitted. -% -% \begin{function}{\str_native_from_UTF_viii:NN} -% \begin{syntax} -% \cs{str_native_from_UTF_viii:NN} \meta{str~var1} \meta{str~var2} -% \end{syntax} -% Reads the contents of the \meta{str~var2} as an \textsc{utf8}-encoded -% sequence of bytes, and stores the resulting characters (which can -% now have any character code) into \meta{str~var1}. -% This function raises an error if the \meta{str~var2} is not -% a valid sequence of bytes in the \textsc{utf8} encoding. -% In the \pdfTeX{} engine, this function raises an error if any of the -% resulting characters is outside the range $0$ to $255$. -% \end{function} +% \section{Encoding functions} % -% \begin{function}{\str_UTF_viii_from_native:NN} -% \begin{syntax} -% \cs{str_UTF_viii_from_native:NN} \meta{str~var1} \meta{str~var2} -% \end{syntax} -% Converts each character of the \meta{str~var2} into -% a sequence of bytes, as defined by the \textsc{utf8} -% encoding, and stores the result in \meta{str~var1}. -% In the \pdfTeX{} engine, this function is of course -% of very little use, but can be used without harm: -% characters in the range $0$ to $127$ are left unchanged, -% and characters in the range $128$ to $255$ become two-byte -% sequences, as per the definition of \texttt{UTF8}. -% \end{function} +% Traditionally, string encodings only specify how strings of characters +% should be stored as bytes. However, the resulting lists of bytes are +% often to be used in contexts where only a restricted subset of bytes +% are permitted (\textsc{pdf} string objects, \textsc{url}s). +% Hence, storing strings of characters has two components. +% \begin{itemize} +% \item The code points (\enquote{character codes}) +% are expressed as bytes following a given +% \enquote{encoding}. This can be \textsc{utf-16}, +% \textsc{iso 8859-1}, \emph{etc.} +% See Table~\ref{tab:encodings} for a list +% of encodings supported. +% \item Bytes are translated to \TeX{} tokens through +% a given \enquote{escaping}. Those are defined for +% the most part by the \texttt{pdf} file format. +% See Table~\ref{tab:escapings} for a list +% of escaping methods supported. +% \end{itemize} % -% \begin{function} -% {\str_bytes_escape_hexadecimal:NN, \str_bytes_unescape_hexadecimal:NN} -% \begin{syntax} -% \cs{str_bytes_escape_hexadecimal:NN} \meta{str var1} \meta{str var2} -% \cs{str_bytes_unescape_hexadecimal:NN} \meta{str var1} \meta{str var2} -% \end{syntax} -% The \texttt{escape} variant takes each character in \meta{str var1} -% and converts it into the hexadecimal representation. The resulting -% string is stored in \meta{str var2} within the current \TeX{} group -% level. The \texttt{unescape} variant reverses this process. -% Thus for example -% \begin{verbatim} -% \str_set:Nn \l_mya_str { Hello ( ) } -% \str_bytes_escape_hexadecimal:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |48656C6C6F2829|, -% while -% \begin{verbatim} -% \str_set:Nn \l_mya_str { 48656C6C6F2829 } -% \str_bytes_unescape_hexadecimal:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello()|. -% See also \tn{pdfescapehex}. -% \end{function} +% \begin{table}\centering +% \caption{\label{tab:encodings}Supported encodings. +% Non-alphanumeric characters are ignored, +% and capital letters are lower-cased.} +% \begin{tabular}{cc} +% \toprule +% \texttt{utf8} & \textsc{utf-8} \\ +% \texttt{utf16} & \textsc{utf-16}, with byte-order mark \\ +% \texttt{utf16be} & \textsc{utf-16}, big-endian \\ +% \texttt{utf16le} & \textsc{utf-16}, little-endian \\ +% \texttt{utf32} & \textsc{utf-32}, with byte-order mark \\ +% \texttt{utf32be} & \textsc{utf-32}, big-endian \\ +% \texttt{utf32le} & \textsc{utf-32}, little-endian \\ +% \texttt{iso88591}, \texttt{latin1} & \textsc{iso 8859-1} \\ +% \texttt{iso88592}, \texttt{latin2} & \textsc{iso 8859-2} \\ +% \texttt{iso88593}, \texttt{latin3} & \textsc{iso 8859-3} \\ +% \texttt{iso88594}, \texttt{latin4} & \textsc{iso 8859-4} \\ +% \texttt{iso88595} & \textsc{iso 8859-5} \\ +% \texttt{iso88596} & \textsc{iso 8859-6} \\ +% \texttt{iso88597} & \textsc{iso 8859-7} \\ +% \texttt{iso88598} & \textsc{iso 8859-8} \\ +% \texttt{iso88599}, \texttt{latin5} & \textsc{iso 8859-9} \\ +% \texttt{iso885910}, \texttt{latin6} & \textsc{iso 8859-10} \\ +% \texttt{iso885911} & \textsc{iso 8859-11} \\ +% \texttt{iso885913}, \texttt{latin7} & \textsc{iso 8859-13} \\ +% \texttt{iso885914}, \texttt{latin8} & \textsc{iso 8859-14} \\ +% \texttt{iso885915}, \texttt{latin9} & \textsc{iso 8859-15} \\ +% \texttt{iso885916}, \texttt{latin10} & \textsc{iso 8859-16} \\ +% \midrule +% \texttt{native} & Native Unicode string. \\ +% \texttt{internal} & Comma-list of integers. \\ +% \bottomrule +% \end{tabular} +% \end{table} % -% \begin{function} -% {\str_bytes_escape_name:NN, \str_bytes_unescape_name:NN} -% \begin{syntax} -% \cs{str_bytes_escape_name:NN} \meta{str var1} \meta{str var2} -% \cs{str_bytes_unescape_name:NN} \meta{str var1} \meta{str var2} -% \end{syntax} -% The \texttt{escape} variant takes each character in \meta{str var1} -% and replaces any which cannot be used directly in a PDF string object -% with the appropriate hexadecimal representation preceded by the |#| -% character. The resulting -% string is stored in \meta{str var2} within the current \TeX{} group -% level. The \texttt{unescape} variant reverses this process. Thus for -% example -% \begin{verbatim} -% \str_set:Nn \l_mya_str { Hello ( ) } -% \str_bytes_escape_name:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello#28#29|, -% while -% \begin{verbatim} -% \char_set_catcode_other:N \# -% \str_set:Nn \l_mya_str { Hello#28#29 } -% \str_bytes_unescape_name:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello()|. -% See also \tn{pdfescapename}. -% \end{function} +% \begin{table}\centering +% \caption{\label{tab:escapings}Supported escapings. +% Non-alphanumeric characters are ignored, +% and capital letters are lower-cased.} +% \begin{tabular}{cc} +% \toprule +% \texttt{bytes}, or empty +% & arbitrary bytes \\ +% \texttt{hex}, \texttt{hexadecimal} +% & byte $=$ two hexadecimal digits \\ +% \texttt{name} +% & see \tn{pdfescapename} \\ +% \texttt{string} +% & see \tn{pdfescapestring} \\ +% \texttt{url}, \texttt{percent} +% & encoding used in \textsc{url}s \\ +% \bottomrule +% \end{tabular} +% \end{table} % -% \begin{function}{\str_bytes_escape_string:NN, \str_bytes_unescape_string:NN} +% \begin{function}{\str_set_convert:Nnnn} % \begin{syntax} -% \cs{str_bytes_escape_string:NN} \meta{str var1} \meta{str var2} -% \cs{str_bytes_unescape_string:NN} \meta{str var1} \meta{str var2} +% \cs{str_set_convert:Nnnn} \meta{str~var} \Arg{string} +% ~~\Arg{name 1} \Arg{name 2} % \end{syntax} -% The \texttt{escape} variant takes each character in \meta{str var1} -% and prepends a |\| to any which cannot be used directly in a PDF string -% object. The resulting string is stored in \meta{str var2} within the -% current \TeX{} group level. The \texttt{unescape} variant reverses this -% process. Thus for example -% \begin{verbatim} -% \str_set:Nn \l_mya_str { Hello [ ] } -% \str_bytes_escape_string:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello\(\)|, -% while -% \begin{verbatim} -% \str_set:Nn \l_mya_str { Hello\(\) } -% \str_bytes_unescape_string:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello()|. -% Some characters (for example a space) are converted to a three-digit -% octal representation, so for example a space gives |\040|. -% See also \tn{pdfescapename}. -% \end{function} +% This function converts the \meta{string} from the encoding +% given by \meta{name 1} to the encoding given by \meta{name 2}, +% and stores the result in the \meta{str~var}. +% Each \meta{name} can have the form \meta{encoding} or +% \meta{encoding}\texttt{/}\meta{escaping}, where the possible values +% of \meta{encoding} and \meta{escaping} are given in +% Tables~\ref{tab:encodings} and~\ref{tab:escapings}, respectively. +% The default escaping is to input and output bytes directly. +% The special encodings \texttt{native} and \texttt{internal} +% do not support a \meta{escaping}, since those formats do not +% correspond to strings of bytes. % -% \begin{function}{\str_bytes_percent_encode:NN, \str_bytes_percent_decode:NN} -% \begin{syntax} -% \cs{str_bytes_percent_encode:NN} meta{str var1} \meta{str var2} -% \cs{str_bytes_percent_decode:NN} meta{str var1} \meta{str var2} -% \end{syntax} -% The \texttt{escape} variant takes each character in \meta{str var1} -% and replaces any which cannot be used directly in a URL -% with the appropriate hexadecimal representation preceded by the |%| -% character. The resulting -% string is stored in \meta{str var2} within the current \TeX{} group -% level. The \texttt{unescape} variant reverses this process. Thus for -% example +% For example, % \begin{verbatim} -% \str_set:Nn \l_mya_str { Hello ( ) } -% \str_bytes_precent_encode:NN \l_mya_str \l_myb_str +% \str_set_convert:Nnnn \l_foo_str { Hello! } +% { native } { utf16/hex } % \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello%28%29|, -% while -% \begin{verbatim} -% \char_set_catcode_other:N \% -% \str_set:Nn \l_mya_str { Hello%28%29 } -% \str_bytes_unescape_name:NN \l_mya_str \l_myb_str -% \end{verbatim} -% will result in \cs{l_myb_str} containing the string |Hello()|. +% results in the variable \cs{l_foo_str} holding the string +% \texttt{FEFF00480065006C006C006F0021}. This is obtained +% by converting each character in the string \texttt{Hello!} +% to the \textsc{utf-16} encoding, and expressing each byte +% as a pair of hexadecimal digits. Note the presence of a +% (big-endian) byte order mark \texttt{FEFF}, which can be +% avoided by specifying the encoding \texttt{utf16be/hex}. +% +% Parts of the \meta{string} which cannot be converted to bytes +% using the \meta{escaping 1} are silently ignored (including non-byte +% characters which could be present in the string). +% Sequences of bytes which are not valid in the \meta{encoding 1} +% are converted to the replacement character |"FFFD|, and raise +% an error. +% Characters which cannot be encoded in the \meta{encoding 2} +% are silently ignored. +%^^A todo: add doc and examples about each encoding and escaping method. % \end{function} % -% \subsection{Internal string functions} +% \section{Internal string functions} % % \begin{function}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n} % \begin{syntax} @@ -531,13 +500,33 @@ % and hands-in the result as arguments to \meta{function}. % \end{function} % -% \begin{function}[rEXP]{\str_map_tokens:Nn} +% \begin{function}{\str_aux_gset_other:Nn} +% \begin{syntax} +% \cs{str_aux_gset_other:Nn} \meta{tl~var} \Arg{token list} +% \end{syntax} +% Converts the \meta{token list} to an \meta{other string}, where +% spaces have category code \enquote{other}, and assigns the result +% to the \meta{tl~var}, globally. +% \end{function} +% +% \begin{function}[rEXP]{\str_map_tokens:Nn, \str_map_tokens:nn} % \begin{syntax} % \cs{str_map_tokens:Nn} \meta{str var} \meta{tokens} % \end{syntax} % Maps the \meta{tokens} over every character in the \meta{str var}. % \end{function} % +% \begin{function}[EXP, pTF] +% {\str_if_contains_char:NN, \str_if_contains_char:nN} +% \begin{syntax} +% \cs{str_if_contains_char:nN} \Arg{token list} \meta{char} +% \end{syntax} +% Converts the \meta{token list} to a \meta{string} +% and tests whether the \meta{char} is present in the \meta{string}. +% The \meta{char} can be given either directly, or as a one +% letter control sequence. +% \end{function} +% % \begin{function}{\str_aux_escape:NNNn} % \begin{syntax} % \cs{str_aux_escape:NNNn} \meta{fn1} \meta{fn2} \meta{fn3} \Arg{token list} @@ -554,26 +543,18 @@ % \meta{fn3}. The result is assigned globally to \cs{g_str_result_tl}. % \end{function} % -% \subsection{Possibilities} +% \section{Possibilities} % % \begin{itemize} % \item More encodings (see Heiko's \pkg{stringenc}). -% In particular, what is needed for pdf: \texttt{UTF-16}? -% \item \cs{str_between:nnn} \Arg{str} -% \Arg{begin delimiter} \Arg{end delimiter} giving the piece of -% \meta{str} between \meta{begin} and \meta{end}; could be used -% with empty \meta{begin} or \meta{end} to indicate that we -% want everything until \meta{end} or starting from \meta{begin}, -% respectively; -% \item \cs{str_count_in:nn} \Arg{str} \Arg{substr} giving the number -% of occurrences of \meta{substr} in \meta{str}; +% In particular, what is needed for pdf: \textsc{utf-16}? +% \item CESU % \item \cs{str_if_head_eq:nN} alias of \cs{tl_if_head_eq_charcode:nN} % \item \cs{str_if_numeric/decimal/integer:n}, perhaps in \pkg{l3fp}? +% \item Should \cs{str_item:Nn} be \cs{str_char:Nn}? % \end{itemize} % Some functionalities of \pkg{stringstrings} and \pkg{xstring} as well. % -%^^A Should "str_item" be "str_char"? -% % \end{documentation} % % \begin{implementation} @@ -611,12 +592,51 @@ % \end{macrocode} % \end{macro} % +% \begin{variable} +% { +% \c_forty_eight, \c_fifty_eight, \c_sixty_five, \c_ninety_one, +% \c_ninety_seven, \c_one_hundred_twenty_three, +% \c_one_hundred_twenty_seven +% } +% We declare here some integer values which delimit ranges of +% ASCII characters of various types. This is mostly used in +% \pkg{l3regex}. +% \begin{macrocode} +\int_const:Nn \c_forty_eight { 48 } +\int_const:Nn \c_fifty_eight { 58 } +\int_const:Nn \c_sixty_five { 65 } +\int_const:Nn \c_ninety_one { 91 } +\int_const:Nn \c_ninety_seven { 97 } +\int_const:Nn \c_one_hundred_twenty_three { 123 } +\int_const:Nn \c_one_hundred_twenty_seven { 127 } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\c_max_char_int} +% The maximum character code depends on the engine. +% \begin{macrocode} +\pdftex_if_engine:TF + { \int_const:Nn \c_max_char_int { 255 } } + { \int_const:Nn \c_max_char_int { 1114111 } } +% \end{macrocode} +% \end{macro} +% +% \begin{variable}{\c_str_replacement_char_int} +% When converting, invalid bytes are replaced by the Unicode +% replacement character \textsc{u-fffd}. +% \begin{macrocode} +\int_const:Nn \c_str_replacement_char_int { "FFFD } +% \end{macrocode} +% \end{variable} +% % \begin{macro} % { % \str_set:Nn, \str_set:Nx, % \str_set:cn, \str_set:cx, % \str_gset:Nn, \str_gset:Nx, % \str_gset:cn, \str_gset:cx, +% \str_const:Nn, \str_const:Nx, +% \str_const:cn, \str_const:cx, % \str_put_left:Nn, \str_put_left:Nx, % \str_put_left:cn, \str_put_left:cx, % \str_gput_left:Nn, \str_gput_left:Nx, @@ -637,6 +657,7 @@ } \str_tmp:w { set } \str_tmp:w { gset } + \str_tmp:w { const } \str_tmp:w { put_left } \str_tmp:w { gput_left } \str_tmp:w { put_right } @@ -648,11 +669,11 @@ % \subsection{Variables and constants} % % \begin{macro}{\str_tmp:w} -% \begin{variable}{\g_str_tmpa_tl} +% \begin{variable}{\l_str_tmpa_tl} % Internal scratch space for some functions. % \begin{macrocode} \cs_new_protected_nopar:Npn \str_tmp:w { } -\tl_new:N \g_str_tmpa_tl +\tl_new:N \l_str_tmpa_tl % \end{macrocode} % \end{variable} % \end{macro} @@ -667,15 +688,12 @@ % \end{macrocode} % \end{variable} % -% \begin{variable}{\l_str_char_int, \l_str_bytes_int} -% When converting from various escaped forms to raw characters, -% we often need to read several digits (hexadecimal or octal depending -% on the case) and keep track of the corresponding character code -% in \cs{l_str_char_int}. For \textsc{utf8} support, the number of bytes -% of for the current character is stored in \cs{l_str_bytes_int}. +% \begin{variable}{\l_str_char_int} +% When converting from various forms to characters, +% \cs{l_str_char_int} is the character code of the +% character which should be created. % \begin{macrocode} \int_new:N \l_str_char_int -\int_new:N \l_str_bytes_int % \end{macrocode} % \end{variable} % @@ -685,6 +703,7 @@ % \c_lbrace_str, % \c_rbrace_str, % \c_hash_str, +% \c_tilde_str, % \c_percent_str % } % For all of those strings, \cs{cs_to_str:N} produce characters @@ -694,11 +713,34 @@ \tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ } \tl_const:Nx \c_rbrace_str { \cs_to_str:N \} } \tl_const:Nx \c_hash_str { \cs_to_str:N \# } +\tl_const:Nx \c_tilde_str { \cs_to_str:N \~ } \tl_const:Nx \c_percent_str { \cs_to_str:N \% } % \end{macrocode} % \end{variable} % -% \subsection{Escaping spaces} +% \begin{variable}{\g_str_aliases_prop} +% To avoid needing one file per encoding/escaping alias, +% we keep track of those in a property list. +% \begin{macrocode} +\prop_new:N \g_str_aliases_prop +\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 } +\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 } +\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 } +\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 } +\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 } +\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 } +\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 } +\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 } +\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 } +\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 } +\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex } +\prop_gput:Nnn \g_str_aliases_prop { percent } { url } +% \end{macrocode} +% \end{variable} +% +% \subsection{Internal functions} +% +% \subsubsection{Escaping spaces} % % \begin{macro}[EXP]{\tl_to_other_str:N, \tl_to_other_str:n} % \begin{macro}[EXP,aux]{\tl_to_other_str_loop:w, \tl_to_other_str_end:w} @@ -748,6 +790,90 @@ % \end{macrocode} % \end{macro} % +% \begin{macro}[int]{\str_aux_gset_other:Nn} +% \begin{macro}[aux,EXP]{\str_aux_gset_other_loop:w} +% \begin{macro}[aux,EXP]{\str_aux_gset_other_end:w} +% \begin{macrocode} +\group_begin: +\char_set_lccode:nn { `\* } { `\ } +\char_set_lccode:nn { `\A } { `\A } +\tl_to_lowercase:n + { + \group_end: + \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2 + { + \tl_gset:Nx #1 + { + \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ % + A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop + } + } + \cs_new_nopar:Npn \str_aux_gset_other_loop:w + #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~ + { + \if_meaning:w A #9 + \str_aux_gset_other_end:w + \fi: + #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9 + \str_aux_gset_other_loop:w * + } + \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop + { \fi: #1 } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsubsection{Mapping} +% +% \begin{macro}[rEXP,int]{\str_map_tokens:nn} +% \begin{macro}[rEXP,int]{\str_map_tokens:Nn} +% \begin{macro}[rEXP,aux]{\str_map_tokens_aux:nn} +% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nw} +% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nN} +% \begin{macro}[rEXP,int]{\str_map_break_do:n} +% We simply need to be careful with spaces. +% Spaces are fed to the mapped tokens with +% category code $12$, not $10$. +% \begin{macrocode} +\cs_new_nopar:Npn \str_map_tokens:Nn + { \exp_args:No \str_map_tokens:nn } +\cs_new:Npn \str_map_tokens:nn #1 + { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } } +\cs_new:Npn \str_map_tokens_aux:nn #1#2 + { + \str_map_tokens_loop:nw {#2} #1 + { ? \use_none_delimit_by_q_recursion_stop:w } + ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop + } +\group_begin: + \char_set_lccode:nn { `* } { `\ } + \tl_to_lowercase:n + { + \group_end: + \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~ + { + \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9* + { ? \use_none_delimit_by_q_stop:w } \q_stop + \str_map_tokens_loop:nw {#1} + } + } +\cs_new:Npn \str_map_tokens_loop:nN #1#2 + { + \use_none:n #2 + #1 #2 + \str_map_tokens_loop:nN {#1} + } +\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% % \subsection{Characters given by their position} % % \begin{macro}[EXP]{\str_length:N} @@ -1113,40 +1239,6 @@ % \end{macro} % \end{macro} % -% \subsection{Internal mapping function} -% -% \begin{macro}[rEXP,int]{\str_map_tokens:nn} -% \begin{macro}[rEXP,int]{\str_map_tokens:Nn} -% \begin{macro}[rEXP,aux]{\str_map_tokens_aux:nn} -% \begin{macro}[rEXP,aux]{\str_map_tokens_loop:nN} -% \begin{macro}[rEXP,int]{\str_map_break_do:n} -% We simply need to be careful with spaces. -% Spaces are fed to the mapped tokens with -% category code $12$, not $10$. -% \begin{macrocode} -\cs_new_nopar:Npn \str_map_tokens:Nn - { \exp_args:No \str_map_tokens:nn } -\cs_new_nopar:Npn \str_map_tokens:nn - { \str_sanitize_args:Nn \str_map_tokens_aux:nn } -\cs_new:Npn \str_map_tokens_aux:nn #1#2 - { - \str_map_tokens_loop:nN {#2} #1 - { ? \use_none_delimit_by_q_recursion_stop:w } \q_recursion_stop - } -\cs_new_nopar:Npn \str_map_tokens_loop:nN #1#2 - { - \use_none:n #2 - #1 #2 - \str_map_tokens_loop:nN {#1} - } -\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw -% \end{macrocode} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% \end{macro} -% % \subsection{String conditionals} % % \begin{macro}[EXP,pTF]{\str_if_eq:NN} @@ -1163,6 +1255,7 @@ % \end{macro} % \end{macro} % +%^^A treat spaces separately, or make that internal and faster. % \begin{macro}[EXP,pTF]{\str_if_contains_char:NN} % \begin{macro}[EXP,pTF]{\str_if_contains_char:nN} % \begin{macro}[EXP,aux]{\str_if_contains_char_aux:NN} @@ -1199,34 +1292,53 @@ % % \begin{macro}[EXP,pTF]{\str_if_bytes:N} % \begin{macro}[EXP,aux]{\str_if_bytes_aux:N} -% Loop over the string, checking if every character code is less than $256$. +% \begin{macro}[EXP,aux]{\str_if_bytes_aux:} +% In the pdf\TeX{} engine, every character is a byte, +% so there is nothing to check. In other engines, we +% loop over the string, checking if every character +% code is less than $256$. % \begin{macrocode} -\prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF } +\pdftex_if_engine:TF { - \str_map_tokens:Nn #1 { \str_if_bytes_aux:N } - \prg_return_true: + \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool + \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn + \cs_new_eq:NN \str_if_bytes:NF \use_none:nn + \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn } -\cs_new_nopar:Npn \str_if_bytes_aux:N #1 { - \int_compare:nNnF {`#1} < \c_two_hundred_fifty_six + \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF } { - \use_i_delimit_by_q_recursion_stop:nw - { \prg_return_false: \use_none:n } + \exp_args:Nf \tl_map_function:nN + { \tl_to_str:N #1 } + \str_if_bytes_aux:N + \prg_return_true: + } + \cs_new_nopar:Npn \str_if_bytes_aux:N #1 + { + \if_num:w `#1 > \c_two_hundred_fifty_five + \exp_after:wN \str_if_bytes_aux: + \fi: } + \cs_new_nopar:Npn \str_if_bytes_aux: + { \tl_map_break:n { \prg_return_false: \use_none:n } } } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} +% +% \subsection{Conversions} % -% \subsection{Internal conditionals} +% \subsubsection{Internal conditionals} % % \begin{macro}[aux]{\str_aux_hexadecimal_test:NTF} % This test is used when reading hexadecimal digits, for the |\x| escape -% sequence, and some conversion functions. It returns \meta{true} if the +% sequence. It returns \meta{true} if the % token is a hexadecimal digit, and \meta{false} otherwise. It has the % additional side-effect of updating the value of % \cs{l_str_char_int} (number formed in base $16$ from the digits % read so far). +%^^A todo: speed tweaks? % \begin{macrocode} \prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF } { @@ -1249,20 +1361,17 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}[aux]{\str_aux_octal_test:NTF} -% This test is used when reading octal digits, for -% some conversion functions. It returns \meta{true} if the -% token is an octal digit, and \meta{false} otherwise. -% It has the additional side-effect of updating the value of -% \cs{l_str_char_int} (number formed in base $8$ from the digits -% read so far). +% \begin{macro}[aux, rEXP]{\str_aux_octal_use:NTF} +% \TeX{} dutifully detects octal digits for us: if |#1| +% is an octal digit, then the right-hand side of the +% comparison is |'1#1|, greater than $1$. Otherwise, +% the right-hand side stops as |'1|, and the conditional +% takes the \texttt{false} branch. % \begin{macrocode} -\prg_new_protected_conditional:Npnn \str_aux_octal_test:N #1 { TF } +\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF } { - \if_num:w \c_seven < '1 \exp_not:N #1 \exp_stop_f: - \int_set:Nn \l_str_char_int - { \c_eight * \l_str_char_int + '#1 } - \prg_return_true: + \if_num:w \c_one < '1 #1 \exp_stop_f: + #1 \prg_return_true: \else: \prg_return_false: \fi: @@ -1270,17 +1379,154 @@ % \end{macrocode} % \end{macro} % -% \begin{macro}{\str_aux_char_if_octal_digit:NTF} -% ^^A to be removed. +% \begin{macro}[aux, rEXP]{\str_aux_hexadecimal_use:NTF} +% \TeX{} detects uppercase hexadecimal digits for us +% (see \cs{str_aux_octal_use:NTF}, but not the +% lowercase letters, which we need to detect and replace +% by their uppercase counterpart. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_aux_char_if_octal_digit:NTF #1 - { \tl_if_in:nnTF { 01234567 } {#1} } +\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF } + { + \if_num:w \c_two < "1 #1 \exp_stop_f: + #1 \prg_return_true: + \else: + \if_case:w \int_eval:w `#1 - `a \int_eval_end: + A \prg_return_true: + \or: B \prg_return_true: + \or: C \prg_return_true: + \or: D \prg_return_true: + \or: E \prg_return_true: + \or: F \prg_return_true: + \else: \prg_return_false: + \fi: + \fi: + } % \end{macrocode} % \end{macro} % -% \subsection{Conversions} +% \subsubsection{Producing one byte or character} % -% \subsubsection{Simple unescaping} +% \begin{variable}{\c_str_byte_0_tl} +% \begin{variable}{\c_str_byte_-1_tl} +% For each integer $N$ in the range $[0,255]$, we create +% a constant token list which holds three character tokens +% with category code other: the character with character +% code $N$, followed by the representation of $N$ as +% two hexadecimal digits. +% The value $-1$ is given a default token list which ensures +% that later functions give an empty result for the input $-1$. +% \begin{macrocode} +\group_begin: + \char_set_catcode_other:n { \c_zero } + \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } } + \tl_map_inline:Nn \g_str_result_tl + { \char_set_lccode:nn {`#1} {`#1} } + \tl_map_inline:Nn \g_str_result_tl + { + \tl_map_inline:Nn \g_str_result_tl + { + \char_set_lccode:nn { \c_zero } {"##1#1} + \tl_to_lowercase:n + { + \tl_const:cx + { c_str_byte_ \int_eval:n {"##1#1} _tl } + { ^^@ ##1 #1 } + } + } + } +\group_end: +\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } } +% \end{macrocode} +% \end{variable} +% \end{variable} +% +% \begin{macro}[int, EXP]{\str_output_byte:n} +% \begin{macro}[aux, EXP]{\str_output_byte:w} +% \begin{macro}[int, EXP]{\str_output_hexadecimal:n} +% \begin{macro}[aux, EXP]{\str_output_hexadecimal:w} +% \begin{macro}[aux, EXP]{\str_output_end:} +% For now, no error detection, we simply pick either +% the byte or the hexadecimal representation of it +% from the three-character token list corresponding +% to the argument. +% The value $-1$ produces an empty result. +% \begin{macrocode} +\cs_new:Npn \str_output_byte:n #1 + { \str_output_byte:w #1 \str_output_end: } +\cs_new_nopar:Npn \str_output_byte:w + { + \exp_after:wN \exp_after:wN + \exp_after:wN \use_i:nnn + \cs:w c_str_byte_ \int_use:N \int_eval:w + } +\cs_new:Npn \str_output_hexadecimal:n #1 + { \str_output_hexadecimal:w #1 \str_output_end: } +\cs_new_nopar:Npn \str_output_hexadecimal:w + { + \exp_after:wN \exp_after:wN + \exp_after:wN \use_none:n + \cs:w c_str_byte_ \int_use:N \int_eval:w + } +\cs_new_nopar:Npn \str_output_end: + { \int_eval_end: _tl \cs_end: } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsubsection{}%^^A here +% +% \begin{macro}{\str_aux_gmap_result:N} +% \begin{macro}[aux, rEXP]{\str_aux_gmap_result_loop:NN} +% +% \begin{macrocode} +\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_gmap_result_loop:NN + \exp_after:wN #1 + \g_str_result_tl { ? \prg_map_break: } + \prg_break_point:n { } + } + } +\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2 + { + \use_none:n #2 + #1#2 + \str_aux_gmap_result_loop:NN #1 + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}{\str_aux_gmap_internal_result:N} +% \begin{macro}[aux, rEXP]{\str_aux_gmap_internal_result_loop:Nw} +% +% \begin{macrocode} +\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_gmap_internal_result_loop:Nw + \exp_after:wN #1 + \g_str_result_tl \q_stop \prg_map_break: , + \prg_break_point:n { } + } + } +\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2, + { + \use_none_delimit_by_q_stop:w #2 \q_stop + #1 {#2} + \str_aux_gmap_internal_result_loop:Nw #1 + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsubsection{Unescape user input} % % The code of this section is used both here for \cs{str_(g)input:Nn}, % and in the regular expression module to go through the regular expression @@ -1340,7 +1586,7 @@ \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 } \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 } \int_set:Nn \tex_escapechar:D { 92 } - \tl_gset:Nx \g_str_result_tl { \tl_to_other_str:n {#4} } + \str_aux_gset_other:Nn \g_str_result_tl {#4} \tl_gset:Nx \g_str_result_tl { \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl @@ -1472,7 +1718,7 @@ \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1 { \exp_not:N \str_aux_escape_x_end: } { - \msg_error:nn { str } { x-missing-brace } + \msg_kernel_error:nn { str } { x-missing-brace } \exp_not:N \str_aux_escape_x_end: #1 } } @@ -1480,11 +1726,9 @@ \char_set_catcode_other:N \^^@ \cs_new_protected_nopar:Npn \str_aux_escape_x_end: { - \group_begin: - \char_set_lccode:nn { \c_zero } { \l_str_char_int } + \tex_lccode:D \c_zero \l_str_char_int \tl_to_lowercase:n { - \group_end: \tl_gput_right:Nx \g_str_result_tl { \if_false: } \fi: \str_aux_escape_raw:N ^^@ @@ -1501,577 +1745,2440 @@ % \end{macro} % \end{macro} % -% \subsubsection{Escape and unescape strings for PDF use} +% \subsubsection{Framework for encoding conversions} % -% \begin{macro}{\str_aux_convert_store:} -% \begin{macro}{\str_aux_convert_store:NNn} -% \begin{macrocode} -\group_begin: - \char_set_catcode_other:n { `\^^@ } - \cs_new_protected_nopar:Npn \str_aux_convert_store: - { - \char_set_lccode:nn { \c_zero } { \l_str_char_int } - \tl_to_lowercase:n { \tl_gput_right:Nx \g_str_result_tl {^^@} } - } - \cs_new_protected_nopar:Npn \str_aux_convert_store:NNn #1#2#3 - { - \char_set_lccode:nn { \c_zero } {#3} - \tl_to_lowercase:n { #1 #2 {^^@} } - } -\group_end: -% \end{macrocode} -% \end{macro} -% \end{macro} +% Most functions in this module expect to be working with +% \enquote{native} strings. Strings can also be stored as +% bytes, in one of many encodings, for instance \textsc{utf8}. +% The bytes themselves can be expressed as character +% tokens with the relevant character code, or as pairs +% of hexadecimal digits, \emph{etc.} The questions of going +% from arbitrary characters to bytes, and from bytes to \TeX{} +% tokens are mostly independent. % -% \begin{macro}{\str_aux_byte_to_hexadecimal:N} -% \begin{macro}{\str_aux_byte_to_octal:N} +% Conversions are done in four steps: +% \begin{itemize} +% \item \enquote{input} produces a string of bytes; +% \item \enquote{from} interprets the byte string in the old encoding, +% and converts it to a comma-list, each character becoming +% \enquote{\ttfamily \meta{character code},}; +% \item \enquote{to} encodes the internal comma list as a byte string +% in the new encoding; +% \item \enquote{output} escapes bytes as requested. +% \end{itemize} +% The process is modified in case one of the encoding is +% \texttt{internal} or \texttt{native}: then the input or +% output step is ignored. +% +% \begin{macro}{\str_set_convert:Nnnn, \str_gset_convert:Nnnn} +% \begin{macro}[aux]{\str_convert_aux_i:NNnnn} +% \begin{macro}[aux]{\str_convert_aux_ii:wwwnn} +% \begin{macro}[aux]{\str_convert_aux_iii:nnF} +% \begin{macro}[aux]{\str_convert_aux_iv:nnnF} +% \begin{macro}[aux]{\str_convert_aux_v:NNnNN} +% The input string is stored in \cs{g_str_result_tl}, +% then we call the various conversion functions, which all +% act on \cs{g_str_result_tl}. The arguments |#3| and |#4| +% of the \texttt{aux_i} function are split into their +% \meta{encoding} and \meta{escaping} parts by \texttt{aux_ii}. +% The conversion function names are built by \texttt{aux_iii}, +% which first tries to use the name as supplied by the user, +% then filters it with \cs{str_aux_lowercase_alphanum:n}, +% and calls \texttt{aux_iv} to load the relevant definition file. +% In all cases, \texttt{aux_iii} ensures that the two +% required conversion functions are defined, with +% a fall-back to the native encoding with no escaping. +% Once this is done, \texttt{aux_v} does the main work: +% \begin{itemize} +% \item |#1| is the encoding conversion function; +% \item |#2| is the escaping function; +% \item |#3| is the escaping name; +% \item |#4| is the \enquote{native} encoding function +% (either \enquote{from} or \enquote{to}); +% \item |#5| is \cs{use_i:nn} or \cs{use_ii:nn}. +% \end{itemize} +% When converting to the internal format, the escaping must be done +% first, then the encoding (this is controlled by |#5|). +% If the encoding is native, then the escaping is ignored and +% must be empty. Otherwise, the escaping |#2| is performed. % \begin{macrocode} -\cs_new_nopar:Npn \str_aux_byte_to_hexadecimal:N #1 +\cs_new_protected:Npn \str_set_convert:Nnnn + { \str_convert_aux_i:NNnnn \str_set:Nx } +\cs_new_protected:Npn \str_gset_convert:Nnnn + { \str_convert_aux_i:NNnnn \str_gset:Nx } +\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5 { - \int_compare:nNnTF {`#1} < { 256 } + \group_begin: + \str_aux_gset_other:Nn \g_str_result_tl {#5} + + \exp_after:wN \str_convert_aux_ii:wwwnn + \tl_to_str:n {#3} /// \q_stop + { from } { input } \str_convert_from_native: \use_i:nn + + \exp_after:wN \str_convert_aux_ii:wwwnn + \tl_to_str:n {#4} /// \q_stop + { to } { output } \str_convert_to_native: \use_ii:nn + + \group_end: + #1 #2 \g_str_result_tl + } +\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5 + { + \str_convert_aux_iii:nnF {#4} {#1} {native} + \str_convert_aux_iii:nnF {#5} {#2} { } + \exp_args:Ncc \str_convert_aux_v:NNnNN + { str_convert_#4_#1: } { str_convert_#5_#2: } {#2} + } +\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3 + { + \cs_if_exist:cF { str_convert_#1_#2: } { - \int_to_letter:n { \int_div_truncate:nn {`#1} \c_sixteen } - \int_to_letter:n { \int_mod:nn {`#1} \c_sixteen } + \exp_args:Nx \str_convert_aux_iv:nnnF + { \str_aux_lowercase_alphanum:n {#2} } + {#1} {#2} {#3} } - { \msg_expandable_error:n { Invalid~byte~`#1'. } } } -\cs_new_nopar:Npn \str_aux_byte_to_octal:N #1 +\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4 { - \int_compare:nNnTF {`#1} < { 64 } + \cs_if_exist:cF { str_convert_#2_#1: } { - 0 - \int_to_letter:n { \int_div_truncate:nn {`#1} \c_eight } - \int_to_letter:n { \int_mod:nn {`#1} \c_eight } + \group_begin: + \str_load_catcodes: + \str_load_one:n {#1} + \group_end: } + \cs_if_exist:cTF { str_convert_#2_#1: } + { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } } { - \int_compare:nNnTF {`#1} < { 256 } - { \int_to_octal:n {`#1} } - { \msg_expandable_error:n { Invalid~byte~`#1'. } } + \msg_kernel_error:nnx { str } { unknown-#2 } {#3} + \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: } } } +\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5 + { + #5 { } {#1} + \cs_if_eq:NNTF #1 #4 + { + \tl_if_empty:nF {#3} + { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} } + } + { #2 } + #5 {#1} { } + } % \end{macrocode} % \end{macro} % \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}{\str_bytes_escape_hexadecimal:NN} +% \begin{macro}[int, rEXP]{\str_aux_lowercase_alphanum:n} +% \begin{macro}[aux, rEXP]{\str_aux_lowercase_alphanum_loop:N} +% This function keeps only letters and digits, with upper case +% letters converted to lower case. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_bytes_escape_hexadecimal:NN #1#2 +\cs_new:Npn \str_aux_lowercase_alphanum:n #1 { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_aux_byte_to_hexadecimal:N } + \exp_after:wN \str_aux_lowercase_alphanum_loop:N + \tl_to_str:n {#1} { ? \prg_map_break: } + \prg_break_point:n { } + } +\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1 + { + \use_none:n #1 + \if_num:w `#1 < \c_ninety_one + \if_num:w `#1 < \c_sixty_five + \if_num:w \c_one < 1#1 \exp_stop_f: + #1 + \fi: + \else: + \str_output_byte:n { `#1 + \c_thirty_two } + \fi: + \else: + \if_num:w `#1 < \c_one_hundred_twenty_three + \if_num:w `#1 < \c_ninety_seven + \else: + #1 + \fi: + \fi: + \fi: + \str_aux_lowercase_alphanum_loop:N } % \end{macrocode} % \end{macro} +% \end{macro} % -% \begin{macro}{\str_bytes_escape_name:NN} +% \begin{macro}[int]{\str_convert_from_internal:} +% \begin{macro}[int]{\str_convert_to_internal:} +% Converting from the internal format to itself is +% of course trivial. % \begin{macrocode} -\tl_const:Nx \c_str_bytes_escape_name_str - { \c_hash_str \c_percent_str \c_lbrace_str \c_rbrace_str ()/<>[] } -\cs_new_protected_nopar:Npn \str_bytes_escape_name:NN #1#2 +\cs_new_protected_nopar:Npn \str_convert_from_internal: { } +\cs_new_protected_nopar:Npn \str_convert_to_internal: { } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsubsection{Loading encoding files} +% +% \begin{macro}{\str_load:n} +% \begin{macro}[int]{\str_load_one:n} +% \begin{macro}[aux]{\str_load_alias:n, \str_load_alias_aux:nnn} +% Loading encoding or escaping files can be done in bulk by +% specifying a comma-list of encodings and escapings. Each +% name is normalized: only alphanumeric characters are kept, +% lowercased. The relevant file is loaded if it was not loaded +% already. +% \begin{macrocode} +\cs_new_protected:Npn \str_load:n #1 { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_bytes_escape_name_aux:N } + \group_begin: + \str_load_catcodes: + \clist_map_inline:nn {#1} + { + \exp_args:Nx \str_load_one:n + { \str_aux_lowercase_alphanum:n {#1} } + } + \group_end: } -\cs_new_nopar:Npn \str_bytes_escape_name_aux:N #1 +\cs_new_protected:Npn \str_load_one:n #1 { - \int_compare:nNnTF {`#1} < { "21 } - { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 } + \cs_if_exist:cF { str_convert_from_ #1 : } { - \int_compare:nNnTF {`#1} > { "7E } - { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 } + \cs_if_exist:cF { str_convert_input_#1: } { - \str_if_contains_char:NNTF \c_str_bytes_escape_name_str #1 - { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 } - {#1} + \file_if_exist:nTF { l3str- #1 .def } + { \file_input:n { l3str- #1 .def } } + { \str_load_alias:n {#1} } } } } +\cs_new_protected:Npn \str_load_alias:n #1 + { + \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl + { + \str_load_one:n { \l_str_tmpa_tl } + \cs_if_exist:cTF + { str_convert_from_ \l_str_tmpa_tl : } + { + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from } + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to } + } + { + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input } + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output } + } + } + { \msg_kernel_error:nnx { str } { file-not-found } {#1} } + } +\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3 + { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}{\str_bytes_escape_string:NN} -% Any character below (and including) space, and any character -% above (and including) \texttt{del}, are converted to octal. -% One backslash is added before each parenthesis and backslash. +% \begin{macro}[int]{\str_load_catcodes:} +% Since encoding files may be loaded at arbitrary places +% in a \TeX{} document, including within verbatim mode, +% we set the catcodes of all characters appearing in any +% encoding definition file. % \begin{macrocode} -\tl_const:Nx \c_str_bytes_escape_string_str { \c_backslash_str ( ) } -\cs_new_protected_nopar:Npn \str_bytes_escape_string:NN #1#2 +\cs_new_protected:Npn \str_load_catcodes: { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_bytes_escape_string_aux:N } + \char_set_catcode_escape:N \\ + \char_set_catcode_group_begin:N \{ + \char_set_catcode_group_end:N \} + \char_set_catcode_math_superscript:N \^ + \char_set_catcode_ignore:N \ % + \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN } + \char_set_catcode_letter:N + \tl_map_function:nN { 0123456789"'?*+ } + \char_set_catcode_other:N + \char_set_catcode_comment:N \% + \int_set:Nn \tex_endlinechar:D {32} } -\cs_new_nopar:Npn \str_bytes_escape_string_aux:N #1 +% \end{macrocode} +% \end{macro} +% +%^^A todo: document, move. +% \begin{macrocode} +\int_new:N \l_str_encoding_int +% \end{macrocode} +% +% \begin{macro}[int]{\str_convert_from_eight_bit:n} +% \begin{macro}[aux]{\str_convert_from_eight_bit_load:nn} +% \begin{macro}[aux]{\str_convert_from_eight_bit_load_missing:n} +% \begin{macro}[aux, EXP]{\str_convert_from_eight_bit_aux:N} +%^^A todo: document +% \begin{macrocode} +\cs_new_protected:Npn \str_convert_from_eight_bit:n #1 { - \int_compare:nNnTF {`#1} < { "21 } - { \c_backslash_str \str_aux_byte_to_octal:N #1 } - { - \int_compare:nNnTF {`#1} > { "7E } - { \c_backslash_str \str_aux_byte_to_octal:N #1 } - { - \str_if_contains_char:NNT \c_str_bytes_escape_string_str #1 - { \c_backslash_str } - #1 - } - } + \group_begin: + \int_zero:N \l_str_encoding_int + \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn + { \tl_use:c { c_str_encoding_#1_tl } } + { \q_stop \prg_map_break: } { } + \prg_break_point:n { } + \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n + { \tl_use:c { c_str_encoding_#1_missing_tl } } + { \q_stop \prg_map_break: } + \prg_break_point:n { } + \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N + \group_end: + } +\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2 + { + \use_none_delimit_by_q_stop:w #1 \q_stop + \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: + \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 } + \tex_advance:D \l_str_encoding_int \c_one + \str_convert_from_eight_bit_load:nn + } +\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1 + { + \use_none_delimit_by_q_stop:w #1 \q_stop + \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: + \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD } + \tex_advance:D \l_str_encoding_int \c_one + \str_convert_from_eight_bit_load_missing:n + } +\cs_new:Npn \str_convert_from_eight_bit_aux:N #1 + { + \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int + \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f: + \tex_the:D \tex_toks:D \tex_dimen:D + \fi: + \fi: + \int_value:w `#1 , } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} % -% \begin{macro}{\str_bytes_unescape_hexadecimal:NN} -% \begin{macro}[aux]{\str_bytes_unescape_hexadecimal_aux:N} -% \begin{macro}[aux]{\str_bytes_unescape_hexadecimal_aux_ii:N} -% Takes chars two by two, and interprets each pair as a hexadecimal code -% for a character. Any non-hexadecimal-digit is ignored. An odd-length -% string gets a |0| appended to it (this is equivalent to appending a |0| -% in all cases, and dropping it if it is alone). +% \begin{macro}[int]{\str_convert_to_eight_bit:n} +% \begin{macro}[aux]{\str_convert_to_eight_bit_load:nn} +% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux:n} +% \begin{macro}[aux, rEXP]{\str_convert_to_eight_bit_aux_ii:n} +%^^A todo: document % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal:NN #1#2 +\cs_new_protected:Npn \str_convert_to_eight_bit:n #1 { \group_begin: - \tl_gclear:N \g_str_result_tl - \int_zero:N \l_str_char_int - \exp_last_unbraced:Nf \str_bytes_unescape_hexadecimal_aux:N - { \tl_to_other_str:N #2 } 0 - \q_recursion_tail \q_recursion_stop + \int_zero:N \l_str_encoding_int + \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn + { \tl_use:c { c_str_encoding_#1_tl } } + { \q_stop \prg_map_break: } { } + \prg_break_point:n { } + \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n \group_end: - \tl_set_eq:NN #1 \g_str_result_tl } -\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal_aux:N #1 +\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2 { - \quark_if_recursion_tail_stop:N #1 - \str_aux_hexadecimal_test:NTF #1 - { \str_bytes_unescape_hexadecimal_aux_ii:N } - { \str_bytes_unescape_hexadecimal_aux:N } + \use_none_delimit_by_q_stop:w #1 \q_stop + \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop: + \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop: + \exp_args:NNf \tex_toks:D \l_str_encoding_int + { \str_output_byte:n { "#1 } } + \tex_advance:D \l_str_encoding_int \c_one + \str_convert_to_eight_bit_load:nn } -\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal_aux_ii:N #1 +\cs_new:Npn \str_convert_to_eight_bit_aux:n #1 { - \quark_if_recursion_tail_stop:N #1 - \str_aux_hexadecimal_test:NTF #1 + \if_num:w #1 < "8000 \exp_stop_f: + \if_num:w \tex_dimen:D #1 < \l_str_encoding_int + \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f: + \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f: + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn + \fi: + \fi: + \str_convert_to_eight_bit_aux_ii:n {#1} + \else: + \msg_expandable_kernel_error:nnn + { str } { eight_bit-to-byte-overflow } {#1} + \fi: + } +\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1 + { + \if_num:w #1 < \c_two_hundred_fifty_six + \str_output_byte:n {#1} + \else: + \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte } + \fi: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_encoding_eight_bit:n} +%^^A todo: document +% \begin{macrocode} +\cs_new_protected:Npn \str_encoding_eight_bit:n #1 + { + \cs_new_protected:cpn { str_convert_from_#1: } + { \str_convert_from_eight_bit:n {#1} } + \cs_new_protected:cpn { str_convert_to_#1: } + { \str_convert_to_eight_bit:n {#1} } + } +% \end{macrocode} +% \end{macro} +% +% \subsubsection{Byte input and output} +% +% Strings of bytes may need to be stored in auxiliary files +% in safe \enquote{escaping} formats. Each such escaping +% is only loaded as needed. By default, on input any non-byte +% is filtered out, while the output simply consists in letting +% bytes through. +% +% \begin{macro}[int, rEXP]{\str_filter_bytes:n} +% \begin{macro}[aux, rEXP]{\str_filter_bytes_aux:N} +% In the case of pdf\TeX{}, every character is a byte. +% For Unicode-aware engines, test the character code. +% Spaces have already been given the correct category +% code when this function is called. +% \begin{macrocode} +\pdftex_if_engine:TF + { \cs_new_eq:NN \str_filter_bytes:n \use:n } + { + \cs_new_nopar:Npn \str_filter_bytes:n #1 + { + \str_filter_bytes_aux:N #1 + { ? \prg_map_break: } + \prg_break_point:n { } + } + \cs_new_nopar:Npn \str_filter_bytes_aux:N #1 { - \str_aux_convert_store: - \int_zero:N \l_str_char_int - \str_bytes_unescape_hexadecimal_aux:N + \use_none:n #1 + \if_num:w `#1 < 256 \exp_stop_f: #1 \fi: + \str_filter_bytes_aux:N } - { \str_bytes_unescape_hexadecimal_aux_ii:N } } % \end{macrocode} % \end{macro} % \end{macro} +% +% \begin{macro}[int]{\str_convert_input_:} +% \begin{macro}[int]{\str_convert_input_bytes:} +% The simplest input method removes non-bytes +% from \cs{g_str_result_tl}. +% \begin{macrocode} +\pdftex_if_engine:TF + { \cs_new_protected_nopar:Npn \str_convert_input_: { } } + { + \cs_new_protected_nopar:Npn \str_convert_input_: + { + \tl_gset:Nx \g_str_result_tl + { \exp_args:No \str_filter_bytes:n \g_str_result_tl } + } + } +\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_: +% \end{macrocode} +% \end{macro} % \end{macro} % -% \begin{macro}{\str_bytes_unescape_name:NN} -% \begin{macro}[aux]{\str_bytes_unescape_name_aux:wN} -% \begin{macro}[aux]{\str_bytes_unescape_name_aux_ii:N} -% This changes all occurrences of |#| followed by two upper- or lowercase -% hexadecimal digits by the corresponding unescaped character. +% \begin{macro}[int]{\str_convert_output_:} +% \begin{macro}[int]{\str_convert_output_bytes:} +% The simplest form of output leaves the bytes from the previous +% step of the conversion unchanged. % \begin{macrocode} +\cs_new_protected_nopar:Npn \str_convert_output_: { } +\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_: +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsubsection{Convert to and from native strings} +% +% \begin{macro}[int]{\str_convert_from_native:} +% \begin{macro}[aux, rEXP]{\str_convert_from_native_aux:N} +% Convert each character to its character code, one at a time. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \str_convert_from_native: + { \str_aux_gmap_result:N \str_convert_from_native_aux:N } +\cs_new:Npn \str_convert_from_native_aux:N #1 + { \int_value:w `#1 , } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_to_native:} +% \begin{macro}[aux]{\str_convert_to_native_aux:w} +% Convert the string from the given encoding +% to the internal format. We grab integers +% from the internal string using the low-level +% method to assign integers. +% Except at the end of the loop, the function +% \cs{str_convert_to_native_aux:w} is followed by a +% comma, hence its argument is empty. +% We are making heavy use of \TeX{} primitives here, +% because the function can become very slow for large +% strings. +% \begin{macrocode} +\cs_new_protected_nopar:Npn \str_convert_to_native: + { + \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl + \tl_gclear:N \g_str_result_tl + \tex_afterassignment:D \str_convert_to_native_aux:w + \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: , + \prg_break_point:n { } + } \group_begin: - \char_set_lccode:nn {`\*} {`\#} - \tl_to_lowercase:n + \char_set_catcode_other:n { `\^^@ } + \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 , { - \group_end: - \cs_new_protected_nopar:Npn \str_bytes_unescape_name:NN #1#2 - { - \group_begin: - \tl_gclear:N \g_str_result_tl - \int_zero:N \l_str_char_int - \exp_last_unbraced:Nf \str_bytes_unescape_name_aux:wN - { \tl_to_other_str:N #2 } - * \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } - \cs_new_protected_nopar:Npn \str_bytes_unescape_name_aux:wN #1 * #2 - { - \tl_gput_right:Nx \g_str_result_tl {#1} - \quark_if_recursion_tail_stop:N #2 - \str_aux_hexadecimal_test:NTF #2 - { \str_bytes_unescape_name_aux_ii:NN #2 } - { - \tl_gput_right:Nx \g_str_result_tl { \c_hash_str } - \str_bytes_unescape_name_aux:wN #2 - } - } + #1 + \if_int_compare:w \l_str_char_int > \c_max_char_int + \msg_kernel_error:nnx { str } { overflow } + { \int_use:N \l_str_char_int } + \else: + \tex_lccode:D \c_zero \l_str_char_int + \tl_to_lowercase:n + { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } } + \fi: + \tex_afterassignment:D \str_convert_to_native_aux:w + \l_str_char_int = } -\cs_new_protected_nopar:Npn \str_bytes_unescape_name_aux_ii:NN #1#2 +\group_end: +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Messages} +% +% \begin{macrocode} +\msg_kernel_new:nnnn { str } { x-missing-brace } + { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. } + { + You~wrote~something~like~ + `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~ + The~closing~brace~is~missing. + } +\msg_kernel_new:nnnn { str } { overflow } + { Character~code~#1~too~big. } { - \str_aux_hexadecimal_test:NTF #2 + \int_compare:nNnTF {#1} > { 1114111 } + { The~Unicode~standard~limits~code~points~to~1114111. } { - \str_aux_convert_store: - \int_zero:N \l_str_char_int - \str_bytes_unescape_name_aux:wN + The~pdfTeX~engine~only~supports~8-bit~characters:~ + valid~character~codes~are~in~the~range~[0,255].~ + To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX. } + } +\msg_kernel_new:nnn { str } { convert-input } + { Input~scheme~`#1'~unknown. } +\msg_kernel_new:nnn { str } { convert-from } + { Encoding~`#1'~unknown. } +\msg_kernel_new:nnn { str } { convert-to } + { Encoding~`#1'~unknown.~Using~UTF-8. } +\msg_kernel_new:nnn { str } { convert-output } + { Output~scheme~`#1~unknown. } +\msg_kernel_new:nnn { str } { to-native } + { Use~\str_set_convert:Nnn to~convert~to~native~strings. } +\msg_kernel_new:nnn { str } { unicode-surrogate } + { Code~point~#1~is~an~unpaired~surrogate. } +\msg_kernel_new:nnn { str } { utf16-surrogate } + { UTF-16~unpaired~surrogate~#1. } +\msg_kernel_new:nnn { str } { utf16-odd } + { UTF-16~dangling~byte~#1. } +\msg_kernel_new:nnn { str } { utf8-extra-conti } + { UTF-8~extra~continuation~byte~#1. } +\msg_kernel_new:nnn { str } { utf8-missing-conti } + { UTF-8~missing~continuation~byte~#1. } +\msg_kernel_new:nnn { str } { utf8-invalid-byte } + { Byte~#1~cannot~appear~in~UTF-8. } +\msg_kernel_new:nnn { str } { utf8-overlong } + { Overlong~UTF-8~byte~sequence~for~code~point~#1. } +\msg_kernel_new:nnn { str } { utf8-premature-end } + { Incomplete~last~UTF-8~character. } +\msg_kernel_new:nnn { str } { utf32-overflow } + { Code~point~too~large~(UTF-32~`#1'). } +\msg_kernel_new:nnn { str } { utf32-truncated } + { Truncated~UTF-32~string~`...#1'. } +% \end{macrocode} +% +% \subsection{Deprecated string functions} +% +% \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n} +% The naming scheme is a little bit more consistent +% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}. +% \begin{macrocode} +\cs_set:Npn \str_length_skip_spaces:N + { \exp_args:No \str_length_skip_spaces:n } +\cs_set_eq:NN \str_length_skip_spaces:n \str_length_ignore_spaces:n +% \end{macrocode} +% \end{macro} +% +% \begin{macrocode} +%</initex|package> +% \end{macrocode} +% +% \subsection{Escaping definition files} +% +% Several of those encodings are defined by the pdf file format. +% The following byte storage methods are defined: +% \begin{itemize} +% \item \texttt{bytes} (default), non-bytes are filtered out, +% and bytes are left untouched; +% \item \texttt{hex} or \texttt{hexadecimal}, +% as per the pdf\TeX{} primitive \tn{pdfescapehex} +% \item \texttt{name}, as per the pdf\TeX{} primitive \tn{pdfescapename} +% \item \texttt{string}, as per the pdf\TeX{} primitive \tn{pdfescapestring} +% \item \texttt{url} or \texttt{percent}, as per the percent encoding of urls. +% \end{itemize} +%^^A todo: in pdfTeX, use the \tn{pdfescape...} +%^^A todo: add various shell escapes, lua escapes etc? +% +% \subsubsection{Input methods} +% +% \begin{macro}[int]{\str_convert_input_hex:} +% \begin{macro}[int]{\str_convert_input_hexadecimal:} +% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux:N} +% \begin{macro}[aux, rEXP]{\str_convert_input_hex_aux_ii:N} +% Take chars two by two, and interpret each pair as +% the hexadecimal code for a byte. Anything else than +% hexadecimal digits is ignored. +% A string which contains an odd number of hexadecimal +% digits gets |0| appended to it: this is equivalent +% to appending a |0| in all cases, and dropping it if +% it is alone. +% \begin{macrocode} +%<*hex> +\cs_new_protected_nopar:Npn \str_convert_input_hex: + { + \tl_gset:Nx \g_str_result_tl { - \tl_gput_right:Nx \g_str_result_tl { \c_hash_str #1 } - \int_zero:N \l_str_char_int - \str_bytes_unescape_name_aux:wN #2 + \str_output_byte:w " + \exp_after:wN \str_convert_input_hex_aux:N + \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } + \prg_break_point:n { \str_output_end: } } } +\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1 + { + \use_none:n #1 + \str_aux_hexadecimal_use:NTF #1 + \str_convert_input_hex_aux_ii:N + \str_convert_input_hex_aux:N + } +\cs_new_nopar:Npn \str_convert_input_hex_aux_ii:N #1 + { + \use_none:n #1 + \str_aux_hexadecimal_use:NTF #1 + { + \str_output_end: + \str_output_byte:w " \str_convert_input_hex_aux:N + } + \str_convert_input_hex_aux_ii:N + } +%</hex> % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} +% \end{macro} % -% \begin{macro}{\str_bytes_unescape_string:NN} -% Here, we need to detect backslashes, which escape characters -% as follows. -% \begin{itemize} -% \item[\texttt{\string\n}] Line feed ($10$) -% \item[\texttt{\string\r}] Carriage return ($13$) -% \item[\texttt{\string\t}] Horizontal tab ($9$) -% \item[\texttt{\string\b}] Backspace ($8$) -% \item[\texttt{\string\f}] Form feed ($12$) -% \item[\texttt{(}] Left parenthesis -% \item[\texttt{)}] Right parenthesis -% \item[\texttt{\string\\}] Backslash -% \item[\texttt{\string\ddd}] Character code \texttt{ddd} (octal) +% \begin{macro}[int]{\str_convert_input_name:} +% \begin{macro}[aux, rEXP]{\str_convert_input_name_aux:wNN} +% \begin{macro}[int]{\str_convert_input_url:} +% \begin{macro}[int]{\str_convert_input_percent:} +% \begin{macro}[aux, rEXP]{\str_convert_input_url_aux:wNN} +% The \cs{str_convert_input_name:} function replaces each +% occurrence of |#| followed by two hexadecimal digits +% in \cs{g_str_result_tl} by the corresponding byte. +% The \texttt{url} function is identical, with escape +% character |%| instead of |#|. Thus we define the two +% together. The arguments of \cs{str_tmp:w} are the character +% code of |#| or |%| in hexadecimal, the name of the main +% function to define, and the name of the auxiliary which +% performs the loop. +% +% The looping auxiliary |#3| finds the next escape character, +% reads the following two characters, and tests them. The test +% \cs{str_aux_hexadecimal_use:NTF} leaves the upper-case digit +% in the input stream, hence we surround the test with +% \cs{str_output_byte:w}~|"| and \cs{str_output_end:}. +% If both characters are hexadecimal digits, they should be +% removed before looping: this is done by \cs{use_i:nnn}. +% If one of the characters is not a hexadecimal digit, +% then feed |"#1| to \cs{str_output_byte:w} to produce the +% escape character, and call the looping function followed +% by the two characters (remove \cs{use_i:nnn}). +% \begin{macrocode} +%<*name|url> +\cs_set_protected:Npn \str_tmp:w #1#2#3 + { + \cs_new_protected_nopar:Npn #2 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN #3 \g_str_result_tl + #1 ? { ? \prg_map_break: } + \prg_break_point:n { } + } + } + \cs_new_nopar:Npn #3 ##1#1##2##3 + { + \str_filter_bytes:n {##1} + \use_none:n ##3 + \str_output_byte:w " + \str_aux_hexadecimal_use:NTF ##2 + { + \str_aux_hexadecimal_use:NTF ##3 + { } + { * \c_zero + `#1 \use_i:nn } + } + { 0 + `#1 \use_i:nn } + \str_output_end: + \use_i:nnn #3 ##2##3 + } + } +%</name|url> +%<name>\exp_after:wN \str_tmp:w \c_hash_str +%<name> \str_convert_input_name: \str_convert_input_name_aux:wNN +%<url>\exp_after:wN \str_tmp:w \c_percent_str +%<url> \str_convert_input_url: \str_convert_input_url_aux:wNN +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_input_string:} +% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wN} +% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:wNNN} +% \begin{macro}[aux, rEXP]{\str_convert_input_string_aux:NNNNNN} +% The \texttt{string} escaping is somewhat similar to +% the \texttt{name} and \texttt{url} escapings, with +% escape character |\|. +% The first step is to convert all three line endings, +% |^^J|, |^^M|, and |^^M^^J| to the common |^^J|, as per +% the \textsc{pdf} specification. +% Then the following escape sequences are decoded. +% \begin{itemize}\def\makelabel#1{\hss\llap{\ttfamily\string#1}} +% \item[\n] Line feed ($10$) +% \item[\r] Carriage return ($13$) +% \item[\t] Horizontal tab ($9$) +% \item[\b] Backspace ($8$) +% \item[\f] Form feed ($12$) +% \item[\(] Left parenthesis +% \item[\)] Right parenthesis +% \item[\\] Backslash +% \item[\ddd] (backslash followed by $1$ to $3$ octal digits) +% Byte \texttt{ddd} (octal), subtracting $256$ in case of overflow. % \end{itemize} % If followed by an end-of-line character, the backslash and % the end-of-line are ignored. If followed by anything else, % the backslash is ignored. -% The PDF specification indicates that LF, CR, and CRLF should be -% converted to LF: \emph{this is not implemented here}. % \begin{macrocode} +%<*string> \group_begin: \char_set_lccode:nn {`\*} {`\\} \char_set_catcode_other:N \^^J \char_set_catcode_other:N \^^M \tl_to_lowercase:n { - \group_end: - \cs_new_protected_nopar:Npn \str_bytes_unescape_string:NN #1#2 + \cs_new_protected_nopar:Npn \str_convert_input_string: { - \group_begin: - \tl_gclear:N \g_str_result_tl - \exp_last_unbraced:Nf \str_bytes_unescape_string_aux:wN - { \tl_to_other_str:N #2 } - * \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } - \cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux:wN #1 * #2 - { - \tl_gput_right:Nx \g_str_result_tl {#1} - \quark_if_recursion_tail_stop:N #2 - \int_zero:N \l_str_char_int - \str_aux_octal_test:NTF #2 - { \str_bytes_unescape_string_aux_d:N } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wN + \g_str_result_tl \prg_map_break: ^^M ? + \prg_break_point:n { } + } + \tl_gset:Nx \g_str_result_tl { - \int_set:Nn \l_str_char_int - { - \prg_case_str:xxn {#2} - { - { n } { 10 } - { r } { 13 } - { t } { 9 } - { b } { 8 } - { f } { 12 } - { ( } { 40 } - { ) } { 41 } - { \c_backslash_str } { 92 } - { ^^J } { -1 } - { ^^M } { -1 } - } - {`#2} - } - \int_compare:nNnF \l_str_char_int = \c_minus_one - { \str_aux_convert_store: } - \str_bytes_unescape_string_aux:wN + \exp_after:wN \str_convert_input_string_aux:wNNN + \g_str_result_tl * ?? { ? \prg_map_break: } + \prg_break_point:n { } } } + \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4 } -\cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux_d:N #1 + { + \str_filter_bytes:n {#1} + \use_none:n #4 + \str_output_byte:w ' + \str_aux_octal_use:NTF #2 + { + \str_aux_octal_use:NTF #3 + { + \str_aux_octal_use:NTF #4 + { + \if_int_compare:w #2 > \c_three + - 256 + \fi: + \str_convert_input_string_aux:NNNNNN + } + { \str_convert_input_string_aux:NNNNNN ? } + } + { \str_convert_input_string_aux:NNNNNN ?? } + } + { + \prg_case_str:xxn {#2} + { + { \c_backslash_str } { 134 } + { ( } { 50 } + { ) } { 51 } + { r } { 15 } + { f } { 14 } + { n } { 12 } + { t } { 11 } + { b } { 10 } + { ^^J } { 0 - \c_one } + } + { 0 - \c_one \use_i:nn } + } + \str_output_end: + \use_i:nn \str_convert_input_string_aux:wNNN #2#3#4 + } + \cs_new_nopar:Npn \str_convert_input_string_aux:NNNNNN #1#2#3#4#5#6 + { \str_output_end: \str_convert_input_string_aux:wNNN } + \cs_new_nopar:Npn \str_convert_input_string_aux:wN #1 ^^M #2 + { + #1 ^^J + \if_charcode:w ^^J #2 + \exp_after:wN \use_i:nn + \fi: + \str_convert_input_string_aux:wN #2 + } +\group_end: +%</string> +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsubsection{Output methods} +% +% \begin{macro}[int]{\str_convert_output_hex:} +% \begin{macro}[int]{\str_convert_output_hexadecimal:} +% \begin{macro}[aux, rEXP]{\str_convert_output_hex_aux:N} +% Loop and convert each byte to hexadecimal. +% \begin{macrocode} +%<*hex> +\cs_new_protected_nopar:Npn \str_convert_output_hex: + { \str_aux_gmap_result:N \str_convert_output_hex_aux:N } +\cs_new_nopar:Npn \str_convert_output_hex_aux:N #1 + { \str_output_hexadecimal:n { `#1 } } +%</hex> +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_output_name:} +% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:N} +% \begin{macro}[aux, rEXP]{\str_convert_output_name_aux:NTF} +% \begin{variable}{\c_str_convert_output_name_str} +% \begin{variable}{\c_str_convert_output_name_not_str} +% For each byte, test whether it should +% be output as is, or be \enquote{hash-encoded}. +% Roughly, bytes outside the range from |"2A| to |"7E| +% are hash-encoded. We keep two lists of exceptions: +% characters in \cs{c_str_convert_output_name_not_str} +% are not hash-encoded, and characters in the +% \cs{c_str_convert_output_name_str} are encoded. +% \begin{macrocode} +%<*name> +\str_const:Nn \c_str_convert_output_name_not_str { ! " $ & ' } %$ +\str_const:Nn \c_str_convert_output_name_str { {}/<>[] } +\cs_new_protected_nopar:Npn \str_convert_output_name: + { \str_aux_gmap_result:N \str_convert_output_name_aux:N } +\cs_new_nopar:Npn \str_convert_output_name_aux:N #1 { - \str_aux_octal_test:NTF #1 - { \str_bytes_unescape_string_aux_dd:N } - { - \str_aux_convert_store: - \str_bytes_unescape_string_aux:wN #1 - } + \str_convert_output_name_aux:NTF #1 {#1} + { \c_hash_str \str_output_hexadecimal:n {`#1} } } -\cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux_dd:N #1 +\prg_new_conditional:Npnn \str_convert_output_name_aux:N #1 { TF } { - \str_aux_octal_test:NTF #1 + \if_num:w `#1 < "2A \exp_stop_f: + \str_if_contains_char:NNTF \c_str_convert_output_name_not_str #1 + \prg_return_true: \prg_return_false: + \else: + \if_num:w `#1 > "7E \exp_stop_f: + \prg_return_false: + \else: + \str_if_contains_char:NNTF \c_str_convert_output_name_str #1 + \prg_return_false: \prg_return_true: + \fi: + \fi: + } +%</name> +% \end{macrocode} +% \end{variable} +% \end{variable} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_output_string:} +% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:N} +% \begin{macro}[aux, rEXP]{\str_convert_output_string_aux:NTF} +% \begin{variable}{\c_str_convert_output_string_str} +% Any character below (and including) space, and any character +% above (and including) \texttt{del}, are converted to octal. +% One backslash is added before each parenthesis and backslash. +% \begin{macrocode} +%<*string> +\str_const:Nx \c_str_convert_output_string_str + { \c_backslash_str ( ) } +\cs_new_protected_nopar:Npn \str_convert_output_string: + { \str_aux_gmap_result:N \str_convert_output_string_aux:N } +\cs_new_nopar:Npn \str_convert_output_string_aux:N #1 + { + \str_convert_output_string_aux:NTF #1 { - \str_aux_convert_store: - \str_bytes_unescape_string_aux:wN + \str_if_contains_char:NNT + \c_str_convert_output_string_str #1 + { \c_backslash_str } + #1 } { - \str_aux_convert_store: - \str_bytes_unescape_string_aux:wN #1 + \c_backslash_str + \int_div_truncate:nn {`#1} {64} + \int_mod:nn { \int_div_truncate:nn {`#1} \c_eight } \c_eight + \int_mod:nn {`#1} \c_eight } } +\prg_new_conditional:Npnn \str_convert_output_string_aux:N #1 { TF } + { + \if_num:w `#1 < "21 \exp_stop_f: + \prg_return_false: + \else: + \if_num:w `#1 > "7E \exp_stop_f: + \prg_return_false: + \else: + \prg_return_true: + \fi: + \fi: + } +%</string> % \end{macrocode} +% \end{variable} +% \end{macro} +% \end{macro} % \end{macro} % -% \subsubsection{Percent encoding} +% \begin{macro}[int]{\str_convert_output_url:} +% \begin{macro}[int]{\str_convert_output_percent:} +% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:N} +% \begin{macro}[aux, rEXP]{\str_convert_output_url_aux:NTF} +%^^A todo: Figure out what bytes should be percent encoded! Safe could be +%^^A !'()*-./0123456789_ +% This function is similar to \cs{str_convert_output_name:}, +% escaping different characters. +% \begin{macrocode} +%<*url> +\cs_new_protected_nopar:Npn \str_convert_output_url: + { \str_aux_gmap_result:N \str_convert_output_url_aux:N } +\cs_new_nopar:Npn \str_convert_output_url_aux:N #1 + { + \str_convert_output_url_aux:NTF #1 {#1} + { \c_percent_str \str_output_hexadecimal:n { `#1 } } + } +\prg_new_conditional:Npnn \str_convert_output_url_aux:N #1 { TF } + { + \if_num:w `#1 < "41 \exp_stop_f: + \str_if_contains_char:nNTF { "-.<> } #1 + \prg_return_true: \prg_return_false: + \else: + \if_num:w `#1 > "7E \exp_stop_f: + \prg_return_false: + \else: + \str_if_contains_char:nNTF { [ ] } #1 + \prg_return_false: \prg_return_true: + \fi: + \fi: + } +%</url> +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \subsection{Encoding definition files} +% +% The \texttt{native} and \texttt{internal} encodings are automatically +% defined. Other encodings are loaded as needed. The following encodings +% are supported: +% \begin{itemize} +% \item \textsc{utf-8}; +% \item \textsc{utf-16}, big-endian, little-endian, or with byte order mark; +% \item \textsc{utf-32}, big-endian, little-endian, or with byte order mark; +% \item the \textsc{iso 8859} code pages, numbered from $1$ to $16$, +% skipping the inexistent \textsc{iso 8859-12}. +% \end{itemize} % -% \begin{macro}{\str_bytes_percent_encode:NN} +% \subsubsection{\textsc{utf-8} support} +% +% \begin{macrocode} +%<*utf8> +% \end{macrocode} +% +% \begin{macro}[int]{\str_convert_to_utf8:} +% \begin{macro}[aux, rEXP] +% { +% \str_aux_to_utf_viii:n, +% \str_aux_to_utf_viii:nnnnw +% } +% Loop through the internal string, and convert each character +% to the \textsc{utf-8} representation. The details are messy. +%^^A todo: document. % \begin{macrocode} -\tl_const:Nx \c_str_bytes_percent_encode_str { \tl_to_str:n { [] } } -\tl_const:Nx \c_str_bytes_percent_encode_not_str { \tl_to_str:n { "-.<> } } -\cs_new_protected_nopar:Npn \str_bytes_percent_encode:NN #1#2 +\cs_new_protected_nopar:cpn { str_convert_to_utf8: } + { \str_aux_gmap_internal_result:N \str_aux_to_utf_viii:n } +\cs_new:Npn \str_aux_to_utf_viii:n #1 { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_bytes_percent_encode_aux:N } + \str_aux_to_utf_viii:nnnnw + {#1} { \c_minus_one + \c_zero * \use_none:n } + { 128 } { \c_zero } + { 32 } { 192 } + { 16 } { 224 } + { 8 } { 240 } + \q_stop } -\cs_new_nopar:Npn \str_bytes_percent_encode_aux:N #1 +\cs_new_nopar:Npn \str_aux_to_utf_viii:nnnnw #1#2#3#4 #5 \q_stop { - \int_compare:nNnTF {`#1} < { "41 } - { - \str_if_contains_char:NNTF \c_str_bytes_percent_encode_not_str #1 - { #1 } - { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 } - } + \if_num:w #1 < #3 \exp_stop_f: + \str_output_byte:n { #1 + #4 } + \else: + \exp_args:Nf \str_aux_to_utf_viii:nnnnw + { \int_div_truncate:nn {#1} {64} } + {#1} + #5 \q_stop + \fi: + \str_output_byte:n { #2 - 64 * ( #1 - \c_two ) } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int]{\str_convert_from_utf8:} +% \begin{macro}[aux, rEXP] +% { +% \str_aux_from_utf_viii:N, +% \str_aux_from_utf_viii:wwNN, +% \str_aux_from_utf_viii:wNNww +% } +% \begin{macro}[aux, rEXP] +% { +% \str_aux_from_utf_viii_error:w, +% \str_aux_from_utf_viii_error:NwNN, +% \str_aux_from_utf_viii_error: +% } +% In the \textsc{utf-8} encoding, bytes in the range $[0,127]$ stand +% for themselves, bytes in the range $[128,191]$ are continuation bytes, +% and larger bytes must be followed by a number of continuation bytes. +% Documentation to come. +% ^^A todo: refuse "D800 -- "DFFF? +% ^^A todo: document +% ^^A todo: what error recovery for code points > "10FFFF? +% \begin{macrocode} +\cs_new_protected_nopar:cpn { str_convert_from_utf8: } + { + \tl_gset:Nx \g_str_result_tl { - \int_compare:nNnTF {`#1} > { "7E } - { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 } - { - \str_if_contains_char:NNTF \c_str_bytes_percent_encode_str #1 - { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 } - { #1 } - } + \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl + { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: } + \prg_break_point:n { } } } +\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1 + { + \use_none:n #1 + \int_use:N \int_eval:w + \if_num:w `#1 < "C0 \exp_stop_f: + \if_num:w `#1 < "80 \exp_stop_f: + `#1 + \else: + \msg_expandable_kernel_error:nnn + { str } { utf8-extra-conti } {#1} + \c_str_replacement_char_int + \fi: + \else: + \if_num:w `#1 < "F5 \exp_stop_f: + \exp_after:wN \str_aux_from_utf_viii:wwNN + \int_use:N \int_eval:w `#1 - "C0 + \else: + \msg_expandable_kernel_error:nnn + { str } { utf8-invalid-byte } {#1} + \c_str_replacement_char_int + \fi: + \fi: + , + \use_none_delimit_by_q_stop:w + "80 , "800 , "10000 , "10FFFF , + \q_stop + \str_aux_from_utf_viii:N + } +\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4 + { + \use_none_delimit_by_q_stop:w #4 \q_stop + \exp_after:wN \str_aux_from_utf_viii:wNNww + \int_use:N \int_eval:w #1 * "40 + `#4 - "80 , + #4 + #2 + \q_stop #3 + } +\cs_new_nopar:Npn \str_aux_from_utf_viii:wNNww #1, #2 #3#4, #5, + { + \if_num:w \int_eval:w ( `#2 + \c_thirty_two ) / "40 = \c_three + \if_num:w #1 < #4 \exp_stop_f: + \str_aux_from_utf_viii_error:w #1 , + \else: + \if_num:w #1 < #5 \exp_stop_f: + #1 + \else: + \exp_after:wN \str_aux_from_utf_viii:wwNN + \int_use:N \int_eval:w #1 - #5 + \fi: + \fi: + \else: + \exp_after:wN \str_aux_from_utf_viii_error:NwN + \exp_after:wN #2 + \fi: + , #3#5, + } +\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 , + { + \msg_expandable_kernel_error:nnn + { str } { utf8-overlong } {#1} + \c_str_replacement_char_int + } +\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3 + { + \msg_expandable_kernel_error:nnn + { str } { utf8-missing-conti } {#1} + \c_str_replacement_char_int , + #3#1 + } +\cs_new_nopar:Npn \str_aux_from_utf_viii_error: + { + \msg_expandable_kernel_error:nn + { str } { utf8-premature-end } + \c_str_replacement_char_int , + \prg_map_break: + } % \end{macrocode} % \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macrocode} +%</utf8> +% \end{macrocode} % -% \begin{macro}{\str_bytes_percent_decode:NN} -% \begin{macro}[aux]{\str_bytes_percent_decode_aux:wN} -% \begin{macro}[aux]{\str_bytes_percent_decode_aux_ii:NN} +% \subsubsection{\textsc{utf-16} support} +% +% The definitions are done in a category code regime where +% the bytes $254$ and $255$ used by the byte order mark +% have catcode \enquote{other}. % \begin{macrocode} +%<*utf16> \group_begin: - \char_set_lccode:nn {`\*} {`\%} - \tl_to_lowercase:n + \char_set_catcode_other:N \^^fe + \char_set_catcode_other:N \^^ff +% \end{macrocode} +% +% \begin{macro}[int]{\str_convert_to_utf16:} +% \begin{macro}[int]{\str_convert_to_utf16be:} +% \begin{macro}[int]{\str_convert_to_utf16le:} +% \begin{macro}[aux, rEXP] +% { +% \str_aux_to_utf_xvi:n, +% \str_aux_to_utf_xvi_be:n, +% \str_aux_to_utf_xvi_le:n +% } +% Convert characters one by one in a loop, with different +% behaviours depending on the character code: +% \begin{itemize} +% \item $[0, 55295]$ converted to two bytes; +% \item $[55296, 57343]$ cannot be converted, replaced by +% the replacement character; +% \item $[57344, 65535]$ converted to two bytes; +% \item $[65536, 1114111]$ converted to a pair of surrogates, +% each two bytes. +% \end{itemize} +% For the duration of this operation, \cs{str_tmp:w} is defined +% as a function to convert a number in the range $[0, 65536]$ +% to a pair of bytes (either big endian or little endian). +% \begin{macrocode} + \cs_new_protected_nopar:cpx { str_convert_to_utf16: } { - \group_end: - \cs_new_protected_nopar:Npn \str_bytes_percent_decode:NN #1#2 + \exp_not:c { str_convert_to_utf16be: } + \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } } + } + \cs_new_protected_nopar:cpn { str_convert_to_utf16be: } + { + \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_be:n + \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n + } + \cs_new_protected_nopar:cpn { str_convert_to_utf16le: } + { + \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_le:n + \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n + } + \cs_new_nopar:Npn \str_aux_to_utf_xvi:n #1 + { + \if_int_compare:w #1 < "D800 \exp_stop_f: + \str_tmp:w {#1} + \else: + \if_int_compare:w #1 < "10000 \exp_stop_f: + \if_int_compare:w #1 < "E000 \exp_stop_f: + \msg_expandable_kernel_error:nnn + { str } { unicode-surrogate } {#1} + \str_tmp:w { \c_str_replacement_char_int } + \else: + \str_tmp:w {#1} + \fi: + \else: + \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D800 } + \exp_args:Nf \str_tmp:w { \int_mod:nn {#1} {"400} + "DC00 } + \fi: + \fi: + } + \cs_new_nopar:Npn \str_aux_to_utf_xvi_be:n #1 + { + \str_output_byte:n { \int_div_truncate:nn {#1} {"100} } + \str_output_byte:n { \int_mod:nn {#1} {"100} } + } + \cs_new_nopar:Npn \str_aux_to_utf_xvi_le:n #1 + { + \str_output_byte:n { \int_mod:nn {#1} {"100} } + \str_output_byte:n { \int_div_truncate:nn {#1} {"100} } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% \begin{macro}[int] +% { +% \str_convert_from_utf16:, +% \str_convert_from_utf16be:, +% \str_convert_from_utf16le: +% } +% \begin{macro}[aux]{\str_aux_from_utf_xvi_bom:NNw} +% \begin{macro}[aux]{\str_aux_from_utf_xvi:No} +% Define \cs{str_tmp:w} to take two arguments and return the +% character code of the first one if the string is big-endian, +% and the second one if the string is little-endian. +% If the endianness is not known, look for a byte order mark +% to decide. +% \begin{macrocode} + \cs_new_protected_nopar:cpn { str_convert_from_utf16be: } + { \str_aux_from_utf_xvi:No 1 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf16le: } + { \str_aux_from_utf_xvi:No 2 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf16: } + { + \exp_after:wN \str_aux_from_utf_xvi_bom:NNw + \g_str_result_tl \scan_stop: \scan_stop: \scan_stop: + } + \cs_new_protected_nopar:Npn + \str_aux_from_utf_xvi_bom:NNw #1#2#3\scan_stop: + { + \str_if_eq:nnTF { #1#2 } { ^^ff ^^fe } + { \str_aux_from_utf_xvi:No 2 {#3} } { - \group_begin: - \tl_gclear:N \g_str_result_tl - \exp_last_unbraced:Nf \str_bytes_percent_decode_aux:wN - { \tl_to_other_str:N #2 } - * \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl + \str_if_eq:nnTF { #1#2 } { ^^fe ^^ff } + { \str_aux_from_utf_xvi:No 1 {#3} } + { \str_aux_from_utf_xvi:No 1 {#1#2#3} } } - \cs_new_protected_nopar:Npn \str_bytes_percent_decode_aux:wN #1 * #2 + } + \cs_new_protected_nopar:Npn \str_aux_from_utf_xvi:No #1#2 + { + \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 } + \tl_gset:Nx \g_str_result_tl { - \tl_gput_right:Nx \g_str_result_tl {#1} - \quark_if_recursion_tail_stop:N #2 - \int_zero:N \l_str_char_int - \str_aux_hexadecimal_test:NTF #2 - { \str_bytes_percent_decode_aux_ii:NN #2 } - { - \tl_gput_right:Nx \g_str_result_tl { \c_percent_str } - \str_bytes_percent_decode_aux:wN #2 - } + \exp_after:wN \str_aux_from_utf_xvi:NN + #2 \scan_stop: \scan_stop: + \prg_break_point:n { } } } -\cs_new_protected_nopar:Npn \str_bytes_percent_decode_aux_ii:NN #1#2 - { - \str_aux_hexadecimal_test:NTF #2 - { - \str_aux_convert_store: - \str_bytes_percent_decode_aux:wN - } - { - \tl_gput_right:Nx \g_str_result_tl {#1} - \str_bytes_percent_decode_aux:wN #2 - } - } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % -% \subsubsection{UTF-8 support} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:NN} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi:wNNN} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_error:wNNw} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:Nw} +% \begin{macro}[aux, rEXP]{\str_aux_from_utf_xvi_end:wNNww} +% When \cs{str_aux_from_utf_xvi:NN} is called, \cs{str_tmp:w} +% is such that |\str_tmp:w #1#2| expands to |`#1| if the string +% is big-endian, and |`#2| if the string is little-endian. +% If the most significant byte is between |"D8| and |"DB| +% inclusive, then the pair of bytes is a high surrogate, +% and we fetch the following pair, which must be a low surrogate, +% \emph{i.e.}, between |"DC| and |"DF| inclusive. +% Unpaired surrogates are errors, handled with +% \cs{str_aux_from_utf_xvi_error:wNNw}. +% \begin{macrocode} + \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2 + { + \if_meaning:w \scan_stop: #2 + \str_aux_from_utf_xvi_end:Nw #1 + \fi: + \int_use:N \int_eval:w + \if_case:w + \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: + \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN + \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw + \fi: + "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , + \str_aux_from_utf_xvi:NN + } + \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4 + { + \if_meaning:w \scan_stop: #4 + \str_aux_from_utf_xvi_end:wNNww #1 , #3 + \fi: + \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one + ( #1 - "D800 ) * "400 + + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , + \exp_after:wN \use_i:nnn + \else: + \str_aux_from_utf_xvi_error:wNNw #1 , + \fi: + #2#3#4 + } + \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4, + { + \msg_expandable_kernel_error:nnn + { str } { utf16-surrogate } { #2#3 } + \c_str_replacement_char_int , + } + \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi: + { + \fi: + \if_meaning:w \scan_stop: #1 + \else: + \msg_expandable_kernel_error:nnn + { str } { utf16-odd } {#1} + \c_str_replacement_char_int , + \fi: + \prg_map_break: + } + \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi: + { + \fi: + \if_meaning:w \scan_stop: #5 + \msg_expandable_kernel_error:nnn + { str } { utf16-surrogate } { #2#3 } + \else: + \msg_expandable_kernel_error:nnn + { str } { utf16-odd } { #2#3#5 } + \fi: + \c_str_replacement_char_int , + \prg_map_break: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% \end{macro} +% +% Restore the original catcodes of bytes $254$ and $255$. +% \begin{macrocode} +\group_end: +%</utf16> +% \end{macrocode} +% +% \subsubsection{\textsc{utf-32} support} +% +% The definitions are done in a category code regime where +% the bytes $0$, $254$ and $255$ used by the byte order mark +% have catcode \enquote{other}. +% \begin{macrocode} +%<*utf32> +\group_begin: + \char_set_catcode_other:N \^^00 + \char_set_catcode_other:N \^^fe + \char_set_catcode_other:N \^^ff +% \end{macrocode} +% +% \begin{macro}[int] +% { +% \str_convert_to_utf32:, +% \str_convert_to_utf32be:, +% \str_convert_to_utf32le: +% } +% \begin{macro}[aux, rEXP] +% { +% \str_aux_to_utf_xxxii_be:n, +% \str_aux_to_utf_xxxii_le:n +% } +% Convert each integer in the comma-list \cs{g_str_result_tl} +% to a sequence of four bytes. The functions for big-endian +% and little-endian encodings are very similar, but the +% \cs{str_output_byte:n} instructions are reversed. +% \begin{macrocode} + \cs_new_protected_nopar:cpx { str_convert_to_utf32: } + { + \exp_not:c { str_convert_to_utf32be: } + \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } } + } + \cs_new_protected_nopar:cpn { str_convert_to_utf32be: } + { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_be:n } + \cs_new_protected_nopar:cpn { str_convert_to_utf32le: } + { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_le:n } + \cs_new:Npn \str_aux_to_utf_xxxii_be:n #1 + { + ^^00 + \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } } + \str_output_byte:n + { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} } + \str_output_byte:n { \int_mod:nn {#1} {"100} } + } + \cs_new:Npn \str_aux_to_utf_xxxii_le:n #1 + { + \str_output_byte:n { \int_mod:nn {#1} {"100} } + \str_output_byte:n + { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} } + \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } } + ^^00 + } +% \end{macrocode} +% \end{macro} +% \end{macro} % -% \begin{macro}{\str_native_from_UTF_viii:NN} +% \begin{macro}[int] +% { +% \str_convert_from_utf32:, +% \str_convert_from_utf32be:, +% \str_convert_from_utf32le: +% } % \begin{macro}[aux] % { -% \str_native_from_UTF_viii_aux_i:N, -% \str_native_from_UTF_viii_aux_ii:nN, -% \str_native_from_UTF_viii_aux_iii:N, -% \str_native_from_UTF_viii_aux_iv:, -% \str_native_from_UTF_viii_aux_v: +% \str_aux_from_utf_xxxii_bom:NNNNw, +% \str_aux_from_utf_xxxii:No +% } +% \begin{macro}[aux, rEXP] +% { +% \str_aux_from_utf_xxxii:NNNN, +% \str_aux_from_utf_xxxii_end:ww % } +% See the conversion functions from \textsc{utf-16} encodings. +% The \cs{str_aux_from_utf_xxxii:No} auxiliary defines +% \cs{str_tmp:w} to take two arguments and give the character +% code of either the first argument (big-endian) or the second +% argument (little-endian). Then we loop over the string, +% four bytes at a time, check for overflow, and otherwise +% produce the number corresponding to the four bytes with +% the given endianness. +% \begin{macrocode} + \cs_new_protected_nopar:cpn { str_convert_from_utf32be: } + { \str_aux_from_utf_xxxii:No 1 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf32le: } + { \str_aux_from_utf_xxxii:No 2 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf32: } + { + \exp_after:wN \str_aux_from_utf_xxxii_bom:NNNNw \g_str_result_tl + \scan_stop: \scan_stop: \scan_stop: \scan_stop: \scan_stop: + } + \cs_new_protected_nopar:Npn + \str_aux_from_utf_xxxii_bom:NNNNw #1#2#3#4#5\scan_stop: + { + \str_if_eq:nnTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 } + { \str_aux_from_utf_xxxii:No 2 {#5} } + { + \str_if_eq:nnTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff } + { \str_aux_from_utf_xxxii:No 1 {#5} } + { \str_aux_from_utf_xxxii:No 1 {#1#2#3#4#5} } + } + } + \cs_new_protected_nopar:Npn \str_aux_from_utf_xxxii:No #1#2 + { + \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_from_utf_xxxii:NNNN + #2 \scan_stop: \scan_stop: \scan_stop: \scan_stop: + \prg_break_point:n { } + } + } + \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4 + { + \if_meaning:w \scan_stop: #4 + \str_aux_from_utf_xxxii_end:ww #1#2#3#4 + \fi: + \int_use:N \int_eval:w + \if_num:w \int_eval:w + \str_tmp:w #1#4 * \c_two_hundred_fifty_six + + \str_tmp:w #2#3 + > \c_sixteen + \msg_expandable_kernel_error:nnn + { str } { utf32-overflow } { #1#2#3#4 } + \c_str_replacement_char_int + \else: + \str_tmp:w #2#3*"10000 + + \str_tmp:w #3#2 * \c_two_hundred_fifty_six + + \str_tmp:w #4#1 + \fi: + , + \str_aux_from_utf_xxxii:NNNN + } + \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi: + { + \fi: + \tl_if_empty:nF {#1} + { + \msg_expandable_kernel_error:nnn + { str } { utf32-truncated } { #1 } + } + \prg_map_break: + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% \end{macro} +% +% Restore the original catcodes of bytes $0$, $254$ and $255$. % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii:NN #1#2 +\group_end: +%</utf32> +% \end{macrocode} +% +% \subsubsection{\textsc{iso 8859} support} +%^^A todo: document +% +% The \textsc{iso-8859-1} encoding exactly matches with the $256$ +% first Unicode characters. For other 8-bit encodings +% of the \textsc{iso-8859} family, we keep track only of +% differences, and of unassigned bytes. +% \begin{macrocode} +%<*iso88591> +\str_encoding_eight_bit:n { iso88591 } +\tl_const:cn { c_str_encoding_iso88591_tl } { - \group_begin: - \tl_gclear:N \g_str_result_tl - \exp_last_unbraced:Nf \str_native_from_UTF_viii_aux_i:N - { \tl_to_other_str:N #2 } - \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_i:N #1 - { - \quark_if_recursion_tail_stop:N #1 - \int_set:Nn \l_str_char_int {`#1} - \str_native_from_UTF_viii_aux_ii:nN { 128 } \c_one - \str_native_from_UTF_viii_aux_ii:nN { 64 } \c_zero - \str_native_from_UTF_viii_aux_ii:nN { 32 } \c_two - \str_native_from_UTF_viii_aux_ii:nN { 16 } \c_three - \str_native_from_UTF_viii_aux_ii:nN { 8 } \c_four - \msg_error:nnx { str } { utf8-invalid-byte } {#1} - \use_i:nnn \str_native_from_UTF_viii_aux_i:N - \q_stop - \str_native_from_UTF_viii_aux_iii:N +\tl_const:cn { c_str_encoding_iso88591_missing_tl } + { + } +%</iso88591> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso88592> +\str_encoding_eight_bit:n { iso88592 } +\tl_const:cn { c_str_encoding_iso88592_tl } + { + { A1 } { 0104 } + { A2 } { 02D8 } + { A3 } { 0141 } + { A5 } { 013D } + { A6 } { 015A } + { A9 } { 0160 } + { AA } { 015E } + { AB } { 0164 } + { AC } { 0179 } + { AE } { 017D } + { AF } { 017B } + { B1 } { 0105 } + { B2 } { 02DB } + { B3 } { 0142 } + { B5 } { 013E } + { B6 } { 015B } + { B7 } { 02C7 } + { B9 } { 0161 } + { BA } { 015F } + { BB } { 0165 } + { BC } { 017A } + { BD } { 02DD } + { BE } { 017E } + { BF } { 017C } + { C0 } { 0154 } + { C3 } { 0102 } + { C5 } { 0139 } + { C6 } { 0106 } + { C8 } { 010C } + { CA } { 0118 } + { CC } { 011A } + { CF } { 010E } + { D0 } { 0110 } + { D1 } { 0143 } + { D2 } { 0147 } + { D5 } { 0150 } + { D8 } { 0158 } + { D9 } { 016E } + { DB } { 0170 } + { DE } { 0162 } + { E0 } { 0155 } + { E3 } { 0103 } + { E5 } { 013A } + { E6 } { 0107 } + { E8 } { 010D } + { EA } { 0119 } + { EC } { 011B } + { EF } { 010F } + { F0 } { 0111 } + { F1 } { 0144 } + { F2 } { 0148 } + { F5 } { 0151 } + { F8 } { 0159 } + { F9 } { 016F } + { FB } { 0171 } + { FE } { 0163 } + { FF } { 02D9 } } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_ii:nN #1#2 +\tl_const:cn { c_str_encoding_iso88592_missing_tl } { - \int_compare:nNnTF \l_str_char_int < {#1} - { - \int_set_eq:NN \l_str_bytes_int #2 - \use_none_delimit_by_q_stop:w - } - { \int_sub:Nn \l_str_char_int {#1} } } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_iii:N #1 +%</iso88592> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso88593> +\str_encoding_eight_bit:n { iso88593 } +\tl_const:cn { c_str_encoding_iso88593_tl } { - \int_compare:nNnTF \l_str_bytes_int < \c_two - { \str_native_from_UTF_viii_aux_iv: #1 } - { - \if_meaning:w \q_recursion_tail #1 - \msg_error:nn { str } { utf8-premature-end } - \exp_after:wN \use_none_delimit_by_q_recursion_stop:w - \fi: - \quark_if_recursion_tail_stop:N #1 - \tex_multiply:D \l_str_char_int by 64 \scan_stop: - \int_decr:N \l_str_bytes_int - \int_compare:nNnTF {`#1} < { 128 } - { \use_ii:nn } - { \int_compare:nNnTF {`#1} < { 192 } } - { - \int_add:Nn \l_str_char_int { `#1 - 128 } - \str_native_from_UTF_viii_aux_iii:N - } - { - \msg_error:nnx { str } { utf8-missing-byte } {#1} - \str_native_from_UTF_viii_aux_i:N #1 - } - } + { A1 } { 0126 } + { A2 } { 02D8 } + { A6 } { 0124 } + { A9 } { 0130 } + { AA } { 015E } + { AB } { 011E } + { AC } { 0134 } + { AF } { 017B } + { B1 } { 0127 } + { B6 } { 0125 } + { B9 } { 0131 } + { BA } { 015F } + { BB } { 011F } + { BC } { 0135 } + { BF } { 017C } + { C5 } { 010A } + { C6 } { 0108 } + { D5 } { 0120 } + { D8 } { 011C } + { DD } { 016C } + { DE } { 015C } + { E5 } { 010B } + { E6 } { 0109 } + { F5 } { 0121 } + { F8 } { 011D } + { FD } { 016D } + { FE } { 015D } + { FF } { 02D9 } } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_iv: +\tl_const:cn { c_str_encoding_iso88593_missing_tl } { - \int_compare:nNnTF \l_str_bytes_int = \c_zero - { - \msg_error:nnx { str } { utf8-extra-byte } - { \int_eval:n { \l_str_char_int + 128 } } - } - { \str_native_from_UTF_viii_aux_v: } - \str_native_from_UTF_viii_aux_i:N + { A5 } + { AE } + { BE } + { C3 } + { D0 } + { E3 } + { F0 } } -\pdftex_if_engine:TF +%</iso88593> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso88594> +\str_encoding_eight_bit:n { iso88594 } +\tl_const:cn { c_str_encoding_iso88594_tl } { - \cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_v: - { - \int_compare:nNnTF { \l_str_char_int } < { 256 } - { \str_aux_convert_store: } - { - \msg_error:nnx { str } { utf8-pdftex-overflow } - { \int_use:N \l_str_char_int } - } - } + { A1 } { 0104 } + { A2 } { 0138 } + { A3 } { 0156 } + { A5 } { 0128 } + { A6 } { 013B } + { A9 } { 0160 } + { AA } { 0112 } + { AB } { 0122 } + { AC } { 0166 } + { AE } { 017D } + { B1 } { 0105 } + { B2 } { 02DB } + { B3 } { 0157 } + { B5 } { 0129 } + { B6 } { 013C } + { B7 } { 02C7 } + { B9 } { 0161 } + { BA } { 0113 } + { BB } { 0123 } + { BC } { 0167 } + { BD } { 014A } + { BE } { 017E } + { BF } { 014B } + { C0 } { 0100 } + { C7 } { 012E } + { C8 } { 010C } + { CA } { 0118 } + { CC } { 0116 } + { CF } { 012A } + { D0 } { 0110 } + { D1 } { 0145 } + { D2 } { 014C } + { D3 } { 0136 } + { D9 } { 0172 } + { DD } { 0168 } + { DE } { 016A } + { E0 } { 0101 } + { E7 } { 012F } + { E8 } { 010D } + { EA } { 0119 } + { EC } { 0117 } + { EF } { 012B } + { F0 } { 0111 } + { F1 } { 0146 } + { F2 } { 014D } + { F3 } { 0137 } + { F9 } { 0173 } + { FD } { 0169 } + { FE } { 016B } + { FF } { 02D9 } } +\tl_const:cn { c_str_encoding_iso88594_missing_tl } { - \cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_v: - { \str_aux_convert_store: } } +%</iso88594> % \end{macrocode} -% \end{macro} -% \end{macro} % -% \begin{macro}{\str_UTF_viii_from_native:NN} % \begin{macrocode} -\cs_new_protected_nopar:Npn \str_UTF_viii_from_native:NN #1#2 +%<*iso88595> +\str_encoding_eight_bit:n { iso88595 } +\tl_const:cn { c_str_encoding_iso88595_tl } { - \group_begin: - \tl_gclear:N \g_str_result_tl - \str_map_tokens:Nn #2 \str_UTF_viii_from_native_aux_i:N - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl + { A1 } { 0401 } + { A2 } { 0402 } + { A3 } { 0403 } + { A4 } { 0404 } + { A5 } { 0405 } + { A6 } { 0406 } + { A7 } { 0407 } + { A8 } { 0408 } + { A9 } { 0409 } + { AA } { 040A } + { AB } { 040B } + { AC } { 040C } + { AE } { 040E } + { AF } { 040F } + { B0 } { 0410 } + { B1 } { 0411 } + { B2 } { 0412 } + { B3 } { 0413 } + { B4 } { 0414 } + { B5 } { 0415 } + { B6 } { 0416 } + { B7 } { 0417 } + { B8 } { 0418 } + { B9 } { 0419 } + { BA } { 041A } + { BB } { 041B } + { BC } { 041C } + { BD } { 041D } + { BE } { 041E } + { BF } { 041F } + { C0 } { 0420 } + { C1 } { 0421 } + { C2 } { 0422 } + { C3 } { 0423 } + { C4 } { 0424 } + { C5 } { 0425 } + { C6 } { 0426 } + { C7 } { 0427 } + { C8 } { 0428 } + { C9 } { 0429 } + { CA } { 042A } + { CB } { 042B } + { CC } { 042C } + { CD } { 042D } + { CE } { 042E } + { CF } { 042F } + { D0 } { 0430 } + { D1 } { 0431 } + { D2 } { 0432 } + { D3 } { 0433 } + { D4 } { 0434 } + { D5 } { 0435 } + { D6 } { 0436 } + { D7 } { 0437 } + { D8 } { 0438 } + { D9 } { 0439 } + { DA } { 043A } + { DB } { 043B } + { DC } { 043C } + { DD } { 043D } + { DE } { 043E } + { DF } { 043F } + { E0 } { 0440 } + { E1 } { 0441 } + { E2 } { 0442 } + { E3 } { 0443 } + { E4 } { 0444 } + { E5 } { 0445 } + { E6 } { 0446 } + { E7 } { 0447 } + { E8 } { 0448 } + { E9 } { 0449 } + { EA } { 044A } + { EB } { 044B } + { EC } { 044C } + { ED } { 044D } + { EE } { 044E } + { EF } { 044F } + { F0 } { 2116 } + { F1 } { 0451 } + { F2 } { 0452 } + { F3 } { 0453 } + { F4 } { 0454 } + { F5 } { 0455 } + { F6 } { 0456 } + { F7 } { 0457 } + { F8 } { 0458 } + { F9 } { 0459 } + { FA } { 045A } + { FB } { 045B } + { FC } { 045C } + { FD } { 00A7 } + { FE } { 045E } + { FF } { 045F } } -\cs_new_protected_nopar:Npn \str_UTF_viii_from_native_aux_i:N #1 +\tl_const:cn { c_str_encoding_iso88595_missing_tl } { - \int_set:Nn \l_str_char_int {`#1} - \int_compare:nNnTF \l_str_char_int < { 128 } - { \tl_gput_right:Nx \g_str_result_tl {#1} } - { - \tl_gclear:N \g_str_tmpa_tl - \int_set:Nn \l_str_bytes_int { 64 } - \str_UTF_viii_from_native_aux_ii:n { 32 } - \str_UTF_viii_from_native_aux_ii:n { 16 } - \str_UTF_viii_from_native_aux_ii:n { 8 } - \ERROR % somehow the unicode char was > "1FFFFF > "10FFFF - \tl_gclear:N \g_str_tmpa_tl - \use_none:n \q_stop - \int_add:Nn \l_str_char_int { 2 * \l_str_bytes_int } - \str_aux_convert_store: - \tl_gput_right:Nx \g_str_result_tl { \g_str_tmpa_tl } - } } -\cs_new_protected_nopar:Npn \str_UTF_viii_from_native_aux_ii:n #1 +%</iso88595> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso88596> +\str_encoding_eight_bit:n { iso88596 } +\tl_const:cn { c_str_encoding_iso88596_tl } { - \str_aux_convert_store:NNn - \tl_gput_left:Nx \g_str_tmpa_tl - { 128 + \int_mod:nn { \l_str_char_int } { 64 } } - \tex_divide:D \l_str_char_int by 64 \scan_stop: - \int_add:Nn \l_str_bytes_int {#1} - \int_compare:nNnT \l_str_char_int < {#1} - { \use_none_delimit_by_q_stop:w } + { AC } { 060C } + { BB } { 061B } + { BF } { 061F } + { C1 } { 0621 } + { C2 } { 0622 } + { C3 } { 0623 } + { C4 } { 0624 } + { C5 } { 0625 } + { C6 } { 0626 } + { C7 } { 0627 } + { C8 } { 0628 } + { C9 } { 0629 } + { CA } { 062A } + { CB } { 062B } + { CC } { 062C } + { CD } { 062D } + { CE } { 062E } + { CF } { 062F } + { D0 } { 0630 } + { D1 } { 0631 } + { D2 } { 0632 } + { D3 } { 0633 } + { D4 } { 0634 } + { D5 } { 0635 } + { D6 } { 0636 } + { D7 } { 0637 } + { D8 } { 0638 } + { D9 } { 0639 } + { DA } { 063A } + { E0 } { 0640 } + { E1 } { 0641 } + { E2 } { 0642 } + { E3 } { 0643 } + { E4 } { 0644 } + { E5 } { 0645 } + { E6 } { 0646 } + { E7 } { 0647 } + { E8 } { 0648 } + { E9 } { 0649 } + { EA } { 064A } + { EB } { 064B } + { EC } { 064C } + { ED } { 064D } + { EE } { 064E } + { EF } { 064F } + { F0 } { 0650 } + { F1 } { 0651 } + { F2 } { 0652 } } +\tl_const:cn { c_str_encoding_iso88596_missing_tl } + { + { A1 } + { A2 } + { A3 } + { A5 } + { A6 } + { A7 } + { A8 } + { A9 } + { AA } + { AB } + { AE } + { AF } + { B0 } + { B1 } + { B2 } + { B3 } + { B4 } + { B5 } + { B6 } + { B7 } + { B8 } + { B9 } + { BA } + { BC } + { BD } + { BE } + { C0 } + { DB } + { DC } + { DD } + { DE } + { DF } + } +%</iso88596> % \end{macrocode} -% \end{macro} -% -% \subsection{Messages} -% +% % \begin{macrocode} -\msg_new:nnnn { str } { x-missing-brace } - { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. } +%<*iso88597> +\str_encoding_eight_bit:n { iso88597 } +\tl_const:cn { c_str_encoding_iso88597_tl } { - You~wrote~something~like~ - `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~ - The~closing~brace~is~missing. + { A1 } { 2018 } + { A2 } { 2019 } + { A4 } { 20AC } + { A5 } { 20AF } + { AA } { 037A } + { AF } { 2015 } + { B4 } { 0384 } + { B5 } { 0385 } + { B6 } { 0386 } + { B8 } { 0388 } + { B9 } { 0389 } + { BA } { 038A } + { BC } { 038C } + { BE } { 038E } + { BF } { 038F } + { C0 } { 0390 } + { C1 } { 0391 } + { C2 } { 0392 } + { C3 } { 0393 } + { C4 } { 0394 } + { C5 } { 0395 } + { C6 } { 0396 } + { C7 } { 0397 } + { C8 } { 0398 } + { C9 } { 0399 } + { CA } { 039A } + { CB } { 039B } + { CC } { 039C } + { CD } { 039D } + { CE } { 039E } + { CF } { 039F } + { D0 } { 03A0 } + { D1 } { 03A1 } + { D3 } { 03A3 } + { D4 } { 03A4 } + { D5 } { 03A5 } + { D6 } { 03A6 } + { D7 } { 03A7 } + { D8 } { 03A8 } + { D9 } { 03A9 } + { DA } { 03AA } + { DB } { 03AB } + { DC } { 03AC } + { DD } { 03AD } + { DE } { 03AE } + { DF } { 03AF } + { E0 } { 03B0 } + { E1 } { 03B1 } + { E2 } { 03B2 } + { E3 } { 03B3 } + { E4 } { 03B4 } + { E5 } { 03B5 } + { E6 } { 03B6 } + { E7 } { 03B7 } + { E8 } { 03B8 } + { E9 } { 03B9 } + { EA } { 03BA } + { EB } { 03BB } + { EC } { 03BC } + { ED } { 03BD } + { EE } { 03BE } + { EF } { 03BF } + { F0 } { 03C0 } + { F1 } { 03C1 } + { F2 } { 03C2 } + { F3 } { 03C3 } + { F4 } { 03C4 } + { F5 } { 03C5 } + { F6 } { 03C6 } + { F7 } { 03C7 } + { F8 } { 03C8 } + { F9 } { 03C9 } + { FA } { 03CA } + { FB } { 03CB } + { FC } { 03CC } + { FD } { 03CD } + { FE } { 03CE } } -\msg_new:nnn { str } { utf8-invalid-byte } +\tl_const:cn { c_str_encoding_iso88597_missing_tl } { - \int_compare:nNnTF {`#1} < { 256 } - { Byte~number~ \int_eval:n {`#1} ~invalid~in~utf-8~encoding. } - { The~character~number~ \int_eval:n {`#1} ~is~not~a~byte. } + { AE } + { D2 } } -\msg_new:nnn { str } { utf8-missing-byte } - { The~byte~number~ \int_eval:n {`#1} ~is~not~a~valid~continuation~byte. } -\msg_new:nnn { str } { utf8-extra-byte } - { The~byte~number~ \int_eval:n {`#1} ~is~only~valid~as~a~continuation~byte. } -\msg_new:nnnn { str } { utf8-premature-end } - { Incomplete~last~UTF8~character. } +%</iso88597> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso88598> +\str_encoding_eight_bit:n { iso88598 } +\tl_const:cn { c_str_encoding_iso88598_tl } { - The~sequence~of~byte~that~to~be~converted~to~UTF8~ - ended~before~the~last~character~was~complete.~Perhaps~ - it~was~mistakenly~truncated? + { AA } { 00D7 } + { BA } { 00F7 } + { DF } { 2017 } + { E0 } { 05D0 } + { E1 } { 05D1 } + { E2 } { 05D2 } + { E3 } { 05D3 } + { E4 } { 05D4 } + { E5 } { 05D5 } + { E6 } { 05D6 } + { E7 } { 05D7 } + { E8 } { 05D8 } + { E9 } { 05D9 } + { EA } { 05DA } + { EB } { 05DB } + { EC } { 05DC } + { ED } { 05DD } + { EE } { 05DE } + { EF } { 05DF } + { F0 } { 05E0 } + { F1 } { 05E1 } + { F2 } { 05E2 } + { F3 } { 05E3 } + { F4 } { 05E4 } + { F5 } { 05E5 } + { F6 } { 05E6 } + { F7 } { 05E7 } + { F8 } { 05E8 } + { F9 } { 05E9 } + { FA } { 05EA } + { FD } { 200E } + { FE } { 200F } } -\msg_new:nnn { str } { utf8-pdftex-overflow } - { The~character~number~#1~is~too~big~for~pdfTeX. } +\tl_const:cn { c_str_encoding_iso88598_missing_tl } + { + { A1 } + { BF } + { C0 } + { C1 } + { C2 } + { C3 } + { C4 } + { C5 } + { C6 } + { C7 } + { C8 } + { C9 } + { CA } + { CB } + { CC } + { CD } + { CE } + { CF } + { D0 } + { D1 } + { D2 } + { D3 } + { D4 } + { D5 } + { D6 } + { D7 } + { D8 } + { D9 } + { DA } + { DB } + { DC } + { DD } + { DE } + { FB } + { FC } + } +%</iso88598> % \end{macrocode} % -% \subsection{Deprecated string functions} +% \begin{macrocode} +%<*iso88599> +\str_encoding_eight_bit:n { iso88599 } +\tl_const:cn { c_str_encoding_iso88599_tl } + { + { D0 } { 011E } + { DD } { 0130 } + { DE } { 015E } + { F0 } { 011F } + { FD } { 0131 } + { FE } { 015F } + } +\tl_const:cn { c_str_encoding_iso88599_missing_tl } + { + } +%</iso88599> +% \end{macrocode} % -% \begin{macro}{\str_length_skip_spaces:N, \str_length_skip_spaces:n} -% The naming scheme is a little bit more consistent -% with \enquote{ignore_spaces} instead of \enquote{skip_spaces}. % \begin{macrocode} -\cs_set:Npn \str_length_skip_spaces:N - { \exp_args:No \str_length_skip_spaces:n } -\cs_set_eq:NN \str_length_skip_spaces:n \str_length_ignore_spaces:n +%<*iso885910> +\str_encoding_eight_bit:n { iso885910 } +\tl_const:cn { c_str_encoding_iso885910_tl } + { + { A1 } { 0104 } + { A2 } { 0112 } + { A3 } { 0122 } + { A4 } { 012A } + { A5 } { 0128 } + { A6 } { 0136 } + { A8 } { 013B } + { A9 } { 0110 } + { AA } { 0160 } + { AB } { 0166 } + { AC } { 017D } + { AE } { 016A } + { AF } { 014A } + { B1 } { 0105 } + { B2 } { 0113 } + { B3 } { 0123 } + { B4 } { 012B } + { B5 } { 0129 } + { B6 } { 0137 } + { B8 } { 013C } + { B9 } { 0111 } + { BA } { 0161 } + { BB } { 0167 } + { BC } { 017E } + { BD } { 2015 } + { BE } { 016B } + { BF } { 014B } + { C0 } { 0100 } + { C7 } { 012E } + { C8 } { 010C } + { CA } { 0118 } + { CC } { 0116 } + { D1 } { 0145 } + { D2 } { 014C } + { D7 } { 0168 } + { D9 } { 0172 } + { E0 } { 0101 } + { E7 } { 012F } + { E8 } { 010D } + { EA } { 0119 } + { EC } { 0117 } + { F1 } { 0146 } + { F2 } { 014D } + { F7 } { 0169 } + { F9 } { 0173 } + { FF } { 0138 } + } +\tl_const:cn { c_str_encoding_iso885910_missing_tl } + { + } +%</iso885910> % \end{macrocode} -% \end{macro} % % \begin{macrocode} -%</initex|package> +%<*iso885911> +\str_encoding_eight_bit:n { iso885911 } +\tl_const:cn { c_str_encoding_iso885911_tl } + { + { A1 } { 0E01 } + { A2 } { 0E02 } + { A3 } { 0E03 } + { A4 } { 0E04 } + { A5 } { 0E05 } + { A6 } { 0E06 } + { A7 } { 0E07 } + { A8 } { 0E08 } + { A9 } { 0E09 } + { AA } { 0E0A } + { AB } { 0E0B } + { AC } { 0E0C } + { AD } { 0E0D } + { AE } { 0E0E } + { AF } { 0E0F } + { B0 } { 0E10 } + { B1 } { 0E11 } + { B2 } { 0E12 } + { B3 } { 0E13 } + { B4 } { 0E14 } + { B5 } { 0E15 } + { B6 } { 0E16 } + { B7 } { 0E17 } + { B8 } { 0E18 } + { B9 } { 0E19 } + { BA } { 0E1A } + { BB } { 0E1B } + { BC } { 0E1C } + { BD } { 0E1D } + { BE } { 0E1E } + { BF } { 0E1F } + { C0 } { 0E20 } + { C1 } { 0E21 } + { C2 } { 0E22 } + { C3 } { 0E23 } + { C4 } { 0E24 } + { C5 } { 0E25 } + { C6 } { 0E26 } + { C7 } { 0E27 } + { C8 } { 0E28 } + { C9 } { 0E29 } + { CA } { 0E2A } + { CB } { 0E2B } + { CC } { 0E2C } + { CD } { 0E2D } + { CE } { 0E2E } + { CF } { 0E2F } + { D0 } { 0E30 } + { D1 } { 0E31 } + { D2 } { 0E32 } + { D3 } { 0E33 } + { D4 } { 0E34 } + { D5 } { 0E35 } + { D6 } { 0E36 } + { D7 } { 0E37 } + { D8 } { 0E38 } + { D9 } { 0E39 } + { DA } { 0E3A } + { DF } { 0E3F } + { E0 } { 0E40 } + { E1 } { 0E41 } + { E2 } { 0E42 } + { E3 } { 0E43 } + { E4 } { 0E44 } + { E5 } { 0E45 } + { E6 } { 0E46 } + { E7 } { 0E47 } + { E8 } { 0E48 } + { E9 } { 0E49 } + { EA } { 0E4A } + { EB } { 0E4B } + { EC } { 0E4C } + { ED } { 0E4D } + { EE } { 0E4E } + { EF } { 0E4F } + { F0 } { 0E50 } + { F1 } { 0E51 } + { F2 } { 0E52 } + { F3 } { 0E53 } + { F4 } { 0E54 } + { F5 } { 0E55 } + { F6 } { 0E56 } + { F7 } { 0E57 } + { F8 } { 0E58 } + { F9 } { 0E59 } + { FA } { 0E5A } + { FB } { 0E5B } + } +\tl_const:cn { c_str_encoding_iso885911_missing_tl } + { + { DB } + { DC } + { DD } + { DE } + } +%</iso885911> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso885913> +\str_encoding_eight_bit:n { iso885913 } +\tl_const:cn { c_str_encoding_iso885913_tl } + { + { A1 } { 201D } + { A5 } { 201E } + { A8 } { 00D8 } + { AA } { 0156 } + { AF } { 00C6 } + { B4 } { 201C } + { B8 } { 00F8 } + { BA } { 0157 } + { BF } { 00E6 } + { C0 } { 0104 } + { C1 } { 012E } + { C2 } { 0100 } + { C3 } { 0106 } + { C6 } { 0118 } + { C7 } { 0112 } + { C8 } { 010C } + { CA } { 0179 } + { CB } { 0116 } + { CC } { 0122 } + { CD } { 0136 } + { CE } { 012A } + { CF } { 013B } + { D0 } { 0160 } + { D1 } { 0143 } + { D2 } { 0145 } + { D4 } { 014C } + { D8 } { 0172 } + { D9 } { 0141 } + { DA } { 015A } + { DB } { 016A } + { DD } { 017B } + { DE } { 017D } + { E0 } { 0105 } + { E1 } { 012F } + { E2 } { 0101 } + { E3 } { 0107 } + { E6 } { 0119 } + { E7 } { 0113 } + { E8 } { 010D } + { EA } { 017A } + { EB } { 0117 } + { EC } { 0123 } + { ED } { 0137 } + { EE } { 012B } + { EF } { 013C } + { F0 } { 0161 } + { F1 } { 0144 } + { F2 } { 0146 } + { F4 } { 014D } + { F8 } { 0173 } + { F9 } { 0142 } + { FA } { 015B } + { FB } { 016B } + { FD } { 017C } + { FE } { 017E } + { FF } { 2019 } + } +\tl_const:cn { c_str_encoding_iso885913_missing_tl } + { + } +%</iso885913> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso885914> +\str_encoding_eight_bit:n { iso885914 } +\tl_const:cn { c_str_encoding_iso885914_tl } + { + { A1 } { 1E02 } + { A2 } { 1E03 } + { A4 } { 010A } + { A5 } { 010B } + { A6 } { 1E0A } + { A8 } { 1E80 } + { AA } { 1E82 } + { AB } { 1E0B } + { AC } { 1EF2 } + { AF } { 0178 } + { B0 } { 1E1E } + { B1 } { 1E1F } + { B2 } { 0120 } + { B3 } { 0121 } + { B4 } { 1E40 } + { B5 } { 1E41 } + { B7 } { 1E56 } + { B8 } { 1E81 } + { B9 } { 1E57 } + { BA } { 1E83 } + { BB } { 1E60 } + { BC } { 1EF3 } + { BD } { 1E84 } + { BE } { 1E85 } + { BF } { 1E61 } + { D0 } { 0174 } + { D7 } { 1E6A } + { DE } { 0176 } + { F0 } { 0175 } + { F7 } { 1E6B } + { FE } { 0177 } + } +\tl_const:cn { c_str_encoding_iso885914_missing_tl } + { + } +%</iso885914> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso885915> +\str_encoding_eight_bit:n { iso885915 } +\tl_const:cn { c_str_encoding_iso885915_tl } + { + { A4 } { 20AC } + { A6 } { 0160 } + { A8 } { 0161 } + { B4 } { 017D } + { B8 } { 017E } + { BC } { 0152 } + { BD } { 0153 } + { BE } { 0178 } + } +\tl_const:cn { c_str_encoding_iso885915_missing_tl } + { + } +%</iso885915> +% \end{macrocode} +% +% \begin{macrocode} +%<*iso885916> +\str_encoding_eight_bit:n { iso885916 } +\tl_const:cn { c_str_encoding_iso885916_tl } + { + { A1 } { 0104 } + { A2 } { 0105 } + { A3 } { 0141 } + { A4 } { 20AC } + { A5 } { 201E } + { A6 } { 0160 } + { A8 } { 0161 } + { AA } { 0218 } + { AC } { 0179 } + { AE } { 017A } + { AF } { 017B } + { B2 } { 010C } + { B3 } { 0142 } + { B4 } { 017D } + { B5 } { 201D } + { B8 } { 017E } + { B9 } { 010D } + { BA } { 0219 } + { BC } { 0152 } + { BD } { 0153 } + { BE } { 0178 } + { BF } { 017C } + { C3 } { 0102 } + { C5 } { 0106 } + { D0 } { 0110 } + { D1 } { 0143 } + { D5 } { 0150 } + { D7 } { 015A } + { D8 } { 0170 } + { DD } { 0118 } + { DE } { 021A } + { E3 } { 0103 } + { E5 } { 0107 } + { F0 } { 0111 } + { F1 } { 0144 } + { F5 } { 0151 } + { F7 } { 015B } + { F8 } { 0171 } + { FD } { 0119 } + { FE } { 021B } + } +\tl_const:cn { c_str_encoding_iso885916_missing_tl } + { + } +%</iso885916> % \end{macrocode} % % \end{implementation} diff --git a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins index 5aef8b5f0fe..cb3e893d52c 100644 --- a/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins +++ b/Master/texmf-dist/source/latex/l3experimental/l3str/l3str.ins @@ -37,7 +37,40 @@ Do not distribute a modified version of this file. \generate{\file{l3str.sty} {\from{l3str.dtx} {package}}} \generate{\file{l3regex.sty} {\from{l3regex.dtx} {package}}} - -\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}} +%\generate{\file{l3regex-trace.sty} {\from{l3regex.dtx} {package,trace}}} + +% Escapings. +\generate{% + \file{l3str-hex.def} {\from{l3str.dtx}{hex}}% + \file{l3str-name.def} {\from{l3str.dtx}{name}}% + \file{l3str-string.def} {\from{l3str.dtx}{string}}% + \file{l3str-url.def} {\from{l3str.dtx}{url}}% +} + +% UTF encodings. +\generate{% + \file{l3str-utf8.def} {\from{l3str.dtx}{utf8}}% + \file{l3str-utf16.def} {\from{l3str.dtx}{utf16}}% + \file{l3str-utf32.def} {\from{l3str.dtx}{utf32}}% +} + +% ISO-8859 encodings. +\generate{% + \file{l3str-iso88591.def} {\from{l3str.dtx}{iso88591}}% + \file{l3str-iso88592.def} {\from{l3str.dtx}{iso88592}}% + \file{l3str-iso88593.def} {\from{l3str.dtx}{iso88593}}% + \file{l3str-iso88594.def} {\from{l3str.dtx}{iso88594}}% + \file{l3str-iso88595.def} {\from{l3str.dtx}{iso88595}}% + \file{l3str-iso88596.def} {\from{l3str.dtx}{iso88596}}% + \file{l3str-iso88597.def} {\from{l3str.dtx}{iso88597}}% + \file{l3str-iso88598.def} {\from{l3str.dtx}{iso88598}}% + \file{l3str-iso88599.def} {\from{l3str.dtx}{iso88599}}% + \file{l3str-iso885910.def} {\from{l3str.dtx}{iso885910}}% + \file{l3str-iso885911.def} {\from{l3str.dtx}{iso885911}}% + \file{l3str-iso885913.def} {\from{l3str.dtx}{iso885913}}% + \file{l3str-iso885914.def} {\from{l3str.dtx}{iso885914}}% + \file{l3str-iso885915.def} {\from{l3str.dtx}{iso885915}}% + \file{l3str-iso885916.def} {\from{l3str.dtx}{iso885916}}% +} \endbatchfile diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex-trace.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex-trace.sty deleted file mode 100644 index 07d568d94e9..00000000000 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex-trace.sty +++ /dev/null @@ -1,1796 +0,0 @@ -%% -%% This is file `l3regex-trace.sty', -%% generated with the docstrip utility. -%% -%% The original source files were: -%% -%% l3regex.dtx (with options: `package,trace') -%% -%% EXPERIMENTAL CODE -%% -%% Do not distribute this file without also distributing the -%% source files specified above. -%% -%% Do not distribute a modified version of this file. -%% -%% File: l3regex.dtx Copyright (C) 2011 The LaTeX3 Project -%% -%% It may be distributed and/or modified under the conditions of the -%% LaTeX Project Public License (LPPL), either version 1.3c of this -%% license or (at your option) any later version. The latest version -%% of this license is in the file -%% -%% http://www.latex-project.org/lppl.txt -%% -%% This file is part of the "l3experimental bundle" (The Work in LPPL) -%% and all files in that bundle must be distributed together. -%% -%% The released version of this bundle is available from CTAN. -%% -%% ----------------------------------------------------------------------- -%% -%% The development version of the bundle can be found at -%% -%% http://www.latex-project.org/svnroot/experimental/trunk/ -%% -%% for those people who are interested. -%% -%%%%%%%%%%% -%% NOTE: %% -%%%%%%%%%%% -%% -%% Snapshots taken from the repository represent work in progress and may -%% not work or may contain conflicting material! We therefore ask -%% people _not_ to put them into distributions, archives, etc. without -%% prior consultation with the LaTeX3 Project. -%% -%% ----------------------------------------------------------------------- -\RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 2943 2011-10-30 15:59:43Z bruno $ - {L3 Experimental Regular Expressions} -\ProvidesExplPackage - {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} -\RequirePackage{l3str} -\int_const:Nn \c_forty_eight { 48 } -\int_const:Nn \c_fifty_eight { 58 } -\int_const:Nn \c_sixty_five { 65 } -\int_const:Nn \c_ninety_one { 91 } -\int_const:Nn \c_ninety_seven { 97 } -\int_const:Nn \c_one_hundred_twenty_three { 123 } -\int_const:Nn \c_one_hundred_twenty_seven { 127 } -\cs_new:Npn \regex_tmp:w { } -\tl_new:N \g_regex_tmpa_tl -\tl_new:N \l_regex_tmpa_tl -\tl_new:N \l_regex_tmpb_tl -\int_new:N \l_regex_tmpa_int -\int_new:N \l_regex_tmpb_int -\int_new:N \l_regex_max_state_int -\int_new:N \l_regex_left_state_int -\int_new:N \l_regex_right_state_int -\seq_new:N \l_regex_group_begin_seq -\seq_new:N \l_regex_group_end_seq -\int_new:N \l_regex_capturing_group_int -\seq_new:N \l_regex_capturing_group_seq -\tl_new:N \l_regex_one_or_group_tl -\cs_new_eq:NN \regex_build_tmp_class:n \use_none:n -\bool_new:N \l_regex_class_bool -\tl_new:N \l_regex_class_tl -\tl_const:Nn \c_regex_d_tl - { - \regex_item_range:nn { \c_forty_eight } { 57 } % 0--9 - } -\tl_const:Nn \c_regex_D_tl - { - \regex_item_geq:n { \c_fifty_eight } % > `9 - \regex_item_range:nn { \c_zero } { 47 } % 0 - } -\tl_const:Nn \c_regex_h_tl - { - \regex_item_equal:n { \c_thirty_two } % space - \regex_item_equal:n { \c_nine } % tab - } -\tl_const:Nn \c_regex_H_tl - { - \regex_item_geq:n { 33 } % > space - \regex_item_range:nn { \c_ten } { 31 } % tab < ... < space - \regex_item_range:nn { \c_zero } { \c_eight } % < tab - } -\tl_const:Nn \c_regex_s_tl - { - \regex_item_equal:n { \c_thirty_two } % space - \regex_item_range:nn { \c_nine } { \c_ten } % tab, lf - \regex_item_range:nn { \c_twelve } { \c_thirteen } % ff, cr - } -\tl_const:Nn \c_regex_S_tl - { - \regex_item_geq:n { 33 } % > space - \regex_item_range:nn { \c_fourteen } { 31 } % tab < ... < space - \regex_item_range:nn { \c_zero } { \c_eight } % < tab - \regex_item_equal:n { \c_eleven } % vtab - } -\tl_const:Nn \c_regex_v_tl - { - \regex_item_range:nn { \c_ten } { \c_thirteen } % lf, vtab, ff, cr - } -\tl_const:Nn \c_regex_V_tl - { - \regex_item_geq:n { \c_fourteen } % >cr - \regex_item_range:nn { \c_zero } { \c_nine } % < lf - } -\tl_const:Nn \c_regex_w_tl - { - \regex_item_range:nn { \c_ninety_seven } { 122 } % a--z - \regex_item_range:nn { \c_sixty_five } { 90 } % A--Z - \regex_item_range:nn { \c_forty_eight } { 57 } % 0--9 - \regex_item_equal:n { 95 } % _ - } -\tl_const:Nn \c_regex_W_tl - { - \regex_item_range:nn { \c_zero } { 47 } % <`0 - \regex_item_range:nn { \c_fifty_eight } { 64 } % (`9+1)--(`A-1) - \regex_item_range:nn { \c_ninety_one } { 94 } % (`Z+1)--(`_-1) - \regex_item_equal:n { 96 } % ` - \regex_item_geq:n { \c_one_hundred_twenty_three } % z - } -\tl_const:Nn \c_regex_N_tl - { - \regex_item_geq:n { \c_eleven } % > lf - \regex_item_range:nn { \c_zero } { \c_nine } % < lf - } -\tl_new:N \l_regex_query_other_str -\int_new:N \l_regex_current_step_int -\int_new:N \l_regex_start_step_int -\int_new:N \l_regex_success_step_int -\int_new:N \l_regex_unique_step_int -\int_new:N \l_regex_current_char_int -\int_new:N \l_regex_last_char_int -\int_new:N \l_regex_case_changed_char_int -\bool_new:N \l_regex_caseless_bool -\int_new:N \l_regex_current_state_int -\prop_new:N \l_regex_current_submatches_prop -\prop_new:N \l_regex_success_submatches_prop -\int_new:N \l_regex_max_index_int -\tl_new:N \l_regex_every_match_tl -\bool_new:N \l_regex_success_bool -\cs_new_protected:Npn \regex_last_match_empty_no:F #1 {#1} -\cs_new_protected:Npn \regex_last_match_empty_yes:F - { \int_compare:nNnF \l_regex_start_step_int = \l_regex_current_step_int } -\cs_new_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F -\bool_new:N \l_regex_success_empty_bool -\bool_new:N \l_regex_fresh_thread_bool -\seq_new:N \g_regex_submatches_seq -\int_new:N \g_regex_match_count_int -\seq_new:N \g_regex_extract_all_seq -\seq_new:N \g_regex_split_seq -\tl_new:N \l_regex_replacement_tl -\tl_new:N \g_regex_replaced_str -\cs_new_eq:NN \s_regex_toks \scan_stop: -\cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 - { - \cs_set_nopar:Npx \regex_tmp:w \s_regex_toks { \s_regex_toks #2 } - \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN - { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } - } -\cs_new_protected:Npn \regex_toks_put_right:Nx #1#2 - { - \cs_set_nopar:Npx \regex_tmp:w {#2} - \tex_toks:D #1 \exp_after:wN - { \tex_the:D \tex_toks:D \exp_after:wN #1 \regex_tmp:w } - } -\cs_new_nopar:Npn \regex_query_substr:NN #1#2 - { - \exp_after:wN \regex_query_substr_aux:NN - \exp_after:wN #1 - \exp_after:wN #2 - \l_regex_query_other_str - \q_stop - } -\cs_new:Npn \regex_query_substr_aux:NN #1#2 - { - \str_skip_do:nn {#1} - { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } } - } -\cs_new_protected_nopar:Npn \regex_seq_pop:N #1 - { - \seq_pop:NN #1 \l_regex_tmpa_tl - \l_regex_tmpa_tl - } -\cs_new_protected_nopar:Npn \regex_seq_get:N #1 - { - \seq_get:NN #1 \l_regex_tmpa_tl - \l_regex_tmpa_tl - } -\cs_new_protected_nopar:Npn \regex_item_dot:T - { \int_compare:nNnF \l_regex_current_char_int = \c_minus_one } -\cs_new_protected_nopar:Npn \regex_break_true:w - #1 \regex_break_point:TF #2 #3 {#2} -\cs_new_protected_nopar:Npn \regex_break_false:w - #1 \regex_break_point:TF #2 #3 {#3} -\cs_new_protected_nopar:Npn \regex_break_point:TF #1 #2 { #2 } -\cs_new_protected_nopar:Npn \regex_item_caseful_equal:n #1 - { - \if_num:w #1 = \l_regex_current_char_int - \exp_after:wN \regex_break_true:w - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_caseful_range:nn #1 #2 - { - \reverse_if:N \if_num:w #1 > \l_regex_current_char_int - \reverse_if:N \if_num:w #2 < \l_regex_current_char_int - \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w - \fi: - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_caseful_geq:n #1 - { - \reverse_if:N \if_num:w #1 > \l_regex_current_char_int - \exp_after:wN \regex_break_true:w - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_caseless_equal:n #1 - { - \if_num:w #1 = \l_regex_current_char_int - \exp_after:wN \regex_break_true:w - \fi: - \if_num:w #1 = \l_regex_case_changed_char_int - \exp_after:wN \regex_break_true:w - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_caseless_range:nn #1 #2 - { - \reverse_if:N \if_num:w #1 > \l_regex_current_char_int - \reverse_if:N \if_num:w #2 < \l_regex_current_char_int - \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w - \fi: - \fi: - \reverse_if:N \if_num:w #1 > \l_regex_case_changed_char_int - \reverse_if:N \if_num:w #2 < \l_regex_case_changed_char_int - \exp_after:wN \exp_after:wN \exp_after:wN \regex_break_true:w - \fi: - \fi: - } -\cs_new_protected_nopar:Npn \regex_item_caseless_geq:n #1 - { - \reverse_if:N \if_num:w #1 > \l_regex_current_char_int - \exp_after:wN \regex_break_true:w - \fi: - \reverse_if:N \if_num:w #1 > \l_regex_case_changed_char_int - \exp_after:wN \regex_break_true:w - \fi: - } -\cs_new:Npn \regex_item_equal:n { \regex_item_caseful_equal:n } -\cs_new:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } -\cs_new:Npn \regex_item_geq:n { \regex_item_caseful_geq:n } -\cs_new_protected_nopar:Npn \regex_build_caseless: - { - \bool_set_true:N \l_regex_caseless_bool - \cs_set:Npn \regex_item_equal:n { \regex_item_caseless_equal:n } - \cs_set:Npn \regex_item_range:nn { \regex_item_caseless_range:nn } - \cs_set:Npn \regex_item_geq:n { \regex_item_caseless_geq:n } - } -\cs_new_protected_nopar:Npn \regex_build_caseful: - { - \bool_set_false:N \l_regex_caseless_bool - \cs_set:Npn \regex_item_equal:n { \regex_item_caseful_equal:n } - \cs_set:Npn \regex_item_range:nn { \regex_item_caseful_range:nn } - \cs_set:Npn \regex_item_geq:n { \regex_item_caseful_geq:n } - } -\cs_new_protected:Npn \regex_get_digits:nw #1 - { - \tex_afterassignment:D \regex_tmp:w - \cs_set_nopar:Npx \regex_tmp:w - { - \exp_not:n {#1} - { \if_false: } } \fi: - \regex_get_digits_aux:NN - } -\cs_new_nopar:Npn \regex_get_digits_aux:NN #1#2 - { - \if_meaning:w \regex_build_raw:N #1 - \if_charcode:w \c_space_token \exp_not:N #2 - \else: - \if_num:w 9 < 1 \exp_not:N #2 \exp_stop_f: - #2 - \else: - \regex_get_digits_end:w #1 #2 - \fi: - \fi: - \else: - \regex_get_digits_end:w #1 #2 - \fi: - \regex_get_digits_aux:NN - } -\cs_new_nopar:Npn \regex_get_digits_end:w #1 \fi: #2 \regex_get_digits_aux:NN - { - \fi: #2 - \if_false: { { \fi: } } - #1 - } -\prg_new_conditional:Npnn \regex_token_if_other_digit:N #1 { TF } - { - \if_num:w \c_nine < 1 \exp_not:N #1 \exp_stop_f: - \prg_return_true: \else: \prg_return_false: \fi: - } -\prg_new_conditional:Npnn \regex_aux_char_if_special:N #1 { TF } - { - \if_num:w `#1 < \c_ninety_one - \if_num:w `#1 < \c_fifty_eight - \if_num:w `#1 < \c_forty_eight - \if_num:w `#1 < \c_thirty_two - \prg_return_false: \else: \prg_return_true: \fi: - \else: \prg_return_false: \fi: - \else: - \if_num:w `#1 < \c_sixty_five - \prg_return_true: \else: \prg_return_false: \fi: - \fi: - \else: - \if_num:w `#1 < \c_one_hundred_twenty_three - \if_num:w `#1 < \c_ninety_seven - \prg_return_true: \else: \prg_return_false: \fi: - \else: - \if_num:w `#1 < \c_one_hundred_twenty_seven - \prg_return_true: \else: \prg_return_false: \fi: - \fi: - \fi: - } -\prg_new_conditional:Npnn \regex_aux_char_if_alphanumeric:N #1 { TF } - { - \if_num:w `#1 < \c_ninety_one - \if_num:w `#1 < \c_fifty_eight - \if_num:w `#1 < \c_forty_eight - \prg_return_false: \else: \prg_return_true: \fi: - \else: - \if_num:w `#1 < \c_sixty_five - \prg_return_false: \else: \prg_return_true: \fi: - \fi: - \else: - \if_num:w `#1 < \c_one_hundred_twenty_three - \if_num:w `#1 < \c_ninety_seven - \prg_return_false: \else: \prg_return_true: \fi: - \else: - \prg_return_false: - \fi: - \fi: - } -\cs_new_protected_nopar:Npn \regex_build_new_state: - { - \int_compare:nNnTF \l_regex_max_state_int > { 32766 } - { \msg_error:nn { regex } { -997 } } - { - \int_incr:N \l_regex_max_state_int - \tex_toks:D \l_regex_max_state_int { \s_regex_toks } - } - \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int - \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int - } -\cs_new_protected_nopar:Npn \regex_build_transition_aux:NN #1#2 - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { #1 { \int_eval:n { #2 - \l_regex_left_state_int } } } - } -\cs_new_protected_nopar:Npn \regex_build_transitions_aux:NNNN #1#2#3#4 - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - #1 { \int_eval:n { #2 - \l_regex_left_state_int } } - #3 { \int_eval:n { #4 - \l_regex_left_state_int } } - } - } -\cs_new_protected:Npn \regex_build:n #1 - { - \regex_build_setup: - \str_aux_escape:NNNn - \regex_build_i_unescaped:N - \regex_build_i_escaped:N - \regex_build_i_raw:N - { #1 } - \regex_build_open_aux: - \g_str_result_tl \prg_do_nothing: \prg_do_nothing: - \seq_push:Nn \l_regex_capturing_group_seq { 0 } - \regex_build_close_aux: \regex_build_group_: - \regex_build_end: - } -\cs_new_nopar:Npn \regex_build_i_unescaped:N #1 - { - \regex_aux_char_if_special:NTF #1 - { \exp_not:N \regex_build_control:N #1 } - { \exp_not:N \regex_build_raw:N #1 } - } -\cs_new_nopar:Npn \regex_build_i_escaped:N #1 - { - \regex_aux_char_if_alphanumeric:NTF #1 - { \exp_not:N \regex_build_control:N #1 } - { \exp_not:N \regex_build_raw:N #1 } - } -\cs_new_nopar:Npn \regex_build_i_raw:N #1 - { \exp_not:N \regex_build_raw:N #1 } -\cs_new_protected_nopar:Npn \regex_build_default_control:N #1 - { - \cs_if_exist_use:cF { regex_build_#1: } - { \regex_build_default_raw:N #1 } - } -\cs_new_protected_nopar:Npn \regex_build_setup: - { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N - \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N - \int_set_eq:NN \l_regex_capturing_group_int \c_zero - \int_zero:N \l_regex_max_state_int - \regex_build_new_state: - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { \regex_action_start_wildcard:nn {0} {1} } - } -\cs_new_protected_nopar:Npn \regex_build_end: - { - \seq_if_empty:NF \l_regex_capturing_group_seq - { - \msg_error:nn { regex } {22} - \prg_replicate:nn - { \seq_length:N \l_regex_capturing_group_seq } - { \regex_build_close_aux: \regex_build_group_: } - } - \regex_toks_put_right:Nx \l_regex_right_state_int - { \regex_action_success: } - } -\cs_new_protected_nopar:cpn { regex_build_^: } - { \regex_build_anchor_start:N \c_zero } -\cs_new_protected_nopar:Npn \regex_build_A: - { \regex_build_anchor_start:N \c_zero } -\cs_new_protected_nopar:Npn \regex_build_G: - { \regex_build_anchor_start:N \l_regex_start_step_int } -\cs_new_protected_nopar:Npn \regex_build_anchor_start:N #1 - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \int_compare:nNnT {#1} = \l_regex_current_step_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } - } -\cs_new_protected_nopar:cpn { regex_build_$: } % $ - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \int_compare:nNnT - \c_minus_one = \l_regex_current_char_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } - } -\cs_new_eq:Nc \regex_build_Z: { regex_build_$: } %$ -\cs_new_eq:Nc \regex_build_z: { regex_build_$: } %$ -\cs_new_protected_nopar:Npn \regex_build_b: - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \regex_if_word_boundary:TF - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - { } - } - } -\cs_new_protected_nopar:Npn \regex_build_B: - { - \regex_build_new_state: - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \exp_not:N \regex_if_word_boundary:TF - { } - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } - } -\cs_new_protected_nopar:Npn \regex_if_word_boundary:TF - { - \group_begin: - \int_set_eq:NN \l_regex_current_char_int \l_regex_last_char_int - \c_regex_w_tl - \regex_break_point:TF - { \group_end: \c_regex_W_tl \regex_item_equal:n { -1 } } - { \group_end: \c_regex_w_tl } - \regex_break_point:TF - } -\cs_new_protected_nopar:Npn \regex_build_default_raw:N #1 - { - \cs_set:Npx \regex_build_tmp_class:n ##1 - { - \regex_item_equal:n { \int_value:w `#1 } - \regex_break_point:TF { \regex_action_cost:n { ##1 } } { } - } - \regex_build_one_quantifier: - } -\cs_new_protected_nopar:cpn { regex_build_.: } - { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { \regex_item_dot:T { \regex_action_cost:n {##1} } } - \regex_build_one_quantifier: - } -\cs_new_protected_nopar:Npn \regex_build_char_type:N #1 - { - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:N #1 - \regex_break_point:TF { \regex_action_cost:n {##1} } { } - } - \regex_build_one_quantifier: - } -\tl_map_inline:nn { dDhHsSvVwWN } - { - \cs_new_protected_nopar:cpx { regex_build_#1: } - { - \exp_not:N \regex_build_char_type:N - \exp_not:c { c_regex_#1_tl } - } - } -\cs_new_protected_nopar:cpn { regex_build_[: } - { - \tl_clear:N \l_regex_class_tl - \cs_set_eq:NN \regex_build_control:N \regex_class_control:N - \cs_set_eq:NN \regex_build_raw:N \regex_class_raw:N - \regex_class_first:NN - } -\cs_new_protected_nopar:Npn \regex_class_control:N #1 - { - \cs_if_exist_use:cF { regex_class_#1: } - { \regex_class_raw:N #1 } - } -\cs_new_protected_nopar:cpn { regex_class_]: } - { - \tl_if_empty:NTF \l_regex_class_tl %[ - { \regex_class_raw:N ] } - { - \cs_set_eq:NN \regex_build_control:N \regex_build_default_control:N - \cs_set_eq:NN \regex_build_raw:N \regex_build_default_raw:N - \cs_set:Npn \regex_build_tmp_class:n ##1 - { - \exp_not:o \l_regex_class_tl - \bool_if:NTF \l_regex_class_bool - { \regex_break_point:TF { \regex_action_cost:n {##1} } { } } - { \regex_break_point:TF { } { \regex_action_cost:n {##1} } } - } - \regex_build_one_quantifier: - } - } -\cs_new_protected_nopar:Npn \regex_class_first:NN #1#2 - { - \str_if_eq:nnTF {#1#2} { \regex_build_control:N ^ } - { \bool_set_false:N \l_regex_class_bool } - { - \bool_set_true:N \l_regex_class_bool - #1 #2 - } - } -\cs_new_protected_nopar:Npn \regex_class_raw:N #1#2#3 - { - \str_if_eq:nnTF {#2#3} { \regex_build_control:N - } - { \regex_class_range:Nw #1 } - { - \regex_class_single:N #1 - #2 #3 - } - } -\cs_new_protected_nopar:Npn \regex_class_single:N #1 - { - \tl_put_right:Nx \l_regex_class_tl - { \regex_item_equal:n { \int_value:w `#1 } } - } -\cs_new_protected_nopar:Npn \regex_class_range:Nw #1#2#3 - { - \token_if_eq_meaning:NNTF #2 \regex_build_control:N - { - \cs_if_exist:cTF { regex_class_#3: } - { - \regex_class_single:N #1 - \regex_class_single:N - - #2#3 - } - { \regex_class_range_put:NN #1#3 } - } - { \regex_class_range_put:NN #1#3 } - } -\cs_new_protected_nopar:Npn \regex_class_range_put:NN #1#2 - { - \if_num:w `#1 > `#2 \exp_stop_f: - \msg_error:nn { regex } {8} - \else: - \tl_put_right:Nx \l_regex_class_tl - { - \if_num:w `#1 = `#2 \exp_stop_f: - \regex_item_equal:n - \else: - \regex_item_range:nn { \int_value:w `#1 } - \fi: - { \int_value:w `#2 } - } - \fi: - } -\tl_map_inline:nn { dDhHsSvVwWN } - { - \cs_new_protected_nopar:cpx { regex_class_#1: } - { - \tl_put_right:Nn \exp_not:N \l_regex_class_tl - { \exp_not:c { c_regex_#1_tl } } - } - } -\cs_new_protected_nopar:Npn \regex_build_quantifier:w #1#2 - { - \token_if_eq_meaning:NNTF #1 \regex_build_control:N - { - \cs_if_exist_use:cF { regex_build_quantifier_#2:w } - { - \regex_build_quantifier_end:n { } - #1 #2 - } - } - { - \regex_build_quantifier_end:n { } - #1 #2 - } - } -\cs_new_protected_nopar:cpn { regex_build_quantifier_?:w } - { \regex_build_quantifier_aux:nnNN { } { ? } } -\cs_new_protected_nopar:cpn { regex_build_quantifier_*:w } - { \regex_build_quantifier_aux:nnNN { } { * } } -\cs_new_protected_nopar:cpn { regex_build_quantifier_+:w } - { \regex_build_quantifier_aux:nnNN { } { + } } -\cs_new_protected_nopar:Npn \regex_build_quantifier_aux:nnNN #1#2#3#4 - { - \str_if_eq:nnTF { #3 #4 } { \regex_build_control:N ? } - { \regex_build_quantifier_end:n { #2 #4 } #1 } - { - \regex_build_quantifier_end:n { #2 } #1 - #3 #4 - } - } -\cs_new_protected_nopar:cpn { regex_build_quantifier_ \c_lbrace_str :w } - { \regex_get_digits:nw { \regex_build_quantifier_lbrace:n } } -\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:n #1 - { - \tl_if_empty:nTF {#1} - { - \regex_build_quantifier_end:n { } - \exp_after:wN \regex_build_raw:N \c_lbrace_str - } - { \regex_build_quantifier_lbrace:nw {#1} } - } -\cs_new_protected_nopar:Npx \regex_build_quantifier_lbrace:nw #1#2#3 - { - \exp_not:N \prg_case_str:nnn { #2 #3 } - { - { \exp_not:N \regex_build_control:N , } - { - \exp_not:N \regex_get_digits:nw - { \exp_not:N \regex_build_quantifier_lbrace:nnw {#1} } - } - { \exp_not:N \regex_build_control:N \c_rbrace_str } - { \exp_not:N \regex_build_quantifier_end:n {n} {#1} } - } - { - \exp_not:N \regex_build_quantifier_end:n { } - \exp_not:N \regex_build_raw:N \c_lbrace_str #1#2 - } - } -\cs_new_protected_nopar:Npn \regex_build_quantifier_lbrace:nnw #1#2#3#4 - { - \str_if_eq:xxTF - { \exp_not:n {#3#4} } - { \exp_not:N \regex_build_control:N \c_rbrace_str } - { - \tl_if_empty:nTF {#2} - { \regex_build_quantifier_aux:nnNN { {#1} } { n* } } - { - \int_compare:nNnT {#1} > {#2} - { \msg_error:nn { regex } {4} } - \regex_build_quantifier_aux:nnNN { {#1} {#2} } { nn } - } - } - { - \regex_build_quantifier_end:n { } - \use:x - { - \exp_args:No \tl_map_function:nN - { \c_lbrace_str #1 , #2 } - \regex_build_raw:N - } - #3 #4 - } - } -\cs_new_protected_nopar:Npn \regex_build_quantifier_end:n #1 - { \use:c { regex_build_ \l_regex_one_or_group_tl _ #1 : } } -\cs_new_protected_nopar:Npn \regex_build_one_quantifier: - { - \tl_set:Nx \l_regex_one_or_group_tl { one } - \regex_build_quantifier:w - } -\cs_new_protected_nopar:Npn \regex_build_one_: - { - \regex_build_transition_aux:NN - \regex_build_tmp_class:n \l_regex_right_state_int - } -\cs_new_protected_nopar:cpn { regex_build_one_?: } - { - \regex_build_transitions_aux:NNNN - \regex_build_tmp_class:n \l_regex_right_state_int - \regex_action_free:n \l_regex_right_state_int - } -\cs_new_protected_nopar:cpn { regex_build_one_??: } - { - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_build_tmp_class:n \l_regex_right_state_int - } -\cs_new_protected_nopar:cpn { regex_build_one_*: } - { - \regex_build_transitions_aux:NNNN - \regex_build_tmp_class:n \l_regex_left_state_int - \regex_action_free:n \l_regex_right_state_int - } -\cs_new_protected_nopar:cpn { regex_build_one_*?: } - { - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_build_tmp_class:n \l_regex_left_state_int - } -\cs_new_protected_nopar:cpn { regex_build_one_+: } - { - \regex_build_one_: - \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_tmpa_int - \regex_action_free:n \l_regex_right_state_int - } -\cs_new_protected_nopar:cpn { regex_build_one_+?: } - { - \regex_build_one_: - \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_action_free:n \l_regex_tmpa_int - } -\cs_new_protected_nopar:Npn \regex_build_one_n: #1 - { \prg_replicate:nn {#1} { \regex_build_one_: } } -\cs_new_eq:cN { regex_build_one_n?: } \regex_build_one_n: -\cs_new_protected_nopar:cpx { regex_build_one_n*: } #1 - { - \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: } - \exp_not:c { regex_build_one_*: } - } -\cs_new_protected_nopar:cpx { regex_build_one_n*?: } #1 - { - \exp_not:N \prg_replicate:nn {#1} { \exp_not:N \regex_build_one_: } - \exp_not:c { regex_build_one_*?: } - } -\cs_new_protected_nopar:Npn \regex_build_one_nn_aux:Nnn #1#2#3 - { - \prg_replicate:nn {#2} { \regex_build_one_: } - \prg_replicate:nn {#3-#2} {#1} - } -\cs_new_protected_nopar:Npx \regex_build_one_nn: - { \regex_build_one_nn_aux:Nnn \exp_not:c { regex_build_one_?: } } -\cs_new_protected_nopar:cpx { regex_build_one_nn?: } - { \regex_build_one_nn_aux:Nnn \exp_not:c { regex_build_one_??: } } -\cs_new_protected_nopar:cpn { regex_build_(: } #1#2 - { - \str_if_eq:nnTF { #1 #2 } { \regex_build_control:N ? } - { \regex_build_special_group:NN } - { - \int_incr:N \l_regex_capturing_group_int - \seq_push:Nx \l_regex_capturing_group_seq - { \int_use:N \l_regex_capturing_group_int } - \regex_build_open_aux: - #1 #2 - } - } -\cs_new_protected_nopar:Npn \regex_build_open_aux: - { - \regex_build_new_state: - \seq_push:Nx \l_regex_group_begin_seq - { - \int_set:Nn \l_regex_left_state_int - { \int_use:N \l_regex_left_state_int } - } - \bool_if:NTF \l_regex_caseless_bool - { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseless: } } - { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseful: } } - \seq_push:Nx \l_regex_group_end_seq - { - \int_set:Nn \l_regex_right_state_int - { \int_use:N \l_regex_right_state_int } - } - \regex_build_begin_alternation: - } -\cs_new_protected_nopar:cpn { regex_build_|: } - { - \regex_build_end_alternation: - \regex_build_begin_alternation: - } -\cs_new_protected_nopar:cpn { regex_build_): } - { - \seq_if_empty:NTF \l_regex_capturing_group_seq - { \msg_error:nn { regex } { 22 } } - { - \regex_build_close_aux: - \regex_build_group_quantifier: - } - } -\cs_new_protected_nopar:Npn \regex_build_close_aux: - { - \regex_build_end_alternation: - \regex_seq_pop:N \l_regex_group_begin_seq - \regex_seq_pop:N \l_regex_group_end_seq - \regex_seq_pop:N \l_regex_group_end_seq - } -\cs_new_protected_nopar:Npn \regex_build_begin_alternation: - { - \regex_build_new_state: - \regex_seq_get:N \l_regex_group_begin_seq - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } -\cs_new_protected_nopar:Npn \regex_build_end_alternation: - { - \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int - \regex_seq_get:N \l_regex_group_end_seq - \regex_toks_put_right:Nx \l_regex_left_state_int - { - \regex_action_free:n - { - \int_eval:n - { \l_regex_right_state_int - \l_regex_left_state_int } - } - } - } -\cs_new_protected_nopar:Npn \regex_build_special_group:NN #1#2 - { - \cs_if_exist_use:cF { regex_build_special_group_\token_to_str:N #2 : } - { - \msg_error:nn { regex } { -998 } - \regex_build_control:N ( % ) - \regex_build_raw:N ? - #1 #2 - } - } -\cs_new_protected_nopar:cpn { regex_build_special_group_:: } - { - \seq_push:Nx \l_regex_capturing_group_seq { * } - \regex_build_open_aux: - } -\cs_new_protected_nopar:Npn \regex_build_special_group_i: - { - \regex_build_options:NNN + - \regex_build_raw:N i - } -\cs_new_protected_nopar:cpn { regex_build_special_group_-: } - { - \regex_build_options:NNN - - } -\cs_new_protected_nopar:Npn \regex_build_options:NNN #1#2#3 - { - \token_if_eq_meaning:NNTF \regex_build_raw:N #2 - { - \cs_if_exist_use:cF { regex_build_option_#1#3: } - { \msg_error:nnx { regex } { unknown-option } { #3 } } - \regex_build_options:NNN #1 - } - { - \prg_case_str:nnn { #3 } - { % ( - { ) } { } - { - } { \regex_build_options:NNN - } - } - { \msg_error:nnx { regex } { invalid-in-option } { #3 } } - } - } -\cs_new_protected_nopar:cpn { regex_build_option_+i: } - { - \regex_build_caseless: - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - } -\cs_new_protected_nopar:cpn { regex_build_option_-i: } - { \regex_build_caseful: } -\cs_new_protected_nopar:Npn \regex_build_group_quantifier: - { - \tl_set:Nn \l_regex_one_or_group_tl { group } - \regex_build_quantifier:w - } -\cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2 - { - \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl - \str_if_eq:xxF { \l_regex_tmpa_tl } { * } - { - \regex_toks_put_left:Nx #1 - { \regex_action_submatch:n { \l_regex_tmpa_tl < } } - \regex_toks_put_left:Nx #2 - { \regex_action_submatch:n { \l_regex_tmpa_tl > } } - } - } -\cs_new_protected_nopar:Npn \regex_build_group_: - { - \regex_build_group_submatches:NN - \l_regex_left_state_int \l_regex_right_state_int - \regex_build_transition_aux:NN - \regex_action_free:n \l_regex_right_state_int - } -\cs_new_protected_nopar:Npn \regex_build_group_shift:N #1 - { - \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_new_state: - \tex_toks:D \l_regex_right_state_int = \tex_toks:D \l_regex_tmpa_int - \regex_toks_put_left:Nx \l_regex_right_state_int - { - \int_set:Nn \l_regex_current_state_int - { \int_use:N \l_regex_tmpa_int } % ^^A here we lie! - } - \cs_set:Npx \regex_tmp:w - { - \tex_toks:D \l_regex_tmpa_int - { - \s_regex_toks - #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } - } - } - \regex_tmp:w - \regex_build_group_submatches:NN - \l_regex_right_state_int \l_regex_left_state_int - } -\cs_new_protected_nopar:Npn \regex_build_group_qs_aux:NN #1#2 - { - \regex_build_group_shift:N \regex_action_free:n - \int_set_eq:NN \l_regex_right_state_int \l_regex_left_state_int - \regex_build_transition_aux:NN \regex_action_free:n #1 - #2 \l_regex_tmpa_int - { - \regex_action_free:n - { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } - } - } -\cs_new_protected_nopar:cpn { regex_build_group_?: } - { - \regex_build_group_qs_aux:NN - \l_regex_right_state_int \regex_toks_put_right:Nx - } -\cs_new_protected_nopar:cpn { regex_build_group_??: } - { - \regex_build_group_qs_aux:NN - \l_regex_right_state_int \regex_toks_put_left:Nx - } -\cs_new_protected_nopar:cpn { regex_build_group_*: } - { - \regex_build_group_qs_aux:NN - \l_regex_tmpa_int \regex_toks_put_right:Nx - } -\cs_new_protected_nopar:cpn { regex_build_group_*?: } - { - \regex_build_group_qs_aux:NN - \l_regex_tmpa_int \regex_toks_put_left:Nx - } -\cs_new_protected_nopar:cpn { regex_build_group_+: } - { - \regex_build_group_submatches:NN - \l_regex_left_state_int \l_regex_right_state_int - \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_tmpa_int - \regex_action_free:n \l_regex_right_state_int - } -\cs_new_protected_nopar:cpn { regex_build_group_+?: } - { - \regex_build_group_submatches:NN - \l_regex_left_state_int \l_regex_right_state_int - \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_action_free:n \l_regex_tmpa_int - } -\cs_new_protected_nopar:Npn \regex_build_group_n_aux:n #1 - { - \regex_toks_put_right:Nx \l_regex_right_state_int - { - \regex_action_free:n - { \int_eval:n { \l_regex_max_state_int - \l_regex_left_state_int } } - } - \int_set_eq:NN \l_regex_tmpa_int \l_regex_left_state_int - \int_set:Nn \l_regex_tmpb_int { \l_regex_max_state_int + \c_one } - \int_set:Nn \l_regex_max_state_int - { - \l_regex_left_state_int - \c_one - + #1 * ( \l_regex_max_state_int - \l_regex_left_state_int + \c_one ) - } - \int_until_do:nNnn \l_regex_tmpb_int > \l_regex_max_state_int - { - \tex_toks:D \l_regex_tmpb_int = \tex_toks:D \l_regex_tmpa_int - \int_incr:N \l_regex_tmpa_int - \int_incr:N \l_regex_tmpb_int - } - } -\cs_new_protected_nopar:Npn \regex_build_group_n: #1 - { % ^^A todo: catch case #1 <= 0. - \regex_build_group_n_aux:n {#1} - \regex_build_transition_aux:NN - \regex_action_free:n \l_regex_right_state_int - \regex_build_group_submatches:NN - \l_regex_tmpa_int \l_regex_left_state_int - } -\cs_new_eq:cN { regex_build_group_n?: } \regex_build_group_n: -\cs_new_protected_nopar:cpn { regex_build_group_n*: } #1 - { % ^^A todo: catch case #1 <= 0. - \regex_build_group_n_aux:n {#1} - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_tmpa_int - \regex_action_free:n \l_regex_right_state_int - \regex_build_group_submatches:NN - \l_regex_tmpa_int \l_regex_left_state_int - } -\cs_new_protected_nopar:cpn { regex_build_group_n*?: } #1 - { % ^^A todo: catch case #1 <= 0. - \regex_build_group_n_aux:n {#1} - \regex_build_transitions_aux:NNNN - \regex_action_free:n \l_regex_right_state_int - \regex_action_free:n \l_regex_tmpa_int - \regex_build_group_submatches:NN - \l_regex_tmpa_int \l_regex_left_state_int - } -\cs_new_protected_nopar:Npn \regex_build_group_nn: #1#2 - { % ^^A Not Implemented Yet! - \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet } - \use:c { regex_build_group_n*: } {#1} - } -\cs_new_protected_nopar:cpn { regex_build_group_nn?: } #1#2 - { % ^^A Not Implemented Yet! - \msg_expandable_error:n { Quantifier~{m,n}~not~implemented~yet } - \use:c { regex_build_group_n*?: } {#1} - } -\cs_new_protected:Npn \regex_store_state:n #1 - { - \int_incr:N \l_regex_max_index_int - \tex_skip:D \l_regex_max_index_int #1 sp \scan_stop: - \regex_store_submatches:n {#1} - } -\cs_new_protected_nopar:Npn \regex_state_use_with_submatches: - { \regex_state_use_aux:n { } } -\cs_new_protected_nopar:Npn \regex_state_use: - { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } } -\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_toks { } -\cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 - { - \if_num:w \tex_dimen:D \l_regex_current_state_int - < \l_regex_unique_step_int - \tex_dimen:D \l_regex_current_state_int - = \l_regex_unique_step_int sp \scan_stop: - #1 \tex_the:D \tex_toks:D \exp_after:wN \l_regex_current_state_int - \fi: - \scan_stop: - } -\cs_new_protected_nopar:Npn \regex_disable_submatches: - { - \cs_set_eq:NN \regex_state_use_with_submatches: \regex_state_use: - \cs_set_eq:NN \regex_store_submatches:n - \regex_protected_use_none:n - \cs_set_eq:NN \regex_action_submatches:n - \regex_protected_use_none:n - } -\cs_new_protected:Npn \regex_protected_use_none:n #1 { } -\cs_new_protected:Npn \regex_store_submatches:n #1 - { - \tex_toks:D #1 \exp_after:wN - { - \tex_romannumeral:D - \exp_after:wN \regex_store_submatches_aux:w - \tex_the:D \tex_toks:D #1 - } - } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_toks - { - \regex_store_submatches_aux_ii:Nnnw - #1 - \regex_state_submatches:nn \c_minus_one \q_prop - \s_regex_toks - } -\cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_regex_toks - { - \exp_after:wN \c_zero - \exp_after:wN \regex_state_submatches:nn \exp_after:wN - { - \int_value:w \int_eval:w - \l_regex_unique_step_int + \c_one - \exp_after:wN - } - \exp_after:wN { \l_regex_current_submatches_prop } - \regex_state_submatches:nn {#1} {#2} - \s_regex_toks - } -\cs_new_protected:Npn \regex_state_submatches:nn #1#2 - { - \if_num:w #1 = \l_regex_unique_step_int - \tl_set:Nn \l_regex_current_submatches_prop { #2 } - \fi: - } -\cs_new_protected:Npn \regex_match:n #1 - { - \tl_set:Nx \l_regex_query_other_str { \tl_to_other_str:n {#1} } - \regex_match_initial_setup: - \regex_match_once: - } -\cs_new_protected_nopar:Npn \regex_match_once: - { - \regex_match_setup: - \exp_after:wN \regex_match_once_aux: \l_regex_query_other_str - \q_recursion_tail \q_recursion_stop - \bool_if:NT \l_regex_success_bool { \l_regex_every_match_tl } - } -\cs_new_protected_nopar:Npn \regex_match_once_aux: - { - \int_compare:nNnTF \l_regex_start_step_int = \c_zero - { - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \regex_match_loop:N - } - { - \str_skip_do:nn - { \l_regex_start_step_int - \c_one } - { \regex_match_once_aux_ii:N } - } - } -\cs_new_protected_nopar:Npn \regex_match_once_aux_ii:N #1 - { - \int_set:Nn \l_regex_current_char_int { `#1 } - \regex_match_loop:N - } -\cs_new_protected_nopar:Npn \regex_match_initial_setup: - { - \prg_stepwise_inline:nnnn {1} {1} { \l_regex_max_state_int } - { \tex_dimen:D ##1 \c_minus_one sp \scan_stop: } - \int_set_eq:NN \l_regex_unique_step_int \c_minus_one - \int_set_eq:NN \l_regex_start_step_int \c_minus_one - \int_set_eq:NN \l_regex_current_step_int \c_zero - \int_set_eq:NN \l_regex_success_step_int \c_zero - \bool_set_false:N \l_regex_success_empty_bool - } -\cs_new_protected_nopar:Npn \regex_match_setup: - { - \prop_clear:N \l_regex_current_submatches_prop - \bool_if:NTF \l_regex_success_empty_bool - { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_yes:F } - { \cs_set_eq:NN \regex_last_match_empty:F \regex_last_match_empty_no:F } - \int_set_eq:NN \l_regex_start_step_int \l_regex_success_step_int - \int_set_eq:NN \l_regex_current_step_int \l_regex_start_step_int - \int_decr:N \l_regex_current_step_int - \bool_set_false:N \l_regex_success_bool - \int_zero:N \l_regex_max_index_int - \regex_store_state:n {1} - } -\cs_new_eq:NN \s_regex_step \scan_stop: -\cs_new:Npn \regex_break_step:w #1 \s_regex_step { } -\cs_new_protected_nopar:Npn \regex_match_loop:N #1 - { - \regex_match_loop_setup:N #1 - \cs_set_nopar:Npx \regex_tmp:w - { - \int_zero:N \l_regex_max_index_int - \prg_stepwise_function:nnnN - {1} {1} { \l_regex_max_index_int } - \regex_match_one_index:n - } - \regex_tmp:w - \s_regex_step - \if_num:w \l_regex_max_index_int = \c_zero - \exp_after:wN \use_none_delimit_by_q_recursion_stop:w - \fi: - \quark_if_recursion_tail_stop:N #1 - \regex_match_loop:N - } -\cs_new_nopar:Npn \regex_match_one_index:n #1 - { - \regex_match_one_index_aux:n - { \int_value:w \tex_skip:D #1 } - } -\cs_new_protected_nopar:Npn \regex_match_one_index_aux:n #1 - { - \int_set:Nn \l_regex_current_state_int {#1} - \prop_clear:N \l_regex_current_submatches_prop - \regex_state_use_with_submatches: - } -\cs_new_protected_nopar:Npn \regex_match_loop_setup:N #1 - { - \int_incr:N \l_regex_current_step_int - \int_incr:N \l_regex_unique_step_int - \bool_set_false:N \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_last_char_int \l_regex_current_char_int - \if_meaning:w #1 \q_recursion_tail - \int_set_eq:NN \l_regex_current_char_int \c_minus_one - \else: - \int_set:Nn \l_regex_current_char_int {`#1} - \fi: - \regex_match_loop_case_hook: - } -\cs_new_protected_nopar:Npn \regex_match_loop_case_hook: { } -\cs_new_protected_nopar:Npn \regex_match_loop_caseless_hook: - { - \int_set_eq:NN \l_regex_case_changed_char_int \l_regex_current_char_int - \if_num:w \l_regex_current_char_int < \c_ninety_one - \if_num:w \l_regex_current_char_int < \c_sixty_five - \else: - \int_add:Nn \l_regex_case_changed_char_int { \c_thirty_two } - \fi: - \else: - \if_num:w \l_regex_current_char_int < \c_one_hundred_twenty_three - \if_num:w \l_regex_current_char_int < \c_ninety_seven - \else: - \int_sub:Nn \l_regex_case_changed_char_int { \c_thirty_two } - \fi: - \fi: - \fi: - } -\cs_new_protected_nopar:Npn \regex_action_start_wildcard:nn #1#2 - { - \bool_set_true:N \l_regex_fresh_thread_bool - \regex_action_free:n {#2} - \bool_set_false:N \l_regex_fresh_thread_bool - \regex_action_cost:n {#1} - } -\cs_new_protected_nopar:Npn \regex_action_cost:n #1 - { - \exp_args:Nf \regex_store_state:n %^^A optimize! - { \int_eval:n { \l_regex_current_state_int + #1 } } - } -\cs_new_protected_nopar:Npn \regex_action_success: - { - \regex_last_match_empty:F - { - \bool_set_true:N \l_regex_success_bool - \bool_set_eq:NN \l_regex_success_empty_bool - \l_regex_fresh_thread_bool - \int_set_eq:NN \l_regex_success_step_int - \l_regex_current_step_int - \prop_set_eq:NN \l_regex_success_submatches_prop - \l_regex_current_submatches_prop - \regex_break_step:w - } - } -\cs_new_protected_nopar:Npn \regex_action_free:n #1 - { - \cs_set_nopar:Npx \regex_tmp:w - { - \int_add:Nn \l_regex_current_state_int {#1} - \regex_state_use: - \int_set:Nn \l_regex_current_state_int - { \int_use:N \l_regex_current_state_int } - \tl_set:Nn \exp_not:N \l_regex_current_submatches_prop - { \exp_not:o \l_regex_current_submatches_prop } - } - \regex_tmp:w - } -\cs_new_protected_nopar:Npn \regex_action_submatch:n #1 - { - \prop_put:Nno \l_regex_current_submatches_prop {#1} - { \int_use:N \l_regex_current_step_int } - } -\cs_new_protected_nopar:Npn \regex_extract: - { - \seq_gclear:N \g_regex_submatches_seq - \prg_stepwise_inline:nnnn - {0} {1} { \l_regex_capturing_group_int } - { - \regex_extract_aux:nTF { ##1 } - { - \seq_gput_right:Nx \g_regex_submatches_seq - { \regex_query_substr:NN \l_regex_tmpa_tl \l_regex_tmpb_tl } - } - { \seq_gput_right:Nn \g_regex_submatches_seq { } } - } - } -\cs_new_protected_nopar:Npn \regex_extract_aux:nTF #1#2#3 - { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 < } \l_regex_tmpa_tl - { - \prop_get:NnNTF \l_regex_success_submatches_prop - { #1 > } \l_regex_tmpb_tl - {#2} - {#3} - } - {#3} - } -\cs_new_protected_nopar:Npn \regex_set:Nn - { \regex_set_aux:NNn \tl_set_eq:NN } -\cs_new_protected_nopar:Npn \regex_gset:Nn - { \regex_set_aux:NNn \tl_gset_eq:NN } -\cs_new_protected:Npn \regex_set_aux:NNn #1#2#3 - { - \group_begin: - \regex_build:n {#3} - \tl_gset:Nx \g_regex_tmpa_tl - { - \exp_not:N \regex_nfa:Nw \exp_not:N #2 - \l_regex_max_state_int - = \int_use:N \l_regex_max_state_int - \l_regex_capturing_group_int - = \int_use:N \l_regex_capturing_group_int - \token_if_eq_meaning:NNT - \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - { - \cs_set_eq:NN \regex_match_loop_case_hook: - \regex_match_loop_caseless_hook: - } - \prg_stepwise_function:nnnN - {1} {1} {\l_regex_max_state_int} - \regex_set_aux:n - \regex_nfa_end: - } - \group_end: - #1 #2 \g_regex_tmpa_tl - } -\cs_new_nopar:Npn \regex_set_aux:n #1 - { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 #2 \regex_nfa_end: - { \msg_expandable_error:n { Automaton~#1 used~incorrectly. } } -\cs_new_eq:NN \regex_nfa_end: \scan_stop: -\prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF } - { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop } -\cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop - { - \if_meaning:w \regex_nfa:Nw #1 - \prg_return_true: - \else: - \msg_error:nnx { regex } { not-nfa } { \token_to_str:N #1 } - \prg_return_false: - \fi: - } -\cs_new_protected_nopar:Npn \regex_use:N #1 - { \exp_after:wN \use_none:nn #1 } -\cs_new_protected:Npn \regex_user_aux:n #1 - { - \group_begin: - \tl_clear:N \l_regex_every_match_tl - #1 - \group_end: - } -\cs_new_protected_nopar:Npn \regex_return_after_group: - { - \if_meaning:w \c_true_bool \l_regex_success_bool - \group_insert_after:N \prg_return_true: - \else: - \group_insert_after:N \prg_return_false: - \fi: - } -\cs_new_protected_nopar:Npn \regex_extract_after_group:N #1 - { - \if_meaning:w \c_true_bool \l_regex_success_bool - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_submatches_seq - \fi: - } -\cs_new_protected_nopar:Npn \regex_count_after_group:N #1 - { - \group_insert_after:N \int_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_match_count_int - } -\prg_new_protected_conditional:Npnn \regex_match:nn #1#2 { T , F , TF } - { - \regex_user_aux:n - { - \regex_disable_submatches: - \regex_build:n {#1} - \regex_match:n {#2} - \regex_return_after_group: - } - } -\prg_new_protected_conditional:Npnn \regex_match:Nn #1#2 { T , F , TF } - { - \regex_check_nfa:NTF #1 - { - \regex_user_aux:n - { - \regex_disable_submatches: - \regex_use:N #1 - \regex_match:n {#2} - \regex_return_after_group: - } - } - { \prg_return_false: } - } -\cs_new_protected:Npn \regex_count:nnN #1#2#3 - { - \regex_user_aux:n - { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_count_after_group:N #3 - } - } -\cs_new_protected:Npn \regex_count:NnN #1#2#3 - { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \regex_disable_submatches: - \int_gzero:N \g_regex_match_count_int - \tl_set:Nn \l_regex_every_match_tl - { - \int_gincr:N \g_regex_match_count_int - \regex_match_once: - } - \regex_use:N #1 - \regex_match:n {#2} - \regex_count_after_group:N #3 - } - } - } -\cs_new_protected:Npn \regex_extract_once:nnN #1#2#3 - { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } - } -\prg_new_protected_conditional:Npnn \regex_extract_once:nnN #1#2#3 - { T , F , TF } - { - \regex_user_aux:n - { - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } - } -\cs_new_protected:Npn \regex_extract_once:NnN #1#2#3 - { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - } - } - } -\prg_new_protected_conditional:Npnn \regex_extract_once:NnN #1#2#3 - { T , F , TF } - { - \regex_check_nfa:NTF #1 - { - \regex_user_aux:n - { - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_after_group:N #3 - \regex_return_after_group: - } - } - { \prg_return_false: } - } -\cs_new_protected:Npn \regex_extract_all:nnN #1#2#3 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } - } -\cs_new_protected:Npn \regex_extract_all:NnN #1#2#3 - { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_extract_all_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_extract_all_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_extract_all_after_group:N #3 - } - } - } -\cs_new_protected_nopar:Npn \regex_extract_all_aux: - { - \regex_extract: - \seq_gconcat:NNN \g_regex_extract_all_seq - \g_regex_extract_all_seq \g_regex_submatches_seq - \regex_match_once: - } -\cs_new_protected_nopar:Npn \regex_extract_all_after_group:N #1 - { - \seq_if_empty:NF \g_regex_extract_all_seq - { - \regex_extract: - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_extract_all_seq - } - } -\cs_new_protected:Npn \regex_split:nnN #1#2#3 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_build:n {#1} - \regex_match:n {#2} - \regex_split_after_group:N #3 - } - } -\cs_new_protected:Npn \regex_split:NnN #1#2#3 - { - \regex_check_nfa:NT #1 - { - \regex_user_aux:n - { - \seq_gclear:N \g_regex_split_seq - \tl_set:Nn \l_regex_every_match_tl { \regex_split_aux: } - \regex_use:N #1 - \regex_match:n {#2} - \regex_split_after_group:N #3 - } - } - } -\cs_new_protected_nopar:Npn \regex_split_aux: - { - \int_compare:nNnF \l_regex_start_step_int = \l_regex_success_step_int - { - \regex_extract: - \seq_pop:NN \g_regex_submatches_seq \l_regex_tmpa_tl - \regex_extract_aux:nTF {0} - { - \seq_gput_left:Nx \g_regex_submatches_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - } - } - { \msg_error:nn { regex } { internal } } - \seq_gconcat:NNN \g_regex_split_seq - \g_regex_split_seq \g_regex_submatches_seq - } - \regex_match_once: - } -\cs_new_protected_nopar:Npn \regex_split_after_group:N #1 - { - \int_compare:nNnTF \l_regex_start_step_int = \l_regex_current_step_int - { - \bool_if:NF \l_regex_success_empty_bool - { \seq_gput_right:Nn \g_regex_split_seq { } } - } - { - \seq_gput_right:Nx \g_regex_split_seq - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_current_step_int - } - } - \group_insert_after:N \seq_set_eq:NN - \group_insert_after:N #1 - \group_insert_after:N \g_regex_split_seq - } -\cs_new_protected:Npn \regex_replacement:n #1 - { - \str_aux_escape:NNNn - \prg_do_nothing: - \regex_replacement_escaped:N - \prg_do_nothing: - {#1} - \tl_set_eq:NN \l_regex_replacement_tl \g_str_result_tl - \tl_set:Nx \l_regex_replacement_tl - { \l_regex_replacement_tl \prg_do_nothing: } - } -\cs_new_nopar:Npn \regex_replacement_escaped:N #1 - { - \regex_token_if_other_digit:NTF #1 - { - \exp_not:n - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } {#1} - } - { - \token_if_eq_charcode:NNTF g #1 - { \exp_not:N \regex_replacement_submatch:w } - { #1 } - } - } -\cs_new_nopar:Npn \regex_replacement_submatch:w #1 - { - \exp_after:wN \token_if_eq_meaning:NNTF \c_lbrace_str #1 - { \regex_replacement_submatch_aux:nN { } } - { - \regex_token_if_other_digit:NTF #1 - { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } - { \msg_expandable_error:n { \g misused~in~replacement~text. } } - #1 - } - } -\cs_new_nopar:Npn \regex_replacement_submatch_aux:nN #1 #2 - { - \regex_token_if_other_digit:NTF #2 - { \regex_replacement_submatch_aux:nN {#1#2} } - { - \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1} - \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2 - { - \msg_expandable_error:n { \g misused~in~replacement~text. } - #2 - } - } - } -\cs_new_protected_nopar:Npn \regex_replace_after_group:N #1 - { - \group_insert_after:N \tl_set:Nx - \group_insert_after:N #1 - \group_insert_after:N { - \group_insert_after:N \tl_to_str:N - \group_insert_after:N \g_regex_replaced_str - \group_insert_after:N } - } -\cs_new_protected:Npn \regex_replace_once_aux:Nn #1#2 - { - \group_begin: - \regex_replace_after_group:N #1 - \tl_clear:N \l_regex_every_match_tl - #2 - \exp_args:No \regex_match:n {#1} - \regex_extract: - \regex_extract_aux:nTF {0} - { - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_tmpb_tl } { } } - \tl_gset:Nx \g_regex_replaced_str - { - \regex_query_substr:NN \c_zero \l_regex_tmpa_tl - \l_regex_replacement_tl - \exp_after:wN \regex_tmp:w \l_regex_query_other_str - } - } - { \tl_gset_eq:NN \g_regex_replaced_str \l_regex_query_other_str } - \group_end: - } -\cs_new_protected:Npn \regex_replace_once:nnN #1#2#3 - { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - } - } -\cs_new_protected:Npn \regex_replace_once:NnN #1#2#3 - { - \regex_replace_once_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - } - } -\prg_new_protected_conditional:Npnn \regex_replace_once:nnN #1#2#3 {T,F,TF} - { - \regex_replace_once_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - \regex_return_after_group: - } - } -\prg_new_protected_conditional:Npnn \regex_replace_once:NnN #1#2#3 {T,F,TF} - { - \regex_replace_once_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - \regex_return_after_group: - } - } -\cs_new_protected:Npn \regex_replace_all_aux:Nn #1#2 - { - \group_begin: - \regex_replace_after_group:N #1 - \tl_set:Nn \l_regex_every_match_tl { \regex_replace_all_aux: } - \tl_gclear:N \g_regex_replaced_str - #2 - \exp_args:No \regex_match:n {#1} - \cs_set_nopar:Npn \regex_tmp:w - { \str_skip_do:nn { \l_regex_start_step_int } { } } - \tl_gput_right:Nx \g_regex_replaced_str - { \exp_after:wN \regex_tmp:w \l_regex_query_other_str } - \group_end: - } -\cs_new_protected_nopar:Npn \regex_replace_all_aux: - { - \regex_extract: - \regex_extract_aux:nTF {0} - { - \tl_gput_right:Nx \g_regex_replaced_str - { - \regex_query_substr:NN - \l_regex_start_step_int \l_regex_tmpa_tl - \l_regex_replacement_tl - } - } - { \msg_error:nn { regex } { internal } } - \regex_match_once: - } -\cs_new_protected:Npn \regex_replace_all:nnN #1#2#3 - { - \regex_replace_all_aux:Nn #3 - { - \regex_build:n {#1} - \regex_replacement:n {#2} - } - } -\cs_new_protected:Npn \regex_replace_all:NnN #1#2#3 - { - \regex_replace_all_aux:Nn #3 - { - \regex_use:N #1 - \regex_replacement:n {#2} - } - } -\msg_new:nnn { regex } { -998 } { Unsupported~construct } -\msg_new:nnn { regex } { -997 } - { The~regular~expression~is~too~large~(32768~states). } -\msg_new:nnn { regex } { 1 } { \iow_char:N\\~at~end~of~pattern } -\msg_new:nnn { regex } { 4 } - { Numbers~out~of~order~in~\iow_char:N\{\iow_char\}~quantifier. } -\msg_new:nnn { regex } { 6 } - { Missing~terminating~\iow_char:N\]~for~character~class } -\msg_new:nnn { regex } { 7 } - { Invalid~escape~sequence~in~character~class } -\msg_new:nnn { regex } { 8 } - { Range~out~of~order~in~character~class } -\msg_new:nnn { regex } { 22 } { Mismatched~parentheses } -\msg_new:nnn { regex } { 34 } - { Character~value~in~\iow_char:N\\x{...}~sequence~is~too~large } -\msg_new:nnn { regex } { not-nfa } - { - LaTeX~was~expecting~a~regular~expression~variable.\\ - Instead,~LaTeX~found~'#1'. - } -%% -%% -%% End of file `l3regex-trace.sty'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty index c57f415bc47..f3dc0b5a358 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3regex.sty @@ -46,18 +46,11 @@ %% %% ----------------------------------------------------------------------- \RequirePackage{expl3} -\GetIdInfo$Id: l3regex.dtx 2943 2011-10-30 15:59:43Z bruno $ +\GetIdInfo$Id: l3regex.dtx 3017 2011-11-28 00:10:02Z bruno $ {L3 Experimental Regular Expressions} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} \RequirePackage{l3str} -\int_const:Nn \c_forty_eight { 48 } -\int_const:Nn \c_fifty_eight { 58 } -\int_const:Nn \c_sixty_five { 65 } -\int_const:Nn \c_ninety_one { 91 } -\int_const:Nn \c_ninety_seven { 97 } -\int_const:Nn \c_one_hundred_twenty_three { 123 } -\int_const:Nn \c_one_hundred_twenty_seven { 127 } \cs_new:Npn \regex_tmp:w { } \tl_new:N \g_regex_tmpa_tl \tl_new:N \l_regex_tmpa_tl @@ -67,10 +60,13 @@ \int_new:N \l_regex_max_state_int \int_new:N \l_regex_left_state_int \int_new:N \l_regex_right_state_int -\seq_new:N \l_regex_group_begin_seq -\seq_new:N \l_regex_group_end_seq +\seq_new:N \l_regex_left_state_seq +\seq_new:N \l_regex_right_state_seq +\seq_new:N \l_regex_end_group_seq +\seq_new:N \l_regex_end_alternation_seq \int_new:N \l_regex_capturing_group_int \seq_new:N \l_regex_capturing_group_seq +\int_new:N \l_regex_capturing_group_max_int \tl_new:N \l_regex_one_or_group_tl \cs_new_eq:NN \regex_build_tmp_class:n \use_none:n \bool_new:N \l_regex_class_bool @@ -164,10 +160,10 @@ \seq_new:N \g_regex_split_seq \tl_new:N \l_regex_replacement_tl \tl_new:N \g_regex_replaced_str -\cs_new_eq:NN \s_regex_toks \scan_stop: +\cs_new_eq:NN \s_regex_stop \scan_stop: \cs_new_protected:Npn \regex_toks_put_left:Nx #1#2 { - \cs_set_nopar:Npx \regex_tmp:w \s_regex_toks { \s_regex_toks #2 } + \cs_set_nopar:Npx \regex_tmp:w \s_regex_stop { \s_regex_stop #2 } \tex_toks:D #1 \exp_after:wN \exp_after:wN \exp_after:wN { \exp_after:wN \regex_tmp:w \tex_the:D \tex_toks:D #1 } } @@ -190,12 +186,24 @@ \str_skip_do:nn {#1} { \str_collect_do:nn { #2 - #1 } { \use_i_delimit_by_q_stop:nw } } } -\cs_new_protected_nopar:Npn \regex_seq_pop:N #1 +\cs_new_protected:Npn \regex_seq_pop_int:NN #1#2 + { + \seq_pop:NN #1 \l_regex_tmpa_tl + \int_set:Nn #2 \l_regex_tmpa_tl + } +\cs_new_protected:Npn \regex_seq_get_int:NN #1#2 + { + \seq_get:NN #1 \l_regex_tmpa_tl + \int_set:Nn #2 \l_regex_tmpa_tl + } +\cs_new_protected:Npn \regex_seq_push_int:NN #1#2 + { \seq_push:No #1 { \int_use:N #2 } } +\cs_new_protected_nopar:Npn \regex_seq_pop_use:N #1 { \seq_pop:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl } -\cs_new_protected_nopar:Npn \regex_seq_get:N #1 +\cs_new_protected_nopar:Npn \regex_seq_get_use:N #1 { \seq_get:NN #1 \l_regex_tmpa_tl \l_regex_tmpa_tl @@ -358,7 +366,7 @@ { \msg_error:nn { regex } { -997 } } { \int_incr:N \l_regex_max_state_int - \tex_toks:D \l_regex_max_state_int { \s_regex_toks } + \tex_toks:D \l_regex_max_state_int { \s_regex_stop } } \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int \int_set_eq:NN \l_regex_right_state_int \l_regex_max_state_int @@ -388,7 +396,7 @@ { #1 } \regex_build_open_aux: \g_str_result_tl \prg_do_nothing: \prg_do_nothing: - \seq_push:Nn \l_regex_capturing_group_seq { 0 } + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_zero \regex_build_close_aux: \regex_build_group_: \regex_build_end: } @@ -519,7 +527,7 @@ { \cs_set:Npx \regex_build_tmp_class:n ##1 { - \regex_item_equal:n { \int_value:w `#1 } + \regex_item_equal:n { \int_value:w `#1 ~ } \regex_break_point:TF { \regex_action_cost:n { ##1 } } { } } \regex_build_one_quantifier: @@ -800,8 +808,8 @@ { \regex_build_special_group:NN } { \int_incr:N \l_regex_capturing_group_int - \seq_push:Nx \l_regex_capturing_group_seq - { \int_use:N \l_regex_capturing_group_int } + \regex_seq_push_int:NN + \l_regex_capturing_group_seq \l_regex_capturing_group_int \regex_build_open_aux: #1 #2 } @@ -809,19 +817,12 @@ \cs_new_protected_nopar:Npn \regex_build_open_aux: { \regex_build_new_state: - \seq_push:Nx \l_regex_group_begin_seq - { - \int_set:Nn \l_regex_left_state_int - { \int_use:N \l_regex_left_state_int } - } + \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int + \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int \bool_if:NTF \l_regex_caseless_bool - { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseless: } } - { \seq_push:Nn \l_regex_group_end_seq { \regex_build_caseful: } } - \seq_push:Nx \l_regex_group_end_seq - { - \int_set:Nn \l_regex_right_state_int - { \int_use:N \l_regex_right_state_int } - } + { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseless: } + { \seq_push:Nn \l_regex_end_group_seq \regex_build_caseful: } + \seq_push:Nn \l_regex_end_alternation_seq { } \regex_build_begin_alternation: } \cs_new_protected_nopar:cpn { regex_build_|: } @@ -841,14 +842,15 @@ \cs_new_protected_nopar:Npn \regex_build_close_aux: { \regex_build_end_alternation: - \regex_seq_pop:N \l_regex_group_begin_seq - \regex_seq_pop:N \l_regex_group_end_seq - \regex_seq_pop:N \l_regex_group_end_seq + \regex_seq_pop_int:NN \l_regex_left_state_seq \l_regex_left_state_int + \regex_seq_pop_int:NN \l_regex_right_state_seq \l_regex_right_state_int + \regex_seq_pop_use:N \l_regex_end_group_seq + \seq_pop:NN \l_regex_end_alternation_seq \l_regex_tmpa_tl } \cs_new_protected_nopar:Npn \regex_build_begin_alternation: { \regex_build_new_state: - \regex_seq_get:N \l_regex_group_begin_seq + \regex_seq_get_int:NN \l_regex_left_state_seq \l_regex_left_state_int \regex_toks_put_right:Nx \l_regex_left_state_int { \regex_action_free:n @@ -861,7 +863,7 @@ \cs_new_protected_nopar:Npn \regex_build_end_alternation: { \int_set_eq:NN \l_regex_left_state_int \l_regex_right_state_int - \regex_seq_get:N \l_regex_group_end_seq + \regex_seq_get_int:NN \l_regex_right_state_seq \l_regex_right_state_int \regex_toks_put_right:Nx \l_regex_left_state_int { \regex_action_free:n @@ -870,6 +872,7 @@ { \l_regex_right_state_int - \l_regex_left_state_int } } } + \regex_seq_get_use:N \l_regex_end_alternation_seq } \cs_new_protected_nopar:Npn \regex_build_special_group:NN #1#2 { @@ -883,9 +886,39 @@ } \cs_new_protected_nopar:cpn { regex_build_special_group_:: } { - \seq_push:Nx \l_regex_capturing_group_seq { * } + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one \regex_build_open_aux: } +\cs_new_protected_nopar:cpn { regex_build_special_group_|: } + { + \regex_seq_push_int:NN \l_regex_capturing_group_seq \c_minus_one + \regex_build_new_state: + \regex_seq_push_int:NN \l_regex_left_state_seq \l_regex_left_state_int + \regex_seq_push_int:NN \l_regex_right_state_seq \l_regex_right_state_int + \seq_push:Nx \l_regex_end_alternation_seq + { + \exp_not:N \int_compare:nNnT + \l_regex_capturing_group_int + > \l_regex_capturing_group_max_int + { + \int_set_eq:NN + \l_regex_capturing_group_max_int + \l_regex_capturing_group_int + } + \int_set:Nn \l_regex_capturing_group_int + { \int_use:N \l_regex_capturing_group_int } + } + \seq_push:Nx \l_regex_end_group_seq + { + \bool_if:NTF \l_regex_caseless_bool + \regex_build_caseless: + \regex_build_caseful: + \int_set_eq:NN + \l_regex_capturing_group_int + \l_regex_capturing_group_max_int + } + \regex_build_begin_alternation: + } \cs_new_protected_nopar:Npn \regex_build_special_group_i: { \regex_build_options:NNN + @@ -928,7 +961,7 @@ \cs_new_protected_nopar:Npn \regex_build_group_submatches:NN #1#2 { \seq_pop:NN \l_regex_capturing_group_seq \l_regex_tmpa_tl - \str_if_eq:xxF { \l_regex_tmpa_tl } { * } + \int_compare:nNnF { \l_regex_tmpa_tl } < \c_zero { \regex_toks_put_left:Nx #1 { \regex_action_submatch:n { \l_regex_tmpa_tl < } } @@ -957,7 +990,7 @@ { \tex_toks:D \l_regex_tmpa_int { - \s_regex_toks + \s_regex_stop #1 { \int_eval:n { \l_regex_right_state_int - \l_regex_tmpa_int } } } } @@ -1082,7 +1115,7 @@ { \regex_state_use_aux:n { } } \cs_new_protected_nopar:Npn \regex_state_use: { \regex_state_use_aux:n { \exp_after:wN \regex_state_use_aux_ii:w } } -\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_toks { } +\cs_new_nopar:Npn \regex_state_use_aux_ii:w #1 \s_regex_stop { } \cs_new_protected_nopar:Npn \regex_state_use_aux:n #1 { \if_num:w \tex_dimen:D \l_regex_current_state_int @@ -1111,15 +1144,15 @@ \tex_the:D \tex_toks:D #1 } } -\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_toks +\cs_new_protected:Npn \regex_store_submatches_aux:w #1 \s_regex_stop { \regex_store_submatches_aux_ii:Nnnw #1 \regex_state_submatches:nn \c_minus_one \q_prop - \s_regex_toks + \s_regex_stop } \cs_new_protected:Npn \regex_store_submatches_aux_ii:Nnnw - \regex_state_submatches:nn #1 #2 #3 \s_regex_toks + \regex_state_submatches:nn #1 #2 #3 \s_regex_stop { \exp_after:wN \c_zero \exp_after:wN \regex_state_submatches:nn \exp_after:wN @@ -1130,7 +1163,7 @@ } \exp_after:wN { \l_regex_current_submatches_prop } \regex_state_submatches:nn {#1} {#2} - \s_regex_toks + \s_regex_stop } \cs_new_protected:Npn \regex_state_submatches:nn #1#2 { @@ -1192,8 +1225,6 @@ \int_zero:N \l_regex_max_index_int \regex_store_state:n {1} } -\cs_new_eq:NN \s_regex_step \scan_stop: -\cs_new:Npn \regex_break_step:w #1 \s_regex_step { } \cs_new_protected_nopar:Npn \regex_match_loop:N #1 { \regex_match_loop_setup:N #1 @@ -1204,8 +1235,7 @@ {1} {1} { \l_regex_max_index_int } \regex_match_one_index:n } - \regex_tmp:w - \s_regex_step + \regex_tmp:w \prg_break_point:n { } \if_num:w \l_regex_max_index_int = \c_zero \exp_after:wN \use_none_delimit_by_q_recursion_stop:w \fi: @@ -1277,7 +1307,7 @@ \l_regex_current_step_int \prop_set_eq:NN \l_regex_success_submatches_prop \l_regex_current_submatches_prop - \regex_break_step:w + \prg_map_break: } } \cs_new_protected_nopar:Npn \regex_action_free:n #1 @@ -1349,16 +1379,17 @@ \prg_stepwise_function:nnnN {1} {1} {\l_regex_max_state_int} \regex_set_aux:n - \regex_nfa_end: + \s_regex_stop } \group_end: #1 #2 \g_regex_tmpa_tl } \cs_new_nopar:Npn \regex_set_aux:n #1 { \tex_toks:D #1 { \tex_the:D \tex_toks:D #1 } } -\cs_new:Npn \regex_nfa:Nw #1 #2 \regex_nfa_end: - { \msg_expandable_error:n { Automaton~#1 used~incorrectly. } } -\cs_new_eq:NN \regex_nfa_end: \scan_stop: +\cs_new:Npn \regex_nfa:Nw #1 #2 \s_regex_stop + { \msg_expandable_kernel_error:nnn { regex } { nfa-misused } {#1} } +\cs_new_protected_nopar:Npn \regex_const:Nn + { \regex_set_aux:NNn \cs_new_eq:NN } \prg_new_protected_conditional:Npnn \regex_check_nfa:N #1 { T , TF } { \exp_after:wN \regex_check_nfa_aux:Nw #1 \q_stop } \cs_new:Npn \regex_check_nfa_aux:Nw #1 #2 \q_stop @@ -1644,7 +1675,7 @@ { \regex_token_if_other_digit:NTF #1 { \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } } - { \msg_expandable_error:n { \g misused~in~replacement~text. } } + { \msg_expandable_kernel_error:nn { regex } { g-misused } } #1 } } @@ -1656,7 +1687,7 @@ \exp_not:n { \seq_item:Nn \g_regex_submatches_seq } {#1} \exp_after:wN \token_if_eq_meaning:NNF \c_rbrace_str #2 { - \msg_expandable_error:n { \g misused~in~replacement~text. } + \msg_expandable_kernel_error:nn { regex } { g-misused } #2 } } @@ -1791,6 +1822,10 @@ LaTeX~was~expecting~a~regular~expression~variable.\\ Instead,~LaTeX~found~'#1'. } +\msg_kernel_new:nnn { regex } { nfa-misused } + { Automaton~#1 used~incorrectly. } +\msg_kernel_new:nnn { regex } { g-misused } + { \g misused~in~replacement~text. } %% %% %% End of file `l3regex.sty'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def new file mode 100644 index 00000000000..9af2e4322ad --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-hex.def @@ -0,0 +1,81 @@ +%% +%% This is file `l3str-hex.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `hex') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\cs_new_protected_nopar:Npn \str_convert_input_hex: + { + \tl_gset:Nx \g_str_result_tl + { + \str_output_byte:w " + \exp_after:wN \str_convert_input_hex_aux:N + \g_str_result_tl 0 { ? 0 - \c_one \prg_map_break: } + \prg_break_point:n { \str_output_end: } + } + } +\cs_new_nopar:Npn \str_convert_input_hex_aux:N #1 + { + \use_none:n #1 + \str_aux_hexadecimal_use:NTF #1 + \str_convert_input_hex_aux_ii:N + \str_convert_input_hex_aux:N + } +\cs_new_nopar:Npn \str_convert_input_hex_aux_ii:N #1 + { + \use_none:n #1 + \str_aux_hexadecimal_use:NTF #1 + { + \str_output_end: + \str_output_byte:w " \str_convert_input_hex_aux:N + } + \str_convert_input_hex_aux_ii:N + } +\cs_new_protected_nopar:Npn \str_convert_output_hex: + { \str_aux_gmap_result:N \str_convert_output_hex_aux:N } +\cs_new_nopar:Npn \str_convert_output_hex_aux:N #1 + { \str_output_hexadecimal:n { `#1 } } +%% +%% +%% End of file `l3str-hex.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88591.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88591.def new file mode 100644 index 00000000000..310226b18d1 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88591.def @@ -0,0 +1,57 @@ +%% +%% This is file `l3str-iso88591.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88591') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88591 } +\tl_const:cn { c_str_encoding_iso88591_tl } + { + } +\tl_const:cn { c_str_encoding_iso88591_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso88591.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885910.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885910.def new file mode 100644 index 00000000000..68e6dc9463e --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885910.def @@ -0,0 +1,103 @@ +%% +%% This is file `l3str-iso885910.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso885910') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso885910 } +\tl_const:cn { c_str_encoding_iso885910_tl } + { + { A1 } { 0104 } + { A2 } { 0112 } + { A3 } { 0122 } + { A4 } { 012A } + { A5 } { 0128 } + { A6 } { 0136 } + { A8 } { 013B } + { A9 } { 0110 } + { AA } { 0160 } + { AB } { 0166 } + { AC } { 017D } + { AE } { 016A } + { AF } { 014A } + { B1 } { 0105 } + { B2 } { 0113 } + { B3 } { 0123 } + { B4 } { 012B } + { B5 } { 0129 } + { B6 } { 0137 } + { B8 } { 013C } + { B9 } { 0111 } + { BA } { 0161 } + { BB } { 0167 } + { BC } { 017E } + { BD } { 2015 } + { BE } { 016B } + { BF } { 014B } + { C0 } { 0100 } + { C7 } { 012E } + { C8 } { 010C } + { CA } { 0118 } + { CC } { 0116 } + { D1 } { 0145 } + { D2 } { 014C } + { D7 } { 0168 } + { D9 } { 0172 } + { E0 } { 0101 } + { E7 } { 012F } + { E8 } { 010D } + { EA } { 0119 } + { EC } { 0117 } + { F1 } { 0146 } + { F2 } { 014D } + { F7 } { 0169 } + { F9 } { 0173 } + { FF } { 0138 } + } +\tl_const:cn { c_str_encoding_iso885910_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso885910.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885911.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885911.def new file mode 100644 index 00000000000..b22795dea38 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885911.def @@ -0,0 +1,148 @@ +%% +%% This is file `l3str-iso885911.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso885911') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso885911 } +\tl_const:cn { c_str_encoding_iso885911_tl } + { + { A1 } { 0E01 } + { A2 } { 0E02 } + { A3 } { 0E03 } + { A4 } { 0E04 } + { A5 } { 0E05 } + { A6 } { 0E06 } + { A7 } { 0E07 } + { A8 } { 0E08 } + { A9 } { 0E09 } + { AA } { 0E0A } + { AB } { 0E0B } + { AC } { 0E0C } + { AD } { 0E0D } + { AE } { 0E0E } + { AF } { 0E0F } + { B0 } { 0E10 } + { B1 } { 0E11 } + { B2 } { 0E12 } + { B3 } { 0E13 } + { B4 } { 0E14 } + { B5 } { 0E15 } + { B6 } { 0E16 } + { B7 } { 0E17 } + { B8 } { 0E18 } + { B9 } { 0E19 } + { BA } { 0E1A } + { BB } { 0E1B } + { BC } { 0E1C } + { BD } { 0E1D } + { BE } { 0E1E } + { BF } { 0E1F } + { C0 } { 0E20 } + { C1 } { 0E21 } + { C2 } { 0E22 } + { C3 } { 0E23 } + { C4 } { 0E24 } + { C5 } { 0E25 } + { C6 } { 0E26 } + { C7 } { 0E27 } + { C8 } { 0E28 } + { C9 } { 0E29 } + { CA } { 0E2A } + { CB } { 0E2B } + { CC } { 0E2C } + { CD } { 0E2D } + { CE } { 0E2E } + { CF } { 0E2F } + { D0 } { 0E30 } + { D1 } { 0E31 } + { D2 } { 0E32 } + { D3 } { 0E33 } + { D4 } { 0E34 } + { D5 } { 0E35 } + { D6 } { 0E36 } + { D7 } { 0E37 } + { D8 } { 0E38 } + { D9 } { 0E39 } + { DA } { 0E3A } + { DF } { 0E3F } + { E0 } { 0E40 } + { E1 } { 0E41 } + { E2 } { 0E42 } + { E3 } { 0E43 } + { E4 } { 0E44 } + { E5 } { 0E45 } + { E6 } { 0E46 } + { E7 } { 0E47 } + { E8 } { 0E48 } + { E9 } { 0E49 } + { EA } { 0E4A } + { EB } { 0E4B } + { EC } { 0E4C } + { ED } { 0E4D } + { EE } { 0E4E } + { EF } { 0E4F } + { F0 } { 0E50 } + { F1 } { 0E51 } + { F2 } { 0E52 } + { F3 } { 0E53 } + { F4 } { 0E54 } + { F5 } { 0E55 } + { F6 } { 0E56 } + { F7 } { 0E57 } + { F8 } { 0E58 } + { F9 } { 0E59 } + { FA } { 0E5A } + { FB } { 0E5B } + } +\tl_const:cn { c_str_encoding_iso885911_missing_tl } + { + { DB } + { DC } + { DD } + { DE } + } +%% +%% +%% End of file `l3str-iso885911.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885913.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885913.def new file mode 100644 index 00000000000..a0904a29c84 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885913.def @@ -0,0 +1,113 @@ +%% +%% This is file `l3str-iso885913.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso885913') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso885913 } +\tl_const:cn { c_str_encoding_iso885913_tl } + { + { A1 } { 201D } + { A5 } { 201E } + { A8 } { 00D8 } + { AA } { 0156 } + { AF } { 00C6 } + { B4 } { 201C } + { B8 } { 00F8 } + { BA } { 0157 } + { BF } { 00E6 } + { C0 } { 0104 } + { C1 } { 012E } + { C2 } { 0100 } + { C3 } { 0106 } + { C6 } { 0118 } + { C7 } { 0112 } + { C8 } { 010C } + { CA } { 0179 } + { CB } { 0116 } + { CC } { 0122 } + { CD } { 0136 } + { CE } { 012A } + { CF } { 013B } + { D0 } { 0160 } + { D1 } { 0143 } + { D2 } { 0145 } + { D4 } { 014C } + { D8 } { 0172 } + { D9 } { 0141 } + { DA } { 015A } + { DB } { 016A } + { DD } { 017B } + { DE } { 017D } + { E0 } { 0105 } + { E1 } { 012F } + { E2 } { 0101 } + { E3 } { 0107 } + { E6 } { 0119 } + { E7 } { 0113 } + { E8 } { 010D } + { EA } { 017A } + { EB } { 0117 } + { EC } { 0123 } + { ED } { 0137 } + { EE } { 012B } + { EF } { 013C } + { F0 } { 0161 } + { F1 } { 0144 } + { F2 } { 0146 } + { F4 } { 014D } + { F8 } { 0173 } + { F9 } { 0142 } + { FA } { 015B } + { FB } { 016B } + { FD } { 017C } + { FE } { 017E } + { FF } { 2019 } + } +\tl_const:cn { c_str_encoding_iso885913_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso885913.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885914.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885914.def new file mode 100644 index 00000000000..9cbba28749a --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885914.def @@ -0,0 +1,88 @@ +%% +%% This is file `l3str-iso885914.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso885914') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso885914 } +\tl_const:cn { c_str_encoding_iso885914_tl } + { + { A1 } { 1E02 } + { A2 } { 1E03 } + { A4 } { 010A } + { A5 } { 010B } + { A6 } { 1E0A } + { A8 } { 1E80 } + { AA } { 1E82 } + { AB } { 1E0B } + { AC } { 1EF2 } + { AF } { 0178 } + { B0 } { 1E1E } + { B1 } { 1E1F } + { B2 } { 0120 } + { B3 } { 0121 } + { B4 } { 1E40 } + { B5 } { 1E41 } + { B7 } { 1E56 } + { B8 } { 1E81 } + { B9 } { 1E57 } + { BA } { 1E83 } + { BB } { 1E60 } + { BC } { 1EF3 } + { BD } { 1E84 } + { BE } { 1E85 } + { BF } { 1E61 } + { D0 } { 0174 } + { D7 } { 1E6A } + { DE } { 0176 } + { F0 } { 0175 } + { F7 } { 1E6B } + { FE } { 0177 } + } +\tl_const:cn { c_str_encoding_iso885914_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso885914.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885915.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885915.def new file mode 100644 index 00000000000..1501f0308c2 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885915.def @@ -0,0 +1,65 @@ +%% +%% This is file `l3str-iso885915.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso885915') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso885915 } +\tl_const:cn { c_str_encoding_iso885915_tl } + { + { A4 } { 20AC } + { A6 } { 0160 } + { A8 } { 0161 } + { B4 } { 017D } + { B8 } { 017E } + { BC } { 0152 } + { BD } { 0153 } + { BE } { 0178 } + } +\tl_const:cn { c_str_encoding_iso885915_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso885915.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885916.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885916.def new file mode 100644 index 00000000000..84d3e86d0e5 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso885916.def @@ -0,0 +1,97 @@ +%% +%% This is file `l3str-iso885916.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso885916') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso885916 } +\tl_const:cn { c_str_encoding_iso885916_tl } + { + { A1 } { 0104 } + { A2 } { 0105 } + { A3 } { 0141 } + { A4 } { 20AC } + { A5 } { 201E } + { A6 } { 0160 } + { A8 } { 0161 } + { AA } { 0218 } + { AC } { 0179 } + { AE } { 017A } + { AF } { 017B } + { B2 } { 010C } + { B3 } { 0142 } + { B4 } { 017D } + { B5 } { 201D } + { B8 } { 017E } + { B9 } { 010D } + { BA } { 0219 } + { BC } { 0152 } + { BD } { 0153 } + { BE } { 0178 } + { BF } { 017C } + { C3 } { 0102 } + { C5 } { 0106 } + { D0 } { 0110 } + { D1 } { 0143 } + { D5 } { 0150 } + { D7 } { 015A } + { D8 } { 0170 } + { DD } { 0118 } + { DE } { 021A } + { E3 } { 0103 } + { E5 } { 0107 } + { F0 } { 0111 } + { F1 } { 0144 } + { F5 } { 0151 } + { F7 } { 015B } + { F8 } { 0171 } + { FD } { 0119 } + { FE } { 021B } + } +\tl_const:cn { c_str_encoding_iso885916_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso885916.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88592.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88592.def new file mode 100644 index 00000000000..cf3fdb20f03 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88592.def @@ -0,0 +1,114 @@ +%% +%% This is file `l3str-iso88592.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88592') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88592 } +\tl_const:cn { c_str_encoding_iso88592_tl } + { + { A1 } { 0104 } + { A2 } { 02D8 } + { A3 } { 0141 } + { A5 } { 013D } + { A6 } { 015A } + { A9 } { 0160 } + { AA } { 015E } + { AB } { 0164 } + { AC } { 0179 } + { AE } { 017D } + { AF } { 017B } + { B1 } { 0105 } + { B2 } { 02DB } + { B3 } { 0142 } + { B5 } { 013E } + { B6 } { 015B } + { B7 } { 02C7 } + { B9 } { 0161 } + { BA } { 015F } + { BB } { 0165 } + { BC } { 017A } + { BD } { 02DD } + { BE } { 017E } + { BF } { 017C } + { C0 } { 0154 } + { C3 } { 0102 } + { C5 } { 0139 } + { C6 } { 0106 } + { C8 } { 010C } + { CA } { 0118 } + { CC } { 011A } + { CF } { 010E } + { D0 } { 0110 } + { D1 } { 0143 } + { D2 } { 0147 } + { D5 } { 0150 } + { D8 } { 0158 } + { D9 } { 016E } + { DB } { 0170 } + { DE } { 0162 } + { E0 } { 0155 } + { E3 } { 0103 } + { E5 } { 013A } + { E6 } { 0107 } + { E8 } { 010D } + { EA } { 0119 } + { EC } { 011B } + { EF } { 010F } + { F0 } { 0111 } + { F1 } { 0144 } + { F2 } { 0148 } + { F5 } { 0151 } + { F8 } { 0159 } + { F9 } { 016F } + { FB } { 0171 } + { FE } { 0163 } + { FF } { 02D9 } + } +\tl_const:cn { c_str_encoding_iso88592_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso88592.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88593.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88593.def new file mode 100644 index 00000000000..f9c9c5cf174 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88593.def @@ -0,0 +1,92 @@ +%% +%% This is file `l3str-iso88593.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88593') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88593 } +\tl_const:cn { c_str_encoding_iso88593_tl } + { + { A1 } { 0126 } + { A2 } { 02D8 } + { A6 } { 0124 } + { A9 } { 0130 } + { AA } { 015E } + { AB } { 011E } + { AC } { 0134 } + { AF } { 017B } + { B1 } { 0127 } + { B6 } { 0125 } + { B9 } { 0131 } + { BA } { 015F } + { BB } { 011F } + { BC } { 0135 } + { BF } { 017C } + { C5 } { 010A } + { C6 } { 0108 } + { D5 } { 0120 } + { D8 } { 011C } + { DD } { 016C } + { DE } { 015C } + { E5 } { 010B } + { E6 } { 0109 } + { F5 } { 0121 } + { F8 } { 011D } + { FD } { 016D } + { FE } { 015D } + { FF } { 02D9 } + } +\tl_const:cn { c_str_encoding_iso88593_missing_tl } + { + { A5 } + { AE } + { BE } + { C3 } + { D0 } + { E3 } + { F0 } + } +%% +%% +%% End of file `l3str-iso88593.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88594.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88594.def new file mode 100644 index 00000000000..77edd29fe07 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88594.def @@ -0,0 +1,107 @@ +%% +%% This is file `l3str-iso88594.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88594') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88594 } +\tl_const:cn { c_str_encoding_iso88594_tl } + { + { A1 } { 0104 } + { A2 } { 0138 } + { A3 } { 0156 } + { A5 } { 0128 } + { A6 } { 013B } + { A9 } { 0160 } + { AA } { 0112 } + { AB } { 0122 } + { AC } { 0166 } + { AE } { 017D } + { B1 } { 0105 } + { B2 } { 02DB } + { B3 } { 0157 } + { B5 } { 0129 } + { B6 } { 013C } + { B7 } { 02C7 } + { B9 } { 0161 } + { BA } { 0113 } + { BB } { 0123 } + { BC } { 0167 } + { BD } { 014A } + { BE } { 017E } + { BF } { 014B } + { C0 } { 0100 } + { C7 } { 012E } + { C8 } { 010C } + { CA } { 0118 } + { CC } { 0116 } + { CF } { 012A } + { D0 } { 0110 } + { D1 } { 0145 } + { D2 } { 014C } + { D3 } { 0136 } + { D9 } { 0172 } + { DD } { 0168 } + { DE } { 016A } + { E0 } { 0101 } + { E7 } { 012F } + { E8 } { 010D } + { EA } { 0119 } + { EC } { 0117 } + { EF } { 012B } + { F0 } { 0111 } + { F1 } { 0146 } + { F2 } { 014D } + { F3 } { 0137 } + { F9 } { 0173 } + { FD } { 0169 } + { FE } { 016B } + { FF } { 02D9 } + } +\tl_const:cn { c_str_encoding_iso88594_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso88594.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88595.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88595.def new file mode 100644 index 00000000000..a05b410dc1b --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88595.def @@ -0,0 +1,151 @@ +%% +%% This is file `l3str-iso88595.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88595') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88595 } +\tl_const:cn { c_str_encoding_iso88595_tl } + { + { A1 } { 0401 } + { A2 } { 0402 } + { A3 } { 0403 } + { A4 } { 0404 } + { A5 } { 0405 } + { A6 } { 0406 } + { A7 } { 0407 } + { A8 } { 0408 } + { A9 } { 0409 } + { AA } { 040A } + { AB } { 040B } + { AC } { 040C } + { AE } { 040E } + { AF } { 040F } + { B0 } { 0410 } + { B1 } { 0411 } + { B2 } { 0412 } + { B3 } { 0413 } + { B4 } { 0414 } + { B5 } { 0415 } + { B6 } { 0416 } + { B7 } { 0417 } + { B8 } { 0418 } + { B9 } { 0419 } + { BA } { 041A } + { BB } { 041B } + { BC } { 041C } + { BD } { 041D } + { BE } { 041E } + { BF } { 041F } + { C0 } { 0420 } + { C1 } { 0421 } + { C2 } { 0422 } + { C3 } { 0423 } + { C4 } { 0424 } + { C5 } { 0425 } + { C6 } { 0426 } + { C7 } { 0427 } + { C8 } { 0428 } + { C9 } { 0429 } + { CA } { 042A } + { CB } { 042B } + { CC } { 042C } + { CD } { 042D } + { CE } { 042E } + { CF } { 042F } + { D0 } { 0430 } + { D1 } { 0431 } + { D2 } { 0432 } + { D3 } { 0433 } + { D4 } { 0434 } + { D5 } { 0435 } + { D6 } { 0436 } + { D7 } { 0437 } + { D8 } { 0438 } + { D9 } { 0439 } + { DA } { 043A } + { DB } { 043B } + { DC } { 043C } + { DD } { 043D } + { DE } { 043E } + { DF } { 043F } + { E0 } { 0440 } + { E1 } { 0441 } + { E2 } { 0442 } + { E3 } { 0443 } + { E4 } { 0444 } + { E5 } { 0445 } + { E6 } { 0446 } + { E7 } { 0447 } + { E8 } { 0448 } + { E9 } { 0449 } + { EA } { 044A } + { EB } { 044B } + { EC } { 044C } + { ED } { 044D } + { EE } { 044E } + { EF } { 044F } + { F0 } { 2116 } + { F1 } { 0451 } + { F2 } { 0452 } + { F3 } { 0453 } + { F4 } { 0454 } + { F5 } { 0455 } + { F6 } { 0456 } + { F7 } { 0457 } + { F8 } { 0458 } + { F9 } { 0459 } + { FA } { 045A } + { FB } { 045B } + { FC } { 045C } + { FD } { 00A7 } + { FE } { 045E } + { FF } { 045F } + } +\tl_const:cn { c_str_encoding_iso88595_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso88595.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88596.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88596.def new file mode 100644 index 00000000000..1cf3ac63bb4 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88596.def @@ -0,0 +1,137 @@ +%% +%% This is file `l3str-iso88596.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88596') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88596 } +\tl_const:cn { c_str_encoding_iso88596_tl } + { + { AC } { 060C } + { BB } { 061B } + { BF } { 061F } + { C1 } { 0621 } + { C2 } { 0622 } + { C3 } { 0623 } + { C4 } { 0624 } + { C5 } { 0625 } + { C6 } { 0626 } + { C7 } { 0627 } + { C8 } { 0628 } + { C9 } { 0629 } + { CA } { 062A } + { CB } { 062B } + { CC } { 062C } + { CD } { 062D } + { CE } { 062E } + { CF } { 062F } + { D0 } { 0630 } + { D1 } { 0631 } + { D2 } { 0632 } + { D3 } { 0633 } + { D4 } { 0634 } + { D5 } { 0635 } + { D6 } { 0636 } + { D7 } { 0637 } + { D8 } { 0638 } + { D9 } { 0639 } + { DA } { 063A } + { E0 } { 0640 } + { E1 } { 0641 } + { E2 } { 0642 } + { E3 } { 0643 } + { E4 } { 0644 } + { E5 } { 0645 } + { E6 } { 0646 } + { E7 } { 0647 } + { E8 } { 0648 } + { E9 } { 0649 } + { EA } { 064A } + { EB } { 064B } + { EC } { 064C } + { ED } { 064D } + { EE } { 064E } + { EF } { 064F } + { F0 } { 0650 } + { F1 } { 0651 } + { F2 } { 0652 } + } +\tl_const:cn { c_str_encoding_iso88596_missing_tl } + { + { A1 } + { A2 } + { A3 } + { A5 } + { A6 } + { A7 } + { A8 } + { A9 } + { AA } + { AB } + { AE } + { AF } + { B0 } + { B1 } + { B2 } + { B3 } + { B4 } + { B5 } + { B6 } + { B7 } + { B8 } + { B9 } + { BA } + { BC } + { BD } + { BE } + { C0 } + { DB } + { DC } + { DD } + { DE } + { DF } + } +%% +%% +%% End of file `l3str-iso88596.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88597.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88597.def new file mode 100644 index 00000000000..3b0b119a49d --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88597.def @@ -0,0 +1,136 @@ +%% +%% This is file `l3str-iso88597.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88597') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88597 } +\tl_const:cn { c_str_encoding_iso88597_tl } + { + { A1 } { 2018 } + { A2 } { 2019 } + { A4 } { 20AC } + { A5 } { 20AF } + { AA } { 037A } + { AF } { 2015 } + { B4 } { 0384 } + { B5 } { 0385 } + { B6 } { 0386 } + { B8 } { 0388 } + { B9 } { 0389 } + { BA } { 038A } + { BC } { 038C } + { BE } { 038E } + { BF } { 038F } + { C0 } { 0390 } + { C1 } { 0391 } + { C2 } { 0392 } + { C3 } { 0393 } + { C4 } { 0394 } + { C5 } { 0395 } + { C6 } { 0396 } + { C7 } { 0397 } + { C8 } { 0398 } + { C9 } { 0399 } + { CA } { 039A } + { CB } { 039B } + { CC } { 039C } + { CD } { 039D } + { CE } { 039E } + { CF } { 039F } + { D0 } { 03A0 } + { D1 } { 03A1 } + { D3 } { 03A3 } + { D4 } { 03A4 } + { D5 } { 03A5 } + { D6 } { 03A6 } + { D7 } { 03A7 } + { D8 } { 03A8 } + { D9 } { 03A9 } + { DA } { 03AA } + { DB } { 03AB } + { DC } { 03AC } + { DD } { 03AD } + { DE } { 03AE } + { DF } { 03AF } + { E0 } { 03B0 } + { E1 } { 03B1 } + { E2 } { 03B2 } + { E3 } { 03B3 } + { E4 } { 03B4 } + { E5 } { 03B5 } + { E6 } { 03B6 } + { E7 } { 03B7 } + { E8 } { 03B8 } + { E9 } { 03B9 } + { EA } { 03BA } + { EB } { 03BB } + { EC } { 03BC } + { ED } { 03BD } + { EE } { 03BE } + { EF } { 03BF } + { F0 } { 03C0 } + { F1 } { 03C1 } + { F2 } { 03C2 } + { F3 } { 03C3 } + { F4 } { 03C4 } + { F5 } { 03C5 } + { F6 } { 03C6 } + { F7 } { 03C7 } + { F8 } { 03C8 } + { F9 } { 03C9 } + { FA } { 03CA } + { FB } { 03CB } + { FC } { 03CC } + { FD } { 03CD } + { FE } { 03CE } + } +\tl_const:cn { c_str_encoding_iso88597_missing_tl } + { + { AE } + { D2 } + } +%% +%% +%% End of file `l3str-iso88597.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88598.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88598.def new file mode 100644 index 00000000000..400122b3c14 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88598.def @@ -0,0 +1,124 @@ +%% +%% This is file `l3str-iso88598.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88598') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88598 } +\tl_const:cn { c_str_encoding_iso88598_tl } + { + { AA } { 00D7 } + { BA } { 00F7 } + { DF } { 2017 } + { E0 } { 05D0 } + { E1 } { 05D1 } + { E2 } { 05D2 } + { E3 } { 05D3 } + { E4 } { 05D4 } + { E5 } { 05D5 } + { E6 } { 05D6 } + { E7 } { 05D7 } + { E8 } { 05D8 } + { E9 } { 05D9 } + { EA } { 05DA } + { EB } { 05DB } + { EC } { 05DC } + { ED } { 05DD } + { EE } { 05DE } + { EF } { 05DF } + { F0 } { 05E0 } + { F1 } { 05E1 } + { F2 } { 05E2 } + { F3 } { 05E3 } + { F4 } { 05E4 } + { F5 } { 05E5 } + { F6 } { 05E6 } + { F7 } { 05E7 } + { F8 } { 05E8 } + { F9 } { 05E9 } + { FA } { 05EA } + { FD } { 200E } + { FE } { 200F } + } +\tl_const:cn { c_str_encoding_iso88598_missing_tl } + { + { A1 } + { BF } + { C0 } + { C1 } + { C2 } + { C3 } + { C4 } + { C5 } + { C6 } + { C7 } + { C8 } + { C9 } + { CA } + { CB } + { CC } + { CD } + { CE } + { CF } + { D0 } + { D1 } + { D2 } + { D3 } + { D4 } + { D5 } + { D6 } + { D7 } + { D8 } + { D9 } + { DA } + { DB } + { DC } + { DD } + { DE } + { FB } + { FC } + } +%% +%% +%% End of file `l3str-iso88598.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88599.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88599.def new file mode 100644 index 00000000000..1d7effe7915 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-iso88599.def @@ -0,0 +1,63 @@ +%% +%% This is file `l3str-iso88599.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `iso88599') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\str_encoding_eight_bit:n { iso88599 } +\tl_const:cn { c_str_encoding_iso88599_tl } + { + { D0 } { 011E } + { DD } { 0130 } + { DE } { 015E } + { F0 } { 011F } + { FD } { 0131 } + { FE } { 015F } + } +\tl_const:cn { c_str_encoding_iso88599_missing_tl } + { + } +%% +%% +%% End of file `l3str-iso88599.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def new file mode 100644 index 00000000000..8d378c87e3d --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-name.def @@ -0,0 +1,102 @@ +%% +%% This is file `l3str-name.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `name') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\cs_set_protected:Npn \str_tmp:w #1#2#3 + { + \cs_new_protected_nopar:Npn #2 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN #3 \g_str_result_tl + #1 ? { ? \prg_map_break: } + \prg_break_point:n { } + } + } + \cs_new_nopar:Npn #3 ##1#1##2##3 + { + \str_filter_bytes:n {##1} + \use_none:n ##3 + \str_output_byte:w " + \str_aux_hexadecimal_use:NTF ##2 + { + \str_aux_hexadecimal_use:NTF ##3 + { } + { * \c_zero + `#1 \use_i:nn } + } + { 0 + `#1 \use_i:nn } + \str_output_end: + \use_i:nnn #3 ##2##3 + } + } +\exp_after:wN \str_tmp:w \c_hash_str + \str_convert_input_name: \str_convert_input_name_aux:wNN +\str_const:Nn \c_str_convert_output_name_not_str { ! " $ & ' } %$ +\str_const:Nn \c_str_convert_output_name_str { {}/<>[] } +\cs_new_protected_nopar:Npn \str_convert_output_name: + { \str_aux_gmap_result:N \str_convert_output_name_aux:N } +\cs_new_nopar:Npn \str_convert_output_name_aux:N #1 + { + \str_convert_output_name_aux:NTF #1 {#1} + { \c_hash_str \str_output_hexadecimal:n {`#1} } + } +\prg_new_conditional:Npnn \str_convert_output_name_aux:N #1 { TF } + { + \if_num:w `#1 < "2A \exp_stop_f: + \str_if_contains_char:NNTF \c_str_convert_output_name_not_str #1 + \prg_return_true: \prg_return_false: + \else: + \if_num:w `#1 > "7E \exp_stop_f: + \prg_return_false: + \else: + \str_if_contains_char:NNTF \c_str_convert_output_name_str #1 + \prg_return_false: \prg_return_true: + \fi: + \fi: + } +%% +%% +%% End of file `l3str-name.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def new file mode 100644 index 00000000000..7718a0d1b25 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-string.def @@ -0,0 +1,153 @@ +%% +%% This is file `l3str-string.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `string') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\group_begin: + \char_set_lccode:nn {`\*} {`\\} + \char_set_catcode_other:N \^^J + \char_set_catcode_other:N \^^M + \tl_to_lowercase:n + { + \cs_new_protected_nopar:Npn \str_convert_input_string: + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wN + \g_str_result_tl \prg_map_break: ^^M ? + \prg_break_point:n { } + } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_convert_input_string_aux:wNNN + \g_str_result_tl * ?? { ? \prg_map_break: } + \prg_break_point:n { } + } + } + \cs_new_nopar:Npn \str_convert_input_string_aux:wNNN #1 *#2#3#4 + } + { + \str_filter_bytes:n {#1} + \use_none:n #4 + \str_output_byte:w ' + \str_aux_octal_use:NTF #2 + { + \str_aux_octal_use:NTF #3 + { + \str_aux_octal_use:NTF #4 + { + \if_int_compare:w #2 > \c_three + - 256 + \fi: + \str_convert_input_string_aux:NNNNNN + } + { \str_convert_input_string_aux:NNNNNN ? } + } + { \str_convert_input_string_aux:NNNNNN ?? } + } + { + \prg_case_str:xxn {#2} + { + { \c_backslash_str } { 134 } + { ( } { 50 } + { ) } { 51 } + { r } { 15 } + { f } { 14 } + { n } { 12 } + { t } { 11 } + { b } { 10 } + { ^^J } { 0 - \c_one } + } + { 0 - \c_one \use_i:nn } + } + \str_output_end: + \use_i:nn \str_convert_input_string_aux:wNNN #2#3#4 + } + \cs_new_nopar:Npn \str_convert_input_string_aux:NNNNNN #1#2#3#4#5#6 + { \str_output_end: \str_convert_input_string_aux:wNNN } + \cs_new_nopar:Npn \str_convert_input_string_aux:wN #1 ^^M #2 + { + #1 ^^J + \if_charcode:w ^^J #2 + \exp_after:wN \use_i:nn + \fi: + \str_convert_input_string_aux:wN #2 + } +\group_end: +\str_const:Nx \c_str_convert_output_string_str + { \c_backslash_str ( ) } +\cs_new_protected_nopar:Npn \str_convert_output_string: + { \str_aux_gmap_result:N \str_convert_output_string_aux:N } +\cs_new_nopar:Npn \str_convert_output_string_aux:N #1 + { + \str_convert_output_string_aux:NTF #1 + { + \str_if_contains_char:NNT + \c_str_convert_output_string_str #1 + { \c_backslash_str } + #1 + } + { + \c_backslash_str + \int_div_truncate:nn {`#1} {64} + \int_mod:nn { \int_div_truncate:nn {`#1} \c_eight } \c_eight + \int_mod:nn {`#1} \c_eight + } + } +\prg_new_conditional:Npnn \str_convert_output_string_aux:N #1 { TF } + { + \if_num:w `#1 < "21 \exp_stop_f: + \prg_return_false: + \else: + \if_num:w `#1 > "7E \exp_stop_f: + \prg_return_false: + \else: + \prg_return_true: + \fi: + \fi: + } +%% +%% +%% End of file `l3str-string.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def new file mode 100644 index 00000000000..bf6bf96514d --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-url.def @@ -0,0 +1,100 @@ +%% +%% This is file `l3str-url.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `url') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\cs_set_protected:Npn \str_tmp:w #1#2#3 + { + \cs_new_protected_nopar:Npn #2 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN #3 \g_str_result_tl + #1 ? { ? \prg_map_break: } + \prg_break_point:n { } + } + } + \cs_new_nopar:Npn #3 ##1#1##2##3 + { + \str_filter_bytes:n {##1} + \use_none:n ##3 + \str_output_byte:w " + \str_aux_hexadecimal_use:NTF ##2 + { + \str_aux_hexadecimal_use:NTF ##3 + { } + { * \c_zero + `#1 \use_i:nn } + } + { 0 + `#1 \use_i:nn } + \str_output_end: + \use_i:nnn #3 ##2##3 + } + } +\exp_after:wN \str_tmp:w \c_percent_str + \str_convert_input_url: \str_convert_input_url_aux:wNN +\cs_new_protected_nopar:Npn \str_convert_output_url: + { \str_aux_gmap_result:N \str_convert_output_url_aux:N } +\cs_new_nopar:Npn \str_convert_output_url_aux:N #1 + { + \str_convert_output_url_aux:NTF #1 {#1} + { \c_percent_str \str_output_hexadecimal:n { `#1 } } + } +\prg_new_conditional:Npnn \str_convert_output_url_aux:N #1 { TF } + { + \if_num:w `#1 < "41 \exp_stop_f: + \str_if_contains_char:nNTF { "-.<> } #1 + \prg_return_true: \prg_return_false: + \else: + \if_num:w `#1 > "7E \exp_stop_f: + \prg_return_false: + \else: + \str_if_contains_char:nNTF { [ ] } #1 + \prg_return_false: \prg_return_true: + \fi: + \fi: + } +%% +%% +%% End of file `l3str-url.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def new file mode 100644 index 00000000000..e2bea7ca224 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf16.def @@ -0,0 +1,186 @@ +%% +%% This is file `l3str-utf16.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `utf16') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\group_begin: + \char_set_catcode_other:N \^^fe + \char_set_catcode_other:N \^^ff + \cs_new_protected_nopar:cpx { str_convert_to_utf16: } + { + \exp_not:c { str_convert_to_utf16be: } + \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^fe ^^ff } } + } + \cs_new_protected_nopar:cpn { str_convert_to_utf16be: } + { + \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_be:n + \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n + } + \cs_new_protected_nopar:cpn { str_convert_to_utf16le: } + { + \cs_set_eq:NN \str_tmp:w \str_aux_to_utf_xvi_le:n + \str_aux_gmap_internal_result:N \str_aux_to_utf_xvi:n + } + \cs_new_nopar:Npn \str_aux_to_utf_xvi:n #1 + { + \if_int_compare:w #1 < "D800 \exp_stop_f: + \str_tmp:w {#1} + \else: + \if_int_compare:w #1 < "10000 \exp_stop_f: + \if_int_compare:w #1 < "E000 \exp_stop_f: + \msg_expandable_kernel_error:nnn + { str } { unicode-surrogate } {#1} + \str_tmp:w { \c_str_replacement_char_int } + \else: + \str_tmp:w {#1} + \fi: + \else: + \exp_args:Nf \str_tmp:w { \int_div_truncate:nn {#1} {"400} + "D800 } + \exp_args:Nf \str_tmp:w { \int_mod:nn {#1} {"400} + "DC00 } + \fi: + \fi: + } + \cs_new_nopar:Npn \str_aux_to_utf_xvi_be:n #1 + { + \str_output_byte:n { \int_div_truncate:nn {#1} {"100} } + \str_output_byte:n { \int_mod:nn {#1} {"100} } + } + \cs_new_nopar:Npn \str_aux_to_utf_xvi_le:n #1 + { + \str_output_byte:n { \int_mod:nn {#1} {"100} } + \str_output_byte:n { \int_div_truncate:nn {#1} {"100} } + } + \cs_new_protected_nopar:cpn { str_convert_from_utf16be: } + { \str_aux_from_utf_xvi:No 1 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf16le: } + { \str_aux_from_utf_xvi:No 2 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf16: } + { + \exp_after:wN \str_aux_from_utf_xvi_bom:NNw + \g_str_result_tl \scan_stop: \scan_stop: \scan_stop: + } + \cs_new_protected_nopar:Npn + \str_aux_from_utf_xvi_bom:NNw #1#2#3\scan_stop: + { + \str_if_eq:nnTF { #1#2 } { ^^ff ^^fe } + { \str_aux_from_utf_xvi:No 2 {#3} } + { + \str_if_eq:nnTF { #1#2 } { ^^fe ^^ff } + { \str_aux_from_utf_xvi:No 1 {#3} } + { \str_aux_from_utf_xvi:No 1 {#1#2#3} } + } + } + \cs_new_protected_nopar:Npn \str_aux_from_utf_xvi:No #1#2 + { + \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_from_utf_xvi:NN + #2 \scan_stop: \scan_stop: + \prg_break_point:n { } + } + } + \cs_new:Npn \str_aux_from_utf_xvi:NN #1#2 + { + \if_meaning:w \scan_stop: #2 + \str_aux_from_utf_xvi_end:Nw #1 + \fi: + \int_use:N \int_eval:w + \if_case:w + \int_eval:w ( \str_tmp:w #1#2 - "D6 ) / \c_four \int_eval_end: + \or: \exp_after:wN \str_aux_from_utf_xvi:wNNN + \or: \exp_after:wN \str_aux_from_utf_xvi_error:wNNw + \fi: + "100 * \str_tmp:w #1#2 + \str_tmp:w #2#1 , + \str_aux_from_utf_xvi:NN + } + \cs_new:Npn \str_aux_from_utf_xvi:wNNN #1, #2#3#4 + { + \if_meaning:w \scan_stop: #4 + \str_aux_from_utf_xvi_end:wNNww #1 , #3 + \fi: + \if_num:w \int_eval:w ( \str_tmp:w #3#4 - "DA ) / \c_four = \c_one + ( #1 - "D800 ) * "400 + + \str_tmp:w #3#4 * "100 + \str_tmp:w #4#3 - "DC00 , + \exp_after:wN \use_i:nnn + \else: + \str_aux_from_utf_xvi_error:wNNw #1 , + \fi: + #2#3#4 + } + \cs_new:Npn \str_aux_from_utf_xvi_error:wNNw #1 \str_tmp:w #2#3 #4, + { + \msg_expandable_kernel_error:nnn + { str } { utf16-surrogate } { #2#3 } + \c_str_replacement_char_int , + } + \cs_new:Npn \str_aux_from_utf_xvi_end:Nw #1 \fi: + { + \fi: + \if_meaning:w \scan_stop: #1 + \else: + \msg_expandable_kernel_error:nnn + { str } { utf16-odd } {#1} + \c_str_replacement_char_int , + \fi: + \prg_map_break: + } + \cs_new:Npn \str_aux_from_utf_xvi_end:wNNww #1 `#2#3 #4, #5 \fi: + { + \fi: + \if_meaning:w \scan_stop: #5 + \msg_expandable_kernel_error:nnn + { str } { utf16-surrogate } { #2#3 } + \else: + \msg_expandable_kernel_error:nnn + { str } { utf16-odd } { #2#3#5 } + \fi: + \c_str_replacement_char_int , + \prg_map_break: + } +\group_end: +%% +%% +%% End of file `l3str-utf16.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def new file mode 100644 index 00000000000..7c56d2e3929 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf32.def @@ -0,0 +1,141 @@ +%% +%% This is file `l3str-utf32.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `utf32') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\group_begin: + \char_set_catcode_other:N \^^00 + \char_set_catcode_other:N \^^fe + \char_set_catcode_other:N \^^ff + \cs_new_protected_nopar:cpx { str_convert_to_utf32: } + { + \exp_not:c { str_convert_to_utf32be: } + \exp_not:n { \tl_gput_left:Nx \g_str_result_tl { ^^00 ^^00 ^^fe ^^ff } } + } + \cs_new_protected_nopar:cpn { str_convert_to_utf32be: } + { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_be:n } + \cs_new_protected_nopar:cpn { str_convert_to_utf32le: } + { \str_aux_gmap_internal_result:N \str_aux_to_utf_xxxii_le:n } + \cs_new:Npn \str_aux_to_utf_xxxii_be:n #1 + { + ^^00 + \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } } + \str_output_byte:n + { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} } + \str_output_byte:n { \int_mod:nn {#1} {"100} } + } + \cs_new:Npn \str_aux_to_utf_xxxii_le:n #1 + { + \str_output_byte:n { \int_mod:nn {#1} {"100} } + \str_output_byte:n + { \int_mod:nn { \int_div_truncate:nn {#1} {"100} } {"100} } + \str_output_byte:n { \int_div_truncate:nn {#1} { "10000 } } + ^^00 + } + \cs_new_protected_nopar:cpn { str_convert_from_utf32be: } + { \str_aux_from_utf_xxxii:No 1 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf32le: } + { \str_aux_from_utf_xxxii:No 2 { \g_str_result_tl } } + \cs_new_protected_nopar:cpn { str_convert_from_utf32: } + { + \exp_after:wN \str_aux_from_utf_xxxii_bom:NNNNw \g_str_result_tl + \scan_stop: \scan_stop: \scan_stop: \scan_stop: \scan_stop: + } + \cs_new_protected_nopar:Npn + \str_aux_from_utf_xxxii_bom:NNNNw #1#2#3#4#5\scan_stop: + { + \str_if_eq:nnTF { #1#2#3#4 } { ^^ff ^^fe ^^00 ^^00 } + { \str_aux_from_utf_xxxii:No 2 {#5} } + { + \str_if_eq:nnTF { #1#2#3#4 } { ^^00 ^^00 ^^fe ^^ff } + { \str_aux_from_utf_xxxii:No 1 {#5} } + { \str_aux_from_utf_xxxii:No 1 {#1#2#3#4#5} } + } + } + \cs_new_protected_nopar:Npn \str_aux_from_utf_xxxii:No #1#2 + { + \cs_set_nopar:Npn \str_tmp:w ##1 ##2 { ` ## #1 } + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_from_utf_xxxii:NNNN + #2 \scan_stop: \scan_stop: \scan_stop: \scan_stop: + \prg_break_point:n { } + } + } + \cs_new:Npn \str_aux_from_utf_xxxii:NNNN #1#2#3#4 + { + \if_meaning:w \scan_stop: #4 + \str_aux_from_utf_xxxii_end:ww #1#2#3#4 + \fi: + \int_use:N \int_eval:w + \if_num:w \int_eval:w + \str_tmp:w #1#4 * \c_two_hundred_fifty_six + + \str_tmp:w #2#3 + > \c_sixteen + \msg_expandable_kernel_error:nnn + { str } { utf32-overflow } { #1#2#3#4 } + \c_str_replacement_char_int + \else: + \str_tmp:w #2#3*"10000 + + \str_tmp:w #3#2 * \c_two_hundred_fifty_six + + \str_tmp:w #4#1 + \fi: + , + \str_aux_from_utf_xxxii:NNNN + } + \cs_new:Npn \str_aux_from_utf_xxxii_end:ww #1 \scan_stop: #2 \fi: + { + \fi: + \tl_if_empty:nF {#1} + { + \msg_expandable_kernel_error:nnn + { str } { utf32-truncated } { #1 } + } + \prg_map_break: + } +\group_end: +%% +%% +%% End of file `l3str-utf32.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def new file mode 100644 index 00000000000..9e0f26c2346 --- /dev/null +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str-utf8.def @@ -0,0 +1,159 @@ +%% +%% This is file `l3str-utf8.def', +%% generated with the docstrip utility. +%% +%% The original source files were: +%% +%% l3str.dtx (with options: `utf8') +%% +%% EXPERIMENTAL CODE +%% +%% Do not distribute this file without also distributing the +%% source files specified above. +%% +%% Do not distribute a modified version of this file. +%% +%% File: l3str.dtx Copyright (C) 2011 The LaTeX3 Project +%% +%% It may be distributed and/or modified under the conditions of the +%% LaTeX Project Public License (LPPL), either version 1.3c of this +%% license or (at your option) any later version. The latest version +%% of this license is in the file +%% +%% http://www.latex-project.org/lppl.txt +%% +%% This file is part of the "l3experimental bundle" (The Work in LPPL) +%% and all files in that bundle must be distributed together. +%% +%% The released version of this bundle is available from CTAN. +%% +%% ----------------------------------------------------------------------- +%% +%% The development version of the bundle can be found at +%% +%% http://www.latex-project.org/svnroot/experimental/trunk/ +%% +%% for those people who are interested. +%% +%%%%%%%%%%% +%% NOTE: %% +%%%%%%%%%%% +%% +%% Snapshots taken from the repository represent work in progress and may +%% not work or may contain conflicting material! We therefore ask +%% people _not_ to put them into distributions, archives, etc. without +%% prior consultation with the LaTeX3 Project. +%% +%% ----------------------------------------------------------------------- +\cs_new_protected_nopar:cpn { str_convert_to_utf8: } + { \str_aux_gmap_internal_result:N \str_aux_to_utf_viii:n } +\cs_new:Npn \str_aux_to_utf_viii:n #1 + { + \str_aux_to_utf_viii:nnnnw + {#1} { \c_minus_one + \c_zero * \use_none:n } + { 128 } { \c_zero } + { 32 } { 192 } + { 16 } { 224 } + { 8 } { 240 } + \q_stop + } +\cs_new_nopar:Npn \str_aux_to_utf_viii:nnnnw #1#2#3#4 #5 \q_stop + { + \if_num:w #1 < #3 \exp_stop_f: + \str_output_byte:n { #1 + #4 } + \else: + \exp_args:Nf \str_aux_to_utf_viii:nnnnw + { \int_div_truncate:nn {#1} {64} } + {#1} + #5 \q_stop + \fi: + \str_output_byte:n { #2 - 64 * ( #1 - \c_two ) } + } +\cs_new_protected_nopar:cpn { str_convert_from_utf8: } + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_from_utf_viii:N \g_str_result_tl + { ? \prg_map_break: \q_stop \str_aux_from_utf_viii_error: } + \prg_break_point:n { } + } + } +\cs_new_nopar:Npn \str_aux_from_utf_viii:N #1 + { + \use_none:n #1 + \int_use:N \int_eval:w + \if_num:w `#1 < "C0 \exp_stop_f: + \if_num:w `#1 < "80 \exp_stop_f: + `#1 + \else: + \msg_expandable_kernel_error:nnn + { str } { utf8-extra-conti } {#1} + \c_str_replacement_char_int + \fi: + \else: + \if_num:w `#1 < "F5 \exp_stop_f: + \exp_after:wN \str_aux_from_utf_viii:wwNN + \int_use:N \int_eval:w `#1 - "C0 + \else: + \msg_expandable_kernel_error:nnn + { str } { utf8-invalid-byte } {#1} + \c_str_replacement_char_int + \fi: + \fi: + , + \use_none_delimit_by_q_stop:w + "80 , "800 , "10000 , "10FFFF , + \q_stop + \str_aux_from_utf_viii:N + } +\cs_new_nopar:Npn \str_aux_from_utf_viii:wwNN #1 , #2 \q_stop #3#4 + { + \use_none_delimit_by_q_stop:w #4 \q_stop + \exp_after:wN \str_aux_from_utf_viii:wNNww + \int_use:N \int_eval:w #1 * "40 + `#4 - "80 , + #4 + #2 + \q_stop #3 + } +\cs_new_nopar:Npn \str_aux_from_utf_viii:wNNww #1, #2 #3#4, #5, + { + \if_num:w \int_eval:w ( `#2 + \c_thirty_two ) / "40 = \c_three + \if_num:w #1 < #4 \exp_stop_f: + \str_aux_from_utf_viii_error:w #1 , + \else: + \if_num:w #1 < #5 \exp_stop_f: + #1 + \else: + \exp_after:wN \str_aux_from_utf_viii:wwNN + \int_use:N \int_eval:w #1 - #5 + \fi: + \fi: + \else: + \exp_after:wN \str_aux_from_utf_viii_error:NwN + \exp_after:wN #2 + \fi: + , #3#5, + } +\cs_new_nopar:Npn \str_aux_from_utf_viii_error:w #1 , + { + \msg_expandable_kernel_error:nnn + { str } { utf8-overlong } {#1} + \c_str_replacement_char_int + } +\cs_new_nopar:Npn \str_aux_from_utf_viii_error:NwN #1 #2 \q_stop #3 + { + \msg_expandable_kernel_error:nnn + { str } { utf8-missing-conti } {#1} + \c_str_replacement_char_int , + #3#1 + } +\cs_new_nopar:Npn \str_aux_from_utf_viii_error: + { + \msg_expandable_kernel_error:nn + { str } { utf8-premature-end } + \c_str_replacement_char_int , + \prg_map_break: + } +%% +%% +%% End of file `l3str-utf8.def'. diff --git a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty index c705f6366b3..c49601841e0 100644 --- a/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty +++ b/Master/texmf-dist/tex/latex/l3experimental/l3str/l3str.sty @@ -46,12 +46,23 @@ %% %% ----------------------------------------------------------------------- \RequirePackage{expl3} -\GetIdInfo$Id: l3str.dtx 2921 2011-10-17 13:23:36Z bruno $ +\GetIdInfo$Id: l3str.dtx 3035 2011-12-07 11:48:46Z bruno $ {L3 Experimental Strings} \ProvidesExplPackage {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription} \cs_if_exist:NF \use_i:nnnnnnnn { \cs_new:Npn \use_i:nnnnnnnn #1#2#3#4#5#6#7#8 {#1} } +\int_const:Nn \c_forty_eight { 48 } +\int_const:Nn \c_fifty_eight { 58 } +\int_const:Nn \c_sixty_five { 65 } +\int_const:Nn \c_ninety_one { 91 } +\int_const:Nn \c_ninety_seven { 97 } +\int_const:Nn \c_one_hundred_twenty_three { 123 } +\int_const:Nn \c_one_hundred_twenty_seven { 127 } +\pdftex_if_engine:TF + { \int_const:Nn \c_max_char_int { 255 } } + { \int_const:Nn \c_max_char_int { 1114111 } } +\int_const:Nn \c_str_replacement_char_int { "FFFD } \group_begin: \cs_set_protected_nopar:Npn \str_tmp:w #1 { @@ -61,21 +72,35 @@ } \str_tmp:w { set } \str_tmp:w { gset } + \str_tmp:w { const } \str_tmp:w { put_left } \str_tmp:w { gput_left } \str_tmp:w { put_right } \str_tmp:w { gput_right } \group_end: \cs_new_protected_nopar:Npn \str_tmp:w { } -\tl_new:N \g_str_tmpa_tl +\tl_new:N \l_str_tmpa_tl \tl_new:N \g_str_result_tl \int_new:N \l_str_char_int -\int_new:N \l_str_bytes_int \tl_const:Nx \c_backslash_str { \cs_to_str:N \\ } \tl_const:Nx \c_lbrace_str { \cs_to_str:N \{ } \tl_const:Nx \c_rbrace_str { \cs_to_str:N \} } \tl_const:Nx \c_hash_str { \cs_to_str:N \# } +\tl_const:Nx \c_tilde_str { \cs_to_str:N \~ } \tl_const:Nx \c_percent_str { \cs_to_str:N \% } +\prop_new:N \g_str_aliases_prop +\prop_gput:Nnn \g_str_aliases_prop { latin1 } { iso88591 } +\prop_gput:Nnn \g_str_aliases_prop { latin2 } { iso88592 } +\prop_gput:Nnn \g_str_aliases_prop { latin3 } { iso88593 } +\prop_gput:Nnn \g_str_aliases_prop { latin4 } { iso88594 } +\prop_gput:Nnn \g_str_aliases_prop { latin5 } { iso88599 } +\prop_gput:Nnn \g_str_aliases_prop { latin6 } { iso885910 } +\prop_gput:Nnn \g_str_aliases_prop { latin7 } { iso885913 } +\prop_gput:Nnn \g_str_aliases_prop { latin8 } { iso885914 } +\prop_gput:Nnn \g_str_aliases_prop { latin9 } { iso885915 } +\prop_gput:Nnn \g_str_aliases_prop { latin10 } { iso885916 } +\prop_gput:Nnn \g_str_aliases_prop { hexadecimal } { hex } +\prop_gput:Nnn \g_str_aliases_prop { percent } { url } \group_begin: \char_set_lccode:nn { `\* } { `\ } \char_set_lccode:nn { `\A } { `\A } @@ -108,6 +133,61 @@ { \tl_to_other_str:n {#2} } { \tl_to_other_str:n {#3} } } +\group_begin: +\char_set_lccode:nn { `\* } { `\ } +\char_set_lccode:nn { `\A } { `\A } +\tl_to_lowercase:n + { + \group_end: + \cs_new_protected:Npn \str_aux_gset_other:Nn #1#2 + { + \tl_gset:Nx #1 + { + \exp_after:wN \str_aux_gset_other_loop:w \tl_to_str:n {#2} ~ % + A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ A ~ \q_mark \q_stop + } + } + \cs_new_nopar:Npn \str_aux_gset_other_loop:w + #1 ~ #2 ~ #3 ~ #4 ~ #5 ~ #6 ~ #7 ~ #8 ~ #9 ~ + { + \if_meaning:w A #9 + \str_aux_gset_other_end:w + \fi: + #1 * #2 * #3 * #4 * #5 * #6 * #7 * #8 * #9 + \str_aux_gset_other_loop:w * + } + \cs_new_nopar:Npn \str_aux_gset_other_end:w \fi: #1 * A #2 \q_stop + { \fi: #1 } + } +\cs_new_nopar:Npn \str_map_tokens:Nn + { \exp_args:No \str_map_tokens:nn } +\cs_new:Npn \str_map_tokens:nn #1 + { \exp_args:No \str_map_tokens_aux:nn { \tl_to_str:n {#1} } } +\cs_new:Npn \str_map_tokens_aux:nn #1#2 + { + \str_map_tokens_loop:nw {#2} #1 + { ? \use_none_delimit_by_q_recursion_stop:w } + ?~ ?~ ?~ ?~ ?~ ?~ ?~ ?~ \q_recursion_stop + } +\group_begin: + \char_set_lccode:nn { `* } { `\ } + \tl_to_lowercase:n + { + \group_end: + \cs_new:Npn \str_map_tokens_loop:nw #1 #2~ #3~ #4~ #5~ #6~ #7~ #8~ #9~ + { + \str_map_tokens_loop:nN {#1} #2* #3* #4* #5* #6* #7* #8* #9* + { ? \use_none_delimit_by_q_stop:w } \q_stop + \str_map_tokens_loop:nw {#1} + } + } +\cs_new:Npn \str_map_tokens_loop:nN #1#2 + { + \use_none:n #2 + #1 #2 + \str_map_tokens_loop:nN {#1} + } +\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw \cs_new_nopar:Npn \str_length:N { \exp_args:No \str_length:n } \cs_new:Npn \str_length:n { \str_sanitize_args:Nn \str_length_unsafe:n } \cs_new_nopar:Npn \str_length_unsafe:n #1 @@ -318,22 +398,6 @@ \fi: } } -\cs_new_nopar:Npn \str_map_tokens:Nn - { \exp_args:No \str_map_tokens:nn } -\cs_new_nopar:Npn \str_map_tokens:nn - { \str_sanitize_args:Nn \str_map_tokens_aux:nn } -\cs_new:Npn \str_map_tokens_aux:nn #1#2 - { - \str_map_tokens_loop:nN {#2} #1 - { ? \use_none_delimit_by_q_recursion_stop:w } \q_recursion_stop - } -\cs_new_nopar:Npn \str_map_tokens_loop:nN #1#2 - { - \use_none:n #2 - #1 #2 - \str_map_tokens_loop:nN {#1} - } -\cs_new_eq:NN \str_map_break_do:n \use_i_delimit_by_q_recursion_stop:nw \prg_new_conditional:Npnn \str_if_eq:NN #1#2 { p , TF , T , F } { \if_int_compare:w \pdftex_strcmp:D { \tl_to_str:N #1 } { \tl_to_str:N #2 } @@ -357,18 +421,29 @@ } \cs_new_nopar:Npn \str_if_contains_char_end:w \fi: #1 \prg_return_false: { \fi: \prg_return_true: } -\prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF } +\pdftex_if_engine:TF { - \str_map_tokens:Nn #1 { \str_if_bytes_aux:N } - \prg_return_true: + \cs_new_eq:NN \str_if_bytes_p:N \c_true_bool + \cs_new_eq:NN \str_if_bytes:NT \use_ii:nn + \cs_new_eq:NN \str_if_bytes:NF \use_none:nn + \cs_new_eq:NN \str_if_bytes:NTF \use_ii:nnn } -\cs_new_nopar:Npn \str_if_bytes_aux:N #1 { - \int_compare:nNnF {`#1} < \c_two_hundred_fifty_six + \prg_new_conditional:Npnn \str_if_bytes:N #1 { p , T , F , TF } { - \use_i_delimit_by_q_recursion_stop:nw - { \prg_return_false: \use_none:n } + \exp_args:Nf \tl_map_function:nN + { \tl_to_str:N #1 } + \str_if_bytes_aux:N + \prg_return_true: + } + \cs_new_nopar:Npn \str_if_bytes_aux:N #1 + { + \if_num:w `#1 > \c_two_hundred_fifty_five + \exp_after:wN \str_if_bytes_aux: + \fi: } + \cs_new_nopar:Npn \str_if_bytes_aux: + { \tl_map_break:n { \prg_return_false: \use_none:n } } } \prg_new_protected_conditional:Npnn \str_aux_hexadecimal_test:N #1 { TF } { @@ -388,18 +463,100 @@ \fi: } } -\prg_new_protected_conditional:Npnn \str_aux_octal_test:N #1 { TF } +\prg_new_conditional:Npnn \str_aux_octal_use:N #1 { TF } { - \if_num:w \c_seven < '1 \exp_not:N #1 \exp_stop_f: - \int_set:Nn \l_str_char_int - { \c_eight * \l_str_char_int + '#1 } - \prg_return_true: + \if_num:w \c_one < '1 #1 \exp_stop_f: + #1 \prg_return_true: \else: \prg_return_false: \fi: } -\cs_new_protected_nopar:Npn \str_aux_char_if_octal_digit:NTF #1 - { \tl_if_in:nnTF { 01234567 } {#1} } +\prg_new_conditional:Npnn \str_aux_hexadecimal_use:N #1 { TF } + { + \if_num:w \c_two < "1 #1 \exp_stop_f: + #1 \prg_return_true: + \else: + \if_case:w \int_eval:w `#1 - `a \int_eval_end: + A \prg_return_true: + \or: B \prg_return_true: + \or: C \prg_return_true: + \or: D \prg_return_true: + \or: E \prg_return_true: + \or: F \prg_return_true: + \else: \prg_return_false: + \fi: + \fi: + } +\group_begin: + \char_set_catcode_other:n { \c_zero } + \tl_gset:Nx \g_str_result_tl { \tl_to_str:n { 0123456789ABCDEF } } + \tl_map_inline:Nn \g_str_result_tl + { \char_set_lccode:nn {`#1} {`#1} } + \tl_map_inline:Nn \g_str_result_tl + { + \tl_map_inline:Nn \g_str_result_tl + { + \char_set_lccode:nn { \c_zero } {"##1#1} + \tl_to_lowercase:n + { + \tl_const:cx + { c_str_byte_ \int_eval:n {"##1#1} _tl } + { ^^@ ##1 #1 } + } + } + } +\group_end: +\tl_const:cn { c_str_byte_-1_tl } { { } \use_none:n { } } +\cs_new:Npn \str_output_byte:n #1 + { \str_output_byte:w #1 \str_output_end: } +\cs_new_nopar:Npn \str_output_byte:w + { + \exp_after:wN \exp_after:wN + \exp_after:wN \use_i:nnn + \cs:w c_str_byte_ \int_use:N \int_eval:w + } +\cs_new:Npn \str_output_hexadecimal:n #1 + { \str_output_hexadecimal:w #1 \str_output_end: } +\cs_new_nopar:Npn \str_output_hexadecimal:w + { + \exp_after:wN \exp_after:wN + \exp_after:wN \use_none:n + \cs:w c_str_byte_ \int_use:N \int_eval:w + } +\cs_new_nopar:Npn \str_output_end: + { \int_eval_end: _tl \cs_end: } +\cs_new_protected_nopar:Npn \str_aux_gmap_result:N #1 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_gmap_result_loop:NN + \exp_after:wN #1 + \g_str_result_tl { ? \prg_map_break: } + \prg_break_point:n { } + } + } +\cs_new_nopar:Npn \str_aux_gmap_result_loop:NN #1#2 + { + \use_none:n #2 + #1#2 + \str_aux_gmap_result_loop:NN #1 + } +\cs_new_protected_nopar:Npn \str_aux_gmap_internal_result:N #1 + { + \tl_gset:Nx \g_str_result_tl + { + \exp_after:wN \str_aux_gmap_internal_result_loop:Nw + \exp_after:wN #1 + \g_str_result_tl \q_stop \prg_map_break: , + \prg_break_point:n { } + } + } +\cs_new_nopar:Npn \str_aux_gmap_internal_result_loop:Nw #1#2, + { + \use_none_delimit_by_q_stop:w #2 \q_stop + #1 {#2} + \str_aux_gmap_internal_result_loop:Nw #1 + } \cs_new_protected:Npn \str_input:Nn #1#2 { \str_aux_escape:NNNn \token_to_str:N \token_to_str:N \token_to_str:N {#2} @@ -420,7 +577,7 @@ \cs_set_nopar:Npn \str_aux_escape_escaped:N { #2 } \cs_set_nopar:Npn \str_aux_escape_raw:N { #3 } \int_set:Nn \tex_escapechar:D { 92 } - \tl_gset:Nx \g_str_result_tl { \tl_to_other_str:n {#4} } + \str_aux_gset_other:Nn \g_str_result_tl {#4} \tl_gset:Nx \g_str_result_tl { \exp_after:wN \str_aux_escape_loop:N \g_str_result_tl @@ -507,7 +664,7 @@ \exp_not:N \token_if_eq_charcode:NNTF \c_rbrace_str #1 { \exp_not:N \str_aux_escape_x_end: } { - \msg_error:nn { str } { x-missing-brace } + \msg_kernel_error:nn { str } { x-missing-brace } \exp_not:N \str_aux_escape_x_end: #1 } } @@ -515,11 +672,9 @@ \char_set_catcode_other:N \^^@ \cs_new_protected_nopar:Npn \str_aux_escape_x_end: { - \group_begin: - \char_set_lccode:nn { \c_zero } { \l_str_char_int } + \tex_lccode:D \c_zero \l_str_char_int \tl_to_lowercase:n { - \group_end: \tl_gput_right:Nx \g_str_result_tl { \if_false: } \fi: \str_aux_escape_raw:N ^^@ @@ -527,440 +682,354 @@ } } \group_end: -\group_begin: - \char_set_catcode_other:n { `\^^@ } - \cs_new_protected_nopar:Npn \str_aux_convert_store: - { - \char_set_lccode:nn { \c_zero } { \l_str_char_int } - \tl_to_lowercase:n { \tl_gput_right:Nx \g_str_result_tl {^^@} } - } - \cs_new_protected_nopar:Npn \str_aux_convert_store:NNn #1#2#3 - { - \char_set_lccode:nn { \c_zero } {#3} - \tl_to_lowercase:n { #1 #2 {^^@} } - } -\group_end: -\cs_new_nopar:Npn \str_aux_byte_to_hexadecimal:N #1 +\cs_new_protected:Npn \str_set_convert:Nnnn + { \str_convert_aux_i:NNnnn \str_set:Nx } +\cs_new_protected:Npn \str_gset_convert:Nnnn + { \str_convert_aux_i:NNnnn \str_gset:Nx } +\cs_new_protected:Npn \str_convert_aux_i:NNnnn #1#2#3#4#5 + { + \group_begin: + \str_aux_gset_other:Nn \g_str_result_tl {#5} + + \exp_after:wN \str_convert_aux_ii:wwwnn + \tl_to_str:n {#3} /// \q_stop + { from } { input } \str_convert_from_native: \use_i:nn + + \exp_after:wN \str_convert_aux_ii:wwwnn + \tl_to_str:n {#4} /// \q_stop + { to } { output } \str_convert_to_native: \use_ii:nn + + \group_end: + #1 #2 \g_str_result_tl + } +\cs_new_protected:Npn \str_convert_aux_ii:wwwnn #1 / #2 // #3 \q_stop #4#5 + { + \str_convert_aux_iii:nnF {#4} {#1} {native} + \str_convert_aux_iii:nnF {#5} {#2} { } + \exp_args:Ncc \str_convert_aux_v:NNnNN + { str_convert_#4_#1: } { str_convert_#5_#2: } {#2} + } +\cs_new_protected:Npn \str_convert_aux_iii:nnF #1#2#3 { - \int_compare:nNnTF {`#1} < { 256 } + \cs_if_exist:cF { str_convert_#1_#2: } { - \int_to_letter:n { \int_div_truncate:nn {`#1} \c_sixteen } - \int_to_letter:n { \int_mod:nn {`#1} \c_sixteen } + \exp_args:Nx \str_convert_aux_iv:nnnF + { \str_aux_lowercase_alphanum:n {#2} } + {#1} {#2} {#3} } - { \msg_expandable_error:n { Invalid~byte~`#1'. } } } -\cs_new_nopar:Npn \str_aux_byte_to_octal:N #1 +\cs_new_protected:Npn \str_convert_aux_iv:nnnF #1#2#3#4 { - \int_compare:nNnTF {`#1} < { 64 } + \cs_if_exist:cF { str_convert_#2_#1: } { - 0 - \int_to_letter:n { \int_div_truncate:nn {`#1} \c_eight } - \int_to_letter:n { \int_mod:nn {`#1} \c_eight } + \group_begin: + \str_load_catcodes: + \str_load_one:n {#1} + \group_end: } + \cs_if_exist:cTF { str_convert_#2_#1: } + { \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#1: } } { - \int_compare:nNnTF {`#1} < { 256 } - { \int_to_octal:n {`#1} } - { \msg_expandable_error:n { Invalid~byte~`#1'. } } + \msg_kernel_error:nnx { str } { unknown-#2 } {#3} + \cs_gset_eq:cc { str_convert_#2_#3: } { str_convert_#2_#4: } } } -\cs_new_protected_nopar:Npn \str_bytes_escape_hexadecimal:NN #1#2 - { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_aux_byte_to_hexadecimal:N } - } -\tl_const:Nx \c_str_bytes_escape_name_str - { \c_hash_str \c_percent_str \c_lbrace_str \c_rbrace_str ()/<>[] } -\cs_new_protected_nopar:Npn \str_bytes_escape_name:NN #1#2 +\cs_new_protected:Npn \str_convert_aux_v:NNnNN #1#2#3#4#5 { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_bytes_escape_name_aux:N } - } -\cs_new_nopar:Npn \str_bytes_escape_name_aux:N #1 - { - \int_compare:nNnTF {`#1} < { "21 } - { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 } + #5 { } {#1} + \cs_if_eq:NNTF #1 #4 { - \int_compare:nNnTF {`#1} > { "7E } - { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 } - { - \str_if_contains_char:NNTF \c_str_bytes_escape_name_str #1 - { \c_hash_str \str_aux_byte_to_hexadecimal:N #1 } - {#1} - } + \tl_if_empty:nF {#3} + { \msg_kernel_error:nnx { str } { native-ignore-escaping } {#3} } } + { #2 } + #5 {#1} { } } -\tl_const:Nx \c_str_bytes_escape_string_str { \c_backslash_str ( ) } -\cs_new_protected_nopar:Npn \str_bytes_escape_string:NN #1#2 +\cs_new:Npn \str_aux_lowercase_alphanum:n #1 { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_bytes_escape_string_aux:N } + \exp_after:wN \str_aux_lowercase_alphanum_loop:N + \tl_to_str:n {#1} { ? \prg_map_break: } + \prg_break_point:n { } } -\cs_new_nopar:Npn \str_bytes_escape_string_aux:N #1 +\cs_new:Npn \str_aux_lowercase_alphanum_loop:N #1 { - \int_compare:nNnTF {`#1} < { "21 } - { \c_backslash_str \str_aux_byte_to_octal:N #1 } - { - \int_compare:nNnTF {`#1} > { "7E } - { \c_backslash_str \str_aux_byte_to_octal:N #1 } - { - \str_if_contains_char:NNT \c_str_bytes_escape_string_str #1 - { \c_backslash_str } - #1 - } - } + \use_none:n #1 + \if_num:w `#1 < \c_ninety_one + \if_num:w `#1 < \c_sixty_five + \if_num:w \c_one < 1#1 \exp_stop_f: + #1 + \fi: + \else: + \str_output_byte:n { `#1 + \c_thirty_two } + \fi: + \else: + \if_num:w `#1 < \c_one_hundred_twenty_three + \if_num:w `#1 < \c_ninety_seven + \else: + #1 + \fi: + \fi: + \fi: + \str_aux_lowercase_alphanum_loop:N } -\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal:NN #1#2 +\cs_new_protected_nopar:Npn \str_convert_from_internal: { } +\cs_new_protected_nopar:Npn \str_convert_to_internal: { } +\cs_new_protected:Npn \str_load:n #1 { \group_begin: - \tl_gclear:N \g_str_result_tl - \int_zero:N \l_str_char_int - \exp_last_unbraced:Nf \str_bytes_unescape_hexadecimal_aux:N - { \tl_to_other_str:N #2 } 0 - \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } -\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal_aux:N #1 - { - \quark_if_recursion_tail_stop:N #1 - \str_aux_hexadecimal_test:NTF #1 - { \str_bytes_unescape_hexadecimal_aux_ii:N } - { \str_bytes_unescape_hexadecimal_aux:N } - } -\cs_new_protected_nopar:Npn \str_bytes_unescape_hexadecimal_aux_ii:N #1 - { - \quark_if_recursion_tail_stop:N #1 - \str_aux_hexadecimal_test:NTF #1 - { - \str_aux_convert_store: - \int_zero:N \l_str_char_int - \str_bytes_unescape_hexadecimal_aux:N - } - { \str_bytes_unescape_hexadecimal_aux_ii:N } - } -\group_begin: - \char_set_lccode:nn {`\*} {`\#} - \tl_to_lowercase:n - { - \group_end: - \cs_new_protected_nopar:Npn \str_bytes_unescape_name:NN #1#2 - { - \group_begin: - \tl_gclear:N \g_str_result_tl - \int_zero:N \l_str_char_int - \exp_last_unbraced:Nf \str_bytes_unescape_name_aux:wN - { \tl_to_other_str:N #2 } - * \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } - \cs_new_protected_nopar:Npn \str_bytes_unescape_name_aux:wN #1 * #2 + \str_load_catcodes: + \clist_map_inline:nn {#1} { - \tl_gput_right:Nx \g_str_result_tl {#1} - \quark_if_recursion_tail_stop:N #2 - \str_aux_hexadecimal_test:NTF #2 - { \str_bytes_unescape_name_aux_ii:NN #2 } - { - \tl_gput_right:Nx \g_str_result_tl { \c_hash_str } - \str_bytes_unescape_name_aux:wN #2 - } + \exp_args:Nx \str_load_one:n + { \str_aux_lowercase_alphanum:n {#1} } } - } -\cs_new_protected_nopar:Npn \str_bytes_unescape_name_aux_ii:NN #1#2 - { - \str_aux_hexadecimal_test:NTF #2 - { - \str_aux_convert_store: - \int_zero:N \l_str_char_int - \str_bytes_unescape_name_aux:wN - } - { - \tl_gput_right:Nx \g_str_result_tl { \c_hash_str #1 } - \int_zero:N \l_str_char_int - \str_bytes_unescape_name_aux:wN #2 - } + \group_end: } -\group_begin: - \char_set_lccode:nn {`\*} {`\\} - \char_set_catcode_other:N \^^J - \char_set_catcode_other:N \^^M - \tl_to_lowercase:n - { - \group_end: - \cs_new_protected_nopar:Npn \str_bytes_unescape_string:NN #1#2 - { - \group_begin: - \tl_gclear:N \g_str_result_tl - \exp_last_unbraced:Nf \str_bytes_unescape_string_aux:wN - { \tl_to_other_str:N #2 } - * \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } - \cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux:wN #1 * #2 - { - \tl_gput_right:Nx \g_str_result_tl {#1} - \quark_if_recursion_tail_stop:N #2 - \int_zero:N \l_str_char_int - \str_aux_octal_test:NTF #2 - { \str_bytes_unescape_string_aux_d:N } - { - \int_set:Nn \l_str_char_int - { - \prg_case_str:xxn {#2} - { - { n } { 10 } - { r } { 13 } - { t } { 9 } - { b } { 8 } - { f } { 12 } - { ( } { 40 } - { ) } { 41 } - { \c_backslash_str } { 92 } - { ^^J } { -1 } - { ^^M } { -1 } - } - {`#2} - } - \int_compare:nNnF \l_str_char_int = \c_minus_one - { \str_aux_convert_store: } - \str_bytes_unescape_string_aux:wN - } - } - } -\cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux_d:N #1 +\cs_new_protected:Npn \str_load_one:n #1 { - \str_aux_octal_test:NTF #1 - { \str_bytes_unescape_string_aux_dd:N } + \cs_if_exist:cF { str_convert_from_ #1 : } { - \str_aux_convert_store: - \str_bytes_unescape_string_aux:wN #1 + \cs_if_exist:cF { str_convert_input_#1: } + { + \file_if_exist:nTF { l3str- #1 .def } + { \file_input:n { l3str- #1 .def } } + { \str_load_alias:n {#1} } + } } } -\cs_new_protected_nopar:Npn \str_bytes_unescape_string_aux_dd:N #1 +\cs_new_protected:Npn \str_load_alias:n #1 { - \str_aux_octal_test:NTF #1 - { - \str_aux_convert_store: - \str_bytes_unescape_string_aux:wN - } + \exp_args:NNx \prop_get:NnNTF \g_str_aliases_prop {#1} \l_str_tmpa_tl { - \str_aux_convert_store: - \str_bytes_unescape_string_aux:wN #1 + \str_load_one:n { \l_str_tmpa_tl } + \cs_if_exist:cTF + { str_convert_from_ \l_str_tmpa_tl : } + { + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { from } + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { to } + } + { + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { input } + \str_load_alias_aux:nnn {#1} { \l_str_tmpa_tl } { output } + } } + { \msg_kernel_error:nnx { str } { file-not-found } {#1} } + } +\cs_new_protected:Npn \str_load_alias_aux:nnn #1#2#3 + { \cs_new_eq:cc { str_convert_#3_#1: } { str_convert_#3_#2: } } +\cs_new_protected:Npn \str_load_catcodes: + { + \char_set_catcode_escape:N \\ + \char_set_catcode_group_begin:N \{ + \char_set_catcode_group_end:N \} + \char_set_catcode_math_superscript:N \^ + \char_set_catcode_ignore:N \ % + \tl_map_function:nN { abcdefghijklmnopqrstuvwxyz_:ABCDEFN } + \char_set_catcode_letter:N + \tl_map_function:nN { 0123456789"'?*+ } + \char_set_catcode_other:N + \char_set_catcode_comment:N \% + \int_set:Nn \tex_endlinechar:D {32} + } +\int_new:N \l_str_encoding_int +\cs_new_protected:Npn \str_convert_from_eight_bit:n #1 + { + \group_begin: + \int_zero:N \l_str_encoding_int + \exp_last_unbraced:Nx \str_convert_from_eight_bit_load:nn + { \tl_use:c { c_str_encoding_#1_tl } } + { \q_stop \prg_map_break: } { } + \prg_break_point:n { } + \exp_last_unbraced:Nx \str_convert_from_eight_bit_load_missing:n + { \tl_use:c { c_str_encoding_#1_missing_tl } } + { \q_stop \prg_map_break: } + \prg_break_point:n { } + \str_aux_gmap_result:N \str_convert_from_eight_bit_aux:N + \group_end: } -\tl_const:Nx \c_str_bytes_percent_encode_str { \tl_to_str:n { [] } } -\tl_const:Nx \c_str_bytes_percent_encode_not_str { \tl_to_str:n { "-.<> } } -\cs_new_protected_nopar:Npn \str_bytes_percent_encode:NN #1#2 +\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load:nn #1#2 { - \str_set:Nx #1 - { \str_map_tokens:Nn #2 \str_bytes_percent_encode_aux:N } + \use_none_delimit_by_q_stop:w #1 \q_stop + \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: + \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "#2 } + \tex_advance:D \l_str_encoding_int \c_one + \str_convert_from_eight_bit_load:nn } -\cs_new_nopar:Npn \str_bytes_percent_encode_aux:N #1 +\cs_new_protected_nopar:Npn \str_convert_from_eight_bit_load_missing:n #1 { - \int_compare:nNnTF {`#1} < { "41 } - { - \str_if_contains_char:NNTF \c_str_bytes_percent_encode_not_str #1 - { #1 } - { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 } - } - { - \int_compare:nNnTF {`#1} > { "7E } - { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 } - { - \str_if_contains_char:NNTF \c_str_bytes_percent_encode_str #1 - { \c_percent_str \str_aux_byte_to_hexadecimal:N #1 } - { #1 } - } - } + \use_none_delimit_by_q_stop:w #1 \q_stop + \tex_dimen:D "#1 = \l_str_encoding_int sp \scan_stop: + \tex_skip:D \l_str_encoding_int = "#1 sp \scan_stop: + \tex_toks:D \l_str_encoding_int \exp_after:wN { \int_value:w "FFFD } + \tex_advance:D \l_str_encoding_int \c_one + \str_convert_from_eight_bit_load_missing:n } -\group_begin: - \char_set_lccode:nn {`\*} {`\%} - \tl_to_lowercase:n - { - \group_end: - \cs_new_protected_nopar:Npn \str_bytes_percent_decode:NN #1#2 - { - \group_begin: - \tl_gclear:N \g_str_result_tl - \exp_last_unbraced:Nf \str_bytes_percent_decode_aux:wN - { \tl_to_other_str:N #2 } - * \q_recursion_tail \q_recursion_stop - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } - \cs_new_protected_nopar:Npn \str_bytes_percent_decode_aux:wN #1 * #2 - { - \tl_gput_right:Nx \g_str_result_tl {#1} - \quark_if_recursion_tail_stop:N #2 - \int_zero:N \l_str_char_int - \str_aux_hexadecimal_test:NTF #2 - { \str_bytes_percent_decode_aux_ii:NN #2 } - { - \tl_gput_right:Nx \g_str_result_tl { \c_percent_str } - \str_bytes_percent_decode_aux:wN #2 - } - } - } -\cs_new_protected_nopar:Npn \str_bytes_percent_decode_aux_ii:NN #1#2 +\cs_new:Npn \str_convert_from_eight_bit_aux:N #1 { - \str_aux_hexadecimal_test:NTF #2 - { - \str_aux_convert_store: - \str_bytes_percent_decode_aux:wN - } - { - \tl_gput_right:Nx \g_str_result_tl {#1} - \str_bytes_percent_decode_aux:wN #2 - } + \if_num:w \tex_dimen:D `#1 < \l_str_encoding_int + \if_num:w \tex_skip:D \tex_dimen:D `#1 = `#1 \exp_stop_f: + \tex_the:D \tex_toks:D \tex_dimen:D + \fi: + \fi: + \int_value:w `#1 , } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii:NN #1#2 +\cs_new_protected:Npn \str_convert_to_eight_bit:n #1 { \group_begin: - \tl_gclear:N \g_str_result_tl - \exp_last_unbraced:Nf \str_native_from_UTF_viii_aux_i:N - { \tl_to_other_str:N #2 } - \q_recursion_tail \q_recursion_stop + \int_zero:N \l_str_encoding_int + \exp_last_unbraced:Nx \str_convert_to_eight_bit_load:nn + { \tl_use:c { c_str_encoding_#1_tl } } + { \q_stop \prg_map_break: } { } + \prg_break_point:n { } + \str_aux_gmap_internal_result:N \str_convert_to_eight_bit_aux:n \group_end: - \tl_set_eq:NN #1 \g_str_result_tl } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_i:N #1 - { - \quark_if_recursion_tail_stop:N #1 - \int_set:Nn \l_str_char_int {`#1} - \str_native_from_UTF_viii_aux_ii:nN { 128 } \c_one - \str_native_from_UTF_viii_aux_ii:nN { 64 } \c_zero - \str_native_from_UTF_viii_aux_ii:nN { 32 } \c_two - \str_native_from_UTF_viii_aux_ii:nN { 16 } \c_three - \str_native_from_UTF_viii_aux_ii:nN { 8 } \c_four - \msg_error:nnx { str } { utf8-invalid-byte } {#1} - \use_i:nnn \str_native_from_UTF_viii_aux_i:N - \q_stop - \str_native_from_UTF_viii_aux_iii:N - } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_ii:nN #1#2 +\cs_new_protected_nopar:Npn \str_convert_to_eight_bit_load:nn #1#2 { - \int_compare:nNnTF \l_str_char_int < {#1} - { - \int_set_eq:NN \l_str_bytes_int #2 - \use_none_delimit_by_q_stop:w - } - { \int_sub:Nn \l_str_char_int {#1} } + \use_none_delimit_by_q_stop:w #1 \q_stop + \tex_dimen:D "#2 = \l_str_encoding_int sp \scan_stop: + \tex_skip:D \l_str_encoding_int = "#2 sp \scan_stop: + \exp_args:NNf \tex_toks:D \l_str_encoding_int + { \str_output_byte:n { "#1 } } + \tex_advance:D \l_str_encoding_int \c_one + \str_convert_to_eight_bit_load:nn } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_iii:N #1 +\cs_new:Npn \str_convert_to_eight_bit_aux:n #1 { - \int_compare:nNnTF \l_str_bytes_int < \c_two - { \str_native_from_UTF_viii_aux_iv: #1 } - { - \if_meaning:w \q_recursion_tail #1 - \msg_error:nn { str } { utf8-premature-end } - \exp_after:wN \use_none_delimit_by_q_recursion_stop:w + \if_num:w #1 < "8000 \exp_stop_f: + \if_num:w \tex_dimen:D #1 < \l_str_encoding_int + \if_num:w \tex_skip:D \tex_dimen:D #1 = #1 \exp_stop_f: + \tex_the:D \tex_toks:D \tex_dimen:D #1 \exp_stop_f: + \exp_after:wN \exp_after:wN \exp_after:wN \use_none:nn \fi: - \quark_if_recursion_tail_stop:N #1 - \tex_multiply:D \l_str_char_int by 64 \scan_stop: - \int_decr:N \l_str_bytes_int - \int_compare:nNnTF {`#1} < { 128 } - { \use_ii:nn } - { \int_compare:nNnTF {`#1} < { 192 } } - { - \int_add:Nn \l_str_char_int { `#1 - 128 } - \str_native_from_UTF_viii_aux_iii:N - } - { - \msg_error:nnx { str } { utf8-missing-byte } {#1} - \str_native_from_UTF_viii_aux_i:N #1 - } - } + \fi: + \str_convert_to_eight_bit_aux_ii:n {#1} + \else: + \msg_expandable_kernel_error:nnn + { str } { eight_bit-to-byte-overflow } {#1} + \fi: } -\cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_iv: +\cs_new:Npn \str_convert_to_eight_bit_aux_ii:n #1 { - \int_compare:nNnTF \l_str_bytes_int = \c_zero - { - \msg_error:nnx { str } { utf8-extra-byte } - { \int_eval:n { \l_str_char_int + 128 } } - } - { \str_native_from_UTF_viii_aux_v: } - \str_native_from_UTF_viii_aux_i:N + \if_num:w #1 < \c_two_hundred_fifty_six + \str_output_byte:n {#1} + \else: + \msg_expandable_kernel_error:nn { str } { eight-bit-to-byte } + \fi: + } +\cs_new_protected:Npn \str_encoding_eight_bit:n #1 + { + \cs_new_protected:cpn { str_convert_from_#1: } + { \str_convert_from_eight_bit:n {#1} } + \cs_new_protected:cpn { str_convert_to_#1: } + { \str_convert_to_eight_bit:n {#1} } } \pdftex_if_engine:TF + { \cs_new_eq:NN \str_filter_bytes:n \use:n } { - \cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_v: + \cs_new_nopar:Npn \str_filter_bytes:n #1 { - \int_compare:nNnTF { \l_str_char_int } < { 256 } - { \str_aux_convert_store: } - { - \msg_error:nnx { str } { utf8-pdftex-overflow } - { \int_use:N \l_str_char_int } - } + \str_filter_bytes_aux:N #1 + { ? \prg_map_break: } + \prg_break_point:n { } + } + \cs_new_nopar:Npn \str_filter_bytes_aux:N #1 + { + \use_none:n #1 + \if_num:w `#1 < 256 \exp_stop_f: #1 \fi: + \str_filter_bytes_aux:N } } +\pdftex_if_engine:TF + { \cs_new_protected_nopar:Npn \str_convert_input_: { } } { - \cs_new_protected_nopar:Npn \str_native_from_UTF_viii_aux_v: - { \str_aux_convert_store: } - } -\cs_new_protected_nopar:Npn \str_UTF_viii_from_native:NN #1#2 - { - \group_begin: - \tl_gclear:N \g_str_result_tl - \str_map_tokens:Nn #2 \str_UTF_viii_from_native_aux_i:N - \group_end: - \tl_set_eq:NN #1 \g_str_result_tl - } -\cs_new_protected_nopar:Npn \str_UTF_viii_from_native_aux_i:N #1 - { - \int_set:Nn \l_str_char_int {`#1} - \int_compare:nNnTF \l_str_char_int < { 128 } - { \tl_gput_right:Nx \g_str_result_tl {#1} } + \cs_new_protected_nopar:Npn \str_convert_input_: { - \tl_gclear:N \g_str_tmpa_tl - \int_set:Nn \l_str_bytes_int { 64 } - \str_UTF_viii_from_native_aux_ii:n { 32 } - \str_UTF_viii_from_native_aux_ii:n { 16 } - \str_UTF_viii_from_native_aux_ii:n { 8 } - \ERROR % somehow the unicode char was > "1FFFFF > "10FFFF - \tl_gclear:N \g_str_tmpa_tl - \use_none:n \q_stop - \int_add:Nn \l_str_char_int { 2 * \l_str_bytes_int } - \str_aux_convert_store: - \tl_gput_right:Nx \g_str_result_tl { \g_str_tmpa_tl } + \tl_gset:Nx \g_str_result_tl + { \exp_args:No \str_filter_bytes:n \g_str_result_tl } } } -\cs_new_protected_nopar:Npn \str_UTF_viii_from_native_aux_ii:n #1 - { - \str_aux_convert_store:NNn - \tl_gput_left:Nx \g_str_tmpa_tl - { 128 + \int_mod:nn { \l_str_char_int } { 64 } } - \tex_divide:D \l_str_char_int by 64 \scan_stop: - \int_add:Nn \l_str_bytes_int {#1} - \int_compare:nNnT \l_str_char_int < {#1} - { \use_none_delimit_by_q_stop:w } +\cs_new_eq:NN \str_convert_input_bytes: \str_convert_input_: +\cs_new_protected_nopar:Npn \str_convert_output_: { } +\cs_new_eq:NN \str_convert_output_bytes: \str_convert_output_: +\cs_new_protected_nopar:Npn \str_convert_from_native: + { \str_aux_gmap_result:N \str_convert_from_native_aux:N } +\cs_new:Npn \str_convert_from_native_aux:N #1 + { \int_value:w `#1 , } +\cs_new_protected_nopar:Npn \str_convert_to_native: + { + \tl_set_eq:NN \l_str_tmpa_tl \g_str_result_tl + \tl_gclear:N \g_str_result_tl + \tex_afterassignment:D \str_convert_to_native_aux:w + \l_str_char_int = \l_str_tmpa_tl \c_zero \prg_map_break: , + \prg_break_point:n { } } -\msg_new:nnnn { str } { x-missing-brace } +\group_begin: + \char_set_catcode_other:n { `\^^@ } + \cs_new_protected_nopar:Npn \str_convert_to_native_aux:w #1 , + { + #1 + \if_int_compare:w \l_str_char_int > \c_max_char_int + \msg_kernel_error:nnx { str } { overflow } + { \int_use:N \l_str_char_int } + \else: + \tex_lccode:D \c_zero \l_str_char_int + \tl_to_lowercase:n + { \tex_xdef:D \g_str_result_tl { \g_str_result_tl ^^@ } } + \fi: + \tex_afterassignment:D \str_convert_to_native_aux:w + \l_str_char_int = + } +\group_end: +\msg_kernel_new:nnnn { str } { x-missing-brace } { Missing~closing~brace~in~ \token_to_str:N \x ~byte~sequence. } { You~wrote~something~like~ `\iow_char:N\\x\{ \int_to_hexadecimal:n { \l_str_char_int }'.~ The~closing~brace~is~missing. } -\msg_new:nnn { str } { utf8-invalid-byte } - { - \int_compare:nNnTF {`#1} < { 256 } - { Byte~number~ \int_eval:n {`#1} ~invalid~in~utf-8~encoding. } - { The~character~number~ \int_eval:n {`#1} ~is~not~a~byte. } - } -\msg_new:nnn { str } { utf8-missing-byte } - { The~byte~number~ \int_eval:n {`#1} ~is~not~a~valid~continuation~byte. } -\msg_new:nnn { str } { utf8-extra-byte } - { The~byte~number~ \int_eval:n {`#1} ~is~only~valid~as~a~continuation~byte. } -\msg_new:nnnn { str } { utf8-premature-end } - { Incomplete~last~UTF8~character. } +\msg_kernel_new:nnnn { str } { overflow } + { Character~code~#1~too~big. } { - The~sequence~of~byte~that~to~be~converted~to~UTF8~ - ended~before~the~last~character~was~complete.~Perhaps~ - it~was~mistakenly~truncated? + \int_compare:nNnTF {#1} > { 1114111 } + { The~Unicode~standard~limits~code~points~to~1114111. } + { + The~pdfTeX~engine~only~supports~8-bit~characters:~ + valid~character~codes~are~in~the~range~[0,255].~ + To~manipulate~arbitrary~Unicode,~use~LuaTeX~or~XeTeX. + } } -\msg_new:nnn { str } { utf8-pdftex-overflow } - { The~character~number~#1~is~too~big~for~pdfTeX. } +\msg_kernel_new:nnn { str } { convert-input } + { Input~scheme~`#1'~unknown. } +\msg_kernel_new:nnn { str } { convert-from } + { Encoding~`#1'~unknown. } +\msg_kernel_new:nnn { str } { convert-to } + { Encoding~`#1'~unknown.~Using~UTF-8. } +\msg_kernel_new:nnn { str } { convert-output } + { Output~scheme~`#1~unknown. } +\msg_kernel_new:nnn { str } { to-native } + { Use~\str_set_convert:Nnn to~convert~to~native~strings. } +\msg_kernel_new:nnn { str } { unicode-surrogate } + { Code~point~#1~is~an~unpaired~surrogate. } +\msg_kernel_new:nnn { str } { utf16-surrogate } + { UTF-16~unpaired~surrogate~#1. } +\msg_kernel_new:nnn { str } { utf16-odd } + { UTF-16~dangling~byte~#1. } +\msg_kernel_new:nnn { str } { utf8-extra-conti } + { UTF-8~extra~continuation~byte~#1. } +\msg_kernel_new:nnn { str } { utf8-missing-conti } + { UTF-8~missing~continuation~byte~#1. } +\msg_kernel_new:nnn { str } { utf8-invalid-byte } + { Byte~#1~cannot~appear~in~UTF-8. } +\msg_kernel_new:nnn { str } { utf8-overlong } + { Overlong~UTF-8~byte~sequence~for~code~point~#1. } +\msg_kernel_new:nnn { str } { utf8-premature-end } + { Incomplete~last~UTF-8~character. } +\msg_kernel_new:nnn { str } { utf32-overflow } + { Code~point~too~large~(UTF-32~`#1'). } +\msg_kernel_new:nnn { str } { utf32-truncated } + { Truncated~UTF-32~string~`...#1'. } \cs_set:Npn \str_length_skip_spaces:N { \exp_args:No \str_length_skip_spaces:n } \cs_set_eq:NN \str_length_skip_spaces:n \str_length_ignore_spaces:n |