summaryrefslogtreecommitdiff
path: root/macros/latex-dev/required/l3kernel/l3cctab.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'macros/latex-dev/required/l3kernel/l3cctab.dtx')
-rw-r--r--macros/latex-dev/required/l3kernel/l3cctab.dtx975
1 files changed, 975 insertions, 0 deletions
diff --git a/macros/latex-dev/required/l3kernel/l3cctab.dtx b/macros/latex-dev/required/l3kernel/l3cctab.dtx
new file mode 100644
index 0000000000..dd35fb00a3
--- /dev/null
+++ b/macros/latex-dev/required/l3kernel/l3cctab.dtx
@@ -0,0 +1,975 @@
+% \iffalse meta-comment
+%
+%% File: l3cctab.dtx
+%
+% Copyright (C) 2018-2024 The LaTeX Project
+%
+% It may be distributed and/or modified under the conditions of the
+% LaTeX Project Public License (LPPL), either version 1.3c of this
+% license or (at your option) any later version. The latest version
+% of this license is in the file
+%
+% http://www.latex-project.org/lppl.txt
+%
+% This file is part of the "l3kernel bundle" (The Work in LPPL)
+% and all files in that bundle must be distributed together.
+%
+% -----------------------------------------------------------------------
+%
+% The development version of the bundle can be found at
+%
+% https://github.com/latex3/latex3
+%
+% for those people who are interested.
+%
+%<*driver>
+\documentclass[full]{l3doc}
+\begin{document}
+ \DocInput{\jobname.dtx}
+\end{document}
+%</driver>
+% \fi
+%
+% \title{^^A
+% The \pkg{l3cctab} module\\ Category code tables^^A
+% }
+%
+% \author{^^A
+% The \LaTeX{} Project\thanks
+% {^^A
+% E-mail:
+% \href{mailto:latex-team@latex-project.org}
+% {latex-team@latex-project.org}^^A
+% }^^A
+% }
+%
+% \date{Released 2024-04-11}
+%
+% \maketitle
+%
+% \begin{documentation}
+%
+% A category code table enables rapid switching of all category codes in
+% one operation. For \LuaTeX{}, this is possible over the entire Unicode
+% range. For other engines, only the $8$-bit range ($0$--$255$) is covered by
+% such tables. The implementation of category code tables in \pkg{expl3}
+% also saves and restores the \TeX{} \tn{endlinechar} primitive value, meaning
+% they could be used for example to implement \cs{ExplSyntaxOn}.
+%
+% \section{Creating and initialising category code tables}
+%
+% \begin{function}[updated = 2020-07-02]{\cctab_new:N,\cctab_new:c}
+% \begin{syntax}
+% \cs{cctab_new:N} \meta{category code table}
+% \end{syntax}
+% Creates a new \meta{category code table} variable or raises an error if
+% the name is already taken. The declaration is global. The
+% \meta{category code table} is initialised with the codes
+% as used by \IniTeX{}.
+% \end{function}
+%
+% \begin{function}[updated = 2020-07-07]{\cctab_const:Nn,\cctab_const:cn}
+% \begin{syntax}
+% \cs{cctab_const:Nn} \meta{category code table} \Arg{category code set up}
+% \end{syntax}
+% Creates a new \meta{category code table}, applies (in a group) the
+% \meta{category code set up} on top of \IniTeX{} settings,
+% then saves them globally as a constant
+% table. The \meta{category code set up} can include a call to
+% \cs{cctab_select:N}.
+% \end{function}
+%
+% \begin{function}[updated = 2020-07-07]{\cctab_gset:Nn,\cctab_gset:cn}
+% \begin{syntax}
+% \cs{cctab_gset:Nn} \meta{category code table} \Arg{category code set up}
+% \end{syntax}
+% Starting from the \IniTeX{} category codes,
+% applies (in a group) the \meta{category code set up}, then saves them
+% globally in the \meta{category code table}. The \meta{category code set up}
+% can include a call to \cs{cctab_select:N}.
+% \end{function}
+%
+% \begin{function}[added = 2023-05-26]
+% {\cctab_gsave_current:N,\cctab_gsave_current:c}
+% \begin{syntax}
+% \cs{cctab_gsave_current:N} \meta{category code table}
+% \end{syntax}
+% Saves the current prevailing category codes in the
+% \meta{category code table}.
+% \end{function}
+%
+% \section{Using category code tables}
+%
+% \begin{function}[updated = 2020-07-02]{\cctab_begin:N,\cctab_begin:c}
+% \begin{syntax}
+% \cs{cctab_begin:N} \meta{category code table}
+% \end{syntax}
+% Switches locally the category codes in force to those stored in the
+% \meta{category code table}. The prevailing codes before the
+% function is called are added to a stack, for use with
+% \cs{cctab_end:}. This function does not start a \TeX{} group.
+% \end{function}
+%
+% \begin{function}[updated = 2020-07-02]{\cctab_end:}
+% \begin{syntax}
+% \cs{cctab_end:}
+% \end{syntax}
+% Ends the scope of a \meta{category code table} started using
+% \cs{cctab_begin:N}, returning the codes to those in force before the
+% matching \cs{cctab_begin:N} was used. This must be used within the
+% same \TeX{} group (and at the same \TeX{} group level) as the
+% matching \cs{cctab_begin:N}.
+% \end{function}
+%
+% \begin{function}[added = 2020-05-19, updated = 2020-07-02]{\cctab_select:N, \cctab_select:c}
+% \begin{syntax}
+% \cs{cctab_select:N} \meta{category code table}
+% \end{syntax}
+% Selects the \meta{category code table} for the scope of the current
+% group. This is in particular useful in the \meta{setup} arguments
+% of \cs{tl_set_rescan:Nnn}, \cs{tl_rescan:nn}, \cs{cctab_const:Nn},
+% and \cs{cctab_gset:Nn}.
+% \end{function}
+%
+% \begin{function}[EXP, added = 2021-05-10]{\cctab_item:Nn, \cctab_item:cn}
+% \begin{syntax}
+% \cs{cctab_item:Nn} \meta{category code table} \Arg{int expr}
+% \end{syntax}
+% Determines the \meta{character} with character code given by the
+% \meta{int expr} and expands to its category code specified
+% by the \meta{category code table}.
+% \end{function}
+%
+% \section{Category code table conditionals}
+%
+% \begin{function}[pTF]{\cctab_if_exist:N, \cctab_if_exist:c}
+% \begin{syntax}
+% \cs{cctab_if_exist_p:N} \meta{category code table}
+% \cs{cctab_if_exist:NTF} \meta{category code table} \Arg{true code} \Arg{false code}
+% \end{syntax}
+% Tests whether the \meta{category code table} is currently defined.
+% This does not check that the \meta{category code table} really is a
+% category code table.
+% \end{function}
+%
+% \section{Constant and scratch category code tables}
+%
+% \begin{variable}[updated = 2020-07-10]{\c_code_cctab}
+% Category code table for the \pkg{expl3} code environment; this does
+% \emph{not} include \texttt{@}, which is retained as an \enquote{other}
+% character. Sets the \tn{endlinechar} value to $32$ (a space).
+% \end{variable}
+%
+% \begin{variable}[updated = 2020-07-08]{\c_document_cctab}
+% Category code table for a standard \LaTeX{} document, as set by the \LaTeX{}
+% kernel. In particular, the upper-half of the $8$-bit range will be set to
+% \enquote{active} with \pdfTeX{} \emph{only}. No \pkg{babel} shorthands
+% will be activated. Sets the \tn{endlinechar} value to $13$ (normal
+% line ending).
+% \end{variable}
+%
+% \begin{variable}[updated = 2020-07-02]{\c_initex_cctab}
+% Category code table as set up by \IniTeX{}.
+% \end{variable}
+%
+% \begin{variable}[updated = 2020-07-02]{\c_other_cctab}
+% Category code table where all characters have category code $12$
+% (other). Sets the \tn{endlinechar} value to $-1$.
+% \end{variable}
+%
+% \begin{variable}[updated = 2020-07-02]{\c_str_cctab}
+% Category code table where all characters have category code $12$
+% (other) with the exception of spaces, which have category code
+% $10$ (space). Sets the \tn{endlinechar} value to $-1$.
+% \end{variable}
+%
+% \begin{variable}[added = 2023-05-26]{\g_tmpa_cctab, \g_tmpb_cctab}
+% Scratch category code tables.
+% \end{variable}
+%
+% \end{documentation}
+%
+% \begin{implementation}
+%
+% \section{\pkg{l3cctab} implementation}
+%
+% \begin{macrocode}
+%<*package>
+% \end{macrocode}
+%
+% \begin{macrocode}
+%<@@=cctab>
+% \end{macrocode}
+%
+% As \LuaTeX{} offers engine support for category code tables, and this
+% is entirely lacking from the other engines, we need two complementary
+% approaches. (Some future \XeTeX{} may add support, at which point the
+% conditionals below would be different.)
+%
+% \subsection{Variables}
+%
+% \begin{variable}{\g_@@_stack_seq, \g_@@_unused_seq}
+% List of catcode tables saved by nested \cs{cctab_begin:N}, to
+% restore catcodes at the matching \cs{cctab_end:}. When popped from
+% the \cs{g_@@_stack_seq} the table numbers are stored in
+% \cs{g_@@_unused_seq} for later reuse.
+% \begin{macrocode}
+\seq_new:N \g_@@_stack_seq
+\seq_new:N \g_@@_unused_seq
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_@@_group_seq}
+% A stack to store the group level when a catcode table started.
+% \begin{macrocode}
+\seq_new:N \g_@@_group_seq
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_@@_allocate_int}
+% Integer to keep track of what category code table to allocate. In
+% \LuaTeX{} it is only used in format mode to implement
+% \cs{cctab_new:N}. In other engines it is used to make csnames for
+% dynamic tables.
+% \begin{macrocode}
+\int_new:N \g_@@_allocate_int
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\l_@@_internal_a_tl,\l_@@_internal_b_tl}
+% Scratch space. For instance, when popping
+% \cs{g_@@_stack_seq}/\cs{g_@@_unused_seq}, consists of the
+% catcodetable number (integer denotation) in \LuaTeX{}, or of an
+% intarray variable (as a single token) in other engines.
+% \begin{macrocode}
+\tl_new:N \l_@@_internal_a_tl
+\tl_new:N \l_@@_internal_b_tl
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_@@_endlinechar_prop}
+% In \LuaTeX{} we store the \tn{endlinechar} associated to each
+% \tn{catcodetable} in a property list, unless it is the default
+% value~$13$.
+% \begin{macrocode}
+\prop_new:N \g_@@_endlinechar_prop
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Allocating category code tables}
+%
+% \begin{macro}{\cctab_new:N, \cctab_new:c, \@@_new:N, \@@_gstore:Nnn}
+% The \cs{@@_new:N} auxiliary allocates a new catcode table but does
+% not attempt to set its value consistently across engines. It is
+% used both in \cs{cctab_new:N}, which sets catcodes to \IniTeX{}
+% values, and in \cs{cctab_begin:N}/\cs{cctab_end:} for dynamically
+% allocated tables.
+%
+% First, the \LuaTeX{} case.
+% Creating a new category code table is done like other registers.
+% In Con\TeX{}t, \tn{newcatcodetable} does not include the initialisation,
+% so that is added explicitly.
+% \begin{macrocode}
+\sys_if_engine_luatex:TF
+ {
+ \cs_new_protected:Npn \cctab_new:N #1
+ {
+ \__kernel_chk_if_free_cs:N #1
+ \@@_new:N #1
+ }
+ \cs_new_protected:Npn \@@_new:N #1
+ {
+ \newcatcodetable #1
+ \tex_initcatcodetable:D #1
+ }
+ }
+% \end{macrocode}
+% Now the case for other engines. Here, each table is an integer
+% array. Following the \LuaTeX{} pattern, a new table starts with
+% \IniTeX{} codes. The \cs{debug_suspend:} and \cs{debug_resume:}
+% functions prevent errors and logging from \texttt{debug} commands
+% which are either duplicate or false when \cs{@@_new:N} is used
+% by \cs{cctab_new:N} or \cs{cctab_const:Nn}.
+% The index base is out-by-one, so we have an
+% internal function to handle that. The \IniTeX{} \tn{endlinechar} is
+% $13$.
+% \begin{macrocode}
+ {
+ \cs_new_protected:Npn \@@_new:N #1
+ {
+ \debug_suspend:
+ \intarray_new:Nn #1 { 257 }
+ \debug_resume:
+ }
+ \cs_new_protected:Npn \@@_gstore:Nnn #1#2#3
+ { \intarray_gset:Nnn #1 { #2 + 1 } {#3} }
+ \cs_new_protected:Npn \cctab_new:N #1
+ {
+ \__kernel_chk_if_free_cs:N #1
+ \@@_new:N #1
+ \int_step_inline:nn { 256 }
+ { \__kernel_intarray_gset:Nnn #1 {##1} { 12 } }
+ \__kernel_intarray_gset:Nnn #1 { 257 } { 13 }
+ \@@_gstore:Nnn #1 { 0 } { 9 }
+ \@@_gstore:Nnn #1 { 13 } { 5 }
+ \@@_gstore:Nnn #1 { 32 } { 10 }
+ \@@_gstore:Nnn #1 { 37 } { 14 }
+ \int_step_inline:nnn { 65 } { 90 }
+ { \@@_gstore:Nnn #1 {##1} { 11 } }
+ \@@_gstore:Nnn #1 { 92 } { 0 }
+ \int_step_inline:nnn { 97 } { 122 }
+ { \@@_gstore:Nnn #1 {##1} { 11 } }
+ \@@_gstore:Nnn #1 { 127 } { 15 }
+ }
+ }
+\cs_generate_variant:Nn \cctab_new:N { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Saving category code tables}
+%
+% \begin{macro}{\@@_gset:n, \@@_gset_aux:n}
+% In various functions we need to save the current catcodes (globally)
+% in a table. In \LuaTeX{}, saving the catcodes is a primitives, but
+% the \tn{endlinechar} needs more work: to avoid filling
+% \cs{g_@@_endlinechar_prop} with many entries we special-case the
+% default value $13$. In other engines we store $256$ current
+% catcodes and the \tn{endlinechar} in an intarray variable.
+% \begin{macrocode}
+\sys_if_engine_luatex:TF
+ {
+ \cs_new_protected:Npn \@@_gset:n #1
+ { \exp_args:Nf \@@_gset_aux:n { \int_eval:n {#1} } }
+ \cs_new_protected:Npn \@@_gset_aux:n #1
+ {
+ \tex_savecatcodetable:D #1 \scan_stop:
+ \int_compare:nNnTF { \tex_endlinechar:D } = { 13 }
+ { \prop_gremove:Nn \g_@@_endlinechar_prop {#1} }
+ {
+ \prop_gput:NnV \g_@@_endlinechar_prop {#1}
+ \tex_endlinechar:D
+ }
+ }
+ }
+ {
+ \cs_new_protected:Npn \@@_gset:n #1
+ {
+ \int_step_inline:nn { 256 }
+ {
+ \__kernel_intarray_gset:Nnn #1 {##1}
+ { \char_value_catcode:n { ##1 - 1 } }
+ }
+ \__kernel_intarray_gset:Nnn #1 { 257 }
+ { \tex_endlinechar:D }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\cctab_gset:Nn, \cctab_gset:cn}
+% Category code tables are always global, so only one version of
+% assignments is needed. Simply run the setup in a group and save the
+% result in a category code table~|#1|, provided it is valid. The
+% internal function is defined above depending on the engine.
+% \begin{macrocode}
+\cs_new_protected:Npn \cctab_gset:Nn #1#2
+ {
+ \@@_chk_if_valid:NT #1
+ {
+ \group_begin:
+ \cctab_select:N \c_initex_cctab
+ #2 \scan_stop:
+ \@@_gset:n {#1}
+ \group_end:
+ }
+ }
+\cs_generate_variant:Nn \cctab_gset:Nn { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\cctab_gsave_current:N, \cctab_gsave_current:c}
+% Very simple.
+% \begin{macrocode}
+\cs_new_protected:Npn \cctab_gsave_current:N #1
+ {
+ \@@_chk_if_valid:NT #1
+ { \@@_gset:n {#1} }
+ }
+\cs_generate_variant:Nn \cctab_gsave_current:N { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Using category code tables}
+%
+% \begin{variable}{\g_@@_internal_cctab}
+% \begin{macro}[EXP]{\@@_internal_cctab_name:}
+% In \LuaTeX{}, we must ensure that the saved tables are read-only.
+% This is done by applying the saved table, then switching immediately
+% to a scratch table. Any later catcode assignment will affect that
+% scratch table rather than the saved one. If we simply switched to
+% the saved tables, then \cs{char_set_catcode_other:N} in the example
+% below would change \cs{c_document_cctab} and a later use of that
+% table would give the wrong category code to |_|.
+% \begin{verbatim}
+% \use:n
+% {
+% \cctab_begin:N \c_document_cctab
+% \char_set_catcode_other:N \_
+% \cctab_end:
+% \cctab_begin:N \c_document_cctab
+% \int_compare:nTF { \char_value_catcode:n { `_ } = 8 }
+% { \TRUE } { \ERROR }
+% \cctab_end:
+% }
+% \end{verbatim}
+% We must also make sure that a scratch table is never reused in a
+% nested group: in the following example, the scratch table used by
+% the first \cs{cctab_begin:N} would be changed globally by the second
+% one issuing \tn{savecatcodetable}, and after \cs{group_end:} the
+% wrong category codes (those of \cs{c_str_cctab}) would be imposed.
+% Note that the inner \cs{cctab_end:} restores the correct catcodes
+% only locally, so the problem really comes up because of the
+% different grouping level. The simplest is to use a scratch table
+% labeled by the \tn{currentgrouplevel}. We initialize one of them as
+% an example.
+% \begin{verbatim}
+% \use:n
+% {
+% \cctab_begin:N \c_document_cctab
+% \group_begin:
+% \cctab_begin:N \c_str_cctab
+% \cctab_end:
+% \group_end:
+% \cctab_end:
+% }
+% \end{verbatim}
+% \begin{macrocode}
+\sys_if_engine_luatex:T
+ {
+ \@@_new:N \g_@@_internal_cctab
+ \cs_new:Npn \@@_internal_cctab_name:
+ {
+ g_@@_internal
+ \tex_romannumeral:D \tex_currentgrouplevel:D
+ _cctab
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+%
+% \begin{macro}{\cctab_select:N, \cctab_select:c}
+% \begin{macro}{\@@_select:N}
+% The public function simply checks the \meta{cctab~var} exists before
+% using the engine-dependent \cs{@@_select:N}. Skipping these checks
+% would result in low-level engine-dependent errors. First, the
+% \LuaTeX{} case. In other engines, selecting a catcode table is a matter
+% of doing $256$ catcode assignments and setting the \tn{endlinechar}.
+% \begin{macrocode}
+\cs_new_protected:Npn \cctab_select:N #1
+ { \@@_chk_if_valid:NT #1 { \@@_select:N #1 } }
+\cs_generate_variant:Nn \cctab_select:N { c }
+\sys_if_engine_luatex:TF
+ {
+ \cs_new_protected:Npn \@@_select:N #1
+ {
+ \tex_catcodetable:D #1
+ \prop_get:NVNTF \g_@@_endlinechar_prop #1 \l_@@_internal_a_tl
+ { \int_set:Nn \tex_endlinechar:D { \l_@@_internal_a_tl } }
+ { \int_set:Nn \tex_endlinechar:D { 13 } }
+ \cs_if_exist:cF { \@@_internal_cctab_name: }
+ { \exp_args:Nc \@@_new:N { \@@_internal_cctab_name: } }
+ \exp_args:Nc \tex_savecatcodetable:D { \@@_internal_cctab_name: }
+ \exp_args:Nc \tex_catcodetable:D { \@@_internal_cctab_name: }
+ }
+ }
+ {
+ \cs_new_protected:Npn \@@_select:N #1
+ {
+ \int_step_inline:nn { 256 }
+ {
+ \char_set_catcode:nn { ##1 - 1 }
+ { \__kernel_intarray_item:Nn #1 {##1} }
+ }
+ \int_set:Nn \tex_endlinechar:D
+ { \__kernel_intarray_item:Nn #1 { 257 } }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \begin{variable}{\g_@@_next_cctab}
+% \begin{macro}{\@@_begin_aux:}
+% For \cs{cctab_begin:N}/\cs{cctab_end:} we will need to allocate
+% dynamic tables. This is done here by \cs{@@_begin_aux:}, which puts
+% a table number (in \LuaTeX{}) or name (in other engines) into
+% \cs{l_@@_internal_a_tl}. In \LuaTeX{} this simply calls \cs{@@_new:N}
+% and uses the resulting catcodetable number; in other engines we need
+% to give a name to the intarray variable and use that. In \LuaTeX{},
+% to restore catcodes at \cs{cctab_end:} we cannot just set
+% \tn{catcodetable} to its value before \cs{cctab_begin:N}, because
+% that table may have been altered by other code in the mean time. So
+% we must make sure to save the catcodes in a table we control and
+% restore them at \cs{cctab_end:}.
+% \begin{macrocode}
+\sys_if_engine_luatex:TF
+ {
+ \cs_new_protected:Npn \@@_begin_aux:
+ {
+ \@@_new:N \g_@@_next_cctab
+ \tl_set:NV \l_@@_internal_a_tl \g_@@_next_cctab
+ \cs_undefine:N \g_@@_next_cctab
+ }
+ }
+ {
+ \cs_new_protected:Npn \@@_begin_aux:
+ {
+ \int_gincr:N \g_@@_allocate_int
+ \exp_args:Nc \@@_new:N
+ { g_@@_ \int_use:N \g_@@_allocate_int _cctab }
+ \exp_args:NNc \tl_set:Nn \l_@@_internal_a_tl
+ { g_@@_ \int_use:N \g_@@_allocate_int _cctab }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{variable}
+%
+% \begin{macro}{\cctab_begin:N, \cctab_begin:c}
+% Check the \meta{cctab~var} exists, to avoid low-level errors. Get
+% in \cs{l_@@_internal_a_tl} the number/name of a dynamic table, either
+% from \cs{g_@@_unused_seq} where we save tables that are not
+% currently in use, or from \cs{@@_begin_aux:} if none are available.
+% Then save the current catcodes into the table (pointed to by)
+% \cs{l_@@_internal_a_tl} and save that table number in a stack before
+% selecting the desired catcodes.
+% \begin{macrocode}
+\cs_new_protected:Npn \cctab_begin:N #1
+ {
+ \@@_chk_if_valid:NT #1
+ {
+ \seq_gpop:NNF \g_@@_unused_seq \l_@@_internal_a_tl
+ { \@@_begin_aux: }
+ \@@_chk_group_begin:e
+ { \@@_nesting_number:N \l_@@_internal_a_tl }
+ \seq_gpush:NV \g_@@_stack_seq \l_@@_internal_a_tl
+ \exp_args:NV \@@_gset:n \l_@@_internal_a_tl
+ \@@_select:N #1
+ }
+ }
+\cs_generate_variant:Nn \cctab_begin:N { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\cctab_end:}
+% Make sure a \cs{cctab_begin:N} was used some time earlier, get in
+% \cs{l_@@_internal_a_tl} the catcode table number/name in which the
+% prevailing catcodes were stored, then restore these catcodes. The
+% dynamic table is now unused hence stored in \cs{g_@@_unused_seq} for
+% recycling by later \cs{cctab_begin:N}.
+% \begin{macrocode}
+\cs_new_protected:Npn \cctab_end:
+ {
+ \seq_gpop:NNTF \g_@@_stack_seq \l_@@_internal_a_tl
+ {
+ \seq_gpush:NV \g_@@_unused_seq \l_@@_internal_a_tl
+ \exp_args:Ne \@@_chk_group_end:n
+ { \@@_nesting_number:N \l_@@_internal_a_tl }
+ \@@_select:N \l_@@_internal_a_tl
+ }
+ { \msg_error:nn { cctab } { extra-end } }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}
+% {\@@_chk_group_begin:n,\@@_chk_group_begin:e, \@@_chk_group_end:n}
+% Catcode tables are not allowed to be intermixed with groups, so here
+% we check that they are properly nested regarding \TeX{} groups.
+% \cs{@@_chk_group_begin:n} stores the current group level in a stack,
+% and locally defines a dummy control sequence
+% \cs[no-index]{@@_group_\meta{cctab-level}_chk:}.
+%
+% \cs{@@_chk_group_end:n} pops the stack, and compares the returned
+% value with \cs{tex_currentgrouplevel:D}. If they differ,
+% \cs{cctab_end:} is in a different grouping level than the matching
+% \cs{cctab_begin:N}. If they are the same, both happened at the same
+% level, however a group might have ended and another started between
+% \cs{cctab_begin:N} and \cs{cctab_end:}:
+% \begin{verbatim}
+% \group_begin:
+% \cctab_begin:N \c_document_cctab
+% \group_end:
+% \group_begin:
+% \cctab_end:
+% \group_end:
+% \end{verbatim}
+% In this case checking \cs{tex_currentgrouplevel:D} is not enough, so
+% we locally define \cs[no-index]{@@_group_\meta{cctab-level}_chk:},
+% and then check if it exist in \cs{cctab_end:}. If it doesn't,
+% we know there was a group end where it shouldn't.
+%
+% The \meta{cctab-level} in the sentinel macro above cannot be
+% replaced by the more convenient \cs{tex_currentgrouplevel:D} because
+% with the latter we might be tricked. Suppose:
+% \begin{verbatim}
+% \group_begin:
+% \cctab_begin:N \c_code_cctab % A
+% \group_end:
+% \group_begin:
+% \cctab_begin:N \c_code_cctab % B
+% \cctab_end: % C
+% \cctab_end: % D
+% \group_end:
+% \end{verbatim}
+% The line marked with |A| would start a |cctab| with a sentinel token
+% named \cs[no-index]{@@_group_1_chk:}, which would disappear at the
+% \cs{group_end:} that follows. But |B| would create the same
+% sentinel token, since both are at the same group level. Line |C|
+% would end the |cctab| from line |B| correctly, but so would line |D|
+% because line |B| created the same sentinel token. Using
+% \meta{cctab-level} works correctly because it signals that certain
+% |cctab| level was activated somewhere, but if it doesn't exist when
+% the \cs{cctab_end:} is reached, we had a problem.
+%
+% Unfortunately these tests only flag the wrong usage at the
+% \cs{cctab_end:}, which might be far from the \cs{cctab_begin:N}.
+% However it isn't possible to signal the wrong usage at the
+% \cs{group_end:} without using \cs{tex_aftergroup:D}, which is
+% unsafe in certain types of groups.
+%
+% The three cases checked here just raise an error, and no recovery is
+% attempted: usually interleaving groups and catcode tables will work
+% predictably.
+% \begin{macrocode}
+\cs_new_protected:Npn \@@_chk_group_begin:n #1
+ {
+ \seq_gpush:Ne \g_@@_group_seq
+ { \int_use:N \tex_currentgrouplevel:D }
+ \cs_set_eq:cN { @@_group_ #1 _chk: } \prg_do_nothing:
+ }
+\cs_generate_variant:Nn \@@_chk_group_begin:n { e }
+\cs_new_protected:Npn \@@_chk_group_end:n #1
+ {
+ \seq_gpop:NN \g_@@_group_seq \l_@@_internal_b_tl
+ \bool_lazy_and:nnF
+ {
+ \int_compare_p:nNn
+ { \tex_currentgrouplevel:D } = { \l_@@_internal_b_tl }
+ }
+ { \cs_if_exist_p:c { @@_group_ #1 _chk: } }
+ {
+ \msg_error:nne { cctab } { group-mismatch }
+ {
+ \int_sign:n
+ { \tex_currentgrouplevel:D - \l_@@_internal_b_tl }
+ }
+ }
+ \cs_undefine:c { @@_group_ #1 _chk: }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}{\@@_nesting_number:N,\@@_nesting_number:w}
+% This macro returns the numeric index of the current catcode table.
+% In \LuaTeX{} this is just the argument, which is a count reference
+% to a \tn{catcodetable} register. In other engines, the number is
+% extracted from the |cctab| variable.
+% \begin{macrocode}
+\sys_if_engine_luatex:TF
+ { \cs_new:Npn \@@_nesting_number:N #1 {#1} }
+ {
+ \cs_new:Npn \@@_nesting_number:N #1
+ {
+ \exp_after:wN \exp_after:wN \exp_after:wN \@@_nesting_number:w
+ \exp_after:wN \token_to_str:N #1
+ }
+ \use:e
+ {
+ \cs_new:Npn \exp_not:N \@@_nesting_number:w
+ #1 \tl_to_str:n { g_@@_ } #2 \tl_to_str:n { _cctab } {#2}
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+%
+% Finally, install some code at the end of the \TeX{} run to check that
+% all \cs{cctab_begin:N} were ended by some \cs{cctab_end:}.
+% \begin{macrocode}
+\cs_if_exist:NT \hook_gput_code:nnn
+ {
+ \hook_gput_code:nnn { enddocument/end } { cctab }
+ {
+ \seq_if_empty:NF \g_@@_stack_seq
+ { \msg_error:nn { cctab } { missing-end } }
+ }
+ }
+% \end{macrocode}
+%
+%
+% \begin{macro}{\cctab_item:Nn, \cctab_item:cn}
+% Evaluate the integer argument only once. In most engines the
+% |cctab| variable only has $256$ entries so we only look up the
+% catcode for these entries, otherwise we use the current catcode. In
+% particular, for out-of-range values we use whatever fall-back
+% \cs{char_value_catcode:n}. In \LuaTeX{}, we use the
+% |tex.getcatcode| function.
+% \begin{macrocode}
+\cs_new:Npn \cctab_item:Nn #1#2
+ { \exp_args:Nf \@@_item:nN { \int_eval:n {#2} } #1 }
+\sys_if_engine_luatex:TF
+ {
+ \cs_new:Npn \@@_item:nN #1#2
+ { \lua_now:e { tex.print(-2, tex.getcatcode(\int_use:N #2, #1)) } }
+ }
+ {
+ \cs_new:Npn \@@_item:nN #1#2
+ {
+ \int_compare:nNnTF {#1} < { 256 }
+ { \intarray_item:Nn #2 { #1 + 1 } }
+ { \char_value_catcode:n {#1} }
+ }
+ }
+\cs_generate_variant:Nn \cctab_item:Nn { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \subsection{Category code table conditionals}
+%
+% \begin{macro}[pTF]{\cctab_if_exist:N,\cctab_if_exist:c}
+% Checks whether a \meta{cctab~var} is defined.
+% \begin{macrocode}
+\prg_new_eq_conditional:NNn \cctab_if_exist:N \cs_if_exist:N
+ { TF , T , F , p }
+\prg_new_eq_conditional:NNn \cctab_if_exist:c \cs_if_exist:c
+ { TF , T , F , p }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{macro}[TF]{\@@_chk_if_valid:N}
+% \begin{macro}{\@@_chk_if_valid_aux:NTF}
+% Checks whether the argument is defined and whether it is a valid
+% \meta{cctab~var}. In \LuaTeX{} the validity of the \meta{cctab~var}
+% is checked by the engine, which complains if the argument is not a
+% \cs{chardef}'ed constant. In other engines, check if the given
+% command is an intarray variable (the underlying definition is a copy
+% of the \texttt{cmr10} font).
+% \begin{macrocode}
+\prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1
+ { TF , T , F }
+ {
+ \cctab_if_exist:NTF #1
+ {
+ \@@_chk_if_valid_aux:NTF #1
+ { \prg_return_true: }
+ {
+ \msg_error:nne { cctab } { invalid-cctab }
+ { \token_to_str:N #1 }
+ \prg_return_false:
+ }
+ }
+ {
+ \msg_error:nne { kernel } { command-not-defined }
+ { \token_to_str:N #1 }
+ \prg_return_false:
+ }
+ }
+\sys_if_engine_luatex:TF
+ {
+ \cs_new_protected:Npn \@@_chk_if_valid_aux:NTF #1
+ {
+ \int_compare:nNnTF {#1-1} < { \e@alloc@ccodetable@count }
+ }
+ \cs_if_exist:NT \c_syst_catcodes_n
+ {
+ \cs_gset_protected:Npn \@@_chk_if_valid_aux:NTF #1
+ {
+ \int_compare:nTF { #1 <= \c_syst_catcodes_n }
+ }
+ }
+ }
+ {
+ \cs_new_protected:Npn \@@_chk_if_valid_aux:NTF #1
+ {
+ \exp_args:Nf \str_if_in:nnTF
+ { \cs_meaning:N #1 }
+ { select~font~cmr10~at~ }
+ }
+ }
+% \end{macrocode}
+% \end{macro}
+% \end{macro}
+%
+% \subsection{Constant category code tables}
+%
+% \begin{macro}{\cctab_const:Nn,\cctab_const:cn}
+% Creates a new \meta{cctab~var} then sets it with the \IniTeX{} and
+% user-supplied codes. To avoid false \texttt{debug} errors, we write
+% out implementation of \cs{cctab_new:N} and \cs{cctab_gset:Nn}
+% instead of directly using them here. The initialization part in
+% \cs{cctab_new:N} in non-\LuaTeX{} is omitted as it's covered by
+% the \IniTeX{} settings.
+% \begin{macrocode}
+\cs_new_protected:Npn \cctab_const:Nn #1#2
+ {
+ \__kernel_chk_if_free_cs:N #1
+ \@@_new:N #1
+ \group_begin:
+ \cctab_select:N \c_initex_cctab
+ #2 \scan_stop:
+ \@@_gset:n {#1}
+ \group_end:
+ }
+\cs_generate_variant:Nn \cctab_const:Nn { c }
+% \end{macrocode}
+% \end{macro}
+%
+% \begin{variable}
+% {
+% \c_initex_cctab ,
+% \c_other_cctab ,
+% \c_str_cctab
+% }
+% Creating category code tables means thinking starting from \IniTeX{}.
+% For all-other and the standard \enquote{string} tables that's easy.
+% \begin{macrocode}
+\cctab_new:N \c_initex_cctab
+\cctab_const:Nn \c_other_cctab
+ {
+ \cctab_select:N \c_initex_cctab
+ \int_set:Nn \tex_endlinechar:D { -1 }
+ \int_step_inline:nnn { 0 } { 127 }
+ { \char_set_catcode_other:n {#1} }
+ }
+\cctab_const:Nn \c_str_cctab
+ {
+ \cctab_select:N \c_other_cctab
+ \char_set_catcode_space:n { 32 }
+ }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\c_code_cctab, \c_document_cctab}
+% To pick up document-level category codes, we need to delay set up to the
+% end of the format, where that's possible. Also, as there are a \emph{lot}
+% of category codes to set, we avoid using the official interface and store the
+% document codes using internal code. Depending on whether we are in the hook
+% or not, the catcodes may be code or document, so we explicitly set up both
+% correctly.
+% \begin{macrocode}
+\cs_if_exist:NTF \@expl@finalise@setup@@@@
+ { \tl_gput_right:Nn \@expl@finalise@setup@@@@ }
+ { \use:n }
+ {
+ \@@_new:N \c_code_cctab
+ \group_begin:
+ \int_set:Nn \tex_endlinechar:D { 32 }
+ \char_set_catcode_invalid:n { 0 }
+ \bool_lazy_or:nnTF
+ { \sys_if_engine_xetex_p: } { \sys_if_engine_luatex_p: }
+ { \int_step_function:nN { 31 } \char_set_catcode_invalid:n }
+ { \int_step_function:nN { 31 } \char_set_catcode_active:n }
+ \int_step_function:nnN { 33 } { 64 } \char_set_catcode_other:n
+ \int_step_function:nnN { 65 } { 90 } \char_set_catcode_letter:n
+ \int_step_function:nnN { 91 } { 96 } \char_set_catcode_other:n
+ \int_step_function:nnN { 97 } { 122 } \char_set_catcode_letter:n
+ \char_set_catcode_ignore:n { 9 } % tab
+ \char_set_catcode_other:n { 10 } % lf
+ \char_set_catcode_active:n { 12 } % ff
+ \char_set_catcode_end_line:n { 13 } % cr
+ \char_set_catcode_ignore:n { 32 } % space
+ \char_set_catcode_parameter:n { 35 } % hash
+ \char_set_catcode_math_toggle:n { 36 } % dollar
+ \char_set_catcode_comment:n { 37 } % percent
+ \char_set_catcode_alignment:n { 38 } % ampersand
+ \char_set_catcode_letter:n { 58 } % colon
+ \char_set_catcode_escape:n { 92 } % backslash
+ \char_set_catcode_math_superscript:n { 94 } % circumflex
+ \char_set_catcode_letter:n { 95 } % underscore
+ \char_set_catcode_group_begin:n { 123 } % left brace
+ \char_set_catcode_other:n { 124 } % pipe
+ \char_set_catcode_group_end:n { 125 } % right brace
+ \char_set_catcode_space:n { 126 } % tilde
+ \char_set_catcode_invalid:n { 127 } % ^^?
+ \bool_lazy_or:nnF
+ { \sys_if_engine_xetex_p: } { \sys_if_engine_luatex_p: }
+ { \int_step_function:nnN { 128 } { 255 } \char_set_catcode_active:n }
+ \@@_gset:n { \c_code_cctab }
+ \group_end:
+ \cctab_const:Nn \c_document_cctab
+ {
+ \cctab_select:N \c_code_cctab
+ \int_set:Nn \tex_endlinechar:D { 13 }
+ \char_set_catcode_space:n { 9 }
+ \char_set_catcode_space:n { 32 }
+ \char_set_catcode_other:n { 58 }
+ \char_set_catcode_math_subscript:n { 95 }
+ \char_set_catcode_active:n { 126 }
+ }
+ }
+% \end{macrocode}
+% \end{variable}
+%
+% \begin{variable}{\g_tmpa_cctab, \g_tmpb_cctab}
+% \begin{macrocode}
+\cctab_new:N \g_tmpa_cctab
+\cctab_new:N \g_tmpb_cctab
+% \end{macrocode}
+% \end{variable}
+%
+% \subsection{Messages}
+%
+% \begin{macrocode}
+\msg_new:nnnn { cctab } { stack-full }
+ { The~category~code~table~stack~is~exhausted. }
+ {
+ LaTeX~has~been~asked~to~switch~to~a~new~category~code~table,~
+ but~there~is~no~more~space~to~do~this!
+ }
+\msg_new:nnnn { cctab } { extra-end }
+ { Extra~\iow_char:N\\cctab_end:~ignored~\msg_line_context:. }
+ {
+ LaTeX~came~across~a~\iow_char:N\\cctab_end:~without~a~matching~
+ \iow_char:N\\cctab_begin:N.~This~command~will~be~ignored.
+ }
+\msg_new:nnnn { cctab } { missing-end }
+ { Missing~\iow_char:N\\cctab_end:~before~end~of~TeX~run. }
+ {
+ LaTeX~came~across~more~\iow_char:N\\cctab_begin:N~than~
+ \iow_char:N\\cctab_end:.
+ }
+\msg_new:nnnn { cctab } { invalid-cctab }
+ { Invalid~\iow_char:N\\catcode~table. }
+ {
+ You~can~only~switch~to~a~\iow_char:N\\catcode~table~that~is~
+ initialized~using~\iow_char:N\\cctab_new:N~or~
+ \iow_char:N\\cctab_const:Nn.
+ }
+\msg_new:nnnn { cctab } { group-mismatch }
+ {
+ \iow_char:N\\cctab_end:~occurred~in~a~
+ \int_case:nn {#1}
+ {
+ { 0 } { different~group }
+ { 1 } { higher~group~level }
+ { -1 } { lower~group~level }
+ } ~than~
+ the~matching~\iow_char:N\\cctab_begin:N.
+ }
+ {
+ Catcode~tables~and~groups~must~be~properly~nested,~but~
+ you~tried~to~interleave~them.~LaTeX~will~try~to~proceed,~
+ but~results~may~be~unexpected.
+ }
+\prop_gput:Nnn \g_msg_module_name_prop { cctab } { LaTeX }
+\prop_gput:Nnn \g_msg_module_type_prop { cctab } { }
+% \end{macrocode}
+%
+% \begin{macrocode}
+%</package>
+% \end{macrocode}
+%
+% \end{implementation}
+%
+%\PrintIndex