diff options
Diffstat (limited to 'macros/latex/required/l3kernel/l3cctab.dtx')
| -rw-r--r-- | macros/latex/required/l3kernel/l3cctab.dtx | 975 |
1 files changed, 975 insertions, 0 deletions
diff --git a/macros/latex/required/l3kernel/l3cctab.dtx b/macros/latex/required/l3kernel/l3cctab.dtx new file mode 100644 index 0000000000..dd35fb00a3 --- /dev/null +++ b/macros/latex/required/l3kernel/l3cctab.dtx @@ -0,0 +1,975 @@ +% \iffalse meta-comment +% +%% File: l3cctab.dtx +% +% Copyright (C) 2018-2024 The LaTeX Project +% +% It may be distributed and/or modified under the conditions of the +% LaTeX Project Public License (LPPL), either version 1.3c of this +% license or (at your option) any later version. The latest version +% of this license is in the file +% +% http://www.latex-project.org/lppl.txt +% +% This file is part of the "l3kernel bundle" (The Work in LPPL) +% and all files in that bundle must be distributed together. +% +% ----------------------------------------------------------------------- +% +% The development version of the bundle can be found at +% +% https://github.com/latex3/latex3 +% +% for those people who are interested. +% +%<*driver> +\documentclass[full]{l3doc} +\begin{document} + \DocInput{\jobname.dtx} +\end{document} +%</driver> +% \fi +% +% \title{^^A +% The \pkg{l3cctab} module\\ Category code tables^^A +% } +% +% \author{^^A +% The \LaTeX{} Project\thanks +% {^^A +% E-mail: +% \href{mailto:latex-team@latex-project.org} +% {latex-team@latex-project.org}^^A +% }^^A +% } +% +% \date{Released 2024-04-11} +% +% \maketitle +% +% \begin{documentation} +% +% A category code table enables rapid switching of all category codes in +% one operation. For \LuaTeX{}, this is possible over the entire Unicode +% range. For other engines, only the $8$-bit range ($0$--$255$) is covered by +% such tables. The implementation of category code tables in \pkg{expl3} +% also saves and restores the \TeX{} \tn{endlinechar} primitive value, meaning +% they could be used for example to implement \cs{ExplSyntaxOn}. +% +% \section{Creating and initialising category code tables} +% +% \begin{function}[updated = 2020-07-02]{\cctab_new:N,\cctab_new:c} +% \begin{syntax} +% \cs{cctab_new:N} \meta{category code table} +% \end{syntax} +% Creates a new \meta{category code table} variable or raises an error if +% the name is already taken. The declaration is global. The +% \meta{category code table} is initialised with the codes +% as used by \IniTeX{}. +% \end{function} +% +% \begin{function}[updated = 2020-07-07]{\cctab_const:Nn,\cctab_const:cn} +% \begin{syntax} +% \cs{cctab_const:Nn} \meta{category code table} \Arg{category code set up} +% \end{syntax} +% Creates a new \meta{category code table}, applies (in a group) the +% \meta{category code set up} on top of \IniTeX{} settings, +% then saves them globally as a constant +% table. The \meta{category code set up} can include a call to +% \cs{cctab_select:N}. +% \end{function} +% +% \begin{function}[updated = 2020-07-07]{\cctab_gset:Nn,\cctab_gset:cn} +% \begin{syntax} +% \cs{cctab_gset:Nn} \meta{category code table} \Arg{category code set up} +% \end{syntax} +% Starting from the \IniTeX{} category codes, +% applies (in a group) the \meta{category code set up}, then saves them +% globally in the \meta{category code table}. The \meta{category code set up} +% can include a call to \cs{cctab_select:N}. +% \end{function} +% +% \begin{function}[added = 2023-05-26] +% {\cctab_gsave_current:N,\cctab_gsave_current:c} +% \begin{syntax} +% \cs{cctab_gsave_current:N} \meta{category code table} +% \end{syntax} +% Saves the current prevailing category codes in the +% \meta{category code table}. +% \end{function} +% +% \section{Using category code tables} +% +% \begin{function}[updated = 2020-07-02]{\cctab_begin:N,\cctab_begin:c} +% \begin{syntax} +% \cs{cctab_begin:N} \meta{category code table} +% \end{syntax} +% Switches locally the category codes in force to those stored in the +% \meta{category code table}. The prevailing codes before the +% function is called are added to a stack, for use with +% \cs{cctab_end:}. This function does not start a \TeX{} group. +% \end{function} +% +% \begin{function}[updated = 2020-07-02]{\cctab_end:} +% \begin{syntax} +% \cs{cctab_end:} +% \end{syntax} +% Ends the scope of a \meta{category code table} started using +% \cs{cctab_begin:N}, returning the codes to those in force before the +% matching \cs{cctab_begin:N} was used. This must be used within the +% same \TeX{} group (and at the same \TeX{} group level) as the +% matching \cs{cctab_begin:N}. +% \end{function} +% +% \begin{function}[added = 2020-05-19, updated = 2020-07-02]{\cctab_select:N, \cctab_select:c} +% \begin{syntax} +% \cs{cctab_select:N} \meta{category code table} +% \end{syntax} +% Selects the \meta{category code table} for the scope of the current +% group. This is in particular useful in the \meta{setup} arguments +% of \cs{tl_set_rescan:Nnn}, \cs{tl_rescan:nn}, \cs{cctab_const:Nn}, +% and \cs{cctab_gset:Nn}. +% \end{function} +% +% \begin{function}[EXP, added = 2021-05-10]{\cctab_item:Nn, \cctab_item:cn} +% \begin{syntax} +% \cs{cctab_item:Nn} \meta{category code table} \Arg{int expr} +% \end{syntax} +% Determines the \meta{character} with character code given by the +% \meta{int expr} and expands to its category code specified +% by the \meta{category code table}. +% \end{function} +% +% \section{Category code table conditionals} +% +% \begin{function}[pTF]{\cctab_if_exist:N, \cctab_if_exist:c} +% \begin{syntax} +% \cs{cctab_if_exist_p:N} \meta{category code table} +% \cs{cctab_if_exist:NTF} \meta{category code table} \Arg{true code} \Arg{false code} +% \end{syntax} +% Tests whether the \meta{category code table} is currently defined. +% This does not check that the \meta{category code table} really is a +% category code table. +% \end{function} +% +% \section{Constant and scratch category code tables} +% +% \begin{variable}[updated = 2020-07-10]{\c_code_cctab} +% Category code table for the \pkg{expl3} code environment; this does +% \emph{not} include \texttt{@}, which is retained as an \enquote{other} +% character. Sets the \tn{endlinechar} value to $32$ (a space). +% \end{variable} +% +% \begin{variable}[updated = 2020-07-08]{\c_document_cctab} +% Category code table for a standard \LaTeX{} document, as set by the \LaTeX{} +% kernel. In particular, the upper-half of the $8$-bit range will be set to +% \enquote{active} with \pdfTeX{} \emph{only}. No \pkg{babel} shorthands +% will be activated. Sets the \tn{endlinechar} value to $13$ (normal +% line ending). +% \end{variable} +% +% \begin{variable}[updated = 2020-07-02]{\c_initex_cctab} +% Category code table as set up by \IniTeX{}. +% \end{variable} +% +% \begin{variable}[updated = 2020-07-02]{\c_other_cctab} +% Category code table where all characters have category code $12$ +% (other). Sets the \tn{endlinechar} value to $-1$. +% \end{variable} +% +% \begin{variable}[updated = 2020-07-02]{\c_str_cctab} +% Category code table where all characters have category code $12$ +% (other) with the exception of spaces, which have category code +% $10$ (space). Sets the \tn{endlinechar} value to $-1$. +% \end{variable} +% +% \begin{variable}[added = 2023-05-26]{\g_tmpa_cctab, \g_tmpb_cctab} +% Scratch category code tables. +% \end{variable} +% +% \end{documentation} +% +% \begin{implementation} +% +% \section{\pkg{l3cctab} implementation} +% +% \begin{macrocode} +%<*package> +% \end{macrocode} +% +% \begin{macrocode} +%<@@=cctab> +% \end{macrocode} +% +% As \LuaTeX{} offers engine support for category code tables, and this +% is entirely lacking from the other engines, we need two complementary +% approaches. (Some future \XeTeX{} may add support, at which point the +% conditionals below would be different.) +% +% \subsection{Variables} +% +% \begin{variable}{\g_@@_stack_seq, \g_@@_unused_seq} +% List of catcode tables saved by nested \cs{cctab_begin:N}, to +% restore catcodes at the matching \cs{cctab_end:}. When popped from +% the \cs{g_@@_stack_seq} the table numbers are stored in +% \cs{g_@@_unused_seq} for later reuse. +% \begin{macrocode} +\seq_new:N \g_@@_stack_seq +\seq_new:N \g_@@_unused_seq +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_@@_group_seq} +% A stack to store the group level when a catcode table started. +% \begin{macrocode} +\seq_new:N \g_@@_group_seq +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_@@_allocate_int} +% Integer to keep track of what category code table to allocate. In +% \LuaTeX{} it is only used in format mode to implement +% \cs{cctab_new:N}. In other engines it is used to make csnames for +% dynamic tables. +% \begin{macrocode} +\int_new:N \g_@@_allocate_int +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\l_@@_internal_a_tl,\l_@@_internal_b_tl} +% Scratch space. For instance, when popping +% \cs{g_@@_stack_seq}/\cs{g_@@_unused_seq}, consists of the +% catcodetable number (integer denotation) in \LuaTeX{}, or of an +% intarray variable (as a single token) in other engines. +% \begin{macrocode} +\tl_new:N \l_@@_internal_a_tl +\tl_new:N \l_@@_internal_b_tl +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_@@_endlinechar_prop} +% In \LuaTeX{} we store the \tn{endlinechar} associated to each +% \tn{catcodetable} in a property list, unless it is the default +% value~$13$. +% \begin{macrocode} +\prop_new:N \g_@@_endlinechar_prop +% \end{macrocode} +% \end{variable} +% +% \subsection{Allocating category code tables} +% +% \begin{macro}{\cctab_new:N, \cctab_new:c, \@@_new:N, \@@_gstore:Nnn} +% The \cs{@@_new:N} auxiliary allocates a new catcode table but does +% not attempt to set its value consistently across engines. It is +% used both in \cs{cctab_new:N}, which sets catcodes to \IniTeX{} +% values, and in \cs{cctab_begin:N}/\cs{cctab_end:} for dynamically +% allocated tables. +% +% First, the \LuaTeX{} case. +% Creating a new category code table is done like other registers. +% In Con\TeX{}t, \tn{newcatcodetable} does not include the initialisation, +% so that is added explicitly. +% \begin{macrocode} +\sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \cctab_new:N #1 + { + \__kernel_chk_if_free_cs:N #1 + \@@_new:N #1 + } + \cs_new_protected:Npn \@@_new:N #1 + { + \newcatcodetable #1 + \tex_initcatcodetable:D #1 + } + } +% \end{macrocode} +% Now the case for other engines. Here, each table is an integer +% array. Following the \LuaTeX{} pattern, a new table starts with +% \IniTeX{} codes. The \cs{debug_suspend:} and \cs{debug_resume:} +% functions prevent errors and logging from \texttt{debug} commands +% which are either duplicate or false when \cs{@@_new:N} is used +% by \cs{cctab_new:N} or \cs{cctab_const:Nn}. +% The index base is out-by-one, so we have an +% internal function to handle that. The \IniTeX{} \tn{endlinechar} is +% $13$. +% \begin{macrocode} + { + \cs_new_protected:Npn \@@_new:N #1 + { + \debug_suspend: + \intarray_new:Nn #1 { 257 } + \debug_resume: + } + \cs_new_protected:Npn \@@_gstore:Nnn #1#2#3 + { \intarray_gset:Nnn #1 { #2 + 1 } {#3} } + \cs_new_protected:Npn \cctab_new:N #1 + { + \__kernel_chk_if_free_cs:N #1 + \@@_new:N #1 + \int_step_inline:nn { 256 } + { \__kernel_intarray_gset:Nnn #1 {##1} { 12 } } + \__kernel_intarray_gset:Nnn #1 { 257 } { 13 } + \@@_gstore:Nnn #1 { 0 } { 9 } + \@@_gstore:Nnn #1 { 13 } { 5 } + \@@_gstore:Nnn #1 { 32 } { 10 } + \@@_gstore:Nnn #1 { 37 } { 14 } + \int_step_inline:nnn { 65 } { 90 } + { \@@_gstore:Nnn #1 {##1} { 11 } } + \@@_gstore:Nnn #1 { 92 } { 0 } + \int_step_inline:nnn { 97 } { 122 } + { \@@_gstore:Nnn #1 {##1} { 11 } } + \@@_gstore:Nnn #1 { 127 } { 15 } + } + } +\cs_generate_variant:Nn \cctab_new:N { c } +% \end{macrocode} +% \end{macro} +% +% \subsection{Saving category code tables} +% +% \begin{macro}{\@@_gset:n, \@@_gset_aux:n} +% In various functions we need to save the current catcodes (globally) +% in a table. In \LuaTeX{}, saving the catcodes is a primitives, but +% the \tn{endlinechar} needs more work: to avoid filling +% \cs{g_@@_endlinechar_prop} with many entries we special-case the +% default value $13$. In other engines we store $256$ current +% catcodes and the \tn{endlinechar} in an intarray variable. +% \begin{macrocode} +\sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \@@_gset:n #1 + { \exp_args:Nf \@@_gset_aux:n { \int_eval:n {#1} } } + \cs_new_protected:Npn \@@_gset_aux:n #1 + { + \tex_savecatcodetable:D #1 \scan_stop: + \int_compare:nNnTF { \tex_endlinechar:D } = { 13 } + { \prop_gremove:Nn \g_@@_endlinechar_prop {#1} } + { + \prop_gput:NnV \g_@@_endlinechar_prop {#1} + \tex_endlinechar:D + } + } + } + { + \cs_new_protected:Npn \@@_gset:n #1 + { + \int_step_inline:nn { 256 } + { + \__kernel_intarray_gset:Nnn #1 {##1} + { \char_value_catcode:n { ##1 - 1 } } + } + \__kernel_intarray_gset:Nnn #1 { 257 } + { \tex_endlinechar:D } + } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\cctab_gset:Nn, \cctab_gset:cn} +% Category code tables are always global, so only one version of +% assignments is needed. Simply run the setup in a group and save the +% result in a category code table~|#1|, provided it is valid. The +% internal function is defined above depending on the engine. +% \begin{macrocode} +\cs_new_protected:Npn \cctab_gset:Nn #1#2 + { + \@@_chk_if_valid:NT #1 + { + \group_begin: + \cctab_select:N \c_initex_cctab + #2 \scan_stop: + \@@_gset:n {#1} + \group_end: + } + } +\cs_generate_variant:Nn \cctab_gset:Nn { c } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\cctab_gsave_current:N, \cctab_gsave_current:c} +% Very simple. +% \begin{macrocode} +\cs_new_protected:Npn \cctab_gsave_current:N #1 + { + \@@_chk_if_valid:NT #1 + { \@@_gset:n {#1} } + } +\cs_generate_variant:Nn \cctab_gsave_current:N { c } +% \end{macrocode} +% \end{macro} +% +% \subsection{Using category code tables} +% +% \begin{variable}{\g_@@_internal_cctab} +% \begin{macro}[EXP]{\@@_internal_cctab_name:} +% In \LuaTeX{}, we must ensure that the saved tables are read-only. +% This is done by applying the saved table, then switching immediately +% to a scratch table. Any later catcode assignment will affect that +% scratch table rather than the saved one. If we simply switched to +% the saved tables, then \cs{char_set_catcode_other:N} in the example +% below would change \cs{c_document_cctab} and a later use of that +% table would give the wrong category code to |_|. +% \begin{verbatim} +% \use:n +% { +% \cctab_begin:N \c_document_cctab +% \char_set_catcode_other:N \_ +% \cctab_end: +% \cctab_begin:N \c_document_cctab +% \int_compare:nTF { \char_value_catcode:n { `_ } = 8 } +% { \TRUE } { \ERROR } +% \cctab_end: +% } +% \end{verbatim} +% We must also make sure that a scratch table is never reused in a +% nested group: in the following example, the scratch table used by +% the first \cs{cctab_begin:N} would be changed globally by the second +% one issuing \tn{savecatcodetable}, and after \cs{group_end:} the +% wrong category codes (those of \cs{c_str_cctab}) would be imposed. +% Note that the inner \cs{cctab_end:} restores the correct catcodes +% only locally, so the problem really comes up because of the +% different grouping level. The simplest is to use a scratch table +% labeled by the \tn{currentgrouplevel}. We initialize one of them as +% an example. +% \begin{verbatim} +% \use:n +% { +% \cctab_begin:N \c_document_cctab +% \group_begin: +% \cctab_begin:N \c_str_cctab +% \cctab_end: +% \group_end: +% \cctab_end: +% } +% \end{verbatim} +% \begin{macrocode} +\sys_if_engine_luatex:T + { + \@@_new:N \g_@@_internal_cctab + \cs_new:Npn \@@_internal_cctab_name: + { + g_@@_internal + \tex_romannumeral:D \tex_currentgrouplevel:D + _cctab + } + } +% \end{macrocode} +% \end{macro} +% \end{variable} +% +% \begin{macro}{\cctab_select:N, \cctab_select:c} +% \begin{macro}{\@@_select:N} +% The public function simply checks the \meta{cctab~var} exists before +% using the engine-dependent \cs{@@_select:N}. Skipping these checks +% would result in low-level engine-dependent errors. First, the +% \LuaTeX{} case. In other engines, selecting a catcode table is a matter +% of doing $256$ catcode assignments and setting the \tn{endlinechar}. +% \begin{macrocode} +\cs_new_protected:Npn \cctab_select:N #1 + { \@@_chk_if_valid:NT #1 { \@@_select:N #1 } } +\cs_generate_variant:Nn \cctab_select:N { c } +\sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \@@_select:N #1 + { + \tex_catcodetable:D #1 + \prop_get:NVNTF \g_@@_endlinechar_prop #1 \l_@@_internal_a_tl + { \int_set:Nn \tex_endlinechar:D { \l_@@_internal_a_tl } } + { \int_set:Nn \tex_endlinechar:D { 13 } } + \cs_if_exist:cF { \@@_internal_cctab_name: } + { \exp_args:Nc \@@_new:N { \@@_internal_cctab_name: } } + \exp_args:Nc \tex_savecatcodetable:D { \@@_internal_cctab_name: } + \exp_args:Nc \tex_catcodetable:D { \@@_internal_cctab_name: } + } + } + { + \cs_new_protected:Npn \@@_select:N #1 + { + \int_step_inline:nn { 256 } + { + \char_set_catcode:nn { ##1 - 1 } + { \__kernel_intarray_item:Nn #1 {##1} } + } + \int_set:Nn \tex_endlinechar:D + { \__kernel_intarray_item:Nn #1 { 257 } } + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \begin{variable}{\g_@@_next_cctab} +% \begin{macro}{\@@_begin_aux:} +% For \cs{cctab_begin:N}/\cs{cctab_end:} we will need to allocate +% dynamic tables. This is done here by \cs{@@_begin_aux:}, which puts +% a table number (in \LuaTeX{}) or name (in other engines) into +% \cs{l_@@_internal_a_tl}. In \LuaTeX{} this simply calls \cs{@@_new:N} +% and uses the resulting catcodetable number; in other engines we need +% to give a name to the intarray variable and use that. In \LuaTeX{}, +% to restore catcodes at \cs{cctab_end:} we cannot just set +% \tn{catcodetable} to its value before \cs{cctab_begin:N}, because +% that table may have been altered by other code in the mean time. So +% we must make sure to save the catcodes in a table we control and +% restore them at \cs{cctab_end:}. +% \begin{macrocode} +\sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \@@_begin_aux: + { + \@@_new:N \g_@@_next_cctab + \tl_set:NV \l_@@_internal_a_tl \g_@@_next_cctab + \cs_undefine:N \g_@@_next_cctab + } + } + { + \cs_new_protected:Npn \@@_begin_aux: + { + \int_gincr:N \g_@@_allocate_int + \exp_args:Nc \@@_new:N + { g_@@_ \int_use:N \g_@@_allocate_int _cctab } + \exp_args:NNc \tl_set:Nn \l_@@_internal_a_tl + { g_@@_ \int_use:N \g_@@_allocate_int _cctab } + } + } +% \end{macrocode} +% \end{macro} +% \end{variable} +% +% \begin{macro}{\cctab_begin:N, \cctab_begin:c} +% Check the \meta{cctab~var} exists, to avoid low-level errors. Get +% in \cs{l_@@_internal_a_tl} the number/name of a dynamic table, either +% from \cs{g_@@_unused_seq} where we save tables that are not +% currently in use, or from \cs{@@_begin_aux:} if none are available. +% Then save the current catcodes into the table (pointed to by) +% \cs{l_@@_internal_a_tl} and save that table number in a stack before +% selecting the desired catcodes. +% \begin{macrocode} +\cs_new_protected:Npn \cctab_begin:N #1 + { + \@@_chk_if_valid:NT #1 + { + \seq_gpop:NNF \g_@@_unused_seq \l_@@_internal_a_tl + { \@@_begin_aux: } + \@@_chk_group_begin:e + { \@@_nesting_number:N \l_@@_internal_a_tl } + \seq_gpush:NV \g_@@_stack_seq \l_@@_internal_a_tl + \exp_args:NV \@@_gset:n \l_@@_internal_a_tl + \@@_select:N #1 + } + } +\cs_generate_variant:Nn \cctab_begin:N { c } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\cctab_end:} +% Make sure a \cs{cctab_begin:N} was used some time earlier, get in +% \cs{l_@@_internal_a_tl} the catcode table number/name in which the +% prevailing catcodes were stored, then restore these catcodes. The +% dynamic table is now unused hence stored in \cs{g_@@_unused_seq} for +% recycling by later \cs{cctab_begin:N}. +% \begin{macrocode} +\cs_new_protected:Npn \cctab_end: + { + \seq_gpop:NNTF \g_@@_stack_seq \l_@@_internal_a_tl + { + \seq_gpush:NV \g_@@_unused_seq \l_@@_internal_a_tl + \exp_args:Ne \@@_chk_group_end:n + { \@@_nesting_number:N \l_@@_internal_a_tl } + \@@_select:N \l_@@_internal_a_tl + } + { \msg_error:nn { cctab } { extra-end } } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro} +% {\@@_chk_group_begin:n,\@@_chk_group_begin:e, \@@_chk_group_end:n} +% Catcode tables are not allowed to be intermixed with groups, so here +% we check that they are properly nested regarding \TeX{} groups. +% \cs{@@_chk_group_begin:n} stores the current group level in a stack, +% and locally defines a dummy control sequence +% \cs[no-index]{@@_group_\meta{cctab-level}_chk:}. +% +% \cs{@@_chk_group_end:n} pops the stack, and compares the returned +% value with \cs{tex_currentgrouplevel:D}. If they differ, +% \cs{cctab_end:} is in a different grouping level than the matching +% \cs{cctab_begin:N}. If they are the same, both happened at the same +% level, however a group might have ended and another started between +% \cs{cctab_begin:N} and \cs{cctab_end:}: +% \begin{verbatim} +% \group_begin: +% \cctab_begin:N \c_document_cctab +% \group_end: +% \group_begin: +% \cctab_end: +% \group_end: +% \end{verbatim} +% In this case checking \cs{tex_currentgrouplevel:D} is not enough, so +% we locally define \cs[no-index]{@@_group_\meta{cctab-level}_chk:}, +% and then check if it exist in \cs{cctab_end:}. If it doesn't, +% we know there was a group end where it shouldn't. +% +% The \meta{cctab-level} in the sentinel macro above cannot be +% replaced by the more convenient \cs{tex_currentgrouplevel:D} because +% with the latter we might be tricked. Suppose: +% \begin{verbatim} +% \group_begin: +% \cctab_begin:N \c_code_cctab % A +% \group_end: +% \group_begin: +% \cctab_begin:N \c_code_cctab % B +% \cctab_end: % C +% \cctab_end: % D +% \group_end: +% \end{verbatim} +% The line marked with |A| would start a |cctab| with a sentinel token +% named \cs[no-index]{@@_group_1_chk:}, which would disappear at the +% \cs{group_end:} that follows. But |B| would create the same +% sentinel token, since both are at the same group level. Line |C| +% would end the |cctab| from line |B| correctly, but so would line |D| +% because line |B| created the same sentinel token. Using +% \meta{cctab-level} works correctly because it signals that certain +% |cctab| level was activated somewhere, but if it doesn't exist when +% the \cs{cctab_end:} is reached, we had a problem. +% +% Unfortunately these tests only flag the wrong usage at the +% \cs{cctab_end:}, which might be far from the \cs{cctab_begin:N}. +% However it isn't possible to signal the wrong usage at the +% \cs{group_end:} without using \cs{tex_aftergroup:D}, which is +% unsafe in certain types of groups. +% +% The three cases checked here just raise an error, and no recovery is +% attempted: usually interleaving groups and catcode tables will work +% predictably. +% \begin{macrocode} +\cs_new_protected:Npn \@@_chk_group_begin:n #1 + { + \seq_gpush:Ne \g_@@_group_seq + { \int_use:N \tex_currentgrouplevel:D } + \cs_set_eq:cN { @@_group_ #1 _chk: } \prg_do_nothing: + } +\cs_generate_variant:Nn \@@_chk_group_begin:n { e } +\cs_new_protected:Npn \@@_chk_group_end:n #1 + { + \seq_gpop:NN \g_@@_group_seq \l_@@_internal_b_tl + \bool_lazy_and:nnF + { + \int_compare_p:nNn + { \tex_currentgrouplevel:D } = { \l_@@_internal_b_tl } + } + { \cs_if_exist_p:c { @@_group_ #1 _chk: } } + { + \msg_error:nne { cctab } { group-mismatch } + { + \int_sign:n + { \tex_currentgrouplevel:D - \l_@@_internal_b_tl } + } + } + \cs_undefine:c { @@_group_ #1 _chk: } + } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}{\@@_nesting_number:N,\@@_nesting_number:w} +% This macro returns the numeric index of the current catcode table. +% In \LuaTeX{} this is just the argument, which is a count reference +% to a \tn{catcodetable} register. In other engines, the number is +% extracted from the |cctab| variable. +% \begin{macrocode} +\sys_if_engine_luatex:TF + { \cs_new:Npn \@@_nesting_number:N #1 {#1} } + { + \cs_new:Npn \@@_nesting_number:N #1 + { + \exp_after:wN \exp_after:wN \exp_after:wN \@@_nesting_number:w + \exp_after:wN \token_to_str:N #1 + } + \use:e + { + \cs_new:Npn \exp_not:N \@@_nesting_number:w + #1 \tl_to_str:n { g_@@_ } #2 \tl_to_str:n { _cctab } {#2} + } + } +% \end{macrocode} +% \end{macro} +% +% Finally, install some code at the end of the \TeX{} run to check that +% all \cs{cctab_begin:N} were ended by some \cs{cctab_end:}. +% \begin{macrocode} +\cs_if_exist:NT \hook_gput_code:nnn + { + \hook_gput_code:nnn { enddocument/end } { cctab } + { + \seq_if_empty:NF \g_@@_stack_seq + { \msg_error:nn { cctab } { missing-end } } + } + } +% \end{macrocode} +% +% +% \begin{macro}{\cctab_item:Nn, \cctab_item:cn} +% Evaluate the integer argument only once. In most engines the +% |cctab| variable only has $256$ entries so we only look up the +% catcode for these entries, otherwise we use the current catcode. In +% particular, for out-of-range values we use whatever fall-back +% \cs{char_value_catcode:n}. In \LuaTeX{}, we use the +% |tex.getcatcode| function. +% \begin{macrocode} +\cs_new:Npn \cctab_item:Nn #1#2 + { \exp_args:Nf \@@_item:nN { \int_eval:n {#2} } #1 } +\sys_if_engine_luatex:TF + { + \cs_new:Npn \@@_item:nN #1#2 + { \lua_now:e { tex.print(-2, tex.getcatcode(\int_use:N #2, #1)) } } + } + { + \cs_new:Npn \@@_item:nN #1#2 + { + \int_compare:nNnTF {#1} < { 256 } + { \intarray_item:Nn #2 { #1 + 1 } } + { \char_value_catcode:n {#1} } + } + } +\cs_generate_variant:Nn \cctab_item:Nn { c } +% \end{macrocode} +% \end{macro} +% +% \subsection{Category code table conditionals} +% +% \begin{macro}[pTF]{\cctab_if_exist:N,\cctab_if_exist:c} +% Checks whether a \meta{cctab~var} is defined. +% \begin{macrocode} +\prg_new_eq_conditional:NNn \cctab_if_exist:N \cs_if_exist:N + { TF , T , F , p } +\prg_new_eq_conditional:NNn \cctab_if_exist:c \cs_if_exist:c + { TF , T , F , p } +% \end{macrocode} +% \end{macro} +% +% \begin{macro}[TF]{\@@_chk_if_valid:N} +% \begin{macro}{\@@_chk_if_valid_aux:NTF} +% Checks whether the argument is defined and whether it is a valid +% \meta{cctab~var}. In \LuaTeX{} the validity of the \meta{cctab~var} +% is checked by the engine, which complains if the argument is not a +% \cs{chardef}'ed constant. In other engines, check if the given +% command is an intarray variable (the underlying definition is a copy +% of the \texttt{cmr10} font). +% \begin{macrocode} +\prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1 + { TF , T , F } + { + \cctab_if_exist:NTF #1 + { + \@@_chk_if_valid_aux:NTF #1 + { \prg_return_true: } + { + \msg_error:nne { cctab } { invalid-cctab } + { \token_to_str:N #1 } + \prg_return_false: + } + } + { + \msg_error:nne { kernel } { command-not-defined } + { \token_to_str:N #1 } + \prg_return_false: + } + } +\sys_if_engine_luatex:TF + { + \cs_new_protected:Npn \@@_chk_if_valid_aux:NTF #1 + { + \int_compare:nNnTF {#1-1} < { \e@alloc@ccodetable@count } + } + \cs_if_exist:NT \c_syst_catcodes_n + { + \cs_gset_protected:Npn \@@_chk_if_valid_aux:NTF #1 + { + \int_compare:nTF { #1 <= \c_syst_catcodes_n } + } + } + } + { + \cs_new_protected:Npn \@@_chk_if_valid_aux:NTF #1 + { + \exp_args:Nf \str_if_in:nnTF + { \cs_meaning:N #1 } + { select~font~cmr10~at~ } + } + } +% \end{macrocode} +% \end{macro} +% \end{macro} +% +% \subsection{Constant category code tables} +% +% \begin{macro}{\cctab_const:Nn,\cctab_const:cn} +% Creates a new \meta{cctab~var} then sets it with the \IniTeX{} and +% user-supplied codes. To avoid false \texttt{debug} errors, we write +% out implementation of \cs{cctab_new:N} and \cs{cctab_gset:Nn} +% instead of directly using them here. The initialization part in +% \cs{cctab_new:N} in non-\LuaTeX{} is omitted as it's covered by +% the \IniTeX{} settings. +% \begin{macrocode} +\cs_new_protected:Npn \cctab_const:Nn #1#2 + { + \__kernel_chk_if_free_cs:N #1 + \@@_new:N #1 + \group_begin: + \cctab_select:N \c_initex_cctab + #2 \scan_stop: + \@@_gset:n {#1} + \group_end: + } +\cs_generate_variant:Nn \cctab_const:Nn { c } +% \end{macrocode} +% \end{macro} +% +% \begin{variable} +% { +% \c_initex_cctab , +% \c_other_cctab , +% \c_str_cctab +% } +% Creating category code tables means thinking starting from \IniTeX{}. +% For all-other and the standard \enquote{string} tables that's easy. +% \begin{macrocode} +\cctab_new:N \c_initex_cctab +\cctab_const:Nn \c_other_cctab + { + \cctab_select:N \c_initex_cctab + \int_set:Nn \tex_endlinechar:D { -1 } + \int_step_inline:nnn { 0 } { 127 } + { \char_set_catcode_other:n {#1} } + } +\cctab_const:Nn \c_str_cctab + { + \cctab_select:N \c_other_cctab + \char_set_catcode_space:n { 32 } + } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\c_code_cctab, \c_document_cctab} +% To pick up document-level category codes, we need to delay set up to the +% end of the format, where that's possible. Also, as there are a \emph{lot} +% of category codes to set, we avoid using the official interface and store the +% document codes using internal code. Depending on whether we are in the hook +% or not, the catcodes may be code or document, so we explicitly set up both +% correctly. +% \begin{macrocode} +\cs_if_exist:NTF \@expl@finalise@setup@@@@ + { \tl_gput_right:Nn \@expl@finalise@setup@@@@ } + { \use:n } + { + \@@_new:N \c_code_cctab + \group_begin: + \int_set:Nn \tex_endlinechar:D { 32 } + \char_set_catcode_invalid:n { 0 } + \bool_lazy_or:nnTF + { \sys_if_engine_xetex_p: } { \sys_if_engine_luatex_p: } + { \int_step_function:nN { 31 } \char_set_catcode_invalid:n } + { \int_step_function:nN { 31 } \char_set_catcode_active:n } + \int_step_function:nnN { 33 } { 64 } \char_set_catcode_other:n + \int_step_function:nnN { 65 } { 90 } \char_set_catcode_letter:n + \int_step_function:nnN { 91 } { 96 } \char_set_catcode_other:n + \int_step_function:nnN { 97 } { 122 } \char_set_catcode_letter:n + \char_set_catcode_ignore:n { 9 } % tab + \char_set_catcode_other:n { 10 } % lf + \char_set_catcode_active:n { 12 } % ff + \char_set_catcode_end_line:n { 13 } % cr + \char_set_catcode_ignore:n { 32 } % space + \char_set_catcode_parameter:n { 35 } % hash + \char_set_catcode_math_toggle:n { 36 } % dollar + \char_set_catcode_comment:n { 37 } % percent + \char_set_catcode_alignment:n { 38 } % ampersand + \char_set_catcode_letter:n { 58 } % colon + \char_set_catcode_escape:n { 92 } % backslash + \char_set_catcode_math_superscript:n { 94 } % circumflex + \char_set_catcode_letter:n { 95 } % underscore + \char_set_catcode_group_begin:n { 123 } % left brace + \char_set_catcode_other:n { 124 } % pipe + \char_set_catcode_group_end:n { 125 } % right brace + \char_set_catcode_space:n { 126 } % tilde + \char_set_catcode_invalid:n { 127 } % ^^? + \bool_lazy_or:nnF + { \sys_if_engine_xetex_p: } { \sys_if_engine_luatex_p: } + { \int_step_function:nnN { 128 } { 255 } \char_set_catcode_active:n } + \@@_gset:n { \c_code_cctab } + \group_end: + \cctab_const:Nn \c_document_cctab + { + \cctab_select:N \c_code_cctab + \int_set:Nn \tex_endlinechar:D { 13 } + \char_set_catcode_space:n { 9 } + \char_set_catcode_space:n { 32 } + \char_set_catcode_other:n { 58 } + \char_set_catcode_math_subscript:n { 95 } + \char_set_catcode_active:n { 126 } + } + } +% \end{macrocode} +% \end{variable} +% +% \begin{variable}{\g_tmpa_cctab, \g_tmpb_cctab} +% \begin{macrocode} +\cctab_new:N \g_tmpa_cctab +\cctab_new:N \g_tmpb_cctab +% \end{macrocode} +% \end{variable} +% +% \subsection{Messages} +% +% \begin{macrocode} +\msg_new:nnnn { cctab } { stack-full } + { The~category~code~table~stack~is~exhausted. } + { + LaTeX~has~been~asked~to~switch~to~a~new~category~code~table,~ + but~there~is~no~more~space~to~do~this! + } +\msg_new:nnnn { cctab } { extra-end } + { Extra~\iow_char:N\\cctab_end:~ignored~\msg_line_context:. } + { + LaTeX~came~across~a~\iow_char:N\\cctab_end:~without~a~matching~ + \iow_char:N\\cctab_begin:N.~This~command~will~be~ignored. + } +\msg_new:nnnn { cctab } { missing-end } + { Missing~\iow_char:N\\cctab_end:~before~end~of~TeX~run. } + { + LaTeX~came~across~more~\iow_char:N\\cctab_begin:N~than~ + \iow_char:N\\cctab_end:. + } +\msg_new:nnnn { cctab } { invalid-cctab } + { Invalid~\iow_char:N\\catcode~table. } + { + You~can~only~switch~to~a~\iow_char:N\\catcode~table~that~is~ + initialized~using~\iow_char:N\\cctab_new:N~or~ + \iow_char:N\\cctab_const:Nn. + } +\msg_new:nnnn { cctab } { group-mismatch } + { + \iow_char:N\\cctab_end:~occurred~in~a~ + \int_case:nn {#1} + { + { 0 } { different~group } + { 1 } { higher~group~level } + { -1 } { lower~group~level } + } ~than~ + the~matching~\iow_char:N\\cctab_begin:N. + } + { + Catcode~tables~and~groups~must~be~properly~nested,~but~ + you~tried~to~interleave~them.~LaTeX~will~try~to~proceed,~ + but~results~may~be~unexpected. + } +\prop_gput:Nnn \g_msg_module_name_prop { cctab } { LaTeX } +\prop_gput:Nnn \g_msg_module_type_prop { cctab } { } +% \end{macrocode} +% +% \begin{macrocode} +%</package> +% \end{macrocode} +% +% \end{implementation} +% +%\PrintIndex |