% \iffalse meta-comment % %% File: l3cctab.dtx % % Copyright (C) 2018-2020 The LaTeX3 Project % % It may be distributed and/or modified under the conditions of the % LaTeX Project Public License (LPPL), either version 1.3c of this % license or (at your option) any later version. The latest version % of this license is in the file % % http://www.latex-project.org/lppl.txt % % This file is part of the "l3experimental bundle" (The Work in LPPL) % and all files in that bundle must be distributed together. % % ----------------------------------------------------------------------- % % The development version of the bundle can be found at % % https://github.com/latex3/latex3 % % for those people who are interested. % %<*driver|package> \RequirePackage{expl3} % %<*driver> \documentclass[full]{l3doc} \begin{document} \DocInput{\jobname.dtx} \end{document} % % \fi % % \title{^^A % The \pkg{l3cctab} package\\ Experimental category code tables^^A % } % % \author{^^A % The \LaTeX3 Project\thanks % {^^A % E-mail: % \href{mailto:latex-team@latex-project.org} % {latex-team@latex-project.org}^^A % }^^A % } % % \date{Released 2020-01-12} % % \maketitle % % \begin{documentation} % % \section{\pkg{l3cctab} documentation} % % A category code table enables rapid switching of all category codes in % one operation. For \LuaTeX{}, this is possible over the entire Unicode % range. For other engines, only the $8$-bit range ($0$-$255$) is covered by % such tables. % % \subsection{Creating and initialising category code tables} % % \begin{function}{\cctab_new:N,\cctab_new:c} % \begin{syntax} % \cs{cctab_new:N} \meta{category code table} % \end{syntax} % Creates a new \meta{category code table} variable or raises an error if % the name is already taken. The declaration is global. The % \meta{category code table} is initialised with the codes % as used by \IniTeX{}. % \end{function} % % \begin{function}{\cctab_const:Nn,\cctab_const:cn} % \begin{syntax} % \cs{cctab_const:Nn} \meta{category code table} \Arg{category code set up} % \end{syntax} % Creates a new \meta{category code table} with the category code settings in % force at the time the function is called, and applies the % \meta{category code set up} on top of prevailing settings, then saves % as a constant table. % \end{function} % % \begin{function}{\cctab_gset:Nn,\cctab_gset:cn} % \begin{syntax} % \cs{cctab_gset:Nn} \meta{category code table} \Arg{category code set up} % \end{syntax} % Sets the \meta{category code table} to apply the category codes % which apply when the prevailing régime is modified by the % \meta{category code set up}. Thus within a standard code block % the starting point will be the code applied by \cs{c_code_cctab}. % The assignment of the table is global: the underlying primitive does % not respect grouping. % \end{function} % % \subsection{Using category code tables} % % \begin{function}{\cctab_begin:N,\cctab_begin:c} % \begin{syntax} % \cs{cctab_begin:N} \meta{category code table} % \end{syntax} % Switches the category codes in force to those stored in the % \meta{category code table}. The prevailing codes before the % function is called are added to a stack, for use with % \cs{cctab_end:}. This function does not start a \TeX{} group. % \end{function} % % \begin{function}{\cctab_end:} % \begin{syntax} % \cs{cctab_end:} % \end{syntax} % Ends the scope of a \meta{category code table} started using % \cs{cctab_begin:N}, retuning the codes to those in force before % the matching \cs{cctab_begin:N} was used. % \end{function} % % \subsection{Category code table conditionals} % % \begin{function}[pTF]{\cctab_if_exist:N,\cctab_if_exist:c} % \begin{syntax} % \cs{cctab_if_exist_p:N} \meta{category code table} % \cs{cctab_if_exist:NTF} \meta{category code table} \Arg{true code} \Arg{false code} % \end{syntax} % Tests whether the \meta{category code table} is currently defined. % This does not check that the \meta{category code table} really is a % category code table. % \end{function} % % \subsection{Constant category code tables} % % \begin{variable}{\c_code_cctab} % Category code table for the code environment. This does not include % setting the behaviour of the line-end character, which is only % altered by \cs{ExplSyntaxOn}. % \end{variable} % % \begin{variable}{\c_document_cctab} % Category code table for a standard \LaTeX{} document. This does not % include setting the behaviour of the line-end character, which is % only altered by \cs{ExplSyntaxOff}. % \end{variable} % % \begin{variable}{\c_initex_cctab} % Category code table as set up by \IniTeX{}. % \end{variable} % % \begin{variable}{\c_other_cctab} % Category code table where all characters have category code $12$ % (other). % \end{variable} % % \begin{variable}{\c_str_cctab} % Category code table where all characters have category code $12$ % (other) with the exception of spaces, which have category code % $10$ (space). % \end{variable} % % \end{documentation} % % \begin{implementation} % % \section{\pkg{l3cctab} implementation} % % \begin{macrocode} %<*initex|package> % \end{macrocode} % % \begin{macrocode} %<@@=cctab> % \end{macrocode} % % \begin{macrocode} %<*package> \ProvidesExplPackage{l3cctab}{2020-01-12}{} {L3 Experimental category code tables} % % \end{macrocode} % % \begin{variable}{\g_@@_allocate_int} % \begin{variable}{\g_@@_stack_int} % \begin{variable}{\g_@@_stack_seq} % To allocate category code tables, both the read-only and stack % tables need to be followed. There is also a sequence stack for the % dynamic tables themselves. % \begin{macrocode} \int_new:N \g_@@_allocate_int \int_gset:Nn \g_@@_allocate_int { -1 } \int_new:N \g_@@_stack_int \seq_new:N \g_@@_stack_seq % \end{macrocode} % \end{variable} % \end{variable} % \end{variable} % % \begin{variable}{\l_@@_tmp_tl} % Scratch space. % \begin{macrocode} \tl_new:N \l_@@_tmp_tl % \end{macrocode} % \end{variable} % % \begin{macro}{\cctab_new:N,\cctab_new:c} % \begin{macro}{\@@_begin:N,\@@_begin:c} % \begin{macro}{\@@_end:} % \begin{macro}{\cctab_gset:Nn,\cctab_gset:cn} % As \LuaTeX{} offers engine support for category code tables, and this is % entirely lacking from the other engines, we need two complementary % approaches here. Rather than intermix them, we split the set up based on % engine. (Some future \XeTeX{} may add support, at which point the % conditional here would be subtly different.) % % First, the \LuaTeX{} case. % \begin{macrocode} \sys_if_engine_luatex:TF { % \end{macrocode} % Creating a new category code table is done slightly differently % from other registers. Low-numbered tables are more efficiently-stored % than high-numbered ones. There is also a need to have a stack of % flexible tables as well as the set of read-only ones. To satisfy both % of these requirements, odd numbered tables are used for read-only % tables, and even ones for the stack. Here, therefore, the odd numbers % are allocated. % \begin{macrocode} \cs_new_protected:Npn \cctab_new:N #1 { \__kernel_chk_if_free_cs:N #1 %<*initex> \int_gadd:Nn \g_@@_allocate_int { 2 } \int_compare:nNnTF \g_@@_allocate_int < { \c_max_register_int + 1 } { \tex_global:D \tex_chardef:D #1 \g_@@_allocate_int \tex_initcatcodetable:D #1 } { \__kernel_msg_fatal:nnx { kernel } { out-of-registers } { cctab } } % %<*package> \newcatcodetable #1 % } % \end{macrocode} % The aim here is to ensure that the saved tables are read-only. This is % done by using a stack of tables which are not read only, and actually % having them as \enquote{in use} copies. % \begin{macrocode} \cs_new_protected:Npn \@@_begin:N #1 { \seq_gpush:Nx \g_@@_stack_seq { \tex_the:D \tex_catcodetable:D } \tex_catcodetable:D #1 \int_gadd:Nn \g_@@_stack_int { 2 } \int_compare:nNnT \g_@@_stack_int > \c_max_register_int { \__kernel_msg_fatal:nn { kernel } { cctab-stack-full } } \tex_savecatcodetable:D \g_@@_stack_int \tex_catcodetable:D \g_@@_stack_int } \cs_new_protected:Npn \@@_end: { \int_gsub:Nn \g_@@_stack_int { 2 } \seq_if_empty:NTF \g_@@_stack_seq { \tl_set:Nn \l_@@_tmp_tl { 0 } } { \seq_gpop:NN \g_@@_stack_seq \l_@@_tmp_tl } \tex_catcodetable:D \l_@@_tmp_tl \scan_stop: } % \end{macrocode} % Category code tables are always global, so only one version is needed. % The set up here is simple, and means that at the point of use there is % no need to worry about escaping category codes. % \begin{macrocode} \cs_new_protected:Npn \cctab_gset:Nn #1#2 { \group_begin: #2 \scan_stop: \tex_savecatcodetable:D #1 \group_end: } } % \end{macrocode} % Now the case for other engines. Here, we use an integer array for each % table. The index base is out-by-one, so we have an internal function to % handle that. The rest of the approach here is pretty simple: use a stack % of tables, and save to them at each |begin|. Unlike the \LuaTeX{} case, % we can't accidentally alter a saved table, which makes life a little % easier. % \begin{macrocode} { \cs_new_protected:Npn \@@_gstore:Nnn #1#2#3 { \intarray_gset:cnn { g_@@_ \int_use:N #1 _cctab } { #2 + 1 } {#3} } % \end{macrocode} % Following the \LuaTeX{} pattern, a new table starts with \IniTeX{} codes. % \begin{macrocode} \cs_new_protected:Npn \cctab_new:N #1 { \int_gadd:Nn \g_@@_allocate_int { 2 } \int_const:Nn #1 { \g_@@_allocate_int } \intarray_new:cn { g_@@_ \int_use:N #1 _cctab } { 256 } \int_step_inline:nn { 256 } { \intarray_gset:cnn { g_@@_ \int_use:N #1 _cctab } {##1} { 12 } } \@@_gstore:Nnn #1 { 0 } { 9 } \@@_gstore:Nnn #1 { 13 } { 5 } \@@_gstore:Nnn #1 { 32 } { 10 } \@@_gstore:Nnn #1 { 37 } { 14 } \int_step_inline:nnn { 65 } { 90 } { \@@_gstore:Nnn #1 {##1} { 11 } } \@@_gstore:Nnn #1 { 92 } { 0 } \int_step_inline:nnn { 97 } { 122 } { \@@_gstore:Nnn #1 {##1} { 11 } } \@@_gstore:Nnn #1 { 127 } { 15 } } \cs_new_protected:Npn \@@_begin:N #1 { \int_gadd:Nn \g_@@_stack_int { 2 } \int_compare:nNnT \g_@@_stack_int > \c_max_register_int { \__kernel_msg_fatal:nn { kernel } { cctab-stack-full } } \cs_if_exist:cF { g_@@_ \int_use:N \g_@@_stack_int _cctab } { \intarray_new:cn { g_@@_ \int_use:N \g_@@_stack_int _cctab } { 256 } } \int_step_inline:nn { 256 } { \intarray_gset:cnn { g_@@_ \int_use:N \g_@@_stack_int _cctab } {##1} { \char_value_catcode:n { ##1 - 1 } } } \int_step_inline:nn { 256 } { \char_set_catcode:nn { ##1 - 1 } { \intarray_item:cn { g_@@_ \int_use:N #1 _cctab } {##1} } } } \cs_generate_variant:Nn \intarray_new:Nn { c } \cs_generate_variant:Nn \intarray_gset:Nnn { c } \cs_new_protected:Npn \@@_end: { \int_step_inline:nn { 256 } { \char_set_catcode:nn { ##1 - 1 } { \intarray_item:cn { g_@@_ \int_use:N \g_@@_stack_int _cctab } {##1} } } \int_gsub:Nn \g_@@_stack_int { 2 } } \cs_generate_variant:Nn \intarray_item:Nn { c } \cs_new_protected:Npn \cctab_gset:Nn #1#2 { \group_begin: #2 \scan_stop: \int_step_inline:nn { 256 } { \intarray_gset:cnn { g_@@_ \int_use:N #1 _cctab } {##1} { \char_value_catcode:n { ##1 - 1 } } } \group_end: } } \cs_generate_variant:Nn \cctab_new:N { c } \cs_generate_variant:Nn \cctab_gset:Nn { c } % \end{macrocode} % \end{macro} % \end{macro} % \end{macro} % \end{macro} % % \begin{macro}{\cctab_begin:N,\cctab_end:} % These functions perform a few sanity checks before calling the % functions which will do the heavy-lifting. % \cs{cctab_begin:N} checks if a \meta{cctab~var} exists before trying % to use it. \cs{cctab_end:} checks if a \cs{cctab_begin:N} was used % some time earlier. % % Skipping these checks would result in an \enquote{Undefined control % sequence} error for \cs{cctab_begin:N} with an undefined % \meta{cctab~var}, and a \enquote{Missing font identifier} for an extra % \cs{cctab_end:}. % \begin{macrocode} \cs_new_protected:Npn \cctab_begin:N #1 { \@@_chk_if_valid:NT #1 { \__cctab_begin:N #1 } } \cs_generate_variant:Nn \cctab_begin:N { c } \cs_new_protected:Npn \cctab_end: { \int_compare:nNnTF { \g_@@_stack_int } = { \c_zero_int } { \__kernel_msg_error:nn { kernel } { cctab-extra-end } } { \@@_end: } } % \end{macrocode} % \end{macro} % % \begin{macro}{\cctab_if_exist:N,\cctab_if_exist:c} % Checks whether a \meta{cctab~var} is defined. % \begin{macrocode} \prg_new_eq_conditional:NNn \cctab_if_exist:N \cs_if_exist:N { TF , T , F , p } \prg_new_eq_conditional:NNn \cctab_if_exist:c \cs_if_exist:c { TF , T , F , p } % \end{macrocode} % \end{macro} % % \begin{macro}{\@@_chk_if_valid:N} % Checks whether the argument is defined and whether it is a valid % \meta{cctab~var}. In \LuaTeX{} the validity of the \meta{cctab~var} % is checked by the engine, which complains if the argument is not % a \cs{chardef}'ed constant. In other engines, check if the % underlying \cs[no-index]{g_@@_\meta{integer}_cctab} % \meta{intarray~var} exists. % \begin{macrocode} \sys_if_engine_luatex:TF { \prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1 { TF , T , F } { \cctab_if_exist:NTF #1 { %<*initex> \bool_lazy_and:nnTF { \int_if_odd_p:n {#1} } { \int_compare_p:nNn {#1-1} < { \g_@@_allocate_int } } % %<*package> \int_compare:nNnTF {#1-1} < { \e@alloc@ccodetable@count } % { \prg_return_true: } { \__kernel_msg_error:nnx { kernel } { invalid-cctab } { \token_to_str:N #1 } \prg_return_false: } } { \__kernel_msg_error:nnx { kernel } { command-not-defined } { \token_to_str:N #1 } \prg_return_false: } } } { \prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1 { TF , T , F } { \cctab_if_exist:NTF #1 { \cs_if_exist:cTF { g_@@_ \int_use:N #1 _cctab } { \prg_return_true: } { \__kernel_msg_error:nnx { kernel } { invalid-cctab } { \token_to_str:N #1 } \prg_return_false: } } { \__kernel_msg_error:nnx { kernel } { command-not-defined } { \token_to_str:N #1 } \prg_return_false: } } } % \end{macrocode} % \end{macro} % % \begin{macro}{\cctab_const:Nn,\cctab_const:cn} % Creates a new \meta{cctab~var} then sets it with the current and % user-supplied codes. % \begin{macrocode} \cs_new_protected:Npn \cctab_const:Nn #1#2 { \cctab_new:N #1 \cctab_gset:Nn #1 {#2} } \cs_generate_variant:Nn \cctab_const:Nn { c } % \end{macrocode} % \end{macro} % % \begin{variable} % { % \c_initex_cctab , % \c_code_cctab , % \c_document_cctab , % \c_other_cctab , % \c_str_cctab % } % Creating category code tables is easy using the function above. % The \texttt{other} and \texttt{string} ones are done by completely % ignoring the existing codes as this makes life a lot less complex. % \begin{macrocode} \cctab_new:N \c_initex_cctab \cctab_const:Nn \c_code_cctab { } \cctab_const:Nn \c_document_cctab { \char_set_catcode_space:n { 9 } \char_set_catcode_space:n { 32 } \char_set_catcode_other:n { 58 } \char_set_catcode_math_subscript:n { 95 } \char_set_catcode_active:n { 126 } } \cctab_const:Nn \c_other_cctab { \int_step_inline:nnn { 0 } { 127 } { \char_set_catcode_other:n {#1} } } \cctab_const:Nn \c_str_cctab { \int_step_inline:nnn { 0 } { 127 } { \char_set_catcode_other:n {#1} } \char_set_catcode_space:n { 32 } } % \end{macrocode} % \end{variable} % % \subsection{Messages} % % \begin{macrocode} \__kernel_msg_new:nnnn { kernel } { cctab-stack-full } { The~category~code~table~stack~is~exhausted. } { LaTeX~has~been~asked~to~switch~to~a~new~category~code~table,~ but~there~is~no~more~space~to~do~this! } \__kernel_msg_new:nnnn { kernel } { cctab-extra-end } { Extra~\iow_char:N\\cctab_end:~ignored~\msg_line_context:. } { LaTeX~came~across~a~\iow_char:N\\cctab_end:~without~a~matching~ \iow_char:N\\cctab_begin:N.~This~command~will~be~ignored. } \__kernel_msg_new:nnnn { kernel } { invalid-cctab } { Invalid~\iow_char:N\\catcode~table. } { You~can~only~switch~to~a~\iow_char:N\\catcode~table~that~is~ initialized~using~\iow_char:N\\cctab_new:N~or~ \iow_char:N\\cctab_const:Nn. } % \end{macrocode} % % \begin{macrocode} % % \end{macrocode} % %\end{implementation} % %\PrintIndex