path: root/Master/texmf-dist/source/latex/l3experimental/l3cctab/l3cctab.dtx

% \iffalse meta-comment
%
%% File: l3cctab.dtx
%
% Copyright (C) 2018,2019 The LaTeX3 Project
%
% It may be distributed and/or modified under the conditions of the
% LaTeX Project Public License (LPPL), either version 1.3c of this
% license or (at your option) any later version.  The latest version
% of this license is in the file
%
%    http://www.latex-project.org/lppl.txt
%
% This file is part of the "l3experimental bundle" (The Work in LPPL)
% and all files in that bundle must be distributed together.
%
% -----------------------------------------------------------------------
%
% The development version of the bundle can be found at
%
%    https://github.com/latex3/latex3
%
% for those people who are interested.
%
%<*driver|package>
\RequirePackage{expl3}
%</driver|package>
%<*driver>
\documentclass[full]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
%   The \pkg{l3cctab} package\\ Experimental category code tables^^A
% }
%
% \author{^^A
%  The \LaTeX3 Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released 2019-09-28}
%
% \maketitle
%
% \begin{documentation}
%
% \section{\pkg{l3cctab} documentation}
%
% A category code table enables rapid switching of all category codes in
% one operation. For \LuaTeX{}, this is possible over the entire Unicode
% range. For other engines, only the $8$-bit range ($0$-$255$) is covered by
% such tables.
%
% \subsection{Creating and initialising category code tables}
%
% \begin{function}{\cctab_new:N,\cctab_new:c}
%   \begin{syntax}
%     \cs{cctab_new:N} \meta{category code table}
%   \end{syntax}
%   Creates a new \meta{category code table} variable or raises an error if
%   the name is already taken. The declaration is global.  The
%   \meta{category code table} is initialised with the codes
%   as used by \IniTeX{}.
% \end{function}
%
% \begin{function}{\cctab_const:Nn,\cctab_const:cn}
%   \begin{syntax}
%     \cs{cctab_const:Nn} \meta{category code table} \Arg{category code set up}
%   \end{syntax}
%   Creates a new \meta{category code table} with the category code settings in
%   force at the time the function is called, and applies the
%   \meta{category code set up} on top of prevailing settings, then saves
%   as a constant table.
% \end{function}
%
% \begin{function}{\cctab_gset:Nn,\cctab_gset:cn}
%   \begin{syntax}
%     \cs{cctab_gset:Nn} \meta{category code table} \Arg{category code set up}
%   \end{syntax}
%   Sets the \meta{category code table} to apply the category codes
%   which apply when the prevailing régime is modified by the
%   \meta{category code set up}. Thus within a standard code block
%   the starting point will be the code applied by \cs{c_code_cctab}.
%   The assignment of the table is global: the underlying primitive does
%   not respect grouping.
% \end{function}
%
% \subsection{Using category code tables}
%
% \begin{function}{\cctab_begin:N,\cctab_begin:c}
%   \begin{syntax}
%     \cs{cctab_begin:N} \meta{category code table}
%   \end{syntax}
%   Switches the category codes in force to those stored in the
%   \meta{category code table}.  The prevailing codes before the
%   function is called are added to a stack, for use with
%   \cs{cctab_end:}. This function does not start a \TeX{} group.
% \end{function}
%
% \begin{function}{\cctab_end:}
%   \begin{syntax}
%     \cs{cctab_end:}
%   \end{syntax}
%   Ends the scope of a \meta{category code table} started using
%   \cs{cctab_begin:N}, retuning the codes to those in force before
%   the matching \cs{cctab_begin:N} was used.
% \end{function}
%
% \subsection{Category code table conditionals}
%
% \begin{function}[pTF]{\cctab_if_exist:N,\cctab_if_exist:c}
%   \begin{syntax}
%     \cs{cctab_if_exist_p:N} \meta{category code table}
%     \cs{cctab_if_exist:NTF} \meta{category code table} \Arg{true code} \Arg{false code}
%   \end{syntax}
%   Tests whether the \meta{category code table} is currently defined.
%   This does not check that the \meta{category code table} really is a
%   category code table.
% \end{function}
%
% \subsection{Constant category code tables}
%
% \begin{variable}{\c_code_cctab}
%   Category code table for the code environment. This does not include
%   setting the behaviour of the line-end character, which is only
%   altered by \cs{ExplSyntaxOn}.
% \end{variable}
%
% \begin{variable}{\c_document_cctab}
%   Category code table for a standard \LaTeX{} document. This does not
%   include setting the behaviour of the line-end character, which is
%   only altered by \cs{ExplSyntaxOff}.
% \end{variable}
%
% \begin{variable}{\c_initex_cctab}
%   Category code table as set up by \IniTeX{}.
% \end{variable}
%
% \begin{variable}{\c_other_cctab}
%   Category code table where all characters have category code $12$
%   (other).
% \end{variable}
%
% \begin{variable}{\c_str_cctab}
%   Category code table where all characters have category code $12$
%   (other) with the exception of spaces, which have category code
%   $10$ (space).
% \end{variable}
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3cctab} implementation}
%
%    \begin{macrocode}
%<*initex|package>
%    \end{macrocode}
%
%    \begin{macrocode}
%<@@=cctab>
%    \end{macrocode}
%
%    \begin{macrocode}
%<*package>
\ProvidesExplPackage{l3cctab}{2019-09-28}{}
  {L3 Experimental category code tables}
%</package>
%    \end{macrocode}
%
% \begin{variable}{\g_@@_allocate_int}
% \begin{variable}{\g_@@_stack_int}
% \begin{variable}{\g_@@_stack_seq}
%   To allocate category code tables, both the read-only and stack
%   tables need to be followed. There is also a sequence stack for the
%   dynamic tables themselves.
%    \begin{macrocode}
\int_new:N  \g_@@_allocate_int
\int_gset:Nn \g_@@_allocate_int { -1 }
\int_new:N \g_@@_stack_int
\seq_new:N \g_@@_stack_seq
%    \end{macrocode}
% \end{variable}
% \end{variable}
% \end{variable}
%
% \begin{variable}{\l_@@_tmp_tl}
%   Scratch space.
%    \begin{macrocode}
\tl_new:N \l_@@_tmp_tl
%    \end{macrocode}
% \end{variable}
%
% \begin{macro}{\cctab_new:N,\cctab_new:c}
% \begin{macro}{\@@_begin:N,\@@_begin:c}
% \begin{macro}{\@@_end:}
% \begin{macro}{\cctab_gset:Nn,\cctab_gset:cn}
%   As \LuaTeX{} offers engine support for category code tables, and this is
%   entirely lacking from the other engines, we need two complementary
%   approaches here. Rather than intermix them, we split the set up based on
%   engine. (Some future \XeTeX{} may add support, at which point the
%   conditional here would be subtly different.)
%
%   First, the \LuaTeX{} case.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
%    \end{macrocode}
%   Creating a new category code table is done slightly differently
%   from other registers. Low-numbered tables are more efficiently-stored
%   than high-numbered ones. There is also a need to have a stack of
%   flexible tables as well as the set of read-only ones. To satisfy both
%   of these requirements, odd numbered tables are used for read-only
%   tables, and even ones for the stack. Here, therefore, the odd numbers
%   are allocated.
%    \begin{macrocode}
    \cs_new_protected:Npn \cctab_new:N #1
      {
        \__kernel_chk_if_free_cs:N #1
%<*initex>
        \int_gadd:Nn \g_@@_allocate_int { 2 }
        \int_compare:nNnTF
          \g_@@_allocate_int < { \c_max_register_int + 1 }
           {
             \tex_global:D \tex_chardef:D #1 \g_@@_allocate_int
             \tex_initcatcodetable:D #1
           }
           {
             \__kernel_msg_fatal:nnx
               { kernel } { out-of-registers } { cctab }
           }
%</initex>
%<*package>
        \newcatcodetable #1
%</package>
      }
%    \end{macrocode}
%   The aim here is to ensure that the saved tables are read-only. This is
%   done by using a stack of tables which are not read only, and actually
%   having them as \enquote{in use} copies.
%    \begin{macrocode}
    \cs_new_protected:Npn \@@_begin:N #1
      {
        \seq_gpush:Nx \g_@@_stack_seq { \tex_the:D \tex_catcodetable:D }
        \tex_catcodetable:D #1
        \int_gadd:Nn \g_@@_stack_int { 2 }
        \int_compare:nNnT \g_@@_stack_int > \c_max_register_int
          { \__kernel_msg_fatal:nn { kernel } { cctab-stack-full } }
        \tex_savecatcodetable:D \g_@@_stack_int
        \tex_catcodetable:D \g_@@_stack_int
      }
    \cs_new_protected:Npn \@@_end:
      {
        \int_gsub:Nn \g_@@_stack_int { 2 }
        \seq_if_empty:NTF \g_@@_stack_seq
          { \tl_set:Nn \l_@@_tmp_tl { 0 } }
          { \seq_gpop:NN \g_@@_stack_seq \l_@@_tmp_tl }
        \tex_catcodetable:D \l_@@_tmp_tl \scan_stop:
      }
%    \end{macrocode}
%   Category code tables are always global, so only one version is needed.
%   The set up here is simple, and means that at the point of use there is
%   no need to worry about escaping category codes.
%    \begin{macrocode}
    \cs_new_protected:Npn \cctab_gset:Nn #1#2
      {
        \group_begin:
          #2 \scan_stop:
          \tex_savecatcodetable:D #1
        \group_end:
      }
  }
%    \end{macrocode}
%   Now the case for other engines. Here, we use an integer array for each
%   table. The index base is out-by-one, so we have an internal function to
%   handle that. The rest of the approach here is pretty simple: use a stack
%   of tables, and save to them at each |begin|. Unlike the \LuaTeX{} case,
%   we can't accidentally alter a saved table, which makes life a little
%   easier.
%    \begin{macrocode}
  {
    \cs_new_protected:Npn \@@_gstore:Nnn #1#2#3
      {
        \intarray_gset:cnn
          { g_@@_ \int_use:N #1 _cctab } { #2 + 1 } {#3}
      }
%    \end{macrocode}
%   Following the \LuaTeX{} pattern, a new table starts with \IniTeX{} codes.
%    \begin{macrocode}
    \cs_new_protected:Npn \cctab_new:N #1
      {
        \int_gadd:Nn \g_@@_allocate_int { 2 }
        \int_const:Nn #1 { \g_@@_allocate_int }
        \intarray_new:cn { g_@@_ \int_use:N #1 _cctab } { 256 }
        \int_step_inline:nn { 256 }
          {
            \intarray_gset:cnn
              { g_@@_ \int_use:N #1 _cctab } {##1} { 12 }
          }
        \@@_gstore:Nnn #1 { 0 } { 9 }
        \@@_gstore:Nnn #1 { 13 } { 5 }
        \@@_gstore:Nnn #1 { 32 } { 10 }
        \@@_gstore:Nnn #1 { 37 } { 14 }
        \int_step_inline:nnn { 65 } { 90 }
          { \@@_gstore:Nnn #1 {##1} { 11 } }
        \@@_gstore:Nnn #1 { 92 } { 0 }
        \int_step_inline:nnn { 97 } { 122 }
          { \@@_gstore:Nnn #1 {##1} { 11 } }
        \@@_gstore:Nnn #1 { 127 } { 15 }
      }
    \cs_new_protected:Npn \@@_begin:N #1
      {
        \int_gadd:Nn \g_@@_stack_int { 2 }
        \int_compare:nNnT \g_@@_stack_int > \c_max_register_int
          { \__kernel_msg_fatal:nn { kernel } { cctab-stack-full } }
        \cs_if_exist:cF { g_@@_ \int_use:N \g_@@_stack_int _cctab }
          {
            \intarray_new:cn
              { g_@@_ \int_use:N \g_@@_stack_int _cctab }
              { 256 }
          }
        \int_step_inline:nn { 256 }
          {
            \intarray_gset:cnn
              { g_@@_ \int_use:N \g_@@_stack_int _cctab }
              {##1}
              { \char_value_catcode:n { ##1 - 1 } }
          }
        \int_step_inline:nn { 256 }
          {
            \char_set_catcode:nn { ##1 - 1 }
              {
                \intarray_item:cn
                  { g_@@_ \int_use:N #1 _cctab } {##1}
              }
          }
      }
    \cs_generate_variant:Nn \intarray_new:Nn { c }
    \cs_generate_variant:Nn \intarray_gset:Nnn { c }
    \cs_new_protected:Npn \@@_end:
      {
        \int_step_inline:nn { 256 }
          {
            \char_set_catcode:nn { ##1 - 1 }
              {
                \intarray_item:cn
                  { g_@@_ \int_use:N \g_@@_stack_int _cctab }
                  {##1}
              }
          }
        \int_gsub:Nn \g_@@_stack_int { 2 }
      }
    \cs_generate_variant:Nn \intarray_item:Nn { c }
    \cs_new_protected:Npn \cctab_gset:Nn #1#2
      {
        \group_begin:
          #2 \scan_stop:
          \int_step_inline:nn { 256 }
            {
              \intarray_gset:cnn { g_@@_ \int_use:N #1 _cctab } {##1}
                { \char_value_catcode:n { ##1 - 1 } }
            }
        \group_end:
      }
  }
\cs_generate_variant:Nn \cctab_new:N { c }
\cs_generate_variant:Nn \cctab_gset:Nn { c }
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\cctab_begin:N,\cctab_end:}
%   These functions perform a few sanity checks before calling the
%   functions which will do the heavy-lifting.
%   \cs{cctab_begin:N} checks if a \meta{cctab~var} exists before trying
%   to use it. \cs{cctab_end:} checks if a \cs{cctab_begin:N} was used
%   some time earlier.
%   
%   Skipping these checks would result in an \enquote{Undefined control
%   sequence} error for \cs{cctab_begin:N} with an undefined
%   \meta{cctab~var}, and a \enquote{Missing font identifier} for an extra
%   \cs{cctab_end:}.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_begin:N #1
  { \@@_chk_if_valid:NT #1 { \__cctab_begin:N #1 } }
\cs_generate_variant:Nn \cctab_begin:N { c }
\cs_new_protected:Npn \cctab_end:
  {
    \int_compare:nNnTF { \g_@@_stack_int } = { \c_zero_int }
      { \__kernel_msg_error:nn { kernel } { cctab-extra-end } }
      { \@@_end: }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\cctab_if_exist:N,\cctab_if_exist:c}
%   Checks whether a \meta{cctab~var} is defined.
%    \begin{macrocode}
\prg_new_eq_conditional:NNn \cctab_if_exist:N \cs_if_exist:N
  { TF , T , F , p }
\prg_new_eq_conditional:NNn \cctab_if_exist:c \cs_if_exist:c
  { TF , T , F , p }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\@@_chk_if_valid:N}
%   Checks whether the argument is defined and whether it is a valid
%   \meta{cctab~var}. In \LuaTeX{} the validity of the \meta{cctab~var}
%   is checked by the engine, which complains if the argument is not
%   a \cs{chardef}'ed constant. In other engines, check if the
%   underlying \cs[no-index]{g_@@_\meta{integer}_cctab}
%   \meta{intarray~var} exists.
%    \begin{macrocode}
\sys_if_engine_luatex:TF
  {
    \prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1
      { TF , T , F }
      {
        \cctab_if_exist:NTF #1
          {
%<*initex>
            \bool_lazy_and:nnTF
              { \int_if_odd_p:n {#1} }
              { \int_compare_p:nNn {#1-1} < { \g_@@_allocate_int } }
%</initex>
%<*package>
            \int_compare:nNnTF {#1-1} < { \e@alloc@ccodetable@count }
%</package>
              { \prg_return_true: }
              {
                \__kernel_msg_error:nnx { kernel } { invalid-cctab }
                  { \token_to_str:N #1 }
                \prg_return_false:
              }
          }
          {
            \__kernel_msg_error:nnx { kernel } { command-not-defined }
              { \token_to_str:N #1 }
            \prg_return_false:
          }
      }
  }
  {
    \prg_new_protected_conditional:Npnn \@@_chk_if_valid:N #1
      { TF , T , F }
      {
        \cctab_if_exist:NTF #1
          {
            \cs_if_exist:cTF { g_@@_ \int_use:N #1 _cctab }
              { \prg_return_true: }
              {
                \__kernel_msg_error:nnx { kernel } { invalid-cctab }
                  { \token_to_str:N #1 }
                \prg_return_false:
              }
          }
          {
            \__kernel_msg_error:nnx { kernel } { command-not-defined }
              { \token_to_str:N #1 }
            \prg_return_false:
          }
      }
  }
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\cctab_const:Nn,\cctab_const:cn}
%  Creates a new \meta{cctab~var} then sets it with the current and
%  user-supplied codes.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_const:Nn #1#2
  {
    \cctab_new:N #1
    \cctab_gset:Nn #1 {#2}
  }
\cs_generate_variant:Nn \cctab_const:Nn { c }
%    \end{macrocode}
% \end{macro}
%
% \begin{variable}
%   {
%     \c_initex_cctab   ,
%     \c_code_cctab     ,
%     \c_document_cctab ,
%     \c_other_cctab    ,
%     \c_str_cctab
%   }  
%   Creating category code tables is easy using the function above.
%   The \texttt{other} and \texttt{string} ones are done by completely
%   ignoring the existing codes as this makes life a lot less complex.
%    \begin{macrocode}
\cctab_new:N \c_initex_cctab
\cctab_const:Nn \c_code_cctab { }
\cctab_const:Nn \c_document_cctab
  {
    \char_set_catcode_space:n          { 9 }
    \char_set_catcode_space:n          { 32 }
    \char_set_catcode_other:n          { 58 }
    \char_set_catcode_math_subscript:n { 95 }
    \char_set_catcode_active:n         { 126 }
  }
\cctab_const:Nn \c_other_cctab
  {
    \int_step_inline:nnn { 0 } { 127 }
      { \char_set_catcode_other:n {#1} }
  }
\cctab_const:Nn \c_str_cctab
  {
    \int_step_inline:nnn { 0 } { 127 }
      { \char_set_catcode_other:n {#1} }
    \char_set_catcode_space:n { 32 }
  }
%    \end{macrocode}
% \end{variable}
%
% \subsection{Messages}
%
%    \begin{macrocode}
\__kernel_msg_new:nnnn { kernel } { cctab-stack-full }
  { The~category~code~table~stack~is~exhausted. }
  {
    LaTeX~has~been~asked~to~switch~to~a~new~category~code~table,~
    but~there~is~no~more~space~to~do~this!
  }
\__kernel_msg_new:nnnn { kernel } { cctab-extra-end }
  { Extra~\iow_char:N\\cctab_end:~ignored~\msg_line_context:. }
  {
    LaTeX~came~across~a~\iow_char:N\\cctab_end:~without~a~matching~
    \iow_char:N\\cctab_begin:N.~This~command~will~be~ignored.
  }
\__kernel_msg_new:nnnn { kernel } { invalid-cctab }
  { Invalid~\iow_char:N\\catcode~table. }
  {
    You~can~only~switch~to~a~\iow_char:N\\catcode~table~that~is~
    initialized~using~\iow_char:N\\cctab_new:N~or~
    \iow_char:N\\cctab_const:Nn.
  }
%    \end{macrocode}
%
%    \begin{macrocode}
%</initex|package>
%    \end{macrocode}
%
%\end{implementation}
%
%\PrintIndex