% \iffalse meta-comment
%
%% File: l3luatex.dtx Copyright (C) 2010,2011 The LaTeX3 Project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
%% license or (at your option) any later version.  The latest version
%% of this license is in the file
%%
%%    http://www.latex-project.org/lppl.txt
%%
%% This file is part of the "expl3 bundle" (The Work in LPPL)
%% and all files in that bundle must be distributed together.
%%
%% The released version of this bundle is available from CTAN.
%%
%% -----------------------------------------------------------------------
%%
%% The development version of the bundle can be found at
%%
%%    http://www.latex-project.org/svnroot/experimental/trunk/
%%
%% for those people who are interested.
%%
%%%%%%%%%%%
%% NOTE: %%
%%%%%%%%%%%
%%
%%   Snapshots taken from the repository represent work in progress and may
%%   not work or may contain conflicting material!  We therefore ask
%%   people _not_ to put them into distributions, archives, etc. without
%%   prior consultation with the LaTeX3 Project.
%%
%% -----------------------------------------------------------------------
%
%<*driver|package>
\RequirePackage{l3names}
\GetIdInfo$Id: l3luatex.dtx 2478 2011-06-19 21:34:23Z joseph $
  {L3 Experimental LuaTeX-specific functions}
%</driver|package>
%<*driver>
\documentclass[full]{l3doc}
\begin{document}
  \DocInput{\jobname.dtx}
\end{document}
%</driver>
% \fi
%
% \title{^^A
%   The \pkg{l3luatex} package\\ LuaTeX-specific functions^^A
%   \thanks{This file describes v\ExplFileVersion,
%      last revised \ExplFileDate.}^^A
% }
%
% \author{^^A
%  The \LaTeX3 Project\thanks
%    {^^A
%      E-mail:
%        \href{mailto:latex-team@latex-project.org}
%          {latex-team@latex-project.org}^^A
%    }^^A
% }
%
% \date{Released \ExplFileDate}
%
% \maketitle
%
% \begin{documentation}
%
% \section{Breaking out to \Lua{}}
%
% The \LuaTeX{} engine provides access to the \Lua{} programming language, 
% and with it access to the \enquote{internals} of \TeX{}. In order to use 
% this within the framework provided here, a family of functions is
% available. When used with \pdfTeX{} or \XeTeX{} these will raise an
% error: use \cs{luatex_if_engine:T} to avoid this. Details of coding
% the \LuaTeX{} engine are detailed in the \LuaTeX{} manual.
% 
% \begin{function}[EXP]{\lua_now:n, \lua_now:x}
%   \begin{syntax}
%     \cs{lua_now:n} \Arg{token list}
%   \end{syntax}
%   The \meta{token list} is first tokenized by \TeX{}, which will include
%   converting line ends to spaces in the usual \TeX{} manner and which
%   respects currently-applicable \TeX\ category codes. The resulting 
%   \meta{\Lua{} input} is passed to the \Lua{} interpreter for processing.
%   Each \cs{lua_now:n} block is treated by \Lua{} as a separate chunk.
%   The \Lua{} interpreter will execute the \meta{\Lua{} input} immediately, 
%   and in an expandable manner.
%   \begin{texnote}
%     \cs{lua_now:x} is the \LuaTeX{} primitive \cs{directlua} renamed.
%   \end{texnote}
% \end{function}
%
% \begin{function}{\lua_shipout:n, \lua_shipout:x}
%   \begin{syntax}
%     \cs{lua_shipout:x} \Arg{token list}
%   \end{syntax}
%   The \meta{token list} is first tokenized by \TeX{}, which will include
%   converting line ends to spaces in the usual \TeX{} manner and which
%   respects currently-applicable \TeX{} category codes.  The resulting 
%   \meta{\Lua{} input} is passed to the \Lua\ interpreter when the 
%   current page is finalised (\emph{i.e.}~at shipout).  Each
%   \cs{lua_shipout:n} block is treated by \Lua{} as a separate chunk.
%   The \Lua{} interpreter will execute the \meta{\Lua{} input} during the
%   page-building routine: no \TeX{} expansion of the \meta{\Lua{} input}
%   will occur at this stage.
%   \begin{texnote}
%     At a \TeX{} level, the \meta{\Lua{} input} is stored as a 
%     \enquote{whatsit}.
%   \end{texnote}
% \end{function}
%
% \begin{function}{\lua_shipout_x:n, \lua_shipout_x:x}
%   \begin{syntax}
%     \cs{lua_shipout:n} \Arg{token list}
%   \end{syntax}
%   The \meta{token list} is first tokenized by \TeX{}, which will include
%   converting line ends to spaces in the usual \TeX{} manner and which
%   respects currently-applicable \TeX{} category codes.  The resulting 
%   \meta{\Lua{} input} is passed to the \Lua{} interpreter when the 
%   current page is finalised (\emph{i.e.}~at shipout).  Each
%   \cs{lua_shipout:n} block is treated by \Lua{} as a separate chunk.
%   The \Lua{} interpreter will execute the \meta{\Lua{} input} during the
%   page-building routine: the \meta{\Lua{} input} is expanded during this
%   process in addition to any expansion when the argument was read. This
%   makes these functions suitable for including material finalised
%   during the page building process (such as the page number).
%   \begin{texnote}
%     \cs{lua_shipout_x:n} is the \LuaTeX{} primitive \cs{latelua}
%     named using the \LaTeX3 scheme.
%  
%     At a \TeX{} level, the \meta{\Lua{} input} is stored as a 
%     \enquote{whatsit}.
%   \end{texnote}
% \end{function}
%
% \section{Category code tables}
%
% As well as providing methods to break out into \Lua{}, there are
% places where additional \LaTeX3 functions are provided by the 
% \LuaTeX{} engine. In particular, \LuaTeX{} provides category code 
% tables. These can be used to ensure that a set of category codes
% are in force in a more robust way than is possible with other 
% engines. These are therefore used by \cs{ExplSyntaxOn} and
% \pkg{ExplSyntaxOff} when using the \LuaTeX{} engine.
% 
% \begin{function}{\cctab_new:N}
%   \begin{syntax}
%     \cs{cctab_new:N} \meta{category code table}
%   \end{syntax}
%   Creates a new category code table, initially with the codes as
%   used by \IniTeX{}.
% \end{function}
%
% \begin{function}{\cctab_gset:Nn}
%   \begin{syntax}
%     \cs{cctab_gset:Nn} \meta{category code table}
%       \Arg{category code set up}
%   \end{syntax}
%   Sets the \meta{category code table} to apply the category codes
%   which apply when the prevailing regime is modified by the
%   \meta{category code set up}. Thus within a standard code block
%   the starting point will be the code applied by \cs{c_code_cctab}.
%   The assignment of the table is global: the underlying primitive does
%   not respect grouping.
% \end{function}
% 
% \begin{function}{\cctab_begin:N}
%   \begin{syntax}
%     \cs{cctab_begin:N} \meta{category code table}
%   \end{syntax}
%   Switches the category codes in force to those stored in the
%   \meta{category code table}.  The prevailing codes before the 
%   function is called are added to a stack, for use with
%   \cs{cctab_end:}.
% \end{function}
%
% \begin{function}{\cctab_end:}
%   \begin{syntax}
%     \cs{cctab_end:}
%   \end{syntax}
%   Ends the scope of a \meta{category code table} started using
%   \cs{cctab_begin:N}, retuning the codes to those in force before the
%   matching \cs{cctab_begin:N} was used.
% \end{function}
%
% \begin{variable}{\c_code_cctab}
%   Category code table for the code environment. This does not include
%   setting the behaviour of the line-end character, which is only
%   altered by \cs{ExplSyntaxOn}.
% \end{variable}
%
% \begin{variable}{\c_document_cctab}
%   Category code table for a standard \LaTeX{} document. This does not 
%   include setting the behaviour of the line-end character, which is 
%   only altered by \cs{ExplSyntaxOff}.
% \end{variable}
%
% \begin{variable}{\c_initex_cctab}
%   Category code table as set up by \IniTeX{}.
% \end{variable}
%
% \begin{variable}{\c_other_cctab}
%   Category code table where all characters have category code $12$
%   (other).
% \end{variable}
%
% \begin{variable}{\c_string_cctab}
%   Category code table where all characters have category code $12$
%   (other) with the exception of spaces, which have category code
%   $10$ (space).
% \end{variable}
%
% \end{documentation}
%
% \begin{implementation}
%
% \section{\pkg{l3luatex} implementation}
% 
%    \begin{macrocode}
%<*initex|package>
%    \end{macrocode}
% 
%    Announce and ensure that the required packages are loaded.
%    \begin{macrocode}
%<*package>
\ProvidesExplPackage
  {\ExplFileName}{\ExplFileDate}{\ExplFileVersion}{\ExplFileDescription}
\package_check_loaded_expl:
%</package>
%<*initex|package>
%    \end{macrocode}
%    
% \begin{macro}{\lua_now:n, \lua_now:x}
% \begin{macro}{\lua_shipout_x:n, \lua_shipout_x:x}
% \begin{macro}{\lua_shipout:n, \lua_shipout:x}
% \begin{macro}[aux]{\lua_wrong_engine:}
%   When \LuaTeX{} is in use, this is all a question of primitives with new
%   names. On the other hand, for \pdfTeX{} and \XeTeX{} the argument should
%   be removed from the input stream before issuing an error. This needs 
%   to be expandable, so the same idea is used as for \texttt{V}-type
%   expansion, with an appropriately-named but undefined function.
%    \begin{macrocode}
\luatex_if_engine:TF
  {
    \cs_new_eq:NN \lua_now:x       \luatex_directlua:D
    \cs_new_eq:NN \lua_shipout_x:n \luatex_latelua:D
  }
  {
    \cs_new:Npn \lua_now:x #1 { \lua_wrong_engine: }
    \cs_new_protected:Npn \lua_shipout_x:n #1 { \lua_wrong_engine: }
  }
\cs_new:Npn \lua_now:n #1
  { \lua_now:x { \exp_not:n {#1} } }
\cs_generate_variant:Nn \lua_shipout_x:n { x }
\cs_new_protected:Npn \lua_shipout:n #1
  { \lua_shipout_x:n { \exp_not:n {#1} } }
\cs_generate_variant:Nn \lua_shipout:n { x }
\group_begin:
\char_set_catcode_letter:N\!
\char_set_catcode_letter:N\ %
\cs_gset:Npn\lua_wrong_engine:{%
\LuaTeX engine not in use!%
}%
\group_end:%
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
% \end{macro}
%
% \subsection{Category code tables}
%
% \begin{variable}{\g_cctab_allocate_int}
% \begin{variable}{\g_cctab_stack_int}
% \begin{variable}{\g_cctab_stack_seq}
%   To allocate category code tables, both the read-only and stack 
%   tables need to be followed. There is also a sequence stack for the
%   dynamic tables themselves.
%    \begin{macrocode}
\int_new:N  \g_cctab_allocate_int
\int_set:Nn \g_cctab_allocate_int { -1 }
\int_new:N \g_cctab_stack_int
\seq_new:N \g_cctab_stack_seq
%    \end{macrocode}
% \end{variable}
% \end{variable} 
% \end{variable}  
%
% \begin{macro}{\cctab_new:N}
%   Creating a new category code table is done slightly differently
%   from other registers. Low-numbered tables are more efficiently-stored
%   than high-numbered ones. There is also a need to have a stack of 
%   flexible tables as well as the set of read-only ones. To satisfy both
%   of these requirements, odd numbered tables are used for read-only
%   tables, and even ones for the stack. Here, therefore, the odd numbers
%   are allocated.
%    \begin{macrocode}
\cs_new_protected_nopar:Npn \cctab_new:N #1
  {
    \cs_if_free:NTF #1
      { 
        \int_gadd:Nn \g_cctab_allocate_int { 2 }
         \int_compare:nNnTF 
           { \g_cctab_allocate_int } <  { \c_max_register_int + 1 }
           { 
             \pref_global:D \tex_mathchardef:D #1 \g_cctab_allocate_int 
             \luatex_initcatcodetable:D #1
           }
           { \msg_kernel_fatal:nnx { alloc } { out-of-registers } { cctab } }  
       }
       {
         \msg_kernel_error:nnx { code } { variable-already-defined } 
           { \token_to_str:N #1 } 
       }
  } 
\luatex_if_engine:F
  { \cs_set_protected_nopar:Npn \cctab_new:N #1 { \lua_wrong_engine: } }
%<*package>
\luatex_if_engine:T
  {
    \cs_set_protected_nopar:Npn \cctab_new:N #1 
      { 
        \newcatcodetable #1
        \luatex_initcatcodetable:D #1
      }
  }
%</package>
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\cctab_begin:N}
% \begin{macro}{\cctab_end:}
% \begin{variable}{\l_cctab_tmp_tl}
%   The aim here is to ensure that the saved tables are read-only. This is
%   done by using a stack of tables which are not read only, and actually
%   having them as \enquote{in use} copies.
%    \begin{macrocode}
\cs_new_protected_nopar:Npn \cctab_begin:N #1
  {
    \seq_gpush:Nx \g_cctab_stack_seq { \tex_the:D \luatex_catcodetable:D }
    \luatex_catcodetable:D #1
    \int_gadd:Nn \g_cctab_stack_int { 2 }
    \int_compare:nNnT { \g_cctab_stack_int } > { 268 435 453 }
      { \msg_kernel_error:nn { code } { cctab-stack-full } }
    \luatex_savecatcodetable:D \g_cctab_stack_int 
    \luatex_catcodetable:D \g_cctab_stack_int 
  }
\cs_new_protected_nopar:Npn \cctab_end:
  {
    \int_gsub:Nn \g_cctab_stack_int { 2 }
    \seq_gpop:NN \g_cctab_stack_seq \l_cctab_tmp_tl
    \quark_if_no_value:NT \l_cctab_tmp_tl
      { \tl_set:Nn \l_cctab_tmp_tl { 0 } }
    \luatex_catcodetable:D \l_cctab_tmp_tl \scan_stop:
  }
\luatex_if_engine:F
  {
    \cs_set_protected_nopar:Npn \cctab_begin:N #1 { \lua_wrong_engine: }
    \cs_set_protected_nopar:Npn \cctab_end: { \lua_wrong_engine: }
  }
%<*package>
\luatex_if_engine:T
  {
    \cs_set_protected_nopar:Npn \cctab_begin:N #1 { \BeginCatcodeRegime #1 }
    \cs_set_protected_nopar:Npn \cctab_end: { \EndCatcodeRegime }
  }
%</package>
\tl_new:N \l_cctab_tmp_tl
%    \end{macrocode}
% \end{variable}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\cctab_gset:Nn}
%   Category code tables are always global, so only one version is needed.
%   The set up here is simple, and means that at the point of use there is
%   no need to worry about escaping category codes.
%    \begin{macrocode}
\cs_new_protected:Npn \cctab_gset:Nn #1#2
  {
    \group_begin:
      #2
      \luatex_savecatcodetable:D #1
    \group_end:
  }
\luatex_if_engine:F
  { \cs_set_protected_nopar:Npn \cctab_gset:Nn #1#2 { \lua_wrong_engine: } }
%    \end{macrocode}
% \end{macro}
%
% \begin{variable}{\c_code_cctab}
% \begin{variable}{\c_document_cctab}
% \begin{variable}{\c_initex_cctab}
% \begin{variable}{\c_other_cctab}
% \begin{variable}{\c_string_cctab}
%   Creating category code tables is easy using the function above.
%   The \texttt{other} and \texttt{string} ones are done by completely
%   ignoring the existing codes as this makes life a lot less complex. The
%   table for \pkg{expl3} category codes is always needed, whereas when in
%   package mode the rest can be copied from the existing \LaTeXe{} package 
%   \pkg{luatex}.
%    \begin{macrocode}
\luatex_if_engine:T
  {
    \cctab_new:N \c_code_cctab
    \cctab_gset:Nn \c_code_cctab { }
  }
%<*package>  
\luatex_if_engine:T
  {
    \cs_new_eq:NN \c_document_cctab \CatcodeTableLaTeX
    \cs_new_eq:NN \c_initex_cctab   \CatcodeTableIniTeX
    \cs_new_eq:NN \c_other_cctab    \CatcodeTableOther
    \cs_new_eq:NN \c_string_cctab   \CatcodeTableString
  }
%</package>   
%<*initex>    
\luatex_if_engine:T
  {
    \cctab_new:N \c_document_cctab
    \cctab_new:N \c_other_cctab
    \cctab_new:N \c_string_cctab
    \cctab_gset:Nn \c_document_cctab
      {
        \char_set_catcode_space:n          { 9 }
        \char_set_catcode_space:n          { 32 }
        \char_set_catcode_other:n          { 58 }
        \char_set_catcode_math_subscript:n { 95 }
        \char_set_catcode_active:n         { 126 }
      }
    \cctab_gset:Nn \c_other_cctab
      {
        \prg_stepwise_inline:nnnn { 0 } { 1 } { 127 }
          { \char_set_catcode_other:n {#1} }
      } 
    \cctab_gset:Nn \c_string_cctab
      {
        \prg_stepwise_inline:nnnn { 0 } { 1 } { 127 }
          { \char_set_catcode_other:n {#1} }
        \char_set_catcode_space:n { 32 }  
      }  
  }
%</initex>
%    \end{macrocode}
% \end{variable}
% \end{variable}
% \end{variable}
% \end{variable}
% \end{variable}
% 
%    \begin{macrocode}
%</initex|package>
%    \end{macrocode}
%
%\end{implementation}
%
%\PrintIndex