% \iffalse
%% File: l3io.dtx Copyright (C) 1990-2006 LaTeX3 project
%%
%% It may be distributed and/or modified under the conditions of the
%% LaTeX Project Public License (LPPL), either version 1.3c of this
%% license or (at your option) any later version.  The latest version
%% of this license is in the file
%%
%%    http://www.latex-project.org/lppl.txt
%%
%% This file is part of the ``expl3 bundle'' (The Work in LPPL)
%% and all files in that bundle must be distributed together.
%%
%% The released version of this bundle is available from CTAN.
%%
%% -----------------------------------------------------------------------
%%
%% The development version of the bundle can be found at
%%
%%    http://www.latex-project.org/cgi-bin/cvsweb.cgi/
%%
%% for those people who are interested.
%%
%%%%%%%%%%%
%% NOTE: %%
%%%%%%%%%%%
%%
%%   Snapshots taken from the repository represent work in progress and may
%%   not work or may contain conflicting material!  We therefore ask
%%   people _not_ to put them into distributions, archives, etc. without
%%   prior consultation with the LaTeX Project Team.
%%
%% -----------------------------------------------------------------------
%
%<*driver|package>
\RequirePackage{l3names}
%</driver|package>
%\fi
\GetIdInfo$Id: l3io.dtx 621 2007-09-01 20:14:19Z morten $
          {L3 Experimental i/o module}
%\iffalse
%<*driver>
%\fi
\ProvidesFile{\filename.\filenameext}
  [\filedate\space v\fileversion\space\filedescription]
%\iffalse
\documentclass{l3doc}
\begin{document}
\DocInput{\filename.\filenameext}
\end{document}
%</driver>
% \fi
%
% \section{Low-level file i/o}
%
% \TeX{} is capable of reading from and writing up to 16 individual
% streams. These i/o operations are accessable in \LaTeX3 with functions
% from the "\io.."  modules. In most cases it will be
% sufficiant for the programmer to use the functions provided by the
% auxiliary file module, but here are the necessary functions for
% manipulating private streams.
%
% Sometimes it is not know beforehand how much text is going to be
% written with a single call. As a result some internal \TeX{} buffer
% may overflow. To avoid this kind of problem, \LaTeX3 maintains beside
% direct write operations like "\iow_expanded:Nn" also so called ``long''
% writes where the output is broken into individual lines on every blank
% in the text to be written. The resulting files are difficult to read
% for humans but since they usually serve only as internal storage this
% poses no problem.
%
% Beside the functions that immediately act (e.g., "\iow_expanded:Nn",
% etc.) we also have deferred operations that are saved away until the
% next page is finished. This allow to expand the <tokens> at the right
% time to get correct page numbers etc.
%
% \subsection{Functions for output streams}
%
% \begin{function}{\iow_new:N |
%                  \iow_new:c
% }
% \begin{syntax}
%   "\iow_new:N" <stream>
% \end{syntax}
% Defines <stream> to be a new identifer denoting an output stream for
% use in subsequent functions.
% \begin{texnote}
% "\iow_new:N" corresponds to the plain \TeX{} \tn{newwrite}
% allocation routine.
% \end{texnote}
% \end{function}
%
% \begin{function}{\iow_open:Nn |
%                  \iow_open:cn
% }
% \begin{syntax}
%   "\iow_open:Nn" <stream> "{" <file name> "}"
% \end{syntax}
% Opens output stream <stream> to write to <file name>. The output
% stream is immediately available for use. If the <stream> was already
% used as an output stream to some other file, this file gets closed
% first.\footnote{This is a precaution since on some OS it is possible
% to open the same file for output more than once which then results in
% some internal errors at the end of the run.} Also, all output streams
% still open at the end of the \TeX{} run will be automatically closed.
% \end{function}
%
% \begin{function}{\iow_expanded:Nn |
%                  \iow_unexpanded:Nn
% }
% \begin{syntax}
%  "\iow_expanded:Nn" <stream> "{" <tokens> "}"
% \end{syntax}
% This function immediately writes the expansion of <tokens> to the
% output stream <stream>. If <stream> is not open output goes to the
% terminal. The variant "\iow_unexpanded:Nn" writes out <tokens> without any
% further expansion (verbatim).
% \end{function}
%
% \begin{function}{\iow_expanded_log:n |
%                  \iow_expanded_term:n |
%                  \iow_unexpanded_term:n
% }
% \begin{syntax}
%  "\iow_expanded_log:n" "{" <tokens> "}"
% \end{syntax}
% These functions write to the transcript or to the terminal
% respectively. So they are equivalent to "\iow_expanded:Nn" where <stream>
% is the transcript file ("\c_iow_log_stream") or the terminal
% ("\c_io_term_stream").
% \end{function}
%
% \begin{function}{\iow_long_expanded:Nx |
%                  \iow_long_unexpanded:Nn
% }
% \begin{syntax}
%  "\iow_long_expanded:Nn" <stream> "{" <tokens> "}"
% \end{syntax}
% Like "\iow_expanded:Nn" but splits <tokens> at every blank into separate
% lines.
% \end{function}
%
% \begin{function}{\iow_unexpanded_if_avail:Nn|
%                  \iow_unexpanded_if_avail:cn|}
% \begin{syntax}
%  "\iow_unexpanded_if_avail:Nn" <stream> "{" <tokens> "}"
% \end{syntax}
% This special function first checks if the <stream> is open of writing.
% If not it does nothing otherwise it behaves like "\iow_unexpanded:Nn".
% \end{function}
%
% \begin{function}{\iow_deferred_expanded:Nn |
%                  \iow_deferred_unexpanded:Nn}
% \begin{syntax}
%   "\iow_deferred_expanded:Nn"  <stream> "{" <tokens> "}"
% \end{syntax}
% These functions save away <tokens> until the next page is ready to be
% shipped out. Then, in case of "\iow_deferred_expanded:Nn" <tokens> get
% expanded and afterwards written to <stream>. "\iow_deferred_expanded:Nn"
% also always needs "{}" around the second argument. The use of
% "\iow_deferred_unexpanded:Nn" is probably seldom necessary.
% \begin{texnote}
% "\iow_deferred_expanded:Nn" was known as \tn{write}.
% \end{texnote}
% \end{function}
%
% \begin{function}{\iow_newline:}
% \begin{syntax}
%   "\iow_newline:"
% \end{syntax}
% Function that produces a new line when used within the <token list>
% that gets written some output stream in non-verbatim mode.
% \end{function}
%
% \subsection{Functions for input streams}
%
% \begin{function}{\ior_new:N
% }
% \begin{syntax}
%   "\ior_new:N" <stream>
% \end{syntax}
% This function defines <stream> to be a new input stream constant.
% \begin{texnote}
% This is the new name and new implementation for plain \TeX's
% \tn{newread}.
% \end{texnote}
% \end{function}
%
% \begin{function}{\ior_open:Nn}
% \begin{syntax}
%   "\ior_open:Nn" <stream> "{" <file name> "}"
% \end{syntax}
% This function opens <stream> as an input stream for the external file
% <file name>. If <file name> doesn't exist or is an empty file the
% stream is considered to be fully read, a condition which can be tested
% with "\ior_eof:NTF" etc. If <stream> was already used to read from some
% other file this file will be closed first. The input stream is ready
% for immediate use.
% \end{function}
%
% \begin{function}{\ior_close:N}
% \begin{syntax}
%   "\ior_close:N" <stream>
% \end{syntax}
% This function closes the read stream <stream>.
% \begin{texnote}
% This is a new name for \tn{closein} but it is considered bad practice
% to make use of this knowledge :-)
% \end{texnote}
% \end{function}
%
% \begin{function}{\ior_eof:NTF |
%                  \ior_eof:NF}
% \begin{syntax}
%   "\ior_eof:NTF" <stream> "{" <true code> "}{" <false code> "}"
% \end{syntax}
% Conditional that tests if some input stream is fully read. The
% condition is also true if the input stream is not open.
% \end{function}
%
% \begin{function}{\if_eof:w}
% \begin{syntax}
%   "\if_eof:w" <stream> <true code> "\else:" <false code> "\fi:"
% \end{syntax}
% \begin{texnote}
% This is the primitive \tn{ifeof} but we allow only a <stream> and not
% a plain number after it.
% \end{texnote}
% \end{function}
%
% \begin{function}{\ior_to:NN |
%                  \ior_gto:NN}
% \begin{syntax}
%   "\ior_to:NN" <stream> <tlp>
% \end{syntax}
% Functions that reads one or more lines (until an equal number of left
% and right braces are found) from the input stream <stream> and places
% the result locally or globally into <tlp>. If <stream> is not open
% input is requested from the terminal.
% \end{function}
%
% \subsection{Constants}
%
% \begin{variable}{\c_iow_comment_char |
%                  \c_iow_lbrace_char  |
%                  \c_iow_rbrace_char}
% Constants that can be used to represent comment character, left and
% right brace in token lists that should be written to a file.
% \end{variable}
%
% \begin{variable}{\c_io_term_stream}
% Input or output stream denoting the terminal. If used as an input
% stream the user is prompted with the name of the <tlp> (that is used
% in the call "\ior_to:NN" or "\ior_gto:NN") followed by an equal sign.
% If you don't want an automatic prompt of this sort ``misuse''
% "\c_iow_log_stream" as an input stream.
% \end{variable}
%
% \begin{variable}{\c_iow_log_stream}
% Output stream that writes only to the transcript file (e.g., the
% {\tt.log} file on most systems). You may ``misuse'' this stream as an
% input stream. In this case it acts as a terminal stream without user
% prompting.
% \end{variable}
%
%
% \subsection{Internal functions}
%
% \begin{function}{\iow_long_expanded_aux:w}
%  Function used to implement immediate writing where a new line is
% started at every blank.
% \end{function}
%
% \begin{function}{%
%                  \tex_read:D |
%                  \tex_immediate:D |
%                  \tex_closeout:D |
%                  \tex_openin:D |
%                  \tex_openout:D |
% }
% These are the functions of the primitive interface to \TeX.
% \begin{texnote}
% The \TeX{} primitives \tn{read}, \tn{immediate}, \tn{closeout},
% \tn{openin}, and \tn{openout} are all renamed and should not be used
% by a programmer since the functionality is covered by the \LaTeX3
% functions above.
% \end{texnote}
% \end{function}
%
% \StopEventually{}
%
% \subsection{The Implementation}
%
%    We start by ensuring that the required packages are loaded.
%    \begin{macrocode}
%<package>\ProvidesExplPackage
%<package>  {\filename}{\filedate}{\fileversion}{\filedescription}
%<package&check>\RequirePackage{l3chk}\par
%<package>\RequirePackage{l3toks}\par
%<*initex|package>
%    \end{macrocode}
%
% This section is primarily concerned with input and output streams.
% The naming conventions for i/o streams is |ior| (for read) and |iow|
% (for write) as module names. e.g.\ |\c_ior_test_stream| is an input stream
% variable called `test'.
%
% \subsubsection{Output streams}
%
% \begin{macro}{\iow_new:N}
% \begin{macro}{\iow_new:c}
%    Allocation of new output streams is done by these functions.
%    As we currently do not distribute a new allocation module we nick
%    the |\newwrite| function.
%    \begin{macrocode}
%<*initex>
\alloc_setup_type:nnn {iow} \c_zero \c_sixteen
\def_new:Npn \iow_new:N #1 {\alloc_reg:NnNN g {iow} \tex_chardef:D #1}
%</initex>
%<package>\let:NN \iow_new:N \newwrite
\def_new:Npn \iow_new:c {\exp_args:Nc \iow_new:N}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\iow_open:Nn}
% \begin{macro}{\iow_open:cn}
%    To open streams for reading or writing the following two functions
%    are provided. The streams are opened immediately.
%
%    From some bad experiences on the mainframe, I learned that it is
%    better to force the close before opening a dataset for writing.
%    We have to check whether this is also necessary in case of
%    |\tex_openin:D|.
%    \begin{macrocode}
\def_new:Npn \iow_open:Nn #1#2{\iow_close:N #1
     \tex_immediate:D\tex_openout:D#1#2\scan_stop:}
\def_new:Npn \iow_open:cn {\exp_args:Nc \iow_open:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\iow_close:N}
%    Since we close output streams prior to opening, a separate closing
%    operation is probably not necessary. But here it is, just in
%    case\ldots. Actually you will need this if you intend to write
%    and then read in the same pass from some stream.
%    \begin{macrocode}
\def_new:Npn \iow_close:N {\tex_immediate:D\tex_closeout:D}
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\c_io_term_stream}
% \begin{macro}{\c_iow_log_stream}
%    Here we allocate two output streams for writing to the transcript
%    file only (|\c_iow_log_stream|) and to both the terminal and transcript
%    file (|\c_io_term_stream|). The latter can also be used to read
%    from therefore it is called |..io_..|.
%    \begin{macrocode}
\let_new:NN \c_io_term_stream \c_sixteen
\let_new:NN \c_iow_log_stream \c_minus_one
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \paragraph{Immediate writing}
%
% \begin{macro}{\iow_expanded:Nn}
%    An abbreviation for an often used operation, which immediately
%    writes its second argument to the output stream.
%    \begin{macrocode}
\def_new:Npn \iow_expanded:Nn {\tex_immediate:D\iow_deferred_expanded:Nn}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\iow_unexpanded:Nn}
%    This routine writes the second argument verbatim onto the output
%    stream. If this stream isn't open, the output goes to the terminal.
%    If the first argument is no output stream at all, we get an
%    internal error.
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded:Nn #1#2{
   \iow_expanded:Nn #1{\exp_not:n{#2}}}
%    \end{macrocode}
% \end{macro}
%
%
%  \begin{macro}{\iow_expanded_log:n}
%  \begin{macro}{\iow_expanded_term:n}
%    Now we redefine two functions for which we needed a definition
%    very early on. They both write their second argument fully
%    expanded to the output stream.
%    \begin{macrocode}
\def:Npn \iow_expanded_log:n {\iow_expanded:Nn \c_iow_log_stream}
\def:Npn \iow_expanded_term:n{\iow_expanded:Nn \c_io_term_stream}
%    \end{macrocode}
%    The second one isn't exactly equivalent to the old |\typeout| since
%    we need to control expansion in the function we provide for the user.
%  \end{macro}
%  \end{macro}

% \begin{macro}{\iow_unexpanded_term:n}
%    This function writes its argument verbatim to the the terminal.
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded_term:n {\iow_unexpanded:Nn \c_io_term_stream}
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\iow_unexpanded_if_avail:Nn}
% \begin{macro}{\iow_unexpanded_if_avail:cn}
%    |\iow_unexpanded_if_avail:Nn | \m{stream} \m{code}.  This routine writes
%    its second argument unexpanded to the stream given by the first
%    argument, provided that this stream was opened for writing. Note,
%    that |#| characters get doubled within \m{code}.
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded_if_avail:Nn #1{
%    \end{macrocode}
%    In this routine we have to check whether or not the output stream
%    that was requested is defined at all.
%    So we check if the name is still free.
%    \begin{macrocode}
  \cs_free:NTF #1\use_none:n {\iow_unexpanded:Nn #1}}
%    \end{macrocode}
%    Note: the next function could be streamlined for speed if we use
%    the faster |\cs_free:cTF|. (space viz time).
%    \begin{macrocode}
\def_new:Npn \iow_unexpanded_if_avail:cn {
         \exp_args:Nc \iow_unexpanded_if_avail:Nn }
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\iow_long_expanded:Nn}
% \begin{macro}{\iow_long_unexpanded:Nn}
% \begin{macro}{\iow_long_expanded_aux:w}
%    Another type of writing onto an output stream is used for
%    potentially long token sequences. We break the output lines at
%    every blank in the second argument. This avoids the problem of
%    buffer overflow when reading back, or badly broken lines on
%    systems with limited file records.  The only thing we have to
%    take care of, is the danger of two blanks in succession since
%    these get converted into a |\par| when we read the stuff back.
%    But this can happen only if things like
%    two spaces find their way into the second argument.
%    Usually, multiple spaces are removed by \TeX's scanner.
%
%    \begin{macrocode}
\def_new:Npn \iow_long_expanded_aux:w #1#2#3{
   \group_begin:\tex_newlinechar:D`\ #1#2{#3}\group_end:}
\def_new:Npn \iow_long_expanded:Nn {\iow_long_expanded_aux:w
                                    \iow_expanded:Nn}
\def_new:Npn \iow_long_unexpanded:Nn {\iow_long_expanded_aux:w
                                      \iow_unexpanded:Nn}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
%
% \paragraph{Deferred writing}
% 
% With \eTeX\ available deferred writing is easy. The comments below
% are old.
%
% Deferred writing to output streams is a bit more complicated because
% there seems to be no nice hack for writing unexpanded. The only
% relatively sure bet is to use |\token_to_meaning:N| expansion of some token
% list.  That's the way the following functions are implemented.
%
% Another possibility would be to reserve a certain number of scratch
% token registers that could be used to hold the tokens until after the
% next |\tex_shipout:D|. But such an approach would probably fail because of
% the limited number of available token registers that would need to
% be reserved for this special application.
%
%
% \begin{macro}{\iow_deferred_expanded:Nn}
%    First the easy part, this is the primitive.
%    \begin{macrocode}
\let:NN \iow_deferred_expanded:Nn \tex_write:D
%    \end{macrocode}
% \end{macro}
%
%
% \begin{macro}{\iow_deferred_unexpanded:Nn}
%    Now the harder part:
%    \begin{macrocode}
\def_new:Npn \iow_deferred_unexpanded:Nn #1#2{
  \iow_deferred_expanded:Nn{\exp_not:n{#2}}
}
%% Old implementation:
%\def_new:Npn \iow_deferred_unexpanded:Nn #1#2{
%  \tlp_set:Nn \l_tmpa_tlp {#2}
%  \tlp_set:Nx \l_tmpb_tlp
%              {\iow_deferred_expanded:Nn #1{\tlp_to_str:N \l_tmpa_tlp}}
%  \l_tmpb_tlp}
%    \end{macrocode}
% \end{macro}
%
% Long forms of these functions are not possible since the deferred
% writing will restore the value of |\tex_newlinechar:D| before it will have
% a chance to act. But on the other hand it is nevertheless possible
% to make all deferred writes long by setting the |\tex_newlinechar:D|
% inside the output routine just before the |\tex_shipout:D|. The only
% disadvantage of this method is the fact that messages to the
% terminal during this time will also then break at spaces. But we
% should consider this.
%
% \paragraph{Special characters for writing}
%
% \begin{macro}{\iow_newline:}
%    Global variable holding the character that forces a new line when
%    something is written to an output stream. 
%    \begin{macrocode}
\def_new:Npn \iow_newline: {^^J}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\c_iow_comment_char}
% \begin{macro}{\c_iow_lbrace_char}
% \begin{macro}{\c_iow_rbrace_char}
%   We also need to be able to write braces and the comment character.  We
%   achieve this by defining global constants to expand into a version of
%   these characters with |\tex_catcode:D|${}=12$.
%    \begin{macrocode}
\tlp_new:Nx \c_iow_comment_char {\cs_to_str:N\%}
%    \end{macrocode}
%    To avoid another allocation function which is probably only
%    necessary here we use the |\def:Npx| command directly.
%    \begin{macrocode}
\tlp_new:Nx \c_iow_lbrace_char{\cs_to_str:N\{}
\tlp_new:Nx \c_iow_rbrace_char{\cs_to_str:N\}}
%    \end{macrocode}
% \end{macro}
% \end{macro}
% \end{macro}
%
%
% \subsubsection{Input streams}
%
% \begin{macro}{\ior_new:N}
%    Allocation of new input streams is done by this function.
%    As we currently do not distribute a new allocation module we nick
%    the |\newwread| function.
%    \begin{macrocode}
%<*initex>
\alloc_setup_type:nnn {ior} \c_zero \c_sixteen
\def_new:Npn \ior_new:N #1 {\alloc_reg:NnNN g {ior} \tex_chardef:D #1}
%</initex>
%<package>\let:NN \ior_new:N \newread
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\ior_open:Nn}
% \begin{macro}{\ior_close:Nn}
%    Processing of input-streams (via |\tex_openin:D| and |closein|) is
%    always `immediate' as far as \TeX{} is concerned. An extra
%    |\tex_immediate:D| is silently ignored.
%    \begin{macrocode}
\let:NN \ior_close:N \tex_closein:D
\def_new:Npn \ior_open:Nn #1#2{\ior_close:N #1\scan_stop:
                               \tex_openin:D#1#2\scan_stop:}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
% \begin{macro}{\ior_eof:NTF}
%    |\ior_eof:NTF | \m{stream} \m{true case} \m{false case}.  To
%    test if some particular input stream is exhausted the following
%    conditional is provided:
%    \begin{macrocode}
\def_new:Npn \ior_eof:NTF #1{\if_eof:w#1
     \exp_after:NN\use_arg_i:nn \else:
     \exp_after:NN\use_arg_ii:nn \fi:}
%    \end{macrocode}
% \end{macro}
%
% \begin{macro}{\ior_eof:NF}
% \begin{macro}{\if_eof:w}
%    |\ior_eof:NF | \m{stream}  \m{false case}.  Do something if
%    if there is still something to read from this file:
%    \begin{macrocode}
\let:NN \if_eof:w \tex_ifeof:D
\def_new:Npn \ior_eof:NF #1{\if_eof:w#1
        \exp_after:NN \use_none:nn \fi: \use_arg_i:n}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
% \begin{macro}{\ior_to:NN}
% \begin{macro}{\ior_gto:NN}
%    And here we read from files.
%    \begin{macrocode}
%<*check>
\def_new:Npn \ior_to:NN #1#2{\tex_read:D#1to#2
   \chk_local_or_pref_global:N #2}
%</check>
%<-check> \def_new:Npn \ior_to:NN #1{\tex_read:D#1to}
\def_new:Npn \ior_gto:NN {
%<*check>
   \pref_global_chk:
%</check>
%<-check> \pref_global:D
   \ior_to:NN}
%    \end{macrocode}
% \end{macro}
% \end{macro}
%
%
%    Show token usage:
%    \begin{macrocode}
%</initex|package>
%<*showmemory>
\showMemUsage
%</showmemory>
%    \end{macrocode}
%
% \endinput
%
% $Log$
% Revision 1.16  2006/03/20 18:26:36  braams
% Updated the copyright notice (2006) and demoted all implementation
% sections to subsections and so on to clean up the toc for source3.tex
%
% Revision 1.15  2005/12/27 09:51:27  morten
% Changed the use of RCS information
%
% Revision 1.14  2005/11/07 17:59:48  morten
% Fixed definition of \iow_newline: (moved it to the right place)
%
% Revision 1.13  2005/04/29 22:29:23  braams
% Replaced the undefined \g_iow_newline_code with \tex_newlinechar:D
% and added the definition for \iow_newline:
%
% Revision 1.12  2005/04/06 22:35:37  braams
% Now we have register allocation fitted
%
% Revision 1.11  2005/03/16 22:36:24  braams
% Added the tweaks necessary to be able to load with initex
%
% Revision 1.10  2005/03/11 21:39:10  braams
% Fixed the use of RCS information; added \StopEventually
%