summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/source/latex/tcldoc/pstokens.dtx
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/source/latex/tcldoc/pstokens.dtx')
-rw-r--r--Master/texmf-dist/source/latex/tcldoc/pstokens.dtx979
1 files changed, 979 insertions, 0 deletions
diff --git a/Master/texmf-dist/source/latex/tcldoc/pstokens.dtx b/Master/texmf-dist/source/latex/tcldoc/pstokens.dtx
new file mode 100644
index 00000000000..26f2a71e28b
--- /dev/null
+++ b/Master/texmf-dist/source/latex/tcldoc/pstokens.dtx
@@ -0,0 +1,979 @@
+%
+% \iffalse
+%<*driver>
+\documentclass{tcldoc}
+
+\usepackage{array}
+
+
+% Logos:
+%
+\DeclareFontFamily{U}{logo}{}
+\DeclareFontShape{U}{logo}{m}{n}{
+ <-9> logo8
+ <9-10> logo9
+ <10-> logo10
+}{}
+\DeclareFontShape{U}{logo}{bx}{n}{<-> logobf10}{}
+\DeclareFontShape{U}{logo}{b}{n}{<-> ssub * logo/bx/n}{}
+\DeclareFontShape{U}{logo}{m}{sl}{<-> logosl10}{}
+\DeclareFontShape{U}{logo}{m}{it}{<-> ssub * logo/m/sl}{}
+
+% \DeclareRobustCommand\META{{%
+% \fontencoding{U}\fontfamily{logo}\selectfont META%
+% }}
+\DeclareRobustCommand\MF{%
+ {\fontencoding{U}\fontfamily{logo}\selectfont META}\-%
+ {\fontencoding{U}\fontfamily{logo}\selectfont FONT}%
+}
+\DeclareRobustCommand\MP{%
+ {\fontencoding{U}\fontfamily{logo}\selectfont META}\-%
+ {\fontencoding{U}\fontfamily{logo}\selectfont POST}%
+}
+\DeclareRobustCommand\PS{PostScript}
+\DeclareRobustCommand\TOne{Type~1}
+\DeclareRobustCommand\package[1]{\textsf{#1}}
+
+
+% Useful environments:
+%
+% \begin{environment}{isyntax}
+% |isyntax| is for ``informal'' syntax specifications, such as
+% \begin{quote}
+% A \meta{varchar command} can be one of\\
+% \begin{quote}
+% |\vartop|\marg{glyph}\\
+% |\varmid|\marg{glyph}\\
+% |\varbot|\marg{glyph}\\
+% |\varrep|\marg{glyph}
+% \end{quote}
+% \end{quote}
+% The implementation is based on that of the \texttt{verse}
+% environment.It usually works just as fine with \texttt{quote}, but
+% \texttt{isyntax} handles breaking long lines better (and more
+% elegantly).
+% \changes{1.37}{2000/05/14}{Increased \cs{rightskip} stretchability
+% and added setting of \cs{linepenalty}. (LH)}
+% \begin{macrocode}
+\makeatletter
+\newenvironment{isyntax}{%
+ \let\\\@centercr
+ \list{}{%
+ \itemsep \z@
+ \itemindent -1.5em%
+ \listparindent \itemindent
+ \advance \leftmargin 1.5em%
+ }%
+ \advance \rightskip \z@\@plus0.7\linewidth \relax
+ \linepenalty=100\relax
+ \item\relax
+}{\endlist}
+\makeatother
+% \end{macrocode}
+% \end{environment}
+
+
+% Tabular notes:
+\makeatletter
+\newcommand\tabnotemark[1]{%
+ \leavevmode
+ \ifhmode\edef\@x@sf{\the\spacefactor}\nobreak\fi
+ \hbox{\@textsuperscript{%
+ \normalfont\itshape
+ \count@=#1\relax\@alph\count@
+ }}%
+ \ifhmode\spacefactor\@x@sf\fi
+ \relax
+}
+\newenvironment{tabnotes}{%
+ \list{\the\c@mpfootnote}{%
+ \usecounter{mpfootnote}%
+ \def\makelabel##1{\hfil\tabnotemark{##1}}%
+ \normalfont\footnotesize
+ \setlength\leftmargin{0pt}%
+ \setlength\labelsep{0pt}%
+ \setlength\labelwidth{1.8em}%
+ \setlength\itemindent{\labelwidth}%
+ \setlength\listparindent{1em}%
+ \setlength\parsep{0pt plus 1pt}%
+ \setlength\itemsep{\parsep}%
+ \setlength\topsep{\footnotesep}%
+ }%
+}{\endlist}
+\makeatother
+
+
+\CodelineIndex
+
+\title{\texttt{pstokens.dtx}}
+\author{Lars Hellstr\"om}
+\date{Work in progress}
+\begin{document}
+\maketitle
+
+\DocInput{pstokens.dtx}
+
+\end{document}
+%</driver>
+% \fi
+%
+% \begin{abstract}
+% \package{pstokens} is a collection of utility routines for
+% \Tcllogo\ programs that need to view a file as a sequence of
+% \PS\ tokens rather than as a sequence of characters or lines.
+% \end{abstract}
+%
+%
+% \setnamespace{pstokens}
+%
+% First of all, the \texttt{pstokens} namespace must be defined.
+% \begin{tcl}
+namespace eval pstokens {}
+% \end{tcl}
+%
+% \section{Reading and writing tokens}
+%
+% This section contains the routines for reading a text file as a
+% sequence of tokens (tokenizing) and writing a sequence of tokens
+% to a text file (detokenizing).
+%
+%
+% \subsection{Data structures for tokens}
+%
+% A \emph{token} is a \Tcllogo\ list with the format
+% \begin{quote}
+% \word{type} \word{data}\regstar
+% \end{quote}
+% The interpretation of the \word{data} depends on what the \word{type}
+% is, as specified below. Some of the token types only occur internally
+% in the tokenizer and some others have no meaning to a \PS\ interpreter,
+% but this organisation seems to be the most convenient here.
+%
+% A primary goal for the format of tokens is that if a sequence of
+% tokens are read from one file and written to another then the only
+% things that may have changed should be the whitespace between the
+% tokens, not the tokens themselves. Therefore the tokenizer generally
+% avoids interpreting the tokens any more than necessary;
+% interpretation of tokens is instead handled in the next section.
+%
+% The token types that occur in \package{pstokens} are
+% \begin{description}
+% \item[\texttt{number}, \texttt{radixnumber}]
+% \describestring[token type]{number}
+% \describestring[token type]{radixnumber}
+% A number (integer or real), and radix number, respectively. There
+% is one \word{data} item, which contains the string that
+% represented the number in the tokenized text.
+% \item[\texttt{string}, \texttt{hexstring}, \texttt{base-85}]
+% \describestring[token type]{string}
+% \describestring[token type]{hexstring}
+% \describestring[token type]{base-85}
+% A parenthesis-delimited string, hexadecimal string, and ASCII
+% base-85 string, respectively. There is one \word{data} item,
+% which contains the string (minus delimiters) that represented
+% the string in the tokenized text.
+% \item[\texttt{executable}, \texttt{literal}, \texttt{immediate}]
+% \describestring[token type]{executable}
+% \describestring[token type]{literal}
+% \describestring[token type]{immediate}
+% An executable (normal), literal, and immediately evaluated name
+% respectively. There is one \word{data} item, which simply contains
+% the name as a \Tcllogo\ string, not including any initial slashes.
+% \item[\texttt{procedure}]
+% \describestring[token type]{procedure}
+% An executionable array. The data is simply the sequence of tokens
+% in the array.
+% \item[\texttt{comment}]
+% \describestring[token type]{comment}
+% This is not a \PS\ token at all, but a representation of a
+% comment. The reason comments are tokenized is that they often
+% contain interesting information (especially structured comments).
+% The structure of this kind of token is
+% \begin{quote}
+% |comment| \word{head} \word{tail}
+% \end{quote}
+% The \word{head} is the part of the line which is after the |%|
+% that started the comment and before the first whitespace character
+% after it. The \word{tail} is the rest of the line (including the
+% whitespace character, if there is one). Any of \word{head} and
+% \word{tail} may be empty. The structured comment
+% \begin{quote}
+% |%%Pages: 2|
+% \end{quote}
+% would have \word{head} equal to \verb*"%Pages:" and \word{tail}
+% equal to \verb*" 2".
+% \item[\texttt{eof}]
+% \describestring[token type]{eof}
+% This is a pseudotoken which denotes that the end of the file was
+% reached while looking for the next token. There structure is
+% \begin{quote}
+% |eof| \word{token}\regopt
+% \end{quote}
+% If the \word{token} is present then it is a composite token which
+% was being read when the file unexpectedly ended. It is an error
+% if the \word{token} is present.
+% \item[\texttt{special}]
+% \describestring[token type]{special}
+% This token type can be used to extend the token type syntax
+% without having to change the actual routines for reading and
+% writing tokens. The structure of a |special| token is
+% \begin{quote}
+% |special| \word{token} \word{extra data}\regstar
+% \end{quote}
+% If a |special| token gets sent to the token writer then it will be
+% treated as the token \word{token} (which usually has another
+% type), but other routines may treat it differently based on the
+% information in the \word{extra data}.
+% \item[\texttt{error}]
+% \describestring[token type]{error}
+% A token of this type is returned from the tokenizer whenever it
+% encounters anything which is syntactically incorrect. The
+% general structure of this kind of token is
+% \begin{quote}
+% |error| \word{error type} \word{data}\regstar
+% \end{quote}
+% where the \word{error type} more precisely specifies the error.
+% So far, the following error types are used
+% \begin{quote}
+% |error| |extra| \word{string} \word{line no.}\\
+% |error| |missing| \word{string} \word{line no.}\\
+% |error| |missing/extra| \word{m-string} \word{e-string}
+% \word{line no.}
+% \end{quote}
+% |extra| means that there is an occurence of \word{string} which
+% shouldn't have been there, whereas |missing| means that there
+% should have been a \word{string} there, but isn't. |missing/extra|
+% is the combination that there either should have been an
+% \word{m-string} there, but isn't, or there is an \word{e-string}
+% there, but there shouldn't be.
+% \word{line no.} is the line on which the error was encountered.
+% \end{description}
+%
+%
+% \subsection{Reading tokens}
+%
+% \begin{arrayvar}{input_line}[file-id]
+% \begin{arrayvar}{input_line_no}[file-id]
+% Since the |gets| procedure always reads an entire line (which
+% usually contain several tokens) each time it is used, it is
+% necessary to store the most recently read line somewhere. This is
+% the purpose of the |input_line| array, which is indexed by the file
+% id. In general, text is removed from the entires in |input_line| as
+% it is tokenized.
+%
+% The |input_line_no| array stores the number of the line in the
+% corresponding |input_line| entry. The first line in a file has
+% number one.
+% \end{arrayvar}\end{arrayvar}
+%
+% \begin{proc}{open_input}
+% \begin{proc}{close_input}
+% These procedures are wrappers around \Tcllogo's |open| and |close|
+% procedures respectively which additionally initialize and unset the
+% relevant entry of the |input_line| and |input_line_no| arrays. Thus
+% their syntaxes are
+% \begin{quote}
+% |pstokens::open_input| \word{file-name} \word{access}\regopt\
+% \word{permissions}\regopt\\
+% |pstokens::close_input| \word{file-id}
+% \end{quote}
+% and they return the same things as |open| and |close| respectively.
+%
+% A possible development of tokenizer is to implement filtered input,
+% in which case more support data structures would have to be
+% initialized, but then the |open_input| and |close_input| procedures
+% would be redefined to cope with this.
+% \begin{tcl}
+%<*tokenio>
+proc pstokens::open_input {args} {
+ set res [eval open $args]
+ global pstokens::input_line pstokens::input_line_no
+ set pstokens::input_line($res) ""
+ set pstokens::input_line_no($res) 0
+ return $res
+}
+% \medskip
+proc pstokens::close_input {file_id} {
+ global pstokens::input_line pstokens::input_line_no
+ unset pstokens::input_line($file_id)
+ unset pstokens::input_line_no($file_id)
+ close $file_id
+}
+% \end{tcl}
+% \end{proc}\end{proc}
+%
+% The tokenizer has a couple of extra internal token types for things
+% which later become parts of other tokens.
+% \begin{description}
+% \item[\texttt{beginproc}, \texttt{endproc}]
+% \describestring[token type]{beginproc}
+% \describestring[token type]{endproc}
+% These are the |{| and |}| at the beginning and end respectively of
+% a \PS\ procedure. There is no \word{data}.
+% \item[\texttt{beginstring}, \texttt{beginhex}, \texttt{beginbase85}]
+% \describestring[token type]{beginstring}
+% \describestring[token type]{beginhex}
+% \describestring[token type]{beginbase85}
+% These token types are the |(|, |<|, and |<~| at the beginning of
+% a \PS\ literal, hex, and base-85 string respectively. There is no
+% \word{data}.
+% \end{description}
+%
+%
+% \begin{proc}{get_primitive_token}
+% The |get_primitive_token| procedure returns the next primitive
+% token in the input file. It takes the file id of the source to read
+% from as its only argument.
+%
+% A \emph{primitive token} is a smallest syntactic unit in the \PS\
+% file. Smallest syntactic units are characterized by that the
+% insertion of a space anywhere in them changes its meaning. One
+% exception is however the left parenthesis characters; it is
+% convenient to include |(| among the primitive tokens even though
+% inserting a space after it will change the meaning of
+% the corresponding token.
+%
+% The first step in |get_primitive_token| is to read ahead until some
+% non-whitespace (or the end of the file) is encountered.
+% \begin{tcl}
+proc pstokens::get_primitive_token {file} {
+ global pstokens::input_line pstokens::input_line_no
+ set pstokens::input_line($file)\
+ [string trimleft [set pstokens::input_line($file)] \t\n\f\r\ ]
+ while {[string length [set pstokens::input_line($file)]]==0} {
+ if {[gets $file pstokens::input_line($file)]==-1} then {
+ return {eof}
+ }
+ incr pstokens::input_line_no($file)
+ set pstokens::input_line($file)\
+ [string trimleft [set pstokens::input_line($file)] \t\n\f\r\ ]
+ }
+% \end{tcl}
+% The second step is to isolate the part of the text that constitutes
+% the next token. This is done using regular expressions, but as these
+% get rather complicated it seems best to explain how they are
+% constructed. The ASCII characters null (|\x00|), tab (|\x09|), line
+% feed (|\x0a|), form feed (|\x0c|), carriage return (|\x0d|), and
+% space (|\x20|) count as whitespace and delimit tokens, but null is
+% currently ignored because \Tcllogo\ before v\,8 cannot keep them in
+% strings anyway. The ASCII characters |(|, |)|, |<|, |>|, |[|, |]|,
+% |{|, |}|, |/|, and |%| also delimit tokens, and therefore the
+% regular expression for a \PS\ name is
+% \begin{quote}
+% |(/?/?)([^][()<>{}/%|\meta{tab}\meta{LF}\meta{FF}\meta{CR}^^A
+% \verb*" ]+)|//?"
+% \end{quote}
+% (Note that the first right bracket above does not end a character
+% range, but is part of the list of characters which are exculded
+% from it. Also note that |/| and |//| are valid \PS\ tokens.)
+% The number of initial slashes determine whether the name is
+% \texttt{executable}, \texttt{literal}, or \texttt{immediate}.
+%
+% The executable tokens are however somewhat tricky, because the above
+% regular expression will also match all numbers. An \emph{integer} is
+% any token which matches the regular expression \verb"[+-]?[0-9]+",
+% whereas a \emph{real number} is anything else which matches the
+% regular expression
+% \begin{quote}
+% \verb"[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?"
+% \end{quote}
+% and a \emph{radix number} is anything which matches the regular
+% expression\footnote{Actually there is also the restriction that
+% none of the characters after the \texttt{\#} may represent a digit
+% greater than or equal to the decimal number before the \texttt{\#},
+% but I'm ignoring that, for now.}
+% \begin{quote}
+% |[0-9][0-9]?#[0-9A-Za-z]+|
+% \end{quote}
+% \begin{tcl}
+ if {[regexp "^(/?/?)(\[^\]\[()<>\{\}/%\t\n\f\r\ \]+)|^//?"\
+ [set pstokens::input_line($file)] tokenstr slashes name]}\
+ then {
+ if {[regexp\
+ {^[+-]?([0-9]+(\.[0-9]*)?|\.[0-9]+)([eE][+-]?[0-9]+)?$}\
+ $tokenstr]}\
+ then {
+ set token [list number $tokenstr]
+ } elseif {[regexp {^[0-9][0-9]?#[0-9A-Za-z]+$} $tokenstr]} then {
+ set token [list radixnumber $tokenstr]
+ } else {
+ if {"$tokenstr"=="/" || "$tokenstr"=="//"}\
+ then {set slashes $tokenstr}
+ set token [list [lindex {executable literal immediate}\
+ [string length $slashes]] $name]
+ }
+ }\
+% \end{tcl}
+% There are also a couple of primitive tokens which consist of
+% non-whitespace delimiting characters, namely
+% \begin{quote}
+% |{|, |}|, |<|, |<~|, |<<|, |>>|, |[|, |]|, |(|,
+% \end{quote}
+% The right parenthesis and greater than characters does not appear in
+% the list because they may not appear outside a string. Finally
+% there are the comments, which have the regular expression
+% \begin{quote}
+% |^%([^|\meta{tab}\meta{LF}\meta{FF}\meta{CR}\verb*" ]*)(.*)$"
+% \end{quote}
+% Here the first parenthesis is the head of the comment and the second
+% is the tail.
+% \begin{tcl}
+ else {
+ switch -regexp -- [set pstokens::input_line($file)] {
+ ^\{ {set token beginproc ; set tokenstr \{}
+ ^\} {set token endproc ; set tokenstr \}}
+ ^\\\[ {set token {executable [} ; set tokenstr \[}
+ ^\\\] {set token {executable ]} ; set tokenstr \]}
+ ^< {set token beginhex ; set tokenstr <}
+ ^<~ {set token beginbase85 ; set tokenstr <~}
+ ^<< {set token {executable <<} ; set tokenstr <<}
+ ^>> {set token {executable >>} ; set tokenstr >>}
+ ^\\( {set token beginstring ; set tokenstr (}
+ ^% {
+ regexp "^%(\[^\t\n\f\r\ \]*)(.*)\$"\
+ [set pstokens::input_line($file)] tokenstr head tail
+ set token [list comment $head $tail]
+ }
+ default {
+% \end{tcl}
+% But if none of the above matched the next character must be
+% erroneous.
+% \begin{tcl}
+ set tokenstr\
+ [string index [set pstokens::input_line($file)] 0]
+ set token [list error extra $tokenstr\
+ [set pstokens::input_line_no($file)] ]
+ }
+ }
+ }
+ set pstokens::input_line($file)\
+ [string range [set pstokens::input_line($file)]\
+ [string length $tokenstr] end]
+ return $token
+}
+% \end{tcl}
+%
+% \end{proc}
+%
+% \begin{proc}{get_string}
+% The |get_string| procedure reads a \PS\ literal string and converts
+% it to a \texttt{string} token, which it returns. The only argument
+% is the file id of the input file. It is assumed that the next thing
+% on the input file line is the character after the left parenthesis
+% that started the string.
+%
+% The only conversion of the data that |get_string| performs is that
+% escaped newlines are skipped and non-escaped newlines are replaced
+% by |\n| escapes. If the end-of-file is encountered while reading a
+% string then the necessary number of right parentheses are added to
+% the string and the resulting string token is built into an |eof|
+% token, which is returned.
+%
+% The local variable |level| keeps track of the parenthesis nesting
+% level, the local variable |data| is the string read so far, and the
+% local variable is a flag which is |1| if the next character should
+% be escaped and |0| otherwise.
+% \begin{tcl}
+proc pstokens::get_string {file} {
+ upvar #0 pstokens::input_line line
+ global pstokens::input_line_no
+ set level 1
+ set data ""
+ set escaped 0
+ while {$level>0} {
+ while {$level>0 && [string length $line($file)]>0} {
+ set c [string index $line($file) 0]
+ if {$escaped} then {
+ append data $c
+ set escaped 0
+ } else {
+ switch -exact -- $c {
+ ( {incr level ; append data $c}
+ ) {
+ incr level -1
+ if {$level>0} then {append data $c}
+ }
+ \\ {set escaped 1 ; append data $c}
+ default {append data $c}
+ }
+ }
+ set line($file) [string range $line($file) 1 end]
+ }
+ if {$level>0} then {
+ if {!$escaped} then {append data \\n}
+ set escaped 0
+ if {[gets $file line($file)]==-1} then {
+ while {$level>1} {append data ) ; incr level -1}
+ return [list eof [list string $data]]
+ }
+ incr pstokens::input_line_no($file)
+ }
+ }
+ list string $data
+}
+% \end{tcl}
+% \end{proc}
+%
+%
+% \begin{proc}{get_hexstring}
+% The |get_hexstring| procedure reads a \PS\ hexadecimal string and
+% converts it to a \texttt{hexstring} token, which it returns. The
+% only argument is the file id of the input file. It is assumed that
+% the next thing on the input file line is the character after the
+% less than character that started the string.
+%
+% The only conversion of the data that |get_hexstring| performs is that
+% embedded whitespace is removed. If a character which is neither
+% whitespace, a hexadecimal digit, or the closing |>| is encountered
+% then an \texttt{error} token is returned and the text read is put
+% back into the |input_line| buffert (followed by an |>| which is
+% inserted to terminate the string. If the end-of-file is
+% encountered while reading a string then what has been read so far
+% is returned as a hexstring token built into an |eof| token.
+% \begin{tcl}
+proc pstokens::get_hexstring {file} {
+ upvar #0 pstokens::input_line line
+ global pstokens::input_line_no
+ set data ""
+ while 1 {
+ while {[string length $line($file)]==0} {
+ if {[gets $file line($file)]==-1} then {
+ return [list eof [list hexstring $data]]
+ }
+ incr pstokens::input_line_no($file)
+ }
+ set c [string index $line($file) 0]
+ set line($file) [string range $line($file) 1 end]
+ switch -regexp -- $c {
+ {[0-9a-fA-F]} {append data $c}
+ \[\t\n\f\r\ \] {}
+ > {break}
+ default {
+ set line($file) <$data>$c$line($file)
+ return [list error missing/extra > $c\
+ [set pstokens::input_line_no($file)]]
+ }
+ }
+ }
+ list hexstring $data
+}
+% \end{tcl}
+% \end{proc}
+%
+%
+% \begin{proc}{get_base85string}
+% The |get_base85string| procedure reads a \PS\ base-85 string and
+% converts it to a \texttt{base-85} token, which it returns. The
+% only argument is the file id of the input file. It is assumed that
+% the next thing on the input file line is the character after the
+% |<~| characters that started the string.
+%
+% The only conversion of the data that |get_base85string| performs is
+% that embedded whitespace is removed. If a character which is neither
+% whitespace, a base-85 digit, the letter |z|, or the closing |~| is
+% encountered then an \texttt{error} token is returned and the text
+% read is put back into the |input_line| buffert (followed by a |~>|
+% that terminates the string). If the end-of-file is encountered while
+% reading a string then what has been read so far is returned as a
+% base-85 string token built into an |eof| token.
+% \begin{tcl}
+proc pstokens::get_base85string {file} {
+ upvar #0 pstokens::input_line line pstokens::input_line_no line_no
+ set data ""
+ while 1 {
+ while {[string length $line($file)]==0} {
+ if {[gets $file line($file)]==-1} then {
+ return [list eof [list base-85 $data]]
+ }
+ incr line_no($file)
+ }
+ set c [string index $line($file) 0]
+ set line($file) [string range $line($file) 1 end]
+ switch -regexp -- $c {
+ {[!-uz]} {append data $c}
+ \[\t\n\f\r\ \] {}
+ ~ {break}
+ default {
+ set line($file) <~$data~>$c$line($file)
+ return [list error missing/extra ~> $c $line_no($file)]
+ }
+ }
+ }
+% \end{tcl}
+% If the character after the closing |~| is not a |>| then the entire
+% string is put back in the |input_line| buffert and a |missing| error
+% is issued.
+% \begin{tcl}
+ if {[string length $line($file)]>0 &&\
+ "[string index $line($file) 0]"==">"} then {
+ set line($file) [string range $line($file) 1 end]
+ list base-85 $data
+ } else {
+ set line($file) <~$data~>$line($file)
+ list error missing > $line_no($file)
+ }
+}
+% \end{tcl}
+% \end{proc}
+%
+%
+% \begin{proc}{get_token}
+% \begin{proc}{get_token_rek}
+% The |get_token| procedure has the syntax
+% \begin{quote}
+% |pstokens::get_token| \word{file-id}
+% \end{quote}
+% and returns the next \PS\ token in the \word{file-id} file.
+%
+% Most of the work is however done by the |get_token_rek| procedure,
+% which is a recursive form of |get_token|. |get_token_rek| handles
+% converting the various |begin|\textellipsis\ token types to their
+% complete forms, and it calls itself recursively to get the tokens
+% that form a procedure. |get_token| only checks that there are no
+% lone |endproc| tokens.
+% \begin{tcl}
+proc pstokens::get_token {file} {
+ set token [pstokens::get_token_rek $file]
+ if {"[lindex $token 0]" == "endproc"} then {
+ global pstokens::input_line_no
+ set token [list error extra \} [set pstokens::input_line_no($file)]]
+ }
+ return $token
+}
+% \medskip
+proc pstokens::get_token_rek {file} {
+ set token [pstokens::get_primitive_token $file]
+ switch -exact [lindex $token 0] {
+ beginproc {
+ set token {procedure}
+ while {"[lindex [set token2 [pstokens::get_token_rek $file]] 0]"\
+ != "endproc" && "[lindex $token2 0]" != "eof"} {
+ lappend token $token2
+ }
+ if {"[lindex $token2 0]" == "eof"} then {
+ if {[llength $token2]>1} then {
+ lappend token [lindex $token2 1]
+ }
+ set token [list eof $token]
+ }
+ }
+ beginstring {set token [pstokens::get_string $file]}
+ beginhex {set token [pstokens::get_hexstring $file]}
+ beginbase85 {set token [pstokens::get_base85string $file]}
+ }
+ return $token
+}
+% \end{tcl}
+%
+% These procedures, and in particular |get_token_rek|, could be made
+% more intelligent when it comes to handling errors that occur inside
+% procedures. Currently they just embed the |error| token in the
+% procedure.
+% \end{proc}\end{proc}
+%
+%
+% \subsection{Writing tokens}
+%
+% Writing tokens is simpler than reading them; the only real
+% complication is that the length of lines must be bounded. In order to
+% achieve this, data is written linewise to the file (much like the way
+% it is read).
+%
+% \begin{arrayvar}{output_line}[file-id]
+% The |output_line| array stores each output line until it is
+% actually written to file. Like with the |input_line| array, each
+% active output file has its own entry in this array.
+% \end{arrayvar}
+%
+% \begin{proc}{flush_line}
+% The |flush_line| procedure has the syntax
+% \begin{quote}
+% |pstokens::flush_line| \word{file-id} \word{physical}\regopt
+% \end{quote}
+% It flushes buffered data that is to be written to the file with id
+% \word{file-id}. The \word{physical} argument controls whether the
+% data actually should be physically written to file (when |1|) or
+% merely flushed from the |output_line| array (when |0|). The default
+% is |0|. Calling |flush_line| starts a new line in the output file
+% unless the |output_line| entry is empty.
+% \begin{tcl}
+proc pstokens::flush_line {fileid {physical 0}} {
+ global pstokens::output_line
+ if {[string length [set pstokens::output_line($fileid)]]>0} then {
+ puts $fileid [set pstokens::output_line($fileid)]
+ set pstokens::output_line($fileid) ""
+ }
+ if {$physical} then {flush $fileid}
+}
+% \end{tcl}
+% \end{proc}
+%
+% \begin{proc}{open_output}
+% \begin{proc}{close_output}
+% These procedures are wrappers around \Tcllogo's |open| and |close|
+% procedures respectively which additionally initialize and unset the
+% relevant entry of the |output_line|. Thus their syntaxes are
+% \begin{quote}
+% |pstokens::open_output| \word{file-name} \word{access}\regopt\
+% \word{permissions}\regopt\\
+% |pstokens::close_output| \word{file-id}
+% \end{quote}
+% and they return the same things as |open| and |close| respectively.
+% Note however that unlike the case with |open|, the \word{access}
+% argument of |open_output| defaults to |w|.
+%
+% A possible development of the routines for writing tokens would be
+% to support output channels other than files. In that case more
+% support data structures would have to be initialized, but then the
+% |open_output| and |close_output| procedures would be redefined to
+% cope with this.
+% \begin{tcl}
+proc pstokens::open_output {name {access w} args} {
+ set res [eval open \$name \$access $args]
+ global pstokens::output_line
+ set pstokens::output_line($res) ""
+ return $res
+}
+% \medskip
+proc pstokens::close_output {file_id} {
+ pstokens::flush_line $file_id 1
+ close $file_id
+ global pstokens::output_line
+ unset pstokens::output_line($file_id)
+}
+% \end{tcl}
+% \end{proc}\end{proc}
+%
+% \begin{variable}{wrap_length}
+% The |wrap_length| variable stores the maximal number of characters
+% that may appear on a single output line. The same value applies for
+% all output files.
+% \begin{tcl}
+set pstokens::wrap_length 72
+% \end{tcl}
+% \end{variable}
+%
+% \begin{proc}{put_word}
+% The |put_word| procedure has the syntax
+% \begin{quote}
+% |pstokens::put_word| \word{file-id} \word{string}
+% \end{quote}
+% It writes \meta{string} to the file with id \word{file-id},
+% preceeded by a whitespace character if the current line is
+% nonempty. The whitespace character is usually a space, but it
+% will be a newline if the length of what has already been written
+% to the current line is too large to also fit the \word{string}
+% within the length specified by the |wrap_length| variable.
+% \begin{tcl}
+proc pstokens::put_word {file str} {
+ global pstokens::output_line pstokens::wrap_length
+ set l [string length [set pstokens::output_line($file)]]
+ if {$l==0} then {
+ set pstokens::output_line($file) $str
+ } elseif {$l + [string length $str] < ${pstokens::wrap_length}}\
+ then {
+ append pstokens::output_line($file) " $str"
+ } else {
+ puts $file [set pstokens::output_line($file)]
+ set pstokens::output_line($file) $str
+ }
+}
+% \end{tcl}
+% \end{proc}
+%
+% \begin{proc}{put_breakable}
+% The |put_breakable| procedure has the syntax
+% \begin{quote}
+% |pstokens::put_breakable| \word{file-id}
+% \word{prefix} \word{text} \word{suffix}
+% \end{quote}
+% It writes \meta{prefix}\meta{text}\meta{suffix} to the file with
+% id \word{file-id} and inserts whitspace (newlines) into the
+% \meta{text} in such a way that the |wrap_length| linewidth isn't
+% exceeded. This is primarily useful for hexadecimal and base-85
+% strings.
+% \begin{tcl}
+proc pstokens::put_breakable {file prefix text suffix} {
+ upvar #0 pstokens::output_line line pstokens::wrap_length wrap
+ if {[string length $line($file)] + [string length $prefix] +\
+ [string length $text] + [string length $suffix] < $wrap} then {
+ append line($file) " $prefix$text$suffix"
+ } else {
+ pstokens::flush_line $file
+ set line($file) $prefix
+ while {[string length $line($file)] + [string length $text] +\
+ [string length $suffix] < $wrap} {
+ set t [expr {$wrap -[string length $line($file)]}]
+ append line($file) [string range $text 0 [expr {$t-1}]]
+ set text [string range $text $t end]
+ pstokens::flush_line $file
+ }
+ append line($file) $text$suffix
+ }
+}
+% \end{tcl}
+% \end{proc}
+%
+% \begin{proc}{put_string}
+% The |put_string| procedure has the syntax
+% \begin{quote}
+% |pstokens::put_string| \word{file-id} \word{text}
+% \end{quote}
+% It writes |(|\meta{text}|)| to the file with id \word{file-id} and
+% inserts backslash+newline items into the \meta{text} in such a way
+% that the |wrap_length| linewidth isn't exceeded and the
+% tokenization of the string is unaffected.
+%
+% The only places where it is unsafe to insert a backslash+newline is
+% inside an escape squence. A position can only be inside such a
+% sequence if the part of the \meta{text} which is before it gets a
+% match against the regular expression
+% \begin{quote}
+% \verb"(^|[^\])(\\\\)*(\\[0-7]?[0-7]?)$"
+% \end{quote}
+% and in this case the position between the second and third
+% parenthesis is safe.
+% \begin{tcl}
+proc pstokens::put_string {file text} {
+ upvar #0 pstokens::output_line line pstokens::wrap_length wrap
+ if {[string length $line($file)] + [string length $text] + 2 <\
+ $wrap} then {
+ append line($file) " ($text)"
+ } else {
+ pstokens::flush_line $file
+ set line($file) (
+ while {[string length $line($file)] + [string length $text] +\
+ 1 < $wrap} {
+ set t [expr {$wrap -[string length $line($file)]}]
+ set s [string range $text 0 [expr {$t-1}]]
+ if {[regexp {(^|[^\])(\\\\)*(\\[0-7]?[0-7]?)$} $s\
+ foo bar baz escape]}\
+ then {
+ incr t -[string length $escape]
+ set s [string range $text [expr {$t-1}]]
+ }
+ append line($file) $s
+ set text [string range $text $t end]
+ pstokens::flush_line $file
+ }
+ append line($file) $text)
+ }
+}
+% \end{tcl}
+% \end{proc}
+%
+% \begin{proc}{put_token}
+% The |put_token| procedure has the syntax
+% \begin{quote}
+% |pstokens::put_token| \word{file-id} \word{token}
+% \end{quote}
+% It writes the \PS\ token \word{token} to the file with id
+% \word{file-id}.
+% \begin{tcl}
+proc pstokens::put_token {file token} {
+ switch [lindex $token 0] {
+ number -
+ radixnumber -
+ executable {pstokens::put_word $file [lindex $token 1]}
+ literal {pstokens::put_word $file /[lindex $token 1]}
+ immediate {pstokens::put_word $file //[lindex $token 1]}
+ string {pstokens::put_string $file [lindex $token 1]}
+ comment {
+ pstokens::flush_line $file 0
+ puts $file %[lindex $token 1][lindex $token 2]
+ }
+ special {pstoken::put_token $file [lindex $token 1]}
+ procedure {
+ pstokens::put_word $file \{
+ foreach subtok [lrange $token 1 end] {
+ pstokens::put_token $file $subtok
+ }
+ pstokens::put_word $file \}
+ }
+ hexstring {pstokens::put_breakable $file < [lindex $token 1] >}
+ base-85 {pstokens::put_breakable $file <~ [lindex $token 1] ~>}
+ eof {
+ if {[llength $token]>1} then {
+ pstokens::put_token $file [lindex $token 1]
+ }
+ pstokens::terminal_print "Token unexpectedly terminated by\
+ end of file."
+ }
+ error {
+ pstokens::terminal_print "Error token intercepted at put:\
+ [switch [lindex $token 1] {
+ extra {expr {"extra [lindex $token 2] at input line\
+ [lindex $token 3]"}}
+ missing {expr {"missing [lindex $token 2] at input\
+ line [lindex $token 3]"}}
+ missing/extra {expr {"missing [lindex $token 2], or\
+ extra [lindex $token 3], at input line\
+ [lindex $token 4]"}}
+ default {expr {"!!!-unimplemented error-!!!"}}
+ }]."
+ }
+ default {
+ pstokens::terminal_print\
+ "Token of unknown type encountered: $token"
+ }
+ }
+}
+% \end{tcl}
+% \end{proc}
+
+
+% \begin{tcl}
+%</tokenio>
+% \end{tcl}
+%
+% \section{Token stream editing}
+%
+% This section I still have ToDo.
+%
+%
+% \begin{thebibliography}{99}
+%
+% \bibitem{PSman}
+% Adobe Systems Incorporated:
+% \textit{\PS\ language reference manual, 3rd ed.},
+% Addison--Wesley, 1999; ISBN 0-201-37922-8;
+% http:/\slash\texttt{partners.adobe.com}\slash
+% \texttt{asn}\slash\texttt{developer}\slash\texttt{PDFS}\slash
+% \texttt{TN}\slash\texttt{PLRM.pdf}.
+%
+% \bibitem{MPinTUG}
+% John D.\ Hobby: \textit{A \MF-like System with \PS\ output},
+% TUGboat \textbf{10} (4) (1989), 505--512.
+% %^^A 2 or 4 ? Hobby's bibliography says 2, but Bebee's
+% %^^A tugboat.bib says 4.
+% %^^A @article{Hobby89a,
+% %^^A author = {John D. Hobby},
+% %^^A title = {A {METAFONT}-like System with PostScript Output},
+% %^^A journal = {{TUG}boat},
+% %^^A volume = {10},
+% %^^A number = {2},
+% %^^A pages = {505--512},
+% %^^A year = {1989}
+% %^^A }
+% \bibitem{MPman}
+% John D.\ Hobby: \textit{A User's Manual for \MP},
+% AT\&T Bell Laboratories Computing Science Technical Report no.~162
+% (1992);
+% \textit{see} http:/\slash \texttt{cm.bell-labs.com}\slash
+% \texttt{who}\slash\texttt{hobby}\slash\texttt{MetaPost.html}.
+% %^^A @techreport{Hobby92,
+% %^^A title = {A User's manual for {MetaPost}},
+% %^^A author = {John D. Hobby},
+% %^^A institution = {AT\&T Bell Laboratories},
+% %^^A address = {Murray Hill, New Jersey},
+% %^^A type = {Computing Science Technical Report},
+% %^^A number = {no.~162},
+% %^^A year = {1992}
+% %^^A }
+%
+% \end{thebibliography}
+%
+%
+% \Finale
+%
+\endinput
generated by cgit v1.2.3 (git 2.39.1) at 2024-12-29 19:05:14 +0000