summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/doc/generic
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2020-10-01 21:05:19 +0000
committerKarl Berry <karl@freefriends.org>2020-10-01 21:05:19 +0000
commitac45ed8c9c41ead15554f580f98dac4fd93a21b8 (patch)
treedebbcd7c58017a3fa625cdbb027e317040c97e3a /Master/texmf-dist/doc/generic
parente66d6693f79686059c6e9fdf9eab8db3b0077c44 (diff)
tokcycle (1oct20)
git-svn-id: svn://tug.org/texlive/trunk@56501 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/doc/generic')
-rw-r--r--Master/texmf-dist/doc/generic/tokcycle/README12
-rw-r--r--Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdfbin255073 -> 258406 bytes
-rw-r--r--Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex38
-rw-r--r--Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdfbin258377 -> 261160 bytes
-rw-r--r--Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex73
5 files changed, 107 insertions, 16 deletions
diff --git a/Master/texmf-dist/doc/generic/tokcycle/README b/Master/texmf-dist/doc/generic/tokcycle/README
index 491a0006977..ee682cbc05c 100644
--- a/Master/texmf-dist/doc/generic/tokcycle/README
+++ b/Master/texmf-dist/doc/generic/tokcycle/README
@@ -3,12 +3,12 @@ The tokcycle package helps one to build tools to process tokens from an input st
Author: Steven B Segletes, Christian Tellechea (contributor)
-Files : 1) tokcycle.tex
- 2) tokcycle.sty
+Files : 1) tokcycle.tex (tex/generic/tokcycle)
+ 2) tokcycle.sty (tex/latex/tokcycle)
3) tokcycle-doc.tex
- 4) tokcycle-doc.pdf
- 5) tokcycle-examples.tex
- 6) tokcycle-examples.pdf
- 7) README
+ 4) tokcycle-doc.pdf (doc/latex/tokcycle)
+ 5) tokcycle-examples.tex (doc/latex/tokcycle)
+ 6) tokcycle-examples.pdf (doc/latex/tokcycle)
+ 7) README (doc/latex/tokcycle)
Released under the LaTeX Project Public License v1.3c or later, see http://www.latex-project.org/lppl.txt \ No newline at end of file
diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf
index 5a86f5c5a2c..c707af4f154 100644
--- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf
+++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex
index 8614827a24f..612ab5baf29 100644
--- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex
+++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex
@@ -80,6 +80,7 @@ Each of these
The package provides a number of options
for handling groups.
\end{quote}
+\enlargethispage{1.4\baselineskip}
\tableofcontents
@@ -299,8 +300,8 @@ An integer option, call it \textit{n}, takes the
The \texttt{[x]} option employs the \macname{expanded} primitive to maximally
expand the argument before appending it to \macname{cytoks}.
-The \texttt{[x]} option will prove useful when the \Character{} directive involves
- a transformation that is fully expandable.
+The \texttt{[x]} option will prove useful when the \Character{} or other
+ directives involve a transformation that is fully expandable.
Its use will allow the expanded
result of the transformation to be placed in the token register, rather
than the unexpanded transformation instructions.
@@ -464,6 +465,35 @@ i)~implicit grouping tokens (e.g., \macname{bgroup} and \macname{egroup})
ii)~implicit space tokens (e.g., \macname{@sptoken}) will be processed
by the \Space{} directive.
+\subsubsection{Active-Implicit Tokens, Including Spaces}
+
+One may occasionally run across a token that is both active and implicit.
+For example, in the following code, \texttt{Q} is made both active and implicit:
+
+\begin{verbbox}[\vbdelim]
+\catcode`Q=\active
+\let Qx
+\end{verbbox}
+\medskip\theverbbox\medskip\noindent
+
+\noindent
+In general, both \macname{ifactivetok} and \macname{ifimplicittok}
+ tests can be performed together to determine such cases.
+
+This is true even in the case of active-implicit catcode-10 spaces, which
+ are always processed through the Space directive.
+However, because of the process \TokCycle{} uses in digesting space
+ tokens, the actual token passed to the Space directive (so-called
+ \verb|#1|), when an active-implicit space is encountered in the input
+ stream, is a generic implicit space token named \macname{tc@sptoken}.
+However, the catcode-12 version of the active character that
+ produced it will be, for that moment, retained in a definition
+ named \macname{theactivespace}.
+This can be useful if detokenization is required of the spaces.
+Such an example is described in the \texttt{tokcycle-examples}
+ adjunct document.
+
+
\subsubsection{Parameter (cat-6) tokens (e.g., \texttt{\#}): \macname{ifcatSIX}}
Typically, category-code 6 tokens (like \texttt{\#}) are used to
@@ -589,7 +619,7 @@ What follows are the known limitations of the package.
Surely, there are more cases associated with arcane catcode-changing
syntax that are not accounted for;
I encourage you to bring them to my attention.
-If I can't fix them, I can at least disclaim and declaim them.
+If I can't fix them, I can at least disclaim and declaim them below:
\begin{itemize}
@@ -633,7 +663,7 @@ The \texttt{tex.stackexchange} site provides a wonderful
\section*{Source Code}
\Large\texttt{tokcycle.sty}
-\verbfilenobox[\footnotesize]{tokcycle.sty}\vspace{2in}
+\verbfilenobox[\footnotesize]{tokcycle.sty}\vspace{0.5in}
\noindent\texttt{tokcycle.tex}
\verbfilenobox[\footnotesize]{tokcycle.tex}
diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf
index 8b92d149766..0c1f565132d 100644
--- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf
+++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf
Binary files differ
diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex
index 0c46129bbd1..49fbe6c66ce 100644
--- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex
+++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex
@@ -194,6 +194,8 @@
\newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}}
\newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}}
+\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
+ {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Examples, examples, and more examples}
@@ -873,6 +875,8 @@ If \macname{let} to a character, they may be identified in the Character
directive;
If \macname{let} to a control sequence or defined via \macname{def},
they may be identified in the Macro directive.
+For information on how to process active spaces, please refer to
+ section~\ref{s:spc}.
\let\svt T
\Characterdirective{\tctestifcon\ifactivetok
{\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}}
@@ -1257,20 +1261,35 @@ Bottom line:
\endgroup
-\subsubsection{Catcode 10 space tokens}
+\subsubsection{Catcode 10 space tokens\label{s:spc}}
\begingroup
Here we demonstrate that \TokCycle{} can handle arbitrary redesignation
- of tokens to cat-10, as well as implicit space tokens.
+ of tokens to cat-10, as well as implicit space tokens (both implicit
+ macro spaces and active-implicit character spaces).
\ddbend While it should seem natural, we note that implicit
space tokens are directed to the Space directive rather than
the Character directive.
However, \macname{ifimplicittok} may still be used to
differentiate an explicit space from an implicit one.
-\medskip
+
+\ddbend
+Also, if the implicit space is an active character, \macname{ifactivetok}
+ is also set, for the user's benefit.
+Likewise, \macname{ifactivechar} is also checked for active spaces, to see
+ if the charcode of the active space is still, indeed, active.
+While \verb|#1| may be used to represent the space in the space directive,
+ a special technique is required to recover the detokenized name of the
+ active space character.
+In particular, \verb|#1| will, for active spaces, contain a generic
+ implicit space token, \macname{tc@sptoken}.
+However, the name of the associated active character in the input stream
+ will be defined as a cat-12 token in \macname{theactivespace}.
+
+\medskip
Note in the following examples that cat-10 tokens do \textit{not}
get under-dots.
The next three examples all use the same input, but with different
@@ -1313,19 +1332,61 @@ catcode settings for the space and the underscore.
\fbox{a_c d} b_g\itshape f\upshape\endtokencycle
\end{exampleA}
+The next two examples introduce implicit and active-implicit spaces.
+The latter example also demonstrates detokenization of such spaces.
+
+\medskip
+Here is a useful macro for detokenizing space tokens in the context of
+ \TokCycle{}.
+It can process not only explicit cat-10 spaces, but also implicit and
+ active-implicit spaces.
+To add it to the \macname{cytoks} token list, make sure you use the
+ \verb|[x]| fully expanded option to \macname{addcytoks}.
+
+\medskip
+\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling
+ implicit and active-implicit spaces}
+\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
+ {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
+\end{exampleC}
+\medskip
+
%%%
\resettokcycle
\Characterdirective{\addcytoks{\underdot{#1}}}
-\begin{exampleA}[lefthand width=10cm]{Implicit spaces work{,} too}
+\begin{exampleA}[lefthand width=10cm]{Implicit spaces also work}
\resettokcycle
\Characterdirective{\addcytoks{\underdot{#1}}}
-\def\:{\let\mysptoken= } \: %
+\def\:{\let\z= } \: %
\catcode`\_=10 %
\catcode`\ =12 %
\tokencyclexpress
-\fbox{a\mysptoken{}c d} b_g\itshape f\upshape
+\fbox{a\z{}c d} b_g\itshape f\upshape
+\endtokencyclexpress
+\end{exampleA}
+
+\begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too}
+\resettokcycle
+\Characterdirective{\addcytoks{\underdot{#1}}}
+\def\:{\let\z= } \: %
+\catcode`Q=\active %
+\def\:{\let Q= } \: %
+\catcode`\_=10 %
+
+\tokencyclexpress
+xQx x_x\z{}x%
+\endtokencyclexpress
+
+\resettokcycle
+\Spacedirective{\addcytoks{\textcolor{cyan}\bgroup}%
+ \addcytoks[x]{(\detokenizespacetok{#1})}%
+ \addcytoks{\egroup}}
+
+\medskip
+\tokencyclexpress
+xQx x_x\z{}x%
\endtokencyclexpress
\end{exampleA}