From ac45ed8c9c41ead15554f580f98dac4fd93a21b8 Mon Sep 17 00:00:00 2001 From: Karl Berry Date: Thu, 1 Oct 2020 21:05:19 +0000 Subject: tokcycle (1oct20) git-svn-id: svn://tug.org/texlive/trunk@56501 c570f23f-e606-0410-a88d-b1316a301751 --- Master/texmf-dist/doc/generic/tokcycle/README | 12 ++-- .../doc/generic/tokcycle/tokcycle-doc.pdf | Bin 255073 -> 258406 bytes .../doc/generic/tokcycle/tokcycle-doc.tex | 38 +++++++++-- .../doc/generic/tokcycle/tokcycle-examples.pdf | Bin 258377 -> 261160 bytes .../doc/generic/tokcycle/tokcycle-examples.tex | 73 +++++++++++++++++++-- 5 files changed, 107 insertions(+), 16 deletions(-) (limited to 'Master/texmf-dist/doc/generic') diff --git a/Master/texmf-dist/doc/generic/tokcycle/README b/Master/texmf-dist/doc/generic/tokcycle/README index 491a0006977..ee682cbc05c 100644 --- a/Master/texmf-dist/doc/generic/tokcycle/README +++ b/Master/texmf-dist/doc/generic/tokcycle/README @@ -3,12 +3,12 @@ The tokcycle package helps one to build tools to process tokens from an input st Author: Steven B Segletes, Christian Tellechea (contributor) -Files : 1) tokcycle.tex - 2) tokcycle.sty +Files : 1) tokcycle.tex (tex/generic/tokcycle) + 2) tokcycle.sty (tex/latex/tokcycle) 3) tokcycle-doc.tex - 4) tokcycle-doc.pdf - 5) tokcycle-examples.tex - 6) tokcycle-examples.pdf - 7) README + 4) tokcycle-doc.pdf (doc/latex/tokcycle) + 5) tokcycle-examples.tex (doc/latex/tokcycle) + 6) tokcycle-examples.pdf (doc/latex/tokcycle) + 7) README (doc/latex/tokcycle) Released under the LaTeX Project Public License v1.3c or later, see http://www.latex-project.org/lppl.txt \ No newline at end of file diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf index 5a86f5c5a2c..c707af4f154 100644 Binary files a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf and b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf differ diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex index 8614827a24f..612ab5baf29 100644 --- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex @@ -80,6 +80,7 @@ Each of these The package provides a number of options for handling groups. \end{quote} +\enlargethispage{1.4\baselineskip} \tableofcontents @@ -299,8 +300,8 @@ An integer option, call it \textit{n}, takes the The \texttt{[x]} option employs the \macname{expanded} primitive to maximally expand the argument before appending it to \macname{cytoks}. -The \texttt{[x]} option will prove useful when the \Character{} directive involves - a transformation that is fully expandable. +The \texttt{[x]} option will prove useful when the \Character{} or other + directives involve a transformation that is fully expandable. Its use will allow the expanded result of the transformation to be placed in the token register, rather than the unexpanded transformation instructions. @@ -464,6 +465,35 @@ i)~implicit grouping tokens (e.g., \macname{bgroup} and \macname{egroup}) ii)~implicit space tokens (e.g., \macname{@sptoken}) will be processed by the \Space{} directive. +\subsubsection{Active-Implicit Tokens, Including Spaces} + +One may occasionally run across a token that is both active and implicit. +For example, in the following code, \texttt{Q} is made both active and implicit: + +\begin{verbbox}[\vbdelim] +\catcode`Q=\active +\let Qx +\end{verbbox} +\medskip\theverbbox\medskip\noindent + +\noindent +In general, both \macname{ifactivetok} and \macname{ifimplicittok} + tests can be performed together to determine such cases. + +This is true even in the case of active-implicit catcode-10 spaces, which + are always processed through the Space directive. +However, because of the process \TokCycle{} uses in digesting space + tokens, the actual token passed to the Space directive (so-called + \verb|#1|), when an active-implicit space is encountered in the input + stream, is a generic implicit space token named \macname{tc@sptoken}. +However, the catcode-12 version of the active character that + produced it will be, for that moment, retained in a definition + named \macname{theactivespace}. +This can be useful if detokenization is required of the spaces. +Such an example is described in the \texttt{tokcycle-examples} + adjunct document. + + \subsubsection{Parameter (cat-6) tokens (e.g., \texttt{\#}): \macname{ifcatSIX}} Typically, category-code 6 tokens (like \texttt{\#}) are used to @@ -589,7 +619,7 @@ What follows are the known limitations of the package. Surely, there are more cases associated with arcane catcode-changing syntax that are not accounted for; I encourage you to bring them to my attention. -If I can't fix them, I can at least disclaim and declaim them. +If I can't fix them, I can at least disclaim and declaim them below: \begin{itemize} @@ -633,7 +663,7 @@ The \texttt{tex.stackexchange} site provides a wonderful \section*{Source Code} \Large\texttt{tokcycle.sty} -\verbfilenobox[\footnotesize]{tokcycle.sty}\vspace{2in} +\verbfilenobox[\footnotesize]{tokcycle.sty}\vspace{0.5in} \noindent\texttt{tokcycle.tex} \verbfilenobox[\footnotesize]{tokcycle.tex} diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf index 8b92d149766..0c1f565132d 100644 Binary files a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf and b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf differ diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex index 0c46129bbd1..49fbe6c66ce 100644 --- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex @@ -194,6 +194,8 @@ \newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}} \newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}} +\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% + {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Examples, examples, and more examples} @@ -873,6 +875,8 @@ If \macname{let} to a character, they may be identified in the Character directive; If \macname{let} to a control sequence or defined via \macname{def}, they may be identified in the Macro directive. +For information on how to process active spaces, please refer to + section~\ref{s:spc}. \let\svt T \Characterdirective{\tctestifcon\ifactivetok {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}} @@ -1257,20 +1261,35 @@ Bottom line: \endgroup -\subsubsection{Catcode 10 space tokens} +\subsubsection{Catcode 10 space tokens\label{s:spc}} \begingroup Here we demonstrate that \TokCycle{} can handle arbitrary redesignation - of tokens to cat-10, as well as implicit space tokens. + of tokens to cat-10, as well as implicit space tokens (both implicit + macro spaces and active-implicit character spaces). \ddbend While it should seem natural, we note that implicit space tokens are directed to the Space directive rather than the Character directive. However, \macname{ifimplicittok} may still be used to differentiate an explicit space from an implicit one. -\medskip + +\ddbend +Also, if the implicit space is an active character, \macname{ifactivetok} + is also set, for the user's benefit. +Likewise, \macname{ifactivechar} is also checked for active spaces, to see + if the charcode of the active space is still, indeed, active. +While \verb|#1| may be used to represent the space in the space directive, + a special technique is required to recover the detokenized name of the + active space character. +In particular, \verb|#1| will, for active spaces, contain a generic + implicit space token, \macname{tc@sptoken}. +However, the name of the associated active character in the input stream + will be defined as a cat-12 token in \macname{theactivespace}. + +\medskip Note in the following examples that cat-10 tokens do \textit{not} get under-dots. The next three examples all use the same input, but with different @@ -1313,19 +1332,61 @@ catcode settings for the space and the underscore. \fbox{a_c d} b_g\itshape f\upshape\endtokencycle \end{exampleA} +The next two examples introduce implicit and active-implicit spaces. +The latter example also demonstrates detokenization of such spaces. + +\medskip +Here is a useful macro for detokenizing space tokens in the context of + \TokCycle{}. +It can process not only explicit cat-10 spaces, but also implicit and + active-implicit spaces. +To add it to the \macname{cytoks} token list, make sure you use the + \verb|[x]| fully expanded option to \macname{addcytoks}. + +\medskip +\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling + implicit and active-implicit spaces} +\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% + {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} +\end{exampleC} +\medskip + %%% \resettokcycle \Characterdirective{\addcytoks{\underdot{#1}}} -\begin{exampleA}[lefthand width=10cm]{Implicit spaces work{,} too} +\begin{exampleA}[lefthand width=10cm]{Implicit spaces also work} \resettokcycle \Characterdirective{\addcytoks{\underdot{#1}}} -\def\:{\let\mysptoken= } \: % +\def\:{\let\z= } \: % \catcode`\_=10 % \catcode`\ =12 % \tokencyclexpress -\fbox{a\mysptoken{}c d} b_g\itshape f\upshape +\fbox{a\z{}c d} b_g\itshape f\upshape +\endtokencyclexpress +\end{exampleA} + +\begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too} +\resettokcycle +\Characterdirective{\addcytoks{\underdot{#1}}} +\def\:{\let\z= } \: % +\catcode`Q=\active % +\def\:{\let Q= } \: % +\catcode`\_=10 % + +\tokencyclexpress +xQx x_x\z{}x% +\endtokencyclexpress + +\resettokcycle +\Spacedirective{\addcytoks{\textcolor{cyan}\bgroup}% + \addcytoks[x]{(\detokenizespacetok{#1})}% + \addcytoks{\egroup}} + +\medskip +\tokencyclexpress +xQx x_x\z{}x% \endtokencyclexpress \end{exampleA} -- cgit v1.2.3