From 37f9a57b2f39142fbccb20b758af8e0502499671 Mon Sep 17 00:00:00 2001 From: Norbert Preining Date: Fri, 2 Oct 2020 03:03:29 +0000 Subject: CTAN sync 202010020303 --- macros/generic/tokcycle/README | 12 ++--- macros/generic/tokcycle/tokcycle-doc.pdf | Bin 255073 -> 258406 bytes macros/generic/tokcycle/tokcycle-doc.tex | 38 ++++++++++++-- macros/generic/tokcycle/tokcycle-examples.pdf | Bin 258377 -> 261160 bytes macros/generic/tokcycle/tokcycle-examples.tex | 73 +++++++++++++++++++++++--- macros/generic/tokcycle/tokcycle.tex | 31 +++++++---- 6 files changed, 129 insertions(+), 25 deletions(-) (limited to 'macros/generic') diff --git a/macros/generic/tokcycle/README b/macros/generic/tokcycle/README index 491a000697..ee682cbc05 100644 --- a/macros/generic/tokcycle/README +++ b/macros/generic/tokcycle/README @@ -3,12 +3,12 @@ The tokcycle package helps one to build tools to process tokens from an input st Author: Steven B Segletes, Christian Tellechea (contributor) -Files : 1) tokcycle.tex - 2) tokcycle.sty +Files : 1) tokcycle.tex (tex/generic/tokcycle) + 2) tokcycle.sty (tex/latex/tokcycle) 3) tokcycle-doc.tex - 4) tokcycle-doc.pdf - 5) tokcycle-examples.tex - 6) tokcycle-examples.pdf - 7) README + 4) tokcycle-doc.pdf (doc/latex/tokcycle) + 5) tokcycle-examples.tex (doc/latex/tokcycle) + 6) tokcycle-examples.pdf (doc/latex/tokcycle) + 7) README (doc/latex/tokcycle) Released under the LaTeX Project Public License v1.3c or later, see http://www.latex-project.org/lppl.txt \ No newline at end of file diff --git a/macros/generic/tokcycle/tokcycle-doc.pdf b/macros/generic/tokcycle/tokcycle-doc.pdf index 5a86f5c5a2..c707af4f15 100644 Binary files a/macros/generic/tokcycle/tokcycle-doc.pdf and b/macros/generic/tokcycle/tokcycle-doc.pdf differ diff --git a/macros/generic/tokcycle/tokcycle-doc.tex b/macros/generic/tokcycle/tokcycle-doc.tex index 8614827a24..612ab5baf2 100644 --- a/macros/generic/tokcycle/tokcycle-doc.tex +++ b/macros/generic/tokcycle/tokcycle-doc.tex @@ -80,6 +80,7 @@ Each of these The package provides a number of options for handling groups. \end{quote} +\enlargethispage{1.4\baselineskip} \tableofcontents @@ -299,8 +300,8 @@ An integer option, call it \textit{n}, takes the The \texttt{[x]} option employs the \macname{expanded} primitive to maximally expand the argument before appending it to \macname{cytoks}. -The \texttt{[x]} option will prove useful when the \Character{} directive involves - a transformation that is fully expandable. +The \texttt{[x]} option will prove useful when the \Character{} or other + directives involve a transformation that is fully expandable. Its use will allow the expanded result of the transformation to be placed in the token register, rather than the unexpanded transformation instructions. @@ -464,6 +465,35 @@ i)~implicit grouping tokens (e.g., \macname{bgroup} and \macname{egroup}) ii)~implicit space tokens (e.g., \macname{@sptoken}) will be processed by the \Space{} directive. +\subsubsection{Active-Implicit Tokens, Including Spaces} + +One may occasionally run across a token that is both active and implicit. +For example, in the following code, \texttt{Q} is made both active and implicit: + +\begin{verbbox}[\vbdelim] +\catcode`Q=\active +\let Qx +\end{verbbox} +\medskip\theverbbox\medskip\noindent + +\noindent +In general, both \macname{ifactivetok} and \macname{ifimplicittok} + tests can be performed together to determine such cases. + +This is true even in the case of active-implicit catcode-10 spaces, which + are always processed through the Space directive. +However, because of the process \TokCycle{} uses in digesting space + tokens, the actual token passed to the Space directive (so-called + \verb|#1|), when an active-implicit space is encountered in the input + stream, is a generic implicit space token named \macname{tc@sptoken}. +However, the catcode-12 version of the active character that + produced it will be, for that moment, retained in a definition + named \macname{theactivespace}. +This can be useful if detokenization is required of the spaces. +Such an example is described in the \texttt{tokcycle-examples} + adjunct document. + + \subsubsection{Parameter (cat-6) tokens (e.g., \texttt{\#}): \macname{ifcatSIX}} Typically, category-code 6 tokens (like \texttt{\#}) are used to @@ -589,7 +619,7 @@ What follows are the known limitations of the package. Surely, there are more cases associated with arcane catcode-changing syntax that are not accounted for; I encourage you to bring them to my attention. -If I can't fix them, I can at least disclaim and declaim them. +If I can't fix them, I can at least disclaim and declaim them below: \begin{itemize} @@ -633,7 +663,7 @@ The \texttt{tex.stackexchange} site provides a wonderful \section*{Source Code} \Large\texttt{tokcycle.sty} -\verbfilenobox[\footnotesize]{tokcycle.sty}\vspace{2in} +\verbfilenobox[\footnotesize]{tokcycle.sty}\vspace{0.5in} \noindent\texttt{tokcycle.tex} \verbfilenobox[\footnotesize]{tokcycle.tex} diff --git a/macros/generic/tokcycle/tokcycle-examples.pdf b/macros/generic/tokcycle/tokcycle-examples.pdf index 8b92d14976..0c1f565132 100644 Binary files a/macros/generic/tokcycle/tokcycle-examples.pdf and b/macros/generic/tokcycle/tokcycle-examples.pdf differ diff --git a/macros/generic/tokcycle/tokcycle-examples.tex b/macros/generic/tokcycle/tokcycle-examples.tex index 0c46129bbd..49fbe6c66c 100644 --- a/macros/generic/tokcycle/tokcycle-examples.tex +++ b/macros/generic/tokcycle/tokcycle-examples.tex @@ -194,6 +194,8 @@ \newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}} \newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}} +\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% + {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Examples, examples, and more examples} @@ -873,6 +875,8 @@ If \macname{let} to a character, they may be identified in the Character directive; If \macname{let} to a control sequence or defined via \macname{def}, they may be identified in the Macro directive. +For information on how to process active spaces, please refer to + section~\ref{s:spc}. \let\svt T \Characterdirective{\tctestifcon\ifactivetok {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}} @@ -1257,20 +1261,35 @@ Bottom line: \endgroup -\subsubsection{Catcode 10 space tokens} +\subsubsection{Catcode 10 space tokens\label{s:spc}} \begingroup Here we demonstrate that \TokCycle{} can handle arbitrary redesignation - of tokens to cat-10, as well as implicit space tokens. + of tokens to cat-10, as well as implicit space tokens (both implicit + macro spaces and active-implicit character spaces). \ddbend While it should seem natural, we note that implicit space tokens are directed to the Space directive rather than the Character directive. However, \macname{ifimplicittok} may still be used to differentiate an explicit space from an implicit one. -\medskip + +\ddbend +Also, if the implicit space is an active character, \macname{ifactivetok} + is also set, for the user's benefit. +Likewise, \macname{ifactivechar} is also checked for active spaces, to see + if the charcode of the active space is still, indeed, active. +While \verb|#1| may be used to represent the space in the space directive, + a special technique is required to recover the detokenized name of the + active space character. +In particular, \verb|#1| will, for active spaces, contain a generic + implicit space token, \macname{tc@sptoken}. +However, the name of the associated active character in the input stream + will be defined as a cat-12 token in \macname{theactivespace}. + +\medskip Note in the following examples that cat-10 tokens do \textit{not} get under-dots. The next three examples all use the same input, but with different @@ -1313,19 +1332,61 @@ catcode settings for the space and the underscore. \fbox{a_c d} b_g\itshape f\upshape\endtokencycle \end{exampleA} +The next two examples introduce implicit and active-implicit spaces. +The latter example also demonstrates detokenization of such spaces. + +\medskip +Here is a useful macro for detokenizing space tokens in the context of + \TokCycle{}. +It can process not only explicit cat-10 spaces, but also implicit and + active-implicit spaces. +To add it to the \macname{cytoks} token list, make sure you use the + \verb|[x]| fully expanded option to \macname{addcytoks}. + +\medskip +\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling + implicit and active-implicit spaces} +\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% + {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} +\end{exampleC} +\medskip + %%% \resettokcycle \Characterdirective{\addcytoks{\underdot{#1}}} -\begin{exampleA}[lefthand width=10cm]{Implicit spaces work{,} too} +\begin{exampleA}[lefthand width=10cm]{Implicit spaces also work} \resettokcycle \Characterdirective{\addcytoks{\underdot{#1}}} -\def\:{\let\mysptoken= } \: % +\def\:{\let\z= } \: % \catcode`\_=10 % \catcode`\ =12 % \tokencyclexpress -\fbox{a\mysptoken{}c d} b_g\itshape f\upshape +\fbox{a\z{}c d} b_g\itshape f\upshape +\endtokencyclexpress +\end{exampleA} + +\begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too} +\resettokcycle +\Characterdirective{\addcytoks{\underdot{#1}}} +\def\:{\let\z= } \: % +\catcode`Q=\active % +\def\:{\let Q= } \: % +\catcode`\_=10 % + +\tokencyclexpress +xQx x_x\z{}x% +\endtokencyclexpress + +\resettokcycle +\Spacedirective{\addcytoks{\textcolor{cyan}\bgroup}% + \addcytoks[x]{(\detokenizespacetok{#1})}% + \addcytoks{\egroup}} + +\medskip +\tokencyclexpress +xQx x_x\z{}x% \endtokencyclexpress \end{exampleA} diff --git a/macros/generic/tokcycle/tokcycle.tex b/macros/generic/tokcycle/tokcycle.tex index 50b9dd4cc9..eeba320682 100644 --- a/macros/generic/tokcycle/tokcycle.tex +++ b/macros/generic/tokcycle/tokcycle.tex @@ -1,7 +1,7 @@ \def\tcname {tokcycle} -\def\tcver {1.12} +\def\tcver {1.2} % -\def\tcdate {2020/02/11} +\def\tcdate {2020/10/01} % % Author : Steven B Segletes, Christian Tellechea (contributor) % Maintainer : Steven B Segletes @@ -57,7 +57,8 @@ \def\addtc@depth{\advance\tc@depth 1} \def\subtc@depth{\tc@depth=\numexpr\tc@depth-1\relax} \def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse - \catSIXfalse\activecharfalse\activetokunexpandablefalse} + \catSIXfalse\activecharfalse\activetokunexpandablefalse + \def\theactivespace{}} \long\def\count@stringtoks#1{\tc@earg\count@toks{\string#1}} \long\def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt} \long\def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}} @@ -92,7 +93,7 @@ \def\endtokcycraw{\subtc@depth\tctestifnum{\tc@depth=0}{\@aftertokcycle}{}} % LOOP ENTRY POINT \def\@tokcycle{\tc@resetifs\futurelet\tc@next\detect@CANTabsorb} -\def\detect@CANTabsorb{\tctestifx{\tc@next\tc@sptoken}{\stringify\@@@@@spcT}% +\def\detect@CANTabsorb{\tctestifx{\tc@next\tc@sptoken}{\stringify\@@@@@@spcT}% {\tctestifx{\tc@next\bgroup}{\stringify\@@@@grpT}{\can@absorb}}} % NON cat1,10 TOKENS \long\def\can@absorb#1{\tc@tok{#1}\trapcatSIX{#1}\expandafter\can@absorb@ @@ -103,12 +104,16 @@ \long\def\can@absorb@@#1{\let\@tmp=#1\test@ifmacro\@tmp{\implicittokfalse \@macT#1}{\trapimplicitegrp#1\implicitgrpfork#1}\@tokcycle} %CONVERT NEXT (SPACE OR BEGIN-GROUP) TOKEN TO STRING -\def\stringify#1{\expandafter#1\string}% #1 WILL BE \@@@@@spcT or \@@@@grpT +\def\stringify#1{\expandafter#1\string}% #1 WILL BE \@@@@@@spcT or \@@@@grpT %SPACE DECODE -\def\@@@@@spcT{\futurelet\tc@str\@@@@spcT} -\def\@@@@spcT{\tctestifx{\tc@str\tc@sptoken}% +\def\@@@@@@spcT{\futurelet\tc@str\@@@@@spcT} +\def\@@@@@spcT{% + \tctestifx{\tc@str\tc@sptoken}% {\def\@tmp{\@@spcT{ }}\expandafter\@tmp\tc@absorbSpace}% EXPLICIT SPACE - {\implicittoktrue\expandafter\@@@spcT\tc@gobble}}% IMPLICIT SPACE + {\implicittoktrue\tctestifcon{\if\expandafter\@firstoftwo\string\\\tc@str}% + {\expandafter\@@@spcT\tc@gobble}% IMPLICIT MACRO SPACE + {\activetoktrue\@@@@spcT}}}% IMPLICIT ACTIVE SPACE +\def\@@@@spcT#1{\def\theactivespace{#1}\trapactivechar{#1}\@@spcT{\tc@sptoken}} \def\@@@spcT{\csmk{\expandafter\@@spcT\thecs}} \def\@@spcT#1{\@spcT{#1}\@tokcycle} % GROUP DECODE @@ -223,4 +228,12 @@ v1.12 2020/02/11 - Documentation correction: \tokcycleenvironment, not \tokencycleenvironment - Documentation correction: misspelling in tokcycle-examples.tex - Redefined \tc@defx and \tc@earg to omit #2 as part of definition - - Corrected \trapcatSIXb definition to account for revised \tc@earg definition. \ No newline at end of file + - Corrected \trapcatSIXb definition to account for revised \tc@earg definition. + +v1.2 2020/10/1 + -Added/fixed capability to handle active-implicit spaces. While the + #1 passed to the \Spacedirective, in such a case, is an implicit + space \tc@sptoken, the name of the active character from whence it + originated in the input stream is stored as an explicit cat-12 token + in the definition \theactivespace. The flags \implicittok and \activetok + are both set true, and the \activechar flag is checked, as well. \ No newline at end of file -- cgit v1.2.3