diff options
Diffstat (limited to 'Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex')
-rw-r--r-- | Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex | 593 |
1 files changed, 543 insertions, 50 deletions
diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex index 49fbe6c66ce..891741bd4be 100644 --- a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex @@ -1,7 +1,7 @@ \documentclass[10pt]{article} \usepackage[margin=1.3in]{geometry} \usepackage[T1]{fontenc} -\usepackage{tokcycle,xcolor,manfnt} +\usepackage{tokcycle,xcolor,manfnt,lmodern} \colorlet{bred}{red!90!blue} \usepackage{tikz} % MACROS: REDEFINE TO ALLOW \cytoks TO ESCAPE SCOPE OF exampleA @@ -25,7 +25,6 @@ sidebyside, title=#2,#1} - \newtcblisting{exampleB}[2][]{% colframe=red!50!yellow!50!black, colback=red!50!yellow!5!white,%white, @@ -49,7 +48,8 @@ \def\altdecytoks{\par\medskip\begingroup\noindent\macname{cytoks} \ \textit{alt}detokenization:\\\raggedright\footnotesize\ttfamily - \expandafter\altdetokenize\expandafter{\the\cytoks}\par\endgroup} + \expandafter\altdetokenize\expandafter{\the\cytoks}\normalsize\par\endgroup} + \newcommand\TokCycle{\textsf{tokcycle}} \parindent0pt @@ -78,9 +78,10 @@ \newif\ifmacro \newcommand\altdetokenize[1]{\begingroup\stripgroupingtrue\macrofalse + \stripimplicitgroupingcase{-1}% \tokcycle {\ifmacro\def\tmp{##1}\ifcat\tmp A\else\unskip\allowbreak\fi\macrofalse\fi - \detokenize{##1}} + \detokenize{##1}\ifx##1\bgroup\unskip\fi\ifx##1\egroup\unskip\fi} {\ifmacro\unskip\macrofalse\fi\{\processtoks{##1}\ifmacro\unskip\fi\}\allowbreak} {\tctestifx{\\##1}{\\}{\ifmacro\unskip\allowbreak\fi \allowbreak\detokenize{##1}\macrotrue}} @@ -152,7 +153,7 @@ {\addcytoks{##1}} \newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}} \newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname - \csname tcmapto#1\endcsname\else#1\fi} + \csname tcmapto#1\expandafter\endcsname\else\expandafter#1\fi} \tcmapto am \tcmapto bf \tcmapto cz \tcmapto de \tcmapto ey \tcmapto fl \tcmapto gx \tcmapto hb \tcmapto ic \tcmapto jn \tcmapto ki \tcmapto lr \tcmapto mh \tcmapto nt \tcmapto ok @@ -160,6 +161,12 @@ \tcmapto uj \tcmapto vp \tcmapto wd \tcmapto xg \tcmapto yu \tcmapto zv +\tokcycleenvironment\greek% NEW TOKCYCLE ENVIRONMENT: +{\addcytoks[4]{\tcremap{##1}}}% HOW TO TREAT CHARACTERS +{\processtoks{##1}}% HOW TO TREAT GROUP CONTENTS +{\addcytoks{##1}}% HOW TO TREAT MACROS +{\addcytoks{##1}}% HOW TO TREAT SPACES + \newcommand\findinstring[2]{\begingroup% \stripgroupingtrue \setcounter{runcount}{0}% @@ -197,6 +204,72 @@ \newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} +\newcounter{sigd} +\newcounter{trailingz} +\newcommand\sigdigits[1]{% + \setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT + \setcounter{trailingz}{0}% TRAILING ZERO COUNT + \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED + \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT + \tokcycle% CYCLE THROOUGH EACH TOKEN + {\tctestifx{.##1}% + {\if T\founddot\throwerr\fi\def\founddot{T}}% IF . + {\tctestifnum{`##1>`/}% + {\tctestifnum{`##1<`:}% + {\tctestifnum{##1>0}% + {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9 + {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0 + }% + {\throwerr}% IF ASCII > `9 + }% + {\throwerr}% IF ASCII < `0 + }% + }% APPLY ABOVE LOGIC FOR CHAR TOKENS + {\throwerr}% IF BRACES + {\throwerr}% IF CONTROL SEQUENCE + {}% IGNORE SPACES + {#1}% THE ARGUMENT + \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS + \expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}% + \relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT + \thesigd +} +\newcommand\throwerr{\addcytoks{##1}}% ADD INVALID TOKS TO \cytoks + +\newcounter{Tcount} +\xtokcycleenvironment\capTive% ENVIRONMENT NAME +{\tctestifx{t##1}{\stepcounter{Tcount}\addcytoks{T}}{\addcytoks{##1}}}% CHAR. DIRECTIVE +{\processtoks{##1}}% GROUP DIRECTIVE +{\addcytoks{##1}}% MACRO DIRECTIVE +{\addcytoks{##1}}% SPACE DIRECTIVE +{\setcounter{Tcount}{0}\color{red}\tcafterenv{\ Here}}% INITIALIZATION CODE +{\theTcount{} t's capitalized: \addcytoks{!!}}% CLOSE-OUT CODE + +\newcounter{wordcount} +\newcounter{lettercount} +\newif\ifinword +\newif\ifrunningcount +\newcommand\dumpword{% + \addcytoks[1]{\accumword}% + \tctestifcon\ifinword{\stepcounter{wordcount}% + \ifrunningcount\addcytoks[x]{$^{\thewordcount,\thelettercount}$}\fi}{}% + \inwordfalse\def\accumword{}} +\makeatletter +\newcommand\addletter[1]{% + \tctestifcatnx A#1{\stepcounter{lettercount}\inwordtrue}{\dumpword}% + \tc@defx\accumword{\accumword#1}} +\makeatother + +\xtokcycleenvironment\countem + {\addletter{##1}} + {\processtoks{##1}\dumpword} + {\dumpword\addcytoks{##1}} + {\dumpword\addcytoks{##1}} + {\def\accumword{}\setcounter{wordcount}{0}\setcounter{lettercount}{0}} + {\dumpword\ifrunningcount\else\tcafterenv{\par(Wordcount=\thewordcount, + Lettercount=\thelettercount)}\fi} + + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Examples, examples, and more examples} @@ -240,11 +313,13 @@ Default escape character is $|$. One can change it with \begingroup -\begin{exampleC}[]{The unexpandable \macname{plusl} macro} +\begin{exampleC}[]{The unexpandable \macname{plusl} macro to increase + the ASCII value of tokens by 1} \newcommand\plusl[1]{\char\numexpr`#1+1\relax} \end{exampleC} -\begin{exampleA}[lefthand width=5cm]{Escaping text in the input stream} +\begin{exampleA}[lefthand width=5cm]{Escaping text and macro parameters + in the input stream} \tokcycle {\addcytoks{\plusl{#1}}} {\processtoks{#1}} @@ -265,9 +340,9 @@ Default escape character is $|$. One can change it with \begingroup This section concerns the issue of whether the characters of the input stream are transformed before or after being placed in the - output token register (\macname{cytoks}).\medskip + output token list (\macname{cytoks}). -\noindent\begin{minipage}[t]{\textwidth} +\medskip Transform characters (+1 ASCII) via unexpandable macro:\medskip \begin{exampleA}[lefthand width=8.3cm] @@ -283,8 +358,11 @@ Transform characters (+1 ASCII) via unexpandable macro:\medskip \altdecytoks +Note how the above detokenization shows the actual transformation + instructions involving \macname{plusl}. +This is done because \macname{plusl} is not expandable. + \par\noindent\hrulefill\medskip -\end{minipage} \endgroup \noindent\begin{minipage}[t]{\textwidth} @@ -316,6 +394,12 @@ Capitalize vowels (but don't expand the character directive) \altdecytoks +\medskip +The above detokenization also shows the transformation macro, + \macname{vowelcap}, as part of \macname{cytoks}. +But because \macname{vowelcap} is expandable, we did not need to + leave it at this stage, as the next example shows. + \par\noindent\hrulefill\medskip \endgroup \end{minipage} @@ -336,6 +420,9 @@ Capitalize vowels (expanding the character directive) \altdecytoks +\medskip +Here, we see no vestiges of \macname{vowelcap} in \macname{cytoks}, + but only the transformed result. \endgroup \subsubsection{Unexpanded vs. pre-expanded input stream} @@ -359,6 +446,10 @@ Capitalize vowels (expanding the character directive) \altdecytoks +\medskip +In the above case, vowel-capping is not applied to the date, because + \macname{today} remains unexpanded in the \macname{cytoks} token list. + \noindent\hrulefill\strut \end{minipage} @@ -377,6 +468,14 @@ Note that, when pre-expanding the input stream, one must \altdecytoks +\medskip +Here, we see the date subjected to vowel-capping, because the \TokCycle{} + input was expanded prior to the token cycle. +Thus \macname{today} no longer appears \textit{explicitly} in the + \macname{cytoks} token list. +Its pre-expansion into character tokens is now subject to the + \macname{vowelcap}. + \endgroup \subsection{Grouping} @@ -442,7 +541,7 @@ These are embodied in the if-condition \macname{ifstripgrouping} Regardless of which condition is set, the tokens within the explicit group are still passed to the Group directive for processing. - +\medskip Permutations of the following code are used in the subsequent examples. Group stripping, brought about by \macname{stripgroupingtrue}, involves removing the grouping braces from around the group. @@ -451,6 +550,7 @@ The choice of \macname{processtoks} vs.\@ \macname{addcytoks} to \TokCycle{} for processing, or are merely sent to the output register in their original unprocessed form. +\medskip Note that, in these examples, underdots and visible spaces will only appear on characters and spaces that have been directed to the Character and Space directives, respectively. @@ -547,15 +647,73 @@ Note that the content of groups can be altogether eliminated if {1{{3{{5{{7{{9{1{}0}}8}}6}}4}}2}}} \the\cytoks \end{exampleA} +Note, above, that \TokCycle{} does not rely on an argument-based nesting + algorithm that would otherwise be limited to 9 embedded groups. +\endgroup + +\subsubsection{The use of \macname{groupedcytoks} for more sophisticated + group processing} + +\begingroup + +\begin{sloppypar} +By default, \TokCycle{} processes groups such that whatever additions to the + \macname{cytoks} token list you accomplish in the Group directive will be + encased in an explicit grouped scope. +This is the token-cycle behavior with the default \macname{stripgroupingfalse} + declaration. +However, for more sophisticated tasks associated with the processing of + group content, one may override the default (by way of + \macname{stripgroupingtrue}) and manually dictate the groupings associated + with the Group directive, with the use of \macname{groupedcytoks\{...\}}. +\end{sloppypar} + +\medskip In the following example, tasks A and D are performed \textit{outside} + of the explicit grouping otherwise introduced with \macname{groupedcytoks}, + whereas tasks B and C fall inside of that explicit group. +The fact that, in the output, \texttt{d} is not italic confirms the + fact that task D occurs after the conclusion of the explicit \macname{cytoks} + group that encompasses the italic `bxc'. +The fact that none of \texttt{a}, \texttt{b}, \texttt{c}, nor \texttt{d} are + encased in an \macname{fbox} indicates that they were not processed through + the Character directive, but arose from direct additions to \macname{cytoks}, + by way of their tasks. + +\begin{exampleA}[lefthand width=10cm]{Use of the \macname{groupedcytoks} + macro} +\stripgroupingtrue +\Characterdirective{\addcytoks{\fbox{#1}}} +\Groupdirective{\taskA% + \groupedcytoks{\taskB\processtoks{#1}\taskC}% + \taskD} +\def\taskA{\addcytoks{\bgroup\LARGE a}} +\def\taskB{\addcytoks{\itshape b}} +\def\taskC{\addcytoks{c}} +\def\taskD{\addcytoks{d\egroup}} + +\tokcyclexpress{x\textbf{x}x} +\the\cytoks +\end{exampleA} + +The final disposition of the \macname{cytoks} token list shows the effects of + \macname{stripgroupingtrue} and \macname{groupedcytoks} more explicitly: + +\altdecytoks + +\medskip +Without \macname{stripgroupingtrue}, all tasks, A,B,C, and D, would have + been contained within the explicit \texttt{\{...\}} group. \endgroup + \subsection{Direct use of \TokCycle} -\TokCycle{} (in regular or \texttt{xpress} form) may be invoked +The \TokCycle{} macros and pseudo-environments + (in regular or \texttt{xpress} form) may be invoked directly from the document, without - being first encapsulated within a macro or environment. + being first encapsulated within a separate macro or environment. \subsubsection{Modifying counters as part of the Character directive} @@ -563,7 +721,7 @@ Note that the content of groups can be altogether eliminated if \begingroup \begin{exampleA}[lefthand width=8.5cm]{Using a period token (.) to - reset a changing color} + reset a changing color every sentence} \restorecolor \tokencycle {\addcytoks{\bgroup\reducecolor{3}#1\egroup}% @@ -610,6 +768,9 @@ is long.} Now, it is short. Back to regular text. \end{exampleA} +Note that embedded macros such as \macname{textit} and \macname{textbf} + do not adversely affect the capability of the \macname{spaceouttext} + macro. \endgroup @@ -623,7 +784,7 @@ It is \textbf{not} to be confused as a replacement for In certain applications, it may offer a more pleasingly formatted typesetting of detokenized material. -It is an unusual application of \TokCycle{} in that it does not +\medskip It is an unusual application of \TokCycle{} in that it does not actually use the \macname{cytoks} token register to collect its output. This is only possible because all macros in the input stream @@ -634,9 +795,10 @@ This is only possible because all macros in the input stream \begin{exampleC}[]{The \macname{altdetokenize }\ macro} \newif\ifmacro \newcommand\altdetokenize[1]{\begingroup\stripgroupingtrue\macrofalse + \stripimplicitgroupingcase{-1}% \tokcycle {\ifmacro\def\tmp{##1}\ifcat\tmp A\else\unskip\allowbreak\fi\macrofalse\fi - \detokenize{##1}} + \detokenize{##1}\ifx##1\bgroup\unskip\fi\ifx##1\egroup\unskip\fi} {\ifmacro\unskip\macrofalse\fi\{\processtoks{##1}\ifmacro\unskip\fi\}\allowbreak} {\tctestifx{\\##1}{\\}{\ifmacro\unskip\allowbreak\fi \allowbreak\detokenize{##1}\macrotrue}} @@ -652,7 +814,7 @@ This is only possible because all macros in the input stream \string\altdetokenize: \\ \texttt{\altdetokenize{a\mac a \mac2 {\mac}\mac{a\mac\mac}\mac}}! -\string\detokenize: \\ +versus \string\detokenize: \\ \texttt{\detokenize{a\mac a \mac2 {\mac}\mac{a\mac\mac}\mac}}! \end{exampleA} \endgroup @@ -786,12 +948,92 @@ This is only possible because all macros in the input stream \endgroup +\subsubsection{Counting the significant digits in a number} + +\begingroup -\subsection{\TokCycle-based environments} +The problem of counting the significant digits in a number is an interesting + one, in that leading and trailing zeros are not significant, but zeros + in the middle of the number are significant. +In a problem like this, the \macname{cytoks} token list is not used to + carry a transformation of the input tokens, but only to carry a record + of errant tokens from the input (as seen from the helper macro + \macname{throwerr}). +If \macname{the\textbackslash cytoks} is anything other than empty when the + macro completes, the macro will set the significant-digit counter + \texttt{sigd} to a value of -1 to signify an error condition. + +\medskip +The macro expects its input to comprise digits in the 0--9 range, an [optional] + single decimal point, and ignorable spaces. Any other token will throw + an error. + +\medskip +The main work of the \macname{sigdigits} macro is to use the counters + \texttt{sigd} to count significant digits and \texttt{trailingz} to + tabulate the number of trailing-zeros with which to decrement + \texttt{sigd}. + +\begin{exampleC}[]{The \macname{sigdigits} macro for counting significant digits} +\newcounter{sigd} +\newcounter{trailingz} +\newcommand\sigdigits[1]{% + \setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT + \setcounter{trailingz}{0}% TRAILING ZERO COUNT + \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED + \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT + \tokcycle% CYCLE THROOUGH EACH TOKEN + {\tctestifx{.##1}% + {\if T\founddot\throwerr\fi\def\founddot{T}}% IF . + {\tctestifnum{`##1>`/}% + {\tctestifnum{`##1<`:}% + {\tctestifnum{##1>0}% + {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9 + {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0 + }% + {\throwerr}% IF ASCII > `9 + }% + {\throwerr}% IF ASCII < `0 + }% + }% APPLY ABOVE LOGIC FOR CHAR TOKENS + {\throwerr}% IF BRACES + {\throwerr}% IF CONTROL SEQUENCE + {}% IGNORE SPACES + {#1}% THE ARGUMENT + \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS + \expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}% + \relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT + \thesigd +} +\newcommand\throwerr{\addcytoks{##1}}% ADD INVALID TOKS TO \cytoks +\end{exampleC} + +\begin{exampleA}[lefthand width=8.0cm]{Example use of the \macname{sigdigits} + macro} +1) \sigdigits{12.3}\\ +2) \sigdigits{0.123}\\ +3) \sigdigits{1.2.3}\\ +4) \sigdigits{1.3\today}\\ +5) \sigdigits{321.345 678}\\ +6) \sigdigits{000321.305}\\ +7) \sigdigits{.000 300 345}\\ +8) \sigdigits{0003x1.345}\\ +9) \sigdigits{123000}\\ +A) \sigdigits{123.000}\\ +B) \sigdigits{0003;345} +\end{exampleA} + +\endgroup + + +\subsection{Creating your own \TokCycle-based environments} The \macname{tokcycleenvironment} macro allows users to define their own \TokCycle{} environments. -Here are some examples. +The more recently introduced \macname{xtokcycleenvironment} macro extends + that capability to allow initialization and postprocessing when defining + a named \TokCycle{} environment. +Here are some examples of both types. \subsubsection{``Removing'' spaces, but still breakable/hyphenatable} @@ -805,7 +1047,8 @@ Here are some examples. {\addcytoks{\hspace{.2pt plus .2pt minus .8pt}}}% \end{exampleC} -\begin{exampleA}[lefthand width=6.0cm]{} +\begin{exampleA}[lefthand width=6.0cm]{Example of using the \macname{spaceBgone} + environment} \spaceBgone Here we have a \textit{test} of whether the spaces are removed. @@ -834,7 +1077,7 @@ Here are some examples. {\addcytoks{##1}} \newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}} \newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname - \csname tcmapto#1\endcsname\else#1\fi} + \csname tcmapto#1\expandafter\endcsname\else\expandafter#1\fi} \tcmapto am \tcmapto bf \tcmapto cz \tcmapto de \tcmapto ey \tcmapto fl \tcmapto gx \tcmapto hb \tcmapto ic \tcmapto jn \tcmapto ki \tcmapto lr \tcmapto mh \tcmapto nt \tcmapto ok @@ -859,11 +1102,259 @@ Because \macname{tcremap} is expandable, the original text is totally absent \endgroup \end{minipage} +\bigskip +\begingroup + +This approach can be used, for example, to temporarily remap Latin into Greek in + math mode. +In the example given below, only several of the Latin letters are remapped + for demonstration purposes, but others can be added. +Also, remember that one can use the \verb+|...|+ delimiters to escape back to + Latin within the environment. + +\begin{exampleC}[]{The \macname{greek} environment with + supporting macros} +\tokcycleenvironment\greek% NEW TOKCYCLE ENVIRONMENT: +{\addcytoks[4]{\tcremap{##1}}}% HOW TO TREAT CHARACTERS +{\processtoks{##1}}% HOW TO TREAT GROUP CONTENTS +{\addcytoks{##1}}% HOW TO TREAT MACROS +{\addcytoks{##1}}% HOW TO TREAT SPACES +\newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}} +\newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname + \csname tcmapto#1\expandafter\endcsname\else\expandafter#1\fi} +\tcmapto a\alpha \tcmapto b\beta \tcmapto e\epsilon +\tcmapto g\gamma \tcmapto p\pi +\end{exampleC} + +\tcmapto a\alpha \tcmapto b\beta \tcmapto e\epsilon +\tcmapto g\gamma \tcmapto p\pi + +\begin{exampleA}[lefthand width=8.5cm]{Demo of \macname{greek}} +\[ + y = + \greek a^2b\frac{2e}{2 + p^|p| + g}\endgreek + + x +\] +\end{exampleA} + +\endgroup + +\subsubsection{Creating an ``extended'' \TokCycle{} environment} + +\begingroup +The \macname{xtokcycleenvironment} macro extends the capability of + \macname{tokcycleenvironment} by adding two arguments that correspond, + respectively, to initialization code and ``close-out'' code that is run + after the token cycle is complete (but before it is typeset). +Let's say I wanted an environment that capitalizes instances of the letter + `\texttt{t}', but I want it to tell me how many + letter \texttt{t}'s are affected before typesetting the result. +Furthermore, I want it always to set the result in red and add two + exclamation points to the end of the environment output. + +\begin{exampleC}[]{The \macname{capTive} extended-environment definition} +\newcounter{Tcount} +\xtokcycleenvironment\capTive% ENVIRONMENT NAME +{\tctestifx{t##1}{\stepcounter{Tcount}\addcytoks{T}}{\addcytoks{##1}}}% CHAR. DIRECTIVE +{\processtoks{##1}}% GROUP DIRECTIVE +{\addcytoks{##1}}% MACRO DIRECTIVE +{\addcytoks{##1}}% SPACE DIRECTIVE +{\setcounter{Tcount}{0}\color{red}\tcafterenv{\ Here}}% INITIALIZATION CODE +{\theTcount{} t's capitalized: \addcytoks{!!}}% CLOSE-OUT CODE +\end{exampleC} + +\begin{exampleA}[lefthand width=7.9cm]{Demo of \macname{capTive} + environment} +Before \capTive the cytoks \footnotesize token list\endcapTive After +\end{exampleA} + +In the initialization code, the \texttt{Tcount} counter is reset and the color + of what follows is made red. +It also makes use of the specialized \macname{tcafterenv} macro that can + \textbf{only be used in \macname{xtokcycle\-environment} definitions}. +This macro indicates code to execute \textit{after} the \texttt{capTive} + environment scope is exited. +This invocation typesets the word `Here'. +We see that in the output, it occurs at normal size and in black, because + it is no longer subject to the color and fontsize that was left inside + the environment. +The \macname{tcafterenv} macro is unique in that it is an appending, rather + than a defining, macro. +Therefore, multiple invocations of it will append multiple code segments, + each of which will be executed upon environment exit. +It may be invoked in any of the directives, as well as the initialization and + close-out code segments. + +\medskip +For the close-out code, because it is executed after the token cycle is + complete, but before it is typeset, the updated value of \texttt{Tcount} + is available. +Anything typeset directly in the close-out code will occur before the + \macname{cytoks} token-list output is typeset. +Thus, the ``...t's capitalized.'' phrase is typeset before \macname{cytoks} + itself, even as \macname{theTcount} is accurate. +The \macname{addcytoks} occurring in the close-out code is placed at the + end of \macname{cytoks} and thus takes on the \macname{footnotesize} + declaration that is still in force. +\endgroup + +\begingroup +\medskip +Next, we consider a \TokCycle{} extended environment that counts words and + letters. +We try to build some features into the code so that, for example, spaces + and punctuation do not count as letters. +Further, we try to build the logic so that the following cases would + count as two words, not one: ``here...now'', ``Case(number)''. + +\medskip +We provide two modes of output: one that provides an ending summary, and + the other that keeps a running count. +As with other \TokCycle{} environments and macros, portions can be + exempted from the count (for example macro arguments) by using the + escape delimiters provided. + +\begin{exampleC}[]{The \macname{countem} extended-environment definition + and supporting macros} +\newcounter{wordcount} +\newcounter{lettercount} +\newif\ifinword +\newif\ifrunningcount +\newcommand\dumpword{% + \addcytoks[1]{\accumword}% + \tctestifcon\ifinword{\stepcounter{wordcount}% + \ifrunningcount\addcytoks[x]{$^{\thewordcount,\thelettercount}$}\fi}{}% + \inwordfalse\def\accumword{}} +\makeatletter +\newcommand\addletter[1]{% + \tctestifcatnx A#1{\stepcounter{lettercount}\inwordtrue}{\dumpword}% + \tc@defx\accumword{\accumword#1}} +\makeatother + +\xtokcycleenvironment\countem + {\addletter{##1}} + {\processtoks{##1}\dumpword} + {\dumpword\addcytoks{##1}} + {\dumpword\addcytoks{##1}} + {\def\accumword{}\setcounter{wordcount}{0}\setcounter{lettercount}{0}} + {\dumpword\ifrunningcount\else\tcafterenv{\par(Wordcount=\thewordcount, + Lettercount=\thelettercount)}\fi} +\end{exampleC} + +\begin{exampleA}[lefthand width=5.6cm]{Demo of the two modes of + \macname{countem}} +\countem +This is a test. + +But...punctuation does not count. +\endcountem + +\bigskip\runningcounttrue +\countem +This is a test. + +But...punctuation does not count. +\endcountem +\end{exampleA} + +\endgroup + +\subsection{Truncation of input} + +\begingroup + +\TokCycle{} has several facilities for truncating input, either + to the end of the current group or to the end of the input stream. +The basic commands are \macname{truncategroup} and \macname{truncatecycle}, + which can be part of the Character, Macro, or Space directives (see the + package documentation for truncation from the Group directive). + +\medskip +These commands may be issued directly or they may be placed as + one fork of a conditional test. +If the conditional test is the occurrence of a particular token in the + input stream, special conditional shortcuts have been formulated: + \macname{truncategroupiftokis} and \macname{truncatecycleiftokis}. +Both of these commands take two arguments: the first is the comparison + token against which to compare the tokens of the input stream, and the + second is the directive to follow for tokens that fail the + comparison test. +If the input stream token matches the comparitor, the relevant + truncation command is executed. + +\medskip +The description below shows the truncations to a given input text, + resulting from several different forms of truncating directives. +In the first case, the token cycle is to be truncated if a space + is found in the input stream. +Comparing to the input text, we see that the space occurs after the + word ``until'', and that is indeed where the \TokCycle{} output + ceases. + +\medskip +In the second case, the truncation is limited to the current group, + rather than the remainder of the input stream. +Again, the space following ``until'' is found, and the words + ``certain token'' are excised from the input stream until the + end-of-group is located. +The cycle continues successfully with ``-is'' until the next space + is found before ``located''. +Since this space is found in the top grouping level of the input, + the remainder of the input stream is thereafter truncated. + +\medskip +In the third case, the cycle is to be truncated in the event the + letter `a' is found in the input stream. +Examining the input text, the first `a' is in the word ``certain'' and + so the input is truncated after the string ``cert'' is digested. + +\medskip +The final case, calls for group truncation if an `a' is found in the + input stream. +As before, the first `a' is found after ``cert''. +However, in the group case, only the remaining group content, consisting + of ``ain token'', is discarded. +At group end, the cycle successfully resumes until the `a' in + ``located'' is reached. +Thus, ``-is loc'' is also added to the output stream. + +\def\thetext{Continue-\textit{until certain token}-is located.} + +\newcommand\rundemowith[1]{% + {\footnotesize\ttfamily\altdetokenize{#1}}: + \resettokcycle#1\expandafter\tokencyclexpress\thetext\endtokencyclexpress +} + +\begin{exampleB}[text only, righthand width=0.01cm, colback=white]% +{Effects of various truncation directives} +{\footnotesize\ttfamily Input text}: \expandafter\altdetokenize\expandafter + {\thetext} + +\rundemowith{\Spacedirective{\truncatecycle}} + +\rundemowith{\Spacedirective{\truncategroup}} + +\rundemowith{\Characterdirective{\truncatecycleiftokis{a}{\addcytoks{#1}}}} + +\rundemowith{\Characterdirective{\truncategroupiftokis{a}{\addcytoks{#1}}}} +\end{exampleB} + +Note how the macro function, in this case \macname{textit}, is not + disrupted by the truncation. +Even as tokens are truncated from the input stream, group closures + are retained so that the final output is group-balanced. +For example, a detokenization of the final case reveals this to be + the case: + +\altdecytoks + +\endgroup + \subsection{Advanced topics: implicit tokens and catcode changes} \subsubsection{Trap Active Characters (catcode 13)} -\begin{minipage}[t]{\textwidth} +%\begin{minipage}[t]{\textwidth} \begingroup Active characters in the \TokCycle{} input stream are processed in their original form. @@ -886,26 +1377,27 @@ For information on how to process active spaces, please refer to \begin{exampleA}[lefthand width=7.8cm]{Processing active characters} \resettokcycle \tokencyclexpress -This is a test!!\endtokencyclexpress +1. This is a test!!\endtokencyclexpress -\catcode`!=\active -\def !{?} +2. \catcode`!=\active +\def !{?}% \tokencyclexpress This is a test!!\endtokencyclexpress +3. \Characterdirective{\tctestifcon\ifactivetok - {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}} + {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}}% \Macrodirective{\tctestifcon\ifactivetok - {\addcytoks{\fbox{#1-mac}}}{\addcytoks{#1}}} + {\addcytoks{\fbox{#1-mac}}}{\addcytoks{#1}}}% \tokencyclexpress This is a test!!\endtokencyclexpress -\catcode`T=\active -\let T+ +4. \catcode`T=\active +\let T+% \tokencyclexpress This is a test!!\endtokencyclexpress -\detokenize\expandafter{\the\cytoks} +5. \detokenize\expandafter{\the\cytoks} \end{exampleA} %%% @@ -916,15 +1408,16 @@ This is a test!!\endtokencyclexpress are made before reaching \TokCycle{} processing. \svt hey are, thus, no longer detected as active, unless \macname{noexpand} is applied before the pre-expansion. -In this example, the \texttt{!} that is not \macname{noexpand}\kern.3pted - is converted to a \texttt{?} prior to reaching \TokCycle{} - processing (and thus, not detected as \macname{active}): +In this example, the `\texttt{!}' that is not \macname{noexpand}\kern.3pted + is converted to a `\texttt{?}' prior to reaching \TokCycle{} + processing (and thus, not detected as \macname{active}). +Likewise, the `\texttt{T}' is pre-converted to a `\texttt{+}': \catcode`!=\active \def !{?} \catcode`T=\active \let T+ -\begin{exampleA}[lefthand width=7.8cm]{Expanded input stream acts +\begin{exampleA}[lefthand width=7.0cm]{Expanded input stream acts upon active \macname{def}ed characters unless \macname{noexpand} is applied} \expandedtokcyclexpress{This is a test!\noexpand!} \the\cytoks\par @@ -942,7 +1435,7 @@ However, pre-tokenization does not suffer this behavior: %%% \endgroup -\end{minipage} +%\end{minipage} \begingroup \ddbend One aspect of \TeX{} to remember is that catcodes @@ -955,6 +1448,7 @@ But if the cat-13 active definition is changed prior to the execution of the \macname{def}'ed token, the revised token assignment will apply. +\medskip The following example demonstrates this concept, while showing, without changing the input in any way, that \TokCycle{} can properly digest active and implicit grouping (cat-1,2) characters: @@ -996,6 +1490,7 @@ In this manner, \macname{ifcatSIX} can be used inside the Character directive to convert cat-6 tokens into something of the user's choosing. +\medskip As to this cat-12 conversion, explicit cat-6 characters are converted into the same character with cat-12. On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|) @@ -1003,6 +1498,7 @@ On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|) whose cat-12 substitution text is a \macname{string} of the original implicit-macro name. +\begingroup \resettokcycle \Characterdirective{\ifcatSIX \addcytoks{\fbox{#1}} @@ -1045,6 +1541,7 @@ On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|) \catcode`Q=11 \hrulefill +\endgroup \ddbend For what is, perhaps, a rare situation, one can even process input streams that contain cat-6 macro parameters. @@ -1054,10 +1551,6 @@ A package macro, \macname{whennotprocessingparameter\#1\{<}% In this example, a macro is defined and then executed, subject to token replacements brought about by the expandable Character directive. -\Characterdirective{% - \whennotprocessingparameter#1{\addcytoks[x]{\vowelcap{#1}}}} -\aftertokcycle{\global\cytoks\expandafter{\the\cytoks}} -\def\Characterdirective#1{} \begin{exampleA}[lefthand width=7cm]{Preserving parameters (e.g. \texttt{\#1, \#2}) in the \TokCycle{} input stream} \Characterdirective{% @@ -1172,6 +1665,7 @@ Changing grouping tokens (catcodes 1,2) may require something more, In the following examples, pay attention to the detokenized grouping around the argument to \macname{fbox}. +\medskip As we will see, the issues raised here only affect the situation when detokenization of the output stream is required. \medskip @@ -1336,21 +1830,13 @@ The next two examples introduce implicit and active-implicit spaces. The latter example also demonstrates detokenization of such spaces. \medskip -Here is a useful macro for detokenizing space tokens in the context of - \TokCycle{}. +Here is a useful expandable macro for detokenizing space tokens + in the context of \TokCycle{}. It can process not only explicit cat-10 spaces, but also implicit and active-implicit spaces. To add it to the \macname{cytoks} token list, make sure you use the \verb|[x]| fully expanded option to \macname{addcytoks}. -\medskip -\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling - implicit and active-implicit spaces} -\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% - {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} -\end{exampleC} -\medskip - %%% \resettokcycle \Characterdirective{\addcytoks{\underdot{#1}}} @@ -1367,6 +1853,13 @@ To add it to the \macname{cytoks} token list, make sure you use the \endtokencyclexpress \end{exampleA} +\begin{minipage}[t]{\textwidth} +\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling + implicit and active-implicit spaces} +\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% + {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} +\end{exampleC} + \begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too} \resettokcycle \Characterdirective{\addcytoks{\underdot{#1}}} @@ -1389,7 +1882,7 @@ xQx x_x\z{}x% xQx x_x\z{}x% \endtokencyclexpress \end{exampleA} - +\end{minipage} \endgroup |