diff options
Diffstat (limited to 'macros/generic/tokcycle/tokcycle-examples.tex')
-rw-r--r-- | macros/generic/tokcycle/tokcycle-examples.tex | 688 |
1 files changed, 577 insertions, 111 deletions
diff --git a/macros/generic/tokcycle/tokcycle-examples.tex b/macros/generic/tokcycle/tokcycle-examples.tex index 891741bd4b..cdce49c48d 100644 --- a/macros/generic/tokcycle/tokcycle-examples.tex +++ b/macros/generic/tokcycle/tokcycle-examples.tex @@ -1,7 +1,11 @@ \documentclass[10pt]{article} \usepackage[margin=1.3in]{geometry} \usepackage[T1]{fontenc} -\usepackage{tokcycle,xcolor,manfnt,lmodern} +\usepackage{tokcycle,xcolor,manfnt,lmodern,mathtools} +%%%%TRIAL PACKAGE CHANGES (MAKE SURE EXAMPLE FILE COMPILES CORRECTLY) +\makeatletter +\makeatother +%%%%END TRIAL \colorlet{bred}{red!90!blue} \usepackage{tikz} % MACROS: REDEFINE TO ALLOW \cytoks TO ESCAPE SCOPE OF exampleA @@ -48,7 +52,8 @@ \def\altdecytoks{\par\medskip\begingroup\noindent\macname{cytoks} \ \textit{alt}detokenization:\\\raggedright\footnotesize\ttfamily - \expandafter\altdetokenize\expandafter{\the\cytoks}\normalsize\par\endgroup} + \expandafter\altdetokenize\expandafter{\the\cytoks}\normalsize\par + \endgroup\medskip} \newcommand\TokCycle{\textsf{tokcycle}} @@ -120,21 +125,21 @@ \newcommand\nextcap[1]{% \edef\tmp{#1}% - \tctestifx{-#1}{\def\capnext{T}}{}% + \tctestifx{-#1}{\def\capnext{T}}{}% <- TEST FOR HYPHEN \tctestifcon{\if T\capnext}% - {\tctestifcon{\ifcat\tmp A}% - {\uppercase{#1}\def\capnext{F}}% - {#1}}% - {#1}% + {\tctestifcon{\ifcat\tmp A}% <- TEST FOR NEXT CAT-11 + {\addcytoks{\uppercase{#1}}\def\capnext{F}}% + {\addcytoks{#1}}}% + {\addcytoks{#1}}% } \newcommand\TitleCase[1]{% - \def\capnext{T} + \def\capnext{T}% <- INITIAL ENTRY \tokcycle - {\addcytoks{\nextcap{##1}}} - {\processtoks{##1}} + {\nextcap{##1}} + {\def\capnext{T}\processtoks{##1}}% <- GROUP ENTRY {\addcytoks{##1}} - {\addcytoks{##1\def\capnext{T}}} + {\addcytoks{##1}\def\capnext{T}}% <-CAT-10 SPACE {#1}% \the\cytoks } @@ -201,24 +206,23 @@ \newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}} \newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}} -\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% - {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} - \newcounter{sigd} \newcounter{trailingz} +\newif\iffounddot +\newif\ifzerosig \newcommand\sigdigits[1]{% \setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT \setcounter{trailingz}{0}% TRAILING ZERO COUNT - \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED - \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT + \founddotfalse% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED + \zerosigfalse% MADE T WHEN ZERO BECOMES SIGNIFICANT \tokcycle% CYCLE THROOUGH EACH TOKEN {\tctestifx{.##1}% - {\if T\founddot\throwerr\fi\def\founddot{T}}% IF . + {\iffounddot\throwerr\fi\founddottrue}% IF . {\tctestifnum{`##1>`/}% {\tctestifnum{`##1<`:}% {\tctestifnum{##1>0}% - {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9 - {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0 + {\stepcounter{sigd}\zerosigtrue\setcounter{trailingz}{0}}% IF 1-9 + {\ifzerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0 }% {\throwerr}% IF ASCII > `9 }% @@ -229,7 +233,7 @@ {\throwerr}% IF CONTROL SEQUENCE {}% IGNORE SPACES {#1}% THE ARGUMENT - \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS + \iffounddot\else\addtocounter{sigd}{-\thetrailingz}\fi% TRAILING ZEROS \expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}% \relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT \thesigd @@ -269,6 +273,81 @@ {\dumpword\ifrunningcount\else\tcafterenv{\par(Wordcount=\thewordcount, Lettercount=\thelettercount)}\fi} +\newcounter{pointcount} +\newcommand\addtopoints[1]{(\textit{#1 point}% + \ifnum#1=1\relax\else\textit{s}\fi)} +\NewEnviron{pointtracker}[2][points]{% + \par\bigskip\resettokcycle + \setcounter{pointcount}{0}% + \Macrodirective{\addcytoks{##1}\tctestifx{\addtopoints##1}{\sumpoints}{}}% + \def\tmp{\tokencyclexpress{\large\bfseries#2: \thepointcount\ #1}}% + \expandafter\tmp\BODY\endtokencyclexpress +} +\def\sumpoints{\tcpop\Q\addtocounter{pointcount}{\Q}\tcpushgroup{\Q}} + +\newcommand\scalerule[1]{% + \tcpeek\zA + \ifx[\zA\tcpopuntil]\opt\fi% IF PRESENT, POP OPTIONAL ARG + \tcpop\ArgA% POP ARG1 + \tcpop\ArgB% POP ARG2 + \tcpushgroup[x]{#1\dimexpr\ArgB}% PUSH BACK IN REVERSE (LIFO) FASHION + \tcpushgroup[x]{#1\dimexpr\ArgA} + \ifx[\zA\tcpush[x]{\expandafter\scaleopt\opt{#1}}\fi +} +\def\scaleopt[#1]#2{[#2\dimexpr#1]} + +\newcommand\protectruleargs[1]{\ifx\rule#1% + \tcpeek\zA + \ifx[\zA\tcpopuntil]\opt + \addcytoks[1]{\opt}\fi% IF OPTIONAL ARG PRESENT, POP AND SAVE TO \cytoks + \tcpopliteral\ArgA% MUST POP ARG 1 WITH BRACES + \addcytoks[1]{\ArgA}% + \tcpopliteral\ArgB% MUST POP ARG 2 WITH BRACES + \addcytoks[1]{\ArgB}% + \fi +} + +\newif\iffoundstring +\newcommand\findstring[4]{% {CURRENT TOK}{MATCH STRING}{SUCCESS OP}{FAIL CMDS} + \foundstringfalse% DEFAULT CONDITION + \def\poptoks{}% INITIALIZATION. IF MATCH FAILS, MUST TO RESTORE POPPED TOKENS + \expandafter\matchnextchar\ifx#1#2\relax\relax\fi\empty\relax + \iffoundstring\addcytoks{#3{#2}}\else\tcpush{\poptoks}#4\fi +} +\def\matchnextchar#1#2\relax{\tctestifx{#1\empty}{}{% NO MATCH FROM 1ST TOKEN + \tctestifx{\relax#1}{% + \foundstringtrue% NOTHING LEFT TO COMPARE; MATCH FOUND + }{% + \tcpeek\Q% WITHOUT POPPING... + \tctestifx{\Q\bgroup}{}{% PEEK FOR GROUPS OR... + \tctestifx{\Q\tcsptoken}{}{% SPACES THAT INDICATE MATCH FAILURE + \tcpopappto\poptoks% POP TOKEN AND APPEND TO \poptoks + \expandafter\tctestifx\expandafter{\Q#1}{% + \matchnextchar#2\relax% STILL MATCHING; KEEP GOING + }{}% + }% + }% + }% +}} + +\newcommand\foo{$\bullet$} +\newcommand\foocheck[1]{\tcpeek\z\ifx\foo\z\else + \ifx\par\z\else\addcytoks{#1}\fi\fi} +\tokcycleenvironment\dofoo + {\addcytoks{##1}} + {\processtoks{##1}} + {\ifx\par##1\foocheck{##1}\else\addcytoks{##1}\fi} + {\foocheck{##1}} + +\newcommand\coleqchek[1]{\tcpeek\z + \ifx=\z\tcpop\z\addcytoks{\coloneqq}\else\addcytoks{#1}\fi} +\tokcycleenvironment\subcoloneqq +{\tctestifx{:##1}{\coleqchek{##1}}{\addcytoks{##1}}} +{\processtoks{##1}} +{\tctestifcon{\if\detokenize{:}\detokenize{##1}} + {\coleqchek{##1}}{\addcytoks{##1}}} +{\addcytoks{##1}} + %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% \section{Examples, examples, and more examples} @@ -292,7 +371,8 @@ Apply different directives to Characters (under-dot), Groups \begingroup -\begin{exampleA}[lefthand width=7.8cm]{} +\begin{exampleA}[lefthand width=7.8cm]{Apply different directives to + characters{,} groups{,} macros{,} and spaces} \tokcycle{\addcytoks{\underdot{#1}}} {\addcytoks{\{}\processtoks{#1} \addcytoks{\}}} @@ -307,9 +387,13 @@ Apply different directives to Characters (under-dot), Groups \subsubsection{Escaping text} Text between two successive escape characters is bypassed - by \TokCycle{} and instead echoed to the output register. -Default escape character is $|$. One can change it with + by \TokCycle{} and instead echoed to the output token list + (\macname{cytoks}). +The default escape character is $|$. +One can change the token-cycle escape character by way of the \macname{settcEscapechar} macro. +Below, the character codes are incremented by one; however, the + arguments of \macname{rule} are protected against that. \begingroup @@ -343,6 +427,7 @@ This section concerns the issue of whether the characters of the input output token list (\macname{cytoks}). \medskip +\begin{minipage}[t]{\textwidth} Transform characters (+1 ASCII) via unexpandable macro:\medskip \begin{exampleA}[lefthand width=8.3cm] @@ -361,6 +446,7 @@ Transform characters (+1 ASCII) via unexpandable macro:\medskip Note how the above detokenization shows the actual transformation instructions involving \macname{plusl}. This is done because \macname{plusl} is not expandable. +\end{minipage} \par\noindent\hrulefill\medskip \endgroup @@ -394,7 +480,6 @@ Capitalize vowels (but don't expand the character directive) \altdecytoks -\medskip The above detokenization also shows the transformation macro, \macname{vowelcap}, as part of \macname{cytoks}. But because \macname{vowelcap} is expandable, we did not need to @@ -404,9 +489,16 @@ But because \macname{vowelcap} is expandable, we did not need to \endgroup \end{minipage} +\begin{minipage}[t]{\textwidth} \begingroup Capitalize vowels (expanding the character directive) +\medskip +By using the \texttt{[x]} option of \macname{addcytoks}, the + \macname{vowelcap} is fully expanded prior to being placed in + the \macname{cytoks} token list. + + \begin{exampleA}[lefthand width=8.3cm] {Expanded Character directive} \tokcycle @@ -420,11 +512,12 @@ Capitalize vowels (expanding the character directive) \altdecytoks -\medskip Here, we see no vestiges of \macname{vowelcap} in \macname{cytoks}, but only the transformed result. \endgroup +\end{minipage} +\enlargethispage{12pt} \subsubsection{Unexpanded vs. pre-expanded input stream} \begingroup @@ -446,7 +539,6 @@ Here, we see no vestiges of \macname{vowelcap} in \macname{cytoks}, \altdecytoks -\medskip In the above case, vowel-capping is not applied to the date, because \macname{today} remains unexpanded in the \macname{cytoks} token list. @@ -468,7 +560,6 @@ Note that, when pre-expanding the input stream, one must \altdecytoks -\medskip Here, we see the date subjected to vowel-capping, because the \TokCycle{} input was expanded prior to the token cycle. Thus \macname{today} no longer appears \textit{explicitly} in the @@ -494,7 +585,7 @@ The desired options are to be set prior to the \TokCycle{} invocation. The macro \macname{stripimplicitgroupingcase} can take three possible integer arguments: 0 (default) to automatically place unaltered - implicit group tokens in the output register; 1 to strip + implicit group tokens in the \macname{cytoks} output; 1 to strip implicit group tokens from the output; or $-$1 to instead pass the implicit group tokens to the Character directive (as implicit tokens) for separate processing (typically, when detokenization @@ -547,8 +638,8 @@ Group stripping, brought about by \macname{stripgroupingtrue}, involves removing the grouping braces from around the group. The choice of \macname{processtoks} vs.\@ \macname{addcytoks} affects whether the tokens inside the group are recommitted - to \TokCycle{} for processing, or are merely sent to the output - register in their original unprocessed form. + to \TokCycle{} for processing, or are merely sent to the + \macname{cytoks} token list in their original unprocessed form. \medskip Note that, in these examples, underdots and visible spaces @@ -700,7 +791,6 @@ The final disposition of the \macname{cytoks} token list shows the effects of \altdecytoks -\medskip Without \macname{stripgroupingtrue}, all tasks, A,B,C, and D, would have been contained within the explicit \texttt{\{...\}} group. @@ -785,7 +875,7 @@ In certain applications, it may offer a more pleasingly formatted typesetting of detokenized material. \medskip It is an unusual application of \TokCycle{} in that it does not - actually use the \macname{cytoks} token register to collect + actually use the \macname{cytoks} token list to collect its output. This is only possible because all macros in the input stream are detokenized, rather than executed. @@ -821,29 +911,29 @@ versus \string\detokenize: \\ \subsubsection{Capitalize all words, including compound and - parenthetical words} + parenthetical words\label{s:cap}} \begingroup \begin{exampleC}[]{The \macname{Titlecase} and \texttt{\string\nextcap} macros} \newcommand\TitleCase[1]{% - \def\capnext{T} + \def\capnext{T}% <- INITIAL ENTRY \tokcycle - {\addcytoks{\nextcap{##1}}} - {\processtoks{##1}} + {\nextcap{##1}} + {\def\capnext{T}\processtoks{##1}}% <- GROUP ENTRY {\addcytoks{##1}} - {\addcytoks{##1\def\capnext{T}}} + {\addcytoks{##1}\def\capnext{T}}% <-CAT-10 SPACE {#1}% \the\cytoks } \newcommand\nextcap[1]{% \edef\tmp{#1}% - \tctestifx{-#1}{\def\capnext{T}}{}% + \tctestifx{-#1}{\def\capnext{T}}{}% <- TEST FOR HYPHEN \tctestifcon{\if T\capnext}% - {\tctestifcon{\ifcat\tmp A}% - {\uppercase{#1}\def\capnext{F}}% - {#1}}% - {#1}% + {\tctestifcon{\ifcat\tmp A}% <- TEST FOR NEXT CAT-11 + {\addcytoks{\uppercase{#1}}\def\capnext{F}}% + {\addcytoks{#1}}}% + {\addcytoks{#1}}% } \end{exampleC} @@ -859,6 +949,19 @@ versus \string\detokenize: \\ (\textit{capitalizing} words).} \end{exampleA} +As it is currently set up, a future-capitalization of the next catcode-11 token will be triggered when any of the following is encountered in the input stream: +\begin{itemize} +\item The initial entry into the routine + +\item a catcode-10 space token + +\item a hyphen \texttt{-} + +\item upon entry into a group \texttt{\{...\}} +\end{itemize} + +Obviously, additional tests could be added. + \endgroup \subsubsection{Scaling rule dimensions} @@ -890,7 +993,7 @@ versus \string\detokenize: \\ \endgroup -\subsubsection{String search, including non-regex material} +\subsubsection{String search, including non-regex material\label{s:searchA}} \begingroup @@ -953,8 +1056,8 @@ versus \string\detokenize: \\ \begingroup The problem of counting the significant digits in a number is an interesting - one, in that leading and trailing zeros are not significant, but zeros - in the middle of the number are significant. + one, in that leading zeros are not significant, trailing zeros are sometimes + significant, but zeros in the middle of the number are significant. In a problem like this, the \macname{cytoks} token list is not used to carry a transformation of the input tokens, but only to carry a record of errant tokens from the input (as seen from the helper macro @@ -977,30 +1080,32 @@ The main work of the \macname{sigdigits} macro is to use the counters \begin{exampleC}[]{The \macname{sigdigits} macro for counting significant digits} \newcounter{sigd} \newcounter{trailingz} +\newif\iffounddot +\newif\ifzerosig \newcommand\sigdigits[1]{% \setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT \setcounter{trailingz}{0}% TRAILING ZERO COUNT - \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED - \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT + \founddotfalse% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED + \zerosigfalse% MADE T WHEN ZERO BECOMES SIGNIFICANT \tokcycle% CYCLE THROOUGH EACH TOKEN - {\tctestifx{.##1}% - {\if T\founddot\throwerr\fi\def\founddot{T}}% IF . - {\tctestifnum{`##1>`/}% - {\tctestifnum{`##1<`:}% - {\tctestifnum{##1>0}% - {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9 - {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0 - }% - {\throwerr}% IF ASCII > `9 - }% - {\throwerr}% IF ASCII < `0 + {\tctestifx{.##1}% + {\iffounddot\throwerr\fi\founddottrue}% IF . + {\tctestifnum{`##1>`/}% + {\tctestifnum{`##1<`:}% + {\tctestifnum{##1>0}% + {\stepcounter{sigd}\zerosigtrue\setcounter{trailingz}{0}}% IF 1-9 + {\ifzerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0 }% - }% APPLY ABOVE LOGIC FOR CHAR TOKENS - {\throwerr}% IF BRACES - {\throwerr}% IF CONTROL SEQUENCE - {}% IGNORE SPACES - {#1}% THE ARGUMENT - \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS + {\throwerr}% IF ASCII > `9 + }% + {\throwerr}% IF ASCII < `0 + }% + }% APPLY ABOVE LOGIC FOR CHAR TOKENS + {\throwerr}% IF BRACES + {\throwerr}% IF CONTROL SEQUENCE + {}% IGNORE SPACES + {#1}% THE ARGUMENT + \iffounddot\else\addtocounter{sigd}{-\thetrailingz}\fi% TRAILING ZEROS \expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}% \relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT \thesigd @@ -1213,6 +1318,11 @@ We provide two modes of output: one that provides an ending summary, and As with other \TokCycle{} environments and macros, portions can be exempted from the count (for example macro arguments) by using the escape delimiters provided. +An even more elaborate version of this environment can be found at +\texttt{https://tex.stackexchange.com/questions/\allowbreak + 44618/dynamically-count-and-return-number-of-words-in-a-section/587567\#587567},% +\hfil in\-cluding a version that runs + in Plain \TeX. \begin{exampleC}[]{The \macname{countem} extended-environment definition and supporting macros} @@ -1326,7 +1436,7 @@ Thus, ``-is loc'' is also added to the output stream. } \begin{exampleB}[text only, righthand width=0.01cm, colback=white]% -{Effects of various truncation directives} +{Example: Effects of various truncation directives} {\footnotesize\ttfamily Input text}: \expandafter\altdetokenize\expandafter {\thetext} @@ -1350,9 +1460,338 @@ For example, a detokenization of the final case reveals this to be \endgroup -\subsection{Advanced topics: implicit tokens and catcode changes} +\subsection{Look-ahead features} + +The whole concept of the \TokCycle{} package is to examine the input stream + one token at a time and use the directives to process the tokens. +Occasionally, however, there are tasks that involve several adjacent tokens. +In the example of section~\ref{s:cap}, for instance, in order to capitalize + words, each space token triggers an action on the character that follows. +To accomplish this without any look-ahead features, the Space directive sets + a flag, \verb|\def\capnext{T}|, in order to communicate with a the Character + directive about a subsequent token. +That is a perfectly valid approach to communicate conditional action. + +\medskip +However, \TokCycle{} also provides look-ahead features, to avoid the necessity + of communicating with future tokens in the cycle by way of flag settings. +These tools allow a directive to directly ``peek'' ahead at the input stream + (without disturbing it) and, if desired, push and pop tokens to and from + the input stream. +The package manual should be consulted for a full list of macros provided to + facilitate look-ahead capability. +But here are some examples to look at. + +\begingroup + +\subsubsection{Count in advance that which follows} + +Specify the point values for questions and let tokcycle perform the cumulative + tallies in advance, using a single-pass compilation. + +\begin{exampleC}[]{The \texttt{pointtracker} environment to count point values + in advance} +\newcounter{pointcount} +\newcommand\addtopoints[1]{(\textit{#1 point}% + \ifnum#1=1\relax\else\textit{s}\fi)} +\NewEnviron{pointtracker}[2][points]{% + \par\bigskip\resettokcycle + \setcounter{pointcount}{0}% + \Macrodirective{\addcytoks{##1}\tctestifx{\addtopoints##1}{\sumpoints}{}}% + \def\tmp{\tokencyclexpress{\large\bfseries#2: \thepointcount\ #1}}% + \expandafter\tmp\BODY\endtokencyclexpress +} +\def\sumpoints{\tcpop\Q\addtocounter{pointcount}{\Q}\tcpushgroup{\Q}} +\end{exampleC} + +\begin{exampleA}[lefthand width=6.2cm]{Demo of the pointtracker environment} +\begin{pointtracker}[total points]{\Large Science Test} +\begin{pointtracker}{Advanced Geology} + \begin{itemize} + \item Count the fingers on your left hand. Are there more than 9? + \addtopoints{1} + \item Draw a rectangle with five edges. \addtopoints{3} + \end{itemize} +\end{pointtracker} +\begin{pointtracker}{Meteorology} + \begin{itemize} + \item How many borgs does it take to brew a Danish beer? + \addtopoints{2} + \item What is the meaning of life? \addtopoints{4} + \end{itemize} +\end{pointtracker} +\end{pointtracker} +\end{exampleA} + +Note that this example has several neat features. +The points for Geology, Meteorology, and the Test total are not specified, + but calculated. +They are calculated based on content that appears later in the environment. +Traditionally, multiple compilations would be required with external data + written to the aux file, but that does not occur here---there is only + a single compilation. +Finally, becaue we are nesting the \TokCycle{} part of \texttt{pointtracker} + inside of a \macname{NewEnviron}, we see that the \texttt{pointtracker} + environment can be nested, without adverse effect. + +\endgroup + +\subsubsection{Modify the arguments of every instance of a particular macro} -\subsubsection{Trap Active Characters (catcode 13)} +\begingroup + +This example, through the use of the \macname{scalerule} macro, shows how + the \macname{rule} arguments, including the optional argument, can be + popped off the input stream, adjusted, and placed back in the stream. + +\begin{exampleC}[]{The \macname{scalerule} macro to scale all + \macname{rule} arguments} +\newcommand\scalerule[1]{% + \tcpeek\zA + \ifx[\zA\tcpopuntil]\opt\fi% IF PRESENT, POP OPTIONAL ARG + \tcpop\ArgA% POP ARG1 + \tcpop\ArgB% POP ARG2 + \tcpushgroup[x]{#1\dimexpr\ArgB}% PUSH BACK IN REVERSE (LIFO) FASHION + \tcpushgroup[x]{#1\dimexpr\ArgA} + \ifx[\zA\tcpush[x]{\expandafter\scaleopt\opt{#1}}\fi +} +\def\scaleopt[#1]#2{[#2\dimexpr#1]} +\end{exampleC} + +\begin{exampleA}[lefthand width=9.2cm]{Scale every \macname{rule} argument} +\resettokcycle +\settcEscapechar{!}% SO AS NOT TO INTERFERE WITH `|' TOKS IN tabular +\Macrodirective{\addcytoks{#1}\ifx\rule#1\scalerule{2}\fi} +\tokencyclexpress +The rules appearing in the following input stream are literally +!\rule[-2pt]{1.5ex}{1.5ex}! and !\rule{2ex}{3pt}!: + +My environment can include all kinds of rules + \rule[-2pt]{1.5ex}{1.5ex} and other macros, such as \today{} and +\begin{tabular}{c|c} +23 & 454\\\hline +x & y +\end{tabular} +but each rule \rule{2ex}{3pt} is grown by a factor of two. +\endtokencyclexpress +\end{exampleA} +\settcEscapechar{|} + +\endgroup + + +\subsubsection{Protect arguments of specified macro from Character + directive modification} + +\begingroup +Tokcycle provides the escape character $|$ to delimit tokens to be protected + from processing by tokcycle directives. +But that requires the input stream to be adjusted to include these delimiters. +Here's an alternative, not requiring input stream adjustment. +The Macro directive protects all \macname{rule} arguments against contamination + by the Character directive. + +\medskip +When a \macname{rule} is encountered, \macname{protectruleargs} is called + to pop the rule arguments from the input stream and place them directly + in the \macname{cytoks} token list, out of reach from the Character directive. + + +\begin{exampleC}[]{The \macname{protectruleargs} macro to prevent Character directive + contamination of arguments} +\newcommand\protectruleargs[1]{\ifx\rule#1% + \tcpeek\zA + \ifx[\zA\tcpopuntil]\opt + \addcytoks[1]{\opt}\fi% IF OPTIONAL ARG PRESENT, POP AND SAVE TO \cytoks + \tcpopliteral\ArgA% MUST POP ARG 1 WITH BRACES + \addcytoks[1]{\ArgA}% + \tcpopliteral\ArgB% MUST POP ARG 2 WITH BRACES + \addcytoks[1]{\ArgB}% + \fi +} +\end{exampleC} + +Here, every character is typeset with a slash suffix. +If this action were to be imposed to \macname{rule} argument characters, + it would destroy their integrity. +Fortunately, when the Macro directive detects a \macname{rule}, it can + take evasive action. + +\begin{exampleA}[lefthand width=9.1cm]{Protect every \macname{rule} argument + from Character directive contamination} +\resettokcycle +\Characterdirective{\addcytoks{#1\textup{/}\allowbreak}} +\Macrodirective{\addcytoks{#1}\protectruleargs{#1}} +\tokencyclexpress +My \textit{environment} modifies each character token, but rule + \rule[-4pt]{1.5ex}{1.5ex} arguments \rule{2ex}{3pt} are protected. +\endtokencyclexpress +\end{exampleA} +\endgroup + + +\begingroup + +\subsubsection{Find a string of character tokens} + +Earlier in this manuscript, in section~\ref{s:searchA}, a token cycle + was developed to search a token stream for a string match. +Here, we do something similar, except we use the new ``look ahead'' features + of \TokCycle{} to achieve it. +In this case, however, the search string cannot cross space or group + boundaries. +The \macname{matchnextchar} macro demonstrates a use of \macname{tcpopappto}, + which is one of the newly introduced \TokCycle{} commands; instead of + popping an argument from the input stream \textit{into} a macro, it pops it + and \textit{appends} it to a macro. + +\begin{exampleC}[]{The \macname{findstring} macro to perform a look{-}ahead string + search} +\newcommand\findstring[4]{% {CURRENT TOK}{MATCH STRING}{SUCCESS OP}{FAIL CMDS} + \foundstringfalse% DEFAULT CONDITION + \def\poptoks{}% INITIALIZATION. IF MATCH FAILS, MUST TO RESTORE POPPED TOKENS + \expandafter\matchnextchar\ifx#1#2\relax\relax\fi\empty\relax + \iffoundstring\addcytoks{#3{#2}}\else\tcpush{\poptoks}#4\fi +} +\def\matchnextchar#1#2\relax{\tctestifx{#1\empty}{}{% NO MATCH FROM 1ST TOKEN + \tctestifx{\relax#1}{% + \foundstringtrue% NOTHING LEFT TO COMPARE; MATCH FOUND + }{% + \tcpeek\Q% WITHOUT POPPING... + \tctestifx{\Q\bgroup}{}{% PEEK FOR GROUPS OR... + \tctestifx{\Q\tcsptoken}{}{% SPACES THAT INDICATE MATCH FAILURE + \tcpopappto\poptoks% POP TOKEN AND APPEND TO \poptoks + \expandafter\tctestifx\expandafter{\Q#1}{% + \matchnextchar#2\relax% STILL MATCHING; KEEP GOING + }{}% + }% + }% + }% +}} +\end{exampleC} + +\begin{exampleA}[lefthand width=8cm]{Search for the string ``red''} +\resettokcycle +\Characterdirective{\findstring{#1}{red}% + {\textcolor{purple}}{\addcytoks{#1}}} +\tokencyclexpress +Can I \textit{find red in the text}, even in the + MIDredDLE of words and groups? +\endtokencyclexpress +\end{exampleA} + +\endgroup + + +\subsubsection{Eliminate \textit{prior} spaces and pars before a particular + macro} + +The goal here is to eliminate spaces and paragraphs that occur + \textit{prior} to the occurrence of the macro \macname{foo}, which + sets a \macname{bullet}. +Looking backwards is not part of the \TeX{} approach. +Nonetheless, the problem can, with a token cycle, be turned on its head, + such that all occurrences of spaces and \macname{par} can be made + to look for a future instance of \macname{foo}. + +\medskip +Here, if a space or \macname{par} is detected, \macname{foocheck} is called. + It peeks at the token that follows the space or \macname{par}. +If the peeked argument is a \macname{par} or \macname{foo} itself, it discards + the current space or \macname{par}. + +\begingroup + +\begin{exampleC}[]{The \macname{dofoo} environment to remove prior spaces and + paragraphs before \macname{foo}} +\newcommand\foo{$\bullet$} +\newcommand\foocheck[1]{\tcpeek\z\ifx\foo\z\else + \ifx\par\z\else\addcytoks{#1}\fi\fi} +\tokcycleenvironment\dofoo + {\addcytoks{##1}} + {\processtoks{##1}} + {\ifx\par##1\foocheck{##1}\else\addcytoks{##1}\fi} + {\foocheck{##1}} +\end{exampleC} + +\begin{exampleA}[lefthand width=7cm]{Remove spaces and paragraphs before + \macname{foo}} +\dofoo +Lorem ipsum dolor sit amet. (No foo) + +Lorem ipsum dolor sit amet. \foo + +Lorem ipsum dolor sit amet.\foo + +Lorem ipsum dolor sit amet. +\foo + +Lorem ipsum dolor sit amet. + +\foo +\enddofoo +\end{exampleA} +\endgroup + + +\subsubsection{Replace := with \macname{coloneqq} (\texttt{mathtools} + package required)} + +In most cases, creating a macro to signify a composite symbol makes the most + sense. +However, sometimes you don't realize that until you've typed in a lot of code + that fails to use such a macro. +In such a situation, \TokCycle{} can help. + +\medskip +In the \texttt{mathtools} package, there is a symbol, \macname{coloneqq}, + a colon followed by an equal sign, in which the colon has been + vertically adjusted to be symmetric with respect to the \texttt{=} sign. +The adjustment is slight, but noticeable. +A token cycle can be set up to look for instances of the \texttt{:} character, + peek ahead to see if it is followed by an \texttt{=} character and, if so, + replace the pair with \macname{coloneqq}. +However, the process is further complicated, if one is a user of +\macname{usepackage[french]\{babel\}}, where the \texttt{:} character is is made + an active character. + +\medskip +Therefore, in this case, not only does \texttt{:} need to be located by + the Character directive (in the normal case), but it also need be searched + for by the Macro directive, in the event that + \macname{usepackage[french]\{babel\}} is being employed. +In the example below, there are two such colons that satisfy the criterion for + replacement. + +\begingroup + +\begin{exampleC}[]{The \macname{coleqchek} macro and the \macname{subcoloneqq} + environment} +\newcommand\coleqchek[1]{\tcpeek\z + \ifx=\z\tcpop\z\addcytoks{\coloneqq}\else\addcytoks{#1}\fi} +\tokcycleenvironment\subcoloneqq +{\tctestifx{:##1}{\coleqchek{##1}}{\addcytoks{##1}}}% <- IF : A CHARACTER +{\processtoks{##1}} +{\tctestifcon{\if\detokenize{:}\detokenize{##1}}% <- IF : AN ACTIVE MACRO + {\coleqchek{##1}}{\addcytoks{##1}}} +{\addcytoks{##1}} +\end{exampleC} + +\begin{exampleA}[lefthand width=6cm]{replace {:=} with \macname{coloneqq}} +\subcoloneqq +\LARGE Compare: $::=$ + +$a := b$ vs + $a \ordinarycolon= b$ vs + $a:b$. +\endsubcoloneqq +\end{exampleA} + +\endgroup + +\section{Advanced topics: implicit tokens and catcode changes} + +\subsection{Trap active characters (catcode 13)} %\begin{minipage}[t]{\textwidth} \begingroup @@ -1408,10 +1847,12 @@ This is a test!!\endtokencyclexpress are made before reaching \TokCycle{} processing. \svt hey are, thus, no longer detected as active, unless \macname{noexpand} is applied before the pre-expansion. -In this example, the `\texttt{!}' that is not \macname{noexpand}\kern.3pted - is converted to a `\texttt{?}' prior to reaching \TokCycle{} - processing (and thus, not detected as \macname{active}). -Likewise, the `\texttt{T}' is pre-converted to a `\texttt{+}': +In this example, the `\texttt{!}' that is \textit{not} subject to + a \macname{noexpand} is converted to a `\texttt{?}' prior to + reaching \TokCycle{} processing (and thus, not detected as + \macname{active}). +However, the `\texttt{T}' is not pre-converted to a `\texttt{+}', + because it is implicit, and not a definition: \catcode`!=\active \def !{?} @@ -1424,7 +1865,13 @@ Likewise, the `\texttt{T}' is pre-converted to a `\texttt{+}': \detokenize\expandafter{\the\cytoks} \end{exampleA} -However, pre-tokenization does not suffer this behavior: +\begingroup +\catcode`T=11 +\catcode`!=12 +However, pre-tokenization does not suffer this behavior. +The first `\texttt{!}' is tokenized as active during + its absorption into \macname{tmp}, as shown in the next example: +\endgroup \begin{exampleA}[lefthand width=7.8cm]{Pre-tokenized input stream does not affect active characters} \def\tmp{This is a test!!} @@ -1473,7 +1920,7 @@ The following example demonstrates this concept, while showing, \end{exampleA} \endgroup -\subsubsection{Trap Catcode 6 (explicit \& implicit) tokens} +\subsection{Trap catcode 6 (explicit \& implicit) tokens} \begingroup @@ -1569,7 +2016,7 @@ In this example, a macro is defined and then executed, subject to \endgroup -\subsubsection{Trap implicit tokens in general} +\subsection{Trap implicit tokens in general} \begingroup @@ -1656,7 +2103,7 @@ Active tokens processed through the character directive are followed with in the input stream as duplicate, e.g. \texttt{\#\#}. -\subsubsection{Changing grouping tokens (catcodes 1,2)} +\subsection{Changing grouping tokens (catcodes 1,2)} \begingroup @@ -1755,7 +2202,7 @@ Bottom line: \endgroup -\subsubsection{Catcode 10 space tokens\label{s:spc}} +\subsection{Catcode 10 space tokens\label{s:spc}} \begingroup @@ -1771,17 +2218,19 @@ However, \macname{ifimplicittok} may still be used to \ddbend -Also, if the implicit space is an active character, \macname{ifactivetok} - is also set, for the user's benefit. -Likewise, \macname{ifactivechar} is also checked for active spaces, to see +If the implicit space is \textit{also} an active character, + \macname{ifactivetok} is also set, for the user's benefit. +Likewise, \macname{ifactivechar} is also checked, to see if the charcode of the active space is still, indeed, active. While \verb|#1| may be used to represent the space in the space directive, - a special technique is required to recover the detokenized name of the + a special technique may be required to recover the detokenized name of the active space character. -In particular, \verb|#1| will, for active spaces, contain a generic - implicit space token, \macname{tc@sptoken}. -However, the name of the associated active character in the input stream - will be defined as a cat-12 token in \macname{theactivespace}. +In particular, if the active token is an implicit space, but the charcode + of that token is no longer active, \verb|#1| will, for this case, + contain a generic implicit space token, \macname{tcsptoken}. +However, for all active space tokens, the name of the associated active + character in the input stream will be defined as a cat-12 token in + \macname{theactivespace}. \medskip Note in the following examples that cat-10 tokens do \textit{not} @@ -1827,15 +2276,9 @@ catcode settings for the space and the underscore. \end{exampleA} The next two examples introduce implicit and active-implicit spaces. -The latter example also demonstrates detokenization of such spaces. - -\medskip -Here is a useful expandable macro for detokenizing space tokens - in the context of \TokCycle{}. -It can process not only explicit cat-10 spaces, but also implicit and - active-implicit spaces. -To add it to the \macname{cytoks} token list, make sure you use the - \verb|[x]| fully expanded option to \macname{addcytoks}. +In the first example, functionally identical to the prior example, + we see that \TokCycle{} can digest and direct such + tokens to the Space directive, here in the form of \macname{z}. %%% \resettokcycle @@ -1853,40 +2296,63 @@ To add it to the \macname{cytoks} token list, make sure you use the \endtokencyclexpress \end{exampleA} -\begin{minipage}[t]{\textwidth} -\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling - implicit and active-implicit spaces} -\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}% - {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}} -\end{exampleC} +In the second example, an active implicit space token is created, \texttt{Q}, + and stored in a definition, \macname{tmp}. +A \macname{string} is applied by the token cycle at one point to the + space tokens and we see that the proper implicit tokens are decoded, whether + they are active or macro. + +\medskip +The one case deserving special note is the final case of the example, + in which the catcode of \texttt{Q}, the active character, is restored + to a value of 12, before the previously tokenized, active-implicit + \texttt{Q} is passed to the token cycle. +In such a case, where future incantations of \texttt{Q} are no longer active, + then the active-implicit space token that gets passed to the + Space directive (by way of \macname{tmp}) is taken as a generic + active-implicit \macname{tcsptoken}, instead of the original + active-implicit token (\texttt{Q}). -\begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too} +\medskip +Nonetheless, the original string of the active-implicit token (that is, a + cat-12 \texttt{Q}) is preserved in \macname{theactivespace}. +The reason for this behavior is that the absorption of spaces in \TokCycle{} + is a destructive process, in which the spaces are absorbed as strings, + so that a proper interpretation of their implicitness can be ascertained. +With the character no longer active, it would be more difficult to + recreate an active-implicit version of \textit{that} token to + pass to the Space directive. +Thus, a generic active-implicit space token is, instead, employed. + + +\begin{exampleA}[lefthand width=9.05cm]{Active Implicit spaces work{,} too} \resettokcycle -\Characterdirective{\addcytoks{\underdot{#1}}} \def\:{\let\z= } \: % \catcode`Q=\active % \def\:{\let Q= } \: % \catcode`\_=10 % -\tokencyclexpress -xQx x_x\z{}x% -\endtokencyclexpress +\def\tmp{xQx x_x\z{}x} +\expandafter\tokencyclexpress\tmp\endtokencyclexpress -\resettokcycle \Spacedirective{\addcytoks{\textcolor{cyan}\bgroup}% - \addcytoks[x]{(\detokenizespacetok{#1})}% + \addcytoks[x]{(\string#1\ifactivetok + \ifimplicittok\ifactivechar\else\noexpand + \textcolor{red}{:\theactivespace}\fi\fi\fi)}% \addcytoks{\egroup}} \medskip -\tokencyclexpress -xQx x_x\z{}x% -\endtokencyclexpress +\expandafter\tokencyclexpress\tmp\endtokencyclexpress + +\medskip +\catcode`Q=12 +\expandafter\tokencyclexpress\tmp\endtokencyclexpress + \end{exampleA} -\end{minipage} \endgroup -\subsubsection{Changes to catcode 0} +\subsection{Changes to catcode 0} \begingroup |