summaryrefslogtreecommitdiff
path: root/macros/generic/tokcycle/tokcycle-examples.tex
diff options
context:
space:
mode:
Diffstat (limited to 'macros/generic/tokcycle/tokcycle-examples.tex')
-rw-r--r--macros/generic/tokcycle/tokcycle-examples.tex688
1 files changed, 577 insertions, 111 deletions
diff --git a/macros/generic/tokcycle/tokcycle-examples.tex b/macros/generic/tokcycle/tokcycle-examples.tex
index 891741bd4b..cdce49c48d 100644
--- a/macros/generic/tokcycle/tokcycle-examples.tex
+++ b/macros/generic/tokcycle/tokcycle-examples.tex
@@ -1,7 +1,11 @@
\documentclass[10pt]{article}
\usepackage[margin=1.3in]{geometry}
\usepackage[T1]{fontenc}
-\usepackage{tokcycle,xcolor,manfnt,lmodern}
+\usepackage{tokcycle,xcolor,manfnt,lmodern,mathtools}
+%%%%TRIAL PACKAGE CHANGES (MAKE SURE EXAMPLE FILE COMPILES CORRECTLY)
+\makeatletter
+\makeatother
+%%%%END TRIAL
\colorlet{bred}{red!90!blue}
\usepackage{tikz}
% MACROS: REDEFINE TO ALLOW \cytoks TO ESCAPE SCOPE OF exampleA
@@ -48,7 +52,8 @@
\def\altdecytoks{\par\medskip\begingroup\noindent\macname{cytoks}
\ \textit{alt}detokenization:\\\raggedright\footnotesize\ttfamily
- \expandafter\altdetokenize\expandafter{\the\cytoks}\normalsize\par\endgroup}
+ \expandafter\altdetokenize\expandafter{\the\cytoks}\normalsize\par
+ \endgroup\medskip}
\newcommand\TokCycle{\textsf{tokcycle}}
@@ -120,21 +125,21 @@
\newcommand\nextcap[1]{%
\edef\tmp{#1}%
- \tctestifx{-#1}{\def\capnext{T}}{}%
+ \tctestifx{-#1}{\def\capnext{T}}{}% <- TEST FOR HYPHEN
\tctestifcon{\if T\capnext}%
- {\tctestifcon{\ifcat\tmp A}%
- {\uppercase{#1}\def\capnext{F}}%
- {#1}}%
- {#1}%
+ {\tctestifcon{\ifcat\tmp A}% <- TEST FOR NEXT CAT-11
+ {\addcytoks{\uppercase{#1}}\def\capnext{F}}%
+ {\addcytoks{#1}}}%
+ {\addcytoks{#1}}%
}
\newcommand\TitleCase[1]{%
- \def\capnext{T}
+ \def\capnext{T}% <- INITIAL ENTRY
\tokcycle
- {\addcytoks{\nextcap{##1}}}
- {\processtoks{##1}}
+ {\nextcap{##1}}
+ {\def\capnext{T}\processtoks{##1}}% <- GROUP ENTRY
{\addcytoks{##1}}
- {\addcytoks{##1\def\capnext{T}}}
+ {\addcytoks{##1}\def\capnext{T}}% <-CAT-10 SPACE
{#1}%
\the\cytoks
}
@@ -201,24 +206,23 @@
\newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}}
\newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}}
-\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
- {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
-
\newcounter{sigd}
\newcounter{trailingz}
+\newif\iffounddot
+\newif\ifzerosig
\newcommand\sigdigits[1]{%
\setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT
\setcounter{trailingz}{0}% TRAILING ZERO COUNT
- \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED
- \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT
+ \founddotfalse% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED
+ \zerosigfalse% MADE T WHEN ZERO BECOMES SIGNIFICANT
\tokcycle% CYCLE THROOUGH EACH TOKEN
{\tctestifx{.##1}%
- {\if T\founddot\throwerr\fi\def\founddot{T}}% IF .
+ {\iffounddot\throwerr\fi\founddottrue}% IF .
{\tctestifnum{`##1>`/}%
{\tctestifnum{`##1<`:}%
{\tctestifnum{##1>0}%
- {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9
- {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0
+ {\stepcounter{sigd}\zerosigtrue\setcounter{trailingz}{0}}% IF 1-9
+ {\ifzerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0
}%
{\throwerr}% IF ASCII > `9
}%
@@ -229,7 +233,7 @@
{\throwerr}% IF CONTROL SEQUENCE
{}% IGNORE SPACES
{#1}% THE ARGUMENT
- \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS
+ \iffounddot\else\addtocounter{sigd}{-\thetrailingz}\fi% TRAILING ZEROS
\expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}%
\relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT
\thesigd
@@ -269,6 +273,81 @@
{\dumpword\ifrunningcount\else\tcafterenv{\par(Wordcount=\thewordcount,
Lettercount=\thelettercount)}\fi}
+\newcounter{pointcount}
+\newcommand\addtopoints[1]{(\textit{#1 point}%
+ \ifnum#1=1\relax\else\textit{s}\fi)}
+\NewEnviron{pointtracker}[2][points]{%
+ \par\bigskip\resettokcycle
+ \setcounter{pointcount}{0}%
+ \Macrodirective{\addcytoks{##1}\tctestifx{\addtopoints##1}{\sumpoints}{}}%
+ \def\tmp{\tokencyclexpress{\large\bfseries#2: \thepointcount\ #1}}%
+ \expandafter\tmp\BODY\endtokencyclexpress
+}
+\def\sumpoints{\tcpop\Q\addtocounter{pointcount}{\Q}\tcpushgroup{\Q}}
+
+\newcommand\scalerule[1]{%
+ \tcpeek\zA
+ \ifx[\zA\tcpopuntil]\opt\fi% IF PRESENT, POP OPTIONAL ARG
+ \tcpop\ArgA% POP ARG1
+ \tcpop\ArgB% POP ARG2
+ \tcpushgroup[x]{#1\dimexpr\ArgB}% PUSH BACK IN REVERSE (LIFO) FASHION
+ \tcpushgroup[x]{#1\dimexpr\ArgA}
+ \ifx[\zA\tcpush[x]{\expandafter\scaleopt\opt{#1}}\fi
+}
+\def\scaleopt[#1]#2{[#2\dimexpr#1]}
+
+\newcommand\protectruleargs[1]{\ifx\rule#1%
+ \tcpeek\zA
+ \ifx[\zA\tcpopuntil]\opt
+ \addcytoks[1]{\opt}\fi% IF OPTIONAL ARG PRESENT, POP AND SAVE TO \cytoks
+ \tcpopliteral\ArgA% MUST POP ARG 1 WITH BRACES
+ \addcytoks[1]{\ArgA}%
+ \tcpopliteral\ArgB% MUST POP ARG 2 WITH BRACES
+ \addcytoks[1]{\ArgB}%
+ \fi
+}
+
+\newif\iffoundstring
+\newcommand\findstring[4]{% {CURRENT TOK}{MATCH STRING}{SUCCESS OP}{FAIL CMDS}
+ \foundstringfalse% DEFAULT CONDITION
+ \def\poptoks{}% INITIALIZATION. IF MATCH FAILS, MUST TO RESTORE POPPED TOKENS
+ \expandafter\matchnextchar\ifx#1#2\relax\relax\fi\empty\relax
+ \iffoundstring\addcytoks{#3{#2}}\else\tcpush{\poptoks}#4\fi
+}
+\def\matchnextchar#1#2\relax{\tctestifx{#1\empty}{}{% NO MATCH FROM 1ST TOKEN
+ \tctestifx{\relax#1}{%
+ \foundstringtrue% NOTHING LEFT TO COMPARE; MATCH FOUND
+ }{%
+ \tcpeek\Q% WITHOUT POPPING...
+ \tctestifx{\Q\bgroup}{}{% PEEK FOR GROUPS OR...
+ \tctestifx{\Q\tcsptoken}{}{% SPACES THAT INDICATE MATCH FAILURE
+ \tcpopappto\poptoks% POP TOKEN AND APPEND TO \poptoks
+ \expandafter\tctestifx\expandafter{\Q#1}{%
+ \matchnextchar#2\relax% STILL MATCHING; KEEP GOING
+ }{}%
+ }%
+ }%
+ }%
+}}
+
+\newcommand\foo{$\bullet$}
+\newcommand\foocheck[1]{\tcpeek\z\ifx\foo\z\else
+ \ifx\par\z\else\addcytoks{#1}\fi\fi}
+\tokcycleenvironment\dofoo
+ {\addcytoks{##1}}
+ {\processtoks{##1}}
+ {\ifx\par##1\foocheck{##1}\else\addcytoks{##1}\fi}
+ {\foocheck{##1}}
+
+\newcommand\coleqchek[1]{\tcpeek\z
+ \ifx=\z\tcpop\z\addcytoks{\coloneqq}\else\addcytoks{#1}\fi}
+\tokcycleenvironment\subcoloneqq
+{\tctestifx{:##1}{\coleqchek{##1}}{\addcytoks{##1}}}
+{\processtoks{##1}}
+{\tctestifcon{\if\detokenize{:}\detokenize{##1}}
+ {\coleqchek{##1}}{\addcytoks{##1}}}
+{\addcytoks{##1}}
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Examples, examples, and more examples}
@@ -292,7 +371,8 @@ Apply different directives to Characters (under-dot), Groups
\begingroup
-\begin{exampleA}[lefthand width=7.8cm]{}
+\begin{exampleA}[lefthand width=7.8cm]{Apply different directives to
+ characters{,} groups{,} macros{,} and spaces}
\tokcycle{\addcytoks{\underdot{#1}}}
{\addcytoks{\{}\processtoks{#1}
\addcytoks{\}}}
@@ -307,9 +387,13 @@ Apply different directives to Characters (under-dot), Groups
\subsubsection{Escaping text}
Text between two successive escape characters is bypassed
- by \TokCycle{} and instead echoed to the output register.
-Default escape character is $|$. One can change it with
+ by \TokCycle{} and instead echoed to the output token list
+ (\macname{cytoks}).
+The default escape character is $|$.
+One can change the token-cycle escape character by way of the
\macname{settcEscapechar} macro.
+Below, the character codes are incremented by one; however, the
+ arguments of \macname{rule} are protected against that.
\begingroup
@@ -343,6 +427,7 @@ This section concerns the issue of whether the characters of the input
output token list (\macname{cytoks}).
\medskip
+\begin{minipage}[t]{\textwidth}
Transform characters (+1 ASCII) via unexpandable macro:\medskip
\begin{exampleA}[lefthand width=8.3cm]
@@ -361,6 +446,7 @@ Transform characters (+1 ASCII) via unexpandable macro:\medskip
Note how the above detokenization shows the actual transformation
instructions involving \macname{plusl}.
This is done because \macname{plusl} is not expandable.
+\end{minipage}
\par\noindent\hrulefill\medskip
\endgroup
@@ -394,7 +480,6 @@ Capitalize vowels (but don't expand the character directive)
\altdecytoks
-\medskip
The above detokenization also shows the transformation macro,
\macname{vowelcap}, as part of \macname{cytoks}.
But because \macname{vowelcap} is expandable, we did not need to
@@ -404,9 +489,16 @@ But because \macname{vowelcap} is expandable, we did not need to
\endgroup
\end{minipage}
+\begin{minipage}[t]{\textwidth}
\begingroup
Capitalize vowels (expanding the character directive)
+\medskip
+By using the \texttt{[x]} option of \macname{addcytoks}, the
+ \macname{vowelcap} is fully expanded prior to being placed in
+ the \macname{cytoks} token list.
+
+
\begin{exampleA}[lefthand width=8.3cm]
{Expanded Character directive}
\tokcycle
@@ -420,11 +512,12 @@ Capitalize vowels (expanding the character directive)
\altdecytoks
-\medskip
Here, we see no vestiges of \macname{vowelcap} in \macname{cytoks},
but only the transformed result.
\endgroup
+\end{minipage}
+\enlargethispage{12pt}
\subsubsection{Unexpanded vs. pre-expanded input stream}
\begingroup
@@ -446,7 +539,6 @@ Here, we see no vestiges of \macname{vowelcap} in \macname{cytoks},
\altdecytoks
-\medskip
In the above case, vowel-capping is not applied to the date, because
\macname{today} remains unexpanded in the \macname{cytoks} token list.
@@ -468,7 +560,6 @@ Note that, when pre-expanding the input stream, one must
\altdecytoks
-\medskip
Here, we see the date subjected to vowel-capping, because the \TokCycle{}
input was expanded prior to the token cycle.
Thus \macname{today} no longer appears \textit{explicitly} in the
@@ -494,7 +585,7 @@ The desired options are to be set prior to the \TokCycle{} invocation.
The macro
\macname{stripimplicitgroupingcase} can take three possible
integer arguments: 0 (default) to automatically place unaltered
- implicit group tokens in the output register; 1 to strip
+ implicit group tokens in the \macname{cytoks} output; 1 to strip
implicit group tokens from the output; or $-$1 to instead pass the
implicit group tokens to the Character directive (as implicit
tokens) for separate processing (typically, when detokenization
@@ -547,8 +638,8 @@ Group stripping, brought about by \macname{stripgroupingtrue},
involves removing the grouping braces from around the group.
The choice of \macname{processtoks} vs.\@ \macname{addcytoks}
affects whether the tokens inside the group are recommitted
- to \TokCycle{} for processing, or are merely sent to the output
- register in their original unprocessed form.
+ to \TokCycle{} for processing, or are merely sent to the
+ \macname{cytoks} token list in their original unprocessed form.
\medskip
Note that, in these examples, underdots and visible spaces
@@ -700,7 +791,6 @@ The final disposition of the \macname{cytoks} token list shows the effects of
\altdecytoks
-\medskip
Without \macname{stripgroupingtrue}, all tasks, A,B,C, and D, would have
been contained within the explicit \texttt{\{...\}} group.
@@ -785,7 +875,7 @@ In certain applications, it may offer a more pleasingly formatted
typesetting of detokenized material.
\medskip It is an unusual application of \TokCycle{} in that it does not
- actually use the \macname{cytoks} token register to collect
+ actually use the \macname{cytoks} token list to collect
its output.
This is only possible because all macros in the input stream
are detokenized, rather than executed.
@@ -821,29 +911,29 @@ versus \string\detokenize: \\
\subsubsection{Capitalize all words, including compound and
- parenthetical words}
+ parenthetical words\label{s:cap}}
\begingroup
\begin{exampleC}[]{The \macname{Titlecase} and \texttt{\string\nextcap} macros}
\newcommand\TitleCase[1]{%
- \def\capnext{T}
+ \def\capnext{T}% <- INITIAL ENTRY
\tokcycle
- {\addcytoks{\nextcap{##1}}}
- {\processtoks{##1}}
+ {\nextcap{##1}}
+ {\def\capnext{T}\processtoks{##1}}% <- GROUP ENTRY
{\addcytoks{##1}}
- {\addcytoks{##1\def\capnext{T}}}
+ {\addcytoks{##1}\def\capnext{T}}% <-CAT-10 SPACE
{#1}%
\the\cytoks
}
\newcommand\nextcap[1]{%
\edef\tmp{#1}%
- \tctestifx{-#1}{\def\capnext{T}}{}%
+ \tctestifx{-#1}{\def\capnext{T}}{}% <- TEST FOR HYPHEN
\tctestifcon{\if T\capnext}%
- {\tctestifcon{\ifcat\tmp A}%
- {\uppercase{#1}\def\capnext{F}}%
- {#1}}%
- {#1}%
+ {\tctestifcon{\ifcat\tmp A}% <- TEST FOR NEXT CAT-11
+ {\addcytoks{\uppercase{#1}}\def\capnext{F}}%
+ {\addcytoks{#1}}}%
+ {\addcytoks{#1}}%
}
\end{exampleC}
@@ -859,6 +949,19 @@ versus \string\detokenize: \\
(\textit{capitalizing} words).}
\end{exampleA}
+As it is currently set up, a future-capitalization of the next catcode-11 token will be triggered when any of the following is encountered in the input stream:
+\begin{itemize}
+\item The initial entry into the routine
+
+\item a catcode-10 space token
+
+\item a hyphen \texttt{-}
+
+\item upon entry into a group \texttt{\{...\}}
+\end{itemize}
+
+Obviously, additional tests could be added.
+
\endgroup
\subsubsection{Scaling rule dimensions}
@@ -890,7 +993,7 @@ versus \string\detokenize: \\
\endgroup
-\subsubsection{String search, including non-regex material}
+\subsubsection{String search, including non-regex material\label{s:searchA}}
\begingroup
@@ -953,8 +1056,8 @@ versus \string\detokenize: \\
\begingroup
The problem of counting the significant digits in a number is an interesting
- one, in that leading and trailing zeros are not significant, but zeros
- in the middle of the number are significant.
+ one, in that leading zeros are not significant, trailing zeros are sometimes
+ significant, but zeros in the middle of the number are significant.
In a problem like this, the \macname{cytoks} token list is not used to
carry a transformation of the input tokens, but only to carry a record
of errant tokens from the input (as seen from the helper macro
@@ -977,30 +1080,32 @@ The main work of the \macname{sigdigits} macro is to use the counters
\begin{exampleC}[]{The \macname{sigdigits} macro for counting significant digits}
\newcounter{sigd}
\newcounter{trailingz}
+\newif\iffounddot
+\newif\ifzerosig
\newcommand\sigdigits[1]{%
\setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT
\setcounter{trailingz}{0}% TRAILING ZERO COUNT
- \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED
- \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT
+ \founddotfalse% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED
+ \zerosigfalse% MADE T WHEN ZERO BECOMES SIGNIFICANT
\tokcycle% CYCLE THROOUGH EACH TOKEN
- {\tctestifx{.##1}%
- {\if T\founddot\throwerr\fi\def\founddot{T}}% IF .
- {\tctestifnum{`##1>`/}%
- {\tctestifnum{`##1<`:}%
- {\tctestifnum{##1>0}%
- {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9
- {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0
- }%
- {\throwerr}% IF ASCII > `9
- }%
- {\throwerr}% IF ASCII < `0
+ {\tctestifx{.##1}%
+ {\iffounddot\throwerr\fi\founddottrue}% IF .
+ {\tctestifnum{`##1>`/}%
+ {\tctestifnum{`##1<`:}%
+ {\tctestifnum{##1>0}%
+ {\stepcounter{sigd}\zerosigtrue\setcounter{trailingz}{0}}% IF 1-9
+ {\ifzerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0
}%
- }% APPLY ABOVE LOGIC FOR CHAR TOKENS
- {\throwerr}% IF BRACES
- {\throwerr}% IF CONTROL SEQUENCE
- {}% IGNORE SPACES
- {#1}% THE ARGUMENT
- \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS
+ {\throwerr}% IF ASCII > `9
+ }%
+ {\throwerr}% IF ASCII < `0
+ }%
+ }% APPLY ABOVE LOGIC FOR CHAR TOKENS
+ {\throwerr}% IF BRACES
+ {\throwerr}% IF CONTROL SEQUENCE
+ {}% IGNORE SPACES
+ {#1}% THE ARGUMENT
+ \iffounddot\else\addtocounter{sigd}{-\thetrailingz}\fi% TRAILING ZEROS
\expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}%
\relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT
\thesigd
@@ -1213,6 +1318,11 @@ We provide two modes of output: one that provides an ending summary, and
As with other \TokCycle{} environments and macros, portions can be
exempted from the count (for example macro arguments) by using the
escape delimiters provided.
+An even more elaborate version of this environment can be found at
+\texttt{https://tex.stackexchange.com/questions/\allowbreak
+ 44618/dynamically-count-and-return-number-of-words-in-a-section/587567\#587567},%
+\hfil in\-cluding a version that runs
+ in Plain \TeX.
\begin{exampleC}[]{The \macname{countem} extended-environment definition
and supporting macros}
@@ -1326,7 +1436,7 @@ Thus, ``-is loc'' is also added to the output stream.
}
\begin{exampleB}[text only, righthand width=0.01cm, colback=white]%
-{Effects of various truncation directives}
+{Example: Effects of various truncation directives}
{\footnotesize\ttfamily Input text}: \expandafter\altdetokenize\expandafter
{\thetext}
@@ -1350,9 +1460,338 @@ For example, a detokenization of the final case reveals this to be
\endgroup
-\subsection{Advanced topics: implicit tokens and catcode changes}
+\subsection{Look-ahead features}
+
+The whole concept of the \TokCycle{} package is to examine the input stream
+ one token at a time and use the directives to process the tokens.
+Occasionally, however, there are tasks that involve several adjacent tokens.
+In the example of section~\ref{s:cap}, for instance, in order to capitalize
+ words, each space token triggers an action on the character that follows.
+To accomplish this without any look-ahead features, the Space directive sets
+ a flag, \verb|\def\capnext{T}|, in order to communicate with a the Character
+ directive about a subsequent token.
+That is a perfectly valid approach to communicate conditional action.
+
+\medskip
+However, \TokCycle{} also provides look-ahead features, to avoid the necessity
+ of communicating with future tokens in the cycle by way of flag settings.
+These tools allow a directive to directly ``peek'' ahead at the input stream
+ (without disturbing it) and, if desired, push and pop tokens to and from
+ the input stream.
+The package manual should be consulted for a full list of macros provided to
+ facilitate look-ahead capability.
+But here are some examples to look at.
+
+\begingroup
+
+\subsubsection{Count in advance that which follows}
+
+Specify the point values for questions and let tokcycle perform the cumulative
+ tallies in advance, using a single-pass compilation.
+
+\begin{exampleC}[]{The \texttt{pointtracker} environment to count point values
+ in advance}
+\newcounter{pointcount}
+\newcommand\addtopoints[1]{(\textit{#1 point}%
+ \ifnum#1=1\relax\else\textit{s}\fi)}
+\NewEnviron{pointtracker}[2][points]{%
+ \par\bigskip\resettokcycle
+ \setcounter{pointcount}{0}%
+ \Macrodirective{\addcytoks{##1}\tctestifx{\addtopoints##1}{\sumpoints}{}}%
+ \def\tmp{\tokencyclexpress{\large\bfseries#2: \thepointcount\ #1}}%
+ \expandafter\tmp\BODY\endtokencyclexpress
+}
+\def\sumpoints{\tcpop\Q\addtocounter{pointcount}{\Q}\tcpushgroup{\Q}}
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=6.2cm]{Demo of the pointtracker environment}
+\begin{pointtracker}[total points]{\Large Science Test}
+\begin{pointtracker}{Advanced Geology}
+ \begin{itemize}
+ \item Count the fingers on your left hand. Are there more than 9?
+ \addtopoints{1}
+ \item Draw a rectangle with five edges. \addtopoints{3}
+ \end{itemize}
+\end{pointtracker}
+\begin{pointtracker}{Meteorology}
+ \begin{itemize}
+ \item How many borgs does it take to brew a Danish beer?
+ \addtopoints{2}
+ \item What is the meaning of life? \addtopoints{4}
+ \end{itemize}
+\end{pointtracker}
+\end{pointtracker}
+\end{exampleA}
+
+Note that this example has several neat features.
+The points for Geology, Meteorology, and the Test total are not specified,
+ but calculated.
+They are calculated based on content that appears later in the environment.
+Traditionally, multiple compilations would be required with external data
+ written to the aux file, but that does not occur here---there is only
+ a single compilation.
+Finally, becaue we are nesting the \TokCycle{} part of \texttt{pointtracker}
+ inside of a \macname{NewEnviron}, we see that the \texttt{pointtracker}
+ environment can be nested, without adverse effect.
+
+\endgroup
+
+\subsubsection{Modify the arguments of every instance of a particular macro}
-\subsubsection{Trap Active Characters (catcode 13)}
+\begingroup
+
+This example, through the use of the \macname{scalerule} macro, shows how
+ the \macname{rule} arguments, including the optional argument, can be
+ popped off the input stream, adjusted, and placed back in the stream.
+
+\begin{exampleC}[]{The \macname{scalerule} macro to scale all
+ \macname{rule} arguments}
+\newcommand\scalerule[1]{%
+ \tcpeek\zA
+ \ifx[\zA\tcpopuntil]\opt\fi% IF PRESENT, POP OPTIONAL ARG
+ \tcpop\ArgA% POP ARG1
+ \tcpop\ArgB% POP ARG2
+ \tcpushgroup[x]{#1\dimexpr\ArgB}% PUSH BACK IN REVERSE (LIFO) FASHION
+ \tcpushgroup[x]{#1\dimexpr\ArgA}
+ \ifx[\zA\tcpush[x]{\expandafter\scaleopt\opt{#1}}\fi
+}
+\def\scaleopt[#1]#2{[#2\dimexpr#1]}
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=9.2cm]{Scale every \macname{rule} argument}
+\resettokcycle
+\settcEscapechar{!}% SO AS NOT TO INTERFERE WITH `|' TOKS IN tabular
+\Macrodirective{\addcytoks{#1}\ifx\rule#1\scalerule{2}\fi}
+\tokencyclexpress
+The rules appearing in the following input stream are literally
+!\rule[-2pt]{1.5ex}{1.5ex}! and !\rule{2ex}{3pt}!:
+
+My environment can include all kinds of rules
+ \rule[-2pt]{1.5ex}{1.5ex} and other macros, such as \today{} and
+\begin{tabular}{c|c}
+23 & 454\\\hline
+x & y
+\end{tabular}
+but each rule \rule{2ex}{3pt} is grown by a factor of two.
+\endtokencyclexpress
+\end{exampleA}
+\settcEscapechar{|}
+
+\endgroup
+
+
+\subsubsection{Protect arguments of specified macro from Character
+ directive modification}
+
+\begingroup
+Tokcycle provides the escape character $|$ to delimit tokens to be protected
+ from processing by tokcycle directives.
+But that requires the input stream to be adjusted to include these delimiters.
+Here's an alternative, not requiring input stream adjustment.
+The Macro directive protects all \macname{rule} arguments against contamination
+ by the Character directive.
+
+\medskip
+When a \macname{rule} is encountered, \macname{protectruleargs} is called
+ to pop the rule arguments from the input stream and place them directly
+ in the \macname{cytoks} token list, out of reach from the Character directive.
+
+
+\begin{exampleC}[]{The \macname{protectruleargs} macro to prevent Character directive
+ contamination of arguments}
+\newcommand\protectruleargs[1]{\ifx\rule#1%
+ \tcpeek\zA
+ \ifx[\zA\tcpopuntil]\opt
+ \addcytoks[1]{\opt}\fi% IF OPTIONAL ARG PRESENT, POP AND SAVE TO \cytoks
+ \tcpopliteral\ArgA% MUST POP ARG 1 WITH BRACES
+ \addcytoks[1]{\ArgA}%
+ \tcpopliteral\ArgB% MUST POP ARG 2 WITH BRACES
+ \addcytoks[1]{\ArgB}%
+ \fi
+}
+\end{exampleC}
+
+Here, every character is typeset with a slash suffix.
+If this action were to be imposed to \macname{rule} argument characters,
+ it would destroy their integrity.
+Fortunately, when the Macro directive detects a \macname{rule}, it can
+ take evasive action.
+
+\begin{exampleA}[lefthand width=9.1cm]{Protect every \macname{rule} argument
+ from Character directive contamination}
+\resettokcycle
+\Characterdirective{\addcytoks{#1\textup{/}\allowbreak}}
+\Macrodirective{\addcytoks{#1}\protectruleargs{#1}}
+\tokencyclexpress
+My \textit{environment} modifies each character token, but rule
+ \rule[-4pt]{1.5ex}{1.5ex} arguments \rule{2ex}{3pt} are protected.
+\endtokencyclexpress
+\end{exampleA}
+\endgroup
+
+
+\begingroup
+
+\subsubsection{Find a string of character tokens}
+
+Earlier in this manuscript, in section~\ref{s:searchA}, a token cycle
+ was developed to search a token stream for a string match.
+Here, we do something similar, except we use the new ``look ahead'' features
+ of \TokCycle{} to achieve it.
+In this case, however, the search string cannot cross space or group
+ boundaries.
+The \macname{matchnextchar} macro demonstrates a use of \macname{tcpopappto},
+ which is one of the newly introduced \TokCycle{} commands; instead of
+ popping an argument from the input stream \textit{into} a macro, it pops it
+ and \textit{appends} it to a macro.
+
+\begin{exampleC}[]{The \macname{findstring} macro to perform a look{-}ahead string
+ search}
+\newcommand\findstring[4]{% {CURRENT TOK}{MATCH STRING}{SUCCESS OP}{FAIL CMDS}
+ \foundstringfalse% DEFAULT CONDITION
+ \def\poptoks{}% INITIALIZATION. IF MATCH FAILS, MUST TO RESTORE POPPED TOKENS
+ \expandafter\matchnextchar\ifx#1#2\relax\relax\fi\empty\relax
+ \iffoundstring\addcytoks{#3{#2}}\else\tcpush{\poptoks}#4\fi
+}
+\def\matchnextchar#1#2\relax{\tctestifx{#1\empty}{}{% NO MATCH FROM 1ST TOKEN
+ \tctestifx{\relax#1}{%
+ \foundstringtrue% NOTHING LEFT TO COMPARE; MATCH FOUND
+ }{%
+ \tcpeek\Q% WITHOUT POPPING...
+ \tctestifx{\Q\bgroup}{}{% PEEK FOR GROUPS OR...
+ \tctestifx{\Q\tcsptoken}{}{% SPACES THAT INDICATE MATCH FAILURE
+ \tcpopappto\poptoks% POP TOKEN AND APPEND TO \poptoks
+ \expandafter\tctestifx\expandafter{\Q#1}{%
+ \matchnextchar#2\relax% STILL MATCHING; KEEP GOING
+ }{}%
+ }%
+ }%
+ }%
+}}
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=8cm]{Search for the string ``red''}
+\resettokcycle
+\Characterdirective{\findstring{#1}{red}%
+ {\textcolor{purple}}{\addcytoks{#1}}}
+\tokencyclexpress
+Can I \textit{find red in the text}, even in the
+ MIDredDLE of words and groups?
+\endtokencyclexpress
+\end{exampleA}
+
+\endgroup
+
+
+\subsubsection{Eliminate \textit{prior} spaces and pars before a particular
+ macro}
+
+The goal here is to eliminate spaces and paragraphs that occur
+ \textit{prior} to the occurrence of the macro \macname{foo}, which
+ sets a \macname{bullet}.
+Looking backwards is not part of the \TeX{} approach.
+Nonetheless, the problem can, with a token cycle, be turned on its head,
+ such that all occurrences of spaces and \macname{par} can be made
+ to look for a future instance of \macname{foo}.
+
+\medskip
+Here, if a space or \macname{par} is detected, \macname{foocheck} is called.
+ It peeks at the token that follows the space or \macname{par}.
+If the peeked argument is a \macname{par} or \macname{foo} itself, it discards
+ the current space or \macname{par}.
+
+\begingroup
+
+\begin{exampleC}[]{The \macname{dofoo} environment to remove prior spaces and
+ paragraphs before \macname{foo}}
+\newcommand\foo{$\bullet$}
+\newcommand\foocheck[1]{\tcpeek\z\ifx\foo\z\else
+ \ifx\par\z\else\addcytoks{#1}\fi\fi}
+\tokcycleenvironment\dofoo
+ {\addcytoks{##1}}
+ {\processtoks{##1}}
+ {\ifx\par##1\foocheck{##1}\else\addcytoks{##1}\fi}
+ {\foocheck{##1}}
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=7cm]{Remove spaces and paragraphs before
+ \macname{foo}}
+\dofoo
+Lorem ipsum dolor sit amet. (No foo)
+
+Lorem ipsum dolor sit amet. \foo
+
+Lorem ipsum dolor sit amet.\foo
+
+Lorem ipsum dolor sit amet.
+\foo
+
+Lorem ipsum dolor sit amet.
+
+\foo
+\enddofoo
+\end{exampleA}
+\endgroup
+
+
+\subsubsection{Replace := with \macname{coloneqq} (\texttt{mathtools}
+ package required)}
+
+In most cases, creating a macro to signify a composite symbol makes the most
+ sense.
+However, sometimes you don't realize that until you've typed in a lot of code
+ that fails to use such a macro.
+In such a situation, \TokCycle{} can help.
+
+\medskip
+In the \texttt{mathtools} package, there is a symbol, \macname{coloneqq},
+ a colon followed by an equal sign, in which the colon has been
+ vertically adjusted to be symmetric with respect to the \texttt{=} sign.
+The adjustment is slight, but noticeable.
+A token cycle can be set up to look for instances of the \texttt{:} character,
+ peek ahead to see if it is followed by an \texttt{=} character and, if so,
+ replace the pair with \macname{coloneqq}.
+However, the process is further complicated, if one is a user of
+\macname{usepackage[french]\{babel\}}, where the \texttt{:} character is is made
+ an active character.
+
+\medskip
+Therefore, in this case, not only does \texttt{:} need to be located by
+ the Character directive (in the normal case), but it also need be searched
+ for by the Macro directive, in the event that
+ \macname{usepackage[french]\{babel\}} is being employed.
+In the example below, there are two such colons that satisfy the criterion for
+ replacement.
+
+\begingroup
+
+\begin{exampleC}[]{The \macname{coleqchek} macro and the \macname{subcoloneqq}
+ environment}
+\newcommand\coleqchek[1]{\tcpeek\z
+ \ifx=\z\tcpop\z\addcytoks{\coloneqq}\else\addcytoks{#1}\fi}
+\tokcycleenvironment\subcoloneqq
+{\tctestifx{:##1}{\coleqchek{##1}}{\addcytoks{##1}}}% <- IF : A CHARACTER
+{\processtoks{##1}}
+{\tctestifcon{\if\detokenize{:}\detokenize{##1}}% <- IF : AN ACTIVE MACRO
+ {\coleqchek{##1}}{\addcytoks{##1}}}
+{\addcytoks{##1}}
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=6cm]{replace {:=} with \macname{coloneqq}}
+\subcoloneqq
+\LARGE Compare: $::=$
+
+$a := b$ vs
+ $a \ordinarycolon= b$ vs
+ $a:b$.
+\endsubcoloneqq
+\end{exampleA}
+
+\endgroup
+
+\section{Advanced topics: implicit tokens and catcode changes}
+
+\subsection{Trap active characters (catcode 13)}
%\begin{minipage}[t]{\textwidth}
\begingroup
@@ -1408,10 +1847,12 @@ This is a test!!\endtokencyclexpress
are made before reaching \TokCycle{} processing.
\svt hey are, thus, no longer detected as active, unless
\macname{noexpand} is applied before the pre-expansion.
-In this example, the `\texttt{!}' that is not \macname{noexpand}\kern.3pted
- is converted to a `\texttt{?}' prior to reaching \TokCycle{}
- processing (and thus, not detected as \macname{active}).
-Likewise, the `\texttt{T}' is pre-converted to a `\texttt{+}':
+In this example, the `\texttt{!}' that is \textit{not} subject to
+ a \macname{noexpand} is converted to a `\texttt{?}' prior to
+ reaching \TokCycle{} processing (and thus, not detected as
+ \macname{active}).
+However, the `\texttt{T}' is not pre-converted to a `\texttt{+}',
+ because it is implicit, and not a definition:
\catcode`!=\active
\def !{?}
@@ -1424,7 +1865,13 @@ Likewise, the `\texttt{T}' is pre-converted to a `\texttt{+}':
\detokenize\expandafter{\the\cytoks}
\end{exampleA}
-However, pre-tokenization does not suffer this behavior:
+\begingroup
+\catcode`T=11
+\catcode`!=12
+However, pre-tokenization does not suffer this behavior.
+The first `\texttt{!}' is tokenized as active during
+ its absorption into \macname{tmp}, as shown in the next example:
+\endgroup
\begin{exampleA}[lefthand width=7.8cm]{Pre-tokenized input stream
does not affect active characters}
\def\tmp{This is a test!!}
@@ -1473,7 +1920,7 @@ The following example demonstrates this concept, while showing,
\end{exampleA}
\endgroup
-\subsubsection{Trap Catcode 6 (explicit \& implicit) tokens}
+\subsection{Trap catcode 6 (explicit \& implicit) tokens}
\begingroup
@@ -1569,7 +2016,7 @@ In this example, a macro is defined and then executed, subject to
\endgroup
-\subsubsection{Trap implicit tokens in general}
+\subsection{Trap implicit tokens in general}
\begingroup
@@ -1656,7 +2103,7 @@ Active tokens processed through the character directive are followed with
in the input stream as duplicate, e.g. \texttt{\#\#}.
-\subsubsection{Changing grouping tokens (catcodes 1,2)}
+\subsection{Changing grouping tokens (catcodes 1,2)}
\begingroup
@@ -1755,7 +2202,7 @@ Bottom line:
\endgroup
-\subsubsection{Catcode 10 space tokens\label{s:spc}}
+\subsection{Catcode 10 space tokens\label{s:spc}}
\begingroup
@@ -1771,17 +2218,19 @@ However, \macname{ifimplicittok} may still be used to
\ddbend
-Also, if the implicit space is an active character, \macname{ifactivetok}
- is also set, for the user's benefit.
-Likewise, \macname{ifactivechar} is also checked for active spaces, to see
+If the implicit space is \textit{also} an active character,
+ \macname{ifactivetok} is also set, for the user's benefit.
+Likewise, \macname{ifactivechar} is also checked, to see
if the charcode of the active space is still, indeed, active.
While \verb|#1| may be used to represent the space in the space directive,
- a special technique is required to recover the detokenized name of the
+ a special technique may be required to recover the detokenized name of the
active space character.
-In particular, \verb|#1| will, for active spaces, contain a generic
- implicit space token, \macname{tc@sptoken}.
-However, the name of the associated active character in the input stream
- will be defined as a cat-12 token in \macname{theactivespace}.
+In particular, if the active token is an implicit space, but the charcode
+ of that token is no longer active, \verb|#1| will, for this case,
+ contain a generic implicit space token, \macname{tcsptoken}.
+However, for all active space tokens, the name of the associated active
+ character in the input stream will be defined as a cat-12 token in
+ \macname{theactivespace}.
\medskip
Note in the following examples that cat-10 tokens do \textit{not}
@@ -1827,15 +2276,9 @@ catcode settings for the space and the underscore.
\end{exampleA}
The next two examples introduce implicit and active-implicit spaces.
-The latter example also demonstrates detokenization of such spaces.
-
-\medskip
-Here is a useful expandable macro for detokenizing space tokens
- in the context of \TokCycle{}.
-It can process not only explicit cat-10 spaces, but also implicit and
- active-implicit spaces.
-To add it to the \macname{cytoks} token list, make sure you use the
- \verb|[x]| fully expanded option to \macname{addcytoks}.
+In the first example, functionally identical to the prior example,
+ we see that \TokCycle{} can digest and direct such
+ tokens to the Space directive, here in the form of \macname{z}.
%%%
\resettokcycle
@@ -1853,40 +2296,63 @@ To add it to the \macname{cytoks} token list, make sure you use the
\endtokencyclexpress
\end{exampleA}
-\begin{minipage}[t]{\textwidth}
-\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling
- implicit and active-implicit spaces}
-\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
- {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
-\end{exampleC}
+In the second example, an active implicit space token is created, \texttt{Q},
+ and stored in a definition, \macname{tmp}.
+A \macname{string} is applied by the token cycle at one point to the
+ space tokens and we see that the proper implicit tokens are decoded, whether
+ they are active or macro.
+
+\medskip
+The one case deserving special note is the final case of the example,
+ in which the catcode of \texttt{Q}, the active character, is restored
+ to a value of 12, before the previously tokenized, active-implicit
+ \texttt{Q} is passed to the token cycle.
+In such a case, where future incantations of \texttt{Q} are no longer active,
+ then the active-implicit space token that gets passed to the
+ Space directive (by way of \macname{tmp}) is taken as a generic
+ active-implicit \macname{tcsptoken}, instead of the original
+ active-implicit token (\texttt{Q}).
-\begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too}
+\medskip
+Nonetheless, the original string of the active-implicit token (that is, a
+ cat-12 \texttt{Q}) is preserved in \macname{theactivespace}.
+The reason for this behavior is that the absorption of spaces in \TokCycle{}
+ is a destructive process, in which the spaces are absorbed as strings,
+ so that a proper interpretation of their implicitness can be ascertained.
+With the character no longer active, it would be more difficult to
+ recreate an active-implicit version of \textit{that} token to
+ pass to the Space directive.
+Thus, a generic active-implicit space token is, instead, employed.
+
+
+\begin{exampleA}[lefthand width=9.05cm]{Active Implicit spaces work{,} too}
\resettokcycle
-\Characterdirective{\addcytoks{\underdot{#1}}}
\def\:{\let\z= } \: %
\catcode`Q=\active %
\def\:{\let Q= } \: %
\catcode`\_=10 %
-\tokencyclexpress
-xQx x_x\z{}x%
-\endtokencyclexpress
+\def\tmp{xQx x_x\z{}x}
+\expandafter\tokencyclexpress\tmp\endtokencyclexpress
-\resettokcycle
\Spacedirective{\addcytoks{\textcolor{cyan}\bgroup}%
- \addcytoks[x]{(\detokenizespacetok{#1})}%
+ \addcytoks[x]{(\string#1\ifactivetok
+ \ifimplicittok\ifactivechar\else\noexpand
+ \textcolor{red}{:\theactivespace}\fi\fi\fi)}%
\addcytoks{\egroup}}
\medskip
-\tokencyclexpress
-xQx x_x\z{}x%
-\endtokencyclexpress
+\expandafter\tokencyclexpress\tmp\endtokencyclexpress
+
+\medskip
+\catcode`Q=12
+\expandafter\tokencyclexpress\tmp\endtokencyclexpress
+
\end{exampleA}
-\end{minipage}
\endgroup
-\subsubsection{Changes to catcode 0}
+\subsection{Changes to catcode 0}
\begingroup