summaryrefslogtreecommitdiff
path: root/macros/generic/tokcycle/tokcycle-examples.tex
diff options
context:
space:
mode:
Diffstat (limited to 'macros/generic/tokcycle/tokcycle-examples.tex')
-rw-r--r--macros/generic/tokcycle/tokcycle-examples.tex593
1 files changed, 543 insertions, 50 deletions
diff --git a/macros/generic/tokcycle/tokcycle-examples.tex b/macros/generic/tokcycle/tokcycle-examples.tex
index 49fbe6c66c..891741bd4b 100644
--- a/macros/generic/tokcycle/tokcycle-examples.tex
+++ b/macros/generic/tokcycle/tokcycle-examples.tex
@@ -1,7 +1,7 @@
\documentclass[10pt]{article}
\usepackage[margin=1.3in]{geometry}
\usepackage[T1]{fontenc}
-\usepackage{tokcycle,xcolor,manfnt}
+\usepackage{tokcycle,xcolor,manfnt,lmodern}
\colorlet{bred}{red!90!blue}
\usepackage{tikz}
% MACROS: REDEFINE TO ALLOW \cytoks TO ESCAPE SCOPE OF exampleA
@@ -25,7 +25,6 @@
sidebyside,
title=#2,#1}
-
\newtcblisting{exampleB}[2][]{%
colframe=red!50!yellow!50!black,
colback=red!50!yellow!5!white,%white,
@@ -49,7 +48,8 @@
\def\altdecytoks{\par\medskip\begingroup\noindent\macname{cytoks}
\ \textit{alt}detokenization:\\\raggedright\footnotesize\ttfamily
- \expandafter\altdetokenize\expandafter{\the\cytoks}\par\endgroup}
+ \expandafter\altdetokenize\expandafter{\the\cytoks}\normalsize\par\endgroup}
+
\newcommand\TokCycle{\textsf{tokcycle}}
\parindent0pt
@@ -78,9 +78,10 @@
\newif\ifmacro
\newcommand\altdetokenize[1]{\begingroup\stripgroupingtrue\macrofalse
+ \stripimplicitgroupingcase{-1}%
\tokcycle
{\ifmacro\def\tmp{##1}\ifcat\tmp A\else\unskip\allowbreak\fi\macrofalse\fi
- \detokenize{##1}}
+ \detokenize{##1}\ifx##1\bgroup\unskip\fi\ifx##1\egroup\unskip\fi}
{\ifmacro\unskip\macrofalse\fi\{\processtoks{##1}\ifmacro\unskip\fi\}\allowbreak}
{\tctestifx{\\##1}{\\}{\ifmacro\unskip\allowbreak\fi
\allowbreak\detokenize{##1}\macrotrue}}
@@ -152,7 +153,7 @@
{\addcytoks{##1}}
\newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}}
\newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname
- \csname tcmapto#1\endcsname\else#1\fi}
+ \csname tcmapto#1\expandafter\endcsname\else\expandafter#1\fi}
\tcmapto am \tcmapto bf \tcmapto cz \tcmapto de \tcmapto ey
\tcmapto fl \tcmapto gx \tcmapto hb \tcmapto ic \tcmapto jn
\tcmapto ki \tcmapto lr \tcmapto mh \tcmapto nt \tcmapto ok
@@ -160,6 +161,12 @@
\tcmapto uj \tcmapto vp \tcmapto wd \tcmapto xg \tcmapto yu
\tcmapto zv
+\tokcycleenvironment\greek% NEW TOKCYCLE ENVIRONMENT:
+{\addcytoks[4]{\tcremap{##1}}}% HOW TO TREAT CHARACTERS
+{\processtoks{##1}}% HOW TO TREAT GROUP CONTENTS
+{\addcytoks{##1}}% HOW TO TREAT MACROS
+{\addcytoks{##1}}% HOW TO TREAT SPACES
+
\newcommand\findinstring[2]{\begingroup%
\stripgroupingtrue
\setcounter{runcount}{0}%
@@ -197,6 +204,72 @@
\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
{\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
+\newcounter{sigd}
+\newcounter{trailingz}
+\newcommand\sigdigits[1]{%
+ \setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT
+ \setcounter{trailingz}{0}% TRAILING ZERO COUNT
+ \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED
+ \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT
+ \tokcycle% CYCLE THROOUGH EACH TOKEN
+ {\tctestifx{.##1}%
+ {\if T\founddot\throwerr\fi\def\founddot{T}}% IF .
+ {\tctestifnum{`##1>`/}%
+ {\tctestifnum{`##1<`:}%
+ {\tctestifnum{##1>0}%
+ {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9
+ {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0
+ }%
+ {\throwerr}% IF ASCII > `9
+ }%
+ {\throwerr}% IF ASCII < `0
+ }%
+ }% APPLY ABOVE LOGIC FOR CHAR TOKENS
+ {\throwerr}% IF BRACES
+ {\throwerr}% IF CONTROL SEQUENCE
+ {}% IGNORE SPACES
+ {#1}% THE ARGUMENT
+ \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS
+ \expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}%
+ \relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT
+ \thesigd
+}
+\newcommand\throwerr{\addcytoks{##1}}% ADD INVALID TOKS TO \cytoks
+
+\newcounter{Tcount}
+\xtokcycleenvironment\capTive% ENVIRONMENT NAME
+{\tctestifx{t##1}{\stepcounter{Tcount}\addcytoks{T}}{\addcytoks{##1}}}% CHAR. DIRECTIVE
+{\processtoks{##1}}% GROUP DIRECTIVE
+{\addcytoks{##1}}% MACRO DIRECTIVE
+{\addcytoks{##1}}% SPACE DIRECTIVE
+{\setcounter{Tcount}{0}\color{red}\tcafterenv{\ Here}}% INITIALIZATION CODE
+{\theTcount{} t's capitalized: \addcytoks{!!}}% CLOSE-OUT CODE
+
+\newcounter{wordcount}
+\newcounter{lettercount}
+\newif\ifinword
+\newif\ifrunningcount
+\newcommand\dumpword{%
+ \addcytoks[1]{\accumword}%
+ \tctestifcon\ifinword{\stepcounter{wordcount}%
+ \ifrunningcount\addcytoks[x]{$^{\thewordcount,\thelettercount}$}\fi}{}%
+ \inwordfalse\def\accumword{}}
+\makeatletter
+\newcommand\addletter[1]{%
+ \tctestifcatnx A#1{\stepcounter{lettercount}\inwordtrue}{\dumpword}%
+ \tc@defx\accumword{\accumword#1}}
+\makeatother
+
+\xtokcycleenvironment\countem
+ {\addletter{##1}}
+ {\processtoks{##1}\dumpword}
+ {\dumpword\addcytoks{##1}}
+ {\dumpword\addcytoks{##1}}
+ {\def\accumword{}\setcounter{wordcount}{0}\setcounter{lettercount}{0}}
+ {\dumpword\ifrunningcount\else\tcafterenv{\par(Wordcount=\thewordcount,
+ Lettercount=\thelettercount)}\fi}
+
+
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
\section{Examples, examples, and more examples}
@@ -240,11 +313,13 @@ Default escape character is $|$. One can change it with
\begingroup
-\begin{exampleC}[]{The unexpandable \macname{plusl} macro}
+\begin{exampleC}[]{The unexpandable \macname{plusl} macro to increase
+ the ASCII value of tokens by 1}
\newcommand\plusl[1]{\char\numexpr`#1+1\relax}
\end{exampleC}
-\begin{exampleA}[lefthand width=5cm]{Escaping text in the input stream}
+\begin{exampleA}[lefthand width=5cm]{Escaping text and macro parameters
+ in the input stream}
\tokcycle
{\addcytoks{\plusl{#1}}}
{\processtoks{#1}}
@@ -265,9 +340,9 @@ Default escape character is $|$. One can change it with
\begingroup
This section concerns the issue of whether the characters of the input
stream are transformed before or after being placed in the
- output token register (\macname{cytoks}).\medskip
+ output token list (\macname{cytoks}).
-\noindent\begin{minipage}[t]{\textwidth}
+\medskip
Transform characters (+1 ASCII) via unexpandable macro:\medskip
\begin{exampleA}[lefthand width=8.3cm]
@@ -283,8 +358,11 @@ Transform characters (+1 ASCII) via unexpandable macro:\medskip
\altdecytoks
+Note how the above detokenization shows the actual transformation
+ instructions involving \macname{plusl}.
+This is done because \macname{plusl} is not expandable.
+
\par\noindent\hrulefill\medskip
-\end{minipage}
\endgroup
\noindent\begin{minipage}[t]{\textwidth}
@@ -316,6 +394,12 @@ Capitalize vowels (but don't expand the character directive)
\altdecytoks
+\medskip
+The above detokenization also shows the transformation macro,
+ \macname{vowelcap}, as part of \macname{cytoks}.
+But because \macname{vowelcap} is expandable, we did not need to
+ leave it at this stage, as the next example shows.
+
\par\noindent\hrulefill\medskip
\endgroup
\end{minipage}
@@ -336,6 +420,9 @@ Capitalize vowels (expanding the character directive)
\altdecytoks
+\medskip
+Here, we see no vestiges of \macname{vowelcap} in \macname{cytoks},
+ but only the transformed result.
\endgroup
\subsubsection{Unexpanded vs. pre-expanded input stream}
@@ -359,6 +446,10 @@ Capitalize vowels (expanding the character directive)
\altdecytoks
+\medskip
+In the above case, vowel-capping is not applied to the date, because
+ \macname{today} remains unexpanded in the \macname{cytoks} token list.
+
\noindent\hrulefill\strut
\end{minipage}
@@ -377,6 +468,14 @@ Note that, when pre-expanding the input stream, one must
\altdecytoks
+\medskip
+Here, we see the date subjected to vowel-capping, because the \TokCycle{}
+ input was expanded prior to the token cycle.
+Thus \macname{today} no longer appears \textit{explicitly} in the
+ \macname{cytoks} token list.
+Its pre-expansion into character tokens is now subject to the
+ \macname{vowelcap}.
+
\endgroup
\subsection{Grouping}
@@ -442,7 +541,7 @@ These are embodied in the if-condition \macname{ifstripgrouping}
Regardless of which condition is set, the tokens within the explicit
group are still passed to the Group directive for processing.
-
+\medskip
Permutations of the following code are used in the subsequent examples.
Group stripping, brought about by \macname{stripgroupingtrue},
involves removing the grouping braces from around the group.
@@ -451,6 +550,7 @@ The choice of \macname{processtoks} vs.\@ \macname{addcytoks}
to \TokCycle{} for processing, or are merely sent to the output
register in their original unprocessed form.
+\medskip
Note that, in these examples, underdots and visible spaces
will only appear on characters and spaces that have been directed
to the Character and Space directives, respectively.
@@ -547,15 +647,73 @@ Note that the content of groups can be altogether eliminated if
{1{{3{{5{{7{{9{1{}0}}8}}6}}4}}2}}}
\the\cytoks
\end{exampleA}
+Note, above, that \TokCycle{} does not rely on an argument-based nesting
+ algorithm that would otherwise be limited to 9 embedded groups.
+\endgroup
+
+\subsubsection{The use of \macname{groupedcytoks} for more sophisticated
+ group processing}
+
+\begingroup
+
+\begin{sloppypar}
+By default, \TokCycle{} processes groups such that whatever additions to the
+ \macname{cytoks} token list you accomplish in the Group directive will be
+ encased in an explicit grouped scope.
+This is the token-cycle behavior with the default \macname{stripgroupingfalse}
+ declaration.
+However, for more sophisticated tasks associated with the processing of
+ group content, one may override the default (by way of
+ \macname{stripgroupingtrue}) and manually dictate the groupings associated
+ with the Group directive, with the use of \macname{groupedcytoks\{...\}}.
+\end{sloppypar}
+
+\medskip In the following example, tasks A and D are performed \textit{outside}
+ of the explicit grouping otherwise introduced with \macname{groupedcytoks},
+ whereas tasks B and C fall inside of that explicit group.
+The fact that, in the output, \texttt{d} is not italic confirms the
+ fact that task D occurs after the conclusion of the explicit \macname{cytoks}
+ group that encompasses the italic `bxc'.
+The fact that none of \texttt{a}, \texttt{b}, \texttt{c}, nor \texttt{d} are
+ encased in an \macname{fbox} indicates that they were not processed through
+ the Character directive, but arose from direct additions to \macname{cytoks},
+ by way of their tasks.
+
+\begin{exampleA}[lefthand width=10cm]{Use of the \macname{groupedcytoks}
+ macro}
+\stripgroupingtrue
+\Characterdirective{\addcytoks{\fbox{#1}}}
+\Groupdirective{\taskA%
+ \groupedcytoks{\taskB\processtoks{#1}\taskC}%
+ \taskD}
+\def\taskA{\addcytoks{\bgroup\LARGE a}}
+\def\taskB{\addcytoks{\itshape b}}
+\def\taskC{\addcytoks{c}}
+\def\taskD{\addcytoks{d\egroup}}
+
+\tokcyclexpress{x\textbf{x}x}
+\the\cytoks
+\end{exampleA}
+
+The final disposition of the \macname{cytoks} token list shows the effects of
+ \macname{stripgroupingtrue} and \macname{groupedcytoks} more explicitly:
+
+\altdecytoks
+
+\medskip
+Without \macname{stripgroupingtrue}, all tasks, A,B,C, and D, would have
+ been contained within the explicit \texttt{\{...\}} group.
\endgroup
+
\subsection{Direct use of \TokCycle}
-\TokCycle{} (in regular or \texttt{xpress} form) may be invoked
+The \TokCycle{} macros and pseudo-environments
+ (in regular or \texttt{xpress} form) may be invoked
directly from the document, without
- being first encapsulated within a macro or environment.
+ being first encapsulated within a separate macro or environment.
\subsubsection{Modifying counters as part of the Character
directive}
@@ -563,7 +721,7 @@ Note that the content of groups can be altogether eliminated if
\begingroup
\begin{exampleA}[lefthand width=8.5cm]{Using a period token (.) to
- reset a changing color}
+ reset a changing color every sentence}
\restorecolor
\tokencycle
{\addcytoks{\bgroup\reducecolor{3}#1\egroup}%
@@ -610,6 +768,9 @@ is long.} Now, it is short.
Back to regular text.
\end{exampleA}
+Note that embedded macros such as \macname{textit} and \macname{textbf}
+ do not adversely affect the capability of the \macname{spaceouttext}
+ macro.
\endgroup
@@ -623,7 +784,7 @@ It is \textbf{not} to be confused as a replacement for
In certain applications, it may offer a more pleasingly formatted
typesetting of detokenized material.
-It is an unusual application of \TokCycle{} in that it does not
+\medskip It is an unusual application of \TokCycle{} in that it does not
actually use the \macname{cytoks} token register to collect
its output.
This is only possible because all macros in the input stream
@@ -634,9 +795,10 @@ This is only possible because all macros in the input stream
\begin{exampleC}[]{The \macname{altdetokenize }\ macro}
\newif\ifmacro
\newcommand\altdetokenize[1]{\begingroup\stripgroupingtrue\macrofalse
+ \stripimplicitgroupingcase{-1}%
\tokcycle
{\ifmacro\def\tmp{##1}\ifcat\tmp A\else\unskip\allowbreak\fi\macrofalse\fi
- \detokenize{##1}}
+ \detokenize{##1}\ifx##1\bgroup\unskip\fi\ifx##1\egroup\unskip\fi}
{\ifmacro\unskip\macrofalse\fi\{\processtoks{##1}\ifmacro\unskip\fi\}\allowbreak}
{\tctestifx{\\##1}{\\}{\ifmacro\unskip\allowbreak\fi
\allowbreak\detokenize{##1}\macrotrue}}
@@ -652,7 +814,7 @@ This is only possible because all macros in the input stream
\string\altdetokenize: \\
\texttt{\altdetokenize{a\mac a \mac2 {\mac}\mac{a\mac\mac}\mac}}!
-\string\detokenize: \\
+versus \string\detokenize: \\
\texttt{\detokenize{a\mac a \mac2 {\mac}\mac{a\mac\mac}\mac}}!
\end{exampleA}
\endgroup
@@ -786,12 +948,92 @@ This is only possible because all macros in the input stream
\endgroup
+\subsubsection{Counting the significant digits in a number}
+
+\begingroup
-\subsection{\TokCycle-based environments}
+The problem of counting the significant digits in a number is an interesting
+ one, in that leading and trailing zeros are not significant, but zeros
+ in the middle of the number are significant.
+In a problem like this, the \macname{cytoks} token list is not used to
+ carry a transformation of the input tokens, but only to carry a record
+ of errant tokens from the input (as seen from the helper macro
+ \macname{throwerr}).
+If \macname{the\textbackslash cytoks} is anything other than empty when the
+ macro completes, the macro will set the significant-digit counter
+ \texttt{sigd} to a value of -1 to signify an error condition.
+
+\medskip
+The macro expects its input to comprise digits in the 0--9 range, an [optional]
+ single decimal point, and ignorable spaces. Any other token will throw
+ an error.
+
+\medskip
+The main work of the \macname{sigdigits} macro is to use the counters
+ \texttt{sigd} to count significant digits and \texttt{trailingz} to
+ tabulate the number of trailing-zeros with which to decrement
+ \texttt{sigd}.
+
+\begin{exampleC}[]{The \macname{sigdigits} macro for counting significant digits}
+\newcounter{sigd}
+\newcounter{trailingz}
+\newcommand\sigdigits[1]{%
+ \setcounter{sigd}{0}% SIGNIFICANT DIGIT COUNT
+ \setcounter{trailingz}{0}% TRAILING ZERO COUNT
+ \def\founddot{F}% MADE T WHEN DECIMAL HAS BEEN 1ST LOCATED
+ \def\zerosig{F}% MADE T WHEN ZERO BECOMES SIGNIFICANT
+ \tokcycle% CYCLE THROOUGH EACH TOKEN
+ {\tctestifx{.##1}%
+ {\if T\founddot\throwerr\fi\def\founddot{T}}% IF .
+ {\tctestifnum{`##1>`/}%
+ {\tctestifnum{`##1<`:}%
+ {\tctestifnum{##1>0}%
+ {\stepcounter{sigd}\def\zerosig{T}\setcounter{trailingz}{0}}% IF 1-9
+ {\if T\zerosig\stepcounter{sigd}\fi\stepcounter{trailingz}}% IF 0
+ }%
+ {\throwerr}% IF ASCII > `9
+ }%
+ {\throwerr}% IF ASCII < `0
+ }%
+ }% APPLY ABOVE LOGIC FOR CHAR TOKENS
+ {\throwerr}% IF BRACES
+ {\throwerr}% IF CONTROL SEQUENCE
+ {}% IGNORE SPACES
+ {#1}% THE ARGUMENT
+ \addtocounter{sigd}{-\thetrailingz}% SUBTRACT TRAILING ZEROS
+ \expandafter\ifx\expandafter\relax\detokenize\expandafter{\the\cytoks}%
+ \relax\else\setcounter{sigd}{-1}\fi% CHECK FOR INVALID INPUT
+ \thesigd
+}
+\newcommand\throwerr{\addcytoks{##1}}% ADD INVALID TOKS TO \cytoks
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=8.0cm]{Example use of the \macname{sigdigits}
+ macro}
+1) \sigdigits{12.3}\\
+2) \sigdigits{0.123}\\
+3) \sigdigits{1.2.3}\\
+4) \sigdigits{1.3\today}\\
+5) \sigdigits{321.345 678}\\
+6) \sigdigits{000321.305}\\
+7) \sigdigits{.000 300 345}\\
+8) \sigdigits{0003x1.345}\\
+9) \sigdigits{123000}\\
+A) \sigdigits{123.000}\\
+B) \sigdigits{0003;345}
+\end{exampleA}
+
+\endgroup
+
+
+\subsection{Creating your own \TokCycle-based environments}
The \macname{tokcycleenvironment} macro allows users to define
their own \TokCycle{} environments.
-Here are some examples.
+The more recently introduced \macname{xtokcycleenvironment} macro extends
+ that capability to allow initialization and postprocessing when defining
+ a named \TokCycle{} environment.
+Here are some examples of both types.
\subsubsection{``Removing'' spaces, but still breakable/hyphenatable}
@@ -805,7 +1047,8 @@ Here are some examples.
{\addcytoks{\hspace{.2pt plus .2pt minus .8pt}}}%
\end{exampleC}
-\begin{exampleA}[lefthand width=6.0cm]{}
+\begin{exampleA}[lefthand width=6.0cm]{Example of using the \macname{spaceBgone}
+ environment}
\spaceBgone
Here we have a \textit{test} of
whether the spaces are removed.
@@ -834,7 +1077,7 @@ Here are some examples.
{\addcytoks{##1}}
\newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}}
\newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname
- \csname tcmapto#1\endcsname\else#1\fi}
+ \csname tcmapto#1\expandafter\endcsname\else\expandafter#1\fi}
\tcmapto am \tcmapto bf \tcmapto cz \tcmapto de \tcmapto ey
\tcmapto fl \tcmapto gx \tcmapto hb \tcmapto ic \tcmapto jn
\tcmapto ki \tcmapto lr \tcmapto mh \tcmapto nt \tcmapto ok
@@ -859,11 +1102,259 @@ Because \macname{tcremap} is expandable, the original text is totally absent
\endgroup
\end{minipage}
+\bigskip
+\begingroup
+
+This approach can be used, for example, to temporarily remap Latin into Greek in
+ math mode.
+In the example given below, only several of the Latin letters are remapped
+ for demonstration purposes, but others can be added.
+Also, remember that one can use the \verb+|...|+ delimiters to escape back to
+ Latin within the environment.
+
+\begin{exampleC}[]{The \macname{greek} environment with
+ supporting macros}
+\tokcycleenvironment\greek% NEW TOKCYCLE ENVIRONMENT:
+{\addcytoks[4]{\tcremap{##1}}}% HOW TO TREAT CHARACTERS
+{\processtoks{##1}}% HOW TO TREAT GROUP CONTENTS
+{\addcytoks{##1}}% HOW TO TREAT MACROS
+{\addcytoks{##1}}% HOW TO TREAT SPACES
+\newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}}
+\newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname
+ \csname tcmapto#1\expandafter\endcsname\else\expandafter#1\fi}
+\tcmapto a\alpha \tcmapto b\beta \tcmapto e\epsilon
+\tcmapto g\gamma \tcmapto p\pi
+\end{exampleC}
+
+\tcmapto a\alpha \tcmapto b\beta \tcmapto e\epsilon
+\tcmapto g\gamma \tcmapto p\pi
+
+\begin{exampleA}[lefthand width=8.5cm]{Demo of \macname{greek}}
+\[
+ y =
+ \greek a^2b\frac{2e}{2 + p^|p| + g}\endgreek
+ + x
+\]
+\end{exampleA}
+
+\endgroup
+
+\subsubsection{Creating an ``extended'' \TokCycle{} environment}
+
+\begingroup
+The \macname{xtokcycleenvironment} macro extends the capability of
+ \macname{tokcycleenvironment} by adding two arguments that correspond,
+ respectively, to initialization code and ``close-out'' code that is run
+ after the token cycle is complete (but before it is typeset).
+Let's say I wanted an environment that capitalizes instances of the letter
+ `\texttt{t}', but I want it to tell me how many
+ letter \texttt{t}'s are affected before typesetting the result.
+Furthermore, I want it always to set the result in red and add two
+ exclamation points to the end of the environment output.
+
+\begin{exampleC}[]{The \macname{capTive} extended-environment definition}
+\newcounter{Tcount}
+\xtokcycleenvironment\capTive% ENVIRONMENT NAME
+{\tctestifx{t##1}{\stepcounter{Tcount}\addcytoks{T}}{\addcytoks{##1}}}% CHAR. DIRECTIVE
+{\processtoks{##1}}% GROUP DIRECTIVE
+{\addcytoks{##1}}% MACRO DIRECTIVE
+{\addcytoks{##1}}% SPACE DIRECTIVE
+{\setcounter{Tcount}{0}\color{red}\tcafterenv{\ Here}}% INITIALIZATION CODE
+{\theTcount{} t's capitalized: \addcytoks{!!}}% CLOSE-OUT CODE
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=7.9cm]{Demo of \macname{capTive}
+ environment}
+Before \capTive the cytoks \footnotesize token list\endcapTive After
+\end{exampleA}
+
+In the initialization code, the \texttt{Tcount} counter is reset and the color
+ of what follows is made red.
+It also makes use of the specialized \macname{tcafterenv} macro that can
+ \textbf{only be used in \macname{xtokcycle\-environment} definitions}.
+This macro indicates code to execute \textit{after} the \texttt{capTive}
+ environment scope is exited.
+This invocation typesets the word `Here'.
+We see that in the output, it occurs at normal size and in black, because
+ it is no longer subject to the color and fontsize that was left inside
+ the environment.
+The \macname{tcafterenv} macro is unique in that it is an appending, rather
+ than a defining, macro.
+Therefore, multiple invocations of it will append multiple code segments,
+ each of which will be executed upon environment exit.
+It may be invoked in any of the directives, as well as the initialization and
+ close-out code segments.
+
+\medskip
+For the close-out code, because it is executed after the token cycle is
+ complete, but before it is typeset, the updated value of \texttt{Tcount}
+ is available.
+Anything typeset directly in the close-out code will occur before the
+ \macname{cytoks} token-list output is typeset.
+Thus, the ``...t's capitalized.'' phrase is typeset before \macname{cytoks}
+ itself, even as \macname{theTcount} is accurate.
+The \macname{addcytoks} occurring in the close-out code is placed at the
+ end of \macname{cytoks} and thus takes on the \macname{footnotesize}
+ declaration that is still in force.
+\endgroup
+
+\begingroup
+\medskip
+Next, we consider a \TokCycle{} extended environment that counts words and
+ letters.
+We try to build some features into the code so that, for example, spaces
+ and punctuation do not count as letters.
+Further, we try to build the logic so that the following cases would
+ count as two words, not one: ``here...now'', ``Case(number)''.
+
+\medskip
+We provide two modes of output: one that provides an ending summary, and
+ the other that keeps a running count.
+As with other \TokCycle{} environments and macros, portions can be
+ exempted from the count (for example macro arguments) by using the
+ escape delimiters provided.
+
+\begin{exampleC}[]{The \macname{countem} extended-environment definition
+ and supporting macros}
+\newcounter{wordcount}
+\newcounter{lettercount}
+\newif\ifinword
+\newif\ifrunningcount
+\newcommand\dumpword{%
+ \addcytoks[1]{\accumword}%
+ \tctestifcon\ifinword{\stepcounter{wordcount}%
+ \ifrunningcount\addcytoks[x]{$^{\thewordcount,\thelettercount}$}\fi}{}%
+ \inwordfalse\def\accumword{}}
+\makeatletter
+\newcommand\addletter[1]{%
+ \tctestifcatnx A#1{\stepcounter{lettercount}\inwordtrue}{\dumpword}%
+ \tc@defx\accumword{\accumword#1}}
+\makeatother
+
+\xtokcycleenvironment\countem
+ {\addletter{##1}}
+ {\processtoks{##1}\dumpword}
+ {\dumpword\addcytoks{##1}}
+ {\dumpword\addcytoks{##1}}
+ {\def\accumword{}\setcounter{wordcount}{0}\setcounter{lettercount}{0}}
+ {\dumpword\ifrunningcount\else\tcafterenv{\par(Wordcount=\thewordcount,
+ Lettercount=\thelettercount)}\fi}
+\end{exampleC}
+
+\begin{exampleA}[lefthand width=5.6cm]{Demo of the two modes of
+ \macname{countem}}
+\countem
+This is a test.
+
+But...punctuation does not count.
+\endcountem
+
+\bigskip\runningcounttrue
+\countem
+This is a test.
+
+But...punctuation does not count.
+\endcountem
+\end{exampleA}
+
+\endgroup
+
+\subsection{Truncation of input}
+
+\begingroup
+
+\TokCycle{} has several facilities for truncating input, either
+ to the end of the current group or to the end of the input stream.
+The basic commands are \macname{truncategroup} and \macname{truncatecycle},
+ which can be part of the Character, Macro, or Space directives (see the
+ package documentation for truncation from the Group directive).
+
+\medskip
+These commands may be issued directly or they may be placed as
+ one fork of a conditional test.
+If the conditional test is the occurrence of a particular token in the
+ input stream, special conditional shortcuts have been formulated:
+ \macname{truncategroupiftokis} and \macname{truncatecycleiftokis}.
+Both of these commands take two arguments: the first is the comparison
+ token against which to compare the tokens of the input stream, and the
+ second is the directive to follow for tokens that fail the
+ comparison test.
+If the input stream token matches the comparitor, the relevant
+ truncation command is executed.
+
+\medskip
+The description below shows the truncations to a given input text,
+ resulting from several different forms of truncating directives.
+In the first case, the token cycle is to be truncated if a space
+ is found in the input stream.
+Comparing to the input text, we see that the space occurs after the
+ word ``until'', and that is indeed where the \TokCycle{} output
+ ceases.
+
+\medskip
+In the second case, the truncation is limited to the current group,
+ rather than the remainder of the input stream.
+Again, the space following ``until'' is found, and the words
+ ``certain token'' are excised from the input stream until the
+ end-of-group is located.
+The cycle continues successfully with ``-is'' until the next space
+ is found before ``located''.
+Since this space is found in the top grouping level of the input,
+ the remainder of the input stream is thereafter truncated.
+
+\medskip
+In the third case, the cycle is to be truncated in the event the
+ letter `a' is found in the input stream.
+Examining the input text, the first `a' is in the word ``certain'' and
+ so the input is truncated after the string ``cert'' is digested.
+
+\medskip
+The final case, calls for group truncation if an `a' is found in the
+ input stream.
+As before, the first `a' is found after ``cert''.
+However, in the group case, only the remaining group content, consisting
+ of ``ain token'', is discarded.
+At group end, the cycle successfully resumes until the `a' in
+ ``located'' is reached.
+Thus, ``-is loc'' is also added to the output stream.
+
+\def\thetext{Continue-\textit{until certain token}-is located.}
+
+\newcommand\rundemowith[1]{%
+ {\footnotesize\ttfamily\altdetokenize{#1}}:
+ \resettokcycle#1\expandafter\tokencyclexpress\thetext\endtokencyclexpress
+}
+
+\begin{exampleB}[text only, righthand width=0.01cm, colback=white]%
+{Effects of various truncation directives}
+{\footnotesize\ttfamily Input text}: \expandafter\altdetokenize\expandafter
+ {\thetext}
+
+\rundemowith{\Spacedirective{\truncatecycle}}
+
+\rundemowith{\Spacedirective{\truncategroup}}
+
+\rundemowith{\Characterdirective{\truncatecycleiftokis{a}{\addcytoks{#1}}}}
+
+\rundemowith{\Characterdirective{\truncategroupiftokis{a}{\addcytoks{#1}}}}
+\end{exampleB}
+
+Note how the macro function, in this case \macname{textit}, is not
+ disrupted by the truncation.
+Even as tokens are truncated from the input stream, group closures
+ are retained so that the final output is group-balanced.
+For example, a detokenization of the final case reveals this to be
+ the case:
+
+\altdecytoks
+
+\endgroup
+
\subsection{Advanced topics: implicit tokens and catcode changes}
\subsubsection{Trap Active Characters (catcode 13)}
-\begin{minipage}[t]{\textwidth}
+%\begin{minipage}[t]{\textwidth}
\begingroup
Active characters in the \TokCycle{} input stream are processed in
their original form.
@@ -886,26 +1377,27 @@ For information on how to process active spaces, please refer to
\begin{exampleA}[lefthand width=7.8cm]{Processing active characters}
\resettokcycle
\tokencyclexpress
-This is a test!!\endtokencyclexpress
+1. This is a test!!\endtokencyclexpress
-\catcode`!=\active
-\def !{?}
+2. \catcode`!=\active
+\def !{?}%
\tokencyclexpress
This is a test!!\endtokencyclexpress
+3.
\Characterdirective{\tctestifcon\ifactivetok
- {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}}
+ {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}}%
\Macrodirective{\tctestifcon\ifactivetok
- {\addcytoks{\fbox{#1-mac}}}{\addcytoks{#1}}}
+ {\addcytoks{\fbox{#1-mac}}}{\addcytoks{#1}}}%
\tokencyclexpress
This is a test!!\endtokencyclexpress
-\catcode`T=\active
-\let T+
+4. \catcode`T=\active
+\let T+%
\tokencyclexpress
This is a test!!\endtokencyclexpress
-\detokenize\expandafter{\the\cytoks}
+5. \detokenize\expandafter{\the\cytoks}
\end{exampleA}
%%%
@@ -916,15 +1408,16 @@ This is a test!!\endtokencyclexpress
are made before reaching \TokCycle{} processing.
\svt hey are, thus, no longer detected as active, unless
\macname{noexpand} is applied before the pre-expansion.
-In this example, the \texttt{!} that is not \macname{noexpand}\kern.3pted
- is converted to a \texttt{?} prior to reaching \TokCycle{}
- processing (and thus, not detected as \macname{active}):
+In this example, the `\texttt{!}' that is not \macname{noexpand}\kern.3pted
+ is converted to a `\texttt{?}' prior to reaching \TokCycle{}
+ processing (and thus, not detected as \macname{active}).
+Likewise, the `\texttt{T}' is pre-converted to a `\texttt{+}':
\catcode`!=\active
\def !{?}
\catcode`T=\active
\let T+
-\begin{exampleA}[lefthand width=7.8cm]{Expanded input stream acts
+\begin{exampleA}[lefthand width=7.0cm]{Expanded input stream acts
upon active \macname{def}ed characters unless \macname{noexpand} is applied}
\expandedtokcyclexpress{This is a test!\noexpand!}
\the\cytoks\par
@@ -942,7 +1435,7 @@ However, pre-tokenization does not suffer this behavior:
%%%
\endgroup
-\end{minipage}
+%\end{minipage}
\begingroup
\ddbend One aspect of \TeX{} to remember is that catcodes
@@ -955,6 +1448,7 @@ But if the cat-13 active definition is changed prior to the
execution of the \macname{def}'ed token, the revised
token assignment will apply.
+\medskip
The following example demonstrates this concept, while showing,
without changing the input in any way, that \TokCycle{}
can properly digest active and implicit grouping (cat-1,2) characters:
@@ -996,6 +1490,7 @@ In this manner, \macname{ifcatSIX} can be used inside the
Character directive to convert cat-6 tokens into something of
the user's choosing.
+\medskip
As to this cat-12 conversion, explicit cat-6 characters are converted
into the same character with cat-12.
On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|)
@@ -1003,6 +1498,7 @@ On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|)
whose cat-12 substitution text is a \macname{string} of the original
implicit-macro name.
+\begingroup
\resettokcycle
\Characterdirective{\ifcatSIX
\addcytoks{\fbox{#1}}
@@ -1045,6 +1541,7 @@ On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|)
\catcode`Q=11
\hrulefill
+\endgroup
\ddbend For what is, perhaps, a rare situation, one can even process input
streams that contain cat-6 macro parameters.
@@ -1054,10 +1551,6 @@ A package macro, \macname{whennotprocessingparameter\#1\{<}%
In this example, a macro is defined and then executed, subject to
token replacements brought about by the expandable Character directive.
-\Characterdirective{%
- \whennotprocessingparameter#1{\addcytoks[x]{\vowelcap{#1}}}}
-\aftertokcycle{\global\cytoks\expandafter{\the\cytoks}}
-\def\Characterdirective#1{}
\begin{exampleA}[lefthand width=7cm]{Preserving parameters (e.g.
\texttt{\#1, \#2}) in the \TokCycle{} input stream}
\Characterdirective{%
@@ -1172,6 +1665,7 @@ Changing grouping tokens (catcodes 1,2) may require something more,
In the following examples, pay attention to the
detokenized grouping around the argument to \macname{fbox}.
+\medskip
As we will see, the issues raised here only affect the situation
when detokenization of the output stream is required.
\medskip
@@ -1336,21 +1830,13 @@ The next two examples introduce implicit and active-implicit spaces.
The latter example also demonstrates detokenization of such spaces.
\medskip
-Here is a useful macro for detokenizing space tokens in the context of
- \TokCycle{}.
+Here is a useful expandable macro for detokenizing space tokens
+ in the context of \TokCycle{}.
It can process not only explicit cat-10 spaces, but also implicit and
active-implicit spaces.
To add it to the \macname{cytoks} token list, make sure you use the
\verb|[x]| fully expanded option to \macname{addcytoks}.
-\medskip
-\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling
- implicit and active-implicit spaces}
-\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
- {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
-\end{exampleC}
-\medskip
-
%%%
\resettokcycle
\Characterdirective{\addcytoks{\underdot{#1}}}
@@ -1367,6 +1853,13 @@ To add it to the \macname{cytoks} token list, make sure you use the
\endtokencyclexpress
\end{exampleA}
+\begin{minipage}[t]{\textwidth}
+\begin{exampleC}[]{The \macname{detokenizespacetok} macro for handling
+ implicit and active-implicit spaces}
+\newcommand\detokenizespacetok[1]{\tctestifcon{\ifimplicittok}%
+ {\tctestifcon{\ifactivetok}{\theactivespace}{\string#1}}{\detokenize{#1}}}
+\end{exampleC}
+
\begin{exampleA}[lefthand width=10cm]{Active Implicit spaces work{,} too}
\resettokcycle
\Characterdirective{\addcytoks{\underdot{#1}}}
@@ -1389,7 +1882,7 @@ xQx x_x\z{}x%
xQx x_x\z{}x%
\endtokencyclexpress
\end{exampleA}
-
+\end{minipage}
\endgroup