summaryrefslogtreecommitdiff
path: root/macros/generic/tokcycle/tokcycle-examples.tex
diff options
context:
space:
mode:
Diffstat (limited to 'macros/generic/tokcycle/tokcycle-examples.tex')
-rw-r--r--macros/generic/tokcycle/tokcycle-examples.tex272
1 files changed, 223 insertions, 49 deletions
diff --git a/macros/generic/tokcycle/tokcycle-examples.tex b/macros/generic/tokcycle/tokcycle-examples.tex
index b73e89168c..ec63d48f94 100644
--- a/macros/generic/tokcycle/tokcycle-examples.tex
+++ b/macros/generic/tokcycle/tokcycle-examples.tex
@@ -50,20 +50,111 @@
\def\altdecytoks{\par\medskip\begingroup\noindent\macname{cytoks}
\ \textit{alt}detokenization:\\\raggedright\footnotesize\ttfamily
\expandafter\altdetokenize\expandafter{\the\cytoks}\par\endgroup}
-\newcommand\TokCycle[1][t]{\textsf{#1okcycle}}
+\newcommand\TokCycle{\textsf{tokcycle}}
\parindent0pt
\parskip0pt
\begin{document}
{\centering\LARGE \TokCycle{} Package Examples\par\smallskip
-\large Steven B. Segletes\\\today\par}
+\large\today\par}
\tableofcontents
\makeatletter
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
-
+% ALREADY PLACED IN tokcycle.tex
+%
+%\def\trapactivetok#1{\tctestifcatnx~#1{\activetoktrue}{\trapactivelet#1}}
+%
+%% WILL ALSO TRAP ACTIVE \let TO PRIMITIVES AS IMPLICIT; UNDO LATER IN \can@absorb@@
+%\def\trapactivelet#1{\tctestifcon{%
+% \expandafter\if\detokenize{#1}#1}{}{\activetoktrue\implicittoktrue}}
+%
+%\makeatletter
+%\long\def\backslashcmds#1{%
+% \test@ifmacro#1{\tctestifcon\ifcatSIX{\implicittoktrue\@chrT#1}{\@macT#1}}%
+% {\implicittoktrue\trapimplicitegrp#1\tctestifcon{\iftc@implicitgrp}%
+% {\ifcase\@implicitgroupingcase\or\addcytoks{#1}\or\@chrT{#1}\fi}%
+% {\@chrT#1}}}
+%
+%\long\def\can@absorb@#1{\tctestifnum{\count@stringtoks{#1}>1}%
+% {\tctestifx{\endtokcycraw#1}{#1}{\backslashcmds#1\@tokcycle}}%
+% {\trapactivetok#1\tc@trapescape#1{\tc@escapecytoks}{\can@absorb@@#1}}}
+%
+%\long\def\can@absorb@@#1{\let\@tmp#1\test@ifmacro\@tmp
+% {\implicittokfalse\@macT#1}{\@chrT#1}\@tokcycle}
+%
+%\long\def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt}
+%\long\def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}}
+%
+% END ALREADY PLACED IN tokcycle.tex
+
+% REVISED ACTIVE TOK TRAP
+%\newif\ifactivechar
+%\newif\ifactivetokunexpandable
+
+%\def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse
+% \catSIXfalse\activecharfalse\activetokunexpandablefalse}
+
+%\long\def\can@absorb@#1{\tctestifnum{\count@stringtoks{#1}>1}%
+% {\tctestifx{\endtokcycraw#1}{#1}{\backslashcmds#1\@tokcycle}}%
+% {\trapactives#1\tc@trapescape#1{\tc@escapecytoks}{\can@absorb@@#1}}}
+
+% \activechartrue: \catcode`X=\active AT TIME OF TEST
+% \activetoktrue: \catcode`X=\active AT TIME OF X's TOKENIZATION
+% \activetokunexpandabletrue: \catcode`X=\active WHEN TOKENIZED AND
+% \let XY or \let X\primitive AT TIME OF TEST
+% \implicittoktrue \let(X or \X)Y WHERE Y IS NOT cat-0
+
+%\def\trapactives#1{\trapactivechar{#1}\trapactivetok{#1}}
+%\def\trapactivechar#1{\tctestifnum{\number\catcode`#1=13}{\activechartrue}{}}
+%\def\trapactivetok#1{\tctestifcatnx~#1{\activetoktrue}{%
+% \trapactivetokunexpandable#1}}
+%%% WILL ALSO TRAP ACTIVE \let TO PRIMITIVES AS IMPLICIT; UNDO LATER IN \can@absorb@@
+%\def\trapactivetokunexpandable#1{\tctestifcon{\expandafter\if
+% \detokenize{#1}#1}{}{\activetoktrue\activetokunexpandabletrue\implicittoktrue}}
+% END REVISED ACTIVE TOK TRAP
+
+% ACTIVE IMPLICIT CAT-6 TRAP
+%\def\trapcatSIXc#1{\tctestifnum{\count@stringtoks{#1}>1}{\tc@defx\six@str{\string#1}%
+% \global\let\implicitsixtok\six@str\tc@tok{\implicitsixtok}}%
+% {\tc@tok\expandafter{\string#1}\tctestifnum{\number\catcode`#1=6}%
+% {}{\activetoktrue\implicittoktrue}}}
+% END ACTIVE IMPLICIT CAT-6 TRAP
+
+% ACTIVE CHAR \bgroup
+%\def\@@@grpT#1{\tctestifnum{\number\catcode`#1=1}%
+% {\expandafter\@@grpT\expandafter{\iffalse}\fi}% {
+% {\implicittoktrue\tc@implicitgrptrue%
+% \tctestifnum{`#1=92}% WORKS EVEN IF CAT-0 HAS CHANGED
+% {\csmk{\expandafter\backslashcmds\thecs\@tokcycle}}% \bgroup
+% {\begingroup\catcode`#1=\active \xdef\@tmp{\scantokens{#1\noexpand}}\endgroup
+% \expandafter\@chrT\@tmp\@tokcycle}% ACTIVE CHAR \bgroup
+% }%
+%}
+% END ACTIVE CHAR \bgroup
+
+%%%%%%%%%%TRAPS ACTIVE-IMPLICIT GROUP TOKS FOR \stripimplicitgroupingcase
+%\def\implicitgrpfork#1{\tctestifcon{\iftc@implicitgrp}{\ifcase
+% \@implicitgroupingcase\or\addcytoks{#1}\or\@chrT{#1}\fi}{\@chrT#1}}
+
+%\def\@@@grpT#1{\tctestifnum{\number\catcode`#1=1}%
+% {\expandafter\@@grpT\expandafter{\iffalse}\fi}% {
+% {\implicittoktrue\tc@implicitgrptrue%
+% \tctestifnum{`#1=92}% WORKS EVEN IF CAT-0 HAS CHANGED
+% {\csmk{\expandafter\backslashcmds\thecs\@tokcycle}}% \bgroup
+% {\begingroup\catcode`#1=\active \xdef\@tmp{\scantokens{#1\noexpand}}\endgroup
+% \expandafter\implicitgrpfork\@tmp\@tokcycle}% ACTIVE CHAR \bgroup
+% }}
+
+%\long\def\can@absorb@@#1{\let\@tmp#1\test@ifmacro\@tmp{\implicittokfalse
+% \@macT#1}{\trapimplicitegrp#1\implicitgrpfork#1}\@tokcycle}
+
+%\long\def\backslashcmds#1{%
+% \test@ifmacro#1{\tctestifcon\ifcatSIX{\implicittoktrue\@chrT#1}{\@macT#1}}%
+% {\implicittoktrue\trapimplicitegrp#1\implicitgrpfork#1}}
+%%%%%%%%%%END TRAPS ACTIVE-IMPLICIT GROUP TOKS FOR \stripimplicitgroupingcase
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
@@ -227,7 +318,7 @@ Apply different directives to Characters (under-dot), Groups
\begingroup
-\begin{exampleA}[lefthand width=7.8cm]{Employing \TokCycle{} directives}
+\begin{exampleA}[lefthand width=7.8cm]{}
\tokcycle{\addcytoks{\underdot{#1}}}
{\addcytoks{\{}\processtoks{#1}
\addcytoks{\}}}
@@ -239,7 +330,7 @@ Apply different directives to Characters (under-dot), Groups
\endgroup
-\subsubsection{Escaping text\label{s:esc}}
+\subsubsection{Escaping text}
Text between two successive escape characters is bypassed
by \TokCycle{} and instead echoed to the output register.
@@ -268,16 +359,15 @@ Default escape character is $|$. One can change it with
\endgroup
\subsubsection{Unexpandable, unexpanded, and expanded Character
- directives\label{s:unex}}
+ directives}
\begingroup
-These examples concern the issue of whether the characters of the input
+This section concerns the issue of whether the characters of the input
stream are transformed before or after being placed in the
output token register (\macname{cytoks}).\medskip
\noindent\begin{minipage}[t]{\textwidth}
-Transform characters (+1 ASCII) via unexpandable macro (see section~%
- \ref{s:esc} for \macname{plusl} definition):\medskip
+Transform characters (+1 ASCII) via unexpandable macro:\medskip
\begin{exampleA}[lefthand width=8.3cm]
{Unexpandable Character directive}
@@ -353,8 +443,6 @@ Capitalize vowels (expanding the character directive)
\Characterdirective{\addcytoks[x]{\vowelcap{#1}}}
\noindent\begin{minipage}[t]{\textwidth}
-See section~\ref{s:unex} for the \macname{vowelcap} definition.
-
\begin{exampleA}[lefthand width=5.5cm]{Normal token cycle (input stream
not pre-expanded)}
\tokcycle
@@ -410,10 +498,7 @@ The macro
implicit group tokens from the output; or $-$1 to instead pass the
implicit group tokens to the Character directive (as implicit
tokens) for separate processing (typically, when detokenization
- is desired).\smallskip
-
-In the following example, see section~\ref{s:unex} for the
- \macname{vowelcap} definition.
+ is desired).
\begin{exampleA}[lefthand width=6cm]{Using
\macname{stripimplicitgroupingcase} to affect treatment of
@@ -438,6 +523,11 @@ In the following example, see section~\ref{s:unex} for the
\detokenize\expandafter{\the\cytoks}
\end{exampleA}
+%\tokcyclexpress{Announcement: {\bfseries\bgroup\itshape
+% Today \egroup it is} \today, a Wednesday.}\the\cytoks
+
+%\detokenize\expandafter{\the\cytoks}
+
\endgroup
\subsubsection{Treatment options for explicit groups}
@@ -529,7 +619,7 @@ Note that the content of groups can be altogether eliminated if
\endgroup
-\subsubsection{Group nesting\label{s:nest}}
+\subsubsection{Group nesting}
\begingroup
@@ -562,18 +652,14 @@ Note that the content of groups can be altogether eliminated if
\subsection{Direct use of \TokCycle}
-While one may find it convenient to encapsulate \TokCycle{} commands
- inside of other macros, it should be obvious that this is not a requirement.
-\TokCycle[T]{} macros and environments (in regular or \texttt{xpress}
- form) may be invoked directly from the document, without
+\TokCycle{} (in regular or \texttt{xpress} form) may be invoked
+ directly from the document, without
being first encapsulated within a macro or environment.
\subsubsection{Modifying counters as part of the Character
directive}
\begingroup
-In the following example, see section~\ref{s:nest} for the definitions
- of \macname{restorecolor} and \macname{reducecolor}.
\begin{exampleA}[lefthand width=8.5cm]{Using a period token (.) to
reset a changing color}
@@ -634,7 +720,7 @@ This macro attempts to give a more natural presentation of
It is \textbf{not} to be confused as a replacement for
\macname{detokenize}.
In certain applications, it may offer a more pleasingly formatted
- typesetting of detokenized material.\smallskip
+ typesetting of detokenized material.
It is an unusual application of \TokCycle{} in that it does not
actually use the \macname{cytoks} token register to collect
@@ -880,14 +966,19 @@ Because \macname{tcremap} is expandable, the original text is totally absent
\begingroup
Active characters in the \TokCycle{} input stream are processed in
their original form.
-Their active substitutions only occur \textit{afterwards}, when
+Their active substitutions arising from \macname{def}s
+ only occur \textit{afterwards}, when
the \TokCycle{} output is typeset.
-They may be identified in the Character directive with
- \macname{ifactivetok}.
-
+They may be identified with the \macname{ifactivetok} test.
+If \macname{let} to a character, they may be identified in the Character
+ directive;
+If \macname{let} to a control sequence or defined via \macname{def},
+ they may be identified in the Macro directive.
\let\svt T
\Characterdirective{\tctestifcon\ifactivetok
- {\addcytoks{\fbox{#1}}}{\addcytoks{#1}}}
+ {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}}
+\Macrodirective{\tctestifcon\ifactivetok
+ {\addcytoks{\fbox{#1-mac}}}{\addcytoks{#1}}}
\begin{exampleA}[lefthand width=7.8cm]{Processing active characters}
\resettokcycle
@@ -900,7 +991,9 @@ This is a test!!\endtokencyclexpress
This is a test!!\endtokencyclexpress
\Characterdirective{\tctestifcon\ifactivetok
- {\addcytoks{\fbox{#1}}}{\addcytoks{#1}}}
+ {\addcytoks{\fbox{#1-chr}}}{\addcytoks{#1}}}
+\Macrodirective{\tctestifcon\ifactivetok
+ {\addcytoks{\fbox{#1-mac}}}{\addcytoks{#1}}}
\tokencyclexpress
This is a test!!\endtokencyclexpress
@@ -913,24 +1006,29 @@ This is a test!!\endtokencyclexpress
\end{exampleA}
%%%
+
+\ddbend If the input stream is pre-\textit{expanded}, any active
+ substitutions that are expandable (i.e., those involving \macname{def}
+ as well as those \macname{let} to something expandable)
+ are made before reaching \TokCycle{} processing.
+\svt hey are, thus, no longer detected as active, unless
+ \macname{noexpand} is applied before the pre-expansion.
+In this example, the \texttt{!} that is not \macname{noexpand}\kern.3pted
+ is converted to a \texttt{?} prior to reaching \TokCycle{}
+ processing (and thus, not detected as \macname{active}):
+
\catcode`!=\active
\def !{?}
\catcode`T=\active
\let T+
-\ddbend If the input stream is pre-\textit{expanded}, the active
- substitutions involving \macname{def} (but not \macname{let})
- are made before reaching {\catcode`T=11 \TokCycle} processing.
-Such tokens are no longer detected as active, unless
- \macname{noexpand} is applied to the pre-expansion:
-
\begin{exampleA}[lefthand width=7.8cm]{Expanded input stream acts
- upon active characters unless \macname{noexpand} is applied}
+ upon active \macname{def}ed characters unless \macname{noexpand} is applied}
\expandedtokcyclexpress{This is a test!\noexpand!}
\the\cytoks\par
\detokenize\expandafter{\the\cytoks}
\end{exampleA}
-However, pre-\textit{tokenization} does not suffer this behavior:
+However, pre-tokenization does not suffer this behavior:
\begin{exampleA}[lefthand width=7.8cm]{Pre-tokenized input stream
does not affect active characters}
\def\tmp{This is a test!!}
@@ -943,6 +1041,41 @@ However, pre-\textit{tokenization} does not suffer this behavior:
\endgroup
\end{minipage}
+\begingroup
+\ddbend One aspect of \TeX{} to remember is that catcodes
+ are assigned at tokenization; however, for active characters,
+ the substitution assignment is evaluated only upon execution.
+So, if a cat-13 token is placed into a \macname{def}, it will
+ remain active even if the catcode of that character code is
+ later changed.
+But if the cat-13 active definition is changed prior to the
+ execution of the \macname{def}'ed token, the revised
+ token assignment will apply.
+
+The following example demonstrates this concept, while showing,
+ whithout changing the input in any way, that \TokCycle{}
+ can properly digest active and implicit grouping (cat-1,2) characters:
+
+\begin{exampleA}[lefthand width=8cm]{Active and implicit grouping
+ tokens digestible by \TokCycle}
+\catcode`Y=13
+\catcode`Z=13
+\let Y{
+\let Z}
+\let\Y{
+\let\Z}
+\def\tmp{\textit YabcZ de\Y\itshape f\Zg}%
+
+\def Y{\bgroup[NEW]}% APPLIES AT EXECUTION
+\catcode`Y=11% DOES NOT AFFECT Y IN \tmp
+
+\expandafter\tokcyclexpress\expandafter{\tmp}
+\the\cytoks
+
+\detokenize\expandafter{\the\cytoks}
+\end{exampleA}
+\endgroup
+
\subsubsection{Trap Catcode 6 (explicit \& implicit) tokens}
\begingroup
@@ -958,11 +1091,11 @@ Since they are unlikely to be used in that capacity inside a
into something cat-12 and set the if-condition \macname{catSIXtrue}.
In this manner, \macname{ifcatSIX} can be used inside the
Character directive to convert cat-6 tokens into something of
- the user's choosing.\smallskip
+ the user's choosing.
As to this cat-12 conversion, explicit cat-6 characters are converted
into the same character with cat-12.
-On the other hand, implicit cat-6 macros (e.g., \verb|\let\myhash#|)
+On the other hand, implicit cat-6 control sequences (e.g., \verb|\let\myhash#|)
are converted into a fixed-name macro, \macname{implicitsixtok},
whose cat-12 substitution text is a \macname{string} of the original
implicit-macro name.
@@ -1018,10 +1151,6 @@ A package macro, \macname{whennotprocessingparameter\#1\{<}%
In this example, a macro is defined and then executed, subject to
token replacements brought about by the expandable Character directive.
-\smallskip
-In the following example, see section~\ref{s:unex} for
- the \macname{vowelcap} definition.
-
\Characterdirective{%
\whennotprocessingparameter#1{\addcytoks[x]{\vowelcap{#1}}}}
\aftertokcycle{\global\cytoks\expandafter{\the\cytoks}}
@@ -1048,17 +1177,24 @@ In the following example, see section~\ref{s:unex} for
\begingroup
-Implicit macros (assigned via \macname{let}) were already
- mentioned in the context of cat-6.
-However, implicit macros can be of any valid catcode.
+Implicit control sequences (assigned via \macname{let} to characters)
+ were already mentioned in the context of cat-6.
+However, implicit control sequences can be of any valid catcode (except
+ for cat-0, which we instead call macros or primitives).
The condition \macname{ifimplicittok} is used to flag such
- tokens for special processing.
+ tokens for special processing, as well as active tokens that
+ are \macname{let} to anything unexpandable.
+
+In the next example, implicit, cat-6 and implicit-cat-6 tokens may
+ all be differentiated, shown here with a multiplicity of
+ \macname{fbox}es.
\fboxsep=1.5pt
\begin{exampleA}[lefthand width=8cm]{Implicit\ensuremath{{}={}}%
single box\char44 \ cat-6\ensuremath{{}={}}double box\char44 \ %
implicit-cat-6\ensuremath{{}={}}triple box}
+\catcode`Q=\active \let QN
\let\littlet=t
\let\littlel=l
\let\svhash#
@@ -1073,10 +1209,48 @@ The condition \macname{ifimplicittok} is used to flag such
This \textit{is a \textbf{big}} \littlet est.
-Next pa#agraph ending with implicit cat six
+Qext pa#agraph ending with implicit cat six
\svhash.\endtokencyclexpress
\end{exampleA}
+In the following example, we use both control sequences and active characters
+ in \macname{def} and \macname{let} capacities, to demonstrate how
+ \TokCycle{} digests things.
+Implicit tokens (tokens \macname{let} to characters) are shown in a box,
+ with both the token name and the implicit value (note that tokens
+ \macname{let} to macros and primitives are not considered implicit).
+Active tokens processed through the character directive are followed with
+ a \dag, whereas those processed through the macro directive are followed
+ with a \ddag.
+
+
+\begin{exampleA}[lefthand width=7.5cm]{Non-active vs{.} active \macname{def}
+ \& \macname{let}}
+\Characterdirective{\ifimplicittok
+ \addcytoks{\fbox{\detokenize{#1}:#1}}%
+ \else\addcytoks{#1}\fi\ifactivetok
+ \addcytoks{\rlap{\dag}}\fi\addcytoks{~\,}}
+\Macrodirective{\ifimplicittok
+ \addcytoks{\fbox{\detokenize{#1}}}%
+ \else\addcytoks{#1}\fi\ifactivetok
+ \addcytoks{\rlap{\ddag}}\fi
+ \ifx\par#1\else\addcytoks{~\,}\fi}
+
+\def\A{a}
+\let\B i
+\let\C\today
+\let\D\relax
+\def\E{\relax}
+\catcode`V=13 \def V{a}
+\catcode`W=13 \let Ww
+\catcode`X=13 \let X\today
+\catcode`Y=13 \let Y\relax
+\catcode`Z=13 \def Z{\relax}
+\tokcyclexpress{\A\B\C\D\E ab\par VWXYZab}
+\the\cytoks
+\end{exampleA}
+
+
\ddbend If the input stream is subject to pre-expansion, one
will require \macname{noexpand} for macros where no pre-expansion
is desired.\medskip
@@ -1093,7 +1267,7 @@ Next pa#agraph ending with implicit cat six
Changing grouping tokens (catcodes 1,2) may require something more,
if the output stream is to be detokenized.
In the following examples, pay attention to the
- detokenized grouping around the argument to \macname{fbox}.\smallskip
+ detokenized grouping around the argument to \macname{fbox}.
As we will see, the issues raised here only affect the situation
when detokenization of the output stream is required.