diff options
| -rw-r--r-- | Master/texmf-dist/doc/generic/tokcycle/README | 14 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf | bin | 0 -> 241795 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex | 593 | ||||
| -rw-r--r-- | Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf | bin | 0 -> 244484 bytes | |||
| -rw-r--r-- | Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex | 1286 | ||||
| -rw-r--r-- | Master/texmf-dist/tex/generic/tokcycle/tokcycle.sty | 4 | ||||
| -rw-r--r-- | Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex | 192 | ||||
| -rwxr-xr-x | Master/tlpkg/bin/tlpkg-ctan-check | 2 | ||||
| -rwxr-xr-x | Master/tlpkg/libexec/ctan2tds | 1 | ||||
| -rw-r--r-- | Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc | 1 | ||||
| -rw-r--r-- | Master/tlpkg/tlpsrc/tokcycle.tlpsrc | 0 |
11 files changed, 2092 insertions, 1 deletions
diff --git a/Master/texmf-dist/doc/generic/tokcycle/README b/Master/texmf-dist/doc/generic/tokcycle/README new file mode 100644 index 00000000000..491a0006977 --- /dev/null +++ b/Master/texmf-dist/doc/generic/tokcycle/README @@ -0,0 +1,14 @@ +The tokcycle package helps one to build tools to process tokens from an input stream, on a token-by-token basis, by applying category-based directives to each successive token. + + +Author: Steven B Segletes, Christian Tellechea (contributor) + +Files : 1) tokcycle.tex + 2) tokcycle.sty + 3) tokcycle-doc.tex + 4) tokcycle-doc.pdf + 5) tokcycle-examples.tex + 6) tokcycle-examples.pdf + 7) README + +Released under the LaTeX Project Public License v1.3c or later, see http://www.latex-project.org/lppl.txt
\ No newline at end of file diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf Binary files differnew file mode 100644 index 00000000000..c36edd5f642 --- /dev/null +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.pdf diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex new file mode 100644 index 00000000000..e7131379cf8 --- /dev/null +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-doc.tex @@ -0,0 +1,593 @@ +\documentclass{article} +\usepackage[T1]{fontenc} +\usepackage{xcolor,tokcycle,verbatimbox,tabstackengine,lipsum} +\colorlet{bred}{red!90!blue} +\usepackage{caption} +\captionsetup{font={sf, footnotesize},labelfont={sf},textfont={footnotesize}} +\fboxsep=1.5pt\relax +\makeatletter +\let\svlt< +\let\svgt> +\catcode`>=\active % +\catcode`<=\active % +\def\openesc#1{\if\svlt#1\itshape\rmfamily\let\xsvgt\@empty\else + \svlt\itshape\rmfamily#1\let\xsvgt\svgt\fi} +\def\closeesc{\upshape\ttfamily\xsvgt} +\def\vbdelim{\small\catcode`<=\active\catcode`>=\active +\def<{\openesc} +\def>{\closeesc}} +\catcode`>=12 % +\catcode`<=12 % +\newcommand\TokCycle{\textsf{tokcycle}} +\newcommand\Character{\textit{Character}} +\newcommand\Group{\textit{Group}} +\newcommand\Macro{\textit{Macro}} +\newcommand\Space{\textit{Space}} +\newcommand\CGMS{\textit{C-G-M-S}} +\newcommand\CGMnS{\textit{C}, \textit{G}, \textit{M}, and \textit{S}} +\newcommand\macname[1]{\texttt{\char92#1}} +\newenvironment{specfig} +{\par\bigskip\vbox\bgroup\small\sffamily\centering} +{\egroup\medskip\par} +\newtoks\gtoks +\newcommand\svgtoks[1]{\global\gtoks\expandafter{\the#1}} +\frenchspacing + +\begin{document} + +%%% AGAIN + +%%% + +\begin{myverbbox}{\argone}#1\end{myverbbox} +\begin{myverbbox}[\vbdelim\normalsize]{\vbnewcytoks} +\newcytoks{<new token register>} +\end{myverbbox} + +{\centering{\LARGE Package \TokCycle{}\par} +Steven B Segletes +{\footnotesize $<$steven.b.segletes.civ@mail.mil$>$} +\\\small{contributor\footnote{ +I am extremely grateful to Christian + for his assistance in the development of this package. +The \macname{addcytoks} macro was provided by him. +He gave constant reminders on what the token parser should be able + to achieve, thus motivating me to spend the extra time striving for + a generality of application that would not otherwise have made it + into the package. +I value highly his collegiality and hold his expertise in the highest + regard. +}% +: Christian Tellechea {\footnotesize$<$unbonpetit@netc.fr$>$}}\\\today\par} + +\begin{quote} +The \TokCycle{} package helps one to build tools to process + tokens from an input stream. +If a macro to process an arbitrary single + (non-macro, non-space) token can be built, then + \TokCycle{} can provide a wrapper for cycling through an + input stream (including macros, spaces, and groups) + on a token-by-token basis, using the + provided macro on each successive character. + +\TokCycle{} characterizes each successive token + in the input stream as a \Character, a \Group, a \Macro, or + a \Space. +Each of these + token categories are processed with a unique directive, + to bring about the desired effect of the token cycle. +\textit{If}-condition flags are provided to identify active, + implicit, and catcode-6 tokens as they are digested. +The package provides a number of options + for handling groups. +\end{quote} + +\tableofcontents + +\makeatother +\section{The \TokCycle{} macros and environments}\label{s:intro} + +The purpose of the \TokCycle{} package is to provide a tool + to assist the user in developing token-processing macros and + environments. +The types of processing is limited only by the creativity of the user, + but examples might include letter-case-operations, letter spacing, + dimensional manipulation, + simple-ciphering, \{group\} manipulation, macro removal, etc. + +The package can be loaded into both plain \TeX{}, by way of the + invocation \verb|\input tokcycle.tex| + as well as \LaTeX, via \verb|\usepackage{tokcycle}|. +It provides a total of 6 macros/pseudo environments, based on + three criteria: +\begin{itemize} +\item Two pseudo-environments with the phrase ``\texttt{tokencycle}'' + in the name, and four macros containing the phrase ``\texttt{tokcycle}''. +The pseudo-\hspace{0pt}environments operate within a group and typeset + their result upon completion. +The macros operate within the document's current scope, but do not typeset the + result automatically. +In the case of both macros and pseudo-environments, the transformed result + is available for later use, being stored in the package + token register named \macname{cytoks}. + +\item Two macros and one pseudo-environment containing the + phrase ``\texttt{xpress}''. +Without the phrase, the macro/environment requires four + \textit{processing directives} to be \textit{explicitly} specified, + followed by the input stream. +With the phrase present, only the input stream is to be provided. +In the \texttt{xpress} case, the \textit{processing directives} are + to have been separately specified via external macro and/or are taken + from the most recent \TokCycle{} macro invocation (failing that, + are taken from the package initialization). + +\item Two macros containing the phrase ``\texttt{expanded}''. +When present, the input stream of the token cycle is subject to the + new \TeX{} primitive, \macname{expanded}, prior to processing by + the \TokCycle{} macro. +Expansion of specific macros in the input stream can be inhibited + in the input stream with the use of \macname{noexpand}. +Note that there are no \texttt{expanded} environments in \TokCycle, + as \TokCycle{} environments do not pre-tokenize their + input stream. + +\end{itemize} + +The basic approach of all macros/environments is to grab each + successive token (or group) from the input stream, + decide what category it is, and use the currently active + processing directives to make any desired transformation of the + token (or group). +Generally, with rare exception, the processed tokens should be stored + in the token register \macname{cytoks}, using the tools provided + by the package. +The cycle continues until the input stream is terminated. + +As tokens/groups are read from the input stream, they are + categorized according to four type classifications, and are + subjected to the user-specified processing directive associated + with that category of token/group. +The \TokCycle{} categories by which the input stream + is dissected include \Character{}, + \Group{}, \Macro{}, and \Space{}. +The syntax of the non-\texttt{xpress} macros/environments is\medskip + +\begin{verbbox}[\vbdelim] +\tokcycle <<or>> \expandedtokcycle + {<Character processing directive>} + {<Group-content processing directive>} + {<Macro processing directive>} + {<Space processing directive>} + {<token input stream>} +\end{verbbox} +\theverbbox\medskip\par\noindent +or, alternately, for the pseudo-environment, + +\begin{verbbox}[\vbdelim] +\tokencycle + {<Character processing directive>} + {<Group-content processing directive>} + {<Macro processing directive>} + {<Space processing directive>}<token input stream>\endtokencycle +\end{verbbox} +\medskip\theverbbox\medskip\par\noindent +For the \texttt{xpress} macros, the syntax is + +\begin{verbbox}[\vbdelim] +\tokcyclexpress <<or>> \expandedtokcyclexpress + {<token input stream>} +\end{verbbox} +\medskip\theverbbox\medskip\par\noindent\medskip +or, alternately, for the \texttt{xpress}-pseudo-environment, + +\begin{verbbox}[\vbdelim] +\tokencycleexpress<token input stream>\endtokencyclexpress +\end{verbbox} +\theverbbox\medskip +\begin{sloppypar} + +In addition to the above macros/environments, the means is + provided to define new \TokCycle{} environments: + +\begin{verbbox}[\vbdelim] +\tokencycleenvironment\environment_name + {<Character processing directive>} + {<Group-content processing directive>} + {<Macro processing directive>} + {<Space processing directive>}% +\end{verbbox} +\medskip\theverbbox\medskip\par\noindent +This will then permit simplified invocations of the form + +\begin{verbbox}[\vbdelim] +\environment_name<token input stream>\endenvironment_name +\end{verbbox} +\medskip\theverbbox\medskip\par\noindent + + +\subsection{Externally specified directives and directive resets} + +For use in \texttt{xpress} mode, the directives for the \CGMS{} + categories may be externally pre-specified, respectively, + via the four macros \macname{Characterdirective}, + \macname{Groupdirective}, \macname{Macrodirective}, + and \macname{Spacedirective}, each taking an argument containing + the particulars of the directive specification. + +Each of these directives may be individually reset to the package + default with the following argument-free invocations: + \macname{resetCharacterdirective}, \macname{resetGroupdirective}, + \macname{resetMacrodirective}, or \macname{resetSpacedirective}. +In addition, \macname{resettokcycle} is also provided, which + not only resets all four directives collectively, but it also + resets, to the default configuration, the manner in which explicit + and implicit group tokens are processed. +Finally, it resets the \macname{aftertokcycle} macro to empty. +\end{sloppypar} + +The default directives at package outset and upon reset are + +\begin{verbbox}[\vbdelim] +\Characterdirective{\addcytoks{#1}} +\Groupdirective{\processtoks{#1}} +\Macrodirective{\addcytoks{#1}} +\Spacedirective{\addcytoks{#1}} +\aftertokcycle{} +\stripgroupingfalse +\stripimplicitgroupingcase{0} +\end{verbbox} +\medskip\theverbbox\medskip\noindent + +The interpretation of these directives will be explained in the + remainder of this document. + + + +\section{Commands in the \TokCycle{} directives}\label{s:coms} + +The command-line token cycling tools to be employed in the package were + listed in section~\ref{s:intro}. +For each of those commands/pseudo-environments provided, the + user must (explicitly or implicitly) detail a set of directives to + specify the manner in which the \Character{}, \Group{}, \Macro{}, + and \Space{} tokens found in the input stream are to be processed. +The \CGMS{} processing + directives consist of normal \TeX/\LaTeX{} commands and user-defined + macros to accomplish the desired effect. +There are, however, several macros provided by the package to assist + the user in this endeavor. + +The recommended way to apply this package is to collect the + \TokCycle-transformed results of the input stream in a token register + provided by the package, named \macname{cytoks}. +Its contents can then be typeset via \verb|\the\cytoks|. +The macro for appending things to \macname{cytoks}, to be used in + the package directives, is \macname{addcytoks}. + +\subsection{Adding tokens to the output stream: \macname{addcytoks}} + +The macro provided to append tokens to the \macname{cytoks} token register is + named \macname{addcytoks[]\{\}}. +Its mandatory argument consists of tokens denoting \textit{what} you + would like to append to the \macname{cytoks} register, while the + optional argument denotes \textit{how} you would like them appended (valid + options include positive integers \verb|[<|\textit{n}\verb|>]| and + the letter \texttt{[x]}). + +When the optional argument is omitted, the specified tokens are appended + \texttt{literally} to the register (without expansion). +An integer option, call it \textit{n}, takes the + the mandatory argument, and expands it $n$-times, and appends the + result to \macname{cytoks}. +The \texttt{[x]} option employs the \macname{expanded} primitive to maximally + expand the argument before appending it to \macname{cytoks}. + +The \texttt{[x]} option will prove useful when the \Character{} directive involves + a transformation that is fully expandable. +Its use will allow the expanded + result of the transformation to be placed in the token register, rather + than the unexpanded transformation instructions. + +\subsubsection{\argone} + +In the context of the \CGMnS{} processing directives, one may refer to + \argone{} (e.g., in the argument to \macname{addcytoks}). +\TeX{} users know that the first parameter to a \TeX{} macro is denoted + as \argone. +The specification of all \TokCycle{} processing directives is structured + in such a way that ``\argone{}'' may be directly employed to indicate + the current token (or group) under consideration in the input stream. + +\subsection{\Group{} directive: \macname{ifstripgrouping}, and \macname{processtoks}} + +The \Group{} directive is unique, in that it is the only directive whose + argument (\texttt{\#1}) may consist of more than a single token. +There are two issues to consider when handling the tokens comprising a group: + do I retain or remove the grouping (cat-1,2 tokens)? +Do I process the group's token content + individually through the token cycle, or collectively as a unit? + +Grouping in the output stream is determined by the externally set + condition \macname{ifstripgrouping}. +The package default is \macname{stripgroupingfalse}, such that any grouping + that appears in the input stream will be echoed in the output stream. +The alternative, \macname{stripgroupingtrue}, is dangerous + in the sense that it will strip the cat-1,2 grouping from the + group's tokens, thereby + affecting or even breaking code that utilizes such grouping for macro arguments. +Modify \macname{ifstripgrouping} with care. + +The issue of treating the tokens comprising the content of a + group individually or collectively + is determined by the choice of macro inside the \Group{} directive. +Within the \Group{} directive, the argument \texttt{\#1} contains the + collective tokens of the group (absent the outer cat-1,2 grouping). +The default directive \macname{processtoks\{\#1\}} will recommit the + group's tokens individually to be processed through the token cycle. +In contrast, the command \macname{addcytoks\{\#1\}} in the \Group{} + directive would directly add the collective group + of tokens to the \macname{cytoks} register, without being processed + individually by \TokCycle. + +\subsubsection{Implicit grouping tokens: \macname{stripimplicitgroupingcase}} + +Implicit grouping tokens (e.g., \macname{bgroup} \& \macname{egroup}) + can be handled in one of \textit{three} separate ways. +Therefore, rather than using an \textit{if}-condition, the external + declaration \macname{stripimplicitgroupingcase\{\}} is provided, which takes + one of 3 integers as its argument (0, 1, or $-$1). +The package-default case of ``0'' indicates that the implicit-group tokens + will not be stripped, but rather echoed directly into the + output stream. +The case of ``1'' indicates that the implicit-group tokens + will be stripped and not placed into the output stream + (as with explicit grouping, this is a dangerous case and should be + specified with care). + +Finally, the special case of $-$1 indicates that the implicit-group + tokens should instead be passed to the \Character{} directive for + further handling (note that the \macname{implicittoktrue} condition will + be set\footnote{as well as the internal condition \macname{tc@implicitgrptrue}}). +Such a special treatment has limited application---for example, + when the intent is to detokenize these tokens. + + +\subsection{Escaping content from \TokCycle{} processing\label{s:escape}} + +There are times you may wish to prevent tokens in the \TokCycle{} + input stream from being operated on by \TokCycle; you just want + the content passed through unchanged to the output; that is, +with the intent to have multi-token content bypass the + \TokCycle{} directives altogether. + +\begin{sloppypar} +The method developed by the package is to enclose the escaped + content in the input stream with the \TokCycle{} escape character, + initially set to a vertical rule character found on the keyboard: + ``$|\mkern1mu$''. +The main proviso is that the escaped content cannot straddle a + group boundary (the complete group may, however, be escaped). +The escape character can be changed with +\begin{verbbox}[\vbdelim] +\settcEscapechar{<escape-token>}\end{verbbox} +\theverbbox +$\!\!\!$. +\end{sloppypar} + +\subsection{Flagged tokens} + +Certain token types are trapped and flagged via true/false + \textit{if}-conditions. +These \textit{if}-conditions can be examined within the appropriate + directive (generally the \Character{} directive), to direct the + course of action within the directive. + +\subsubsection{Active characters: \macname{ifactivetok}} + +Active (cat-13) characters that occur in the input stream result in the + flag \macname{ifactivetok} being set \macname{activetoktrue}. +Note that the substitution of the token's active value occurs \textit{after} + \TokCycle{} processing, and so \TokCycle{} only sees it as + \macname{active}, regardless of its later substitution or assignment. +The only exception to this rule is with pre-expanded input, + \macname{expandedtokcycle[xpress]}. +If an active token's substitution is governed by a \macname{def} (but + not \macname{let}), the text substitution will have occurred before + reaching the token cycle. + +\subsubsection{Implicit tokens: \macname{ifimplicittok}} + +Implicit tokens, those assigned via \macname{let}, are flagged with + a \texttt{true} setting for \macname{ifimplicittok}. +Generally, implicit tokens will be directed to the \Character{} + directive for processing. +There are, however, two exceptions: +i) implicit grouping tokens (e.g., \macname{bgroup} and \macname{egroup}) + will not appear in any directive unless the + \macname{stripimplicitgroupingcase} has been set to $-$1; and +ii) implicit space tokens (e.g., \macname{@sptoken}) will be processed + by the \Space{} directive. + +\subsubsection{Parameter (cat-6) tokens (e.g., \texttt{\#}): \macname{ifcatSIX}} + +Typically, category-code 6 tokens (like \texttt{\#}) are used to + designate a parameter (e.g., \texttt{\#1}--\texttt{\#9}). +Since they are unlikely to be used in that capacity inside a + tokcycle input stream, the package behavior is to convert them + into something cat-12 and set the \textit{if}-condition \macname{catSIXtrue}. +In this manner, \macname{ifcatSIX} can be used inside the + Character directive to trap and convert cat-6 tokens into something of + the user's choosing. + +\begin{sloppypar} +As to the default nature of this conversion (if no special actions are + taken), explicit cat-6 characters are converted + into the identical character with category code of 12. +On the other hand, implicit cat-6 macros (e.g., \verb|\let\myhash#|) + are converted into a fixed-name implicit macro, \macname{implicitsixtok}, + whose cat-12 substitution text is a \macname{string} of the original + implicit-macro name. + +As previously mentioned, active characters whose active value is + \macname{let} to cat-6 (e.g., \verb|\catcode`X=\active \letX#|) will be + flagged as \macname{activetoktrue}, but not \macname{catSIXtrue}. +\end{sloppypar} + +\subsubsection{Parameters (\texttt{\#1}--\texttt{\#9}): +\macname{whennotprocessingparameter}} + +While, generally, one would not intend to use parameters + in the \TokCycle{} input stream, the package provides, not a flag, + but a macro to allow it. +The macro is to be used \textit{within} the \Character{} directive + with the syntax:\smallskip + +\begin{verbbox}[\vbdelim] +\whennotprocessingparameter#1{<non-parameter-processing-directive>}\end{verbbox} +\theverbbox\smallskip + +\noindent Here, the \texttt{\#1} doesn't refer to \texttt{\#1} + as it appears in the input stream, but to the sole parameter of + the \Character{} directive, referring to the current token being + processed. +With this syntax, when the token under consideration is a + parameter (e.g., \texttt{\#1}--\texttt{\#9}), that parameter is + added to the \macname{cytoks} register. +If the token under consideration is not a parameter, the code + in the final argument is executed. + +\subsection{Misc: general \textit{if}-condition tools} + +\TeX{} comes equipped with a variety of \macname{if...} condition + primitives. +When dealing with macros for which the order of expansion is important, + the \macname{else} and \macname{fi} can sometimes get in the way + of proper expansion and execution. +These four restructured \textit{if} macros came in handy writing the + package, and may be of use in creating your directives, + to prevent \macname{else} and \macname{fi} from getting in the way:\medskip + +\begin{verbbox}[\vbdelim] +\tctestifcon{<TeX-if-condition>}{<true-code>}{<false-code>} +\tctestifx{<\ifx-comparison-toks>}{<true-code>}{<false-code>} +\tctestifnum{<\ifnum-condition>}{<true-code>}{<false-code>} +\tctestifcatnx<\ifcat\noexpand-comparison-toks>{<true-code>}{<false-code>} +\end{verbbox} +\theverbbox + + +\subsection{\TokCycle{} macro close-out: \macname{aftertokcycle}} + +The \TokCycle{} macros, upon completion, do nothing. +Unlike \macname{tokencycle} environments, they don't even + output \macname{the}\macname{cytoks} token register. +A command has been provided, \macname{aftertokcycle}, which takes + an argument to denote the actions to be executed following + completion of \textit{all} subsequent \TokCycle{} invocations. +It might be as simple as \verb|\aftertokcycle{\the\cytoks}|, to + have the output stream automatically typeset. + +The meaning of \macname{aftertokcycle} can be reset with + \macname{aftertokcycle\{\}}, but is also reset as part of + \macname{resettokcycle}. +Unlike macros, the \TokCycle{} environments are unaffected + by \macname{aftertokcycle}, as they actually set it locally + (within their scope) to accomplish their own purposes. + +\subsection{Accommodating catcode-1,2 changes: \macname{settcGrouping}} + +In order to avoid making the \TokCycle{} parser overly complex, requiring + multiple passes of the input stream, the package defaults to using + catcode-1,2 braces \texttt{\{~\}} to bring about grouping in the + output stream, regardless of what the actual cat-1,2 tokens are in + the input stream. +As long as their sole purpose in the token + cycle is for grouping and scoping, this arrangement will produce + the expected output. + +However, if the actual character-code of these tokens is important + to the result (e.g., when detokenized), there is one other option. +The package allows the external specification of which cat-1,2 + tokens should be used in the \TokCycle{} output stream. +The syntax is \verb|\settcGrouping{{#1}}|, to use the standard braces + for this purpose (default). +If angle-brackets \verb|< >| were to be the \textit{new} + grouping tokens, then, after their catcodes were changed, and + \macname{bgroup} and \macname{egroup} were reassigned, one would + invoke \verb|\settcGrouping<<#1>>|. +These will then be the grouping tokens in the \TokCycle{} output + stream until set to something else. +The grouping tokens are \textit{not} reset as part of + \macname{resettokcycle}. + +\section{Usage Examples} + +See the adjunct file, \texttt{tokcycle-examples.pdf}, for a wide + array of \TokCycle{} examples. + +\section{Summary of known package limitations} + +The goal of this package is not to build the perfect token-stream + parser. +It is, rather, to provide the means for users to build useful + token-processing tools for their \TeX/\LaTeX{} documents. + +What follows are the known limitations of the package. + which arise, in part, from the single-pass parsing algorithm + embedded in the package. +Surely, there are more cases associated with arcane catcode-changing + syntax that are not accounted for; +I encourage you to bring them to my attention. +If I can't fix them, I can at least disclaim and declaim them. + +\begin{itemize} + +\item One must inform the package (via \macname{settcGrouping}) of + changes to the cat-1,2 tokens \textit{if} + there is a need to detokenize the output with the specified + bracing group; however, + grouping will still be handled properly (i.e., cat-1,2 tokens will + be detected), even if the package is not notified. + +\item Should one need to keep track of the \textit{names} of implicit tokens, + then only one named implicit cat-6 token may appear in the input + stream (though it can appear multiple times). +There is no limit on explicit cat-6 toks. +This limitation occurs because implicit cat-6 tokens are converted + into the fixed-name implicit macro \macname{implicitsixtok} which + contains the \macname{string} of the most recently used + implicit cat-6 token name. +In any event, all cat-6 tokens are trapped and flag \texttt{true} + the \macname{ifcatSIX} condition. + +\end{itemize} + +\section*{Acknowledgments} + +In addition to Christian Tellechea, a contributor to this + package, +the author would like to thank Dr.\@ David Carlisle for his + assistance in understanding some of the nuances of token + registers. +Likewise, his explanation about how a space token is defined + in \TeX{} (see \verb|https://tex.stackexchange.com|\allowbreak + \verb|/questions/64197/|\allowbreak\verb|pgfparser-module-and-blank-spaces/64200#64200|) + proved to be useful here. +The \texttt{tex.stackexchange} site provides a wonderful + opportunity to interact with the leading developers and + practitioners of \TeX{} and \LaTeX{}. + +\section*{Source Code} + +\Large\texttt{tokcycle.sty} + +\verbfilenobox[\footnotesize]{tokcycle.sty}\bigskip + +\noindent\texttt{tokcycle.tex} + +\verbfilenobox[\footnotesize]{tokcycle.tex} + + +\end{document} + diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf Binary files differnew file mode 100644 index 00000000000..af6e951390b --- /dev/null +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.pdf diff --git a/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex new file mode 100644 index 00000000000..b73e89168cf --- /dev/null +++ b/Master/texmf-dist/doc/generic/tokcycle/tokcycle-examples.tex @@ -0,0 +1,1286 @@ +\documentclass[10pt]{article} +\usepackage[margin=1.3in]{geometry} +\usepackage[T1]{fontenc} +\usepackage{tokcycle,xcolor,manfnt} +\colorlet{bred}{red!90!blue} +\usepackage{tikz} +% MACROS: REDEFINE TO ALLOW \cytoks TO ESCAPE SCOPE OF exampleA +\aftertokcycle{\global\cytoks\expandafter{\the\cytoks}} +% ENVIRONMENTS: REDEFINE TO ALLOW \cytoks TO ESCAPE SCOPE OF exampleA +\makeatletter\renewcommand\tcenvscope{\global\cytoks}\makeatother + +\def\ddbend{\setbox0=\hbox{\dbend\dbend}\leavevmode\llap{% + \raisebox{10pt}{\box0~}}} + +\usepackage[skins,listings]{tcolorbox} + +\newtcblisting{exampleA}[2][]{% + colframe=red!70!yellow!50!black, + colback=red!75!yellow!5!white, + coltitle=red!50!yellow!3!white, + bicolor,colbacklower=white, + top=0mm,bottom=0mm,left=0mm,right=0mm, + arc=1mm,boxrule=1pt, + fonttitle=\sffamily\bfseries, + sidebyside, + title=#2,#1} + + +\newtcblisting{exampleB}[2][]{% + colframe=red!50!yellow!50!black, + colback=red!50!yellow!5!white,%white, + coltitle=red!50!yellow!3!white, + bicolor,colbacklower=white,%red!50!yellow!5!white, + fonttitle=\sffamily\bfseries, + sidebyside,text and listing, + top=0mm,bottom=0mm,left=0mm,right=0mm, + arc=1mm,boxrule=1pt, + title=#2,#1} + +\newtcblisting{exampleC}[2][]{% + colframe=cyan!70!yellow!50!black, + colback=cyan!30!yellow!5!white,%white, + coltitle=cyan!50!yellow!3!white, + fonttitle=\sffamily\bfseries, + listing only, + top=0mm,bottom=0mm,left=0mm,right=0mm, + arc=1mm,boxrule=1pt, + title=#2,#1} + +\def\altdecytoks{\par\medskip\begingroup\noindent\macname{cytoks} + \ \textit{alt}detokenization:\\\raggedright\footnotesize\ttfamily + \expandafter\altdetokenize\expandafter{\the\cytoks}\par\endgroup} +\newcommand\TokCycle[1][t]{\textsf{#1okcycle}} + +\parindent0pt +\parskip0pt +\begin{document} +{\centering\LARGE \TokCycle{} Package Examples\par\smallskip +\large Steven B. Segletes\\\today\par} + +\tableofcontents + +\makeatletter +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% + + +\makeatother + +%%%%%%%%%%%%%% SUPPORT MACRO + +\newcommand\macname[1]{\texttt{\char92#1}} + +\newcommand\underdot[1]{\ooalign{#1\cr\hfil{\raisebox{-5pt}{.}}\hfil}} + +\newcounter{colorindex} + +\newcommand\restorecolor{\setcounter{colorindex}{100}} + +\newcommand\reducecolor[1]{% + \color{red!\thecolorindex!cyan}% + \addtocounter{colorindex}{-#1}% + \ifnum\thecolorindex<1\relax\setcounter{colorindex}{1}\fi +} + +\newif\ifmacro +\newcommand\altdetokenize[1]{\begingroup\stripgroupingtrue\macrofalse + \tokcycle + {\ifmacro\def\tmp{##1}\ifcat\tmp A\else\unskip\allowbreak\fi\macrofalse\fi + \detokenize{##1}} + {\ifmacro\unskip\macrofalse\fi\{\processtoks{##1}\ifmacro\unskip\fi\}\allowbreak} + {\tctestifx{\\##1}{\\}{\ifmacro\unskip\allowbreak\fi + \allowbreak\detokenize{##1}\macrotrue}} + { \hspace{0pt plus 3em minus .3ex}} + {#1}% + \unskip +\endgroup} + +\newcommand\plusl[1]{\char\numexpr`#1+1\relax} + +\newcommand\vowelcap[1]{% + \ifx a#1A\else + \ifx e#1E\else + \ifx i#1I\else + \ifx o#1O\else + \ifx u#1U\else + #1\fi\fi\fi\fi\fi +} + +\newcommand\spaceouttext[2]{% + \tokcycle + {\addcytoks{##1\nobreak\hspace{#1}}}% + {\processtoks{##1}} + {\addcytoks{##1}}% + {\addcytoks{##1\hspace{#1}}} + {#2}% + \the\cytoks\unskip} + +\newcommand\growdim[2]{% +\tokcycle{\addcytoks{##1}} + {\addcytoks{#1\dimexpr##1}} + {\addcytoks{##1}} + {\addcytoks{##1}}{% + #2}% +\the\cytoks} + +\newcommand\nextcap[1]{% + \edef\tmp{#1}% + \tctestifx{-#1}{\def\capnext{T}}{}% + \tctestifcon{\if T\capnext}% + {\tctestifcon{\ifcat\tmp A}% + {\uppercase{#1}\def\capnext{F}}% + {#1}}% + {#1}% +} + +\newcommand\TitleCase[1]{% + \def\capnext{T} + \tokcycle + {\addcytoks{\nextcap{##1}}} + {\processtoks{##1}} + {\addcytoks{##1}} + {\addcytoks{##1\def\capnext{T}}} + {#1}% + \the\cytoks +} + +\tokcycleenvironment\spaceBgone + {\addcytoks{##1}} + {\processtoks{##1}} + {\addcytoks{##1}} + {\addcytoks{\hspace{.2pt plus .2pt minus .8pt}}}% + + +\tokcycleenvironment\remaptext + {\addcytoks[x]{\tcremap{##1}}} + {\processtoks{##1}} + {\addcytoks{##1}} + {\addcytoks{##1}} +\newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}} +\newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname + \csname tcmapto#1\endcsname\else#1\fi} +\tcmapto am \tcmapto bf \tcmapto cz \tcmapto de \tcmapto ey +\tcmapto fl \tcmapto gx \tcmapto hb \tcmapto ic \tcmapto jn +\tcmapto ki \tcmapto lr \tcmapto mh \tcmapto nt \tcmapto ok +\tcmapto ps \tcmapto qa \tcmapto ro \tcmapto sq \tcmapto tw +\tcmapto uj \tcmapto vp \tcmapto wd \tcmapto xg \tcmapto yu +\tcmapto zv + +\newcommand\findinstring[2]{\begingroup% + \stripgroupingtrue + \setcounter{runcount}{0}% + \tokcycle + {\nextctltok{##1}} + {\nextctltok{\opengroup}\processtoks{##1}\nextctltok{\closegroup}} + {\nextctltok{##1}} + {\nextctltok{\tcspace}} + {#1}% + \edef\numlet{\theruncount}% + \expandafter\def\expandafter\searchword\expandafter{\the\cytoks}% +% + \aftertokcycle{\matchfound}% + \setcounter{runcount}{0}% + \def\matchfound{F}% + \tokcycle + {\nextcmptok{##1}} + {\nextcmptok{\opengroup}\processtoks{##1}\nextcmptok{\closegroup}} + {\nextcmptok{##1}} + {\nextcmptok{\tcspace}} + {#2}% +\endgroup} +\newcounter{runcount} +\makeatletter +\newcommand\rotcytoks[1]{\cytoks\expandafter\expandafter\expandafter{% + \expandafter\tc@gobble\the\cytoks#1}} +\makeatother +\newcommand\testmatch[1]{\ifx#1\searchword\gdef\matchfound{T}\fi}% +\newcommand\rotoradd[2]{\stepcounter{runcount}% + \ifnum\theruncount>\numlet\relax#1\else#2\fi + \expandafter\def\expandafter\tmp\expandafter{\the\cytoks}} +\newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}} +\newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}} + + +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +\section{Examples, examples, and more examples} + +Often, the best way to learn a new tool is to see examples of + it being used. +Here, a number of examples are gathered that span the spectrum + of \TokCycle{} usage. + +\subsection{Application basics} + +\subsubsection{Using the CGMS directives} + +Apply different directives to Characters (under-dot), Groups + (visible braces), Macros (boxed, detokenized), and + Spaces (visible space). + +\begin{exampleC}[]{The \macname{underdot} macro} +\newcommand\underdot[1]{\ooalign{#1\cr\hfil{\raisebox{-5pt}{.}}\hfil}} +\end{exampleC} + + +\begingroup +\begin{exampleA}[lefthand width=7.8cm]{Employing \TokCycle{} directives} +\tokcycle{\addcytoks{\underdot{#1}}} + {\addcytoks{\{}\processtoks{#1} + \addcytoks{\}}} + {\addcytoks{\fbox{\detokenize{#1}}}} + {\addcytoks{\textvisiblespace}} +{This \textit{is \textbf{a}} test.} +\the\cytoks +\end{exampleA} + +\endgroup + +\subsubsection{Escaping text\label{s:esc}} + +Text between two successive escape characters is bypassed + by \TokCycle{} and instead echoed to the output register. +Default escape character is $|$. One can change it with + \macname{settcEscapechar} macro. + +\begingroup + +\begin{exampleC}[]{The unexpandable \macname{plusl} macro} +\newcommand\plusl[1]{\char\numexpr`#1+1\relax} +\end{exampleC} + +\begin{exampleA}[lefthand width=5cm]{Escaping text in the input stream} +\tokcycle +{\addcytoks{\plusl{#1}}} +{\processtoks{#1}} +{\addcytoks{#1}} +{\addcytoks{#1}} +{This \fbox{code is a test + |(I can also escape text)|} + of |\rule{1em}{.5em}| + {\bfseries mine}.} +\the\cytoks +\end{exampleA} + +\endgroup + +\subsubsection{Unexpandable, unexpanded, and expanded Character + directives\label{s:unex}} + +\begingroup +These examples concern the issue of whether the characters of the input + stream are transformed before or after being placed in the + output token register (\macname{cytoks}).\medskip + +\noindent\begin{minipage}[t]{\textwidth} +Transform characters (+1 ASCII) via unexpandable macro (see section~% + \ref{s:esc} for \macname{plusl} definition):\medskip + +\begin{exampleA}[lefthand width=8.3cm] + {Unexpandable Character directive} +\tokcycle +{\addcytoks{\plusl{#1}}} +{\processtoks{#1}} +{\addcytoks{#1}} +{\addcytoks{#1}}{% + This \textit{code \textup{is} a test} of mine.} +\the\cytoks +\end{exampleA} + +\altdecytoks + +\par\noindent\hrulefill\medskip +\end{minipage} +\endgroup + +\noindent\begin{minipage}[t]{\textwidth} +Capitalize vowels (but don't expand the character directive) + +\begingroup + +\begin{exampleC}[]{The expandable\macname{vowelcap} macro} +\newcommand\vowelcap[1]{% + \ifx a#1A\else + \ifx e#1E\else + \ifx i#1I\else + \ifx o#1O\else + \ifx u#1U\else + #1\fi\fi\fi\fi\fi +} +\end{exampleC} + +\begin{exampleA}[lefthand width=8.3cm] + {Not expanded Character directive} +\tokcycle +{\addcytoks{\vowelcap{#1}}} +{\processtoks{#1}} +{\addcytoks{#1}} +{\addcytoks{#1}}{% + This \textit{code \textup{is} a test} of mine.} +\the\cytoks +\end{exampleA} + +\altdecytoks + +\par\noindent\hrulefill\medskip +\endgroup +\end{minipage} + +\begingroup +Capitalize vowels (expanding the character directive) + +\begin{exampleA}[lefthand width=8.3cm] + {Expanded Character directive} +\tokcycle +{\addcytoks[x]{\vowelcap{#1}}} +{\processtoks{#1}} +{\addcytoks{#1}} +{\addcytoks{#1}}{% + This \textit{code \textup{is} a test} of mine.} +\the\cytoks +\end{exampleA} + +\altdecytoks + +\endgroup + +\subsubsection{Unexpanded vs. pre-expanded input stream} + +\begingroup +\Characterdirective{\addcytoks[x]{\vowelcap{#1}}} + +\noindent\begin{minipage}[t]{\textwidth} +See section~\ref{s:unex} for the \macname{vowelcap} definition. + +\begin{exampleA}[lefthand width=5.5cm]{Normal token cycle (input stream + not pre-expanded)} +\tokcycle +{\addcytoks[x]{\vowelcap{#1}}} +{\processtoks{#1}} +{\addcytoks{#1}} +{\addcytoks{#1}}% +{This \fbox{code + is a test \today} of + {\bfseries mine}.} +\the\cytoks +\end{exampleA} + +\altdecytoks + +\noindent\hrulefill\strut +\end{minipage} + +Note that, when pre-expanding the input stream, one must + \macname{noexpand} the macros that are \textit{not} + to be pre-expanded. + +\begin{exampleA}[lefthand width=5.5cm]{Pre-\macname{expanded} token cycle + input stream} +\expandedtokcyclexpress +{This \noexpand\fbox{code + is a test \today} of + {\noexpand\bfseries mine}.} +\the\cytoks +\end{exampleA} + +\altdecytoks + +\endgroup + +\subsection{Grouping} + +Differentiating explicit groups, e.g., \{...\}, from + implicit groups, e.g. \macname{bgroup}...\macname{egroup}, + is done automatically by \TokCycle. +The user has options on how \TokCycle{} should treat these tokens. +The desired options are to be set prior to the \TokCycle{} invocation. + + +\subsubsection{Treatment options for implicit groups} + +\begingroup + +The macro +\macname{stripimplicitgroupingcase} can take three possible + integer arguments: 0 (default) to automatically place unaltered + implicit group tokens in the output register; 1 to strip + implicit group tokens from the output; or $-$1 to instead pass the + implicit group tokens to the Character directive (as implicit + tokens) for separate processing (typically, when detokenization + is desired).\smallskip + +In the following example, see section~\ref{s:unex} for the + \macname{vowelcap} definition. + +\begin{exampleA}[lefthand width=6cm]{Using + \macname{stripimplicitgroupingcase} to affect treatment of + implicit grouping tokens} +\resettokcycle +\Characterdirective{\addcytoks[x]{% + \vowelcap{#1}}} +\def\z{Announcement: + {\bfseries\bgroup\itshape + Today \egroup it is} \today, + a Wednesday} +\expandafter\tokencyclexpress\z +\endtokencyclexpress\medskip + +\detokenize\expandafter{\the\cytoks} +\bigskip + +\stripimplicitgroupingcase{1} +\expandafter\tokencyclexpress\z +\endtokencyclexpress\medskip + +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +\endgroup + +\subsubsection{Treatment options for explicit groups} + +\begingroup + +For explicit group tokens, e.g., \{ \}, there are only two + options to be had. +These are embodied in the if-condition \macname{ifstripgrouping} + (default \macname{stripgroupingfalse}). +Regardless of which condition is set, the tokens within the explicit + group are still passed to the Group directive for processing. + + +Permutations of the following code are used in the subsequent examples. +Group stripping, brought about by \macname{stripgroupingtrue}, + involves removing the grouping braces from around the group. +The choice of \macname{processtoks} vs.\@ \macname{addcytoks} + affects whether the tokens inside the group are recommitted + to \TokCycle{} for processing, or are merely sent to the output + register in their original unprocessed form. + +Note that, in these examples, underdots and visible spaces + will only appear on characters and spaces that have been directed + to the Character and Space directives, respectively. +Without \macname{processtoks}, that will not occur to tokens + \textit{inside} of groups. + +\begin{exampleB}[listing only,lefthand width = \dimexpr\textwidth-19pt] +{Code permutations on group stripping and inner-group token processing} +\stripgroupingfalse OR \stripgroupingtrue +\tokcycle{\addcytoks{\underdot{#1}}} + {\processtoks{#1}} OR {\addcytoks{#1}}} + {\addcytoks{#1}} + {\addcytoks{\textvisiblespace}} +{This \fbox{is a \fbox{token}} test.} +\the\cytoks +\end{exampleB} + + +\begin{exampleB}[text only, colback=white]% +{\macname{stripgroupingfalse \string\processtoks} } +\stripgroupingfalse% RETAIN GROUPING (PACKAGE DEFAULT) +\tokcycle{\addcytoks{\underdot{#1}}} + {\processtoks{#1}}% GROUP TOKENS ARE PROCESSED + {\addcytoks{#1}} + {\addcytoks{\textvisiblespace}}{% +This \fbox{is a \fbox{token}} test.} +\the\cytoks +\end{exampleB} + +\begin{exampleB}[text only, colback=white]% +{\macname{stripgroupingfalse \string\addcytoks} } +\tokcycle{\addcytoks{\underdot{#1}}} + {\addcytoks{#1}}% GROUP TOKENS OUTPUT DIRECTLY + {\addcytoks{#1}} + {\addcytoks{\textvisiblespace}}{% +This \fbox{is a \fbox{token}} test.} +\the\cytoks +\end{exampleB} + +\begin{exampleB}[text only, colback=white]% +{\macname{stripgroupingtrue \string\processtoks} } +\stripgroupingtrue% DANGEROUS AS IT REMOVES GROUPING FROM ARGUMENTS +\tokcycle{\addcytoks{{\underdot{#1}}}}% EXTRA GROUPING TO COMPENSATE + {\processtoks{#1}}% GROUP TOKENS PROCESSED + {\addcytoks{#1}} + {\addcytoks{\textvisiblespace}}{% +This \fbox{is a \fbox{token}} test.} +\the\cytoks +\end{exampleB} + +\begin{exampleB}[text only, colback=white]% +{\macname{stripgroupingtrue \string\addcytoks} } +\stripgroupingtrue% DANGEROUS AS IT REMOVES GROUPING FROM ARGUMENTS +\tokcycle{\addcytoks{\underdot{#1}}} + {\addcytoks{#1}}% GROUP TOKENS OUTPUT DIRECTLY + {\addcytoks{#1}} + {\addcytoks{\textvisiblespace}}{% +This \fbox{is a \fbox{token}} test.} +\the\cytoks +\end{exampleB} + +Note that the content of groups can be altogether eliminated if + \textit{neither} \macname{processtoks\{\#1\}} nor + \macname{addcytoks\{\#1\}} are used in the Group directive. + + + +\endgroup + +\subsubsection{Group nesting\label{s:nest}} + +\begingroup + +\begin{exampleC}[]{The \macname{reducecolor} and \macname{restorecolor} + macros} +\newcounter{colorindex} +\newcommand\restorecolor{\setcounter{colorindex}{100}} +\newcommand\reducecolor[1]{% + \color{red!\thecolorindex!cyan}% + \addtocounter{colorindex}{-#1}% + \ifnum\thecolorindex<1\relax\setcounter{colorindex}{1}\fi} +\end{exampleC} + +\begin{exampleA}[lefthand width=6cm]{Group nesting is no impediment to + tokcycle} +\restorecolor +\tokcycle + {\addcytoks{(#1)}} + {\addcytoks{\reducecolor{11}}% + \addcytoks{[}\processtoks{#1}% + \addcytoks{]}} + {\addcytoks{#1}} + {}{% + {1{{3{{5{{7{{9{1{}0}}8}}6}}4}}2}}} +\the\cytoks +\end{exampleA} + +\endgroup + + +\subsection{Direct use of \TokCycle} + +While one may find it convenient to encapsulate \TokCycle{} commands + inside of other macros, it should be obvious that this is not a requirement. +\TokCycle[T]{} macros and environments (in regular or \texttt{xpress} + form) may be invoked directly from the document, without + being first encapsulated within a macro or environment. + +\subsubsection{Modifying counters as part of the Character + directive} + +\begingroup +In the following example, see section~\ref{s:nest} for the definitions + of \macname{restorecolor} and \macname{reducecolor}. + +\begin{exampleA}[lefthand width=8.5cm]{Using a period token (.) to + reset a changing color} +\restorecolor +\tokencycle + {\addcytoks{\bgroup\reducecolor{3}#1\egroup}% + \ifx.#1\addcytoks{\restorecolor}\fi} + {\processtoks{#1}} + {\addcytoks{#1}} + {\addcytoks{#1}}% +This right \textit{here is a sentence in italic}. +And \textbf{here we have another sentence in bold}. + +{\scshape Now in a new paragraph, the sentence +is long.} Now, it is short. +\endtokencycle +\end{exampleA} + +\endgroup + +\subsection{Macro encapsulation of \TokCycle} + +\subsubsection{Spacing out text} + +\begingroup + +\begin{exampleC}[]{The \macname{spaceouttext} macro} +\newcommand\spaceouttext[2]{% + \tokcycle + {\addcytoks{##1\nobreak\hspace{#1}}}% + {\processtoks{##1}} + {\addcytoks{##1}}% + {\addcytoks{##1\hspace{#1}}} + {#2}% + \the\cytoks\unskip} +\end{exampleC} + +\begin{exampleA}[lefthand width=6cm]{\macname{spaceouttext} demo} +\spaceouttext{3pt plus 3pt}{This + \textit{text \textbf{is} + very} spaced out}. Back + to regular text. + +\spaceouttext{1.5pt}{This + \textit{text \textbf{is} + somewhat} spaced out}. + Back to regular text. +\end{exampleA} + +\endgroup + + +\subsubsection{Alternate presentation of detokenized content} + +\begingroup +This macro attempts to give a more natural presentation of + \macname{detokenize}'d material. +It is \textbf{not} to be confused as a replacement for + \macname{detokenize}. +In certain applications, it may offer a more pleasingly formatted + typesetting of detokenized material.\smallskip + +It is an unusual application of \TokCycle{} in that it does not + actually use the \macname{cytoks} token register to collect + its output. +This is only possible because all macros in the input stream + are detokenized, rather than executed. + +\medskip + +\begin{exampleC}[]{The \macname{altdetokenize }\ macro} +\newif\ifmacro +\newcommand\altdetokenize[1]{\begingroup\stripgroupingtrue\macrofalse + \tokcycle + {\ifmacro\def\tmp{##1}\ifcat\tmp A\else\unskip\allowbreak\fi\macrofalse\fi + \detokenize{##1}} + {\ifmacro\unskip\macrofalse\fi\{\processtoks{##1}\ifmacro\unskip\fi\}\allowbreak} + {\tctestifx{\\##1}{\\}{\ifmacro\unskip\allowbreak\fi + \allowbreak\detokenize{##1}\macrotrue}} + { \hspace{0pt plus 3em minus .3ex}} + {#1}% + \unskip +\endgroup} +\end{exampleC} + +\let\mac\relax +\begin{exampleA}[lefthand width=6.3cm] + {\macname{altdetokenize} demo} +\string\altdetokenize: \\ +\texttt{\altdetokenize{a\mac a \mac2 {\mac}\mac{a\mac\mac}\mac}}! + +\string\detokenize: \\ + \texttt{\detokenize{a\mac a \mac2 {\mac}\mac{a\mac\mac}\mac}}! +\end{exampleA} +\endgroup + + +\subsubsection{Capitalize all words, including compound and + parenthetical words} + +\begingroup + +\begin{exampleC}[]{The \macname{Titlecase} and \texttt{\string\nextcap} macros} +\newcommand\TitleCase[1]{% + \def\capnext{T} + \tokcycle + {\addcytoks{\nextcap{##1}}} + {\processtoks{##1}} + {\addcytoks{##1}} + {\addcytoks{##1\def\capnext{T}}} + {#1}% + \the\cytoks +} +\newcommand\nextcap[1]{% + \edef\tmp{#1}% + \tctestifx{-#1}{\def\capnext{T}}{}% + \tctestifcon{\if T\capnext}% + {\tctestifcon{\ifcat\tmp A}% + {\uppercase{#1}\def\capnext{F}}% + {#1}}% + {#1}% +} +\end{exampleC} + +\begin{exampleA}[]{A demo of \macname{Titlecase} showing raw (escaped) + input and processed output} +\TitleCase{% +|here, {\bfseries\today{}, is [my]} + really-big-test + (\textit{capitalizing} words).| + + here, {\bfseries\today{}, is [my]} + really-big-test + (\textit{capitalizing} words).} +\end{exampleA} + +\endgroup + +\subsubsection{Scaling rule dimensions} +\begingroup + +\ddbend This example only applies if one can guarantee that + the input stream will contain only text and rules... + +\begin{exampleC}[]{The \macname{growdim} macro} +\newcommand\growdim[2]{% +\tokcycle{\addcytoks{##1}} + {\addcytoks{#1\dimexpr##1}} + {\addcytoks{##1}} + {\addcytoks{##1}}{% + #2}% +\the\cytoks} +\end{exampleC} + +\begin{exampleA}[]{Using \TokCycle{} to change \macname{rule} dimensions} +\growdim{2}{This rule is exactly 4pt: + \rule|{4pt}{4pt}| , whereas this + rule is 2x bigger than 4pt: + \rule{4pt}{4pt} .}\par +\growdim{4}{This rule is exactly 5pt: + \rule|{5pt}{5pt}| , whereas this + rule is 4x bigger than 5pt: + \rule{5pt}{5pt} .} +\end{exampleA} + +\endgroup + +\subsubsection{String search, including non-regex material} + +\begingroup + +\begin{exampleC}[]{The \macname{findinstring} macro for string searches} +\newcommand\findinstring[2]{\begingroup% + \stripgroupingtrue + \setcounter{runcount}{0}% + \tokcycle + {\nextctltok{##1}} + {\nextctltok{\opengroup}\processtoks{##1}\nextctltok{\closegroup}} + {\nextctltok{##1}} + {\nextctltok{\tcspace}} + {#1}% + \edef\numlet{\theruncount}% + \expandafter\def\expandafter\searchword\expandafter{\the\cytoks}% +% + \aftertokcycle{\matchfound}% + \setcounter{runcount}{0}% + \def\matchfound{F}% + \tokcycle + {\nextcmptok{##1}} + {\nextcmptok{\opengroup}\processtoks{##1}\nextcmptok{\closegroup}} + {\nextcmptok{##1}} + {\nextcmptok{\tcspace}} + {#2}% +\endgroup} +\newcounter{runcount} +\makeatletter +\newcommand\rotcytoks[1]{\cytoks\expandafter\expandafter\expandafter{% + \expandafter\tc@gobble\the\cytoks#1}} +\makeatother +\newcommand\testmatch[1]{\ifx#1\searchword\gdef\matchfound{T}\fi}% +\newcommand\rotoradd[2]{\stepcounter{runcount}% + \ifnum\theruncount>\numlet\relax#1\else#2\fi + \expandafter\def\expandafter\tmp\expandafter{\the\cytoks}} +\newcommand\nextcmptok[1]{\rotoradd{\rotcytoks{#1}}{\addcytoks{#1}}\testmatch{\tmp}} +\newcommand\nextctltok[1]{\stepcounter{runcount}\addcytoks{#1}} +\end{exampleC} + +\begin{exampleA}[lefthand width=11.5cm]{Demo of the + \macname{findinstring} macro} +1. \findinstring{this}{A test of the times} + \findinstring{the} {A test of the times}\par +2. \findinstring{This is}{Here, This is a test} + \findinstring{Thisis} {Here, This is a test}\par +3. \findinstring{the} {This is the\bfseries{} test} + \findinstring{he\bfseries}{This is the\bfseries{} test}\par +4. \findinstring{a{bc}} {gf{vf{a{b c}g}gh}hn} + \findinstring{a{b c}}{gf{vf{a{b c}g}gh}hn}\par +5. \findinstring{a\notmymac{b c}}{gf{vf{a\mymac{b c}g}gh}hn} + \findinstring{a\mymac{b c}} {gf{vf{a\mymac{b c}g}gh}hn}\par +6. \findinstring{\textit{Italic}}{this is an \textit{italic} test} + \findinstring{\textit{italic}}{this is an \textit{italic} test} +\end{exampleA} + +\endgroup + + +\subsection{\TokCycle-based environments} + +The \macname{tokcycleenvironment} macro allows users to define + their own \TokCycle{} environments. +Here are some examples. + +\subsubsection{``Removing'' spaces, but still breakable/hyphenatable} + +\begingroup + +\begin{exampleC}[]{The \macname{spaceBgone} environment} +\tokcycleenvironment\spaceBgone + {\addcytoks{##1}} + {\processtoks{##1}} + {\addcytoks{##1}} + {\addcytoks{\hspace{.2pt plus .2pt minus .8pt}}}% +\end{exampleC} + +\begin{exampleA}[lefthand width=6.0cm]{} +\spaceBgone + Here we have a \textit{test} of + whether the spaces are removed. + We are choosing to use the + tokencycle environment. + + We are also testing the use of + paragraph breaks in the + environment. +\endspaceBgone +\end{exampleA} + +\endgroup + +\subsubsection{Remapping text} + +\begin{minipage}[t]{\textwidth} +\begingroup + +\begin{exampleC}[]{The \macname{remaptext} environment with + supporting macros} +\tokcycleenvironment\remaptext + {\addcytoks[x]{\tcremap{##1}}} + {\processtoks{##1}} + {\addcytoks{##1}} + {\addcytoks{##1}} +\newcommand*\tcmapto[2]{\expandafter\def\csname tcmapto#1\endcsname{#2}} +\newcommand*\tcremap[1]{\ifcsname tcmapto#1\endcsname + \csname tcmapto#1\endcsname\else#1\fi} +\tcmapto am \tcmapto bf \tcmapto cz \tcmapto de \tcmapto ey +\tcmapto fl \tcmapto gx \tcmapto hb \tcmapto ic \tcmapto jn +\tcmapto ki \tcmapto lr \tcmapto mh \tcmapto nt \tcmapto ok +\tcmapto ps \tcmapto qa \tcmapto ro \tcmapto sq \tcmapto tw +\tcmapto uj \tcmapto vp \tcmapto wd \tcmapto xg \tcmapto yu +\tcmapto zv +\end{exampleC} + +\begin{exampleA}[lefthand width=8.5cm]{Demo of \macname{remaptext}} +\remaptext +What can't we \textit{accomplish} if we try? + +Let us be of good spirit and put our minds to it! +\endremaptext +\end{exampleA} + +Because \macname{tcremap} is expandable, the original text is totally absent + from the processed output: + +\altdecytoks + +\endgroup +\end{minipage} + +\subsection{Advanced topics: implicit tokens and catcode changes} + +\subsubsection{Trap Active Characters (catcode 13)} + +\begin{minipage}[t]{\textwidth} +\begingroup +Active characters in the \TokCycle{} input stream are processed in + their original form. +Their active substitutions only occur \textit{afterwards}, when + the \TokCycle{} output is typeset. +They may be identified in the Character directive with + \macname{ifactivetok}. + +\let\svt T +\Characterdirective{\tctestifcon\ifactivetok + {\addcytoks{\fbox{#1}}}{\addcytoks{#1}}} + +\begin{exampleA}[lefthand width=7.8cm]{Processing active characters} +\resettokcycle +\tokencyclexpress +This is a test!!\endtokencyclexpress + +\catcode`!=\active +\def !{?} +\tokencyclexpress +This is a test!!\endtokencyclexpress + +\Characterdirective{\tctestifcon\ifactivetok + {\addcytoks{\fbox{#1}}}{\addcytoks{#1}}} +\tokencyclexpress +This is a test!!\endtokencyclexpress + +\catcode`T=\active +\let T+ +\tokencyclexpress +This is a test!!\endtokencyclexpress + +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +%%% +\catcode`!=\active +\def !{?} +\catcode`T=\active +\let T+ +\ddbend If the input stream is pre-\textit{expanded}, the active + substitutions involving \macname{def} (but not \macname{let}) + are made before reaching {\catcode`T=11 \TokCycle} processing. +Such tokens are no longer detected as active, unless + \macname{noexpand} is applied to the pre-expansion: + +\begin{exampleA}[lefthand width=7.8cm]{Expanded input stream acts + upon active characters unless \macname{noexpand} is applied} +\expandedtokcyclexpress{This is a test!\noexpand!} +\the\cytoks\par +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +However, pre-\textit{tokenization} does not suffer this behavior: +\begin{exampleA}[lefthand width=7.8cm]{Pre-tokenized input stream + does not affect active characters} +\def\tmp{This is a test!!} +\expandafter\tokcyclexpress\expandafter{\tmp} +\the\cytoks\par +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +%%% +\endgroup +\end{minipage} + +\subsubsection{Trap Catcode 6 (explicit \& implicit) tokens} + +\begingroup + +\let\myhash# + +\medskip + +Typically, cat-6 tokens (like \texttt{\#}) are used to designate the + following digit (1-9) as a parameter. +Since they are unlikely to be used in that capacity inside a + \TokCycle{} input stream, the package behavior is to convert them + into something cat-12 and set the if-condition \macname{catSIXtrue}. +In this manner, \macname{ifcatSIX} can be used inside the + Character directive to convert cat-6 tokens into something of + the user's choosing.\smallskip + +As to this cat-12 conversion, explicit cat-6 characters are converted + into the same character with cat-12. +On the other hand, implicit cat-6 macros (e.g., \verb|\let\myhash#|) + are converted into a fixed-name macro, \macname{implicitsixtok}, + whose cat-12 substitution text is a \macname{string} of the original + implicit-macro name. + +\resettokcycle +\Characterdirective{\ifcatSIX + \addcytoks{\fbox{#1}} + \else\addcytoks{#1}\fi} + +%\tokencyclexpress This# isQ \textit{a Q# test\myhash}!\endtokencyclexpress + +\begin{exampleA}[lefthand width=6.5cm]{Treatment of cat-6 tokens} +\resettokcycle +\Characterdirective{\ifcatSIX + \addcytoks{\fbox{#1}} + \else\addcytoks{#1}\fi} +\let\myhash# +\tokcyclexpress{This# isQ + \textit{a Q# test\myhash}!} +\the\cytoks\bigskip\par +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +%\tokcyclexpress{This# isQ \textit{a Q# t\,e\, s\,t\myhash}!}\the\cytoks{} +%with 1-character macros + +%%% +%\expandedtokcyclexpress{This# isQ \noexpand\textit{a Q# test\myhash}!}\the\cytoks +%{} PRE-EXPANDED +%%% + + +%\tokencyclexpress This# isQ \textit{a Q# test}!\endtokencyclexpress + +%\Characterdirective{\ifcatSIX\addcytoks{\fbox{#1}}\else\addcytoks{#1}\fi} + +\begin{exampleA}[lefthand width=6.5cm]{Multiple explicit cat-6 tokens are not + a problem} +\catcode`Q=6 +\tokcyclexpress{This# isQ + \textit{a Q# test\myhash}!} +\the\cytoks +\end{exampleA} + +\catcode`Q=11 +\hrulefill + +\ddbend For what is, perhaps, a rare situation, one can even process input + streams that contain cat-6 macro parameters. +A package macro, \macname{whennotprocessingparameter\#1\{<}% + \textit{directive when not a parameter}\verb|>}|, can be + used inside of the Character directive to intercept parameters. +In this example, a macro is defined and then executed, subject to + token replacements brought about by the expandable Character directive. + +\smallskip +In the following example, see section~\ref{s:unex} for + the \macname{vowelcap} definition. + +\Characterdirective{% + \whennotprocessingparameter#1{\addcytoks[x]{\vowelcap{#1}}}} +\aftertokcycle{\global\cytoks\expandafter{\the\cytoks}} +\def\Characterdirective#1{} +\begin{exampleA}[lefthand width=7cm]{Preserving parameters (e.g. + \texttt{\#1, \#2}) in the \TokCycle{} input stream} +\Characterdirective{% + \whennotprocessingparameter#1{% + \addcytoks[x]{\vowelcap{#1}}}} +\tokcyclexpress{% + \def\zQ#1#2{[one:#1](two:#2)} + This is a \zQ big test. + + \renewcommand\zQ[2]{\ifx t#1[#1]\fi(#2)} + This is a \zQ test.} +\the\cytoks +\end{exampleA} + +\altdecytoks + +\endgroup + +\subsubsection{Trap implicit tokens in general} + +\begingroup + +Implicit macros (assigned via \macname{let}) were already + mentioned in the context of cat-6. +However, implicit macros can be of any valid catcode. +The condition \macname{ifimplicittok} is used to flag such + tokens for special processing. + +\fboxsep=1.5pt + +\begin{exampleA}[lefthand width=8cm]{Implicit\ensuremath{{}={}}% + single box\char44 \ cat-6\ensuremath{{}={}}double box\char44 \ % + implicit-cat-6\ensuremath{{}={}}triple box} +\let\littlet=t +\let\littlel=l +\let\svhash# +\Characterdirective{\ifimplicittok + \ifcatSIX\addcytoks{\fbox{\fbox{\fbox{#1}}}}% + \else\addcytoks{\fbox{#1}}\fi\else\ifcatSIX + \addcytoks{\fbox{\fbox{#1}}}\else + \addcytoks{#1}\fi\fi} + +\tokencyclexpress We wi\littlel\littlel# + \textit{ make a \littlet est #} \littlet + +This \textit{is a \textbf{big}} \littlet est. + +Next pa#agraph ending with implicit cat six + \svhash.\endtokencyclexpress +\end{exampleA} + +\ddbend If the input stream is subject to pre-expansion, one + will require \macname{noexpand} for macros where no pre-expansion + is desired.\medskip + +\ddbend If the input stream is provided pre-tokenized via + \macname{def}, \TeX{} convention requires cat-6 tokens to appear + in the input stream as duplicate, e.g. \texttt{\#\#}. + + +\subsubsection{Changing grouping tokens (catcodes 1,2)} + +\begingroup + +Changing grouping tokens (catcodes 1,2) may require something more, + if the output stream is to be detokenized. +In the following examples, pay attention to the + detokenized grouping around the argument to \macname{fbox}.\smallskip + +As we will see, the issues raised here only affect the situation + when detokenization of the output stream is required. +\medskip + +\begin{exampleA}[]{\TokCycle{} defaults grouping tokens to braces:} +\tokencycle + {\addcytoks{(#1)}} + {\processtoks{#1}} + {\addcytoks{#1}} + {\addcytoks{ }} +This \fbox{is a} test. +\endtokencycle\medskip + +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +One can make brackets cat-1,2, redefining bgroup/egroup to [ ]. +However, while one can now use brackets in input stream, braces + will still appear in the detokenized \TokCycle{} output stream: + +\begin{exampleA}[lefthand width=6.5cm] +{\TokCycle{} will not automatically change its grouping tokens} +\catcode`\[=1 +\catcode`\]=2 +\let\bgroup[ +\let\egroup] +\tokencycle + {\addcytoks{(#1)}} + {\processtoks{#1}} + {\addcytoks{#1}} + {\addcytoks{ }} +This \fbox[is a] test. +\endtokencycle\medskip + +\detokenize\expandafter{\the\cytoks} +\end{exampleA} + +If it is necessary to reflect revised grouping tokens in the + output stream, the \macname{settcgrouping} macro is + to be used. + +\begin{exampleA}[lefthand width=6.5cm] +{Redefine \TokCycle{} grouping tokens as angle + brackets using \macname{settcGrouping}} +\catcode`\<=1 +\catcode`\>=2 +\catcode`\{=12 +\catcode`\}=12 +\let\bgroup< +\let\egroup> +\settcGrouping<<#1>> +\tokencycle + <\addcytoks<(#1)>> + <\processtoks<#1>> + <\addcytoks<#1>> + <\addcytoks< >> +This \fbox<is a> test. +\endtokencycle\medskip + +\detokenize\expandafter<\the\cytoks> +\end{exampleA} + +Angle brackets are now seen in the above detokenization. +Until subsequently changed, cat-1,2 angle brackets now appear in + detokenized \TokCycle{} groups, even if other cat-1,2 tokens were + used in the input stream. +Bottom line: +\begin{itemize} +\item adding, deleting, or changing catcode 1,2 explicit + grouping tokens, e.g., \{\}, (in conjunction with their + associated implicit \macname{bgroup}\macname{egroup}) + tokens will not affect \TokCycle's ability to digest proper + grouping of the input stream, regardless of which tokens + are catcode 1,2 at the moment. + +\item The grouping tokens used in \TokCycle's output + default to \verb|{}| braces (with cat-1,2), but can be + changed deliberately using \macname{settcGrouping}. + +\item The package, currently, has no way to reproduce in the + output stream the actual grouping tokens that occur in the + input stream, but one should ask, for the particular + application, if it really matters, as long as the the + proper catcodes-1,2 are preserved? + +\end{itemize} + +\endgroup + +\subsubsection{Catcode 10 space tokens} + +\begingroup + +Here we demonstrate that \TokCycle{} can handle arbitrary redesignation + of tokens to cat-10, as well as implicit space tokens. + +\ddbend While it should seem natural, we note that implicit + space tokens are directed to the Space directive rather than + the Character directive. +However, \macname{ifimplicittok} may still be used to + differentiate an explicit space from an implicit one. +\medskip + +Note in the following examples that cat-10 tokens do \textit{not} + get under-dots. +The next three examples all use the same input, but with different +catcode settings for the space and the underscore. + +\fboxsep=1pt + +\begin{exampleA}[lefthand width=10cm]{space cat-10{,} underscore cat-12} +\catcode`\_=12 % +\catcode`\ =10 % + +\tokencycle{\addcytoks{\underdot{#1}}}% +{\processtoks{#1}}% +{\addcytoks{#1}}% +{\addcytoks{#1}}% +\fbox{a_c d} b_g\itshape f\upshape\endtokencycle +\end{exampleA} +% +\begin{exampleA}[lefthand width=10cm]{space cat-10{,} underscore cat-10} +\catcode`\_=10 % +\catcode`\ =10 % + +\tokencycle{\addcytoks{\underdot{#1}}}% +{\processtoks{#1}}% +{\addcytoks{#1}}% +{\addcytoks{#1}}% +\fbox{a_c d} b_g\itshape f\upshape\endtokencycle +\end{exampleA} + +% + +\begin{exampleA}[lefthand width=10cm]{space cat-12{,} underscore cat-10} +\catcode`\_=10 % +\catcode`\ =12 % + +\tokencycle{\addcytoks{\underdot{#1}}}% +{\processtoks{#1}}% +{\addcytoks{#1}}% +{\addcytoks{#1}}% +\fbox{a_c d} b_g\itshape f\upshape\endtokencycle +\end{exampleA} + +%%% +\resettokcycle +\Characterdirective{\addcytoks{\underdot{#1}}} + +\begin{exampleA}[lefthand width=10cm]{Implicit spaces work{,} too} +\resettokcycle +\Characterdirective{\addcytoks{\underdot{#1}}} +\def\:{\let\mysptoken= } \: % +\catcode`\_=10 % +\catcode`\ =12 % + +\tokencyclexpress +\fbox{a\mysptoken{}c d} b_g\itshape f\upshape +\endtokencyclexpress +\end{exampleA} + +\endgroup + + +\subsubsection{Changes to catcode 0} + +\begingroup + +\begin{exampleA}[]{Cat-0 changes are not a hindrance to \TokCycle} +\let\littlet=t +\catcode`\! 0 !catcode`!\ 12 +!Characterdirective{!ifimplicittok + !addcytoks{!fbox{#1}}!else!ifcatSIX + !addcytoks{!fbox{!fbox{#1}}} + !else!addcytoks{#1}!fi!fi} +!tokencyclexpress Here, {!scshape!bgroup + on !today!itshape{} we are !egroup + !littlet es!littlet ing} cat-0 + changes{!bgroup}!egroup +!endtokencyclexpress!medskip + +!detokenize!expandafter{!the!cytoks} +\end{exampleA} + +\endgroup + + +\end{document} + diff --git a/Master/texmf-dist/tex/generic/tokcycle/tokcycle.sty b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.sty new file mode 100644 index 00000000000..c61abb55e4b --- /dev/null +++ b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.sty @@ -0,0 +1,4 @@ +\input tokcycle.tex +\ProvidesPackage\tcname[\tcdate\space V\tcver\space Cycle through and transform + a stream of tokens] +\endinput
\ No newline at end of file diff --git a/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex new file mode 100644 index 00000000000..d6f1b4003ef --- /dev/null +++ b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex @@ -0,0 +1,192 @@ +\def\tcname {tokcycle} +\def\tcver {1.0} +% +\def\tcdate {2019/8/21} +% +% Author : Steven B Segletes, Christian Tellechea (contributor) +% Maintainer : Steven B Segletes +% License : Released under the LaTeX Project Public License v1.3c +% or later, see http://www.latex-project.org/lppl.txt +% Files : 1) tokcycle.tex +% 2) tokcycle.sty +% 3) tokcycle-doc.tex +% 4) tokcycle-doc.pdf +% 5) tokcycle-examples.tex +% 6) tokcycle-examples.pdf +% 7) README +%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% +% MACRO FORM +\long\def\tokcycle#1#2#3#4#5{\tokcycraw{#1}{#2}{#3}{#4}#5\endtokcycraw} +% \expanded-ARGUMENT MACRO FORM +\long\def\expandedtokcycle#1#2#3#4#5{\cytoks{\tokcycraw{#1}{#2}{#3}{#4}}% + \expandafter\the\expandafter\cytoks\expanded{#5}\endtokcycraw} +% ENVIRONMENT FORM +\long\def\tokencycle#1#2#3#4{\begingroup\let\endtokencycle\endtokcycraw + \aftertokcycle{\the\cytoks\expandafter\endgroup\expandafter\tcenvscope + \expandafter{\the\cytoks}}\tokcycraw{#1}{#2}{#3}{#4}} +% XPRESS-INTERFACE MACRO FORM +\long\def\tokcyclexpress#1{\tokcycrawxpress#1\endtokcycraw} +% XPRESS-INTERFACE \expanded-ARGUMENT MACRO FORM +\long\def\expandedtokcyclexpress#1{% + \expandafter\tokcycrawxpress\expanded{#1}\endtokcycraw} +% XPRESS-INTERFACE ENVIRONMENT FORM +\def\tokencyclexpress{\begingroup\let\endtokencyclexpress\endtokcycraw + \aftertokcycle{\the\cytoks\expandafter\endgroup\expandafter\tcenvscope + \expandafter{\the\cytoks}}\tokcycrawxpress} +% INITIALIZATION & INTERNAL TOOLS +\def\tcenvscope{\cytoks}% CAN SET TO \global\cytoks TO OVERCOME SCOPE LIMITS +\edef\restorecatcode{\catcode\number`\@=\number\catcode`\@\relax} +\catcode`\@11 +\newif\iftc@implicitgrp +\newif\if@argnext +\newtoks\tc@tok +\newcount\tc@depth +\def\tc@gobble#1{} +\def\tc@deftok#1#2{\let#1= #2\empty} +\tc@deftok\tc@sptoken{ } +\expandafter\def\expandafter\tc@absorbSpace\space{} +\def\tc@ifempty#1{\tc@testxifx{\expandafter\relax\detokenize{#1}\relax}} +\def\tc@defx#1#2{\tc@earg{\def\expandafter#1}{#2}} +\long\def\tc@earg#1#2{\expandafter#1\expandafter{#2}} +\long\def\tc@xarg#1#2{\tc@earg#1{\expanded{#2}}} +\long\def\tc@exfirst#1#2{#1} +\long\def\tc@exsecond#1#2{#2} +\long\def\tc@testxifx{\tc@earg\tctestifx} +\def\tc@addtoks#1#2{\toks#1\expandafter{\the\toks#1#2}} +\def\addtc@depth{\advance\tc@depth 1} +\def\subtc@depth{\tc@depth=\numexpr\tc@depth-1\relax} +\def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse\catSIXfalse} +\long\def\count@stringtoks#1{\tc@earg\count@toks{\string#1}} +\def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt} +\def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}} +\def\sv@hash{##} +% EXTERNAL TOOLS +\long\def\tctestifcon#1{#1\expandafter\tc@exfirst\else\expandafter\tc@exsecond\fi} +\long\def\tctestifcatnx#1#2{\tctestifcon{\ifcat\noexpand#1\noexpand#2}} +\long\def\tctestifx#1{\tctestifcon{\ifx#1}} +\long\def\tctestifnum#1{\tctestifcon{\ifnum#1\relax}} +\newif\ifstripgrouping +\def\stripimplicitgroupingcase#1{\edef\@implicitgroupingcase{\the\numexpr1-#1}} +\newif\ifcatSIX +\newif\ifimplicittok +\newif\ifactivetok +\newtoks\cytoks +\long\def\tokcycleenvironment#1#2#3#4#5{\expandafter\def\expandafter#1% + \expandafter{\expandafter\let\csname end\expandafter\tc@gobble + \string#1\endcsname\endtokcycraw\tokencycle{#2}{#3}{#4}{#5}}} +\long\def\processtoks#1{\addtc@depth\@tokcycle#1\endtokcycraw } +\def\whennotprocessingparameter#1#2{\tctestifcon\if@argnext{\@argnextfalse\cytoks + \expandafter{\the\cytoks###1}}{\tctestifcon\ifcatSIX{\@argnexttrue}{#2}}} +% ESSENTIAL METHOD: STREAMING MACRO WITH TERMINATOR: +% \tokcycraw{<Char>}{<Group>}{<Macro>}{<Space>}<input-stream>\endtokcycraw +\long\def\tokcycraw#1#2#3#4{\def\@chrT##1{#1}\long\def\@grpT##1{#2}% + \long\def\@macT##1{#3}\def\@spcT##1{#4}\tokcycrawxpress} +% ENTRY POINT FOR XPRESS METHOD WITHOUT EXPLICIT ARGUMENTS +\def\tokcycrawxpress{\cytoks{}\tc@depth=1\relax\@tokcycle} +% CODE TO EXECUTE AT COMPLETION +\long\def\aftertokcycle#1{\def\@aftertokcycle{#1}} +\def\endtokcycraw{\subtc@depth\tctestifnum{\tc@depth=0}{\@aftertokcycle}{}} +% LOOP ENTRY POINT +\def\@tokcycle{\tc@resetifs\futurelet\tc@next\detect@CANTabsorb} +\def\detect@CANTabsorb{\tctestifx{\tc@next\tc@sptoken}{\stringify\@@@@@spcT}% + {\tctestifx{\tc@next\bgroup}{\stringify\@@@@grpT}% + {\can@absorb}}} +% NON cat1,10 TOKENS +\long\def\can@absorb#1{\tc@tok{#1}\trapcatSIX{#1}\expandafter\can@absorb@ + \the\tc@tok} +\long\def\can@absorb@#1{\tctestifnum{\count@stringtoks{#1}>1}% + {\tctestifx{\endtokcycraw#1}{#1}{\backslashcmds#1\@tokcycle}}% + {\trapactivetok#1\tc@trapescape#1{\tc@escapecytoks}{\@chrT#1\@tokcycle}}} +%CONVERT NEXT (SPACE OR BEGIN-GROUP) TOKEN TO STRING +\def\stringify#1{\expandafter#1\string}% #1 WILL BE \@@@@@spcT or \@@@@grpT +%SPACE DECODE +\def\@@@@@spcT{\futurelet\tc@str\@@@@spcT} +\def\@@@@spcT{\tctestifx{\tc@str\tc@sptoken}% + {\def\@tmp{\@@spcT{ }}\expandafter\@tmp\tc@absorbSpace}% EXPLICIT SPACE + {\implicittoktrue\expandafter\@@@spcT\tc@gobble}}% IMPLICIT SPACE +\def\@@@spcT{\csmk{\expandafter\@@spcT\thecs}} +\def\@@spcT#1{\@spcT{#1}\@tokcycle} +% GROUP DECODE +\def\@@@@grpT{\futurelet\tc@str\@@@grpT} +\def\@@@grpT#1{\tctestifnum{\number\catcode`#1=1}% + {\expandafter\@@grpT\expandafter{\iffalse}\fi}% { + {\implicittoktrue\tc@implicitgrptrue + \csmk{\expandafter\backslashcmds\thecs\@tokcycle}}}% \bgroup +\def\@@grpT#1{\tctestifcon\ifstripgrouping{\@grpT{#1}}% + {\groupedcytoks{\@grpT{#1}}}\@tokcycle} +% \ COMMANDS (MACROS AND IMPLICITS) +\long\def\test@ifmacro#1{\tctestifcatnx#1\relax}% REVISED! +\long\def\backslashcmds#1{\test@ifmacro#1{\tctestifcon\ifcatSIX{% + \implicittoktrue\@chrT#1}{\@macT#1}}{\implicittoktrue + \trapimplicitegrp#1\tctestifcon{\iftc@implicitgrp}% + {\ifcase\@implicitgroupingcase\or\addcytoks{#1}\or\@chrT{#1}\fi}% + {\@chrT#1}{\@chrT#1}}} +% SET UP ESCAPE MECHANISM +\def\settcEscapechar#1{\let\@tcEscapeptr#1% + \def\tc@escapecytoks##1#1{\addcytoks{##1}\@tokcycle}} +\def\tc@trapescape#1{\tctestifx{\@tcEscapeptr#1}} +% TRAP CAT-6 +\long\def\trapcatSIX#1{\tctestifcatnx#1\relax{}{\trapcatSIXb#1}} +\def\trapcatSIXb#1{\tc@earg\tctestifcatnx\sv@hash#1{\catSIXtrue\trapcatSIXc#1}{}} +\def\trapcatSIXc#1{\tctestifnum{\count@stringtoks{#1}>1}{\tc@defx\six@str{\string#1}% + \global\let\implicitsixtok\six@str\tc@tok{\implicitsixtok}}% + {\tc@tok\expandafter{\string#1}}} +% DIRECTIVES FOR HANDLING GROUPS RECURSIVELY; DEFINE tokcycle GROUPING CHARS +\def\@defgroupedcytoks#1{\long\def\groupedcytoks##1{% + \begingroup\cytoks{}##1\expandafter\endgroup\expandafter + \addcytoks\expandafter{\expandafter#1}}} +\def\settcGrouping#1{\def\@tmp##1{#1}\tc@defx\@@tmp{\@tmp{\the\cytoks}}% + \tc@earg\@defgroupedcytoks{\@@tmp}} +% FAUX TOKENIZATION OF COMMAND NAME (WHEN cat0 TOKEN HAS BEEN MADE cat12) +\def\csmk#1{\def\csaftermk{#1}\toks0{}\@csmkA} +\def\@csmkA{\futurelet\@tmp\@csmkB} +\def\@csmkB{\tctestifx{\@tmp\tc@sptoken}% + {\toks0{ }\expandafter\@csmkF\tc@absorbSpace}{\@csmkCA}} +\def\@csmkCA#1{\tc@addtoks0{#1}\tctestifnum{\number\catcode`#1=11}% + {\futurelet\@tmp\@csmkD}{\@csmkF}} +\def\@csmkC#1{\tctestifnum{\number\catcode`#1=11} + {\tc@addtoks0{#1}\futurelet\@tmp\@csmkD}{\@csmkE#1}} +\def\@csmkD{\tctestifcatnx 0\@tmp\@csmkC\@csmkE} +\def\@csmkE{\tctestifx{\@tmp\tc@sptoken}% + {\expandafter\@csmkF\tc@absorbSpace}{\@csmkF}} +\def\@csmkF{\tc@defx\thecs{\csname\the\toks0\endcsname}\csaftermk} +% TRAP IMPLICIT END GROUP TOK (e.g., \egroup); SET \iftc@implicitgrp +\def\trapimplicitegrp#1{\tctestifx{#1\egroup}{% + \implicittoktrue\tc@implicitgrptrue}{}} +% TRAP ACTIVE TOK +\def\trapactivetok#1{\tctestifnum{\number\catcode`#1=13}{\activetoktrue}{}} +% EXPRESS-INTERFACE - ALLOWS TO EXTERNALLY DEFINE DIRECTIVES +\def\Characterdirective{\def\@chrT##1} +\def\Groupdirective{\long\def\@grpT##1} +\def\Macrodirective{\long\def\@macT##1} +\def\Spacedirective{\def\@spcT##1} +% EXPRESS-INTERFACE - DEFAULT DIRECTIVES +\def\resetCharacterdirective{\Characterdirective{\addcytoks{##1}}} +\def\resetGroupdirective{\Groupdirective{\processtoks{##1}}} +\def\resetMacrodirective{\Macrodirective{\addcytoks{##1}}} +\def\resetSpacedirective{\Spacedirective{\addcytoks{##1}}} +\def\resettokcycle{\resetCharacterdirective\resetGroupdirective + \resetMacrodirective\resetSpacedirective\aftertokcycle{}% + \stripgroupingfalse\stripimplicitgroupingcase{0}} +% SUPPORT MACROS FOR TOKENIZED OUTPUT: \addcytoks[<expansion level>]{<arg>} +% (CONTRIBUTED BY CHRISTIAN TELLECHEA) +\def\addcytoks{\futurelet\nxttok\addcytoks@A} +\long\def\tc@addtotoks#1{\cytoks\expandafter{\the\cytoks#1}} +\def\addcytoks@A{\tctestifx{[\nxttok}\addcytoks@B\tc@addtotoks} +\long\def\addcytoks@B[#1]#2{\tc@ifempty{#1}\tc@addtotoks + {\tctestifx{x#1}{\tc@xarg\tc@addtotoks}{\addcytoks@C{#1}}}{#2}} +\def\addcytoks@C#1{\tctestifnum{#1>0}{\tc@earg\addcytoks@C + {\the\numexpr#1-1\expandafter}\expandafter}\tc@addtotoks} +% SET INITIAL PARAMETERS +\settcGrouping{{#1}}% E.G. <<#1>> IF cat-1,2 SET TO < AND > +\settcEscapechar{|}% BYPASS TOKCYCLE PROCESSING BETWEEN |...| +\resettokcycle% WHICH ALSO CONTAINS THE FOLLOWING 3 RESETS: +% \stripimplicitgroupingcase{0}% DEFAULT RETAIN UNALTERED \b/e-groups +% \stripgroupingfalse% DEFAULT RETAIN UNALTERED {} GROUPING +% \aftertokcycle{}% NO DEFAULT CODE EXECUTED AFTER EACH TOKCYCLE INVOCATION +\restorecatcode +\endinput + +EDIT HISTORY +v1.0 2019/8/21 + - Initial release diff --git a/Master/tlpkg/bin/tlpkg-ctan-check b/Master/tlpkg/bin/tlpkg-ctan-check index 16f4c76312d..4eb079b3486 100755 --- a/Master/tlpkg/bin/tlpkg-ctan-check +++ b/Master/tlpkg/bin/tlpkg-ctan-check @@ -692,7 +692,7 @@ my @TLP_working = qw( tkz-kiviat tkz-linknodes tkz-orm tikz-page tkz-tab tlc-article tlc2 tlcockpit tocbibind tocdata tocloft tocvsec2 todo todonotes - tokenizer toolbox tools topfloat topiclongtable topletter + tokcycle tokenizer toolbox tools topfloat topiclongtable topletter totcount totpages toptesi tpic2pdftex tpslifonts tqft tracklang trajan tram diff --git a/Master/tlpkg/libexec/ctan2tds b/Master/tlpkg/libexec/ctan2tds index ea89f91a204..6e28d1572ff 100755 --- a/Master/tlpkg/libexec/ctan2tds +++ b/Master/tlpkg/libexec/ctan2tds @@ -2026,6 +2026,7 @@ $standardtex 'tikz-feynman','\.lua|' . $standardtex, 'tkz-orm', 'tkz.*\.sty', # no pgfmanual.sty 'tlc2', 'NULL', # doc package + 'tokcycle', 'tokcycle\.tex|' . $standardtex, 'tools', '\.cfg|\.sty|\.clo|\.cls|\.def|\.fd$|.?\.tex|^\.tex$', 'toptesi', 'toptesi\.cfg|\.sty|\.cls', 'translator', '\.dict|translator-language-mappings.tex|' . $standardtex, diff --git a/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc b/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc index dea3c6df718..a3b001481d8 100644 --- a/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc +++ b/Master/tlpkg/tlpsrc/collection-latexextra.tlpsrc @@ -1146,6 +1146,7 @@ depend tocloft depend tocvsec2 depend todo depend todonotes +depend tokcycle depend tokenizer depend toolbox depend topfloat diff --git a/Master/tlpkg/tlpsrc/tokcycle.tlpsrc b/Master/tlpkg/tlpsrc/tokcycle.tlpsrc new file mode 100644 index 00000000000..e69de29bb2d --- /dev/null +++ b/Master/tlpkg/tlpsrc/tokcycle.tlpsrc |