\def\tcname {tokcycle} \def\tcver {1.0} % \def\tcdate {2019/8/21} % % Author : Steven B Segletes, Christian Tellechea (contributor) % Maintainer : Steven B Segletes % License : Released under the LaTeX Project Public License v1.3c % or later, see http://www.latex-project.org/lppl.txt % Files : 1) tokcycle.tex % 2) tokcycle.sty % 3) tokcycle-doc.tex % 4) tokcycle-doc.pdf % 5) tokcycle-examples.tex % 6) tokcycle-examples.pdf % 7) README %%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%% % MACRO FORM \long\def\tokcycle#1#2#3#4#5{\tokcycraw{#1}{#2}{#3}{#4}#5\endtokcycraw} % \expanded-ARGUMENT MACRO FORM \long\def\expandedtokcycle#1#2#3#4#5{\cytoks{\tokcycraw{#1}{#2}{#3}{#4}}% \expandafter\the\expandafter\cytoks\expanded{#5}\endtokcycraw} % ENVIRONMENT FORM \long\def\tokencycle#1#2#3#4{\begingroup\let\endtokencycle\endtokcycraw \aftertokcycle{\the\cytoks\expandafter\endgroup\expandafter\tcenvscope \expandafter{\the\cytoks}}\tokcycraw{#1}{#2}{#3}{#4}} % XPRESS-INTERFACE MACRO FORM \long\def\tokcyclexpress#1{\tokcycrawxpress#1\endtokcycraw} % XPRESS-INTERFACE \expanded-ARGUMENT MACRO FORM \long\def\expandedtokcyclexpress#1{% \expandafter\tokcycrawxpress\expanded{#1}\endtokcycraw} % XPRESS-INTERFACE ENVIRONMENT FORM \def\tokencyclexpress{\begingroup\let\endtokencyclexpress\endtokcycraw \aftertokcycle{\the\cytoks\expandafter\endgroup\expandafter\tcenvscope \expandafter{\the\cytoks}}\tokcycrawxpress} % INITIALIZATION & INTERNAL TOOLS \def\tcenvscope{\cytoks}% CAN SET TO \global\cytoks TO OVERCOME SCOPE LIMITS \edef\restorecatcode{\catcode\number`\@=\number\catcode`\@\relax} \catcode`\@11 \newif\iftc@implicitgrp \newif\if@argnext \newtoks\tc@tok \newcount\tc@depth \def\tc@gobble#1{} \def\tc@deftok#1#2{\let#1= #2\empty} \tc@deftok\tc@sptoken{ } \expandafter\def\expandafter\tc@absorbSpace\space{} \def\tc@ifempty#1{\tc@testxifx{\expandafter\relax\detokenize{#1}\relax}} \def\tc@defx#1#2{\tc@earg{\def\expandafter#1}{#2}} \long\def\tc@earg#1#2{\expandafter#1\expandafter{#2}} \long\def\tc@xarg#1#2{\tc@earg#1{\expanded{#2}}} \long\def\tc@exfirst#1#2{#1} \long\def\tc@exsecond#1#2{#2} \long\def\tc@testxifx{\tc@earg\tctestifx} \def\tc@addtoks#1#2{\toks#1\expandafter{\the\toks#1#2}} \def\addtc@depth{\advance\tc@depth 1} \def\subtc@depth{\tc@depth=\numexpr\tc@depth-1\relax} \def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse\catSIXfalse} \long\def\count@stringtoks#1{\tc@earg\count@toks{\string#1}} \def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt} \def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}} \def\sv@hash{##} % EXTERNAL TOOLS \long\def\tctestifcon#1{#1\expandafter\tc@exfirst\else\expandafter\tc@exsecond\fi} \long\def\tctestifcatnx#1#2{\tctestifcon{\ifcat\noexpand#1\noexpand#2}} \long\def\tctestifx#1{\tctestifcon{\ifx#1}} \long\def\tctestifnum#1{\tctestifcon{\ifnum#1\relax}} \newif\ifstripgrouping \def\stripimplicitgroupingcase#1{\edef\@implicitgroupingcase{\the\numexpr1-#1}} \newif\ifcatSIX \newif\ifimplicittok \newif\ifactivetok \newtoks\cytoks \long\def\tokcycleenvironment#1#2#3#4#5{\expandafter\def\expandafter#1% \expandafter{\expandafter\let\csname end\expandafter\tc@gobble \string#1\endcsname\endtokcycraw\tokencycle{#2}{#3}{#4}{#5}}} \long\def\processtoks#1{\addtc@depth\@tokcycle#1\endtokcycraw } \def\whennotprocessingparameter#1#2{\tctestifcon\if@argnext{\@argnextfalse\cytoks \expandafter{\the\cytoks###1}}{\tctestifcon\ifcatSIX{\@argnexttrue}{#2}}} % ESSENTIAL METHOD: STREAMING MACRO WITH TERMINATOR: % \tokcycraw{}{}{}{}\endtokcycraw \long\def\tokcycraw#1#2#3#4{\def\@chrT##1{#1}\long\def\@grpT##1{#2}% \long\def\@macT##1{#3}\def\@spcT##1{#4}\tokcycrawxpress} % ENTRY POINT FOR XPRESS METHOD WITHOUT EXPLICIT ARGUMENTS \def\tokcycrawxpress{\cytoks{}\tc@depth=1\relax\@tokcycle} % CODE TO EXECUTE AT COMPLETION \long\def\aftertokcycle#1{\def\@aftertokcycle{#1}} \def\endtokcycraw{\subtc@depth\tctestifnum{\tc@depth=0}{\@aftertokcycle}{}} % LOOP ENTRY POINT \def\@tokcycle{\tc@resetifs\futurelet\tc@next\detect@CANTabsorb} \def\detect@CANTabsorb{\tctestifx{\tc@next\tc@sptoken}{\stringify\@@@@@spcT}% {\tctestifx{\tc@next\bgroup}{\stringify\@@@@grpT}% {\can@absorb}}} % NON cat1,10 TOKENS \long\def\can@absorb#1{\tc@tok{#1}\trapcatSIX{#1}\expandafter\can@absorb@ \the\tc@tok} \long\def\can@absorb@#1{\tctestifnum{\count@stringtoks{#1}>1}% {\tctestifx{\endtokcycraw#1}{#1}{\backslashcmds#1\@tokcycle}}% {\trapactivetok#1\tc@trapescape#1{\tc@escapecytoks}{\@chrT#1\@tokcycle}}} %CONVERT NEXT (SPACE OR BEGIN-GROUP) TOKEN TO STRING \def\stringify#1{\expandafter#1\string}% #1 WILL BE \@@@@@spcT or \@@@@grpT %SPACE DECODE \def\@@@@@spcT{\futurelet\tc@str\@@@@spcT} \def\@@@@spcT{\tctestifx{\tc@str\tc@sptoken}% {\def\@tmp{\@@spcT{ }}\expandafter\@tmp\tc@absorbSpace}% EXPLICIT SPACE {\implicittoktrue\expandafter\@@@spcT\tc@gobble}}% IMPLICIT SPACE \def\@@@spcT{\csmk{\expandafter\@@spcT\thecs}} \def\@@spcT#1{\@spcT{#1}\@tokcycle} % GROUP DECODE \def\@@@@grpT{\futurelet\tc@str\@@@grpT} \def\@@@grpT#1{\tctestifnum{\number\catcode`#1=1}% {\expandafter\@@grpT\expandafter{\iffalse}\fi}% { {\implicittoktrue\tc@implicitgrptrue \csmk{\expandafter\backslashcmds\thecs\@tokcycle}}}% \bgroup \def\@@grpT#1{\tctestifcon\ifstripgrouping{\@grpT{#1}}% {\groupedcytoks{\@grpT{#1}}}\@tokcycle} % \ COMMANDS (MACROS AND IMPLICITS) \long\def\test@ifmacro#1{\tctestifcatnx#1\relax}% REVISED! \long\def\backslashcmds#1{\test@ifmacro#1{\tctestifcon\ifcatSIX{% \implicittoktrue\@chrT#1}{\@macT#1}}{\implicittoktrue \trapimplicitegrp#1\tctestifcon{\iftc@implicitgrp}% {\ifcase\@implicitgroupingcase\or\addcytoks{#1}\or\@chrT{#1}\fi}% {\@chrT#1}{\@chrT#1}}} % SET UP ESCAPE MECHANISM \def\settcEscapechar#1{\let\@tcEscapeptr#1% \def\tc@escapecytoks##1#1{\addcytoks{##1}\@tokcycle}} \def\tc@trapescape#1{\tctestifx{\@tcEscapeptr#1}} % TRAP CAT-6 \long\def\trapcatSIX#1{\tctestifcatnx#1\relax{}{\trapcatSIXb#1}} \def\trapcatSIXb#1{\tc@earg\tctestifcatnx\sv@hash#1{\catSIXtrue\trapcatSIXc#1}{}} \def\trapcatSIXc#1{\tctestifnum{\count@stringtoks{#1}>1}{\tc@defx\six@str{\string#1}% \global\let\implicitsixtok\six@str\tc@tok{\implicitsixtok}}% {\tc@tok\expandafter{\string#1}}} % DIRECTIVES FOR HANDLING GROUPS RECURSIVELY; DEFINE tokcycle GROUPING CHARS \def\@defgroupedcytoks#1{\long\def\groupedcytoks##1{% \begingroup\cytoks{}##1\expandafter\endgroup\expandafter \addcytoks\expandafter{\expandafter#1}}} \def\settcGrouping#1{\def\@tmp##1{#1}\tc@defx\@@tmp{\@tmp{\the\cytoks}}% \tc@earg\@defgroupedcytoks{\@@tmp}} % FAUX TOKENIZATION OF COMMAND NAME (WHEN cat0 TOKEN HAS BEEN MADE cat12) \def\csmk#1{\def\csaftermk{#1}\toks0{}\@csmkA} \def\@csmkA{\futurelet\@tmp\@csmkB} \def\@csmkB{\tctestifx{\@tmp\tc@sptoken}% {\toks0{ }\expandafter\@csmkF\tc@absorbSpace}{\@csmkCA}} \def\@csmkCA#1{\tc@addtoks0{#1}\tctestifnum{\number\catcode`#1=11}% {\futurelet\@tmp\@csmkD}{\@csmkF}} \def\@csmkC#1{\tctestifnum{\number\catcode`#1=11} {\tc@addtoks0{#1}\futurelet\@tmp\@csmkD}{\@csmkE#1}} \def\@csmkD{\tctestifcatnx 0\@tmp\@csmkC\@csmkE} \def\@csmkE{\tctestifx{\@tmp\tc@sptoken}% {\expandafter\@csmkF\tc@absorbSpace}{\@csmkF}} \def\@csmkF{\tc@defx\thecs{\csname\the\toks0\endcsname}\csaftermk} % TRAP IMPLICIT END GROUP TOK (e.g., \egroup); SET \iftc@implicitgrp \def\trapimplicitegrp#1{\tctestifx{#1\egroup}{% \implicittoktrue\tc@implicitgrptrue}{}} % TRAP ACTIVE TOK \def\trapactivetok#1{\tctestifnum{\number\catcode`#1=13}{\activetoktrue}{}} % EXPRESS-INTERFACE - ALLOWS TO EXTERNALLY DEFINE DIRECTIVES \def\Characterdirective{\def\@chrT##1} \def\Groupdirective{\long\def\@grpT##1} \def\Macrodirective{\long\def\@macT##1} \def\Spacedirective{\def\@spcT##1} % EXPRESS-INTERFACE - DEFAULT DIRECTIVES \def\resetCharacterdirective{\Characterdirective{\addcytoks{##1}}} \def\resetGroupdirective{\Groupdirective{\processtoks{##1}}} \def\resetMacrodirective{\Macrodirective{\addcytoks{##1}}} \def\resetSpacedirective{\Spacedirective{\addcytoks{##1}}} \def\resettokcycle{\resetCharacterdirective\resetGroupdirective \resetMacrodirective\resetSpacedirective\aftertokcycle{}% \stripgroupingfalse\stripimplicitgroupingcase{0}} % SUPPORT MACROS FOR TOKENIZED OUTPUT: \addcytoks[]{} % (CONTRIBUTED BY CHRISTIAN TELLECHEA) \def\addcytoks{\futurelet\nxttok\addcytoks@A} \long\def\tc@addtotoks#1{\cytoks\expandafter{\the\cytoks#1}} \def\addcytoks@A{\tctestifx{[\nxttok}\addcytoks@B\tc@addtotoks} \long\def\addcytoks@B[#1]#2{\tc@ifempty{#1}\tc@addtotoks {\tctestifx{x#1}{\tc@xarg\tc@addtotoks}{\addcytoks@C{#1}}}{#2}} \def\addcytoks@C#1{\tctestifnum{#1>0}{\tc@earg\addcytoks@C {\the\numexpr#1-1\expandafter}\expandafter}\tc@addtotoks} % SET INITIAL PARAMETERS \settcGrouping{{#1}}% E.G. <<#1>> IF cat-1,2 SET TO < AND > \settcEscapechar{|}% BYPASS TOKCYCLE PROCESSING BETWEEN |...| \resettokcycle% WHICH ALSO CONTAINS THE FOLLOWING 3 RESETS: % \stripimplicitgroupingcase{0}% DEFAULT RETAIN UNALTERED \b/e-groups % \stripgroupingfalse% DEFAULT RETAIN UNALTERED {} GROUPING % \aftertokcycle{}% NO DEFAULT CODE EXECUTED AFTER EACH TOKCYCLE INVOCATION \restorecatcode \endinput EDIT HISTORY v1.0 2019/8/21 - Initial release