tokcycle (28may21)

git-svn-id: svn://tug.org/texlive/trunk@59377 c570f23f-e606-0410-a88d-b1316a301751

1 files changed, 59 insertions, 9 deletions

diff --git a/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex
index 9aaec68b728..d2e3a059cad 100644
--- a/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex
+++ b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex
@@ -1,7 +1,7 @@
 \def\tcname                     {tokcycle}
-\def\tcver                        {1.3}
+\def\tcver                        {1.4}
 %
-\def\tcdate                    {2021/03/10}
+\def\tcdate                    {2021/05/27}
 %
 % Author     : Steven B Segletes, Christian Tellechea (contributor)
 % Maintainer : Steven B Segletes
@@ -52,16 +52,18 @@
 \long\def\tc@exsecond#1#2{#2}
 \long\def\tc@testxifx{\tc@earg\tctestifx}
 \long\def\test@ifmacro#1{\tctestifcatnx#1\relax}
-\def\tc@addtoks#1#2{\toks#1\expandafter{\the\toks#1#2}}
+\def\tc@addtoks#1#2{\toks#1\expandafter{\the\toks#1 #2}}
 \def\add@tcdepth{\advance\tcdepth 1\relax}
-\def\sub@tcdepth{\advance\tcdepth-1\relax}\def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse
-  \catSIXfalse\activecharfalse\activetokunexpandablefalse
-  \def\theactivespace{}}
+\def\sub@tcdepth{\advance\tcdepth-1\relax}
+\def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse\catSIXfalse
+  \activecharfalse\activetokunexpandablefalse\def\theactivespace{}}
 \long\def\count@stringtoks#1{\tc@earg\count@toks{\string#1}}
 \long\def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt}
 \long\def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}}
 \def\sv@hash{##}
+\def\def@popname{\expandafter\def\tc@popname}
 % EXTERNAL TOOLS
+\let\tcsptoken= \tc@sptoken
 \long\def\tctestifcon#1{#1\expandafter\tc@exfirst\else\expandafter\tc@exsecond\fi}
 \long\def\tctestifcatnx#1#2{\tctestifcon{\ifcat\noexpand#1\noexpand#2}}
 \long\def\tctestifx#1{\tctestifcon{\ifx#1}}
@@ -73,6 +75,7 @@
 \newif\ifactivetok
 \newif\ifactivechar
 \newif\ifactivetokunexpandable
+\newif\ifspacepopped
 \newtoks\cytoks
 \long\def\tokcycleenvironment#1#2#3#4#5{\expandafter\def\expandafter#1%
   \expandafter{\expandafter\let\csname end\expandafter\tc@gobble
@@ -122,7 +125,9 @@
  {\implicittoktrue\tctestifcon{\if\expandafter\@firstoftwo\string\\\tc@str}%
    {\expandafter\@@@spcT\tc@gobble}% IMPLICIT MACRO SPACE
    {\activetoktrue\@@@@spcT}}}% IMPLICIT ACTIVE SPACE
-\def\@@@@spcT#1{\def\theactivespace{#1}\trapactivechar{#1}\@@spcT{\tc@sptoken}}
+\def\@@@@spcT#1{\def\theactivespace{#1}\trapactivechar{#1}\ifactivechar\edef\tc@spc
+  {\scantokens{#1\noexpand}}\else\def\tc@spc{\tcsptoken}\fi\expandafter\@@spcT
+  \expandafter{\tc@spc}}
 \def\@@@spcT{\csmk{\expandafter\@@spcT\thecs}}
 \def\@@spcT#1{\@spcT{#1}\@tokcycle}
 % GROUP DECODE
@@ -161,7 +166,7 @@
   \addcytoks\expandafter{\expandafter#1}}}
 \def\settcGrouping#1{\def\@tmp##1{#1}\tc@defx\@@tmp{\@tmp{\the\cytoks}}%
   \tc@earg\defineexit@grouped{\@@tmp}}
-% FAUX TOKENIZATION OF COMMAND NAME (WHEN cat0 TOKEN HAS BEEN MADE cat12)
+% FAUX TOKENIZATION OF COMMAND NAME (WHEN \ AND COMMAND-NAME TOKS ARE NOW cat12)
 \def\csmk#1{\def\csaftermk{#1}\toks0{}\@csmkA}
 \def\@csmkA{\futurelet\@tmp\@csmkB}
 \def\@csmkB{\tctestifx{\@tmp\tc@sptoken}%
@@ -184,6 +189,38 @@
 %% WILL ALSO TRAP ACTIVE \let TO PRIMITIVES AS IMPLICIT; UNDO LATER IN \can@absorb@@
 \def\trapactivetokunexpandable#1{\tctestifcon{\expandafter\if
   \detokenize{#1}#1}{}{\activetoktrue\activetokunexpandabletrue\implicittoktrue}}
+% FEATURES TO LOOK-AHEAD INTO THE INPUT STREAM (INTRODUCED v1.4)
+\long\def\tcpeek#1#2\@tokcycle{\def\tc@tmp{\ifx#1\endtokcycraw
+  \let#1=\empty\fi#2\@tokcycle}\futurelet#1\tc@tmp}%___________________________PEEK_
+\def\tcpopliteral#1{\tcpopwhitespace#1\tcpeek\@tmp\ifx\@tmp\bgroup
+  \tcpop\@tmp\def\tc@tmp{\tcappto#1from}\expandafter\tc@tmp\expandafter
+  {\expandafter{\@tmp}}\else\tcpopappto#1\fi}
+\def\tcpop{\long\def\tc@@@pop##1{\tctestifx{\endtokcycraw##1}{\def@popname{}%
+  \tc@tmp\endtokcycraw}{\def@popname{##1}\tc@tmp}}\tc@pop}
+\long\def\tcpopuntil#1{\long\def\tc@@@pop##1#1{\def@popname{##1#1}\tc@tmp}\tc@pop}
+\long\def\tc@pop#1#2\@tokcycle{\def\tc@popname{#1}\def\tc@tmp
+    {#2\@tokcycle}\futurelet\tc@futuretok\tc@@pop}
+\def\tc@@pop{\tc@trapescape\tc@futuretok{\def@popname{}\tc@tmp}{\tctestifx{%
+  \endtokcycraw\tc@futuretok}{\def@popname{}\tc@tmp}{\tctestifx{\tc@sptoken%
+  \tc@futuretok}{\spacepoppedtrue\tc@@@pop}{\spacepoppedfalse\tc@@@pop}}}}%_____POP_
+\def\tcappto#1from#2{%
+  \expandafter\tc@defx\expandafter#1\expandafter{\expandafter#1#2}}
+\def\tcpopliteralappto#1{\tcpopliteral\@tmp\tcappto#1from\@tmp}
+\def\tcpopappto#1{\tcpop\@tmp\tcappto#1from\@tmp}%___________________________APPEND_
+\long\def\tcpopwhitespace#1#2\@tokcycle{\def\tc@popname{#1}\def@popname{}%
+  \def\tc@tmp{#2\@tokcycle}\futurelet\tc@futuretok\tc@popspc}
+\def\tc@popspc{\tctestifx{\tc@sptoken\tc@futuretok}{\discern@space}{\tc@tmp}}
+\def\discern@space{\begingroup\def\@@spcT##1{\tctestifcon\ifimplicittok{\gdef
+  \tc@nxt{\tc@tmp##1}}{\gdef\tc@nxt{\def@popname{##1}\tc@tmp}}\endgroup
+  \tc@nxt}\stringify\@@@@@@spcT}%_____________________________________________SPACE_
+\long\def\@tcpush#1#2\@tokcycle{\def\tc@tmp{#2\@tokcycle}\expandafter\tc@tmp#1}
+\long\def\@tcpushgroup#1#2\@tokcycle{\def\tc@tmp{#2\@tokcycle}\expandafter\tc@tmp
+  \expandafter{#1}}
+% ...BORROW \addcytoks OPTIONAL ARGUMENT EXPANSION FEATURE FOR \tcpush[group]
+\def\tcpush{\bgroup\long\def\tc@addtotoks##1{\egroup
+  \@tcpush{##1}}\futurelet\nxttok\addcytoks@A}
+\def\tcpushgroup{\bgroup\long\def\tc@addtotoks##1{\egroup
+  \@tcpushgroup{##1}}\futurelet\nxttok\addcytoks@A}%___________________________PUSH_
 % EXPRESS-INTERFACE - ALLOWS TO EXTERNALLY DEFINE DIRECTIVES
 \def\Characterdirective{\def\@chrT##1}
 \def\Groupdirective{\long\def\@grpT##1}
@@ -247,7 +284,7 @@ v1.2  2020/10/01
         in the definition \theactivespace.  The flags \implicittok and \activetok
         are both set true, and the \activechar flag is checked, as well.
 
-v1.3  2021/03/09
+v1.3  2021/03/10
       - Introduced \xtokcycleenvironment, similar to \tokcycleenvironment, but
         allows two additional arguments, defining the setup and trailing code
         that will be run prior to the invocation and following conclusion of the 
@@ -283,3 +320,16 @@ v1.3  2021/03/09
         \tcdepth associated with the Group directive (also, outer token cycle 
         operates with depth = 0 rather than 1).
       - Renamed \@defgroupedcytoks to \defineexit@grouped, to better match function.
+
+v1.4  2021/05/26
+      - Concerning the \@@@@spcT macro: previously, active-implicit spaces 
+        were passed to the Spacedirective as \tc@sptoken, with the \string of the 
+        active char passed in \theactivespace. Now, the active-implicit space token 
+        itself is passed to the Spacedirective instead of \tc@sptoken, but only *IF* 
+        the charcode of that character is currently active; otherwise, a generic 
+        implicit space, \tcsptoken is passed.
+      - Introduced a set of "look ahead" macros: \tcpeek, \tcpop, \tcpopliteral,
+        \tcpopappto, \tcpopliteralappto, \tcappto#1from#2, \tcpopuntil, 
+        \tcpopwhitespace, \tcpush, and \tcpushgroup.
+      - Bug fixed in \tc@addtoks definition, in the event that #2 was a number.
+