From 48c9716c5344efef4d29348600bb0b638e176d13 Mon Sep 17 00:00:00 2001
From: Karl Berry <karl@freefriends.org>
Date: Fri, 27 Sep 2019 21:24:43 +0000
Subject: tokcycle (27sep19)

git-svn-id: svn://tug.org/texlive/trunk@52191 c570f23f-e606-0410-a88d-b1316a301751
---
 .../texmf-dist/tex/generic/tokcycle/tokcycle.tex   | 60 +++++++++++++++-------
 1 file changed, 42 insertions(+), 18 deletions(-)

(limited to 'Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex')

diff --git a/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex
index d6f1b4003ef..5a33109df65 100644
--- a/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex
+++ b/Master/texmf-dist/tex/generic/tokcycle/tokcycle.tex
@@ -1,7 +1,7 @@
 \def\tcname                     {tokcycle}
-\def\tcver                        {1.0}
+\def\tcver                        {1.1}
 %
-\def\tcdate                    {2019/8/21}
+\def\tcdate                    {2019/9/27}
 %
 % Author     : Steven B Segletes, Christian Tellechea (contributor)
 % Maintainer : Steven B Segletes
@@ -52,13 +52,15 @@
 \long\def\tc@exfirst#1#2{#1}
 \long\def\tc@exsecond#1#2{#2}
 \long\def\tc@testxifx{\tc@earg\tctestifx}
+\long\def\test@ifmacro#1{\tctestifcatnx#1\relax}
 \def\tc@addtoks#1#2{\toks#1\expandafter{\the\toks#1#2}}
 \def\addtc@depth{\advance\tc@depth 1}
 \def\subtc@depth{\tc@depth=\numexpr\tc@depth-1\relax}
-\def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse\catSIXfalse}
+\def\tc@resetifs{\activetokfalse\implicittokfalse\tc@implicitgrpfalse
+  \catSIXfalse\activecharfalse\activetokunexpandablefalse}
 \long\def\count@stringtoks#1{\tc@earg\count@toks{\string#1}}
-\def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt}
-\def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}}
+\long\def\count@toks#1{\the\numexpr-1\count@@toks#1.\tc@endcnt}
+\long\def\count@@toks#1#2\tc@endcnt{+1\tc@ifempty{#2}{\relax}{\count@@toks#2\tc@endcnt}}
 \def\sv@hash{##}
 % EXTERNAL TOOLS
 \long\def\tctestifcon#1{#1\expandafter\tc@exfirst\else\expandafter\tc@exsecond\fi}
@@ -70,6 +72,8 @@
 \newif\ifcatSIX
 \newif\ifimplicittok
 \newif\ifactivetok
+\newif\ifactivechar
+\newif\ifactivetokunexpandable
 \newtoks\cytoks
 \long\def\tokcycleenvironment#1#2#3#4#5{\expandafter\def\expandafter#1%
   \expandafter{\expandafter\let\csname end\expandafter\tc@gobble
@@ -89,14 +93,15 @@
 % LOOP ENTRY POINT
 \def\@tokcycle{\tc@resetifs\futurelet\tc@next\detect@CANTabsorb}
 \def\detect@CANTabsorb{\tctestifx{\tc@next\tc@sptoken}{\stringify\@@@@@spcT}%
-  {\tctestifx{\tc@next\bgroup}{\stringify\@@@@grpT}%
-  {\can@absorb}}}
+  {\tctestifx{\tc@next\bgroup}{\stringify\@@@@grpT}{\can@absorb}}}
 % NON cat1,10 TOKENS
 \long\def\can@absorb#1{\tc@tok{#1}\trapcatSIX{#1}\expandafter\can@absorb@
   \the\tc@tok}
 \long\def\can@absorb@#1{\tctestifnum{\count@stringtoks{#1}>1}%
   {\tctestifx{\endtokcycraw#1}{#1}{\backslashcmds#1\@tokcycle}}%
-  {\trapactivetok#1\tc@trapescape#1{\tc@escapecytoks}{\@chrT#1\@tokcycle}}}
+  {\trapactives#1\tc@trapescape#1{\tc@escapecytoks}{\can@absorb@@#1}}}
+\long\def\can@absorb@@#1{\let\@tmp#1\test@ifmacro\@tmp{\implicittokfalse
+  \@macT#1}{\trapimplicitegrp#1\implicitgrpfork#1}\@tokcycle}
 %CONVERT NEXT (SPACE OR BEGIN-GROUP) TOKEN TO STRING
 \def\stringify#1{\expandafter#1\string}% #1 WILL BE \@@@@@spcT or \@@@@grpT
 %SPACE DECODE
@@ -110,17 +115,21 @@
 \def\@@@@grpT{\futurelet\tc@str\@@@grpT}
 \def\@@@grpT#1{\tctestifnum{\number\catcode`#1=1}%
   {\expandafter\@@grpT\expandafter{\iffalse}\fi}% {
-  {\implicittoktrue\tc@implicitgrptrue
-    \csmk{\expandafter\backslashcmds\thecs\@tokcycle}}}% \bgroup
+  {\implicittoktrue\tc@implicitgrptrue%
+    \tctestifnum{`#1=92}% WORKS EVEN IF CAT-0 HAS CHANGED
+    {\csmk{\expandafter\backslashcmds\thecs\@tokcycle}}% \bgroup
+    {\begingroup\catcode`#1=\active \xdef\@tmp{\scantokens{#1\noexpand}}\endgroup
+      \expandafter\implicitgrpfork\@tmp\@tokcycle}% ACTIVE CHAR \bgroup
+  }}
 \def\@@grpT#1{\tctestifcon\ifstripgrouping{\@grpT{#1}}%
   {\groupedcytoks{\@grpT{#1}}}\@tokcycle}
 % \ COMMANDS (MACROS AND IMPLICITS)
-\long\def\test@ifmacro#1{\tctestifcatnx#1\relax}% REVISED!
-\long\def\backslashcmds#1{\test@ifmacro#1{\tctestifcon\ifcatSIX{%
-  \implicittoktrue\@chrT#1}{\@macT#1}}{\implicittoktrue
-  \trapimplicitegrp#1\tctestifcon{\iftc@implicitgrp}%
-    {\ifcase\@implicitgroupingcase\or\addcytoks{#1}\or\@chrT{#1}\fi}%
-      {\@chrT#1}{\@chrT#1}}}
+\long\def\backslashcmds#1{%
+  \test@ifmacro#1{\tctestifcon\ifcatSIX{\implicittoktrue\@chrT#1}{\@macT#1}}%
+    {\implicittoktrue\trapimplicitegrp#1\implicitgrpfork#1}}
+% FORK BASED ON IMPLICIT GROUP TREATMENT
+\def\implicitgrpfork#1{\tctestifcon{\iftc@implicitgrp}{\ifcase
+  \@implicitgroupingcase\or\addcytoks{#1}\or\@chrT{#1}\fi}{\@chrT#1}}
 % SET UP ESCAPE MECHANISM
 \def\settcEscapechar#1{\let\@tcEscapeptr#1%
   \def\tc@escapecytoks##1#1{\addcytoks{##1}\@tokcycle}}
@@ -130,7 +139,8 @@
 \def\trapcatSIXb#1{\tc@earg\tctestifcatnx\sv@hash#1{\catSIXtrue\trapcatSIXc#1}{}}
 \def\trapcatSIXc#1{\tctestifnum{\count@stringtoks{#1}>1}{\tc@defx\six@str{\string#1}%
   \global\let\implicitsixtok\six@str\tc@tok{\implicitsixtok}}%
-  {\tc@tok\expandafter{\string#1}}}
+  {\tc@tok\expandafter{\string#1}\tctestifnum{\number\catcode`#1=6}%
+  {}{\activetoktrue\implicittoktrue}}}
 % DIRECTIVES FOR HANDLING GROUPS RECURSIVELY; DEFINE tokcycle GROUPING CHARS
 \def\@defgroupedcytoks#1{\long\def\groupedcytoks##1{%
   \begingroup\cytoks{}##1\expandafter\endgroup\expandafter
@@ -154,7 +164,12 @@
 \def\trapimplicitegrp#1{\tctestifx{#1\egroup}{%
   \implicittoktrue\tc@implicitgrptrue}{}}
 % TRAP ACTIVE TOK
-\def\trapactivetok#1{\tctestifnum{\number\catcode`#1=13}{\activetoktrue}{}}
+\def\trapactives#1{\trapactivechar{#1}\trapactivetok{#1}}
+\def\trapactivechar#1{\tctestifnum{\number\catcode`#1=13}{\activechartrue}{}}
+\def\trapactivetok#1{\tctestifcatnx~#1{\activetoktrue}{\trapactivetokunexpandable#1}}
+%% WILL ALSO TRAP ACTIVE \let TO PRIMITIVES AS IMPLICIT; UNDO LATER IN \can@absorb@@
+\def\trapactivetokunexpandable#1{\tctestifcon{\expandafter\if
+  \detokenize{#1}#1}{}{\activetoktrue\activetokunexpandabletrue\implicittoktrue}}
 % EXPRESS-INTERFACE - ALLOWS TO EXTERNALLY DEFINE DIRECTIVES
 \def\Characterdirective{\def\@chrT##1}
 \def\Groupdirective{\long\def\@grpT##1}
@@ -190,3 +205,12 @@
 EDIT HISTORY
 v1.0  2019/8/21
       - Initial release
+
+v1.1  2019/9/27
+      - Introduced \ifactivechar, \ifactivetokunexpandable
+      - Tightened up consistent definition of implicit (to exclude primitives)
+      - Rewrote active token trapping logic, to differentiate between active 
+        token vs. active character code, in the event that an earlier tokenized 
+        token no longer shares the current characteristics of the character code
+      - Added ability to handle active-implicit grouping tokens
+      - Added ability to handle active-implicit cat-6 tokens
\ No newline at end of file
-- 
cgit v1.2.3

