diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/tex2tok/tex2tok.tex |
Initial commit
Diffstat (limited to 'support/tex2tok/tex2tok.tex')
-rw-r--r-- | support/tex2tok/tex2tok.tex | 502 |
1 files changed, 502 insertions, 0 deletions
diff --git a/support/tex2tok/tex2tok.tex b/support/tex2tok/tex2tok.tex new file mode 100644 index 0000000000..44701bc998 --- /dev/null +++ b/support/tex2tok/tex2tok.tex @@ -0,0 +1,502 @@ +% Automatically generated from _tex2tok.tex by helper.py +\catcode`\!=11 +\let\!let\let +\!let\-\undefined +\!let\/\undefined +\!let\ \undefined +\!let\!above\above \!let\above\undefined +\!let\!abovedisplayshortskip\abovedisplayshortskip \!let\abovedisplayshortskip\undefined +\!let\!abovedisplayskip\abovedisplayskip \!let\abovedisplayskip\undefined +\!let\!abovewithdelims\abovewithdelims \!let\abovewithdelims\undefined +\!let\!accent\accent \!let\accent\undefined +\!let\!adjdemerits\adjdemerits \!let\adjdemerits\undefined +\!let\!advance\advance \!let\advance\undefined +\!let\!afterassignment\afterassignment \!let\afterassignment\undefined +\!let\!aftergroup\aftergroup \!let\aftergroup\undefined +\!let\!atop\atop \!let\atop\undefined +\!let\!atopwithdelims\atopwithdelims \!let\atopwithdelims\undefined +\!let\!badness\badness \!let\badness\undefined +\!let\!baselineskip\baselineskip \!let\baselineskip\undefined +\!let\!batchmode\batchmode \!let\batchmode\undefined +\!let\!begingroup\begingroup \!let\begingroup\undefined +\!let\!belowdisplayshortskip\belowdisplayshortskip \!let\belowdisplayshortskip\undefined +\!let\!belowdisplayskip\belowdisplayskip \!let\belowdisplayskip\undefined +\!let\!binoppenalty\binoppenalty \!let\binoppenalty\undefined +\!let\!botmark\botmark \!let\botmark\undefined +\!let\!box\box \!let\box\undefined +\!let\!boxmaxdepth\boxmaxdepth \!let\boxmaxdepth\undefined +\!let\!brokenpenalty\brokenpenalty \!let\brokenpenalty\undefined +\!let\!catcode\catcode \!let\catcode\undefined +\!let\!char\char \!let\char\undefined +\!let\!chardef\chardef \!let\chardef\undefined +\!let\!cleaders\cleaders \!let\cleaders\undefined +\!let\!closein\closein \!let\closein\undefined +\!let\!closeout\closeout \!let\closeout\undefined +\!let\!clubpenalty\clubpenalty \!let\clubpenalty\undefined +\!let\!copy\copy \!let\copy\undefined +\!let\!count\count \!let\count\undefined +\!let\!countdef\countdef \!let\countdef\undefined +\!let\!cr\cr \!let\cr\undefined +\!let\!crcr\crcr \!let\crcr\undefined +\!let\!csname\csname \!let\csname\undefined +\!let\!day\day \!let\day\undefined +\!let\!deadcycles\deadcycles \!let\deadcycles\undefined +\!let\!def\def \!let\def\undefined +\!let\!defaulthyphenchar\defaulthyphenchar \!let\defaulthyphenchar\undefined +\!let\!defaultskewchar\defaultskewchar \!let\defaultskewchar\undefined +\!let\!delcode\delcode \!let\delcode\undefined +\!let\!delimiter\delimiter \!let\delimiter\undefined +\!let\!delimiterfactor\delimiterfactor \!let\delimiterfactor\undefined +\!let\!delimitershortfall\delimitershortfall \!let\delimitershortfall\undefined +\!let\!dimen\dimen \!let\dimen\undefined +\!let\!dimendef\dimendef \!let\dimendef\undefined +\!let\!discretionary\discretionary \!let\discretionary\undefined +\!let\!displayindent\displayindent \!let\displayindent\undefined +\!let\!displaylimits\displaylimits \!let\displaylimits\undefined +\!let\!displaystyle\displaystyle \!let\displaystyle\undefined +\!let\!displaywidowpenalty\displaywidowpenalty \!let\displaywidowpenalty\undefined +\!let\!displaywidth\displaywidth \!let\displaywidth\undefined +\!let\!divide\divide \!let\divide\undefined +\!let\!doublehyphendemerits\doublehyphendemerits \!let\doublehyphendemerits\undefined +\!let\!dp\dp \!let\dp\undefined +\!let\!dump\dump \!let\dump\undefined +\!let\!edef\edef \!let\edef\undefined +\!let\!else\else \!let\else\undefined +\!let\!emergencystretch\emergencystretch \!let\emergencystretch\undefined +\!let\!end\end \!let\end\undefined +\!let\!endcsname\endcsname \!let\endcsname\undefined +\!let\!endgroup\endgroup \!let\endgroup\undefined +\!let\!endinput\endinput \!let\endinput\undefined +\!let\!endlinechar\endlinechar \!let\endlinechar\undefined +\!let\!eqno\eqno \!let\eqno\undefined +\!let\!errhelp\errhelp \!let\errhelp\undefined +\!let\!errmessage\errmessage \!let\errmessage\undefined +\!let\!errorcontextlines\errorcontextlines \!let\errorcontextlines\undefined +\!let\!errorstopmode\errorstopmode \!let\errorstopmode\undefined +\!let\!escapechar\escapechar \!let\escapechar\undefined +\!let\!everycr\everycr \!let\everycr\undefined +\!let\!everydisplay\everydisplay \!let\everydisplay\undefined +\!let\!everyhbox\everyhbox \!let\everyhbox\undefined +\!let\!everyjob\everyjob \!let\everyjob\undefined +\!let\!everymath\everymath \!let\everymath\undefined +\!let\!everypar\everypar \!let\everypar\undefined +\!let\!everyvbox\everyvbox \!let\everyvbox\undefined +\!let\!exhyphenpenalty\exhyphenpenalty \!let\exhyphenpenalty\undefined +\!let\!expandafter\expandafter \!let\expandafter\undefined +\!let\!fam\fam \!let\fam\undefined +\!let\!fi\fi \!let\fi\undefined +\!let\!finalhyphendemerits\finalhyphendemerits \!let\finalhyphendemerits\undefined +\!let\!firstmark\firstmark \!let\firstmark\undefined +\!let\!floatingpenalty\floatingpenalty \!let\floatingpenalty\undefined +\!let\!font\font \!let\font\undefined +\!let\!fontdimen\fontdimen \!let\fontdimen\undefined +\!let\!fontname\fontname \!let\fontname\undefined +\!let\!futurelet\futurelet \!let\futurelet\undefined +\!let\!gdef\gdef \!let\gdef\undefined +\!let\!global\global \!let\global\undefined +\!let\!globaldefs\globaldefs \!let\globaldefs\undefined +\!let\!halign\halign \!let\halign\undefined +\!let\!hangafter\hangafter \!let\hangafter\undefined +\!let\!hangindent\hangindent \!let\hangindent\undefined +\!let\!hbadness\hbadness \!let\hbadness\undefined +\!let\!hbox\hbox \!let\hbox\undefined +\!let\!hfil\hfil \!let\hfil\undefined +\!let\!hfill\hfill \!let\hfill\undefined +\!let\!hfilneg\hfilneg \!let\hfilneg\undefined +\!let\!hfuzz\hfuzz \!let\hfuzz\undefined +\!let\!hoffset\hoffset \!let\hoffset\undefined +\!let\!holdinginserts\holdinginserts \!let\holdinginserts\undefined +\!let\!hrule\hrule \!let\hrule\undefined +\!let\!hsize\hsize \!let\hsize\undefined +\!let\!hskip\hskip \!let\hskip\undefined +\!let\!hss\hss \!let\hss\undefined +\!let\!ht\ht \!let\ht\undefined +\!let\!hyphenation\hyphenation \!let\hyphenation\undefined +\!let\!hyphenchar\hyphenchar \!let\hyphenchar\undefined +\!let\!hyphenpenalty\hyphenpenalty \!let\hyphenpenalty\undefined +\!let\!if\if \!let\if\undefined +\!let\!ifcase\ifcase \!let\ifcase\undefined +\!let\!ifcat\ifcat \!let\ifcat\undefined +\!let\!ifdim\ifdim \!let\ifdim\undefined +\!let\!ifeof\ifeof \!let\ifeof\undefined +\!let\!iffalse\iffalse \!let\iffalse\undefined +\!let\!ifhbox\ifhbox \!let\ifhbox\undefined +\!let\!ifhmode\ifhmode \!let\ifhmode\undefined +\!let\!ifinner\ifinner \!let\ifinner\undefined +\!let\!ifmmode\ifmmode \!let\ifmmode\undefined +\!let\!ifnum\ifnum \!let\ifnum\undefined +\!let\!ifodd\ifodd \!let\ifodd\undefined +\!let\!iftrue\iftrue \!let\iftrue\undefined +\!let\!ifvbox\ifvbox \!let\ifvbox\undefined +\!let\!ifvmode\ifvmode \!let\ifvmode\undefined +\!let\!ifvoid\ifvoid \!let\ifvoid\undefined +\!let\!ifx\ifx \!let\ifx\undefined +\!let\!ignorespaces\ignorespaces \!let\ignorespaces\undefined +\!let\!immediate\immediate \!let\immediate\undefined +\!let\!indent\indent \!let\indent\undefined +\!let\!input\input \!let\input\undefined +\!let\!inputlineno\inputlineno \!let\inputlineno\undefined +\!let\!insert\insert \!let\insert\undefined +\!let\!insertpenalties\insertpenalties \!let\insertpenalties\undefined +\!let\!interlinepenalty\interlinepenalty \!let\interlinepenalty\undefined +\!let\!jobname\jobname \!let\jobname\undefined +\!let\!kern\kern \!let\kern\undefined +\!let\!language\language \!let\language\undefined +\!let\!lastbox\lastbox \!let\lastbox\undefined +\!let\!lastkern\lastkern \!let\lastkern\undefined +\!let\!lastpenalty\lastpenalty \!let\lastpenalty\undefined +\!let\!lastskip\lastskip \!let\lastskip\undefined +\!let\!lccode\lccode \!let\lccode\undefined +\!let\!leaders\leaders \!let\leaders\undefined +\!let\!left\left \!let\left\undefined +\!let\!lefthyphenmin\lefthyphenmin \!let\lefthyphenmin\undefined +\!let\!leftskip\leftskip \!let\leftskip\undefined +\!let\!leqno\leqno \!let\leqno\undefined +\!let\!let\let \!let\let\undefined +\!let\!limits\limits \!let\limits\undefined +\!let\!linepenalty\linepenalty \!let\linepenalty\undefined +\!let\!lineskip\lineskip \!let\lineskip\undefined +\!let\!lineskiplimit\lineskiplimit \!let\lineskiplimit\undefined +\!let\!long\long \!let\long\undefined +\!let\!looseness\looseness \!let\looseness\undefined +\!let\!lower\lower \!let\lower\undefined +\!let\!lowercase\lowercase \!let\lowercase\undefined +\!let\!mag\mag \!let\mag\undefined +\!let\!mark\mark \!let\mark\undefined +\!let\!mathaccent\mathaccent \!let\mathaccent\undefined +\!let\!mathbin\mathbin \!let\mathbin\undefined +\!let\!mathchar\mathchar \!let\mathchar\undefined +\!let\!mathchardef\mathchardef \!let\mathchardef\undefined +\!let\!mathchoice\mathchoice \!let\mathchoice\undefined +\!let\!mathclose\mathclose \!let\mathclose\undefined +\!let\!mathcode\mathcode \!let\mathcode\undefined +\!let\!mathinner\mathinner \!let\mathinner\undefined +\!let\!mathop\mathop \!let\mathop\undefined +\!let\!mathopen\mathopen \!let\mathopen\undefined +\!let\!mathord\mathord \!let\mathord\undefined +\!let\!mathpunct\mathpunct \!let\mathpunct\undefined +\!let\!mathrel\mathrel \!let\mathrel\undefined +\!let\!mathsurround\mathsurround \!let\mathsurround\undefined +\!let\!maxdeadcycles\maxdeadcycles \!let\maxdeadcycles\undefined +\!let\!maxdepth\maxdepth \!let\maxdepth\undefined +\!let\!meaning\meaning \!let\meaning\undefined +\!let\!medmuskip\medmuskip \!let\medmuskip\undefined +\!let\!message\message \!let\message\undefined +\!let\!mkern\mkern \!let\mkern\undefined +\!let\!month\month \!let\month\undefined +\!let\!moveleft\moveleft \!let\moveleft\undefined +\!let\!moveright\moveright \!let\moveright\undefined +\!let\!mskip\mskip \!let\mskip\undefined +\!let\!multiply\multiply \!let\multiply\undefined +\!let\!muskip\muskip \!let\muskip\undefined +\!let\!muskipdef\muskipdef \!let\muskipdef\undefined +\!let\!newlinechar\newlinechar \!let\newlinechar\undefined +\!let\!noalign\noalign \!let\noalign\undefined +\!let\!noboundary\noboundary \!let\noboundary\undefined +\!let\!noexpand\noexpand \!let\noexpand\undefined +\!let\!noindent\noindent \!let\noindent\undefined +\!let\!nolimits\nolimits \!let\nolimits\undefined +\!let\!nonscript\nonscript \!let\nonscript\undefined +\!let\!nonstopmode\nonstopmode \!let\nonstopmode\undefined +\!let\!nulldelimiterspace\nulldelimiterspace \!let\nulldelimiterspace\undefined +\!let\!nullfont\nullfont \!let\nullfont\undefined +\!let\!number\number \!let\number\undefined +\!let\!omit\omit \!let\omit\undefined +\!let\!openin\openin \!let\openin\undefined +\!let\!openout\openout \!let\openout\undefined +\!let\!or\or \!let\or\undefined +\!let\!outer\outer \!let\outer\undefined +\!let\!output\output \!let\output\undefined +\!let\!outputpenalty\outputpenalty \!let\outputpenalty\undefined +\!let\!over\over \!let\over\undefined +\!let\!overfullrule\overfullrule \!let\overfullrule\undefined +\!let\!overline\overline \!let\overline\undefined +\!let\!overwithdelims\overwithdelims \!let\overwithdelims\undefined +\!let\!pagedepth\pagedepth \!let\pagedepth\undefined +\!let\!pagefilllstretch\pagefilllstretch \!let\pagefilllstretch\undefined +\!let\!pagefillstretch\pagefillstretch \!let\pagefillstretch\undefined +\!let\!pagefilstretch\pagefilstretch \!let\pagefilstretch\undefined +\!let\!pagegoal\pagegoal \!let\pagegoal\undefined +\!let\!pageshrink\pageshrink \!let\pageshrink\undefined +\!let\!pagestretch\pagestretch \!let\pagestretch\undefined +\!let\!pagetotal\pagetotal \!let\pagetotal\undefined +\!let\!parfillskip\parfillskip \!let\parfillskip\undefined +\!let\!parindent\parindent \!let\parindent\undefined +\!let\!parshape\parshape \!let\parshape\undefined +\!let\!parskip\parskip \!let\parskip\undefined +\!let\!patterns\patterns \!let\patterns\undefined +\!let\!pausing\pausing \!let\pausing\undefined +\!let\!penalty\penalty \!let\penalty\undefined +\!let\!postdisplaypenalty\postdisplaypenalty \!let\postdisplaypenalty\undefined +\!let\!predisplaypenalty\predisplaypenalty \!let\predisplaypenalty\undefined +\!let\!predisplaysize\predisplaysize \!let\predisplaysize\undefined +\!let\!pretolerance\pretolerance \!let\pretolerance\undefined +\!let\!prevdepth\prevdepth \!let\prevdepth\undefined +\!let\!prevgraf\prevgraf \!let\prevgraf\undefined +\!let\!radical\radical \!let\radical\undefined +\!let\!raise\raise \!let\raise\undefined +\!let\!read\read \!let\read\undefined +\!let\!relax\relax \!let\relax\undefined +\!let\!relpenalty\relpenalty \!let\relpenalty\undefined +\!let\!right\right \!let\right\undefined +\!let\!righthyphenmin\righthyphenmin \!let\righthyphenmin\undefined +\!let\!rightskip\rightskip \!let\rightskip\undefined +\!let\!romannumeral\romannumeral \!let\romannumeral\undefined +\!let\!scriptfont\scriptfont \!let\scriptfont\undefined +\!let\!scriptscriptfont\scriptscriptfont \!let\scriptscriptfont\undefined +\!let\!scriptscriptstyle\scriptscriptstyle \!let\scriptscriptstyle\undefined +\!let\!scriptspace\scriptspace \!let\scriptspace\undefined +\!let\!scriptstyle\scriptstyle \!let\scriptstyle\undefined +\!let\!scrollmode\scrollmode \!let\scrollmode\undefined +\!let\!setbox\setbox \!let\setbox\undefined +\!let\!setlanguage\setlanguage \!let\setlanguage\undefined +\!let\!sfcode\sfcode \!let\sfcode\undefined +\!let\!shipout\shipout \!let\shipout\undefined +\!let\!show\show \!let\show\undefined +\!let\!showbox\showbox \!let\showbox\undefined +\!let\!showboxbreadth\showboxbreadth \!let\showboxbreadth\undefined +\!let\!showboxdepth\showboxdepth \!let\showboxdepth\undefined +\!let\!showlists\showlists \!let\showlists\undefined +\!let\!showthe\showthe \!let\showthe\undefined +\!let\!skewchar\skewchar \!let\skewchar\undefined +\!let\!skip\skip \!let\skip\undefined +\!let\!skipdef\skipdef \!let\skipdef\undefined +\!let\!spacefactor\spacefactor \!let\spacefactor\undefined +\!let\!spaceskip\spaceskip \!let\spaceskip\undefined +\!let\!span\span \!let\span\undefined +\!let\!special\special \!let\special\undefined +\!let\!splitbotmark\splitbotmark \!let\splitbotmark\undefined +\!let\!splitfirstmark\splitfirstmark \!let\splitfirstmark\undefined +\!let\!splitmaxdepth\splitmaxdepth \!let\splitmaxdepth\undefined +\!let\!splittopskip\splittopskip \!let\splittopskip\undefined +\!let\!string\string \!let\string\undefined +\!let\!tabskip\tabskip \!let\tabskip\undefined +\!let\!textfont\textfont \!let\textfont\undefined +\!let\!textstyle\textstyle \!let\textstyle\undefined +\!let\!the\the \!let\the\undefined +\!let\!thickmuskip\thickmuskip \!let\thickmuskip\undefined +\!let\!thinmuskip\thinmuskip \!let\thinmuskip\undefined +\!let\!time\time \!let\time\undefined +\!let\!toks\toks \!let\toks\undefined +\!let\!toksdef\toksdef \!let\toksdef\undefined +\!let\!tolerance\tolerance \!let\tolerance\undefined +\!let\!topmark\topmark \!let\topmark\undefined +\!let\!topskip\topskip \!let\topskip\undefined +\!let\!tracingcommands\tracingcommands \!let\tracingcommands\undefined +\!let\!tracinglostchars\tracinglostchars \!let\tracinglostchars\undefined +\!let\!tracingmacros\tracingmacros \!let\tracingmacros\undefined +\!let\!tracingonline\tracingonline \!let\tracingonline\undefined +\!let\!tracingoutput\tracingoutput \!let\tracingoutput\undefined +\!let\!tracingpages\tracingpages \!let\tracingpages\undefined +\!let\!tracingparagraphs\tracingparagraphs \!let\tracingparagraphs\undefined +\!let\!tracingrestores\tracingrestores \!let\tracingrestores\undefined +\!let\!tracingstats\tracingstats \!let\tracingstats\undefined +\!let\!uccode\uccode \!let\uccode\undefined +\!let\!uchyph\uchyph \!let\uchyph\undefined +\!let\!underline\underline \!let\underline\undefined +\!let\!unhbox\unhbox \!let\unhbox\undefined +\!let\!unhcopy\unhcopy \!let\unhcopy\undefined +\!let\!unkern\unkern \!let\unkern\undefined +\!let\!unpenalty\unpenalty \!let\unpenalty\undefined +\!let\!unskip\unskip \!let\unskip\undefined +\!let\!unvbox\unvbox \!let\unvbox\undefined +\!let\!unvcopy\unvcopy \!let\unvcopy\undefined +\!let\!uppercase\uppercase \!let\uppercase\undefined +\!let\!vadjust\vadjust \!let\vadjust\undefined +\!let\!valign\valign \!let\valign\undefined +\!let\!vbadness\vbadness \!let\vbadness\undefined +\!let\!vbox\vbox \!let\vbox\undefined +\!let\!vcenter\vcenter \!let\vcenter\undefined +\!let\!vfil\vfil \!let\vfil\undefined +\!let\!vfill\vfill \!let\vfill\undefined +\!let\!vfilneg\vfilneg \!let\vfilneg\undefined +\!let\!vfuzz\vfuzz \!let\vfuzz\undefined +\!let\!voffset\voffset \!let\voffset\undefined +\!let\!vrule\vrule \!let\vrule\undefined +\!let\!vsize\vsize \!let\vsize\undefined +\!let\!vskip\vskip \!let\vskip\undefined +\!let\!vsplit\vsplit \!let\vsplit\undefined +\!let\!vss\vss \!let\vss\undefined +\!let\!vtop\vtop \!let\vtop\undefined +\!let\!wd\wd \!let\wd\undefined +\!let\!widowpenalty\widowpenalty \!let\widowpenalty\undefined +\!let\!write\write \!let\write\undefined +\!let\!xdef\xdef \!let\xdef\undefined +\!let\!xleaders\xleaders \!let\xleaders\undefined +\!let\!xspaceskip\xspaceskip \!let\xspaceskip\undefined +\!let\!year\year \!let\year\undefined +% Copyright (c) 2005 Jonathan Fine <jfine@pytex.org> +% License: GPL version 2 or (at your option) any later version. +% $Source: /cvsroot/pytex/pytex/tex2tok/tex2tok.tex,v $ + +% usage: '\!LEX{story} + +\!catcode`|=0 % for document control sequences +\!catcode`{=1 +\!catcode`}=2 +\!catcode`$=3 +\!catcode`&=4 +\!catcode`#=6 +\!catcode`^=7 +\!catcode`_=8 +\!catcode`@=11 + +% useful constants +\!let\!bgroup { +\!let\!egroup } +\!chardef\!zero 0 +\!chardef\!one 1 + +\!def\?{\!let\!space = } \? % define \!space +\!let\? \!undefined % clean up afterwards + +\!chardef\!tokfile 16 % change if you wish + + +\!def\!loop +% loop through sequence of tokens +% assign to \!value the meaning of the next token +{% + \!futurelet\!value\!loop@A +} + + +\!def\!loop@continue +% pick up the next token, and continue the loop +% required, for dealing with space and brace characters +{% + \!afterassignment\!loop + \!let\!@temp = +} + + +\!def\!loop@A +% is the next token unexpandable? +% always, unexpandable tokens are characters +% always, control sequences are expandable (or undefined) +% sometimes, a character may be expandable +{% + \!expandafter\!ifx\!noexpand\!value\!value + \!expandafter\!loop@char + \!else + \!expandafter\!loop@macro + \!fi +} + + +\!def\!loop@macro +% suppress implicit \!outer token at end of file +{\!expandafter\!loop@macro@A\!noexpand} + +\!long\!def\!loop@macro@A #1% +% #1 - expandable token, picked up from input stream +% execute the token, and resume the loop +% if the token is undefined, call +{ + \!ifx #1\!undefined + \!process@undefined #1% + \!expandafter\!loop + \!else + \!expandafter #1% + \!fi +} + + +\!long\!def\!process@undefined #1% +% #1 - undefined control sequence +% default value - can be redefined +{% + \!immediate\!write\!tokfile{\!string #1}% +} + + +\!def\!endloop +% when read by \!loop, this ends its execution +{} + + +\!def\!loop@char +% process next token, which is a character +% might be letter, other, brace or whatever +% afterwards, continue with the loop +{% + % some tricky code to deal with common cases at high speed + \!ifcase + % first deal with the common cases, at high speed + \!ifcat a\!value \!one\!fi % it's a letter + \!ifcat ?\!value \!one\!fi % it's punctuation, digit, etc. + \!zero + \!expandafter\!loop@char@A % it's something else + \!or % case of letter or digit + \!expandafter\!loop@char@default + \!fi +} + + +\!def\!loop@char@default #1% +% #1 - a character we can pick up in this way +{% + \!immediate\!write\!tokfile{#1}% + \!loop +} + + +\!def\!loop@char@A +% next token is a special character +% pick it up, and process it +% we already have the \ +{% + \!ifx\!value + \!ifx\!value\!@space + \!immediate\!write16{ \!@percent} + \!else + \!immediate\!write16{% + \!expandafter\!@char@strip\!meaning\!value + \!@percent + }% + \!fi +} + + +\!def\!empty{} +\!def\!loop@char@A +{% + \!edef\!temp + {% + \!ifx\!value\!space + \!if11 \!fi + \!else\!ifx\!value\!bgroup + +\!string{% + \!else\!ifx\!value\!egroup + -\!string}% + \!fi\!fi\!fi + } + \!ifx\!temp\!empty + \!expandafter\!loop@char@default + \!else + \!immediate\!write\!tokfile{\!temp}% + \!expandafter\!loop@continue + \!fi +} + +\!def |LEX #1% +{% + \!begingroup + \!let|par\!undefined % blank lines in input document file + \!let\!LEX\!undefined % in case it appears in document file + \!catcode`!=12 % restore to normal value + \!catcode`|=12 % restore to normal value + \!catcode`@=12 % restore to normal value + \!chardef\!tokfile 15 + \!def\!next{\!immediate\!openout\!tokfile \!jobname.tok }% + \!expandafter\!next + \!expandafter\!loop + \!input #1 % space to terminate file name + \!endloop + \!immediate\!closeout\!tokfile + \!endgroup + \!end +} + +\!dump + |