summaryrefslogtreecommitdiff
path: root/support/tex2tok/tex2tok.tex
diff options
context:
space:
mode:
authorNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
committerNorbert Preining <norbert@preining.info>2019-09-02 13:46:59 +0900
commite0c6872cf40896c7be36b11dcc744620f10adf1d (patch)
tree60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/tex2tok/tex2tok.tex
Initial commit
Diffstat (limited to 'support/tex2tok/tex2tok.tex')
-rw-r--r--support/tex2tok/tex2tok.tex502
1 files changed, 502 insertions, 0 deletions
diff --git a/support/tex2tok/tex2tok.tex b/support/tex2tok/tex2tok.tex
new file mode 100644
index 0000000000..44701bc998
--- /dev/null
+++ b/support/tex2tok/tex2tok.tex
@@ -0,0 +1,502 @@
+% Automatically generated from _tex2tok.tex by helper.py
+\catcode`\!=11
+\let\!let\let
+\!let\-\undefined
+\!let\/\undefined
+\!let\ \undefined
+\!let\!above\above \!let\above\undefined
+\!let\!abovedisplayshortskip\abovedisplayshortskip \!let\abovedisplayshortskip\undefined
+\!let\!abovedisplayskip\abovedisplayskip \!let\abovedisplayskip\undefined
+\!let\!abovewithdelims\abovewithdelims \!let\abovewithdelims\undefined
+\!let\!accent\accent \!let\accent\undefined
+\!let\!adjdemerits\adjdemerits \!let\adjdemerits\undefined
+\!let\!advance\advance \!let\advance\undefined
+\!let\!afterassignment\afterassignment \!let\afterassignment\undefined
+\!let\!aftergroup\aftergroup \!let\aftergroup\undefined
+\!let\!atop\atop \!let\atop\undefined
+\!let\!atopwithdelims\atopwithdelims \!let\atopwithdelims\undefined
+\!let\!badness\badness \!let\badness\undefined
+\!let\!baselineskip\baselineskip \!let\baselineskip\undefined
+\!let\!batchmode\batchmode \!let\batchmode\undefined
+\!let\!begingroup\begingroup \!let\begingroup\undefined
+\!let\!belowdisplayshortskip\belowdisplayshortskip \!let\belowdisplayshortskip\undefined
+\!let\!belowdisplayskip\belowdisplayskip \!let\belowdisplayskip\undefined
+\!let\!binoppenalty\binoppenalty \!let\binoppenalty\undefined
+\!let\!botmark\botmark \!let\botmark\undefined
+\!let\!box\box \!let\box\undefined
+\!let\!boxmaxdepth\boxmaxdepth \!let\boxmaxdepth\undefined
+\!let\!brokenpenalty\brokenpenalty \!let\brokenpenalty\undefined
+\!let\!catcode\catcode \!let\catcode\undefined
+\!let\!char\char \!let\char\undefined
+\!let\!chardef\chardef \!let\chardef\undefined
+\!let\!cleaders\cleaders \!let\cleaders\undefined
+\!let\!closein\closein \!let\closein\undefined
+\!let\!closeout\closeout \!let\closeout\undefined
+\!let\!clubpenalty\clubpenalty \!let\clubpenalty\undefined
+\!let\!copy\copy \!let\copy\undefined
+\!let\!count\count \!let\count\undefined
+\!let\!countdef\countdef \!let\countdef\undefined
+\!let\!cr\cr \!let\cr\undefined
+\!let\!crcr\crcr \!let\crcr\undefined
+\!let\!csname\csname \!let\csname\undefined
+\!let\!day\day \!let\day\undefined
+\!let\!deadcycles\deadcycles \!let\deadcycles\undefined
+\!let\!def\def \!let\def\undefined
+\!let\!defaulthyphenchar\defaulthyphenchar \!let\defaulthyphenchar\undefined
+\!let\!defaultskewchar\defaultskewchar \!let\defaultskewchar\undefined
+\!let\!delcode\delcode \!let\delcode\undefined
+\!let\!delimiter\delimiter \!let\delimiter\undefined
+\!let\!delimiterfactor\delimiterfactor \!let\delimiterfactor\undefined
+\!let\!delimitershortfall\delimitershortfall \!let\delimitershortfall\undefined
+\!let\!dimen\dimen \!let\dimen\undefined
+\!let\!dimendef\dimendef \!let\dimendef\undefined
+\!let\!discretionary\discretionary \!let\discretionary\undefined
+\!let\!displayindent\displayindent \!let\displayindent\undefined
+\!let\!displaylimits\displaylimits \!let\displaylimits\undefined
+\!let\!displaystyle\displaystyle \!let\displaystyle\undefined
+\!let\!displaywidowpenalty\displaywidowpenalty \!let\displaywidowpenalty\undefined
+\!let\!displaywidth\displaywidth \!let\displaywidth\undefined
+\!let\!divide\divide \!let\divide\undefined
+\!let\!doublehyphendemerits\doublehyphendemerits \!let\doublehyphendemerits\undefined
+\!let\!dp\dp \!let\dp\undefined
+\!let\!dump\dump \!let\dump\undefined
+\!let\!edef\edef \!let\edef\undefined
+\!let\!else\else \!let\else\undefined
+\!let\!emergencystretch\emergencystretch \!let\emergencystretch\undefined
+\!let\!end\end \!let\end\undefined
+\!let\!endcsname\endcsname \!let\endcsname\undefined
+\!let\!endgroup\endgroup \!let\endgroup\undefined
+\!let\!endinput\endinput \!let\endinput\undefined
+\!let\!endlinechar\endlinechar \!let\endlinechar\undefined
+\!let\!eqno\eqno \!let\eqno\undefined
+\!let\!errhelp\errhelp \!let\errhelp\undefined
+\!let\!errmessage\errmessage \!let\errmessage\undefined
+\!let\!errorcontextlines\errorcontextlines \!let\errorcontextlines\undefined
+\!let\!errorstopmode\errorstopmode \!let\errorstopmode\undefined
+\!let\!escapechar\escapechar \!let\escapechar\undefined
+\!let\!everycr\everycr \!let\everycr\undefined
+\!let\!everydisplay\everydisplay \!let\everydisplay\undefined
+\!let\!everyhbox\everyhbox \!let\everyhbox\undefined
+\!let\!everyjob\everyjob \!let\everyjob\undefined
+\!let\!everymath\everymath \!let\everymath\undefined
+\!let\!everypar\everypar \!let\everypar\undefined
+\!let\!everyvbox\everyvbox \!let\everyvbox\undefined
+\!let\!exhyphenpenalty\exhyphenpenalty \!let\exhyphenpenalty\undefined
+\!let\!expandafter\expandafter \!let\expandafter\undefined
+\!let\!fam\fam \!let\fam\undefined
+\!let\!fi\fi \!let\fi\undefined
+\!let\!finalhyphendemerits\finalhyphendemerits \!let\finalhyphendemerits\undefined
+\!let\!firstmark\firstmark \!let\firstmark\undefined
+\!let\!floatingpenalty\floatingpenalty \!let\floatingpenalty\undefined
+\!let\!font\font \!let\font\undefined
+\!let\!fontdimen\fontdimen \!let\fontdimen\undefined
+\!let\!fontname\fontname \!let\fontname\undefined
+\!let\!futurelet\futurelet \!let\futurelet\undefined
+\!let\!gdef\gdef \!let\gdef\undefined
+\!let\!global\global \!let\global\undefined
+\!let\!globaldefs\globaldefs \!let\globaldefs\undefined
+\!let\!halign\halign \!let\halign\undefined
+\!let\!hangafter\hangafter \!let\hangafter\undefined
+\!let\!hangindent\hangindent \!let\hangindent\undefined
+\!let\!hbadness\hbadness \!let\hbadness\undefined
+\!let\!hbox\hbox \!let\hbox\undefined
+\!let\!hfil\hfil \!let\hfil\undefined
+\!let\!hfill\hfill \!let\hfill\undefined
+\!let\!hfilneg\hfilneg \!let\hfilneg\undefined
+\!let\!hfuzz\hfuzz \!let\hfuzz\undefined
+\!let\!hoffset\hoffset \!let\hoffset\undefined
+\!let\!holdinginserts\holdinginserts \!let\holdinginserts\undefined
+\!let\!hrule\hrule \!let\hrule\undefined
+\!let\!hsize\hsize \!let\hsize\undefined
+\!let\!hskip\hskip \!let\hskip\undefined
+\!let\!hss\hss \!let\hss\undefined
+\!let\!ht\ht \!let\ht\undefined
+\!let\!hyphenation\hyphenation \!let\hyphenation\undefined
+\!let\!hyphenchar\hyphenchar \!let\hyphenchar\undefined
+\!let\!hyphenpenalty\hyphenpenalty \!let\hyphenpenalty\undefined
+\!let\!if\if \!let\if\undefined
+\!let\!ifcase\ifcase \!let\ifcase\undefined
+\!let\!ifcat\ifcat \!let\ifcat\undefined
+\!let\!ifdim\ifdim \!let\ifdim\undefined
+\!let\!ifeof\ifeof \!let\ifeof\undefined
+\!let\!iffalse\iffalse \!let\iffalse\undefined
+\!let\!ifhbox\ifhbox \!let\ifhbox\undefined
+\!let\!ifhmode\ifhmode \!let\ifhmode\undefined
+\!let\!ifinner\ifinner \!let\ifinner\undefined
+\!let\!ifmmode\ifmmode \!let\ifmmode\undefined
+\!let\!ifnum\ifnum \!let\ifnum\undefined
+\!let\!ifodd\ifodd \!let\ifodd\undefined
+\!let\!iftrue\iftrue \!let\iftrue\undefined
+\!let\!ifvbox\ifvbox \!let\ifvbox\undefined
+\!let\!ifvmode\ifvmode \!let\ifvmode\undefined
+\!let\!ifvoid\ifvoid \!let\ifvoid\undefined
+\!let\!ifx\ifx \!let\ifx\undefined
+\!let\!ignorespaces\ignorespaces \!let\ignorespaces\undefined
+\!let\!immediate\immediate \!let\immediate\undefined
+\!let\!indent\indent \!let\indent\undefined
+\!let\!input\input \!let\input\undefined
+\!let\!inputlineno\inputlineno \!let\inputlineno\undefined
+\!let\!insert\insert \!let\insert\undefined
+\!let\!insertpenalties\insertpenalties \!let\insertpenalties\undefined
+\!let\!interlinepenalty\interlinepenalty \!let\interlinepenalty\undefined
+\!let\!jobname\jobname \!let\jobname\undefined
+\!let\!kern\kern \!let\kern\undefined
+\!let\!language\language \!let\language\undefined
+\!let\!lastbox\lastbox \!let\lastbox\undefined
+\!let\!lastkern\lastkern \!let\lastkern\undefined
+\!let\!lastpenalty\lastpenalty \!let\lastpenalty\undefined
+\!let\!lastskip\lastskip \!let\lastskip\undefined
+\!let\!lccode\lccode \!let\lccode\undefined
+\!let\!leaders\leaders \!let\leaders\undefined
+\!let\!left\left \!let\left\undefined
+\!let\!lefthyphenmin\lefthyphenmin \!let\lefthyphenmin\undefined
+\!let\!leftskip\leftskip \!let\leftskip\undefined
+\!let\!leqno\leqno \!let\leqno\undefined
+\!let\!let\let \!let\let\undefined
+\!let\!limits\limits \!let\limits\undefined
+\!let\!linepenalty\linepenalty \!let\linepenalty\undefined
+\!let\!lineskip\lineskip \!let\lineskip\undefined
+\!let\!lineskiplimit\lineskiplimit \!let\lineskiplimit\undefined
+\!let\!long\long \!let\long\undefined
+\!let\!looseness\looseness \!let\looseness\undefined
+\!let\!lower\lower \!let\lower\undefined
+\!let\!lowercase\lowercase \!let\lowercase\undefined
+\!let\!mag\mag \!let\mag\undefined
+\!let\!mark\mark \!let\mark\undefined
+\!let\!mathaccent\mathaccent \!let\mathaccent\undefined
+\!let\!mathbin\mathbin \!let\mathbin\undefined
+\!let\!mathchar\mathchar \!let\mathchar\undefined
+\!let\!mathchardef\mathchardef \!let\mathchardef\undefined
+\!let\!mathchoice\mathchoice \!let\mathchoice\undefined
+\!let\!mathclose\mathclose \!let\mathclose\undefined
+\!let\!mathcode\mathcode \!let\mathcode\undefined
+\!let\!mathinner\mathinner \!let\mathinner\undefined
+\!let\!mathop\mathop \!let\mathop\undefined
+\!let\!mathopen\mathopen \!let\mathopen\undefined
+\!let\!mathord\mathord \!let\mathord\undefined
+\!let\!mathpunct\mathpunct \!let\mathpunct\undefined
+\!let\!mathrel\mathrel \!let\mathrel\undefined
+\!let\!mathsurround\mathsurround \!let\mathsurround\undefined
+\!let\!maxdeadcycles\maxdeadcycles \!let\maxdeadcycles\undefined
+\!let\!maxdepth\maxdepth \!let\maxdepth\undefined
+\!let\!meaning\meaning \!let\meaning\undefined
+\!let\!medmuskip\medmuskip \!let\medmuskip\undefined
+\!let\!message\message \!let\message\undefined
+\!let\!mkern\mkern \!let\mkern\undefined
+\!let\!month\month \!let\month\undefined
+\!let\!moveleft\moveleft \!let\moveleft\undefined
+\!let\!moveright\moveright \!let\moveright\undefined
+\!let\!mskip\mskip \!let\mskip\undefined
+\!let\!multiply\multiply \!let\multiply\undefined
+\!let\!muskip\muskip \!let\muskip\undefined
+\!let\!muskipdef\muskipdef \!let\muskipdef\undefined
+\!let\!newlinechar\newlinechar \!let\newlinechar\undefined
+\!let\!noalign\noalign \!let\noalign\undefined
+\!let\!noboundary\noboundary \!let\noboundary\undefined
+\!let\!noexpand\noexpand \!let\noexpand\undefined
+\!let\!noindent\noindent \!let\noindent\undefined
+\!let\!nolimits\nolimits \!let\nolimits\undefined
+\!let\!nonscript\nonscript \!let\nonscript\undefined
+\!let\!nonstopmode\nonstopmode \!let\nonstopmode\undefined
+\!let\!nulldelimiterspace\nulldelimiterspace \!let\nulldelimiterspace\undefined
+\!let\!nullfont\nullfont \!let\nullfont\undefined
+\!let\!number\number \!let\number\undefined
+\!let\!omit\omit \!let\omit\undefined
+\!let\!openin\openin \!let\openin\undefined
+\!let\!openout\openout \!let\openout\undefined
+\!let\!or\or \!let\or\undefined
+\!let\!outer\outer \!let\outer\undefined
+\!let\!output\output \!let\output\undefined
+\!let\!outputpenalty\outputpenalty \!let\outputpenalty\undefined
+\!let\!over\over \!let\over\undefined
+\!let\!overfullrule\overfullrule \!let\overfullrule\undefined
+\!let\!overline\overline \!let\overline\undefined
+\!let\!overwithdelims\overwithdelims \!let\overwithdelims\undefined
+\!let\!pagedepth\pagedepth \!let\pagedepth\undefined
+\!let\!pagefilllstretch\pagefilllstretch \!let\pagefilllstretch\undefined
+\!let\!pagefillstretch\pagefillstretch \!let\pagefillstretch\undefined
+\!let\!pagefilstretch\pagefilstretch \!let\pagefilstretch\undefined
+\!let\!pagegoal\pagegoal \!let\pagegoal\undefined
+\!let\!pageshrink\pageshrink \!let\pageshrink\undefined
+\!let\!pagestretch\pagestretch \!let\pagestretch\undefined
+\!let\!pagetotal\pagetotal \!let\pagetotal\undefined
+\!let\!parfillskip\parfillskip \!let\parfillskip\undefined
+\!let\!parindent\parindent \!let\parindent\undefined
+\!let\!parshape\parshape \!let\parshape\undefined
+\!let\!parskip\parskip \!let\parskip\undefined
+\!let\!patterns\patterns \!let\patterns\undefined
+\!let\!pausing\pausing \!let\pausing\undefined
+\!let\!penalty\penalty \!let\penalty\undefined
+\!let\!postdisplaypenalty\postdisplaypenalty \!let\postdisplaypenalty\undefined
+\!let\!predisplaypenalty\predisplaypenalty \!let\predisplaypenalty\undefined
+\!let\!predisplaysize\predisplaysize \!let\predisplaysize\undefined
+\!let\!pretolerance\pretolerance \!let\pretolerance\undefined
+\!let\!prevdepth\prevdepth \!let\prevdepth\undefined
+\!let\!prevgraf\prevgraf \!let\prevgraf\undefined
+\!let\!radical\radical \!let\radical\undefined
+\!let\!raise\raise \!let\raise\undefined
+\!let\!read\read \!let\read\undefined
+\!let\!relax\relax \!let\relax\undefined
+\!let\!relpenalty\relpenalty \!let\relpenalty\undefined
+\!let\!right\right \!let\right\undefined
+\!let\!righthyphenmin\righthyphenmin \!let\righthyphenmin\undefined
+\!let\!rightskip\rightskip \!let\rightskip\undefined
+\!let\!romannumeral\romannumeral \!let\romannumeral\undefined
+\!let\!scriptfont\scriptfont \!let\scriptfont\undefined
+\!let\!scriptscriptfont\scriptscriptfont \!let\scriptscriptfont\undefined
+\!let\!scriptscriptstyle\scriptscriptstyle \!let\scriptscriptstyle\undefined
+\!let\!scriptspace\scriptspace \!let\scriptspace\undefined
+\!let\!scriptstyle\scriptstyle \!let\scriptstyle\undefined
+\!let\!scrollmode\scrollmode \!let\scrollmode\undefined
+\!let\!setbox\setbox \!let\setbox\undefined
+\!let\!setlanguage\setlanguage \!let\setlanguage\undefined
+\!let\!sfcode\sfcode \!let\sfcode\undefined
+\!let\!shipout\shipout \!let\shipout\undefined
+\!let\!show\show \!let\show\undefined
+\!let\!showbox\showbox \!let\showbox\undefined
+\!let\!showboxbreadth\showboxbreadth \!let\showboxbreadth\undefined
+\!let\!showboxdepth\showboxdepth \!let\showboxdepth\undefined
+\!let\!showlists\showlists \!let\showlists\undefined
+\!let\!showthe\showthe \!let\showthe\undefined
+\!let\!skewchar\skewchar \!let\skewchar\undefined
+\!let\!skip\skip \!let\skip\undefined
+\!let\!skipdef\skipdef \!let\skipdef\undefined
+\!let\!spacefactor\spacefactor \!let\spacefactor\undefined
+\!let\!spaceskip\spaceskip \!let\spaceskip\undefined
+\!let\!span\span \!let\span\undefined
+\!let\!special\special \!let\special\undefined
+\!let\!splitbotmark\splitbotmark \!let\splitbotmark\undefined
+\!let\!splitfirstmark\splitfirstmark \!let\splitfirstmark\undefined
+\!let\!splitmaxdepth\splitmaxdepth \!let\splitmaxdepth\undefined
+\!let\!splittopskip\splittopskip \!let\splittopskip\undefined
+\!let\!string\string \!let\string\undefined
+\!let\!tabskip\tabskip \!let\tabskip\undefined
+\!let\!textfont\textfont \!let\textfont\undefined
+\!let\!textstyle\textstyle \!let\textstyle\undefined
+\!let\!the\the \!let\the\undefined
+\!let\!thickmuskip\thickmuskip \!let\thickmuskip\undefined
+\!let\!thinmuskip\thinmuskip \!let\thinmuskip\undefined
+\!let\!time\time \!let\time\undefined
+\!let\!toks\toks \!let\toks\undefined
+\!let\!toksdef\toksdef \!let\toksdef\undefined
+\!let\!tolerance\tolerance \!let\tolerance\undefined
+\!let\!topmark\topmark \!let\topmark\undefined
+\!let\!topskip\topskip \!let\topskip\undefined
+\!let\!tracingcommands\tracingcommands \!let\tracingcommands\undefined
+\!let\!tracinglostchars\tracinglostchars \!let\tracinglostchars\undefined
+\!let\!tracingmacros\tracingmacros \!let\tracingmacros\undefined
+\!let\!tracingonline\tracingonline \!let\tracingonline\undefined
+\!let\!tracingoutput\tracingoutput \!let\tracingoutput\undefined
+\!let\!tracingpages\tracingpages \!let\tracingpages\undefined
+\!let\!tracingparagraphs\tracingparagraphs \!let\tracingparagraphs\undefined
+\!let\!tracingrestores\tracingrestores \!let\tracingrestores\undefined
+\!let\!tracingstats\tracingstats \!let\tracingstats\undefined
+\!let\!uccode\uccode \!let\uccode\undefined
+\!let\!uchyph\uchyph \!let\uchyph\undefined
+\!let\!underline\underline \!let\underline\undefined
+\!let\!unhbox\unhbox \!let\unhbox\undefined
+\!let\!unhcopy\unhcopy \!let\unhcopy\undefined
+\!let\!unkern\unkern \!let\unkern\undefined
+\!let\!unpenalty\unpenalty \!let\unpenalty\undefined
+\!let\!unskip\unskip \!let\unskip\undefined
+\!let\!unvbox\unvbox \!let\unvbox\undefined
+\!let\!unvcopy\unvcopy \!let\unvcopy\undefined
+\!let\!uppercase\uppercase \!let\uppercase\undefined
+\!let\!vadjust\vadjust \!let\vadjust\undefined
+\!let\!valign\valign \!let\valign\undefined
+\!let\!vbadness\vbadness \!let\vbadness\undefined
+\!let\!vbox\vbox \!let\vbox\undefined
+\!let\!vcenter\vcenter \!let\vcenter\undefined
+\!let\!vfil\vfil \!let\vfil\undefined
+\!let\!vfill\vfill \!let\vfill\undefined
+\!let\!vfilneg\vfilneg \!let\vfilneg\undefined
+\!let\!vfuzz\vfuzz \!let\vfuzz\undefined
+\!let\!voffset\voffset \!let\voffset\undefined
+\!let\!vrule\vrule \!let\vrule\undefined
+\!let\!vsize\vsize \!let\vsize\undefined
+\!let\!vskip\vskip \!let\vskip\undefined
+\!let\!vsplit\vsplit \!let\vsplit\undefined
+\!let\!vss\vss \!let\vss\undefined
+\!let\!vtop\vtop \!let\vtop\undefined
+\!let\!wd\wd \!let\wd\undefined
+\!let\!widowpenalty\widowpenalty \!let\widowpenalty\undefined
+\!let\!write\write \!let\write\undefined
+\!let\!xdef\xdef \!let\xdef\undefined
+\!let\!xleaders\xleaders \!let\xleaders\undefined
+\!let\!xspaceskip\xspaceskip \!let\xspaceskip\undefined
+\!let\!year\year \!let\year\undefined
+% Copyright (c) 2005 Jonathan Fine <jfine@pytex.org>
+% License: GPL version 2 or (at your option) any later version.
+% $Source: /cvsroot/pytex/pytex/tex2tok/tex2tok.tex,v $
+
+% usage: '\!LEX{story}
+
+\!catcode`|=0 % for document control sequences
+\!catcode`{=1
+\!catcode`}=2
+\!catcode`$=3
+\!catcode`&=4
+\!catcode`#=6
+\!catcode`^=7
+\!catcode`_=8
+\!catcode`@=11
+
+% useful constants
+\!let\!bgroup {
+\!let\!egroup }
+\!chardef\!zero 0
+\!chardef\!one 1
+
+\!def\?{\!let\!space = } \? % define \!space
+\!let\? \!undefined % clean up afterwards
+
+\!chardef\!tokfile 16 % change if you wish
+
+
+\!def\!loop
+% loop through sequence of tokens
+% assign to \!value the meaning of the next token
+{%
+ \!futurelet\!value\!loop@A
+}
+
+
+\!def\!loop@continue
+% pick up the next token, and continue the loop
+% required, for dealing with space and brace characters
+{%
+ \!afterassignment\!loop
+ \!let\!@temp =
+}
+
+
+\!def\!loop@A
+% is the next token unexpandable?
+% always, unexpandable tokens are characters
+% always, control sequences are expandable (or undefined)
+% sometimes, a character may be expandable
+{%
+ \!expandafter\!ifx\!noexpand\!value\!value
+ \!expandafter\!loop@char
+ \!else
+ \!expandafter\!loop@macro
+ \!fi
+}
+
+
+\!def\!loop@macro
+% suppress implicit \!outer token at end of file
+{\!expandafter\!loop@macro@A\!noexpand}
+
+\!long\!def\!loop@macro@A #1%
+% #1 - expandable token, picked up from input stream
+% execute the token, and resume the loop
+% if the token is undefined, call
+{
+ \!ifx #1\!undefined
+ \!process@undefined #1%
+ \!expandafter\!loop
+ \!else
+ \!expandafter #1%
+ \!fi
+}
+
+
+\!long\!def\!process@undefined #1%
+% #1 - undefined control sequence
+% default value - can be redefined
+{%
+ \!immediate\!write\!tokfile{\!string #1}%
+}
+
+
+\!def\!endloop
+% when read by \!loop, this ends its execution
+{}
+
+
+\!def\!loop@char
+% process next token, which is a character
+% might be letter, other, brace or whatever
+% afterwards, continue with the loop
+{%
+ % some tricky code to deal with common cases at high speed
+ \!ifcase
+ % first deal with the common cases, at high speed
+ \!ifcat a\!value \!one\!fi % it's a letter
+ \!ifcat ?\!value \!one\!fi % it's punctuation, digit, etc.
+ \!zero
+ \!expandafter\!loop@char@A % it's something else
+ \!or % case of letter or digit
+ \!expandafter\!loop@char@default
+ \!fi
+}
+
+
+\!def\!loop@char@default #1%
+% #1 - a character we can pick up in this way
+{%
+ \!immediate\!write\!tokfile{#1}%
+ \!loop
+}
+
+
+\!def\!loop@char@A
+% next token is a special character
+% pick it up, and process it
+% we already have the \
+{%
+ \!ifx\!value
+ \!ifx\!value\!@space
+ \!immediate\!write16{ \!@percent}
+ \!else
+ \!immediate\!write16{%
+ \!expandafter\!@char@strip\!meaning\!value
+ \!@percent
+ }%
+ \!fi
+}
+
+
+\!def\!empty{}
+\!def\!loop@char@A
+{%
+ \!edef\!temp
+ {%
+ \!ifx\!value\!space
+ \!if11 \!fi
+ \!else\!ifx\!value\!bgroup
+ +\!string{%
+ \!else\!ifx\!value\!egroup
+ -\!string}%
+ \!fi\!fi\!fi
+ }
+ \!ifx\!temp\!empty
+ \!expandafter\!loop@char@default
+ \!else
+ \!immediate\!write\!tokfile{\!temp}%
+ \!expandafter\!loop@continue
+ \!fi
+}
+
+\!def |LEX #1%
+{%
+ \!begingroup
+ \!let|par\!undefined % blank lines in input document file
+ \!let\!LEX\!undefined % in case it appears in document file
+ \!catcode`!=12 % restore to normal value
+ \!catcode`|=12 % restore to normal value
+ \!catcode`@=12 % restore to normal value
+ \!chardef\!tokfile 15
+ \!def\!next{\!immediate\!openout\!tokfile \!jobname.tok }%
+ \!expandafter\!next
+ \!expandafter\!loop
+ \!input #1 % space to terminate file name
+ \!endloop
+ \!immediate\!closeout\!tokfile
+ \!endgroup
+ \!end
+}
+
+\!dump
+