diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /support/tex2tok |
Initial commit
Diffstat (limited to 'support/tex2tok')
-rw-r--r-- | support/tex2tok/README | 15 | ||||
-rw-r--r-- | support/tex2tok/_tex2tok.tex | 174 | ||||
-rw-r--r-- | support/tex2tok/helper.py | 97 | ||||
-rw-r--r-- | support/tex2tok/tex2tok.tex | 502 |
4 files changed, 788 insertions, 0 deletions
diff --git a/support/tex2tok/README b/support/tex2tok/README new file mode 100644 index 0000000000..be8689cf41 --- /dev/null +++ b/support/tex2tok/README @@ -0,0 +1,15 @@ +# Copyright (c) 2005 Jonathan Fine <jfine@pytex.org> +# License: GPL version 2 or (at your option) any later version. +# $Source: /cvsroot/pytex/pytex/tex2tok/README,v $ + + +tex2tok: convert a TeX source file into tokens + +usage: +$ tex '&tex2tok' '\LEX{story}' +This will create a file story.tok. +This file will contain one line for each token. + +installation: +$ ./helper.py # create tex2tok.tex from _tex2tok.tex +$ initex tex2tok.tex # create 'tex2tok.fmt' diff --git a/support/tex2tok/_tex2tok.tex b/support/tex2tok/_tex2tok.tex new file mode 100644 index 0000000000..a8bc41aa1c --- /dev/null +++ b/support/tex2tok/_tex2tok.tex @@ -0,0 +1,174 @@ +% Copyright (c) 2005 Jonathan Fine <jfine@pytex.org> +% License: GPL version 2 or (at your option) any later version. +% $Source: /cvsroot/pytex/pytex/tex2tok/_tex2tok.tex,v $ + +% usage: '\LEX{story} + +\catcode`|=0 % for document control sequences +\catcode`{=1 +\catcode`}=2 +\catcode`$=3 +\catcode`&=4 +\catcode`#=6 +\catcode`^=7 +\catcode`_=8 +\catcode`@=11 + +% useful constants +\let\bgroup { +\let\egroup } +\chardef\zero 0 +\chardef\one 1 + +\def\?{\let\space = } \? % define \space +\let\? \undefined % clean up afterwards + +\chardef\tokfile 16 % change if you wish + + +\def\loop +% loop through sequence of tokens +% assign to \value the meaning of the next token +{% + \futurelet\value\loop@A +} + + +\def\loop@continue +% pick up the next token, and continue the loop +% required, for dealing with space and brace characters +{% + \afterassignment\loop + \let\@temp = +} + + +\def\loop@A +% is the next token unexpandable? +% always, unexpandable tokens are characters +% always, control sequences are expandable (or undefined) +% sometimes, a character may be expandable +{% + \expandafter\ifx\noexpand\value\value + \expandafter\loop@char + \else + \expandafter\loop@macro + \fi +} + + +\def\loop@macro +% suppress implicit \outer token at end of file +{\expandafter\loop@macro@A\noexpand} + +\long\def\loop@macro@A #1% +% #1 - expandable token, picked up from input stream +% execute the token, and resume the loop +% if the token is undefined, call +{ + \ifx #1\undefined + \process@undefined #1% + \expandafter\loop + \else + \expandafter #1% + \fi +} + + +\long\def\process@undefined #1% +% #1 - undefined control sequence +% default value - can be redefined +{% + \immediate\write\tokfile{\string #1}% +} + + +\def\endloop +% when read by \loop, this ends its execution +{} + + +\def\loop@char +% process next token, which is a character +% might be letter, other, brace or whatever +% afterwards, continue with the loop +{% + % some tricky code to deal with common cases at high speed + \ifcase + % first deal with the common cases, at high speed + \ifcat a\value \one\fi % it's a letter + \ifcat ?\value \one\fi % it's punctuation, digit, etc. + \zero + \expandafter\loop@char@A % it's something else + \or % case of letter or digit + \expandafter\loop@char@default + \fi +} + + +\def\loop@char@default #1% +% #1 - a character we can pick up in this way +{% + \immediate\write\tokfile{#1}% + \loop +} + + +\def\loop@char@A +% next token is a special character +% pick it up, and process it +% we already have the \ +{% + \ifx\value + \ifx\value\@space + \immediate\write16{ \@percent} + \else + \immediate\write16{% + \expandafter\@char@strip\meaning\value + \@percent + }% + \fi +} + + +\def\empty{} +\def\loop@char@A +{% + \edef\temp + {% + \ifx\value\space + \if11 \fi + \else\ifx\value\bgroup + +\string{% + \else\ifx\value\egroup + -\string}% + \fi\fi\fi + } + \ifx\temp\empty + \expandafter\loop@char@default + \else + \immediate\write\tokfile{\temp}% + \expandafter\loop@continue + \fi +} + +\def |LEX #1% +{% + \begingroup + \let|par\undefined % blank lines in input document file + \let\LEX\undefined % in case it appears in document file + \catcode`!=12 % restore to normal value + \catcode`|=12 % restore to normal value + \catcode`@=12 % restore to normal value + \chardef\tokfile 15 + \def\next{\immediate\openout\tokfile \jobname.tok }% + \expandafter\next + \expandafter\loop + \input #1 % space to terminate file name + \endloop + \immediate\closeout\tokfile + \endgroup + \end +} + +\dump diff --git a/support/tex2tok/helper.py b/support/tex2tok/helper.py new file mode 100644 index 0000000000..83a52ea25c --- /dev/null +++ b/support/tex2tok/helper.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python +# Copyright (c) 2005 Jonathan Fine <jfine@pytex.org> +# License: GPL version 2 or (at your option) any later version. +# $Source: /cvsroot/pytex/pytex/tex2tok/helper.py,v $ + +__version__ = '$Revision: 1.1 $' + + +import sys +import re + +# list of all TeX primitive control sequences +# par, \-, \/ and control space are not in the list + +primitives = ['above', 'abovedisplayshortskip', 'abovedisplayskip', +'abovewithdelims', 'accent', 'adjdemerits', 'advance', +'afterassignment', 'aftergroup', 'atop', 'atopwithdelims', 'badness', +'baselineskip', 'batchmode', 'begingroup', 'belowdisplayshortskip', +'belowdisplayskip', 'binoppenalty', 'botmark', 'box', 'boxmaxdepth', +'brokenpenalty', 'catcode', 'char', 'chardef', 'cleaders', 'closein', +'closeout', 'clubpenalty', 'copy', 'count', 'countdef', 'cr', 'crcr', +'csname', 'day', 'deadcycles', 'def', 'defaulthyphenchar', +'defaultskewchar', 'delcode', 'delimiter', 'delimiterfactor', +'delimitershortfall', 'dimen', 'dimendef', 'discretionary', +'displayindent', 'displaylimits', 'displaystyle', +'displaywidowpenalty', 'displaywidth', 'divide', +'doublehyphendemerits', 'dp', 'dump', 'edef', 'else', +'emergencystretch', 'end', 'endcsname', 'endgroup', 'endinput', +'endlinechar', 'eqno', 'errhelp', 'errmessage', 'errorcontextlines', +'errorstopmode', 'escapechar', 'everycr', 'everydisplay', 'everyhbox', +'everyjob', 'everymath', 'everypar', 'everyvbox', 'exhyphenpenalty', +'expandafter', 'fam', 'fi', 'finalhyphendemerits', 'firstmark', +'floatingpenalty', 'font', 'fontdimen', 'fontname', 'futurelet', +'gdef', 'global', 'globaldefs', 'halign', 'hangafter', 'hangindent', +'hbadness', 'hbox', 'hfil', 'hfill', 'hfilneg', 'hfuzz', 'hoffset', +'holdinginserts', 'hrule', 'hsize', 'hskip', 'hss', 'ht', +'hyphenation', 'hyphenchar', 'hyphenpenalty', 'if', 'ifcase', 'ifcat', +'ifdim', 'ifeof', 'iffalse', 'ifhbox', 'ifhmode', 'ifinner', +'ifmmode', 'ifnum', 'ifodd', 'iftrue', 'ifvbox', 'ifvmode', 'ifvoid', +'ifx', 'ignorespaces', 'immediate', 'indent', 'input', 'inputlineno', +'insert', 'insertpenalties', 'interlinepenalty', 'jobname', 'kern', +'language', 'lastbox', 'lastkern', 'lastpenalty', 'lastskip', +'lccode', 'leaders', 'left', 'lefthyphenmin', 'leftskip', 'leqno', +'let', 'limits', 'linepenalty', 'lineskip', 'lineskiplimit', 'long', +'looseness', 'lower', 'lowercase', 'mag', 'mark', 'mathaccent', +'mathbin', 'mathchar', 'mathchardef', 'mathchoice', 'mathclose', +'mathcode', 'mathinner', 'mathop', 'mathopen', 'mathord', 'mathpunct', +'mathrel', 'mathsurround', 'maxdeadcycles', 'maxdepth', 'meaning', +'medmuskip', 'message', 'mkern', 'month', 'moveleft', 'moveright', +'mskip', 'multiply', 'muskip', 'muskipdef', 'newlinechar', 'noalign', +'noboundary', 'noexpand', 'noindent', 'nolimits', 'nonscript', +'nonstopmode', 'nulldelimiterspace', 'nullfont', 'number', 'omit', +'openin', 'openout', 'or', 'outer', 'output', 'outputpenalty', 'over', +'overfullrule', 'overline', 'overwithdelims', 'pagedepth', +'pagefilllstretch', 'pagefillstretch', 'pagefilstretch', 'pagegoal', +'pageshrink', 'pagestretch', 'pagetotal', 'parfillskip', 'parindent', +'parshape', 'parskip', 'patterns', 'pausing', 'penalty', +'postdisplaypenalty', 'predisplaypenalty', 'predisplaysize', +'pretolerance', 'prevdepth', 'prevgraf', 'radical', 'raise', 'read', +'relax', 'relpenalty', 'right', 'righthyphenmin', 'rightskip', +'romannumeral', 'scriptfont', 'scriptscriptfont', 'scriptscriptstyle', +'scriptspace', 'scriptstyle', 'scrollmode', 'setbox', 'setlanguage', +'sfcode', 'shipout', 'show', 'showbox', 'showboxbreadth', +'showboxdepth', 'showlists', 'showthe', 'skewchar', 'skip', 'skipdef', +'spacefactor', 'spaceskip', 'span', 'special', 'splitbotmark', +'splitfirstmark', 'splitmaxdepth', 'splittopskip', 'string', +'tabskip', 'textfont', 'textstyle', 'the', 'thickmuskip', +'thinmuskip', 'time', 'toks', 'toksdef', 'tolerance', 'topmark', +'topskip', 'tracingcommands', 'tracinglostchars', 'tracingmacros', +'tracingonline', 'tracingoutput', 'tracingpages', 'tracingparagraphs', +'tracingrestores', 'tracingstats', 'uccode', 'uchyph', 'underline', +'unhbox', 'unhcopy', 'unkern', 'unpenalty', 'unskip', 'unvbox', +'unvcopy', 'uppercase', 'vadjust', 'valign', 'vbadness', 'vbox', +'vcenter', 'vfil', 'vfill', 'vfilneg', 'vfuzz', 'voffset', 'vrule', +'vsize', 'vskip', 'vsplit', 'vss', 'vtop', 'wd', 'widowpenalty', +'write', 'xdef', 'xleaders', 'xspaceskip', 'year'] + + + +data = open('_tex2tok.tex').read() +sys.stdout = open('tex2tok.tex', 'wt') + +print '% Automatically generated from _tex2tok.tex by helper.py' +print '\catcode`\!=11' +print '\let\!let\let' +for char in '-/ ': + print r'\!let\%s\undefined' % char +for name in primitives: + print r'\!let\!%s\%s \!let\%s\undefined' % (name, name, name) + + +cs_re = re.compile(r'\\([@a-zA-Z])') +transform = cs_re.sub(r'\\!\1', data) + + + +print transform diff --git a/support/tex2tok/tex2tok.tex b/support/tex2tok/tex2tok.tex new file mode 100644 index 0000000000..44701bc998 --- /dev/null +++ b/support/tex2tok/tex2tok.tex @@ -0,0 +1,502 @@ +% Automatically generated from _tex2tok.tex by helper.py +\catcode`\!=11 +\let\!let\let +\!let\-\undefined +\!let\/\undefined +\!let\ \undefined +\!let\!above\above \!let\above\undefined +\!let\!abovedisplayshortskip\abovedisplayshortskip \!let\abovedisplayshortskip\undefined +\!let\!abovedisplayskip\abovedisplayskip \!let\abovedisplayskip\undefined +\!let\!abovewithdelims\abovewithdelims \!let\abovewithdelims\undefined +\!let\!accent\accent \!let\accent\undefined +\!let\!adjdemerits\adjdemerits \!let\adjdemerits\undefined +\!let\!advance\advance \!let\advance\undefined +\!let\!afterassignment\afterassignment \!let\afterassignment\undefined +\!let\!aftergroup\aftergroup \!let\aftergroup\undefined +\!let\!atop\atop \!let\atop\undefined +\!let\!atopwithdelims\atopwithdelims \!let\atopwithdelims\undefined +\!let\!badness\badness \!let\badness\undefined +\!let\!baselineskip\baselineskip \!let\baselineskip\undefined +\!let\!batchmode\batchmode \!let\batchmode\undefined +\!let\!begingroup\begingroup \!let\begingroup\undefined +\!let\!belowdisplayshortskip\belowdisplayshortskip \!let\belowdisplayshortskip\undefined +\!let\!belowdisplayskip\belowdisplayskip \!let\belowdisplayskip\undefined +\!let\!binoppenalty\binoppenalty \!let\binoppenalty\undefined +\!let\!botmark\botmark \!let\botmark\undefined +\!let\!box\box \!let\box\undefined +\!let\!boxmaxdepth\boxmaxdepth \!let\boxmaxdepth\undefined +\!let\!brokenpenalty\brokenpenalty \!let\brokenpenalty\undefined +\!let\!catcode\catcode \!let\catcode\undefined +\!let\!char\char \!let\char\undefined +\!let\!chardef\chardef \!let\chardef\undefined +\!let\!cleaders\cleaders \!let\cleaders\undefined +\!let\!closein\closein \!let\closein\undefined +\!let\!closeout\closeout \!let\closeout\undefined +\!let\!clubpenalty\clubpenalty \!let\clubpenalty\undefined +\!let\!copy\copy \!let\copy\undefined +\!let\!count\count \!let\count\undefined +\!let\!countdef\countdef \!let\countdef\undefined +\!let\!cr\cr \!let\cr\undefined +\!let\!crcr\crcr \!let\crcr\undefined +\!let\!csname\csname \!let\csname\undefined +\!let\!day\day \!let\day\undefined +\!let\!deadcycles\deadcycles \!let\deadcycles\undefined +\!let\!def\def \!let\def\undefined +\!let\!defaulthyphenchar\defaulthyphenchar \!let\defaulthyphenchar\undefined +\!let\!defaultskewchar\defaultskewchar \!let\defaultskewchar\undefined +\!let\!delcode\delcode \!let\delcode\undefined +\!let\!delimiter\delimiter \!let\delimiter\undefined +\!let\!delimiterfactor\delimiterfactor \!let\delimiterfactor\undefined +\!let\!delimitershortfall\delimitershortfall \!let\delimitershortfall\undefined +\!let\!dimen\dimen \!let\dimen\undefined +\!let\!dimendef\dimendef \!let\dimendef\undefined +\!let\!discretionary\discretionary \!let\discretionary\undefined +\!let\!displayindent\displayindent \!let\displayindent\undefined +\!let\!displaylimits\displaylimits \!let\displaylimits\undefined +\!let\!displaystyle\displaystyle \!let\displaystyle\undefined +\!let\!displaywidowpenalty\displaywidowpenalty \!let\displaywidowpenalty\undefined +\!let\!displaywidth\displaywidth \!let\displaywidth\undefined +\!let\!divide\divide \!let\divide\undefined +\!let\!doublehyphendemerits\doublehyphendemerits \!let\doublehyphendemerits\undefined +\!let\!dp\dp \!let\dp\undefined +\!let\!dump\dump \!let\dump\undefined +\!let\!edef\edef \!let\edef\undefined +\!let\!else\else \!let\else\undefined +\!let\!emergencystretch\emergencystretch \!let\emergencystretch\undefined +\!let\!end\end \!let\end\undefined +\!let\!endcsname\endcsname \!let\endcsname\undefined +\!let\!endgroup\endgroup \!let\endgroup\undefined +\!let\!endinput\endinput \!let\endinput\undefined +\!let\!endlinechar\endlinechar \!let\endlinechar\undefined +\!let\!eqno\eqno \!let\eqno\undefined +\!let\!errhelp\errhelp \!let\errhelp\undefined +\!let\!errmessage\errmessage \!let\errmessage\undefined +\!let\!errorcontextlines\errorcontextlines \!let\errorcontextlines\undefined +\!let\!errorstopmode\errorstopmode \!let\errorstopmode\undefined +\!let\!escapechar\escapechar \!let\escapechar\undefined +\!let\!everycr\everycr \!let\everycr\undefined +\!let\!everydisplay\everydisplay \!let\everydisplay\undefined +\!let\!everyhbox\everyhbox \!let\everyhbox\undefined +\!let\!everyjob\everyjob \!let\everyjob\undefined +\!let\!everymath\everymath \!let\everymath\undefined +\!let\!everypar\everypar \!let\everypar\undefined +\!let\!everyvbox\everyvbox \!let\everyvbox\undefined +\!let\!exhyphenpenalty\exhyphenpenalty \!let\exhyphenpenalty\undefined +\!let\!expandafter\expandafter \!let\expandafter\undefined +\!let\!fam\fam \!let\fam\undefined +\!let\!fi\fi \!let\fi\undefined +\!let\!finalhyphendemerits\finalhyphendemerits \!let\finalhyphendemerits\undefined +\!let\!firstmark\firstmark \!let\firstmark\undefined +\!let\!floatingpenalty\floatingpenalty \!let\floatingpenalty\undefined +\!let\!font\font \!let\font\undefined +\!let\!fontdimen\fontdimen \!let\fontdimen\undefined +\!let\!fontname\fontname \!let\fontname\undefined +\!let\!futurelet\futurelet \!let\futurelet\undefined +\!let\!gdef\gdef \!let\gdef\undefined +\!let\!global\global \!let\global\undefined +\!let\!globaldefs\globaldefs \!let\globaldefs\undefined +\!let\!halign\halign \!let\halign\undefined +\!let\!hangafter\hangafter \!let\hangafter\undefined +\!let\!hangindent\hangindent \!let\hangindent\undefined +\!let\!hbadness\hbadness \!let\hbadness\undefined +\!let\!hbox\hbox \!let\hbox\undefined +\!let\!hfil\hfil \!let\hfil\undefined +\!let\!hfill\hfill \!let\hfill\undefined +\!let\!hfilneg\hfilneg \!let\hfilneg\undefined +\!let\!hfuzz\hfuzz \!let\hfuzz\undefined +\!let\!hoffset\hoffset \!let\hoffset\undefined +\!let\!holdinginserts\holdinginserts \!let\holdinginserts\undefined +\!let\!hrule\hrule \!let\hrule\undefined +\!let\!hsize\hsize \!let\hsize\undefined +\!let\!hskip\hskip \!let\hskip\undefined +\!let\!hss\hss \!let\hss\undefined +\!let\!ht\ht \!let\ht\undefined +\!let\!hyphenation\hyphenation \!let\hyphenation\undefined +\!let\!hyphenchar\hyphenchar \!let\hyphenchar\undefined +\!let\!hyphenpenalty\hyphenpenalty \!let\hyphenpenalty\undefined +\!let\!if\if \!let\if\undefined +\!let\!ifcase\ifcase \!let\ifcase\undefined +\!let\!ifcat\ifcat \!let\ifcat\undefined +\!let\!ifdim\ifdim \!let\ifdim\undefined +\!let\!ifeof\ifeof \!let\ifeof\undefined +\!let\!iffalse\iffalse \!let\iffalse\undefined +\!let\!ifhbox\ifhbox \!let\ifhbox\undefined +\!let\!ifhmode\ifhmode \!let\ifhmode\undefined +\!let\!ifinner\ifinner \!let\ifinner\undefined +\!let\!ifmmode\ifmmode \!let\ifmmode\undefined +\!let\!ifnum\ifnum \!let\ifnum\undefined +\!let\!ifodd\ifodd \!let\ifodd\undefined +\!let\!iftrue\iftrue \!let\iftrue\undefined +\!let\!ifvbox\ifvbox \!let\ifvbox\undefined +\!let\!ifvmode\ifvmode \!let\ifvmode\undefined +\!let\!ifvoid\ifvoid \!let\ifvoid\undefined +\!let\!ifx\ifx \!let\ifx\undefined +\!let\!ignorespaces\ignorespaces \!let\ignorespaces\undefined +\!let\!immediate\immediate \!let\immediate\undefined +\!let\!indent\indent \!let\indent\undefined +\!let\!input\input \!let\input\undefined +\!let\!inputlineno\inputlineno \!let\inputlineno\undefined +\!let\!insert\insert \!let\insert\undefined +\!let\!insertpenalties\insertpenalties \!let\insertpenalties\undefined +\!let\!interlinepenalty\interlinepenalty \!let\interlinepenalty\undefined +\!let\!jobname\jobname \!let\jobname\undefined +\!let\!kern\kern \!let\kern\undefined +\!let\!language\language \!let\language\undefined +\!let\!lastbox\lastbox \!let\lastbox\undefined +\!let\!lastkern\lastkern \!let\lastkern\undefined +\!let\!lastpenalty\lastpenalty \!let\lastpenalty\undefined +\!let\!lastskip\lastskip \!let\lastskip\undefined +\!let\!lccode\lccode \!let\lccode\undefined +\!let\!leaders\leaders \!let\leaders\undefined +\!let\!left\left \!let\left\undefined +\!let\!lefthyphenmin\lefthyphenmin \!let\lefthyphenmin\undefined +\!let\!leftskip\leftskip \!let\leftskip\undefined +\!let\!leqno\leqno \!let\leqno\undefined +\!let\!let\let \!let\let\undefined +\!let\!limits\limits \!let\limits\undefined +\!let\!linepenalty\linepenalty \!let\linepenalty\undefined +\!let\!lineskip\lineskip \!let\lineskip\undefined +\!let\!lineskiplimit\lineskiplimit \!let\lineskiplimit\undefined +\!let\!long\long \!let\long\undefined +\!let\!looseness\looseness \!let\looseness\undefined +\!let\!lower\lower \!let\lower\undefined +\!let\!lowercase\lowercase \!let\lowercase\undefined +\!let\!mag\mag \!let\mag\undefined +\!let\!mark\mark \!let\mark\undefined +\!let\!mathaccent\mathaccent \!let\mathaccent\undefined +\!let\!mathbin\mathbin \!let\mathbin\undefined +\!let\!mathchar\mathchar \!let\mathchar\undefined +\!let\!mathchardef\mathchardef \!let\mathchardef\undefined +\!let\!mathchoice\mathchoice \!let\mathchoice\undefined +\!let\!mathclose\mathclose \!let\mathclose\undefined +\!let\!mathcode\mathcode \!let\mathcode\undefined +\!let\!mathinner\mathinner \!let\mathinner\undefined +\!let\!mathop\mathop \!let\mathop\undefined +\!let\!mathopen\mathopen \!let\mathopen\undefined +\!let\!mathord\mathord \!let\mathord\undefined +\!let\!mathpunct\mathpunct \!let\mathpunct\undefined +\!let\!mathrel\mathrel \!let\mathrel\undefined +\!let\!mathsurround\mathsurround \!let\mathsurround\undefined +\!let\!maxdeadcycles\maxdeadcycles \!let\maxdeadcycles\undefined +\!let\!maxdepth\maxdepth \!let\maxdepth\undefined +\!let\!meaning\meaning \!let\meaning\undefined +\!let\!medmuskip\medmuskip \!let\medmuskip\undefined +\!let\!message\message \!let\message\undefined +\!let\!mkern\mkern \!let\mkern\undefined +\!let\!month\month \!let\month\undefined +\!let\!moveleft\moveleft \!let\moveleft\undefined +\!let\!moveright\moveright \!let\moveright\undefined +\!let\!mskip\mskip \!let\mskip\undefined +\!let\!multiply\multiply \!let\multiply\undefined +\!let\!muskip\muskip \!let\muskip\undefined +\!let\!muskipdef\muskipdef \!let\muskipdef\undefined +\!let\!newlinechar\newlinechar \!let\newlinechar\undefined +\!let\!noalign\noalign \!let\noalign\undefined +\!let\!noboundary\noboundary \!let\noboundary\undefined +\!let\!noexpand\noexpand \!let\noexpand\undefined +\!let\!noindent\noindent \!let\noindent\undefined +\!let\!nolimits\nolimits \!let\nolimits\undefined +\!let\!nonscript\nonscript \!let\nonscript\undefined +\!let\!nonstopmode\nonstopmode \!let\nonstopmode\undefined +\!let\!nulldelimiterspace\nulldelimiterspace \!let\nulldelimiterspace\undefined +\!let\!nullfont\nullfont \!let\nullfont\undefined +\!let\!number\number \!let\number\undefined +\!let\!omit\omit \!let\omit\undefined +\!let\!openin\openin \!let\openin\undefined +\!let\!openout\openout \!let\openout\undefined +\!let\!or\or \!let\or\undefined +\!let\!outer\outer \!let\outer\undefined +\!let\!output\output \!let\output\undefined +\!let\!outputpenalty\outputpenalty \!let\outputpenalty\undefined +\!let\!over\over \!let\over\undefined +\!let\!overfullrule\overfullrule \!let\overfullrule\undefined +\!let\!overline\overline \!let\overline\undefined +\!let\!overwithdelims\overwithdelims \!let\overwithdelims\undefined +\!let\!pagedepth\pagedepth \!let\pagedepth\undefined +\!let\!pagefilllstretch\pagefilllstretch \!let\pagefilllstretch\undefined +\!let\!pagefillstretch\pagefillstretch \!let\pagefillstretch\undefined +\!let\!pagefilstretch\pagefilstretch \!let\pagefilstretch\undefined +\!let\!pagegoal\pagegoal \!let\pagegoal\undefined +\!let\!pageshrink\pageshrink \!let\pageshrink\undefined +\!let\!pagestretch\pagestretch \!let\pagestretch\undefined +\!let\!pagetotal\pagetotal \!let\pagetotal\undefined +\!let\!parfillskip\parfillskip \!let\parfillskip\undefined +\!let\!parindent\parindent \!let\parindent\undefined +\!let\!parshape\parshape \!let\parshape\undefined +\!let\!parskip\parskip \!let\parskip\undefined +\!let\!patterns\patterns \!let\patterns\undefined +\!let\!pausing\pausing \!let\pausing\undefined +\!let\!penalty\penalty \!let\penalty\undefined +\!let\!postdisplaypenalty\postdisplaypenalty \!let\postdisplaypenalty\undefined +\!let\!predisplaypenalty\predisplaypenalty \!let\predisplaypenalty\undefined +\!let\!predisplaysize\predisplaysize \!let\predisplaysize\undefined +\!let\!pretolerance\pretolerance \!let\pretolerance\undefined +\!let\!prevdepth\prevdepth \!let\prevdepth\undefined +\!let\!prevgraf\prevgraf \!let\prevgraf\undefined +\!let\!radical\radical \!let\radical\undefined +\!let\!raise\raise \!let\raise\undefined +\!let\!read\read \!let\read\undefined +\!let\!relax\relax \!let\relax\undefined +\!let\!relpenalty\relpenalty \!let\relpenalty\undefined +\!let\!right\right \!let\right\undefined +\!let\!righthyphenmin\righthyphenmin \!let\righthyphenmin\undefined +\!let\!rightskip\rightskip \!let\rightskip\undefined +\!let\!romannumeral\romannumeral \!let\romannumeral\undefined +\!let\!scriptfont\scriptfont \!let\scriptfont\undefined +\!let\!scriptscriptfont\scriptscriptfont \!let\scriptscriptfont\undefined +\!let\!scriptscriptstyle\scriptscriptstyle \!let\scriptscriptstyle\undefined +\!let\!scriptspace\scriptspace \!let\scriptspace\undefined +\!let\!scriptstyle\scriptstyle \!let\scriptstyle\undefined +\!let\!scrollmode\scrollmode \!let\scrollmode\undefined +\!let\!setbox\setbox \!let\setbox\undefined +\!let\!setlanguage\setlanguage \!let\setlanguage\undefined +\!let\!sfcode\sfcode \!let\sfcode\undefined +\!let\!shipout\shipout \!let\shipout\undefined +\!let\!show\show \!let\show\undefined +\!let\!showbox\showbox \!let\showbox\undefined +\!let\!showboxbreadth\showboxbreadth \!let\showboxbreadth\undefined +\!let\!showboxdepth\showboxdepth \!let\showboxdepth\undefined +\!let\!showlists\showlists \!let\showlists\undefined +\!let\!showthe\showthe \!let\showthe\undefined +\!let\!skewchar\skewchar \!let\skewchar\undefined +\!let\!skip\skip \!let\skip\undefined +\!let\!skipdef\skipdef \!let\skipdef\undefined +\!let\!spacefactor\spacefactor \!let\spacefactor\undefined +\!let\!spaceskip\spaceskip \!let\spaceskip\undefined +\!let\!span\span \!let\span\undefined +\!let\!special\special \!let\special\undefined +\!let\!splitbotmark\splitbotmark \!let\splitbotmark\undefined +\!let\!splitfirstmark\splitfirstmark \!let\splitfirstmark\undefined +\!let\!splitmaxdepth\splitmaxdepth \!let\splitmaxdepth\undefined +\!let\!splittopskip\splittopskip \!let\splittopskip\undefined +\!let\!string\string \!let\string\undefined +\!let\!tabskip\tabskip \!let\tabskip\undefined +\!let\!textfont\textfont \!let\textfont\undefined +\!let\!textstyle\textstyle \!let\textstyle\undefined +\!let\!the\the \!let\the\undefined +\!let\!thickmuskip\thickmuskip \!let\thickmuskip\undefined +\!let\!thinmuskip\thinmuskip \!let\thinmuskip\undefined +\!let\!time\time \!let\time\undefined +\!let\!toks\toks \!let\toks\undefined +\!let\!toksdef\toksdef \!let\toksdef\undefined +\!let\!tolerance\tolerance \!let\tolerance\undefined +\!let\!topmark\topmark \!let\topmark\undefined +\!let\!topskip\topskip \!let\topskip\undefined +\!let\!tracingcommands\tracingcommands \!let\tracingcommands\undefined +\!let\!tracinglostchars\tracinglostchars \!let\tracinglostchars\undefined +\!let\!tracingmacros\tracingmacros \!let\tracingmacros\undefined +\!let\!tracingonline\tracingonline \!let\tracingonline\undefined +\!let\!tracingoutput\tracingoutput \!let\tracingoutput\undefined +\!let\!tracingpages\tracingpages \!let\tracingpages\undefined +\!let\!tracingparagraphs\tracingparagraphs \!let\tracingparagraphs\undefined +\!let\!tracingrestores\tracingrestores \!let\tracingrestores\undefined +\!let\!tracingstats\tracingstats \!let\tracingstats\undefined +\!let\!uccode\uccode \!let\uccode\undefined +\!let\!uchyph\uchyph \!let\uchyph\undefined +\!let\!underline\underline \!let\underline\undefined +\!let\!unhbox\unhbox \!let\unhbox\undefined +\!let\!unhcopy\unhcopy \!let\unhcopy\undefined +\!let\!unkern\unkern \!let\unkern\undefined +\!let\!unpenalty\unpenalty \!let\unpenalty\undefined +\!let\!unskip\unskip \!let\unskip\undefined +\!let\!unvbox\unvbox \!let\unvbox\undefined +\!let\!unvcopy\unvcopy \!let\unvcopy\undefined +\!let\!uppercase\uppercase \!let\uppercase\undefined +\!let\!vadjust\vadjust \!let\vadjust\undefined +\!let\!valign\valign \!let\valign\undefined +\!let\!vbadness\vbadness \!let\vbadness\undefined +\!let\!vbox\vbox \!let\vbox\undefined +\!let\!vcenter\vcenter \!let\vcenter\undefined +\!let\!vfil\vfil \!let\vfil\undefined +\!let\!vfill\vfill \!let\vfill\undefined +\!let\!vfilneg\vfilneg \!let\vfilneg\undefined +\!let\!vfuzz\vfuzz \!let\vfuzz\undefined +\!let\!voffset\voffset \!let\voffset\undefined +\!let\!vrule\vrule \!let\vrule\undefined +\!let\!vsize\vsize \!let\vsize\undefined +\!let\!vskip\vskip \!let\vskip\undefined +\!let\!vsplit\vsplit \!let\vsplit\undefined +\!let\!vss\vss \!let\vss\undefined +\!let\!vtop\vtop \!let\vtop\undefined +\!let\!wd\wd \!let\wd\undefined +\!let\!widowpenalty\widowpenalty \!let\widowpenalty\undefined +\!let\!write\write \!let\write\undefined +\!let\!xdef\xdef \!let\xdef\undefined +\!let\!xleaders\xleaders \!let\xleaders\undefined +\!let\!xspaceskip\xspaceskip \!let\xspaceskip\undefined +\!let\!year\year \!let\year\undefined +% Copyright (c) 2005 Jonathan Fine <jfine@pytex.org> +% License: GPL version 2 or (at your option) any later version. +% $Source: /cvsroot/pytex/pytex/tex2tok/tex2tok.tex,v $ + +% usage: '\!LEX{story} + +\!catcode`|=0 % for document control sequences +\!catcode`{=1 +\!catcode`}=2 +\!catcode`$=3 +\!catcode`&=4 +\!catcode`#=6 +\!catcode`^=7 +\!catcode`_=8 +\!catcode`@=11 + +% useful constants +\!let\!bgroup { +\!let\!egroup } +\!chardef\!zero 0 +\!chardef\!one 1 + +\!def\?{\!let\!space = } \? % define \!space +\!let\? \!undefined % clean up afterwards + +\!chardef\!tokfile 16 % change if you wish + + +\!def\!loop +% loop through sequence of tokens +% assign to \!value the meaning of the next token +{% + \!futurelet\!value\!loop@A +} + + +\!def\!loop@continue +% pick up the next token, and continue the loop +% required, for dealing with space and brace characters +{% + \!afterassignment\!loop + \!let\!@temp = +} + + +\!def\!loop@A +% is the next token unexpandable? +% always, unexpandable tokens are characters +% always, control sequences are expandable (or undefined) +% sometimes, a character may be expandable +{% + \!expandafter\!ifx\!noexpand\!value\!value + \!expandafter\!loop@char + \!else + \!expandafter\!loop@macro + \!fi +} + + +\!def\!loop@macro +% suppress implicit \!outer token at end of file +{\!expandafter\!loop@macro@A\!noexpand} + +\!long\!def\!loop@macro@A #1% +% #1 - expandable token, picked up from input stream +% execute the token, and resume the loop +% if the token is undefined, call +{ + \!ifx #1\!undefined + \!process@undefined #1% + \!expandafter\!loop + \!else + \!expandafter #1% + \!fi +} + + +\!long\!def\!process@undefined #1% +% #1 - undefined control sequence +% default value - can be redefined +{% + \!immediate\!write\!tokfile{\!string #1}% +} + + +\!def\!endloop +% when read by \!loop, this ends its execution +{} + + +\!def\!loop@char +% process next token, which is a character +% might be letter, other, brace or whatever +% afterwards, continue with the loop +{% + % some tricky code to deal with common cases at high speed + \!ifcase + % first deal with the common cases, at high speed + \!ifcat a\!value \!one\!fi % it's a letter + \!ifcat ?\!value \!one\!fi % it's punctuation, digit, etc. + \!zero + \!expandafter\!loop@char@A % it's something else + \!or % case of letter or digit + \!expandafter\!loop@char@default + \!fi +} + + +\!def\!loop@char@default #1% +% #1 - a character we can pick up in this way +{% + \!immediate\!write\!tokfile{#1}% + \!loop +} + + +\!def\!loop@char@A +% next token is a special character +% pick it up, and process it +% we already have the \ +{% + \!ifx\!value + \!ifx\!value\!@space + \!immediate\!write16{ \!@percent} + \!else + \!immediate\!write16{% + \!expandafter\!@char@strip\!meaning\!value + \!@percent + }% + \!fi +} + + +\!def\!empty{} +\!def\!loop@char@A +{% + \!edef\!temp + {% + \!ifx\!value\!space + \!if11 \!fi + \!else\!ifx\!value\!bgroup + +\!string{% + \!else\!ifx\!value\!egroup + -\!string}% + \!fi\!fi\!fi + } + \!ifx\!temp\!empty + \!expandafter\!loop@char@default + \!else + \!immediate\!write\!tokfile{\!temp}% + \!expandafter\!loop@continue + \!fi +} + +\!def |LEX #1% +{% + \!begingroup + \!let|par\!undefined % blank lines in input document file + \!let\!LEX\!undefined % in case it appears in document file + \!catcode`!=12 % restore to normal value + \!catcode`|=12 % restore to normal value + \!catcode`@=12 % restore to normal value + \!chardef\!tokfile 15 + \!def\!next{\!immediate\!openout\!tokfile \!jobname.tok }% + \!expandafter\!next + \!expandafter\!loop + \!input #1 % space to terminate file name + \!endloop + \!immediate\!closeout\!tokfile + \!endgroup + \!end +} + +\!dump + |