% Copyright 2012-2020, Alexander Shibakov % This file is part of SPLinT % % SPLinT is free software: you can redistribute it and/or modify % it under the terms of the GNU General Public License as published by % the Free Software Foundation, either version 3 of the License, or % (at your option) any later version. % % SPLinT is distributed in the hope that it will be useful, % but WITHOUT ANY WARRANTY; without even the implied warranty of % MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the % GNU General Public License for more details. % % You should have received a copy of the GNU General Public License % along with SPLinT. If not, see . %\def\mkpurebyte{\uccode`\@=\yycp@\uppercase{\yybytepure{@}}\uccode`\@=`\@} % make all symbol characters category 12; most macros are indifferent to this % however, if delimited macros are used to process the mathched text, the option % makes it easier to write such macros; \def\mkpurebyte{\uccode`\.=\yycp@\uppercase{\yybytepure{.}}\uccode`\.=`\.} \def\yyinput{\futurelet\next\yyinp@t} % get the code of the next character ... \def\yyinp@t{% test the code and decide whether to continue lexing the % token or return to the parser \ifcat\noexpand\next\bgroup \yybreak\yyinputgroup \else \if\noexpand\next\space % code 32 character token is assumed to % be an ordinary space (while category 10 tokens are not necessarily); % the reason for this choice is the way code 32 characters are treated % by \string (turned into `real' spaces); `funny' spaces of both kinds % (a character that differs from a `real' space (\catcode`\ ==10 % \number`\ ==32) in either category or character code) can be % created, however, category 10 characters of character code other % than 32 are very rare (and take some effort to produce), while % category 12, character code 32 tokens are poduced as a result of % `sanitizing' input all the time; note that the spaces are processed % at the lowest level of the input routine and thus cannot be % substituted by the switches below; this choice is not bullet proof % but should suffice for most uses; in the case of more polluted % input, an extra `sanitation' step can be performed first; if % desired, category 10 charactes can be assumed to be spaces at this % point, as well, although this will introduce an extra test, affecting % the (already poor) efficiency of the input macros. \yycp@=`\ % \yybyte={ }% \mkpurebyte \yybreak@\yyskipspace % return the space \else \if\noexpand\next\eolletter % cannot use ^^M here since TeX will simply drop the rest of the line \yycp@=\n \yybyte={\n}% \mkpurebyte \yybreak@@\yyskipspace % return the end of line character \else \yybreak@@\yy@np@t \fi \fi \yycontinue } % some \Cee\ escape characters; the rest are either silly (like \a and \b) or % are already defined to have other important functions in \TeX\ (such as \v and \t) \chardef\n=`\^^J \chardef\r=`\^^M \chardef\f=`\^^L \chardef\HT=`\^^I % ASCII horizontal tab \chardef\charseq1 \chardef\charac2 \chardef\chargroup3 \def\yyskipspace{\afterassignment\yyreturn\let\next= } \def\yyinputgroup#1{% \yycp@=\chargroup \yybyte={{#1}}% \mkpurebyte \yyreturn } \newif\ifyyinputdebug \def\yy@np@t#1{% start lexing: % whitespace (category 10) tokens are skipped % automatically \yybyte{#1}% \ifyyinputdebug \immediate\write16{read: \the\yybyte\space after: \the\yytext@seen}% \fi \ifx#1\ % a space token \yybreak\returnexplicitspace \else \expandafter\ischar\string#100\end \ifchar % a single character (not a control sequence) \ifcat\noexpand#1\noexpand~% is it an active character? \yycp@=\charac % yes, return it \mkpurebyte \yybreak@@\yyreturn \else % it is a non-active character ... % ... or a control sequence with an empty name (obtained by \csname\endcsname or % \enlinechar=-1 \toks0={\ % } this case can be handled by the switch below but is ignored for the moment) \let\default\achardefault \yybreak@@{\switchon{\the\yybyte}\in\acharswitch}% \fi \else % it is a control sequence, return it \expandafter\ischar\string#10\end \ifchar % it is a one-char control sequence \let\default\onechardefault \yybreak@@{\switchon{\the\yybyte}\in\onecharswitch}% \else \let\default\multichardefault \yybreak@@{\switchon{\the\yybyte}\in\multicharswitch}% \fi \fi \yycontinue } \chardef\explicitspacecode=`\ % \def\returnexplicitspace{% \yycp@=\explicitspacecode \mkpurebyte \yyreturn % keep looking for a non-whitespace token } \def\onecharswitch{ \raw \n \raw {% \yycp@=\n \mkpurebyte \yyreturn } \raw \^^M\raw {% \returnexplicitspace } } \def\onechardefault{% \expandafter\yycp@\expandafter`\the\yybyte\relax \mkpurebyte \yyreturn }% \def\multicharswitch{ \raw\vb\raw {% \vbunwrap } \raw\insertraw\raw {% \insertrawnext } \raw\stashed\raw {% \stashnext } \raw\format \formatlocal\raw {% \formatnext } \raw\formatbegin\raw {% \fmtbegin } \raw\formatp\raw {% \fmtparam } \raw\sflush\raw {% \sflushnext } \raw\yyeof\raw {% \yycp@=\YYENDOFBUFFERCHAR\relax \yybytepure={}% \yyreturn } \raw\inputboundary\raw {% \inputboundarynext } \raw\flatten \resetf \inline \skipheader\raw {% \expandafter\yyinput\expandafter\format\expandafter{\the\yybyte}% } \raw\fold \breakline\raw {% \expandafter\yyinput\expandafter\formatlocal\expandafter{\the\yybyte}% } \raw\breakahead\raw {% \expandafter\yyinput\expandafter\formatp\the\yybyte } \raw\break\raw {% for testing purposes \yycp@=`\ % \yybytepure={ }% \yyreturn } \raw\squashtermstrue\raw {% \yycp@=`\ % \yybytepure={ }% \squashtermstrue \yyreturn } \raw\endparse \endparseinput\raw {% \errmessage{internal error: reading past the end of the input buffer}% } } \def\multichardefault{% \yycp@\charseq \mkpurebyte \yyreturn }% \def\achardefault{% \expandafter\yycp@\expandafter`\the\yybyte\relax \mkpurebyte \yyreturn } \def\acharswitch{} % the following commands are only here for debugging purposes % they slow down the input \setspecialcharsfrom\onecharswitch \setspecialcharsfrom\multicharswitch \setspecialcharsfrom\acharswitch \def\insertrawnext#1{% insert a command #1\yyinput } \def\vbunwrap#1#2\vb{% \yyinput#1\stashed{#2}\vb } \chardef\stashchar=`\ % \chardef\formatchar=`\ % \newcount\stashmarker \newcount\formatmarker \def\stashnextwithspace#1{% \yybytepure{ }\yycp@\stashchar \yybyte\expandafter{\the\yybyte{#1}}% \advance\stashmarker\@ne \edef\next{\yysbyte{\noexpand\strm[\the\stashmarker]{\the\yybyte}.[\the\stashmarker]}}\next % the extra `.' is to avoid the stripping of braces by the parameter scanning mechanism of TeX \yyreturn } % the mechanism for stash processing making stash invisible \def\stashnextwithnothing#1{% \advance\stashmarker\@ne \yybyte\expandafter{\the\yybyte{#1}}\concat\yysubtext\yybyte \appendr\yysbyte{\noexpand\strm[\the\stashmarker]{\the\yybyte}.[\the\stashmarker]}% \ifyyinputdebug \immediate\write16{stash byte: \the\yysbyte mid text: \the\yysubtext}% \fi % the extra `.' is to avoid the stripping of braces by the parameter scanning mechanism of TeX \yyinput } % while collecting the stash, the macro below packages the stash contents into a command sequence % to add to the \yystash stream; the stash is presented to the lexer unpackaged though; this % introduces a minor inefficiency while ensuring that exactly one level of packaging is present % (otherwise the lexer may back up and repackage already packaged stash); the inefficiency only % persists while the lexer scans the current token (and possibly backs up). \def\stashnextwithnothingnx#1{% \advance\stashmarker\@ne \yypush{#1}\on\astarray \appendr\yysbyte{\noexpand\strm[\the\stashmarker]{\the\yybyte{% \expandafter\noexpand\romannumeral\gettopofstackcsx\astarray}}.[\the\stashmarker]}% \yybyte\expandafter{\the\yybyte{#1}}% do not package the input \concat\yysubtext\yybyte \ifyyinputdebug \immediate\write16{stash byte: \the\yysbyte mid text: \the\yysubtext}% \fi % the extra `.' is to avoid the stripping of braces by the parameter scanning mechanism of TeX \yyinput } \let\stashnext\stashnextwithnothingnx \def\formatnext#1{% \yybytepure{ }\yycp@\formatchar \toksa{#1}% \advance\formatmarker\@ne \edef\next{\yyfbyte{\noexpand\strm[\the\formatmarker]{\the\yybyte{\the\toksa}}.[\the\formatmarker]}}\next % the extra `.' is to avoid the stripping of braces by the parameter scanning mechanism of TeX \yybyte\expandafter{\the\yybyte{#1}}% \yyreturn } \def\fmtbegin#1\fmtend{\formatnext{#1}} % multiparameter format sequences \def\fmtparam#1#2{\formatnext{#1{#2}}} % single parameter format sequences \chardef\boundarychar=`\ % % the following is a minimal setup of a parsing boundary \def\inputboundarynext#1{% `l' for left boundary \yybytepure{ }\yycp@\boundarychar \yybyte\expandafter{\the\yybyte{#1}}% \yyreturn % inserting something here will effectively insert it into the input stream } \def\sflushnext#1#2{% #1 is the marker % #2 is the contents \yybytepure{ }\yycp@\stashchar \edef\next{\toksc{\the\yystash\the\yystashseen}}\next \yyfifolastidx\toksc\in\toksc \toksa{#1}\toksb{#2}% \appendr\yysbyte{\noexpand\strm[\the\toksa]{\the\yybyte{{\nx\it flushing}: ``\the\toksb'', {\nx\it last index}: $[\the\toksc]$. }}.[\the\toksa]}% % the extra `.' is to avoid the stripping of braces by the parameter scanning mechanism of TeX \yybyte\expandafter{\the\yybyte{#1}{#2}}% \concat\yysubtext\yybyte \ifyyinputdebug \immediate\write16{stash byte: \the\yysbyte mid text: \the\yysubtext}% \fi \yyreturn } % the following implementation is marginally cleaner as it does not redefine \yyr@@dfifo % it also makes it almost transparent that \yyreadfifo does not modify any token registers % except for the two of its parameters % TODO: replace \yyr@@dfifo with \yyr@adfifo \def\yyreadfifo#1\to#2\in#3{% %{% to reduce the side effects to the redefinition of the input and output % token registers \def\yyr@adfifo##1\strm[#2]##2[#2]##3\end{% \yystringempty{##3}{#3{}}% there is no such marker in the fifo {% #3{##1\strm[#2]##2[#2]}% \def\yyr@@dfifo####1\strm[#2]####2[#2]\end{#1{####1}}% strip off the inserted string \yyr@@dfifo##3\end % TODO: \def\yyr@dfifo####1\strm[#2].[#2]\end{#1{####1}}% strip off the inserted string % \yyr@dfifo##3\end }% }% \expandafter\yyr@adfifo\the#1\strm[#2].[#2]\end %\edef\next{#1{\the#1}#3{\the#3}}\expandafter}\next } \def\yytrimfifo#1\to#2\in#3{% just save the initial segment of #1 in #3 \def\yyr@adfifo##1\strm[#2]##2[#2]##3\end{% \yystringempty{##3}{#3{}}% there is no such marker in the fifo {% #3{##1\strm[#2]##2[#2]}% }% }% \expandafter\yyr@adfifo\the#1\strm[#2].[#2]\end } \def\yyreadfifox#1\to#2\in#3{% read the fifo to a marker in #2 \expandafter\yyreadfifo\expandafter#1\expandafter\to\number#2\in#3% } \def\yyreadfifoxx#1\to#2\in#3{% read the fifo to a marker in #2 \expandafter\yyreadfifo\expandafter#1\expandafter\to#2\in#3% } \def\yyreadfifot#1\to#2\in#3\with#4{% read fifo with a test: \def\yyr@adfifo##1\strm[#2]##2[#2]##3\end{% #4 is a \macro[##1]##2.[##3] which \def\yyr@@dfifo{##3}% looks at its parameters and fills up \ifx\yyr@@dfifo\empty % \othertoks (to be attached at the beginning of #1 % after the initial segment has been removed) % and \fifotoks (to be used by another macro) #3{}% there is no such marker in the fifo \else \let\fifotoks#3% \let\othertoks#1 \let\@@strm\strm \let\strm#4% \fifotoks{}% \othertoks{}% ##1\strm[#2]##2[#2]% run all the tests \let\strm\@@strm \def\yyr@@dfifo####1\strm[#2]####2[#2]\end{#1\expandafter{\the\othertoks####1}}% \yyr@@dfifo##3\end \fi }% \expandafter\yyr@adfifo\the#1\strm[#2].[#2]\end } \def\yyreadfifotx#1\to#2\in#3\with#4{% read the fifo to a marker in #2 \expandafter\yyreadfifot\expandafter#1\expandafter\to\number#2\in#3\with{#4}% } % an example of a sorting macro: extract all the local formatting \def\formatsort[#1]#2.[#3]{% clobbers \toksb, \toksc \formatgrab#2\end \toksc{\strm[#1]{#2}.[#3]}% \let\default\formatdefaultact \switchon{\the\toksb}\in\formatswitch } \def\formatgrab#1#2\end{\toksb{#1}} \def\formatswitch{% \formatlocal {% \edef\next{\fifotoks{\the\fifotoks\the\toksc}}\next } } \def\formatdefaultact{% \edef\next{\othertoks{\the\othertoks\the\toksc}}\next } \def\yygrabfifoelem#1\at#2\in#3{% \def\yyr@adfifo##1\strm[#2]##2[#2]##3\end{% \def\yyr@@dfifo{##3}% \ifx\yyr@@dfifo\empty #3{}% there is no such marker in the fifo \else #3{\strm[#2]##2[#2]}% \def\yyr@@dfifo####1\strm[#2]####2[#2]\end{#1{####1}}% strip off the inserted string \yyr@@dfifo##1##3\end \fi }% \expandafter\yyr@adfifo\the#1\strm[#2].[#2]\end } \def\yygrabfifoelemx#1\at#2\in#3\with#4{% read the fifo to a marker in #2 \expandafter\yygrabfifoelem\expandafter#1\expandafter\at\number#2\in#3\with{#4}% } \def\yypeekfifoelem#1\at#2\in#3{% just read the first element that matches the marker \def\yyr@adfifo##1\strm[#2]##2[#2]##3\end{% \def\yyr@@dfifo{##3}% \ifx\yyr@@dfifo\empty #3{}% there is no such marker in the fifo \else #3{##2}% just get the contents \fi }% \expandafter\yyr@adfifo\the#1\strm[#2].[#2]\end } \def\yypeekfifoelemx#1\at#2\in#3\with#4{% read the fifo to a marker in #2 \expandafter\yypeekfifoelem\expandafter#1\expandafter\at\number#2\in#3\with{#4}% } \def\strmgetfirstidx[#1]#2.[#3]{\toksa{#1}} \def\yyfifolastidx#1\in#2{% \let\next\strm \let\strm\strmgetfirstidx \toksa{}\the#1% #2\toksa \let\strm\next } \def\z@rotest{0} \newif\ifchar \def\ischar#1#2#3\end{% three parameters because #1 can be an % \escapechar \def\lastnamechar{#3}% \ifx\lastnamechar\z@rotest\chartrue\else\charfalse\fi } % trivial input routine \def\yyinputtrivial{\futurelet\next\yyinp@ttrivial} % get the code of the next character ... \def\yyinp@ttrivial{ \ifcat\noexpand\next\space % category 10 token \yycp@=`\ % \yybyte={ }% \mkpurebyte \yybreak\yyskipspace % return the space \else \yybreak\yy@np@ttrivial \yycontinue } \def\yy@np@ttrivial#1{% \ifcat\noexpand#1a% \yycp@`#1% \yybyte{#1}% \yybytepure{#1}% \else \if\noexpand#1\eolletter \yycp@=\n \yybyte\expandafter{\eolletter}% \mkpurebyte \else \ifx#1\yyeof \yycp@=\YYENDOFBUFFERCHAR\relax \yybyte{#1}% \yybytepure={}% \else \yycp@\charseq \mkpurebyte \fi \fi \fi \yyreturn }