% Copyright 2012-2015, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
% it under the terms of the GNU General Public License as published by
% the Free Software Foundation, either version 3 of the License, or
% (at your option) any later version.
%
% SPLinT is distributed in the hope that it will be useful,
% but WITHOUT ANY WARRANTY; without even the implied warranty of
% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
% GNU General Public License for more details.
%
% You should have received a copy of the GNU General Public License
% along with SPLinT.  If not, see <http://www.gnu.org/licenses/>.

% prototypes of all the macros produced by the parser
% we will follow the convention: 12string is a string of category 12 characters and spaces, tex_string: a string
% of TeX tokens; sptr is a pointer to the stash stream, fptr is a pointer to the format stream

\def\yyuniontag{\yyunion}
\def\parserstrictnamespace{parser-strict}
\def\parserprototypesnamespace{parser-strict:headers}

\def\yyunion{\currentyyunionnamespace}
\def\currentyyunionnamespace{generic}

% types inserted by the input routines: these should not be tampered with randomly
% therefore they are not included in the yyunion list and only defined once below and redefined later
% permanently

% insidemost

\long\def\stashed#1{} % stashed material (usually \Cee\ code) :: \stashed{tex_string}
\long\def\sflush#1#2{} % stash flushing marker :: \sflush{tex_string}
\def\format#1{} % format material triggered by braces and the start of production :: \format{tex_string}
\def\formatlocal#1{} % format material triggered by every term :: \formatlocal{tex_string}

% level 1

\long\def\strm[#1]#2.[#3]{} % a stream element :: \strm[digits]{\stashed|\format|\formatlocal}.[digits]

% types returned by the lexer (* marks the types that get removed by the parser in some cases)

\def\midf#1#2{} % rule separator :: \midf{fptr}{sptr}
\defp\stringify#1#2#3#4{} % STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\defp\idit#1#2#3#4{} % ID :: \idit{12string}{tex_string}{fptr}{sptr}
\defp\tagit#1#2#3#4{} % TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\defp\charit#1#2#3#4{} % CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\defp\anint#1#2#3{} % INT :: \anint{digits}{fptr}{sptr}
\defp\hexint#1#2#3{} % INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
\defp\braceit#1#2#3{} % *BRACED_CODE :: \braceit{tex_string}{fptr}{sptr}
\defp\preckind#1#2#3{} % *precedence operator :: \preckind{12string}{fptr}{sptr}
\defp\flexoptionpair#1#2{} % a flex option :: \flexoptoinpair{\idit}{\idit|\stringify}

% types inserted by the parser

% insidemost

\defp\termname#1#2{} % a production term :: \termname{\idit|\stringify|\charit}{\idit}
\defp\mergeop#1#2#3{} % merge directive :: \mergeop{\tagit}{fptr}{sptr}
\defp\dprecop#1#2#3{} % dprec directive :: \dprecop{\anint|\hexint}{fptr}{sptr}
\defp\sprecop#1#2#3{} % prec directive :: \sprecop{\idit|\stringify|\charit}{fptr}{sptr}
\defp\rrhssep#1#2{} %  :: \rrhssep{fptr}{sptr}
\defp\arhssep#1#2{} %  :: \arhssep{fptr}{sptr}
\defp\rarhssep#1#2{} %  :: \rarhssep{fptr}{sptr}
\defp\hspace#1#2{} % a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
\defp\emptyterm{}
\defp\onesymbol#1#2#3{} % a symbol definition :: \onesymbol{\idit}{\anint|\hexint}{\stringify}
\defp\bracedvalue#1#2#3{} % a braced variable value :: \bracedvalue{tex_string}{fptr}{sptr}

%level 1

\defp\rhs#1#2#3{} % a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
\defp\actbraces#1#2#3#4#5\bdend{} % action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
\defp\bpredicate#1#2#3#4#5\bdend{} % predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend

% level 2

\defp\prodheader#1#2#3#4{} % a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
\defp\rules#1#2#3{} % a complete right-hand side :: \rules{[\rhs|\rrhssep]}{max fptr}{max sptr}
\defp\symbolprec#1#2{} % symbol precedence :: \symbolprec{\idit|\stringify|\charit}{\anint|\hexint}
%\def\pcluster#1#2{} % a production cluster :: \pcluster{\prodheader}{\termname|...} should never appear in the final output

% level 3

\defp\oneproduction#1#2#3{} % a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}
\defp\prodprodsepsmall{} % production separator
\defp\prodprodsep{} % production separator
\defp\prodprodseplarge{} % production separator
\defp\optoptsepsmall{} % separator between options
\defp\posmark#1#2{} % the position marker :: \posmark{fptr}{sptr}
\defp\tokendecls#1#2#3{} % token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
\defp\ntermdecls#1#2#3{} % nterm declarations :: \ntermdecls{[\onesymbol]+}{fptr}{sptr}
\defp\typedecls#1#2#3#4{} % type declarations :: \typedecls{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\precdecls#1#2#3#4#5{} % precedence declarations :: \precdecls{left|right|nonassoc|precedence}{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\flexsstatedecls#1#2#3{} % flex state declarations (nonexlusive) :: \flexsstatedecls{[\idit\stringify]+}{fptr}{sptr}
\defp\flexxstatedecls#1#2#3{} % flex state declarations (exclusive) :: \flexxstatedecls{[\idit\stringify]+}{fptr}{sptr}
\defp\flexoptiondecls#1#2#3{} % flex state declarations :: \flexoptiondecls{[\flexoptionpair]+}{fptr}{sptr}
\defp\oneparametricoption#1#2#3#4{} % option :: \oneparametricoption{option-name}{\stringify|\idit}{fptr}{sptr}
\defp\optionflag#1#2#3#4{} % option flag :: \optionflag{flag name}{abbreviated name}{fptr}{sptr}
\defp\paramdef#1#2#3#4{} % parameters :: \paramdef{\braceit}{parameter domain}{fptr}{sptr}
\defp\codepropstype#1#2#3#4#5#6#7{} % term method :: \codepropstype{printer|destructor}{tex_string}{\stringit|\idit}{fptr}{sptr}{fptr}{sptr}
\defp\initaction#1#2#3{} % initial action :: \initaction{tex_string}{fptr}{sptr}
\defp\codeassoc#1#2#3#4#5#6#7{} % code association :: \codeassoc{code kind}{|\idit}{tex_string}{fptr}{sptr}{fptr}{sptr}
\defp\vardef#1#2#3#4{} % variable definition :: \vardef{\idit|\stringify}{|\idit|\stringify|\bracedvalue}{fptr}{sptr}
\defp\prologuecode#1#2#3#4#5{} % prologue code :: \prologuecode{tex_string}{fptr}{sptr}{fptr}{sptr}

% level 4

%\def\grammar#1#2{} % a grammar: should never appear in the final output

\savecslist{parser-prototypes}\yyunion

% table construction: the macros run to build the contents of the table of productions to be
% typeset.

\newtoks\table
\newtoks\tokdectoks
\newtoks\opttable
\newtoks\typestable
\newtoks\prectable

\newtoks\tokstoks
\newtoks\typesset

\let\rhscont\getsecond
\let\rhscnct\getthird
\let\rhsbool\getfourth

\newif\ifrhsfull

\newtoks\yystashlocal
\newtoks\yyformatlocal

\newif\iftermindex

\def\doparse#1{%
    \table{}\typestable{}\prectable{}\opttable{}%
    \parserinit\yyparse#1\yyeof\yyeof\endparseinput\endparse
}

\long\def\strm[#1]#2.[#3]{#2}
\long\def\inmath#1{$#1$}

\long\def\stashed#1{\toksa\expandafter{\the\toksa#1}}
\long\def\sflush#1#2{}

\def\ntt{\noexpand\ntt@} % a convenient shortcut
\def\ntt@{\tt\let\_\UL}

\def\prodheader#1#2#3#4{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}\the\yyformatlocal
    \let\termindex\writeidxhentry
    \prodhead@r{#1}{#2}%
    \let\termindex\eatone
}

\def\headeridxrank{1} % index rank of lhs

\def\writeidxhentry#1{\write\gindex{%
    \nx\nx
    \expandafter
        \nx
    \csname\bisonidxdomain TI\endcsname\headeridxrank#1}%
}

\def\prodhead@r#1#2{%
    \let\idit\prodhdridit
    #1\relax\concat\table\toksa\hrhss@p
}

\def\hrhss@p{\toksa{&\omit\hfil&\omit\hfil&\omit\hfil\cr\noalign{\penalty100}\omit\hfil&\termspostformat}\concat\table\toksa}

\def\rrhssep#1#2{%
    \yyreadfifot\yyformat\to#1\in\yyformatlocal\with\formatsort\relax
    \the\yyformatlocal % this will have a chance to change the definition of \rrhss@p
    \rrhss@p
}

\def\rrhss@p{\toksa{\omit\hfil&}\concat\table\toksa}

\def\arhssep#1#2{%
    \yyreadfifo\yyformat\to#1\in\yyformatlocal
    \the\yyformatlocal % this will have a chance to change the definition of \arhss@p
    \arhss@p
}

%% between the inline action and next term:

\def\arhss@p{\toksa{\omit\hfil&\hbox to 2em{\hfil}}\concat\table\toksa}

\def\rarhssep#1#2{%
    \yyreadfifo\yyformat\to#1\in\yyformatlocal
    \the\yyformatlocal % this will have a chance to change the definition of \rarhss@p
    \rarhss@p
}

%% between a term and the following action

\def\rarhss@p{\appendr\table&}

\newtoks\hstoks

\def\hspace#1#2{%
    \hstoks\yyformat
    \yyreadfifot\yyformat\to#1\in\yyformatlocal\with\formatsort\relax
    \the\yyformatlocal % may change \hspac@
    \hspac@
}

\def\hspac@{%
    \appendr\table{ }%
}

\def\rhs#1#2#3{#1}

\def\supplybdirective#1#2{%
    \edef\next{\nx\s@pplybdirective#1{\the#2}\the#1\nx\bdend}\next
}

\def\s@pplybdirective#1#2#3\bdend\bdend{%
    #1{#3#2\bdend}%
}

\def\termname#1#2{%
    \let\idit\termnameidit
    \let\stringify\termnamestringify
    \let\optstrextra\optstrextraesc
    \let\termindex\writeidxentry
    #1\relax\concat\table\toksa
    \let\termindex\eatone
}

\def\bisonidxdomain{B} % marker fo the domain (section of the index)
\def\termidxrank{2} % rank (affects the order in which the entries are
                    % listed in the index and the listing style)

\def\writeidxentry#1{\write\gindex{%
    \nx\nx
    \expandafter
    \nx
        \csname\bisonidxdomain TI\endcsname\termidxrank#1}%
}

\let\termindex\eatone

% index implementation
% note that the rank and domain parameters are handed down by the
% `outer' macro while the `qualifier' (\term...string) is decided
% at the point of invocation

\def\gidxentry#1#2{% a generic index entry 
    \iftermindex
        \termindex{%
            {\nx\nx\nx#1{\charstonumberse#2\end}}%
            {\hostparsernamespace}{\nx\the\pageno}{\secno}%
        }%
    \fi
}

\def\tidxentry#1#2{% a text index entry 
    \iftermindex
        \write\gindex{%
            \nx\nx\expandafter\nx\csname\bisonidxdomain TI\endcsname\termidxrank
            {\nx\nx\nx#1{\charstonumberse#2\end}}%
            {\indexpseudonamespace}{\nx\the\pageno}{\secno}%
        }%
    \fi
}

%

\newtoks\bdirects

\def\actbraces#1#2#3#4#5\bdend{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    #5% accumulate all the directives
    \edef\next{\the\bdirects}%
    \ifx\next\empty
    \else
        \edef\next{\bdirects{%
             $\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr\the\bdirects}}$}}\next
    \fi
    \the\yyformatlocal
    \actbrac@s{#1}{#2}{#4}%
}

\def\actbrac@s#1#2#3{%
    \yyreadfifo\yystash\to#3\in\yystashlocal
    \unmathstash{#3}%
    \edef\next{\toksa{\the\bdirects&\the\yystashlocal\cr}}\next
    \bdirects{}%
    \concat\table\toksa
}

\def\unmathstash#1{%
    \setbox\z@=\vbox{
        \hsize\maxdimen\emergencystretch\maxdimen
        \rightskip\z@ plus 1fill
        \let\TeXx\relax\let\stashed\copystashed
        \the\yystashlocal\relax
        \ifmmode
            {}$\aftergroup\unm@thstash % $o that the box can be finished
        \else
            \aftergroup\eatone
        \fi}{#1}%
}

\def\copystashed#1{#1}

\def\unm@thstash#1{%
    \toksa{\strm[NaN]{\stashed{{}$}}.[NaN]}% $o it cannot be matched if \yystashlocal is read again
    \concat\yystashlocal\toksa
    \toksa{\strm[#1]{\stashed{${}}}.[#1]}%$
    \appendl\yystash{\the\toksa}%
}

\def\bpredicate#1#2#3#4#5\bdend{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    #5% accumulate all the directives
    \edef\next{\the\bdirects}%
    \ifx\next\empty
        \bdirects{\quad$\dashv\ \nx\vdash$}%
    \else
        \edef\next{\bdirects{%
             $\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr$\dashv\ $&\omit\nx\hfil$\nx\vdash$\cr
                                                                \the\bdirects}}$}}\next
    \fi
    \the\yyformatlocal
    \bpredicat@{#1}{#2}{#4}%
}

\def\bpredicat@#1#2#3{%
    \yyreadfifo\yystash\to#3\in\yystashlocal
    \unmathstash{#3}%
    \edef\next{\toksa{\the\bdirects&\the\yystashlocal\cr}}\next
    \bdirects{}%
    \concat\table\toksa
}

\def\bdirective#1#2#3#4{%
    \savecslist{local-namespace}\yyunion
    \restorecslist{table-render:directives}\yyunion
        \toksa{}%
        #2% make a directive list
    \edef\next{\toksa{$\nx\langle${\ntt #1} &\the\toksa$\nx\rangle$\cr}}\next
    \concat\bdirects\toksa
    \restorecslist{local-namespace}\yyunion
}

\def\mergeop#1#2#3{%
    \bdirective{merge}{#1}{#2}{#3}%
}

\def\dprecop#1#2#3{%
    \bdirective{dprec}{#1}{#2}{#3}%
}

\def\sprecop#1#2#3{%
    \bdirective{prec}{#1}{#2}{#3}%
}

\def\rules#1#2#3{#1}

\def\oneproduction#1#2#3{#1\setrulecontext{}\stopproduction}

% to make it possible to typeset tokens depending on their context
% the context name is passed to the token typeseting macros; the most
% common context name is the left hand side of a production; special
% context names (such as *left, *right, etc) are defined for terms that are
% not part of a rule; all such context names start with `*' since this
% character cannot be part of a term name

\def\setrulecontext#1{\def\currentrulecontext{#1}}

\let\currentrulecontext\empty

\def\emptyterm{%
    \edef\next{\toksa{$\nx\circ$\tidxentry{\termidstring}{emptyrhs}}}\next
    \concat\table\toksa
}

\def\termnameidit#1#2#3#4{%
    \yyreadfifot\yyformat\to#3\in\yyformatlocal\with\formatsort\relax
    \the\yyformatlocal
    \nameproc{#2}%
    \peekstash{#4}%
    \settermstyle{#1}%
 }

\def\settermstyle#1{%
    \edef\next{\toksa{\termmetastyle{%
            \gidxentry{\termidstring}{#1}%
            \let\nx\idxfont\nx\empty\nx\it\the\toksa\nx\/%
        }}}\next
}

\def\idxinline#1{% index the term in text
    \edef\next{\tidxentry{\termidstring}{#1}}\next
}

\def\termnamestringify#1#2#3#4{%
    \nameproc{#2}%
    \edef\next{\toksa{{%
            \gidxentry{\termstring}{#1}%
            \ntt\the\toksa
        }}}\next
}

\def\termmetastyle#1{{#1}}

\let\peekstash\eatone

\newif\ifinheader

\def\prodhdridit#1#2#3#4{%
    \setrulecontext{}%
    \nameproc{#2}%
    \expandafter\setrulecontext\expandafter{\the\toksc}%
    \edef\next{\toksa{{%
            \gidxentry{\termhdrstring}{#1}%
            \nx\inheadertrue\let\nx\idxfont\nx\bf\nx\itbold\the\toksa\nx\/%
        }}}\next
}

\def\charit#1#2#3#4{%
    \expandafter\ifx\csname'parser'\hostparsernamespace#1\endcsname\relax
        \edef\next{\toksa{{\ntt #1}\gidxentry{\termvstring}{'#1'}}}%
    \else
        \edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
            \csname'parser'\hostparsernamespace#1\endcsname{\currentrulecontext}\gidxentry{\termvstring}{'#1'}}}%
    \fi
    \next
}

\def\anint#1#2#3{%
    \toksa{#1}\concat\table\toksa
}

\def\hexint#1#2#3{%
     \toksa\expandafter{\eattwo#1$_{16}$}\concat\table\toksa
}

\def\optoptsepnone{%
    \concat\opttable\postoks
}

\def\optoptsepsmall{% keep the alignment going but insert a small space
    \concat\opttable\postoks
    \toksa{\noalign{\smallskip}}\concat\opttable\toksa
}

\def\prodprodsepsmall{%
    \pushothertables
    \toksa{\noalign{\penalty-150 \smallskip}}\concat\table\toksa
}

\def\prodprodsep{%
    \pushothertables
    \toksa{\noalign{\medskip}}\concat\table\toksa
}

\def\prodprodseplarge{%
    \pushothertables
    \toksa{\noalign{\bigskip}}\concat\table\toksa
}

\def\separatorswitcheq{%
    \oneproduction {%
        \toksd{\prodprodsepsmall}%
    }
    \codepropstype\codeassoc {%
        \toksd{\optoptsepsmall}%
    }
}

\def\separatorswitchdefaulteq{%
    \toksd{\optoptsepnone}%
}

% a switch for inserting position information into the parsed table
% clobbers \toksd

\newtoks\postoks

\def\positionswitch{
    \empty
    {%
        \postoks{}% semicolons should be invisible
    }
    \paramdef\oneparametricoption\optionflag\typedecls\vardef
    {%
        \getfourth{\yy(2)}\to\postoks % the format marker
        \getfifth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \prologuecode\ntermdecls\tokendecls
        \flexsstatedecls\flexxstatedecls\flexoptiondecls
    {%
        \getthird{\yy(2)}\to\postoks % the format marker
        \getfourth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \precdecls
    {%
        \getfifth{\yy(2)}\to\postoks % the format marker
        \getsixth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
    \oneproduction
    {%
        \getsecond{\yy(2)}\to\postoks % the \prodheader
        \getfourth{\postoks}\to\toksd % the format marker
        \getfifth{\postoks}\to\postoks % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\toksd}{\the\postoks}}}\next
    }
    \codeassoc\codepropstype
    {%
        \getseventh{\yy(2)}\to\postoks % the format marker
        \geteightth{\yy(2)}\to\toksd % the stash marker
        \edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
    }
}

\def\positionswitchdefault{%
    \errmessage{unknown chunk type in: \the\yy(2)}%
}

\def\posmark#1#2{%
%    \yyreadfifo\yyformat\to#1\in\yyformatlocal
%    no formattig for locations currently
    \restorecs{table-render}{\strm}%
%    \the\yyformatlocal
    \posm@rk{#2}%
}

\def\posm@rk#1{%
    \yyreadfifo\yystash\to#1\in\yystashlocal
    \edef\next{\the\yystashlocal}%
    \ifx\next\empty
        \postoks{}%
    \else
        \unmathstash{#1}%
        % TODO: move this out
        \toksa{}\the\yystashlocal\cleanstash\stripstash
        \setbox\z@=\vbox{\setlazyc\the\toksa}%
        \ifnum\ht\z@=\z@
            \postoks{}%
        \else
            \edef\next{\postoks{\noalign{\indent\toksa{}\the\yystashlocal\nx\makestashbox}}}\next
        \fi
    \fi
}

\def\codeoptionlist{%
    \codepropstype\codeassoc\prologuecode
}

\def\simpleoptionsplit{%
    \oneparametricoption\vardef\paramdef\optionflag
}

\def\noncodeoptionlist{%
    \oneparametricoption\vardef\paramdef\optionflag
    \codepropstype\codeassoc\prologuecode
}

\def\alloptionlist{%
    \oneparametricoption\vardef\paramdef\optionflag
    \codepropstype\codeassoc\prologuecode
    \typedecls\tokendecls\precdecls\oneproduction
    \flexsstatedecls\flexxstatedecls\flexoptiondecls
}

\def\emptyoption{\empty}

\diagprod\codeoptionlist\noncodeoptionlist\in\codenoncodeseplist
\diagprod\simpleoptionsplit\simpleoptionsplit\in\noncodenoncodeseplist
\diagprod\alloptionlist\emptyoption\in\emptyotherseplist

% this is a switch for inserting separators between dissimilar items

\def\separatorswitchneq{% 
    \classexpand\codenoncodeseplist
    {% separate options with code (\codepropstype, \codeassoc) from other options by a small space
        \toksd{\optoptsepsmall}%
    }
    \classexpand\noncodenoncodeseplist
    {%
        \toksd{\optoptsepnone}%
    }
    % a separator after the first rule or before or after a semicolon
    \classexpand\emptyotherseplist
    {%
        \toksd{\optoptsepnone}%
    }
}

\setspecialcharsfrom\separatorswitchneq

\def\separatorswitchdefaultneq{%
    \toksd{\prodprodsep}%
}

\def\pushothertables{%
    %token's
    \edef\next{\the\tokdectoks}%
    \ifx\next\empty
    \else
        \attachtokentable
    \fi
    %type's
    \edef\next{\the\typestable}%
    \ifx\next\empty
    \else
        \attachtypestable
    \fi
    %precedence's
    \edef\next{\the\prectable}%
    \ifx\next\empty
    \else
        \attachprectable
    \fi
    % options
    \edef\next{\the\opttable}%
    \ifx\next\empty
    \else
        \attachoptionstable
    \fi
    \concat\table\postoks
} %

% the various \attach... macros above are defined elsewhere and are the macros that 
% determine the actual typesetting of the corresponding tables.

\let\stopproduction\relax

\def\tokendecls#1#2#3{%
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:token-declarations}\yyunion
    \let\termindex\writetokenidxentry
    \displaytokenrawtrue
        \let\currenttokentype\empty#1%
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\typedecls#1#2#3#4{%
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:type-declarations}\yyunion
    \let\termindex\writetokenidxentry
        \typesset{}%
        #2\relax#1%
        \edef\next{\typestable{\the\typestable\nx\onetype{\the\toksa}{\the\typesset}}}\next
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

% \onetype is a typeseting macro defined elsewhere

\def\flexsstatedecls#1#2#3{% 
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:flex-state-declarations}\yyunion
    \let\termindex\writetokenidxentry
        \typesset{}%
        #1\relax
        \edef\next{\typestable{\the\typestable\nx\flexsstatelist{\the\typesset}}}\next
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\flexxstatedecls#1#2#3{% 
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:flex-state-declarations}\yyunion
    \let\termindex\writetokenidxentry
        \typesset{}%
        #1\relax
        \edef\next{\typestable{\the\typestable\nx\flexxstatelist{\the\typesset}}}\next
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

% \flexsstatelist and \flexxstatelist are typesetting macros defined elsewhere

\def\flexoptiondecls#1#2#3{% 
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:flex-option-definitions}\yyunion
    \let\termindex\writetokenidxentry
        \toksa{}%
        #1\relax
    \restorecslist{local-namespace}{\yyunion\termindex}%
}

\def\precdecls#1#2#3#4#5{% #1 type
                         % #2 tag
                         % #3 list of symbols
                         % #4 format pointer
                         % #5 stash pointer
    \savecslist{local-namespace}{\yyunion\termindex}%
    \restorecslist{table-render:prec-declarations}\yyunion
    \setrulecontext{*#1}%
    \let\termindex\writetokenidxentry
        \typesset{}#3\toksa{}#2%
        \edef\next{\prectable{\the\prectable\noexpand\oneprec{#1}{\the\toksa}{\the\typesset}}}\next
    \restorecslist{local-namespace}{\yyunion\termindex}%
    \setrulecontext{}%
}

% \oneprec is a typesetting macro defined elsewhere

% options typesetting

\def\oneparametricoption#1#2#3#4{%
    \restorecs{table-render:token-declarations}{\braceit\stringify\idit\anint\hexint}%
        \toksa{}\toksb{}\toksc{}%
        #2%
    \edef\next{\noexpand\oneparametricopti@n{#1}{\the\toksa}{\the\toksb}{\the\toksc}}\next
}

\def\oneparametricopti@n#1#2#3#4{%
    \edef\next{#2}%
    \ifx\next\empty % there is no identifier
        \def\next{#4}%
        \ifx\next\empty % there is no digit
            \edef\next{\toksa{{\ntt"\the\toksb"}}}\next
        \else
            \edef\next{\toksa{{\nx\rm\the\toksc}}}\next
        \fi
    \else
        \nameproc{#2}%
    \fi % now \toksa contains the value of the parameter
    \toksb{#1}%
    \let\termindex\writetokenidxentry
        \edef\next{\toksa{\gidxentry{\termostring}{#1}$\nx\langle${\nx\bf\the\toksb}$\nx\rangle$&\the\toksa\nx\/\nx\cr}}\next
    \let\termindex\eatone
    \concat\opttable\toksa
}

\def\paramdef#1#2#3#4{%
    \restorecs{table-render:token-declarations}\braceit
    \let\termindex\writetokenidxentry
        \edef\next{\toksa{\gidxentry{\termostring}{#2}}}\next
        \toksb\expandafter{\the\toksa$\langle${\bf #2}$\rangle$}\toksa{}%
        #1%
    \let\termindex\eatone
    \concat\toksb\toksa
    \concat\opttable\toksb
}

\def\optionflag#1#2#3#4{%
    \def\next{#2}%
    \ifx\next\empty
        \toksb{}%
    \else
        \toksb{ {\rm(set as $\langle${\bf#2}$\rangle$)}}%
    \fi
    \let\termindex\writetokenidxentry
        \edef\next{\toksa{\gidxentry{\termostring}{#1}$\nx\langle${\nx\bf#1}$\nx\rangle$\nx\rlap{$\nx\,\star$}&\the\toksb\cr}}\next
    \let\termindex\eatone
    \concat\opttable\toksa
}

\def\codepropstype#1#2#3#4#5#6#7{%
    \yyreadfifo\yyformat\to#4\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    \the\yyformatlocal % this can change \codepropstyp@
    \codepropstyp@{#2}{#3}{#1}{#5}%
}

\def\codepropstyp@#1#2#3#4{%
    \yyreadfifo\yystash\to#4\in\yystashlocal
    \unmathstash{#4}%
    \restorecs{table-render:type-declarations}{\idit\stringify\hspace}%
    \let\termindex\writetokenidxentry
    \typesset{}%
    #2% collect all the symbols in \typesset
    \edef\next{\toksa{\gidxentry{\termostring}{#3}$\nx\langle${\nx\bf #3}$\nx\rangle$&%
                 \toksa{}\the\yystashlocal\nx\makestashbox\nx\hfil\cr
                 \noalign{\nx\smallskip}%
                 \noalign{\indent$\nx\rlap{\hbox to2em{\nx\hfil$\nx\star$\nx\hfil}}\vcenter{
                     \advance\hsize by-\parindent
                     \emergencystretch10pt\nx\raggedright\noindent\hangafter\z@\hangindent2em\nx\strut\the\typesset\nx\strut}$}%
               }}\next
    \let\termindex\eatone
    \concat\opttable\toksa
}

\def\codeassoc#1#2#3#4#5#6#7{%
    \yyreadfifo\yyformat\to#4\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    \the\yyformatlocal % this can change \codeass@c
    \codeass@c{#1}{#2}{#3}{#5}%
}

\def\codeass@c#1#2#3#4{%
    \yyreadfifo\yystash\to#4\in\yystashlocal
    \unmathstash{#4}%
    \let\idit\termnameidit
    \let\termindex\writetokenidxentry
        \toksa{}% in case #1 is empty
        #2% put the typeset form of the identifie in \toksa
        \toksb\toksa
        \edef\next{\toksa{\yystringempty{#1}{}{\gidxentry{\termostring}{#1}}%
                          $\nx\langle${\nx\bf #1}$\nx\rangle$\nx\quad{\the\toksb}&%
                     \toksa{}\the\yystashlocal
                     \nx\makestashbox
                     \nx\hfil\cr
                  }}\next
    \concat\opttable\toksa
    \let\termindex\eatone
}

\def\vardef#1#2#3#4{%
    \yyreadfifo\yyformat\to#3\in\yyformatlocal
    \restorecs{table-render}{\strm}%
    \the\yyformatlocal % this can change \vard@f
    \vard@f{#1}{#2}{#4}%
}

\def\vard@f#1#2#3{%
    \let\idit\termnameidit
    \let\stringify\termnamestringify
    \let\termindex\writetokenidxentry
        #1% put the typeset form of the identifier or string in \toksa
        \toksb\toksa
        \savecslist{local-namespace}\yyunion
        \restorecslist{table-render:variable-definitions}\yyunion
            \toksa{}% in case #2 is empty
            #2% put the value in toksa
            \toksc\toksa
        \restorecslist{local-namespace}\yyunion
    \let\termindex\eatone
    \edef\next{\toksa{$\nx\langle${\nx\bf define}$\nx\rangle$\nx\quad{\the\toksb}&%
                 \the\toksc\cr
               }}\next
    \concat\opttable\toksa
}

\def\prologuecode#1#2#3#4#5{% similar to \actbrac@s
    \yyreadfifo\yystash\to#5\in\yystashlocal
    \unmathstash{#5}%
    \restorecs{table-render}{\strm\stashed}%
    \toksa{}\the\yystashlocal
    \edef\next{\toksa{\omit\span\omit\nx\cdotfill\cr
                      \omit\span\omit\toksa{\the\toksa}\nx\makestashbox\nx\cr
                      \omit\span\omit\nx\cdotfill\cr}}\next
    \concat\opttable\toksa
}

\def\yyunion@tablerender{%
    \strm\stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\actbrac@s\hspac@\codepropstyp@\vard@f\stopproduction
    \termmetastyle\peekstash
}

\savecslist{table-render}\yyunion
\savecslist{table-render}\yyunion@tablerender

\def\setprodtable{%
    \restorecslist{table-render}\yyunion
    \restorecslist{table-render}\yyunion@tablerender
}

% formatting macros

\def\midf#1#2{}

\def\format#1{%
    #1%
}
\let\formatlocal\format

\newtoks\yystashlastcontent % contents of the last action
\newif\ifshowlastaction % if true, show the last action of a flattened set of rules

\def\flatten{%
    \let\stopproduction\stopproduction@flatten
    \let\actbrac@s\actbrac@s@flatten
    \let\rrhss@p\rrhss@p@flatten
    \let\arhss@p\relax
    \let\rarhss@p\relax
}

\def\actbrac@s@flatten#1#2#3{% do everything but add the stash contents to the table
    \yyreadfifo\yystash\to#3\in\yystashlocal
    \unmathstash{#3}%
    \yystashlastcontent\yystashlocal
    \futurelet\actsep\actbrac@s@fl@tten
}

\def\actbrac@s@fl@tten{%
    \ifx\actsep\arhssep
        \edef\next{%
            \toksa{$\nx\ \nx\diamond\nx\ $\tidxentry{\termidstring}{inline_action}}}\next
        \concat\table\toksa
    \fi
}

\def\rrhss@p@flatten{%
    \toksa{$\ \vert\ $}\concat\table\toksa
}%

\def\stopproduction@flatten{%
    \ifshowlastaction
        \toksa\yystashlastcontent
    \else
        \toksa{\actionfiller}%
    \fi
    \toksa\expandafter{\expandafter&\expandafter&\the\toksa\cr}\concat\table\toksa
}

\def\fold{%
    \restorecs{table-render}{\actbrac@s\rrhss@p\arhss@p\rarhss@p\stopproduction}%
    \ifshowlastaction
        \toksa\yystashlastcontent
    \else
        \toksa{\actionfiller}%
    \fi
    \toksa\expandafter{\expandafter&\expandafter&\the\toksa\cr}\concat\table\toksa
}

\def\actionfiller{\omit\quad$\ldots$\hfil}

\def\inline{%
    \let\prodhead@r\prodhead@r@inline
    \let\hrhss@p\hrhss@p@inline
}

\def\prodhead@r@inline#1#2{%
    \let\idit\prodhdridit
    #1\relax
    \setbox\z@=\hbox{\strut\the\toksa\/$\,$\rm:\quad}%
    \tempda=\wd\z@\relax
    \advance\tempda-2em
    \edef\next{\table{\the\table{\the\toksa}}}\next
    \hrhss@p
}

\newif\ifsquashterms

\def\termspostformat{\ifsquashterms\aftergroup\noexpand\squashtermstrue\fi}

\def\hrhss@p@inline{%
    \edef\next{\toksa{&\termspostformat\hbox to \the\tempda{\hss}}}\next\concat\table\toksa
}

\def\breakline{%
    \let\hspac@\hspac@@breakline
}

\def\hspac@@breakline{%
    \toksa{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}\concat\table\toksa
    \restorecs{table-render}\hspac@
}

\def\skipheader{%
    \let\prodhead@r\prodhead@r@skipheader
    \let\hrhss@p\hrhss@p@skipheader
}

\def\prodhead@r@skipheader#1#2{%
    \hrhss@p
}

\def\hrhss@p@skipheader{\toksa{\omit\hbox to 2em{\hfil}&\termspostformat}\concat\table\toksa\restorecs{table-render}{\hrhss@p}}

\def\stashtoterm#1{% in case one needs to insert previous stash as a term
                   % usage: \let\peekstash\stashtoterm
    \yyreadfifo\yystash\to#1\in\yystashlocal
    \unmathstash{#1}%
    \edef\next{\toksb{{%
        \nx\restorecs{table-render}{\nx\stashed\nx\strm}%
        \toksa{}\the\yystashlocal\nx\makestashbox}}}\next
    \def\termmetastyle##1{\the\toksb}%
    \restorecs{table-render}\peekstash
}

\def\resetf{%
    \restorecs{table-render}{\prodheader\prodhead@r\hrhss@p\rrhss@p\arhss@p\rhs\termname\actbraces\actbrac@s\rarhss@p\hspac@\rules
        \oneproduction\emptyterm\stringify\idit\charit\prodprodsep\onesymbol\strm\stopproduction}%
    \squashtermsfalse
}

\def\prodstyle#1{% to typeset token names in text
    {%  
        \let\optstrextra\optstrextraesc
        \nameproc{#1}%
        \settermstyle{#1}%
        \the\toksa\nobreak
    }%
}

% macros used in typesetting the result of the prologue parsing

%type declarations

\restorecslist{parser-strict}\yyunion


\def\idit#1#2#3#4{%
    \termnameidit{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\stringify#1#2#3#4{%
    \termnamestringify{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\hspace#1#2{%
    \appendr\typesset{ }%
}

\def\charit#1#2#3#4{%
    \toksa{\.{#2}}%
    \concat\typesset\toksa
}

\def\tagit#1#2#3#4{%
    \toksa{\\{#2}}%
}

\toyyunion{table-render:type-declarations}

%token declarations

\newif\iftracetokennames

\restorecslist{parser-strict}\yyunion

\def\hspace#1#2{}

\def\idit#1#2#3#4{\toksa{#2}}

\def\charit#1#2#3#4{\toksa{'#1'}}

\def\stringify#1#2#3#4{\toksb{#2}}

\def\anint#1#2#3{\toksc{#1}}

\def\hexint#1#2#3{\toksc\expandafter{\eattwo#1$_{16}$}}

\def\tagit#1#2#3#4{%
    \def\currenttokentype{#2}%
}

\def\braceit#1#2#3{% similar to \actbrac@s
    \yyreadfifo\yystash\to#3\in\yystashlocal
    \unmathstash{#3}%
    \restorecs{table-render}{\strm\stashed}%
    \toksa{}\the\yystashlocal
    \edef\next{\toksa{&\toksa{\the\toksa}\nx\makestashbox\nx\cr}}\next
}

\def\onesymbol#1#2#3{% #1 is the macro name used internally by bison (\idit{}{}{}{})
                     % #2 is the integer value (\anint{}{}{})
                     % #3 is the string value (\stringify{}{}{}{})
    \toksa{}\toksb{}\toksc{}%
    #1#2#3\relax
    \edef\next{\noexpand\onesymb@l{\the\toksa}{\the\toksc}{\the\toksb}}\next
}

\def\onesymb@l#1#2#3{%         #1 is the macro name used internally by bison 
                             % #2 is the integer value 
                             % #3 is the string value 
    \let\optstrextra\optstrextraesc
    \yystringempty{#3}{}{\nameproc{#3}\tokstoks\toksa}%
    \yystringempty{#1}{}{\nameproc{#1}}%
    \iftracetokennames\derrmessage{token: \the\toksa, string: \the\tokstoks}\fi
    \toksb\expandafter{\currenttokentype}%
    \edef\next{%
        \tokdectoks{\the\tokdectoks
                    \noexpand\toksdefline
                             {\the\toksa}%
                             {% 
                                 \yystringempty{#1}{}{\gidxentry{\termidstring}{#1}}%
                                 \yystringempty{#3}{}{\gidxentry{\termstring}{#3}}%
                             }% 
                             {\yytoksempty\toksb{auto}{\the\toksb}}%
                             {#2}%
                             {\the\tokstoks}%
                 }%
    }\next
    \restorecs{nameparser}{\optstrextra}%
}

\def\defidxrank{0} % index rank of definitions

\def\writetokenidxentry#1{\write\gindex{%
    \nx\nx
    \expandafter
    \nx
        \csname\bisonidxdomain TI\endcsname\defidxrank#1}%
}

\toyyunion{table-render:token-declarations}

%precedence declarations

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{%
    \termnameidit{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\stringify#1#2#3#4{%
    \termnamestringify{#1}{#2}{#3}{#4}%
    \concat\typesset\toksa
}

\def\charit#1#2#3#4{%
    \expandafter\ifx\csname'parser'\hostparsernamespace#1\endcsname\relax
        \edef\next{\toksa{{\ntt #1}\gidxentry{\termvstring}{'#1'}}}%
    \else
        \edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
            \csname'parser'\hostparsernamespace#1\endcsname{\currentrulecontext}\gidxentry{\termvstring}{'#1'}}}%
    \fi
    \next
    \concat\typesset\toksa
}

\def\anint#1#2#3{%
    \edef\next{\toksa{${}=\hbox{\ntt#1}$}}\next
    \concat\typesset\toksa
}

\def\hexint#1#2#3{%
    \toksa\expandafter{\eattwo#1}%
    \edef\next{\toksa{${}=\hbox{\ntt\the\toksa}_{16}$}}\next
    \concat\typesset\toksa
}

\def\hspace#1#2{%
    \appendr\typesset{ }%
}

\def\tagit#1#2#3#4{%
    \toksa{\\{#2}}%
}

\def\symbolprec#1#2{%
    #1#2%
}

\toyyunion{table-render:prec-declarations}

% variable definitions

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{\toksa{#2}}

\def\stringify#1#2#3#4{\toksa{#2}}

\def\bracedvalue#1#2#3{% similar to \actbrac@s
    \yyreadfifo\yystash\to#3\in\yystashlocal
    \unmathstash{#3}%
    \restorecs{table-render}{\strm\stashed}%
    \toksa{}\the\yystashlocal
    \edef\next{\toksa{\toksa{\the\toksa}\nx\makestashbox}}\next
}

\toyyunion{table-render:variable-definitions}

% directives

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{%
    \termnameidit{#1}{#2}{#3}{#4}%
}

\def\stringify#1#2#3#4{%
    \termnamestringify{#1}{#2}{#3}{#4}%
}

\def\tagit#1#2#3#4{%
    \toksa{\\{#2}}%
}

\def\anint#1#2#3{%
    \toksa{#1}%
}

\def\hexint#1#2#3{%
     \toksa\expandafter{\eattwo#1$_{16}$}%
}

\toyyunion{table-render:directives}

% flex state declarations

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{%
    \nameproc{#2}%
    \edef\next{\toksa{{%
            \gidxentry{\termttstring}{#1}%
            \ntt\the\toksa
        }}}\next
    \concat\typesset\toksa
}

\def\hspace#1#2{%
    \appendr\typesset{ }%
}

\toyyunion{table-render:flex-state-declarations}

% flex option definitions

\restorecslist{parser-strict}\yyunion

\def\flexoptionpair#1#2{%
    #1%
    \edef\next{#2}%
    \ifx\next\empty % there is no rihgt hand side
        \edef\next{\toksa{$\nx\langle${\ntt\the\toksa}$\nx\rangle_{\rm f}\,\star$&\cr}}\next
    \else
        \toksb\toksa\toksa{}#2%
        \edef\next{\toksa{$\nx\langle${\ntt\the\toksb}$\nx\rangle_{\rm f}$&\nx\it\the\toksa\nx\/\nx\cr}}\next
    \fi 
    \concat\opttable\toksa
}

\def\idit#1#2#3#4{\toksa{#2}}

\def\stringify#1#2#3#4{\toksa{{\ntt@"#2"}}}

\def\hspace#1#2{}

\toyyunion{table-render:flex-option-definitions}

% rule listing macros

\restorecslist{parser-prototypes}\yyunion % we just want to ignore the non-rule related code

\def\prodheader#1#2#3#4{%
    #1\relax
    \lastlastname\lastname
    \setnumname\numname
    \lastname{}%
    #2\relax
    \edef\next{\toksc{\numname:}\toksd{\noexpand\lhs{\the\lastlastname}{\the\lastname}}}\next
}

\def\rrhssep#1#2{}
\def\arhssep#1#2{}
\def\rarhssep#1#2{}
\def\hspace#1#2{}

\def\rhs#1#2#3{%
    \concat\toksa\toksc
    \concat\toksb\toksd
    #1\relax
    \edef\next{\toksb{\noexpand\onerule{\the\toksb}}}\next
    \appendr\toksa\def
    \possibleimplicitfalse
    \yyreplacestring\toksa\in\newsymswitch\with\toksb
    \toksa{}\toksb{}%
}

\newtoks\lastname
\newtoks\lastlastname

\def\termname#1#2{%
    \flushact
    #1\relax
    \lastlastname\lastname
    \setnumname\numname
    \lastname{}%
    #2\relax
    \appendr\toksa{ \numname}\appendr\toksb{\noexpand\term{\the\lastlastname}{\the\lastname}}%
}

\def\setnumname#1{%
    \expandafter\let\expandafter\next\csname term\parsernamespace\the\lastname \endcsname
    \ifx\next\relax % this is a terminal that was renamed
        \expandafter\ifx\csname token\parsernamespace\the\lastname \endcsname\relax % this is probably a symbol from another grammar
            \def#1{-1}%
            \errmessage{token \csname token\parsernamespace\the\lastname \endcsname\space is undefined}%
        \else
            \expandafter\tempca\csname token\parsernamespace\the\lastname \endcsname
            \edef#1{\fgetelemof{yytranslate}\at\tempca}%
        \fi
    \else
        \let#1\next
    \fi
}

\def\actbraces#1#2#3#4#5\bdend{%
    \flushact
    \lastname{}%
    #2\relax
    \edef\next{\lastactsym{\the\lastname}}\next
    \possibleimplicittrue
}

\def\rules#1#2#3{#1}

\def\oneproduction#1#2#3{#1}

\def\emptyterm{%
    \flushact
    \lastname{}%
}

\def\stringify#1#2#3#4{%
    \lastname{"#1"}%
}

\def\idit#1#2#3#4{%
    \lastname{#1}%
}

\def\charit#1#2#3#4{%
    \lastname{'#1'}%
}

\def\prodprodsep{}

\def\onesymbol#1#2#3{}

\def\anint#1#2#3{%
    \lastname{#1}%
}

\def\hexint#1#2#3{%
     \lastname\expandafter{\expandafter\number\eattwo#1$_{16}$}%
}

\newtoks\lastactsym
\newif\ifpossibleimplicit

\def\flushact{%
    \ifpossibleimplicit
        \appendr\toksa{ @implicit@}\appendr\toksb{\noexpand\implicitterm{@implicit@}{\the\lastactsym}}%
        \lastactsym{}%
        \possibleimplicitfalse
    \fi
}

\savecslist{symbols}\yyunion

% name parser macros

\def\idstr#1#2{%
   \toksb{#1}%
   \concat\toksa\toksb
   \toksb{#2}%
   \concat\toksc\toksb
}

\def\chstr#1#2{%
   \toksb{{\ntt@#1}}%
   \concat\toksa\toksb
   \toksb{#2}%
   \concat\toksc\toksb
}

\def\sfxi#1#2{\toksb{\/${}_{\idxfont#1}$}\concat\toksa\toksb}

\def\sfxn#1#2{\toksb{#1}\concat\toksa\toksb}

\def\dotsp{\futurelet\next\d@tsp}

\def\d@tsp{%
    \ifx\next\sfxi
    \else
        \ifx\next\qual
        \else
            \toksb{.}\concat\toksa\toksb
        \fi
    \fi
}

\def\optstr#1#2{%
   \toksb\expandafter{\eatone#1}%
   \optstrextra
   \concat\toksa\toksb
   \toksb{#2}%
   \concat\toksc\toksb
}

\def\optstrextra{%
    \edef\next{\toksb{{$\nx\ulcorner$\ntt\the\toksb$\nx\urcorner$}}}\next
}

\def\optstrextraesc{%
    \edef\next{\toksb{{$\nx\langle$\ntt\the\toksb$\nx\rangle$}}}\next
}

\def\qual#1#2{\toksb{\/${}_{\rm#2}$}\concat\toksd\toksb}

\let\idxfont\relax

\savecs{nameparser}{\idstr\chstr\sfxi\sfxn\qual\optstr\dotsp\optstringextra\idxfont}

\newtoks\namechars

\newif\iftracebadnames
\newif\iftracenames

\def\nameproc#1{%
    \ifbootstrapmode
        \toksa{}%
    \else
        \savecs{local-namespace}\getcurrentparser
        \tosmallparser
        \basicparserinit
        \yyparse#1\yyeof\yyeof\endparseinput\endparse
        \ifyyparsefail
            \iftracebadnames
                \ferrmessage{bad name: #1}%
            \fi
            \toksa{#1}\toksc{}%
        \else
            \toksa{}\toksb{}\toksc{}\toksd{}%
            \iftracenames
                \ferrmessage{parsed name: \the\namechars}%
            \fi
            \the\namechars\relax
            \iftracenames
                \ferrmessage{processed name: \the\toksa<identifier>, \the\toksb<suffices>, \the\toksc<hash name>, \the\toksd<options>}%
            \fi
            \expandafter\ifx\csname'parser'\hostparsernamespace\the\toksc \endcsname\relax
            \else
                \toksa\expandafter{\csname'parser'\hostparsernamespace\the\toksc \endcsname}%
                \appendr\toksa{{\currentrulecontext}}%
            \fi
            \concat\toksa\toksd
        \fi
        \restorecs{local-namespace}\getcurrentparser
        \getcurrentparser
    \fi
}

%% token typesetting macros

\def\setsafe#1{\def\saferword{#1}\setspecialcharsfrom\saferword}
\def\prettyword#1{\setsafe{#1}\expandafter\edef\csname'parser'\parsernamespace\saferword\endcsname##1{{\ntt\lowercase{\saferword}}}}
\def\prettytoken#1{\setsafe{#1}\expandafter\edef\csname'parser'\parsernamespace\saferword\endcsname##1{{\ntt\saferword}}}
\def\prettytoken@#1{\toksa{}\numberstocharsandspaces#1\end\expandafter\edef
    \csname'parser'\parsernamespace\the\toksa\endcsname##1{{\ntt\the\toksa}}%
}
\def\prettywordpair#1#2{\setsafe{#1}\expandafter\def\csname'parser'\parsernamespace\saferword\endcsname##1{{\ntt@#2}}}
\def\prettywordpair@#1#2{%
    \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end\expandafter\nameproc\expandafter{\the\toksa}%
    \expandafter\edef\csname'parser'\parsernamespace\saferword\endcsname##1{%
        \noexpand\ifdisplaytokenraw
            {\ntt\saferword}%
        \noexpand\else
            {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa}%
        \noexpand\fi
    }%
}
\def\prettywordpair@@#1#2{\setsafe{#1}\expandafter\def\csname'parser'\parsernamespace\saferword\endcsname##1{{#2}}}

% the bootstrap macros

\def\initbootstrap{%
    \restorecslist{bootstrap}\yyunion
}

\restorecslist{parser-prototypes}\yyunion

\def\tokendecls#1#2#3{#1}

\def\stringify#1#2#3#4{\toksa{"#1"}}

\def\idit#1#2#3#4{\toksb{#1}}

%\def\anint#1#2#3{\toksc{#1}}% this definition is not needed as all
                     %information is extracted from yytname

% note that the definitions below do not handle cases such as 
%token '{' "left brace" since the lexer knows how to return '{'

\def\onesymbol#1#2#3{%
    \toksa{}\toksb{}%\toksc{}%
    #1#2#3%
    \yytoksempty\toksa{\expandafter\charstonumbers\the\toksb\end}%
                      {\expandafter\toksa\expandafter{\expandafter}\expandafter\charstonumbers\the\toksa\end}%
    \yytoksempty\toksb{%
        \yytoksempty\toksa{}{%
            \immediate\write\tokendefs{\noexpand\tokeneq{\the\toksb}{\the\toksa}}%
        }%
    }{%
        \immediate\write\tokendefs{\noexpand\tokeneq{\the\toksb}{\the\toksa}}%
    }%
}

\def\flexsstatedecls#1#2#3{% bootstrap parser does not produce state declarations so it is fine to share states
    \savecslist{local-namespace}\yyunion
    \restorecslist{bootstrap:flex-state-declarations}\yyunion
        #1\relax
    \restorecslist{local-namespace}\yyunion
}

\let\flexxstatedecls\flexsstatedecls

\savecslist{bootstrap}\yyunion

\restorecslist{parser-strict}\yyunion

\def\idit#1#2#3#4{{\immediate\write\stlist{_register_name(#1);}}}

\def\hspace#1#2{}

\savecslist{bootstrap:flex-state-declarations}\yyunion

\def\charstonumberse#1\end{%
    \yystringempty{#1}{}%
        {\yystartsinspace{#1}{{\number`\ }\charstonumber@@#1\end}{\charstonumbers@#1\end}}%
}

\def\charstonumbers@#1{%
    {\number`#1}\charstonumberse
}

\def\charstonumber@@#1 {\charstonumberse}

\def\charstonumbers#1\end{\edef\next{\toksa{\charstonumberse#1\end}}\next}
