summaryrefslogtreecommitdiff
path: root/support/splint/tex/yyunion.sty
diff options
context:
space:
mode:
Diffstat (limited to 'support/splint/tex/yyunion.sty')
-rw-r--r--support/splint/tex/yyunion.sty1314
1 files changed, 954 insertions, 360 deletions
diff --git a/support/splint/tex/yyunion.sty b/support/splint/tex/yyunion.sty
index 7f4937a4e3..81d69b3789 100644
--- a/support/splint/tex/yyunion.sty
+++ b/support/splint/tex/yyunion.sty
@@ -1,4 +1,4 @@
-% Copyright 2012-2020, Alexander Shibakov
+% Copyright 2012-2024, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
@@ -34,17 +34,14 @@
\|\`\\\{\}\~\ \_\&\^\$\#\n% the last one is inserted by the preprocessing scripts
}
-
-% insidemost
+% insidemost (level 0)
\long\def\stashed#1{} % stashed material (usually \Cee\ code) :: \stashed{tex_string}
\long\def\sflush#1#2{} % stash flushing marker :: \sflush{tex_string}
\def\format#1{} % format material triggered by braces and the start of production :: \format{tex_string}
\def\formatlocal#1{} % format material triggered by every term :: \formatlocal{tex_string}
-% level 1
-
-\long\def\strm[#1]#2.[#3]{} % a stream element :: \strm[digits]{\stashed|\format|\formatlocal}.[digits]
+% level 1---atoms minimally processed by the parser
% types returned by the lexer; the integer type (hex vs. decimal) is determined at the lexer level so
% these are tagged by the lexer as well
@@ -63,33 +60,32 @@
\defp\midf#1#2{} % *rule separator :: \midf{fptr}{sptr}
\defp\preckind#1#2#3{} % *precedence operator :: \preckind{12string}{fptr}{sptr}
%
-\defp\flexoptionpair#1#2{} % a flex option :: \flexoptoinpair{\idit}{\idit|\stringify}
\defp\termname#1#2{} % a production term :: \termname{\idit|\stringify|\charit}{\idit}
\defp\mergeop#1#2#3{} % merge directive :: \mergeop{\tagit}{fptr}{sptr}
\defp\dprecop#1#2#3{} % dprec directive :: \dprecop{\anint|\hexint}{fptr}{sptr}
\defp\sprecop#1#2#3{} % prec directive :: \sprecop{\idit|\stringify|\charit}{fptr}{sptr}
-\defp\rrhssep#1#2{} % :: \rrhssep{fptr}{sptr}
-\defp\arhssep#1#2{} % :: \arhssep{fptr}{sptr}
-\defp\rarhssep#1#2{} % :: \rarhssep{fptr}{sptr}
+\defp\rrhssep#1#2{} % separator :: \rrhssep{fptr}{sptr}
+\defp\arhssep#1#2{} % separator (between the action and the right hand side) :: \arhssep{fptr}{sptr}
+\defp\rarhssep#1#2{} % separator :: \rarhssep{fptr}{sptr}
\defp\hspace#1#2{} % a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
-\defp\emptyterm{}
+\defp\emptyterm{} % implicit empty right hand side :: \emptyterm
\defp\onesymbol#1#2#3{} % a symbol definition :: \onesymbol{\idit}{\anint|\hexint}{\stringify}
\defp\bracedvalue#1#2#3{} % a braced variable value :: \bracedvalue{tex_string}{fptr}{sptr}
-%level 1
+% level 2---the structures built by the parser
+\defp\prodheader#1#2#3#4{} % a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
\defp\rhs#1#2#3{} % a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
\defp\actbraces#1#2#3#4#5\bdend{} % action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
\defp\bpredicate#1#2#3#4#5\bdend{} % predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend
-% level 2
+% level 3---the structures composed of structures above
-\defp\prodheader#1#2#3#4{} % a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
-\defp\rules#1#2#3{} % a complete right-hand side :: \rules{[\rhs|\rrhssep]}{max fptr}{max sptr}
+\defp\rules#1{} % a complete right-hand side :: \rules{[\rhs|\rrhssep]}
\defp\symbolprec#1#2{} % symbol precedence :: \symbolprec{\idit|\stringify|\charit}{\anint|\hexint}
%\def\pcluster#1#2{} % a production cluster :: \pcluster{\prodheader}{\termname|...} should never appear in the final output
-% level 3
+% level 4---larger structures and top level constructions
\defp\oneproduction#1#2#3{} % a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}
\defp\prodprodsepsmall{} % production separator
@@ -102,9 +98,6 @@
\defp\ntermdecls#1#2#3{} % nterm declarations :: \ntermdecls{[\onesymbol]+}{fptr}{sptr}
\defp\typedecls#1#2#3#4{} % type declarations :: \typedecls{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\precdecls#1#2#3#4#5{} % precedence declarations :: \precdecls{left|right|nonassoc|precedence}{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
-\defp\flexsstatedecls#1#2#3{} % flex state declarations (nonexlusive) :: \flexsstatedecls{[\idit\stringify]+}{fptr}{sptr}
-\defp\flexxstatedecls#1#2#3{} % flex state declarations (exclusive) :: \flexxstatedecls{[\idit\stringify]+}{fptr}{sptr}
-\defp\flexoptiondecls#1#2#3{} % flex options :: \flexoptiondecls{[\flexoptionpair]+}{fptr}{sptr}
\defp\oneparametricoption#1#2#3#4{} % option :: \oneparametricoption{option-name}{\stringify|\idit}{fptr}{sptr}
\defp\optionflag#1#2#3#4{} % option flag :: \optionflag{flag name}{abbreviated name}{fptr}{sptr}
\defp\paramdef#1#2#3#4{} % parameters :: \paramdef{\braceit}{parameter domain}{fptr}{sptr}
@@ -114,10 +107,6 @@
\defp\vardef#1#2#3#4{} % variable definition :: \vardef{\idit|\stringify}{|\idit|\stringify|\bracedvalue}{fptr}{sptr}
\defp\prologuecode#1#2#3#4#5{} % prologue code :: \prologuecode{tex_string}{fptr}{sptr}{fptr}{sptr}
-% level 4
-
-%\def\grammar#1#2{} % a grammar: should never appear in the final output
-
\savecslist{parser-prototypes}\yyunion
% table construction: the macros run to build the contents of the table of productions to be
@@ -141,6 +130,10 @@
\newtoks\yystashlocal
\newtoks\yyformatlocal
+\def\astappend{\appendtolist\aststream}
+\def\astappendx{\appendtolistx\aststream}
+\def\astisempty{\listisempty\aststream}
+
% index implementation
% note that the rank and domain parameters are handed down by the
% `outer' macro while the `qualifier' (\term...string) is decided
@@ -194,7 +187,7 @@
% must take place)
% o finally, the entries that must be expanded at a specific point between the point
% of invocation and the \shipout
-% the indexing macro below treats the first three cases for the entries that the macro
+% the indexing macro below treats the first three cases for the entries that the macros
% below produce; the last case can be implemented by redefining the \stickycs macro
% at the appropriate point
@@ -256,9 +249,11 @@
\toksa{\ifcase#1<NUL>}
\tempcb=\uccode`.
+% a loop that builds a macro to produce a category 12 character with the given character code
+
\bloop
\uccode`.=\tempca
-\uppercase{\toksb\expandafter{.}}%
+\uppercase{\toksb{.}}%
\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa\expandafter\or\the\toksb}%
\advance\tempca by\@ne
\ifnum\tempca>"FF
@@ -266,6 +261,15 @@
\else
\repeat
+% defining the macro that was built above
+
+\edef\next{\toksa{\def\noexpand\asciiprint##1{\the\toksa}}}\next
+\the\toksa
+
+% return . to its regular state
+
+\uccode`.=\tempcb
+
\def\charstocharsx#1\end{% expandable version of a `safety' macro, ascii characters only, no tricky space characters
\yystartsinspace{#1.}{%
\space\ch@rstocharsx#1\end
@@ -280,9 +284,6 @@
\def\ch@rst@charsx#1#2\end{\asciiprint{`#1}\charstocharsx#2\end}
-\edef\next{\toksa{\def\noexpand\asciiprint##1{\the\toksa}}}\next
-\the\toksa
-
\def\writeidxhentry#1{%
\indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\headeridxrank}#1}%
}
@@ -304,6 +305,40 @@
\fi
}
+\def\ridxentry#1#2#3{% index entry that allows raw input (see
+ \gidxentry{#1}{\end0 \vend{32}\end\toksa{#2}\noexpand\noexpand\noexpand\eatone\vend\eatone}{#3}%
+ % the first \end and \eatone will be removed by \charsto..., the next (innermost) by \numbersto... inside \FI
+}
+
+\def\didxentry#1#2{% (another) index entry that allows raw input (see \csname flex@ption_other\endcsname below)
+ \gidxentryxb{\termraw}{\number#1}{#2}%
+}
+
+\def\gtextidx#1#2#3{% #1 is the entry
+ % #2 is the visual key
+ % #3 is the domain
+ \expandafter\ifx\csname indexterms<count>\endcsname\relax
+ \else
+ {%
+ \storerawidxentry{#1}%
+ \edef\next{%
+ \noexpand\indxe\gindex{{\secno}{{}{\indexpseudonamespace}}{#3}{\termidxrank}{}%
+ {\noexpand\termraw}{\number\getstackpointer\indexterms}{#2}}%
+ }\next
+ \edef\next{\next}\next
+ }%
+ \fi
+}
+
+% a shortcut to simplify using the macro above
+
+\def\storerawidxentry#1{%
+ {%
+ \toksa{#1}%
+ \let\parsernamespace\empty\yypushg{\the\toksa}\on\indexterms
+ }%
+}
+
% a convenient shortcut, as many visual keys come in a token register
\def\gidxentryxv#1#2#3{% a generic index entry, e(x)pand (v)isual key:
@@ -338,62 +373,152 @@
\edef\next{\tidxentry{\termidstring}{#1}}\next
}
-\def\gatherstash#1#2\strm[#3]#4.[#5]{% #1 contains the stash gathered
- \yystringempty{#3}{ #1}%
- {%
- \expandafter\g@therstash\expandafter{\romannumeral0\unstash#4}{#1}%
- }%
+% stash processing wrapper macros
+
+\def\yyreadstash#1\to#2\in#3\with#4{%
+ \consumelistupto{#1}{#2}\to{#3}%
+ #4{#3}%
+ \mathsplittoks{#3}{\yyr@adstash}{\eatthree}{#1}{#2}{#3}%
}
-% stash processing wrapper macros
+% this macro is based on the assumption that all the math mode
+% changes are introduced by unadulterated $ tokens (as is done in
+% \CWEB); \CWEB's handling of math mode is a mistake so there is
+% no need to introduce another one by using clever math switches;
+% note that this is not affected by \inmath (see somewhere in this file),
+% since (as long as!) it is only inserted in the verbatim mode ...
+% ... or at least it should, so never do something like
+% @t}\inmath{{@> ... @t}}@>
+% unless you really know what you are doing (hint: it is only ok
+% to do this if there are no intervening \vb{...} groups inside
+% the ... portion above.
-\def\g@therstash#1#2{\gatherstash{#2#1}}
+\def\mathsplittoks#1#2#3{% execute actions depending on whether
+ % token register #1 contains
+ % odd (then insert #2)
+ % or even (insert #3) number of $'s
+ \expandafter\evenmathify\the#1$%
+ \evenmather\end$\evenmather\end$\evenmather\end$%
+ \evenmather\end$\evenmather\end$\evenmather\end$%
+ \emo$\eme$\emo$\eme$\emo$\eme$\emo$\eme$\emo$\end
+ {#2}{#3}%
+}
-\def\unstash#1#2{\expandafter\space#2}% #1 is \stashed or \sflush
+\def\evenmathcontainer{\evenmather\end}
-\def\yyreadstash#1\to#2\in#3\with#4{%
- \yyreadfifo#1\to#2\in#3%
- #4{#3}%
- \setbox\z@=\vbox{
- \hsize\maxdimen\emergencystretch\maxdimen
- \rightskip\z@ plus 1fill
- \insertstash{#3}%
- \ifmmode
- {}$\aftergroup\yyr@adstash % $o that the box can be finished
+\def\evenmather#1$#2$#3$#4$#5$#6${}
+
+\def\emo#1\end{\yyfirstoftwo}
+
+\def\eme#1\end{\yysecondoftwo}
+
+\def\evenmathify#1$#2$#3$#4$#5$#6${%
+ {% make all changes to \next local
+ \def\next{#6}%
+ \ifx\next\evenmathcontainer
+ \yybreak{}%
\else
- \aftergroup\eatthree
- \fi
- }{#1}{#2}{#3}%
+ \yybreak{\aftergroup\evenmathify}%
+ \yycontinue
+ }%
}
\def\yyr@adstash#1#2#3{%
- #3\expandafter{\the#3{}$}% $
- \appendlnx#1{\strm[NaN]{\stashed{{}${}}}.[NaN]}% {}, in case \yystash ends with $
+ #3\expandafter{\the#3{}$}% $upply the math mode switch
+ {%
+ \tempca#2\relax
+ \advance\tempca\@ne
+ \edef\next{%
+ \noexpand\changelistelem{#1}{#2}{}{{}$}% change the consumed stash, so it can be read again
+ \noexpand\changelistelem{#1}{\the\tempca}{{}${}}{}%
+ }\expandafter
+ }\next
}
-% the sequence below may be redefined so that the #2 parameter to \yyreadstash may be a list of
-% token registers
-
-\def\insertstash#1{\the#1\relax}
-
-\def\readstash#1{\yyreadstash\yystash\to#1\in\yystashlocal\with\unpackstash}
+\def\readstash#1{%
+ \yyreadstash\yystash\to#1\in\yystashlocal\with\eatone
+}
\def\readstashwredirect#1#2#3{% helper macro to read the stash when the stash and the format pointers appear together
% #1(the format pointer) is ignored
% #2 is the stash marker
% #3 is the local stash register
- \yyreadstash\yystash\to#2\in#3\with\unpackstash
+ \yyreadstash\yystash\to#2\in#3\with\eatone
+}
+
+% format processing wrapper macros
+
+\def\readformat#1{%
+ \let\listiterator\listiteratorsimple
+ \consumelistupto\yyformat{#1}\to\yyformatlocal
+}
+
+\def\readformatlocalonly#1{%
+ \let\listiterator\listiteratorfmtloc
+ \let\wrapupfmtlocal\wrapupyyformatlocal
+ \yyformatlocal{}%
+ \evallistbetween\yyformat{\showlistpointer\yyformat}{#1}%
+}
+
+% iterator to extract \formatlocal
+
+\def\listiteratorfmtloc#1#2{%
+ \lookforfmtloc{#1}{}{}.#2\formatlocal{\formatlocal}%
+}
+
+\def\lookforfmtl@c#1#2#3#4#5{%
+ \isthisfmtlocal#5.\formatlocal{%
+ \wrapupfmtlocal{#2}{#3#1}{#4}%
+ }{%
+ \lookforfmtloc{#2}{#3#1}{#4\formatlocal{#5}}.%
+ }%
}
-\def\unpackstash#1{#1\expandafter{\romannumeral0\expandafter\unp@ckstash\the#1\strm[].[]}}
+\def\lookforfmtloc#1#2#3#4\formatlocal#5{% remove the guard .
+ \expandafter\lookforfmtl@c\expandafter{\eatone#4}{#1}{#2}{#3}{#5}%
+}
-\def\unp@ckstash{\gatherstash{}}
-\def\unwrapstash#1{% to use where only the removal of \strm and \stashed is needed
- #1\expandafter{\romannumeral0\expandafter\unwr@pstash\the#1\strm[].[]}%
+\def\isthisfmtlocal#1\formatlocal{%
+ \yystringempty{#1}{% a match, clean up the inserted \formatlocal
+ \endoffmtlocal
+ }{% not a match yet, \lookforfmtloc again
+ \yysecondoftwo
+ }%
}
-\def\unwr@pstash{\gatherstash{}}
+\def\endoffmtlocal#1\formatlocal{\yyfirstoftwo}
+
+\def\wrapupfmtlocal#1#2#3{% a postprocessing macro, to be changed by the one that
+ % invokes the iterator
+ \fdef#1{#2}%
+}
+
+\def\wrapupyyformatlocal#1#2#3{% macro used to save the \formatlocal contents in \yyformatlocal
+ \fdef#1{#2}\yyformatlocal\expandafter{\the\yyformatlocal#3}%
+}
+
+\let\wrapupfmtlocal\wrapupyyformatlocal
+
+% use example for the macros above:
+%\initlist{table}
+%\toksa{#}
+%\let\listiterator\listiteratorfmtloc
+%\appendtolisti{table}{..}
+%\appendtolisti{table}{a,}
+%\appendtolisti{table}{\formatlocal{}{b}\formatlocal{k} , }
+%\appendtolisti{table}{{c},}
+%\appendtolisti{table}{ \space d,}
+%\appendtolisti{table}{e\formatlocal{d},}
+%\appendtolisti{table}{\format{}{f}\formatlocal{j}}
+%\appendtolisti{table}{.}
+%\appendtolisti{table}{\end}
+%\finishlist{table}
+%\storelist{table}\to\toksa
+%\the\toksa
+%\storelist{table}\to\toksa
+%\message{full list: \the\toksa}
+%\errmessage{separated: \the\yyformatlocal}
% parser invocation
@@ -401,9 +526,12 @@
\def\d@parse#1{\yyparse#1\yyeof\yyeof\endparseinput\endparse}
-\def\astarraylastcs{\expandafter\noexpand\romannumeral\gettopofstackcsx\astarray}
-
-\long\def\strm[#1]#2.[#3]{#2} % this is used with both the format and the stash streams; TODO: deprecate
+% as an alternative to lists, stacks (as arrays) can be used to speed up the parser;
+% the following setup (along with a convenient shortcut to reference the most
+% recent store) can be employed if desired; unlike lists, stacks do not have
+% a built-in mechanism for slice expansion.
+%\yyinitstack\astarray % flattened AST to (possibly) speed up the parser
+%\def\astarraylastcs{\expandafter\noexpand\romannumeral\gettopofstackcsx\astarray}
\long\def\inmath#1{$#1$}
@@ -416,9 +544,14 @@
\def\ntt{\noexpand\ntt@} % a convenient shortcut
\def\ntt@{\tt\let\_\UL}
-\def\prodheader#1#2#3#4{%
- \yyreadfifo\yyformat\to#3\in\yyformatlocal
- \restorecs{table-render}{\strm}\the\yyformatlocal
+% macros that process the bison grammar terms;
+% the higher level macros (like the \prodheader below) set up the sequence in
+% which the processing takes place; the actual typesetting (or whatever other
+% task is desired, like reference creation) is done by the lower level macros
+% like \prodhead@r in this case.
+
+\defc\prodheader{% a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
+ \readformat{#3}\the\yyformatlocal
\let\termindex\writeidxhentry
\prodhead@r{#1}{#2}%
\let\termindex\eatone
@@ -426,53 +559,50 @@
\def\prodhead@r#1#2{%
\let\idit\prodhdridit
- #1\relax\concat\table\toksa\hrhss@p
+ #1\relax\astappendx{\the\toksa}\hrhss@p
}
-\def\hrhss@p{\toksa{&\omit\hfil&\omit\hfil&\omit\hfil\cr\noalign{\penalty100}\omit\hfil&\termspostformat}\concat\table\toksa}
+\def\hrhss@p{\toksa{&\omit\hfil&\omit\hfil&\omit\hfil\cr\noalign{\penalty100}\omit\hfil&\termspostformat}\astappendx{\the\toksa}}
-\def\rrhssep#1#2{%
- \yyreadfifot\yyformat\to#1\in\yyformatlocal\with\formatsort\relax
+\defc\rrhssep{% separator :: \rrhssep{fptr}{sptr}
+ \readformatlocalonly{#1}%
\the\yyformatlocal % this will have a chance to change the definition of \rrhss@p
\rrhss@p
}
-\def\rrhss@p{\toksa{\omit\hfil&}\concat\table\toksa}
+\def\rrhss@p{\toksa{\omit\hfil&}\astappendx{\the\toksa}}
-\def\arhssep#1#2{%
- \yyreadfifo\yyformat\to#1\in\yyformatlocal
+\defc\arhssep{% separator (between the action and the right hand side) :: \arhssep{fptr}{sptr}
+ \readformat{#1}%
\the\yyformatlocal % this will have a chance to change the definition of \arhss@p
\arhss@p
}
%% between the inline action and next term:
-\def\arhss@p{\toksa{\omit\hfil&\hbox to 2em{\hfil}}\concat\table\toksa}
+\def\arhss@p{\toksa{\omit\hfil&\hbox to 2em{\hfil}}\astappendx{\the\toksa}}
-\def\rarhssep#1#2{%
- \yyreadfifo\yyformat\to#1\in\yyformatlocal
+\defc\rarhssep{% separator :: \rarhssep{fptr}{sptr}
+ \readformat{#1}%
\the\yyformatlocal % this will have a chance to change the definition of \rarhss@p
\rarhss@p
}
%% between a term and the following action
-\def\rarhss@p{\appendr\table&}
-
-\newtoks\hstoks
+\def\rarhss@p{\astappend&}
-\def\hspace#1#2{%
- \hstoks\yyformat
- \yyreadfifot\yyformat\to#1\in\yyformatlocal\with\formatsort\relax
+\defc\hspace{% a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
+ \readformatlocalonly{#1}%
\the\yyformatlocal % may change \hspac@
\hspac@
}
\def\hspac@{%
- \appendr\table{ }%
+ \astappend{ }%
}
-\def\rhs#1#2#3{#1}
+\defc\rhs{#1}% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
\def\supplybdirective#1#2{%
\edef\next{\nx\s@pplybdirective#1{\the#2}\the#1\nx\bdend}\next
@@ -482,7 +612,7 @@
#1{#3#2\bdend}%
}
-\def\termname#1#2{%
+\defc\termname{% a production term :: \termname{\idit|\stringify|\charit}{\idit}
\let\idit\termnameidit
\let\stringify\termnamestringify
\let\optstrextra\optstrextraesc
@@ -499,7 +629,7 @@
\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
\yystringempty{#1}{\toksa{#2}}{\toksa{#2\hbox{$\langle${}#1{}$\rangle$}}}%
- \concat\table\toksa
+ \astappendx{\the\toksa}%
}
\newif\ifbracketedvisible % decide whether to show the bracketed name in productions
@@ -512,16 +642,15 @@
\toksa{#2}%
\fi
}%
- \concat\table\toksa
+ \astappendx{\the\toksa}%
}
%
\newtoks\bdirects
-\def\actbraces#1#2#3#4#5\bdend{%
- \yyreadfifo\yyformat\to#3\in\yyformatlocal
- \restorecs{table-render}{\strm}%
+\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
+ \readformat{#3}%
#5% accumulate all the directives
\yytoksempty\bdirects{}{%
\edef\next{\bdirects{%
@@ -533,13 +662,12 @@
\def\actbrac@s#1#2#3{%
\readstash{#3}%
- \appendr\table{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
+ \astappendx{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
\bdirects{}%
}
\def\bpredicate#1#2#3#4#5\bdend{%
- \yyreadfifo\yyformat\to#3\in\yyformatlocal
- \restorecs{table-render}{\strm}%
+ \readformat{#3}%
#5% accumulate all the directives
\yytoksempty\bdirects{%
\bdirects{\quad$\dashv\ \nx\vdash$}%
@@ -554,7 +682,7 @@
\def\bpredicat@#1#2#3{%
\readstash{#3}%
- \appendr\table{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
+ \astappendx{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
\bdirects{}%
}
@@ -578,9 +706,9 @@
\bdirective{prec}{#1}{#2}{#3}%
}
-\def\rules#1#2#3{#1}
+\defc\rules{#1}% a complete right-hand side :: \rules{[\rhs|\rrhssep]}
-\def\oneproduction#1#2#3{#1\setrulecontext{}\stopproduction}
+\defc\oneproduction{#1\setrulecontext{}\stopproduction}% a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}
% to make it possible to typeset tokens depending on their context
% the context name is passed to the token typeseting macros; the most
@@ -599,10 +727,13 @@
% because of this it would be unnatural to define a \prettywordpair sequence for it in the main
% parser namespace; using a special non-\bison\ name should avoid any confusion with
% the regular grammar terms
-\def\emptyterm{\appendr\table{$\nx\circ$\tidxentry{\termexception}{emptyrhs&}}}
+
+\def\emptyrhssymbol{$\circ$}
+
+\defc\emptyterm{\astappendx{\nx\emptyrhssymbol\tidxentry{\termexception}{emptyrhs&}}}
\def\termnameidit#1#2#3#4{%
- \yyreadfifot\yyformat\to#3\in\yyformatlocal\with\formatsort\relax
+ \readformatlocalonly{#3}%
\the\yyformatlocal
\nameproc{#2}\with\parsebin
\peekstash{#4}% does not clobber \toksa and \toksb
@@ -641,7 +772,7 @@
\newif\ifinheader
\def\prodhdridit#1#2#3#4{%
- \setrulecontext{}% TODO: ?
+ \setrulecontext{}% TODO: perhaps some special context for the left hand side terms is appropriate
\nameproc{#2}\with\parsebin
\setrulecontext{#1}%
\edef\next{\toksa{{%
@@ -650,7 +781,7 @@
}}}\next
}
-\def\charit#1#2#3#4{%
+\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
\toksa{#1}\sansfirst\toksa
\edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
@@ -668,12 +799,12 @@
\next
}
-\def\anint#1#2#3{%
- \appendrnx\table{#1}%
+\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
+ \astappend{#1}%
}
-\def\hexint#1#2#3{%
- \expandafter\appendrnx\expandafter\table\expandafter{\eattwo#1$_{16}$}%
+\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
+ \expandafter\astappend\expandafter{\eattwo#1$_{16}$}%
}
\def\optoptsepnone{%
@@ -687,17 +818,17 @@
\def\prodprodsepsmall{%
\pushothertables
- \appendrnx\table{\noalign{\penalty-150 \smallskip}}%
+ \astappend{\noalign{\penalty-150 \smallskip}}%
}
\def\prodprodsep{%
\pushothertables
- \appendrnx\table{\noalign{\medskip}}%
+ \astappend{\noalign{\medskip}}%
}
\def\prodprodseplarge{%
\pushothertables
- \appendrnx\table{\noalign{\bigskip}}%
+ \astappend{\noalign{\bigskip}}%
}
\def\separatorswitcheq{%
@@ -730,7 +861,6 @@
\edef\next{\postoks{\nx\posmark{\the\postoks}{\the\toksd}}}\next
}
\prologuecode\ntermdecls\tokendecls
- \flexsstatedecls\flexxstatedecls\flexoptiondecls
{%
\getthird{\yy(2)}\to\postoks % the format marker
\getfourth{\yy(2)}\to\toksd % the stash marker
@@ -762,7 +892,7 @@
}
\def\posmark#1#2{%
-% \yyreadfifo\yyformat\to#1\in\yyformatlocal
+% \readformat{#1}%
% no formatting for locations currently
% \the\yyformatlocal
\posm@rk{#2}%
@@ -798,7 +928,6 @@
\oneparametricoption\vardef\paramdef\optionflag
\codepropstype\codeassoc\prologuecode
\typedecls\tokendecls\precdecls\oneproduction
- \flexsstatedecls\flexxstatedecls\flexoptiondecls
}
\def\emptyoption{\empty}
@@ -840,15 +969,117 @@
\yytoksempty\prectable{}{\attachprectable}%
% options
\yytoksempty\opttable{}{\attachoptionstable}%
- \concat\table\postoks
+ \astappendx{\the\postoks}%
} %
-% the various \attach... macros above are defined elsewhere and are the macros that
-% determine the actual typesetting of the corresponding tables.
+% typesetting token lists, precedence declarations, etc
+
+\def\attachtokentable{% typesetting the token declarations as part of the grammar
+ \tempcb3 \tempca\tempcb
+ \expandafter\tokdectoks\expandafter{\expandafter}\the\tokdectoks
+ \ifnum\tempca=\tempcb
+ \else
+ \bloop
+ \tokdectoks\expandafter{\the\tokdectoks&\omit\hfil}%
+ \ifnum\tempca=\z@
+ \else
+ \advance\tempca\m@ne
+ \repeat
+ \tokdectoks\expandafter{\the\tokdectoks\cr}%
+ \fi
+ \toksa{\tt\strut##\hfil\tabskip=0pt plus 1fil}%
+ \bloop
+ \toksa\expandafter{\the\toksa&\tt\strut##\hfil}%
+ \ifnum\tempcb=\tw@
+ \else
+ \advance\tempcb\m@ne
+ \repeat
+ \toksa\expandafter{\the\toksa&\tt\strut##\hfil\tabskip\z@\cr}%
+ \concat\toksa\tokdectoks
+ \tokdectoks\toksa
+ \astappendx{\noalign{%
+ \tabskip\parindent
+ %\nx\displaytokenrawtrue % this controls how tokens are displayed
+ % in the declarations: if true, the macro names will be shown
+ \halign to\hsize{\the\tokdectoks}%
+ }%
+ }%
+ \tokdectoks{}%
+}
+
+\def\toksdefline#1#2#3#4#5{% #1 is the parsed version of the internal bison name
+ % #2 is the index entry
+ % #3 is the type
+ % #4 is the explicit value
+ % #5 is the parsed version of the string value
+ \ifnum\tempca=\z@
+ \tokdectoks\expandafter{\the\tokdectoks&{#2}#1\hfil\cr}%
+ \tempca\tempcb
+ \else
+ \ifnum\tempcb=\tempca
+ \tokdectoks\expandafter{\the\tokdectoks{#2}#1\hfil}%
+ \else
+ \tokdectoks\expandafter{\the\tokdectoks&{#2}#1\hfil}%
+ \fi
+ \advance\tempca\m@ne
+ \fi
+}
+
+\def\attachtypestable{% typesetting the type declarations as part of the grammar
+ \toksa{}\the\typestable
+ \astappendx{\noalign{{\the\toksa}}}\typestable{}%
+}
+
+\long\def\onetype#1#2{%
+ \toksb{{%
+ \hbadness\@M
+ \rightskip=\z@
+ \tempda\hsize \advance\tempda-1in
+ \indent{$\langle$\tt union$\rangle.#1$:}\ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
+ \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #2\strut}$}\par
+ }}%
+ \concat\toksa\toksb
+}
+
+\def\attachprectable{% typesetting the precedence declarations as part of the grammar
+ \toksa{}\the\prectable
+ \astappendx{\noalign{{\the\toksa}}}\prectable{}%
+}
+
+\long\def\oneprec#1#2#3{%
+ \toksc{%
+ \hbadness\@M
+ \rightskip=\z@
+ \tempda\hsize \advance\tempda-1in
+ \indent
+ }%
+ \toksd{%
+ \ \hfill\nobreak\hfill\nobreak\null\penalty1\relax\null\nobreak\hfill
+ \hbox to\tempda{$\vtop{\noindent\hsize\tempda\raggedright #3\strut}$}\par
+ }
+ \def\next{#2}%
+ \ifx\next\empty
+ \toksb{}%
+ \else
+ \toksb{${}:{}$#2}%
+ \fi
+ \edef\next{\toksb{{$\nx\langle$\nx\bf #1\the\toksb$\nx\rangle$}}}\next
+ \appendr\toksa{{\the\toksc\the\toksb\the\toksd}}%
+}
+
+\def\attachoptionstable{% typesetting the options as part of the grammar
+ \astappendx{\noalign{%
+ \tabskip\parindent
+ \halign{####\nx\hfil\tabskip\z@&\nx\qquad\nx\it####\nx\hfil\cr
+ \the\opttable
+ }%
+ }%
+ }\opttable{}%
+}
\let\stopproduction\relax
-\def\tokendecls#1#2#3{%
+\defc\tokendecls{% token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:token-declarations}\yyunion
\let\termindex\writetokenidxentry
@@ -856,7 +1087,7 @@
\restorecslist{local-namespace}{\yyunion\termindex}%
}
-\def\typedecls#1#2#3#4{%
+\defc\typedecls{% type declarations :: \typedecls{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:type-declarations}\yyunion
\let\termindex\writetokenidxentry
@@ -865,37 +1096,6 @@
\restorecslist{local-namespace}{\yyunion\termindex}%
}
-% \onetype is a typeseting macro defined elsewhere
-% TODO: move these to the \flex\ section
-
-\def\flexsstatedecls#1#2#3{%
- \savecslist{local-namespace}{\yyunion\termindex}%
- \restorecslist{table-render:flex-state-declarations}\yyunion
- \let\termindex\writeidxfsdentry%\writetokenidxentry
- \typesset{}#1\relax
- \appendr\typestable{\nx\flexsstatelist{\the\typesset}}%
- \restorecslist{local-namespace}{\yyunion\termindex}%
-}
-
-\def\flexxstatedecls#1#2#3{%
- \savecslist{local-namespace}{\yyunion\termindex}%
- \restorecslist{table-render:flex-state-declarations}\yyunion
- \let\termindex\writeidxfsdentry%\writetokenidxentry
- \typesset{}#1\relax
- \appendr\typestable{\nx\flexxstatelist{\the\typesset}}%
- \restorecslist{local-namespace}{\yyunion\termindex}%
-}
-
-% \flexsstatelist and \flexxstatelist are typesetting macros defined elsewhere
-
-\def\flexoptiondecls#1#2#3{%
- \savecslist{local-namespace}{\yyunion\termindex}%
- \restorecslist{table-render:flex-option-definitions}\yyunion
- \let\termindex\writeidxfsrnentry%\writetokenidxentry
- \toksa{}#1\relax
- \restorecslist{local-namespace}{\yyunion\termindex}%
-}
-
\def\precdecls#1#2#3#4#5{% #1 type
% #2 tag
% #3 list of symbols
@@ -911,8 +1111,6 @@
\setrulecontext{}%
}
-% \oneprec is a typesetting macro defined elsewhere
-
% options typesetting
\def\oneparametricoption#1#2#3#4{%
@@ -959,9 +1157,7 @@
}
\def\codepropstype#1#2#3#4#5#6#7{%
- \yyreadfifo\yyformat\to#4\in\yyformatlocal
- \restorecs{table-render}{\strm}%
- \the\yyformatlocal % this can change \codepropstyp@
+ \readformat{#4}\the\yyformatlocal % this can change \codepropstyp@
\codepropstyp@{#2}{#3}{#1}{#5}%
}
@@ -981,9 +1177,7 @@
}
\def\codeassoc#1#2#3#4#5#6#7{%
- \yyreadfifo\yyformat\to#4\in\yyformatlocal
- \restorecs{table-render}{\strm}%
- \the\yyformatlocal % this can change \codeass@c
+ \readformat{#4}\the\yyformatlocal % this can change \codeass@c
\codeass@c{#1}{#2}{#3}{#5}%
}
@@ -1001,9 +1195,7 @@
}
\def\vardef#1#2#3#4{%
- \yyreadfifo\yyformat\to#3\in\yyformatlocal
- \restorecs{table-render}{\strm}%
- \the\yyformatlocal % this can change \vard@f
+ \readformat{#3}\the\yyformatlocal % this can change \vard@f
\vard@f{#1}{#2}{#4}%
}
@@ -1022,7 +1214,7 @@
\appendr\opttable{$\nx\langle${\nx\bf define}$\nx\rangle$\nx\quad{\the\toksb}&\the\toksc\cr}%
}
-\def\prologuecode#1#2#3#4#5{% similar to \actbrac@s
+\defc\prologuecode{% prologue code :: \prologuecode{tex_string}{fptr}{sptr}{fptr}{sptr}
\readstash{#5}%
\appendr\opttable{%
\omit\span\omit\nx\cdotfill\cr
@@ -1032,7 +1224,7 @@
}
\def\yyunion@tablerender{%
- \strm\stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\termnam@\termn@m@\t@rmn@m@
+ \stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\termnam@\termn@m@\t@rmn@m@
\actbrac@s\hspac@\codepropstyp@\vard@f\termmetastyle\peekstash\stopproduction
}
@@ -1040,6 +1232,7 @@
\savecslist{table-render}\yyunion@tablerender
\def\setprodtable{%
+ \def\aststream{bison.rules}% the list where the AST output is gathered
\restorecslist{table-render}\yyunion
\restorecslist{table-render}\yyunion@tablerender
}
@@ -1068,10 +1261,12 @@
\let\rrhss@p\rrhss@p@flatten
\let\arhss@p\relax
\let\rarhss@p\relax
+ \def\flattenactiontrace{}%
}
\def\actbrac@s@flatten#1#2#3{% do everything but add the stash contents to the table
\readstash{#3}%
+ \edef\flattenactiontrace{\flattenactiontrace*}%
\yystashlastcontent\yystashlocal % keep the contents of the braces
% in case we want to show the last action
\futurelet\actsep\actbrac@s@fl@tten
@@ -1079,23 +1274,38 @@
% see the remarks before \emptyterm above for the reasons for choosing the index
% pseudonamespace for this term
+
+\def\inlineactionsymbol{$\diamond$}
+
\def\actbrac@s@fl@tten{%
\ifx\actsep\arhssep
- \appendr\table{$\nx\ \nx\diamond\nx\ $\tidxentry{\termexception}{inline_action&}}%
+ \astappendx{\nx\ \nx\inlineactionsymbol\nx\ \tidxentry{\termexception}{inline_action&}}%
\fi
}
\def\rrhss@p@flatten{%
- \appendrnx\table{$\ \vert\ $}%
+ \astappend{$\ \vert\ $}%
}%
\def\stopproduction@flatten{%
\ifshowlastaction
+ {%
+ \toksa\yystashlastcontent\cleanstash
+ \yytoksempty\toksa{}{% there is a non-empty action
+ \def\next{*}%
+ \ifx\flattenactiontrace\next % if there were more than one action
+ \else % preceeding it, prepend an indicator (ellipsis)
+ \appendlnx\toksa{\hbox{$\ldots\ \vert\ $}{}\ignorespaces}%
+ \fi
+ }%
+ \expandafter
+ }\expandafter\yystashlastcontent\expandafter{\the\toksa}%
\yystashlastcontent\expandafter{\expandafter\toksa\expandafter{\the\yystashlastcontent}}%
\else
\yystashlastcontent{\actionfiller}%
\fi
- \appendr\table{&&\the\yystashlastcontent\nx\cr}%
+ \def\flattenactiontrace{}%
+ \astappendx{&&\the\yystashlastcontent\nx\cr}%
}
% fold: resume normal display
@@ -1117,7 +1327,7 @@
\def\prodhead@r@inline#1#2{%
\let\idit\prodhdridit
#1\relax
- \concat\table\toksa
+ \astappendx{\the\toksa}%
\setbox\z@=\hbox{\strut\the\toksa\/$\,$\rm:\quad}%
\tempda=\wd\z@\relax
\advance\tempda-2em
@@ -1129,7 +1339,7 @@
\def\termspostformat{\ifsquashterms\aftergroup\noexpand\squashtermstrue\fi}
\def\hrhss@p@inline{%
- \appendr\table{&\termspostformat\hbox to \the\tempda{\hss}}%
+ \astappendx{&\termspostformat\hbox to \the\tempda{\hss}}%
}
% breakline: break the production line
@@ -1139,7 +1349,7 @@
}
\def\hspac@@breakline{%
- \appendrnx\table{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
+ \astappend{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
\restorecs{table-render}\hspac@
}
@@ -1157,7 +1367,7 @@
\def\hspac@@breakahead{%
\ifnum\breakaheadcount=\@ne
\yybreak{%
- \appendrnx\table{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
+ \astappend{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
\let\hspac@\bahspac@
}%
\else
@@ -1177,7 +1387,7 @@
\hrhss@p
}
-\def\hrhss@p@skipheader{\appendrnx\table{\omit\hbox to 2em{\hfil}&\termspostformat}\restorecs{table-render}{\hrhss@p}}
+\def\hrhss@p@skipheader{\astappend{\omit\hbox to 2em{\hfil}&\termspostformat}\restorecs{table-render}{\hrhss@p}}
\def\stashtoterm#1{% in case one needs to insert previous stash as a term
% usage: \let\peekstash\stashtoterm
@@ -1193,7 +1403,7 @@
\def\resetf{%
\restorecs{table-render}{\prodheader\prodhead@r\hrhss@p\rrhss@p\arhss@p\rhs\termname\actbraces\actbrac@s\rarhss@p\hspac@\rules
- \oneproduction\emptyterm\stringify\idit\charit\prodprodsep\onesymbol\strm\stopproduction}%
+ \oneproduction\emptyterm\stringify\idit\charit\prodprodsep\onesymbol\stopproduction}%
\squashtermsfalse
}
@@ -1263,26 +1473,26 @@
\restorecslist{parser-strict}\yyunion
-\def\idit#1#2#3#4{%
+\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
\termnameidit{#1}{#2}{#3}{#4}%
\concat\typesset\toksa
}
-\def\stringify#1#2#3#4{%
+\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\termnamestringify{#1}{#2}{#3}{#4}%
\concat\typesset\toksa
}
-\def\hspace#1#2{%
+\defc\hspace{% a space has to take two parameters just like any connector :: \hspace{fptr}{sptr}
\appendrnx\typesset{ }%
}
-\def\charit#1#2#3#4{%
+\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\toksa{\.{#2}}%
\concat\typesset\toksa
}
-\def\tagit#1#2#3#4{%
+\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\toksa{\\{#2}}%
}
@@ -1294,19 +1504,19 @@
\restorecslist{parser-strict}\yyunion
-\def\hspace#1#2{}
+\defc\hspace{}
-\def\idit#1#2#3#4{\toksa{#2}}
+\defc\idit{\toksa{#2}}% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
-\def\charit#1#2#3#4{\toksa{'#1'}}
+\defc\charit{\toksa{'#1'}}% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
-\def\stringify#1#2#3#4{\toksb{#2}}
+\defc\stringify{\toksb{#2}}% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
-\def\anint#1#2#3{\toksc{#1}}
+\defc\anint{\toksc{#1}}% INT :: \anint{digits}{fptr}{sptr}
-\def\hexint#1#2#3{\toksc\expandafter{\eattwo#1$_{16}$}}
+\defc\hexint{\toksc\expandafter{\eattwo#1$_{16}$}}% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
-\def\tagit#1#2#3#4{%
+\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\def\currenttokentype{#2}%
}
@@ -1323,9 +1533,9 @@
\edef\next{\nx\onesymb@l{\the\toksa}{\the\toksc}{\the\toksb}}\next
}
-\def\onesymb@l#1#2#3{% #1 is the macro name used internally by bison
- % #2 is the integer value
- % #3 is the string value
+\def\onesymb@l#1#2#3{% #1 is the macro name used internally by bison
+ % #2 is the integer value
+ % #3 is the string value
\let\optstrextra\optstrextraesc
\tokse{}\toksf{}\toksg{}% these registers will contain the appropriate visual keys
% and index entry processing control sequences
@@ -1362,17 +1572,17 @@
\restorecslist{parser-strict}\yyunion
-\def\idit#1#2#3#4{%
+\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
\termnameidit{#1}{#2}{#3}{#4}%
\concat\typesset\toksa
}
-\def\stringify#1#2#3#4{%
+\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\termnamestringify{#1}{#2}{#3}{#4}%
\concat\typesset\toksa
}
-\def\charit#1#2#3#4{%
+\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
\toksa{#1}\sansfirst\toksa
\edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
@@ -1391,20 +1601,20 @@
\concat\typesset\toksa
}
-\def\anint#1#2#3{%
+\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
\appendrnx\typesset{${}=\hbox{\ntt#1}$}%
}
-\def\hexint#1#2#3{%
+\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
\toksa\expandafter{\eattwo#1}%
\appendr\typesset{${}=\hbox{\ntt\the\toksa}_{16}$}%
}
-\def\hspace#1#2{%
+\defc\hspace{%
\appendrnx\typesset{ }%
}
-\def\tagit#1#2#3#4{%
+\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\toksa{\\{#2}}%
}
@@ -1418,9 +1628,9 @@
\restorecslist{parser-strict}\yyunion
-\def\idit#1#2#3#4{\toksa{#2}}
+\defc\idit{\toksa{#2}}% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
-\def\stringify#1#2#3#4{\toksa{#2}}
+\defc\stringify{\toksa{#2}}% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\def\bracedvalue#1#2#3{% similar to \actbrac@s
\readstash{#3}%
@@ -1433,79 +1643,33 @@
\restorecslist{parser-strict}\yyunion
-\def\idit#1#2#3#4{%
+\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
\termnameidit{#1}{#2}{#3}{#4}%
}
-\def\stringify#1#2#3#4{%
+\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\termnamestringify{#1}{#2}{#3}{#4}%
}
-\def\tagit#1#2#3#4{%
+\defc\tagit{% *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\toksa{\\{#2}}%
}
-\def\anint#1#2#3{%
+\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
\toksa{#1}%
}
-\def\hexint#1#2#3{%
+\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
\toksa\expandafter{\eattwo#1$_{16}$}%
}
\toyyunion{table-render:directives}
-% flex state declarations
-
-\restorecslist{parser-strict}\yyunion
-
-\def\idit#1#2#3#4{%
- \nameproc{#2}\with\parsebin
- \appendr\typesset{{%
- \gidxentryxv{\termttstring}{#1}{\the\toksb}%
- \ntt\the\toksa
- }}%
-}
-
-\def\hspace#1#2{%
- \appendr\typesset{ }%
-}
-
-\toyyunion{table-render:flex-state-declarations}
-
-% flex option definitions
-
-\restorecslist{parser-strict}\yyunion
-
-\def\flexoptionpair#1#2{%
- #1%
- \yystringempty{#2}{% there is no right hand side
- \appendr\opttable{$\nx\langle${\ntt\the\toksa}$\nx\rangle_{\rm f}\,\star$&\cr}%
- }{%
- \toksb\toksa\toksa{}#2%
- \appendr\opttable{$\nx\langle${\ntt\the\toksb}$\nx\rangle_{\rm f}$&\nx\it\the\toksa\nx\/\nx\cr}%
- }%
-}
-
-\def\idit#1#2#3#4{%
- \toksa{#2}%
- \edef\next{\toksa{{%
- \gidxentry{\termfsopstring}{#2}{}%
- \ntt\the\toksa
- }}}\next
-}
-
-\def\stringify#1#2#3#4{\toksa{{\ntt@"#2"}}}
-
-\def\hspace#1#2{}
-
-\toyyunion{table-render:flex-option-definitions}
-
% rule listing macros
\restorecslist{parser-prototypes}\yyunion % we just want to ignore the non-rule related code
-\def\prodheader#1#2#3#4{%
+\defc\prodheader{% a `left-hand side' of a production :: \prodheader{\idit}{\idit}{fptr}{sptr}
#1\relax
\lastlastname\lastname
\setnumname\numname
@@ -1515,14 +1679,14 @@
}
% the macros below are defined in `parser-prototypes'
-%\def\rrhssep#1#2{}
-%\def\arhssep#1#2{}
-%\def\rarhssep#1#2{}
-%\def\hspace#1#2{}
-%\def\prodprodsep{}
-%\def\onesymbol#1#2#3{}
-
-\def\rhs#1#2#3{%
+%\defp\rrhssep#1#2{}
+%\defp\arhssep#1#2{}
+%\defp\rarhssep#1#2{}
+%\defp\hspace#1#2{}
+%\defp\prodprodsep{}
+%\defp\onesymbol#1#2#3{}
+
+\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
\concat\toksa\toksc
\concat\toksb\toksd
#1\relax
@@ -1536,7 +1700,7 @@
\newtoks\lastname
\newtoks\lastlastname
-\def\termname#1#2{%
+\defc\termname{% a production term :: \termname{\idit|\stringify|\charit}{\idit}
\flushact
#1\relax
\lastlastname\lastname
@@ -1551,7 +1715,7 @@
\ifx\next\relax % this is a terminal that was renamed
\expandafter\ifx\csname token\parsernamespace\the\lastname \endcsname\relax % this is probably a symbol from another grammar
\def#1{-1}%
- \errmessage{token \csname token\parsernamespace\the\lastname \endcsname\space is undefined}%
+ \errmessage{token \the\lastname\space is undefined}%
\else
\expandafter\tempca\csname token\parsernamespace\the\lastname \endcsname
\edef#1{\fgetelemof{yytranslate}\at\tempca}%
@@ -1561,7 +1725,7 @@
\fi
}
-\def\actbraces#1#2#3#4#5\bdend{%
+\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
\flushact
\lastname{}%
#2\relax
@@ -1569,32 +1733,32 @@
\possibleimplicittrue
}
-\def\rules#1#2#3{#1}
+\defc\rules{#1}% a complete right-hand side :: \rules{[\rhs|\rrhssep]}
-\def\oneproduction#1#2#3{#1}
+\defc\oneproduction{#1}% a complete production :: \oneproduction{\prodheader\rules}{fptr}{sptr}
-\def\emptyterm{%
+\defc\emptyterm{%
\flushact
\lastname{}%
}
-\def\stringify#1#2#3#4{%
+\defc\stringify{% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
\lastname{"#1"}%
}
-\def\idit#1#2#3#4{%
+\defc\idit{% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
\lastname{#1}%
}
-\def\charit#1#2#3#4{%
+\defc\charit{% *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\lastname{'#1'}%
}
-\def\anint#1#2#3{%
+\defc\anint{% INT :: \anint{digits}{fptr}{sptr}
\lastname{#1}%
}
-\def\hexint#1#2#3{%
+\defc\hexint{% INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
\lastname\expandafter{\expandafter\number\eattwo#1$_{16}$}%
}
@@ -1626,6 +1790,8 @@
\def\prettywordpair#1#2{\prettywordpair@@{#1}{{\ntt@#2}}}
+\newif\ifdisplaytokenraw
+
\def\prettywordpair@#1#2{% a `weak symbol' version of the above macro, so that the new typesetting can be overriden
\setsafe{#1}\toksa{}\numberstocharsandspaces#2\end\expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
@@ -1686,11 +1852,11 @@
\restorecslist{parser-prototypes}\yyunion
-\def\tokendecls#1#2#3{#1}
+\defc\tokendecls{#1}% token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
-\def\stringify#1#2#3#4{\toksa{"#1"}}
+\defc\stringify{\toksa{"#1"}}% *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
-\def\idit#1#2#3#4{\toksb{#1}}
+\defc\idit{\toksb{#1}}% *ID :: \idit{12string}{tex_string}{fptr}{sptr}
%\def\anint#1#2#3{\toksc{#1}}% this definition is not needed as all
% the information is extracted from yytname
@@ -1698,7 +1864,7 @@
% note that the definitions below do not handle cases such as
%token '{' "left brace" since the lexer knows how to return '{'
-\def\onesymbol#1#2#3{%
+\defc\onesymbol{%#1#2#3 % a symbol definition :: \onesymbol{\idit}{\anint|\hexint}{\stringify}
\toksa{}\toksb{}%\toksc{}%
#1#2#3%
\yytoksempty\toksa{\expandafter\charstonumbers\the\toksb\end}%
@@ -1712,24 +1878,27 @@
}%
}
-\def\flexsstatedecls#1#2#3{% bootstrap parser does not produce state declarations so it is fine to share states
- \savecslist{local-namespace}\yyunion
- \restorecslist{bootstrap:flex-state-declarations}\yyunion
- #1\relax
- \restorecslist{local-namespace}\yyunion
-}
-
-\let\flexxstatedecls\flexsstatedecls
-
-\savecslist{bootstrap}\yyunion
-
-\restorecslist{parser-strict}\yyunion
+% the original, minimal bootstrapping macros were designed to process
+% \prodstyle{\%token} declarations only and are enough to establish the
+% interface between the \bison\ parser and the \bison\ lexer; to serve
+% the secondary task of providing typesetting information to
+% the \bison\ parser, all forms of token declarations must be processed;
+%
+% note that the bootstrap parser cannot process these additional declarations
+% so these macros are designed to work with a full or prologue parser
+% in a bootstrap mode; see \.{\\modebootstrap} in ldman.w and
+% its use in \.{ldlex.w} and \.{ldgram.w} for an example of such
+% an arrangement.
-\def\idit#1#2#3#4{{\immediate\write\stlist{_register_name(#1);}}}
+\def\precdecls#1#2#3#4#5{#3}
-\def\hspace#1#2{}
+\def\symbolprec#1#2{%
+ \toksa{}\toksb{}%
+ #1%
+ \yytoksempty\toksb{}{\immediate\write\tokendefs{\noexpand\tokenpp{\the\toksb}}}%
+}
-\savecslist{bootstrap:flex-state-declarations}\yyunion
+\savecslist{bootstrap}\yyunion
% correct a few macros for debugging
@@ -1872,7 +2041,7 @@
\def\ignoreexplicitspace{\yyinput}
% the main name processing routine
-% does not have any side effects;
+% does not have any side effects (other than the ones explicitly produced by the final call of #2);
% the parsed term is returned as #1 of the sequence in #2; #2 of #2 will hold a visual key if
% one exists, otherwise it will be empty; #3 is \yyparsefailtrue or empty
% #4 is the prettyfied version of the term if one exists; #5 is the typesetting hint from \visflag sequences;
@@ -2013,12 +2182,24 @@
% \flex\ regex typesetting routine and \flex\ parser value stack types
+\def\flrecharadjust{
+ `{%
+ \yybyte{|}%
+ \expandafter\yycp@\expandafter`\the\yybyte\relax
+ \mkpurebyte
+ \yyreturn
+ }%
+}
+
+% no need for \setspecialcharsfrom\flrecharadjust
+
\def\frexproc#1\with#2{%
{% keeping all changes local
\iftracenames
\toksa{#1}\ferrmessage{original regex: \the\toksa}%
\fi
\toflexreparser
+ \let\acharswitch\flrecharadjust
\basicparserinit
\flexreparserinit
\flexreparserdatainit
@@ -2054,11 +2235,13 @@
\toksc{#2}%
}
-\def\setregextable{%
+\def\setregextable{% activate the appropriate AST control sequences
+ \def\aststream{flex.sect2}% the list where the AST output is gathered
\restorecslist{flexparser-sect2}\yyflunion
}
\def\setregexdeftable{%
+ \def\aststream{flex.sect1}% the list where the AST output is gathered
\restorecslist{flexparser-sect1}\yyflunion
}
@@ -2113,14 +2296,14 @@
\defp\flbracecclneg#1#2#3{} % negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
\defp\flcclunion#1#2{} % union of character classes :: \flcclunion{ccl}{ccl}
\defp\flccldiff#1#2{} % difference of character classes :: \flccldiff{ccl}{ccl}
-\defp\flrepeat#1{} % asterisk operation :: \flrepeat{re}
-\defp\flrepeatstrict#1{} % plus operation :: \flrepeatstrict{re}
+\defp\flrepeat#1{} % * (asterisk) operation :: \flrepeat{re}
+\defp\flrepeatstrict#1{} % + (plus) operation :: \flrepeatstrict{re}
\defp\flrepeatonce#1{} % ? operation :: \flrepeatonce{re}
\defp\flrepeatnm#1#2#3{} % {n, m} operation :: \flrepeatnm{re}{num}{num}
\defp\flrepeatgen#1#2{} % {n,} operation :: \flrepeatgen{re}{num}
\defp\flrepeatn#1#2{} % {n} operation :: \flrepeatn{re}{num}
\defp\flparens#1#2#3#4{} % parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
-\defp\fldot#1#2{} % dot expression :: \fldot{fptr}{sptr}
+\defp\fldot#1#2{} % . (dot) expression :: \fldot{fptr}{sptr}
\defp\flor#1#2{} % `|' expression :: \flor{fptr}{sptr}
\defp\fltrail#1#2{} % regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
\defp\flretrail#1#2#3{} % regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
@@ -2397,10 +2580,13 @@ x
}
\defc\fltrail{%#1#2% regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
- \errmessage{\nx\fltrail\space macro has not been implemented (see yyunion.sty).}
+ \errmessage{\nx\fltrail\space macro has should not be expanded.}
}
\defc\flretrail{%#1#2#3% regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
+ % #1 is the regular expression to be matched
+ % #2 is the pointer to the context marker (/)
+ % #3 is the context
\errmessage{\nx\flretrail\space macro has not been implemented (see yyunion.sty).}
}
@@ -2412,7 +2598,7 @@ x
\toyyunion{flexparser-re}
\def\flexsndisplay#1{% state name typesetting style
- \hbox{\tt#1}%
+ {\let\ntt@\relax\idss#1}%
}
\def\flexrendisplay#1{% regular expression name typesetting style
@@ -2429,7 +2615,7 @@ x
% the parsed name goes to \toksd and the visual key is put in \tokse
\let\termindex\writeidxfstextentry
\edef\next{\toksd{\nx\flexsndisplay{\the\toksd}%
- \gidxentryxv{\termvstring}{#1}{\the\tokse}}}\next
+ \gidxentryxv{\termfsscstring}{#1}{\the\tokse}}}\next
\expandafter
}\the\toksd
}
@@ -2458,16 +2644,18 @@ x
\hbox{\textretrue\tt\the\toksc}%
}
+% \flex\ section~2 typesetting
+
\defc\flactiongroup{% #1#2#3#4#5#6% a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
\toksb{}#1% collect all the state name information in \toksb
% gather the preceeding stash
- \readstash{#3}% read stash late, after \flsconslist (ignore it)
+ \readstash{#3}% read stash late, after \flsconlist (ignore it)
\edef\next{%
\toksb{\noalign{\hbox to\nx\hsize{\hskip\the\rgindent
- \nx\flexsndisplay{\the\toksb}:\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil}}}%
+ \nx\flexsndisplay{\the\toksb$^{++}$}\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil}}}%
}\next
- \yytoksempty\table{}{\appendrnx\table{\noalign{\medskip}}}%
- \concat\table\toksb
+ \astisempty{}{\astappend{\noalign{\medskip}}}%
+ \astappendx{\the\toksb}%
\advance\rgindent by\parindent
#4%
\advance\rgindent by-\parindent
@@ -2475,84 +2663,75 @@ x
\defc\flsconlist{% #1#2#3% start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
\readstashwredirect#1\toksg % save the stash preceeding the conditions list to reinsert later
+ \savecslist{local-namespace}\yyflunion
+ \restorecslist{flexparser-sect2:state-name-display}\yyflunion
#2% collect the state names (and the stash in between)
+ \restorecslist{local-namespace}\yyflunion
\readstashwredirect#3\toksf
\appendr\toksb{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
}
-\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
- {%
- \let\parsernamespace\flexpseudonamespace
- \nameproc{#2}\with\parsebin
- \edef\next{%
- \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
- }\next % a trick to `reshuffle' the output of \nameproc:
- % the parsed name goes to \toksd and the visual key is put in \tokse
- \expandafter
- }\the\toksd
- \let\termindex\writeidxfsentry
- \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
- \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
- \the\toksd\gidxentryxv{\termvstring}{#1}{\the\tokse}}%
- \let\termindex\eatone
-}
-
-\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
- \toksb\expandafter{\the\toksb\ }%
-}
-
\newskip\rgindent
-
\rgindent=\parindent
-\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
- \toksb{}#1%
+\def\appendinteractionskip{%
+ \ifnum\prevdepth<\dp\strutbox
+ \vskip-\prevdepth\vskip\dp\strutbox
+ \fi
+ \nointerlineskip
+}
+
+\def\flaction@generic#1#2#3#4#5{% #1 is the #1 of \flaction.*
+ % #2 is the #2 of \flaction.*
+ % #3 is the #6 of \flaction.*
+ % #4 is the #7 of \flaction.*
+ % #5 is the \toksa prefix (should be expandable)
+ \toksb{}#1% collect all the state name information in \toksb
\toksc{}#2%
- #7% apply formatting
- \readstash{#6}% read stash late, after \flsconlist and \fl...rule
- \yytoksempty\toksb{%
- \appendr\table{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
- }{%
- \appendr\table{%
+ #4% apply formatting
+ \readstash{#3}% read stash late, after \flsconlist and \fl...rule
+ \yytoksempty\toksb{% no state names were collected
+ \astappendx{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
+ }{% put state names first
+ \astisempty{}{\astappend{\noalign{\medskip}}}%
+ \astappendx{%
+ \noalign{\nx\appendinteractionskip}%
\hskip\the\rgindent
\vbox{%
- {\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
+ %{\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
+ % the line above would allow a smaller vertical skip to be inserted in case
+ % the font for state names is reduced in size;
\halign{&####\hfil\cr
- \hbox to\rgindent{\nx\sscmd\nx\tt\nx\strut\the\toksb\hss}\cr
- \nx\tt\nx\strut\the\toksc\cr
+ \flexstatelisttypeset\cr
+ \hskip\parindent\nx\tt\nx\strut\the\toksc\cr
}%
}%
}%
}%
- \appendr\table{&\toksa{\relax\the\yystashlocal}\nx\cr}%
+ \astappendx{&\toksa{#5\relax\the\yystashlocal}\nx\cr}%
}
-\defc\flactionc{%#1#2#3#4#5#6#7% \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
- \toksb{}#1%
- \toksc{}#2%
- \readstash{#6}% read stash late, after \flsconlist and \fl...rule
- #7% apply formatting
- \yytoksempty\toksb{%
- \appendr\table{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
- }{%
- \appendr\table{%
- \hskip\the\rgindent
- \vbox{%
- {\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
- \halign{&####\hfil\cr
- \hbox to\rgindent{\nx\sscmd\nx\tt\nx\strut\the\toksb\hss}\cr
- \nx\tt\nx\strut\the\toksc\cr
- }%
- }%
- }%
+\def\flexstatelisttypeset{%
+ \hbox to\rgindent{%
+ \vbox{\advance\hsize by -\rgindent
+ \noindent
+ \nx\strut\nx\flexsndisplay{\the\toksb$^+$}
+ }\hss
}%
- \appendr\table{&\toksa{{}$\nx\hookleftarrow$\relax\the\yystashlocal}\nx\cr}% in the event \cr has been redefined
+}
+
+\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+ \flaction@generic{#1}{#2}{#6}{#7}{}%
+}
+
+\defc\flactionc{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+ \flaction@generic{#1}{#2}{#6}{#7}{{}$\nx\hookleftarrow$}%
}
\def\inscomment#1{%
\def\astformat@flaction{%
\toksd{#1}%
- \appendr\table{\noalign{\nx\smallskip\noindent\hskip\the\rgindent
+ \astappendx{\noalign{\nx\smallskip\noindent\hskip\the\rgindent
$\nx\triangleright\;$\the\toksd\nx\smallskip}}%
}%
}
@@ -2561,9 +2740,34 @@ x
\toyyunion{flexparser-sect2}
-% \flex\ setion~1 typesetting
-% currently, all \flex\state declarations are handled by an extended \bison\ parser
-% to collect the state data; we thus only implement regular expression definitions;
+% flex state names display inside action groups
+
+\restorecslist{flexparser-strict}\yyflunion
+
+\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
+ {%
+ \let\parsernamespace\flexpseudonamespace
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \expandafter
+ }\the\toksd
+ \let\termindex\writeidxfsentry
+ \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
+ \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
+ \the\toksd\gidxentryxv{\termfsscstring}{#1}{\the\tokse}}%
+ \let\termindex\eatone
+}
+
+\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
+ \toksb\expandafter{\the\toksb\ }%
+}
+
+\toyyunion{flexparser-sect2:state-name-display}
+
+% \flex\ section~1 typesetting
\restorecslist{flexparser-strict}\yyflunion
@@ -2579,24 +2783,90 @@ x
\expandafter
}\the\toksd
\frexproc{#5}\with\ppregex
- \appendr\table{{}$\nx\langle\hbox{\ntt\the\toksd}\nx\rangle${}%
- \gidxentryxv{\termfsrestring}{#1}{\the\tokse}&\the\toksc\cr% TODO: make a special type processor for regex names
+ \astappendx{{}$\nx\langle\hbox{\ntt\the\toksd}\nx\rangle${}%
+ \gidxentryxv{\termfsrestring}{#1}{\the\tokse}&\the\toksc\cr
}%
\let\termindex\eatone
}
-\defc\flopt{%#1#2#3#4#5% \flex option :: \flopt{option}{text}{text12}{fptr}{sptr}
- \csname flex@ption_#1\endcsname{#2}{#3}%
+% \flex\ options
+
+\defc\floptions{%#1 \flex options :: \floptions{\flopt ...}
+ #1%
+}
+
+\defc\flopt{%#1#2#3#4#5% \flex option :: \flopt{option}{text:(no||opt_name)}{text12}{fptr}{sptr}
+ \expandafter\ifx\csname flex@ption_#1\endcsname\relax
+ \yybreak{%
+ \errmessage{!option <#1> with value (#3) is not handled.}%
+ }%
+ \else
+ \yybreak{%
+ \csname flex@ption_#1\endcsname{#2}{#3}{#4}{#5}%
+ }%
+ \yycontinue
+}
+
+% specific options
+\expandafter\def\csname flex@ption_other\endcsname#1#2#3#4{%
+ \let\termindex\writeidxfsrnentry
+ \storerawidxentry{\tt{\rm(}#1{\rm)}#2}%
+ \astappendx{$\nx\langle${\ntt option}$\nx\rangle_{\nx\rm f}$%
+ \gidxentryxv{\termfsopstring}{option}{option}%
+ \yystringempty{#1}{%
+ \gidxentry{\termttstring}{#2}{#2}% index the <option> keyword
+ }{%
+ \didxentry{\getstackpointer\indexterms}{#2}% index the parameter
+ }%
+ &\nx\quad{\ntt#1#2}\nx\cr}%
+ \let\termindex\empty
+}
+
+\expandafter\def\csname flex@ption_file\endcsname#1#2#3#4{%
+ \let\termindex\writeidxfsrnentry
+ \astappendx{$\nx\langle${\ntt output to}$\nx\rangle_{\nx\rm f}$%
+ \gidxentry{\termfsopstring}{output to}{output to}%
+ \gidxentryxv{\termttstring}{#2}{\eatone#2}% remove the first quote for sorting
+ &\nx\quad{\ntt#1}\nx\cr}%
+ \let\termindex\empty
}
-\expandafter\def\csname flex@ption_deprecated\endcsname#1#2{%
+\expandafter\def\csname flex@ption_header\endcsname#1#2#3#4{%
+ \let\termindex\writeidxfsrnentry
+ \astappendx{$\nx\langle${\ntt header file}$\nx\rangle_{\nx\rm f}$%
+ \gidxentry{\termfsopstring}{header}{header}%
+ \gidxentryxv{\termttstring}{#2}{\eatone#2}% remove the first quote for sorting
+ &\nx\quad{\ntt#1}\nx\cr}%
+ \let\termindex\empty
+}
+
+\expandafter\def\csname flex@ption_prefix\endcsname#1#2#3#4{%
+ \let\termindex\writeidxfsrnentry
+ \astappendx{$\nx\langle${\ntt prefix}$\nx\rangle_{\nx\rm f}$%
+ \gidxentry{\termfsopstring}{prefix}{prefix}%
+ \gidxentry{\termttstring}{#2}{#2}%
+ &\nx\quad{\ntt#1}\nx\cr}%
+ \let\termindex\empty
+}
+
+% the option below processes the parameter in C mode
+\expandafter\def\csname flex@ption_xtype\endcsname#1#2#3#4{%
+ \readstash{#4}%
+ \let\termindex\writeidxfsrnentry
+ \astappendx{$\nx\langle${\ntt extra type}$\nx\rangle_{\nx\rm f}$%
+ \gidxentry{\termfsopstring}{extra type}{extra type}%
+ &\nx\quad{\toksa{\the\yystashlocal}\nx\makestashbox}\nx\cr}%
+ \let\termindex\empty
+}
+
+\expandafter\def\csname flex@ption_deprecated\endcsname#1#2#3#4{%
\flex@opt@deprecated#2%
}
\def\flex@opt@deprecated#1#2#3^^J{%
\let\termindex\writeidxfsrnentry
- \appendr\table{$\langle${\nx\tt #2}$\rangle_{\rm f}$\gidxentryxv{\termfsopstring}{#2}{#2}%
- \the\toksg&\number\flexgetnumber#3.\nx\cr}%
+ \astappendx{$\nx\langle${\ntt #2}$\nx\rangle_{\nx\rm f}$\gidxentry{\termfsopstring}{#2}{#2}%
+ &\number\flexgetnumber#3.\nx\cr}%
\let\termindex\empty
}
@@ -2620,8 +2890,49 @@ x
\yycontinue
}
+\defc\flscondecl{%#1#2#3#4% start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
+ \readstashwredirect{#2}{#3}\toksg % save the stash preceeding the conditions list to reinsert later
+ \savecslist{local-namespace}\yyflunion
+ \restorecslist{flexparser-sect1:state-name-display}\yyflunion
+ \toksb{}#4% collect the state names (and the stash in between) in \toksb
+ \restorecslist{local-namespace}\yyflunion
+ %\readstashwredirect#3\toksf
+ %\appendr\toksb{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
+ \let\termindex\writeidxfsentry
+ \astappendx{$\langle${\bf state-#1}$\rangle_{\rm f}$%
+ \gidxentryxv{\termfsopstring}{state-#1}{state-#1}%
+ \nx\quad\nx\flexsndisplay{\the\toksb}\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil&\nx\cr}%
+}
+
\toyyunion{flexparser-sect1}
+% inside state name declarations
+
+\restorecslist{flexparser-strict}\yyflunion
+
+\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
+ {%
+ \let\parsernamespace\flexpseudonamespace
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \expandafter
+ }\the\toksd
+ \let\termindex\writeidxfsdentry
+ \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
+ \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
+ \the\toksd\gidxentryxv{\termfsscstring}{#1}{\the\tokse}}%
+ \let\termindex\eatone
+}
+
+\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
+ \toksb\expandafter{\the\toksb\ }%
+}
+
+\toyyunion{flexparser-sect1:state-name-display}
+
% correct a few sequences for debugging purposes
\restorecslist{flexparser-debug}\yyflunion
@@ -2651,3 +2962,286 @@ x
}
\toyyunion{flexparser-debug}
+
+\restorecslist{flexparser-prototypes}\yyflunion
+
+\defc\flscondecl{%#1#2#3#4% start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
+ #4%
+}
+
+\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
+ {\immediate\write\stlist{_register_name(#2);}}%
+}
+
+\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
+}
+
+\toyyunion{bootstrap-flex}
+
+% remembering the rule associated with a specific section reference
+
+\newtoks\currentheader
+\newtoks\currentterm
+\newtoks\singlerule
+\newtoks\ruleset
+\newcount\refrulecount
+
+\restorecslist{parser-prototypes}\yyunion % we just want to ignore the non-rule related code
+\def\parserprototypesnamespace{parser-strict:headers} % make sure the prototypes come from the correct namespace
+
+\defc\prodheader{#1\currentheader\currentterm}
+
+% the macros below are defined in `parser-prototypes'
+%\defp\rrhssep#1#2{}
+%\defp\arhssep#1#2{}
+%\defp\rarhssep#1#2{}
+%\defp\prodprodsep{}
+%\defp\onesymbol#1#2#3{}
+
+\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
+ \singlerule{}#1%
+ \savecs{local-namespace}{\_}%
+ \let\_\uscoreletter
+ \saveruleforheader
+ \restorecs{local-namespace}{\_}%
+}
+
+\defc\termname{% a production term :: \termname{\idit|\stringify|\charit}{\idit}
+ #1\appendr\singlerule{\the\currentterm}\yystringempty{#2}{}{#2\appendr\singlerule{[\the\currentterm] }}%
+}
+
+\defc\actbraces{%
+ \readformat{#3}%
+ \actbraces@refinit
+ \readstash{#4}%
+ \savecs{local-namespace}{\format\formatlocal}%
+ \let\format\formatreinsert
+ \let\formatlocal\formatreinsertlocal
+ \the\yyformatlocal
+ \appendr\singlerule{ \nx\{\nx\stashed{\the\yystashlocal}\nx\} }%
+ \expandafter\lookforX\the\yystashlocal\X:\X\end
+ \expandafter\lookforXmark\the\yystashlocal\Xmark:\Xmark\end
+ \restorecs{local-namespace}{\format\formatlocal}%
+}
+
+\let\actbraces@refinit\empty
+
+% sometimes all format hints have to be erased (an example would be when collecting
+% productions marked by \Xmark and omitting an \inline while keeping a \fold)
+\newif\iferaselocalformat
+\newif\iferaseformat
+
+\def\formatreinsert#1{\iferaseformat\else\appendrnx\singlerule{\format{#1}}\fi}
+\def\formatreinsertlocal#1{\iferaselocalformat\else\appendrnx\singlerule{\formatlocal{#1}}\fi}
+
+% macros below (normally) package each production into an individual rule, complete with the header;
+% there are two ways to avoid this (i.e. to package the whole production set for a single lhs):
+% o using the header term for the label (in this case every production for the given lhs will
+% be collected in the appropriate command sequence, see \saveruleforheader below)
+% o using the \Xmark label with the empty name part (e.g. \Xmark label:\Xmark) to mark the
+% productions one wishes to collect for this term; note that the \Xmark mechanism does not
+% verify that the given production is for the same lhs
+
+\def\lookforX#1\X#2:#3\X#4\end{%
+ \yystringempty{#2}{}{%
+ \expandafter\ifx\csname prodrulefor[#2]\endcsname\relax
+ \yybreak{%
+ \expandafter\xdef\csname prodrulefor[#2]\endcsname{\the\currentheader: }%
+ \addtoprodrulefor{#2}{\the\singlerule}%
+ }%
+ \else
+ \yybreak{%
+ \addtoprodrulefor{#2}{\the\currentheader: }%
+ \addtoprodrulefor{#2}{\the\singlerule}%
+ }%
+ \yycontinue
+ \lookforX#4\end
+ }%
+}
+
+\def\lookforXmark#1\Xmark#2:#3\Xmark#4\end{%
+ \yystringempty{#2}{}{%
+ \expandafter\ifx\csname prodrulefor[#2]\endcsname\relax
+ \yybreak{%
+ \expandafter\xdef\csname prodrulefor[#2]\endcsname{\the\currentheader: }%
+ \addtoprodrulefor{#2}{\the\singlerule}%
+ }%
+ \else
+ \yybreak{%
+ \yystringempty{#3}{%
+ \addtoprodrulefor{#2}{|}%
+ }{%
+ \addtoprodrulefor{#2}{\the\currentheader: }%
+ }%
+ \addtoprodrulefor{#2}{\the\singlerule}%
+ }%
+ \yycontinue
+ \lookforXmark#4\end
+ }%
+}
+
+\def\saveruleforheader{%
+ \expandafter\ifx\csname prodrulefor[\the\currentheader]\endcsname\relax
+ \yybreak{%
+ \expandafter\xdef\csname prodrulefor[\the\currentheader]\endcsname{\the\currentheader: \the\singlerule}%
+ }%
+ \else
+ \yybreak{%
+ \addtoprodrulefor{\the\currentheader}{\expandafter|\the\singlerule}%
+ }%
+ \yycontinue
+}
+
+\def\addtoprodrulefor#1#2{%
+ \appendxrt{\gdef}{prodrulefor[#1]}{#2}{}%
+}
+
+\def\mergeformatting#1{%
+ \readformat{#1}%
+ \savecs{local-namespace}{\format\formatlocal}%
+ \let\format\formatreinsert
+ \let\formatlocal\formatreinsertlocal
+ \the\yyformatlocal
+ \restorecs{local-namespace}{\format\formatlocal}%
+}
+
+\defc\rules{#1}
+\defc\oneproduction{#1}
+\defc\emptyterm{\appendr\singlerule{ }}
+\defc\hspace{\appendr\singlerule{ }}
+\defc\stringify{\mergeformatting{#3}\currentterm{"#2"}}
+\defc\idit{\mergeformatting{#3}\currentterm{#2}}
+\defc\charit{\mergeformatting{#3}\currentterm{'#2'}}
+\defc\anint{\mergeformatting{#3}\currentterm{#1}}
+\defc\hexint{\mergeformatting{#3}\currentterm{#1}}
+
+\savecslist{reference:delayed}\yyunion
+
+% using the macros above
+
+\def\extractprodrefs{% this is the command sequence that performs the preprocessing of the
+ % parsed table to set the rule references
+ \restorecslist{reference:delayed}\yyunion
+ \savecs{local-names}{\*}\let\*\empty % in case the name of the section comes from a change file
+ \the\table
+ \restorecs{local-names}{\*}%
+ \resetlistpointer\yystash
+ \resetlistpointer\yyformat
+}
+
+\def\PB@#1{\PB@@#1\end}
+\def\PB@@$#1$\end{#1}
+\def\X@#1:#2\X{#1}
+\def\Xmark#1\Xmark{}
+
+\def\rulerefrequests{} % here we collect all the reference `requests' for the rules
+ % (parts of) which may be defined later in the file
+
+% referencing previously processed rules:
+% #1 is additional setup code (such as \showlastactiontrue)
+% #2 is formatting commands (typical use is to make it \nx\flatten)
+% #3 is a command sequence that expands to the desired section number
+% (the macros just above make is possible to use |@<Section name@>|
+% for that)
+
+\def\rulereferencex#1#2#3{%
+ \begingroup
+ \savecs{local-names}{\PB\X\Xmark}%
+ \let\PB\PB@ \let\X\X@ \let\Xmark\X@
+ \let\textproductionsetup\rulereftextproductionsetup
+ {%
+ \toksa\expandafter{\rulerefrequests}%
+ \edef\rulerefrequests{\the\toksa{#3}}%
+ \expandafter
+ }\expandafter\gdef\expandafter\rulerefrequests\expandafter{\rulerefrequests}%
+ \expandafter\ifx\csname full.prodrulefor[#3]\endcsname\relax
+ \else
+ \expandafter\expandafter\expandafter
+ \singlerule\expandafter\expandafter
+ \expandafter{\csname full.prodrulefor[#3]\endcsname}%
+ \restorecs{local-names}{\PB\X\Xmark}%
+ #1%
+ \edef\next{\nx\beginprod#2\the\singlerule\nx\endprod}\next
+ \fi
+ \endgroup
+}
+
+\def\rulereftextproductionsetup{%
+ \ninepoint
+ \let\returnexplicitspace\splitexplicitspace
+ %\let\acharswitch\texcharadjust
+ %\let\onecharswitch\texcsadjust% we do not need the adjustments
+ %since the names have already gone through the lexer
+ \let\extractprodtableinfo\empty % we do not prepocess the table
+ \let\actionfiller\empty
+ \fillpstack{b}{%
+ \preparsebisongrammar
+ \preparsebisonprologue
+ {\preparsefallbacktext{**}}%
+ \relax % this \relax is necessary so that the braces above
+ % are not stripped by \poppstack
+ }%
+}
+
+\def\rulereference#1#2{\rulereferencex{\showlastactiontrue}{#1}{#2}}
+
+\def\thisrulereference#1{\rulereference{#1}{\secno}}
+
+% we use lazy evaluation for the references: reference requests are collected
+% and put into the auxilary file at the end of the job
+
+\def\lookupruleref#1{%
+ \yystringempty{#1}{}% done
+ {%
+ \expandafter\expandafter\expandafter
+ \toksa
+ \expandafter\expandafter\expandafter{\csname prodrulefor[#1]\endcsname}%
+ \immediate\write\auxstream{\noexpand\getrulereffor{#1}{\the\toksa}}%
+ \lookupruleref
+ }%
+}
+
+\def\yyuniontag{\auxunionctl}
+\defp\gramrefunionctl{}% sequence to activate/deactivate/take other action for
+ % gramref set of macros
+
+\defc\gramrefunionctl{%
+ \restorecslist{aux:global:gramrefs}\gramrefunion
+}
+\savecs{aux:global:activate}\gramrefunionctl
+
+\defc\gramrefunionctl{%
+ \restorecslist{aux:global:gramrefs:neutral}\gramrefunion
+}
+\savecs{aux:global:deactivate}\gramrefunionctl
+
+\defc\gramrefunionctl{% this command should not make any page material contributions
+ \message{gramrefs...}%
+}
+\savecs{aux:global:preend}\gramrefunionctl
+
+\defc\gramrefunionctl{% this command should not make any page material
+ % contributions
+ \message{gramrefs.store...}%
+ \expandafter\lookupruleref\rulerefrequests{}%
+}
+\savecs{aux:global:precutoff}\gramrefunionctl
+
+\defc\gramrefunionctl{% writing a headers, etc.
+ \message{gramrefs.done...}%
+ \toksa\expandafter{\gramrefunion}%
+ \immediate\write\auxstream{\harmlesscomment\the\toksa}%
+}
+\savecs{aux:global:prestart}\gramrefunionctl
+
+\yyuniondeclare\gramrefunion{aux:global:gramrefs}
+
+\defp\getrulereffor#1#2{}
+
+\toyyunion{aux:global:gramrefs:neutral}
+
+\defc\getrulereffor#1#2{%
+ \expandafter\def\csname full.prodrulefor[#1]\endcsname{#2}%
+}
+
+\toyyunion{aux:global:gramrefs} % do nothing for now