summaryrefslogtreecommitdiff
path: root/support/splint/tex/yyunion.sty
diff options
context:
space:
mode:
Diffstat (limited to 'support/splint/tex/yyunion.sty')
-rw-r--r--support/splint/tex/yyunion.sty2024
1 files changed, 1577 insertions, 447 deletions
diff --git a/support/splint/tex/yyunion.sty b/support/splint/tex/yyunion.sty
index d9749a86b9..7f4937a4e3 100644
--- a/support/splint/tex/yyunion.sty
+++ b/support/splint/tex/yyunion.sty
@@ -1,4 +1,4 @@
-% Copyright 2012-2015, Alexander Shibakov
+% Copyright 2012-2020, Alexander Shibakov
% This file is part of SPLinT
%
% SPLinT is free software: you can redistribute it and/or modify
@@ -21,6 +21,7 @@
\def\yyuniontag{\yyunion}
\def\parserstrictnamespace{parser-strict}
\def\parserprototypesnamespace{parser-strict:headers}
+\def\parserdebugnamespace{parser-debug}
\def\yyunion{\currentyyunionnamespace}
\def\currentyyunionnamespace{generic}
@@ -29,6 +30,11 @@
% therefore they are not included in the yyunion list and only defined once below and redefined later
% permanently
+\def\cwebstreamchars{% characters seen by the input routine
+ \|\`\\\{\}\~\ \_\&\^\$\#\n% the last one is inserted by the preprocessing scripts
+}
+
+
% insidemost
\long\def\stashed#1{} % stashed material (usually \Cee\ code) :: \stashed{tex_string}
@@ -40,23 +46,24 @@
\long\def\strm[#1]#2.[#3]{} % a stream element :: \strm[digits]{\stashed|\format|\formatlocal}.[digits]
-% types returned by the lexer (* marks the types that get removed by the parser in some cases)
+% types returned by the lexer; the integer type (hex vs. decimal) is determined at the lexer level so
+% these are tagged by the lexer as well
-\def\midf#1#2{} % rule separator :: \midf{fptr}{sptr}
-\defp\stringify#1#2#3#4{} % STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
-\defp\idit#1#2#3#4{} % ID :: \idit{12string}{tex_string}{fptr}{sptr}
-\defp\tagit#1#2#3#4{} % TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
-\defp\charit#1#2#3#4{} % CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
\defp\anint#1#2#3{} % INT :: \anint{digits}{fptr}{sptr}
\defp\hexint#1#2#3{} % INT :: \hexint{0[xX][0-9A-F]+}{fptr}{sptr}
+
+% types inserted by the parser (* marks the types that are essentially returned by the lexer
+% and `tagged' by the parser)
+
+\defp\idit#1#2#3#4{} % *ID :: \idit{12string}{tex_string}{fptr}{sptr}
+\defp\charit#1#2#3#4{} % *CHAR :: \charit{12string}{tex_string}{fptr}{sptr}
+\defp\stringify#1#2#3#4{} % *STRING :: \stringify{12string}{tex_string}{fptr}{sptr}
+\defp\tagit#1#2#3#4{} % *TAG :: \tagit{12string}{tex_string}{fptr}{sptr}
\defp\braceit#1#2#3{} % *BRACED_CODE :: \braceit{tex_string}{fptr}{sptr}
+\defp\midf#1#2{} % *rule separator :: \midf{fptr}{sptr}
\defp\preckind#1#2#3{} % *precedence operator :: \preckind{12string}{fptr}{sptr}
+%
\defp\flexoptionpair#1#2{} % a flex option :: \flexoptoinpair{\idit}{\idit|\stringify}
-
-% types inserted by the parser
-
-% insidemost
-
\defp\termname#1#2{} % a production term :: \termname{\idit|\stringify|\charit}{\idit}
\defp\mergeop#1#2#3{} % merge directive :: \mergeop{\tagit}{fptr}{sptr}
\defp\dprecop#1#2#3{} % dprec directive :: \dprecop{\anint|\hexint}{fptr}{sptr}
@@ -89,6 +96,7 @@
\defp\prodprodsep{} % production separator
\defp\prodprodseplarge{} % production separator
\defp\optoptsepsmall{} % separator between options
+\defp\optoptsepnone{} % (trivial) separator between options
\defp\posmark#1#2{} % the position marker :: \posmark{fptr}{sptr}
\defp\tokendecls#1#2#3{} % token declarations :: \tokendecls{[\onesymbol]+}{fptr}{sptr}
\defp\ntermdecls#1#2#3{} % nterm declarations :: \ntermdecls{[\onesymbol]+}{fptr}{sptr}
@@ -96,7 +104,7 @@
\defp\precdecls#1#2#3#4#5{} % precedence declarations :: \precdecls{left|right|nonassoc|precedence}{\tagit}{[\idit\stringify\charit]+}{fptr}{sptr}
\defp\flexsstatedecls#1#2#3{} % flex state declarations (nonexlusive) :: \flexsstatedecls{[\idit\stringify]+}{fptr}{sptr}
\defp\flexxstatedecls#1#2#3{} % flex state declarations (exclusive) :: \flexxstatedecls{[\idit\stringify]+}{fptr}{sptr}
-\defp\flexoptiondecls#1#2#3{} % flex state declarations :: \flexoptiondecls{[\flexoptionpair]+}{fptr}{sptr}
+\defp\flexoptiondecls#1#2#3{} % flex options :: \flexoptiondecls{[\flexoptionpair]+}{fptr}{sptr}
\defp\oneparametricoption#1#2#3#4{} % option :: \oneparametricoption{option-name}{\stringify|\idit}{fptr}{sptr}
\defp\optionflag#1#2#3#4{} % option flag :: \optionflag{flag name}{abbreviated name}{fptr}{sptr}
\defp\paramdef#1#2#3#4{} % parameters :: \paramdef{\braceit}{parameter domain}{fptr}{sptr}
@@ -133,17 +141,276 @@
\newtoks\yystashlocal
\newtoks\yyformatlocal
+% index implementation
+% note that the rank and domain parameters are handed down by the
+% `outer' macro while the `qualifier' (\term...string) is decided
+% at the point of invocation; some auxilary sequences first
+
+% a macro that always produces the current value of the register
+
+\def\thewrap#1#2{%
+ \noexpand\thewrap{\noexpand#1}{\the#1}%
+}
+
+% a macro that always produces the current page number.
+
+\def\selfpageno#1{%
+ \noexpand\selfpageno{\the\pageno}%
+}
+
+% a macro that keeps the (single) argument intact
+
+\def\stickycs#1{\noexpand\stickycs\noexpand#1}
+
+% for finer control over indexing, there are two levels on which
+% the programmer can disable the production of a whatsit node (\write):
+% (1) by setting the \iftermindex conditional to \iffalse, and (2) by defining
+% the \termindex macro to ignore its argument; both of these are expanded
+% at the point where the index entry is made; the intention is that
+% the conditional controls indexing at large (say, the index entries
+% inside the index should not be indexed again), while redefining the macro
+% provides a finer control over the indexing style
+
\newif\iftermindex
-\def\doparse#1{%
- \table{}\typestable{}\prectable{}\opttable{}%
- \parserinit\yyparse#1\yyeof\yyeof\endparseinput\endparse
+\let\termindex\eatone
+
+% the index is split into different domains: say, the \bison\ domain contains
+% the entries for the terms of a grammar;
+
+\def\bisonidxdomain{B} % marker for the domain (section of the index)
+
+\def\defidxrank{0} % index rank of definitions is lowest so they are listed first
+\def\headeridxrank{1} % index rank of lhs
+\def\termidxrank{2} % ordinary term rank
+
+% the expansion of the various components of an index entry must be carefully
+% controlled: it is not possible to predict how many times the entry will be expanded;
+% there are four types of expandable entries in general:
+% o the entries that must be expanded at the point of invocation (say, \secno)
+% o the entries that must be expanded only when the \write is inside a \box that is
+% being shipped out (say, the \pageno)
+% o the `lazy' entries that can be expanded any time (although at least one expansion
+% must take place)
+% o finally, the entries that must be expanded at a specific point between the point
+% of invocation and the \shipout
+% the indexing macro below treats the first three cases for the entries that the macro
+% below produce; the last case can be implemented by redefining the \stickycs macro
+% at the appropriate point
+
+\def\indxe#1#2{% index entry:
+ % #1 is the output stream
+ % #2 is the entry (see below for the proper form)
+ \expandafter\ind@@\expandafter#2\end{#1}%
+}
+
+\def\ind@@#1\end#2{%
+ \ind@e{#2}{#1}%
+}
+
+\def\ind@e#1#2{% #1 is the output stream
+ % #2 is a 8-parameter sequence, see below
+ \write#1{\in@@e#2\in@@ecomment#2}%
+}
+
+\def\in@@e#1#2#3#4#5#6#7#8{% #1: early expandable (like \secno)
+ % #2: auxilary info (expandable, invariant)
+ % #3: domain
+ % #4: rank
+ % #5: type1
+ % #6: type2
+ % #7: key1
+ % #8: key2
+ \i@@@e{#1}{\errmessage{\noexpand\\in@@e error: expanded page number placeholder}}%
+ {#2}{#3}{#4}{#5}{#6}{\charstonumberse#7\end}{\charstonumberse#8\end}%
+}
+
+\def\in@@ecomment#1#2#3#4#5#6#7#8{% #1: early expandable (like \secno)
+ % #2: auxilary info (expandable, invariant)
+ % #3: domain
+ % #4: rank
+ % #5: type1
+ % #6: type2
+ % #7: key1
+ % #8: key2
+ \harmlesscomment\space\charstocharsx#7\end :: \charstocharsx#8\end%
+}
+
+\def\i@@@e#1#2#3#4#5#6#7#8#9{% #1: early expandable (like \secno)
+ % #2: page number placeholder
+ % #3: auxilary info
+ % #4: domain
+ % #5: rank
+ % #6: type1
+ % #7: type2
+ % #8: key1
+ % #9: key2
+ \noexpand\i@@@e{#1}{\the\pageno}{#3}{#4}{#5}{\noexpand#6}{\noexpand#7}{#8}{#9}%
+}
+
+\def\appendfitotoksa{\toksa\expandafter{\the\toksa\else<>\fi}} % hide \fi inside the loop
+
+% defining expandable macros for string transformations
+
+\tempca\@ne
+\toksa{\ifcase#1<NUL>}
+\tempcb=\uccode`.
+
+\bloop
+\uccode`.=\tempca
+\uppercase{\toksb\expandafter{.}}%
+\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa\expandafter\or\the\toksb}%
+\advance\tempca by\@ne
+\ifnum\tempca>"FF
+ \appendfitotoksa
+\else
+\repeat
+
+\def\charstocharsx#1\end{% expandable version of a `safety' macro, ascii characters only, no tricky space characters
+ \yystartsinspace{#1.}{%
+ \space\ch@rstocharsx#1\end
+ }{%
+ \yystringempty{#1}{}{%
+ \ch@rst@charsx#1\end
+ }%
+ }%
+}
+
+\def\ch@rstocharsx#1 #2\end{\charstocharsx#2\end}
+
+\def\ch@rst@charsx#1#2\end{\asciiprint{`#1}\charstocharsx#2\end}
+
+\edef\next{\toksa{\def\noexpand\asciiprint##1{\the\toksa}}}\next
+\the\toksa
+
+\def\writeidxhentry#1{%
+ \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\headeridxrank}#1}%
+}
+
+\def\writeidxentry#1{%
+ \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\termidxrank}#1}%
+}
+
+\def\writetokenidxentry#1{%
+ \indxe\gindex{{\secno}{{\currentrulecontext}{\hostparsernamespace}}{\bisonidxdomain}{\defidxrank}#1}%
+}
+
+\def\gidxentry#1#2#3{% a generic index entry:
+ % #1 is the type processor,
+ % #2 is the key
+ % #3 is the visual key
+ \iftermindex
+ \termindex{{}{#1}{#2}{#3}}%
+ \fi
+}
+
+% a convenient shortcut, as many visual keys come in a token register
+
+\def\gidxentryxv#1#2#3{% a generic index entry, e(x)pand (v)isual key:
+ % #1 is the type processor,
+ % #2 is the key
+ % #3 is the visual key
+ \expandafter\gidx@ntryxv\expandafter{#3}{#1}{#2}%
+}
+
+\def\gidxentryxb#1#2#3{% a generic index entry, e(x)pand (b)oth keys:
+ % #1 is the type processor,
+ % #2 is the key
+ % #3 is the visual key
+ \expandafter\gidx@ntryxb\expandafter{#2}{#3}{#1}%
+}
+
+\def\gidx@ntryxb#1#2#3{%
+ \expandafter\gidx@ntryxv\expandafter{#2}{#3}{#1}%
+}
+
+\def\gidx@ntryxv#1#2#3{%
+ \gidxentry{#2}{#3}{#1}%
+}
+
+\def\tidxentry#1#2{% a text index entry, TODO: expand \currentrulecontext
+ \iftermindex
+ \indxe\gindex{{\secno}{{\currentrulecontext}{\indexpseudonamespace}}{\bisonidxdomain}{\termidxrank}{}{#1}{#2}{}}%
+ \fi
+}
+
+\def\idxinline#1{% index the term in text
+ \edef\next{\tidxentry{\termidstring}{#1}}\next
+}
+
+\def\gatherstash#1#2\strm[#3]#4.[#5]{% #1 contains the stash gathered
+ \yystringempty{#3}{ #1}%
+ {%
+ \expandafter\g@therstash\expandafter{\romannumeral0\unstash#4}{#1}%
+ }%
+}
+
+% stash processing wrapper macros
+
+\def\g@therstash#1#2{\gatherstash{#2#1}}
+
+\def\unstash#1#2{\expandafter\space#2}% #1 is \stashed or \sflush
+
+\def\yyreadstash#1\to#2\in#3\with#4{%
+ \yyreadfifo#1\to#2\in#3%
+ #4{#3}%
+ \setbox\z@=\vbox{
+ \hsize\maxdimen\emergencystretch\maxdimen
+ \rightskip\z@ plus 1fill
+ \insertstash{#3}%
+ \ifmmode
+ {}$\aftergroup\yyr@adstash % $o that the box can be finished
+ \else
+ \aftergroup\eatthree
+ \fi
+ }{#1}{#2}{#3}%
+}
+
+\def\yyr@adstash#1#2#3{%
+ #3\expandafter{\the#3{}$}% $
+ \appendlnx#1{\strm[NaN]{\stashed{{}${}}}.[NaN]}% {}, in case \yystash ends with $
+}
+
+% the sequence below may be redefined so that the #2 parameter to \yyreadstash may be a list of
+% token registers
+
+\def\insertstash#1{\the#1\relax}
+
+\def\readstash#1{\yyreadstash\yystash\to#1\in\yystashlocal\with\unpackstash}
+
+\def\readstashwredirect#1#2#3{% helper macro to read the stash when the stash and the format pointers appear together
+ % #1(the format pointer) is ignored
+ % #2 is the stash marker
+ % #3 is the local stash register
+ \yyreadstash\yystash\to#2\in#3\with\unpackstash
+}
+
+\def\unpackstash#1{#1\expandafter{\romannumeral0\expandafter\unp@ckstash\the#1\strm[].[]}}
+
+\def\unp@ckstash{\gatherstash{}}
+
+\def\unwrapstash#1{% to use where only the removal of \strm and \stashed is needed
+ #1\expandafter{\romannumeral0\expandafter\unwr@pstash\the#1\strm[].[]}%
}
-\long\def\strm[#1]#2.[#3]{#2}
+\def\unwr@pstash{\gatherstash{}}
+
+% parser invocation
+
+\def\newparserstateextra{}
+
+\def\d@parse#1{\yyparse#1\yyeof\yyeof\endparseinput\endparse}
+
+\def\astarraylastcs{\expandafter\noexpand\romannumeral\gettopofstackcsx\astarray}
+
+\long\def\strm[#1]#2.[#3]{#2} % this is used with both the format and the stash streams; TODO: deprecate
+
\long\def\inmath#1{$#1$}
-\long\def\stashed#1{\toksa\expandafter{\the\toksa#1}}
+%\long\def\stashed#1{\toksa\expandafter{\the\toksa#1}} % to be used with unpackaged stash
+%\long\def\stashed#1{\toksa\expandafter\expandafter\expandafter{\expandafter\the\expandafter\toksa#1}} % to use with AST array
+\long\def\stashed#1{\errmessage{making \noexpand\stashed\space expandable is deprecated}}
+
\long\def\sflush#1#2{}
\def\ntt{\noexpand\ntt@} % a convenient shortcut
@@ -157,15 +424,6 @@
\let\termindex\eatone
}
-\def\headeridxrank{1} % index rank of lhs
-
-\def\writeidxhentry#1{\write\gindex{%
- \nx\nx
- \expandafter
- \nx
- \csname\bisonidxdomain TI\endcsname\headeridxrank#1}%
-}
-
\def\prodhead@r#1#2{%
\let\idit\prodhdridit
#1\relax\concat\table\toksa\hrhss@p
@@ -229,45 +487,32 @@
\let\stringify\termnamestringify
\let\optstrextra\optstrextraesc
\let\termindex\writeidxentry
- #1\relax\concat\table\toksa
+ \termnam@{#1}{#2}%
\let\termindex\eatone
}
-\def\bisonidxdomain{B} % marker fo the domain (section of the index)
-\def\termidxrank{2} % rank (affects the order in which the entries are
- % listed in the index and the listing style)
-
-\def\writeidxentry#1{\write\gindex{%
- \nx\nx
- \expandafter
- \nx
- \csname\bisonidxdomain TI\endcsname\termidxrank#1}%
-}
+\def\termnam@#1#2{\toksa{}\toksb{}\toksc{}#1\expandafter\termn@m@\expandafter{\the\toksa}{#2}}
-\let\termindex\eatone
+\def\termn@m@#1#2{\toksa{}\toksb{}\toksc{}#2\expandafter\t@rmn@m@\expandafter{\the\toksa}{#1}}
-% index implementation
-% note that the rank and domain parameters are handed down by the
-% `outer' macro while the `qualifier' (\term...string) is decided
-% at the point of invocation
+% several versions of term typesetting
-\def\gidxentry#1#2{% a generic index entry
- \iftermindex
- \termindex{%
- {\nx\nx\nx#1{\charstonumberse#2\end}}%
- {\hostparsernamespace}{\nx\the\pageno}{\secno}%
- }%
- \fi
+\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
+ \yystringempty{#1}{\toksa{#2}}{\toksa{#2\hbox{$\langle${}#1{}$\rangle$}}}%
+ \concat\table\toksa
}
-\def\tidxentry#1#2{% a text index entry
- \iftermindex
- \write\gindex{%
- \nx\nx\expandafter\nx\csname\bisonidxdomain TI\endcsname\termidxrank
- {\nx\nx\nx#1{\charstonumberse#2\end}}%
- {\indexpseudonamespace}{\nx\the\pageno}{\secno}%
- }%
- \fi
+\newif\ifbracketedvisible % decide whether to show the bracketed name in productions
+
+\def\t@rmn@m@#1#2{% the termname is #2, the bracketed part is in #1
+ \yystringempty{#1}{\toksa{#2}}{%
+ \ifbracketedvisible
+ \toksa{#2\hbox{$\,\cdot$#1}}%
+ \else
+ \toksa{#2}%
+ \fi
+ }%
+ \concat\table\toksa
}
%
@@ -278,77 +523,46 @@
\yyreadfifo\yyformat\to#3\in\yyformatlocal
\restorecs{table-render}{\strm}%
#5% accumulate all the directives
- \edef\next{\the\bdirects}%
- \ifx\next\empty
- \else
+ \yytoksempty\bdirects{}{%
\edef\next{\bdirects{%
$\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr\the\bdirects}}$}}\next
- \fi
+ }%
\the\yyformatlocal
\actbrac@s{#1}{#2}{#4}%
}
\def\actbrac@s#1#2#3{%
- \yyreadfifo\yystash\to#3\in\yystashlocal
- \unmathstash{#3}%
- \edef\next{\toksa{\the\bdirects&\the\yystashlocal\cr}}\next
+ \readstash{#3}%
+ \appendr\table{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
\bdirects{}%
- \concat\table\toksa
-}
-
-\def\unmathstash#1{%
- \setbox\z@=\vbox{
- \hsize\maxdimen\emergencystretch\maxdimen
- \rightskip\z@ plus 1fill
- \let\TeXx\relax\let\stashed\copystashed
- \the\yystashlocal\relax
- \ifmmode
- {}$\aftergroup\unm@thstash % $o that the box can be finished
- \else
- \aftergroup\eatone
- \fi}{#1}%
-}
-
-\def\copystashed#1{#1}
-
-\def\unm@thstash#1{%
- \toksa{\strm[NaN]{\stashed{{}$}}.[NaN]}% $o it cannot be matched if \yystashlocal is read again
- \concat\yystashlocal\toksa
- \toksa{\strm[#1]{\stashed{${}}}.[#1]}%$
- \appendl\yystash{\the\toksa}%
}
\def\bpredicate#1#2#3#4#5\bdend{%
\yyreadfifo\yyformat\to#3\in\yyformatlocal
\restorecs{table-render}{\strm}%
#5% accumulate all the directives
- \edef\next{\the\bdirects}%
- \ifx\next\empty
+ \yytoksempty\bdirects{%
\bdirects{\quad$\dashv\ \nx\vdash$}%
- \else
+ }{%
\edef\next{\bdirects{%
$\vtop{\halign{\nx\quad####\nx\hfil&####\nx\hfil\cr$\dashv\ $&\omit\nx\hfil$\nx\vdash$\cr
\the\bdirects}}$}}\next
- \fi
+ }%
\the\yyformatlocal
\bpredicat@{#1}{#2}{#4}%
}
-\def\bpredicat@#1#2#3{%
- \yyreadfifo\yystash\to#3\in\yystashlocal
- \unmathstash{#3}%
- \edef\next{\toksa{\the\bdirects&\the\yystashlocal\cr}}\next
+\def\bpredicat@#1#2#3{%
+ \readstash{#3}%
+ \appendr\table{\the\bdirects&\toksa{\the\yystashlocal}\cr}%
\bdirects{}%
- \concat\table\toksa
}
\def\bdirective#1#2#3#4{%
\savecslist{local-namespace}\yyunion
\restorecslist{table-render:directives}\yyunion
- \toksa{}%
- #2% make a directive list
- \edef\next{\toksa{$\nx\langle${\ntt #1} &\the\toksa$\nx\rangle$\cr}}\next
- \concat\bdirects\toksa
+ \toksa{}#2% make a directive list
+ \appendr\bdirects{$\nx\langle${\ntt #1} &\the\toksa$\nx\rangle$\nx\cr}%
\restorecslist{local-namespace}\yyunion
}
@@ -379,34 +593,43 @@
\let\currentrulecontext\empty
-\def\emptyterm{%
- \edef\next{\toksa{$\nx\circ$\tidxentry{\termidstring}{emptyrhs}}}\next
- \concat\table\toksa
-}
+% an empty left hand side is indexed in the index pseudonamespace and not in the main
+% parser namespace, since the name parser never encounters this term (as the \emptyterm
+% control sequence is inserted by the main parser) while typesetting productions;
+% because of this it would be unnatural to define a \prettywordpair sequence for it in the main
+% parser namespace; using a special non-\bison\ name should avoid any confusion with
+% the regular grammar terms
+\def\emptyterm{\appendr\table{$\nx\circ$\tidxentry{\termexception}{emptyrhs&}}}
\def\termnameidit#1#2#3#4{%
\yyreadfifot\yyformat\to#3\in\yyformatlocal\with\formatsort\relax
\the\yyformatlocal
- \nameproc{#2}%
- \peekstash{#4}%
- \settermstyle{#1}%
+ \nameproc{#2}\with\parsebin
+ \peekstash{#4}% does not clobber \toksa and \toksb
+ \expandafter\settermstyle\expandafter{\the\toksb}{#1}%
}
-\def\settermstyle#1{%
- \edef\next{\toksa{\termmetastyle{%
- \gidxentry{\termidstring}{#1}%
- \let\nx\idxfont\nx\empty\nx\it\the\toksa\nx\/%
- }}}\next
-}
-
-\def\idxinline#1{% index the term in text
- \edef\next{\tidxentry{\termidstring}{#1}}\next
+\def\settermstyle#1#2{% put the \whatsit node (\write) at the end so that the
+ % term macros can remove the whitespace preceeding it
+ \edef\next{%
+ \toksa{%
+ \termmetastyle{%
+ \let\nx\idxfont\nx\empty\nx\it\the\toksa
+ \gidxentry{\termidstring}{#2}{#1}\nx\/%
+ }%
+ }%
+ }\next
}
\def\termnamestringify#1#2#3#4{%
- \nameproc{#2}%
+ \nameproc{#2}\with\parsebin
+ \ifyyparsefail
+ \toksb{"#1"}\toksc{\termstring}%
+ \else
+ \toksc{\termidstring}%
+ \fi
\edef\next{\toksa{{%
- \gidxentry{\termstring}{#1}%
+ \expandafter\gidxentryxv\expandafter{\the\toksc}{#1}{\the\toksb}%
\ntt\the\toksa
}}}\next
}
@@ -418,31 +641,39 @@
\newif\ifinheader
\def\prodhdridit#1#2#3#4{%
- \setrulecontext{}%
- \nameproc{#2}%
- \expandafter\setrulecontext\expandafter{\the\toksc}%
+ \setrulecontext{}% TODO: ?
+ \nameproc{#2}\with\parsebin
+ \setrulecontext{#1}%
\edef\next{\toksa{{%
- \gidxentry{\termhdrstring}{#1}%
+ \gidxentryxv{\termidstring}{#1}{\the\toksb}%
\nx\inheadertrue\let\nx\idxfont\nx\bf\nx\itbold\the\toksa\nx\/%
}}}\next
}
\def\charit#1#2#3#4{%
- \expandafter\ifx\csname'parser'\hostparsernamespace#1\endcsname\relax
- \edef\next{\toksa{{\ntt #1}\gidxentry{\termvstring}{'#1'}}}%
+ \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
+ \toksa{#1}\sansfirst\toksa
+ \edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
\else
+ \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
+ \tokse{#1}% we are stripping the quotes so the visible key is adjusted
+ \else
+ \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
+ \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
+ \fi
\edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
- \csname'parser'\hostparsernamespace#1\endcsname{\currentrulecontext}\gidxentry{\termvstring}{'#1'}}}%
+ %\csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termvstring}{'#1'}{\the\tokse}}}%
+ \csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termidstring}{'#1'}{\the\tokse}}}%
\fi
\next
}
\def\anint#1#2#3{%
- \toksa{#1}\concat\table\toksa
+ \appendrnx\table{#1}%
}
\def\hexint#1#2#3{%
- \toksa\expandafter{\eattwo#1$_{16}$}\concat\table\toksa
+ \expandafter\appendrnx\expandafter\table\expandafter{\eattwo#1$_{16}$}%
}
\def\optoptsepnone{%
@@ -451,22 +682,22 @@
\def\optoptsepsmall{% keep the alignment going but insert a small space
\concat\opttable\postoks
- \toksa{\noalign{\smallskip}}\concat\opttable\toksa
+ \appendrnx\opttable{\noalign{\smallskip}}%
}
\def\prodprodsepsmall{%
\pushothertables
- \toksa{\noalign{\penalty-150 \smallskip}}\concat\table\toksa
+ \appendrnx\table{\noalign{\penalty-150 \smallskip}}%
}
\def\prodprodsep{%
\pushothertables
- \toksa{\noalign{\medskip}}\concat\table\toksa
+ \appendrnx\table{\noalign{\medskip}}%
}
\def\prodprodseplarge{%
\pushothertables
- \toksa{\noalign{\bigskip}}\concat\table\toksa
+ \appendrnx\table{\noalign{\bigskip}}%
}
\def\separatorswitcheq{%
@@ -532,28 +763,22 @@
\def\posmark#1#2{%
% \yyreadfifo\yyformat\to#1\in\yyformatlocal
-% no formattig for locations currently
- \restorecs{table-render}{\strm}%
+% no formatting for locations currently
% \the\yyformatlocal
\posm@rk{#2}%
}
\def\posm@rk#1{%
- \yyreadfifo\yystash\to#1\in\yystashlocal
- \edef\next{\the\yystashlocal}%
- \ifx\next\empty
- \postoks{}%
- \else
- \unmathstash{#1}%
- % TODO: move this out
- \toksa{}\the\yystashlocal\cleanstash\stripstash
+ \readstash{#1}%
+ \yytoksempty\yystashlocal{\postoks{}}{%
+ \toksa\expandafter{\the\yystashlocal}\cleanstash\stripstash
\setbox\z@=\vbox{\setlazyc\the\toksa}%
\ifnum\ht\z@=\z@
\postoks{}%
\else
- \edef\next{\postoks{\noalign{\indent\toksa{}\the\yystashlocal\nx\makestashbox}}}\next
+ \edef\next{\postoks{\noalign{\indent\toksa{\the\yystashlocal}\nx\makestashbox}}}\next
\fi
- \fi
+ }%
}
\def\codeoptionlist{%
@@ -608,29 +833,13 @@
\def\pushothertables{%
%token's
- \edef\next{\the\tokdectoks}%
- \ifx\next\empty
- \else
- \attachtokentable
- \fi
+ \yytoksempty\tokdectoks{}{\attachtokentable}%
%type's
- \edef\next{\the\typestable}%
- \ifx\next\empty
- \else
- \attachtypestable
- \fi
+ \yytoksempty\typestable{}{\attachtypestable}%
%precedence's
- \edef\next{\the\prectable}%
- \ifx\next\empty
- \else
- \attachprectable
- \fi
+ \yytoksempty\prectable{}{\attachprectable}%
% options
- \edef\next{\the\opttable}%
- \ifx\next\empty
- \else
- \attachoptionstable
- \fi
+ \yytoksempty\opttable{}{\attachoptionstable}%
\concat\table\postoks
} %
@@ -643,8 +852,7 @@
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:token-declarations}\yyunion
\let\termindex\writetokenidxentry
- \displaytokenrawtrue
- \let\currenttokentype\empty#1%
+ \let\currenttokentype\empty#1%
\restorecslist{local-namespace}{\yyunion\termindex}%
}
@@ -652,31 +860,29 @@
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:type-declarations}\yyunion
\let\termindex\writetokenidxentry
- \typesset{}%
- #2\relax#1%
- \edef\next{\typestable{\the\typestable\nx\onetype{\the\toksa}{\the\typesset}}}\next
+ \typesset{}#2\relax#1%
+ \appendr\typestable{\nx\onetype{\the\toksa}{\the\typesset}}%
\restorecslist{local-namespace}{\yyunion\termindex}%
}
% \onetype is a typeseting macro defined elsewhere
+% TODO: move these to the \flex\ section
\def\flexsstatedecls#1#2#3{%
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:flex-state-declarations}\yyunion
- \let\termindex\writetokenidxentry
- \typesset{}%
- #1\relax
- \edef\next{\typestable{\the\typestable\nx\flexsstatelist{\the\typesset}}}\next
+ \let\termindex\writeidxfsdentry%\writetokenidxentry
+ \typesset{}#1\relax
+ \appendr\typestable{\nx\flexsstatelist{\the\typesset}}%
\restorecslist{local-namespace}{\yyunion\termindex}%
}
\def\flexxstatedecls#1#2#3{%
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:flex-state-declarations}\yyunion
- \let\termindex\writetokenidxentry
- \typesset{}%
- #1\relax
- \edef\next{\typestable{\the\typestable\nx\flexxstatelist{\the\typesset}}}\next
+ \let\termindex\writeidxfsdentry%\writetokenidxentry
+ \typesset{}#1\relax
+ \appendr\typestable{\nx\flexxstatelist{\the\typesset}}%
\restorecslist{local-namespace}{\yyunion\termindex}%
}
@@ -685,9 +891,8 @@
\def\flexoptiondecls#1#2#3{%
\savecslist{local-namespace}{\yyunion\termindex}%
\restorecslist{table-render:flex-option-definitions}\yyunion
- \let\termindex\writetokenidxentry
- \toksa{}%
- #1\relax
+ \let\termindex\writeidxfsrnentry%\writetokenidxentry
+ \toksa{}#1\relax
\restorecslist{local-namespace}{\yyunion\termindex}%
}
@@ -700,8 +905,8 @@
\restorecslist{table-render:prec-declarations}\yyunion
\setrulecontext{*#1}%
\let\termindex\writetokenidxentry
- \typesset{}#3\toksa{}#2%
- \edef\next{\prectable{\the\prectable\noexpand\oneprec{#1}{\the\toksa}{\the\typesset}}}\next
+ \typesset{}#3\toksa{}#2%
+ \appendr\prectable{\nx\oneprec{#1}{\the\toksa}{\the\typesset}}%
\restorecslist{local-namespace}{\yyunion\termindex}%
\setrulecontext{}%
}
@@ -712,52 +917,45 @@
\def\oneparametricoption#1#2#3#4{%
\restorecs{table-render:token-declarations}{\braceit\stringify\idit\anint\hexint}%
- \toksa{}\toksb{}\toksc{}%
- #2%
- \edef\next{\noexpand\oneparametricopti@n{#1}{\the\toksa}{\the\toksb}{\the\toksc}}\next
+ \toksa{}\toksb{}\toksc{}#2%
+ \edef\next{\nx\oneparametricopti@n{#1}{\the\toksa}{\the\toksb}{\the\toksc}}\next
}
\def\oneparametricopti@n#1#2#3#4{%
- \edef\next{#2}%
- \ifx\next\empty % there is no identifier
- \def\next{#4}%
- \ifx\next\empty % there is no digit
+ \yystringempty{#2}{% there is no identifier
+ \yystringempty{#4}{% there is no digit
\edef\next{\toksa{{\ntt"\the\toksb"}}}\next
- \else
+ }{%
\edef\next{\toksa{{\nx\rm\the\toksc}}}\next
- \fi
- \else
- \nameproc{#2}%
- \fi % now \toksa contains the value of the parameter
+ }%
+ }{%
+ \nameproc{#2}\with\parsebin
+ }% now \toksa contains the value of the parameter
\toksb{#1}%
\let\termindex\writetokenidxentry
- \edef\next{\toksa{\gidxentry{\termostring}{#1}$\nx\langle${\nx\bf\the\toksb}$\nx\rangle$&\the\toksa\nx\/\nx\cr}}\next
+ \appendr\opttable{\gidxentry{\termostring}{#1}{}$\nx\langle${\nx\bf\the\toksb}$\nx\rangle$&\the\toksa\nx\/\nx\cr}%
\let\termindex\eatone
- \concat\opttable\toksa
}
\def\paramdef#1#2#3#4{%
\restorecs{table-render:token-declarations}\braceit
\let\termindex\writetokenidxentry
- \edef\next{\toksa{\gidxentry{\termostring}{#2}}}\next
- \toksb\expandafter{\the\toksa$\langle${\bf #2}$\rangle$}\toksa{}%
- #1%
+ \appendr\opttable{\gidxentry{\termostring}{#2}{}}%
\let\termindex\eatone
- \concat\toksb\toksa
- \concat\opttable\toksb
+ \appendrnx\opttable{$\langle${\bf #2}$\rangle$}%
+ \toksa{}#1%
+ \concat\opttable\toksa
}
\def\optionflag#1#2#3#4{%
- \def\next{#2}%
- \ifx\next\empty
- \toksb{}%
- \else
- \toksb{ {\rm(set as $\langle${\bf#2}$\rangle$)}}%
- \fi
\let\termindex\writetokenidxentry
- \edef\next{\toksa{\gidxentry{\termostring}{#1}$\nx\langle${\nx\bf#1}$\nx\rangle$\nx\rlap{$\nx\,\star$}&\the\toksb\cr}}\next
+ \appendr\opttable{\gidxentry{\termostring}{#1}{}}%
\let\termindex\eatone
- \concat\opttable\toksa
+ \yystringempty{#2}{%
+ \appendrnx\opttable{$\langle${\bf#1}$\rangle$\rlap{$\,\star$}&\cr}%
+ }{%
+ \appendrnx\opttable{$\langle${\bf#1}$\rangle$\rlap{$\,\star$}& {\rm(set as $\langle${\bf#2}$\rangle$)}\cr}%
+ }%
}
\def\codepropstype#1#2#3#4#5#6#7{%
@@ -768,21 +966,18 @@
}
\def\codepropstyp@#1#2#3#4{%
- \yyreadfifo\yystash\to#4\in\yystashlocal
- \unmathstash{#4}%
+ \readstash{#4}%
\restorecs{table-render:type-declarations}{\idit\stringify\hspace}%
\let\termindex\writetokenidxentry
- \typesset{}%
- #2% collect all the symbols in \typesset
- \edef\next{\toksa{\gidxentry{\termostring}{#3}$\nx\langle${\nx\bf #3}$\nx\rangle$&%
- \toksa{}\the\yystashlocal\nx\makestashbox\nx\hfil\cr
- \noalign{\nx\smallskip}%
- \noalign{\indent$\nx\rlap{\hbox to2em{\nx\hfil$\nx\star$\nx\hfil}}\vcenter{
- \advance\hsize by-\parindent
- \emergencystretch10pt\nx\raggedright\noindent\hangafter\z@\hangindent2em\nx\strut\the\typesset\nx\strut}$}%
- }}\next
+ \typesset{}#2% collect all the symbols in \typesset
+ \appendr\opttable{\gidxentry{\termostring}{#3}{}$\nx\langle${\nx\bf #3}$\nx\rangle$&%
+ \toksa{\the\yystashlocal}\nx\makestashbox\nx\hfil\cr
+ \noalign{\nx\smallskip}%
+ \noalign{\indent$\nx\rlap{\hbox to2em{\nx\hfil$\nx\star$\nx\hfil}}\vcenter{
+ \advance\hsize by-\parindent
+ \emergencystretch10pt\nx\raggedright\noindent\hangafter\z@\hangindent2em\nx\strut\the\typesset\nx\strut}$}%
+ }%
\let\termindex\eatone
- \concat\opttable\toksa
}
\def\codeassoc#1#2#3#4#5#6#7{%
@@ -793,20 +988,15 @@
}
\def\codeass@c#1#2#3#4{%
- \yyreadfifo\yystash\to#4\in\yystashlocal
- \unmathstash{#4}%
+ \readstash{#4}%
\let\idit\termnameidit
\let\termindex\writetokenidxentry
- \toksa{}% in case #1 is empty
- #2% put the typeset form of the identifie in \toksa
- \toksb\toksa
- \edef\next{\toksa{\yystringempty{#1}{}{\gidxentry{\termostring}{#1}}%
- $\nx\langle${\nx\bf #1}$\nx\rangle$\nx\quad{\the\toksb}&%
- \toksa{}\the\yystashlocal
- \nx\makestashbox
- \nx\hfil\cr
- }}\next
- \concat\opttable\toksa
+ \toksa{}#2% put the typeset form of the identifier in \toksa
+ \appendr\opttable{%
+ \yystringempty{#1}{}{\gidxentry{\termostring}{#1}{}}%
+ $\nx\langle${\nx\bf #1}$\nx\rangle$\nx\quad{\the\toksa}&%
+ \toksa{\the\yystashlocal}\nx\makestashbox\nx\hfil\cr
+ }%
\let\termindex\eatone
}
@@ -821,35 +1011,29 @@
\let\idit\termnameidit
\let\stringify\termnamestringify
\let\termindex\writetokenidxentry
- #1% put the typeset form of the identifier or string in \toksa
- \toksb\toksa
- \savecslist{local-namespace}\yyunion
- \restorecslist{table-render:variable-definitions}\yyunion
- \toksa{}% in case #2 is empty
- #2% put the value in toksa
- \toksc\toksa
- \restorecslist{local-namespace}\yyunion
+ #1% put the typeset form of the identifier or string in \toksa
+ \toksb\toksa
+ \savecslist{local-namespace}\yyunion
+ \restorecslist{table-render:variable-definitions}\yyunion
+ \toksa{}#2% put the value in toksa
+ \toksc\toksa
+ \restorecslist{local-namespace}\yyunion
\let\termindex\eatone
- \edef\next{\toksa{$\nx\langle${\nx\bf define}$\nx\rangle$\nx\quad{\the\toksb}&%
- \the\toksc\cr
- }}\next
- \concat\opttable\toksa
+ \appendr\opttable{$\nx\langle${\nx\bf define}$\nx\rangle$\nx\quad{\the\toksb}&\the\toksc\cr}%
}
\def\prologuecode#1#2#3#4#5{% similar to \actbrac@s
- \yyreadfifo\yystash\to#5\in\yystashlocal
- \unmathstash{#5}%
- \restorecs{table-render}{\strm\stashed}%
- \toksa{}\the\yystashlocal
- \edef\next{\toksa{\omit\span\omit\nx\cdotfill\cr
- \omit\span\omit\toksa{\the\toksa}\nx\makestashbox\nx\cr
- \omit\span\omit\nx\cdotfill\cr}}\next
- \concat\opttable\toksa
+ \readstash{#5}%
+ \appendr\opttable{%
+ \omit\span\omit\nx\cdotfill\cr
+ \omit\span\omit\toksa{\the\yystashlocal}\nx\makestashbox\nx\cr
+ \omit\span\omit\nx\cdotfill\cr
+ }%
}
\def\yyunion@tablerender{%
- \strm\stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\actbrac@s\hspac@\codepropstyp@\vard@f\stopproduction
- \termmetastyle\peekstash
+ \strm\stashed\rarhss@p\rrhss@p\prodhead@r\hrhss@p\arhss@p\termnam@\termn@m@\t@rmn@m@
+ \actbrac@s\hspac@\codepropstyp@\vard@f\termmetastyle\peekstash\stopproduction
}
\savecslist{table-render}\yyunion
@@ -868,10 +1052,16 @@
#1%
}
\let\formatlocal\format
+\let\formatbegin\format
+\let\formatp\format
\newtoks\yystashlastcontent % contents of the last action
\newif\ifshowlastaction % if true, show the last action of a flattened set of rules
+% flatten: omit the actions, just list all the rules in a single line;
+% if \showlastactiontrue show just the last action, otherwise display \actionfiller
+% instead
+
\def\flatten{%
\let\stopproduction\stopproduction@flatten
\let\actbrac@s\actbrac@s@flatten
@@ -881,45 +1071,44 @@
}
\def\actbrac@s@flatten#1#2#3{% do everything but add the stash contents to the table
- \yyreadfifo\yystash\to#3\in\yystashlocal
- \unmathstash{#3}%
- \yystashlastcontent\yystashlocal
+ \readstash{#3}%
+ \yystashlastcontent\yystashlocal % keep the contents of the braces
+ % in case we want to show the last action
\futurelet\actsep\actbrac@s@fl@tten
}
+% see the remarks before \emptyterm above for the reasons for choosing the index
+% pseudonamespace for this term
\def\actbrac@s@fl@tten{%
\ifx\actsep\arhssep
- \edef\next{%
- \toksa{$\nx\ \nx\diamond\nx\ $\tidxentry{\termidstring}{inline_action}}}\next
- \concat\table\toksa
+ \appendr\table{$\nx\ \nx\diamond\nx\ $\tidxentry{\termexception}{inline_action&}}%
\fi
}
\def\rrhss@p@flatten{%
- \toksa{$\ \vert\ $}\concat\table\toksa
+ \appendrnx\table{$\ \vert\ $}%
}%
\def\stopproduction@flatten{%
\ifshowlastaction
- \toksa\yystashlastcontent
+ \yystashlastcontent\expandafter{\expandafter\toksa\expandafter{\the\yystashlastcontent}}%
\else
- \toksa{\actionfiller}%
+ \yystashlastcontent{\actionfiller}%
\fi
- \toksa\expandafter{\expandafter&\expandafter&\the\toksa\cr}\concat\table\toksa
+ \appendr\table{&&\the\yystashlastcontent\nx\cr}%
}
+% fold: resume normal display
+
\def\fold{%
\restorecs{table-render}{\actbrac@s\rrhss@p\arhss@p\rarhss@p\stopproduction}%
- \ifshowlastaction
- \toksa\yystashlastcontent
- \else
- \toksa{\actionfiller}%
- \fi
- \toksa\expandafter{\expandafter&\expandafter&\the\toksa\cr}\concat\table\toksa
+ \stopproduction@flatten
}
\def\actionfiller{\omit\quad$\ldots$\hfil}
+% inline: put the left hand side of the production on the same line as the rule
+
\def\inline{%
\let\prodhead@r\prodhead@r@inline
\let\hrhss@p\hrhss@p@inline
@@ -928,10 +1117,10 @@
\def\prodhead@r@inline#1#2{%
\let\idit\prodhdridit
#1\relax
+ \concat\table\toksa
\setbox\z@=\hbox{\strut\the\toksa\/$\,$\rm:\quad}%
\tempda=\wd\z@\relax
\advance\tempda-2em
- \edef\next{\table{\the\table{\the\toksa}}}\next
\hrhss@p
}
@@ -940,18 +1129,45 @@
\def\termspostformat{\ifsquashterms\aftergroup\noexpand\squashtermstrue\fi}
\def\hrhss@p@inline{%
- \edef\next{\toksa{&\termspostformat\hbox to \the\tempda{\hss}}}\next\concat\table\toksa
+ \appendr\table{&\termspostformat\hbox to \the\tempda{\hss}}%
}
+% breakline: break the production line
+
\def\breakline{%
\let\hspac@\hspac@@breakline
}
\def\hspac@@breakline{%
- \toksa{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}\concat\table\toksa
+ \appendrnx\table{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
\restorecs{table-render}\hspac@
}
+% breakahead: break the production line after #1 terms
+% use: \breakahead{[0-9]+}
+
+\newcount\breakaheadcount
+
+\def\breakahead#1{%
+ \let\bahspac@\hspac@
+ \let\hspac@\hspac@@breakahead
+ \breakaheadcount#1\relax
+}
+
+\def\hspac@@breakahead{%
+ \ifnum\breakaheadcount=\@ne
+ \yybreak{%
+ \appendrnx\table{{$\,\hookleftarrow$}&\omit\hfil&\omit\hfil\cr\omit\hfil&\hbox to 2em{\hfil}}%
+ \let\hspac@\bahspac@
+ }%
+ \else
+ \advance\breakaheadcount by \m@ne
+ \yybreak{\bahspac@}%
+ \yycontinue
+}
+
+% skipheader: do not show the left hand side
+
\def\skipheader{%
\let\prodhead@r\prodhead@r@skipheader
\let\hrhss@p\hrhss@p@skipheader
@@ -961,34 +1177,85 @@
\hrhss@p
}
-\def\hrhss@p@skipheader{\toksa{\omit\hbox to 2em{\hfil}&\termspostformat}\concat\table\toksa\restorecs{table-render}{\hrhss@p}}
+\def\hrhss@p@skipheader{\appendrnx\table{\omit\hbox to 2em{\hfil}&\termspostformat}\restorecs{table-render}{\hrhss@p}}
\def\stashtoterm#1{% in case one needs to insert previous stash as a term
% usage: \let\peekstash\stashtoterm
- \yyreadfifo\yystash\to#1\in\yystashlocal
- \unmathstash{#1}%
- \edef\next{\toksb{{%
- \nx\restorecs{table-render}{\nx\stashed\nx\strm}%
- \toksa{}\the\yystashlocal\nx\makestashbox}}}\next
- \def\termmetastyle##1{\the\toksb}%
+ \readstash{#1}%
+ {\edef\next{\toksc{{\toksa{\the\yystashlocal}\nx\makestashbox}}}\expandafter}\next
+ \let\termmetastyle\xtoksc
\restorecs{table-render}\peekstash
}
+\def\xtoksc#1{\the\toksc}%
+
+% resetf: resume normal formatting
+
\def\resetf{%
\restorecs{table-render}{\prodheader\prodhead@r\hrhss@p\rrhss@p\arhss@p\rhs\termname\actbraces\actbrac@s\rarhss@p\hspac@\rules
\oneproduction\emptyterm\stringify\idit\charit\prodprodsep\onesymbol\strm\stopproduction}%
\squashtermsfalse
}
-\def\prodstyle#1{% to typeset token names in text
+\def\prodstyle#1{% to typeset token names in text;
+ % CAUTION: when many parsers are at play at the same time,
+ % one should use the version of this macro below that takes a
+ % namespace as a parameter, to make sure the term is typeset properly
+ % when another namespace might be in effect (such as when section names are output)
{%
\let\optstrextra\optstrextraesc
- \nameproc{#1}%
- \settermstyle{#1}%
+ \def\termidxrank{5}%
+ \let\termindex\writeidxentry
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksa{%
+ \termmetastyle{%
+ \let\nx\idxfont\nx\empty\nx\it\the\toksa
+ \expandafter\gidxentryxv\expandafter{\romannumeral
+ \ifyyparsefail
+ \yybreak{0 \termidstring}%
+ \else
+ \yybreak{\expandafter\tssextract\the\toksa\end}%
+ \yycontinue}{#1}{\the\toksb}\nx\/%
+ }%
+ }%
+ }\next
+ \let\_\uscore
\the\toksa\nobreak
}%
}
+\def\tssextract#1#2#3\end{%
+ \yystringempty{#2}{0 \termidstring}{0 #2}%
+}
+
+\def\prodtstyle#1{% to typeset terminal string names in text;
+ {%
+ \let\optstrextra\optstrextraesc
+ \def\termidxrank{5}%
+ \let\termindex\writeidxentry
+ \nameproc{#1}\with\parsebin
+ \yytoksempty\toksb{\toksb{"#1"}}{}%
+ \edef\next{%
+ \toksa{%
+ \termmetastyle{%
+ \let\nx\idxfont\nx\empty\ntt\the\toksa
+ \gidxentryxv{\termstring}{#1}{\the\toksb}%
+ }%
+ }%
+ }\next
+ \let\_\uscore
+ \the\toksa\nobreak
+ }%
+}
+
+\def\prodstylens#1#2{% to typeset token names in text, in a predefined namespace
+ {%
+ \def\hostparsernamespace{#2}%
+ \prodstyle{#1}%
+ }%
+}
+
% macros used in typesetting the result of the prologue parsing
%type declarations
@@ -1007,7 +1274,7 @@
}
\def\hspace#1#2{%
- \appendr\typesset{ }%
+ \appendrnx\typesset{ }%
}
\def\charit#1#2#3#4{%
@@ -1044,11 +1311,8 @@
}
\def\braceit#1#2#3{% similar to \actbrac@s
- \yyreadfifo\yystash\to#3\in\yystashlocal
- \unmathstash{#3}%
- \restorecs{table-render}{\strm\stashed}%
- \toksa{}\the\yystashlocal
- \edef\next{\toksa{&\toksa{\the\toksa}\nx\makestashbox\nx\cr}}\next
+ \readstash{#3}%
+ \edef\next{\toksa{&\toksa{\the\yystashlocal}\nx\makestashbox\nx\cr}}\next
}
\def\onesymbol#1#2#3{% #1 is the macro name used internally by bison (\idit{}{}{}{})
@@ -1056,42 +1320,42 @@
% #3 is the string value (\stringify{}{}{}{})
\toksa{}\toksb{}\toksc{}%
#1#2#3\relax
- \edef\next{\noexpand\onesymb@l{\the\toksa}{\the\toksc}{\the\toksb}}\next
+ \edef\next{\nx\onesymb@l{\the\toksa}{\the\toksc}{\the\toksb}}\next
}
\def\onesymb@l#1#2#3{% #1 is the macro name used internally by bison
% #2 is the integer value
% #3 is the string value
\let\optstrextra\optstrextraesc
- \yystringempty{#3}{}{\nameproc{#3}\tokstoks\toksa}%
- \yystringempty{#1}{}{\nameproc{#1}}%
- \iftracetokennames\derrmessage{token: \the\toksa, string: \the\tokstoks}\fi
+ \tokse{}\toksf{}\toksg{}% these registers will contain the appropriate visual keys
+ % and index entry processing control sequences
+ \yystringempty{#3}{}{%
+ \nameproc{#3}\with\parsebin
+ \ifyyparsefail % if the parsing failed, create a visible key, process the
+ % index entry as a \.{\\tt} string
+ \tokse{"#3"}\toksf{\termstring}%
+ \else % otherwise use the value of the string as the visual key,
+ % process the string as a \bison\ identifier
+ \tokse\toksb\toksf{\termidstring}%
+ \fi
+ \tokstoks\toksa
+ }%
+ \yystringempty{#1}{}{\nameproc{#1}\with\parsebin\toksg\toksb}%
+ \iftracetokennames\ferrmessage{token: \the\toksa, string: \the\tokstoks}\fi
\toksb\expandafter{\currenttokentype}%
- \edef\next{%
- \tokdectoks{\the\tokdectoks
- \noexpand\toksdefline
- {\the\toksa}%
- {%
- \yystringempty{#1}{}{\gidxentry{\termidstring}{#1}}%
- \yystringempty{#3}{}{\gidxentry{\termstring}{#3}}%
- }%
- {\yytoksempty\toksb{auto}{\the\toksb}}%
- {#2}%
- {\the\tokstoks}%
- }%
- }\next
+ \appendr\tokdectoks{\nx\toksdefline
+ {\the\toksa}%
+ {%
+ \yystringempty{#1}{}{\gidxentryxv{\termidstring}{#1}{\the\toksg}}%
+ \yystringempty{#3}{}{\expandafter\gidxentryxv\expandafter{\the\toksf}{#3}{\the\tokse}}%
+ }%
+ {\yytoksempty\toksb{auto}{\the\toksb}}%
+ {#2}%
+ {\the\tokstoks}%
+ }%
\restorecs{nameparser}{\optstrextra}%
}
-\def\defidxrank{0} % index rank of definitions
-
-\def\writetokenidxentry#1{\write\gindex{%
- \nx\nx
- \expandafter
- \nx
- \csname\bisonidxdomain TI\endcsname\defidxrank#1}%
-}
-
\toyyunion{table-render:token-declarations}
%precedence declarations
@@ -1109,29 +1373,35 @@
}
\def\charit#1#2#3#4{%
- \expandafter\ifx\csname'parser'\hostparsernamespace#1\endcsname\relax
- \edef\next{\toksa{{\ntt #1}\gidxentry{\termvstring}{'#1'}}}%
+ \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
+ \toksa{#1}\sansfirst\toksa
+ \edef\next{\toksa{{\ntt\the\toksa}\gidxentryxb{\termvstring}{\the\toksa}{}}}% remove the possible \\
\else
+ \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
+ \tokse{#1}%
+ \else
+ \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
+ \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
+ \fi
\edef\next{\expandafter\toksa\expandafter{\expandafter\noexpand
- \csname'parser'\hostparsernamespace#1\endcsname{\currentrulecontext}\gidxentry{\termvstring}{'#1'}}}%
+ %\csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termvstring}{'#1'}{\the\tokse}}}%
+ \csname\prettynamecs\hostparsernamespace{#1}\endcsname{\currentrulecontext}\gidxentryxv{\termidstring}{'#1'}{\the\tokse}}}%
\fi
\next
\concat\typesset\toksa
}
\def\anint#1#2#3{%
- \edef\next{\toksa{${}=\hbox{\ntt#1}$}}\next
- \concat\typesset\toksa
+ \appendrnx\typesset{${}=\hbox{\ntt#1}$}%
}
\def\hexint#1#2#3{%
\toksa\expandafter{\eattwo#1}%
- \edef\next{\toksa{${}=\hbox{\ntt\the\toksa}_{16}$}}\next
- \concat\typesset\toksa
+ \appendr\typesset{${}=\hbox{\ntt\the\toksa}_{16}$}%
}
\def\hspace#1#2{%
- \appendr\typesset{ }%
+ \appendrnx\typesset{ }%
}
\def\tagit#1#2#3#4{%
@@ -1153,11 +1423,8 @@
\def\stringify#1#2#3#4{\toksa{#2}}
\def\bracedvalue#1#2#3{% similar to \actbrac@s
- \yyreadfifo\yystash\to#3\in\yystashlocal
- \unmathstash{#3}%
- \restorecs{table-render}{\strm\stashed}%
- \toksa{}\the\yystashlocal
- \edef\next{\toksa{\toksa{\the\toksa}\nx\makestashbox}}\next
+ \readstash{#3}%
+ \edef\next{\toksa{\toksa{\the\yystashlocal}\nx\makestashbox}}\next
}
\toyyunion{table-render:variable-definitions}
@@ -1193,12 +1460,11 @@
\restorecslist{parser-strict}\yyunion
\def\idit#1#2#3#4{%
- \nameproc{#2}%
- \edef\next{\toksa{{%
- \gidxentry{\termttstring}{#1}%
+ \nameproc{#2}\with\parsebin
+ \appendr\typesset{{%
+ \gidxentryxv{\termttstring}{#1}{\the\toksb}%
\ntt\the\toksa
- }}}\next
- \concat\typesset\toksa
+ }}%
}
\def\hspace#1#2{%
@@ -1213,17 +1479,21 @@
\def\flexoptionpair#1#2{%
#1%
- \edef\next{#2}%
- \ifx\next\empty % there is no rihgt hand side
- \edef\next{\toksa{$\nx\langle${\ntt\the\toksa}$\nx\rangle_{\rm f}\,\star$&\cr}}\next
- \else
+ \yystringempty{#2}{% there is no right hand side
+ \appendr\opttable{$\nx\langle${\ntt\the\toksa}$\nx\rangle_{\rm f}\,\star$&\cr}%
+ }{%
\toksb\toksa\toksa{}#2%
- \edef\next{\toksa{$\nx\langle${\ntt\the\toksb}$\nx\rangle_{\rm f}$&\nx\it\the\toksa\nx\/\nx\cr}}\next
- \fi
- \concat\opttable\toksa
+ \appendr\opttable{$\nx\langle${\ntt\the\toksb}$\nx\rangle_{\rm f}$&\nx\it\the\toksa\nx\/\nx\cr}%
+ }%
}
-\def\idit#1#2#3#4{\toksa{#2}}
+\def\idit#1#2#3#4{%
+ \toksa{#2}%
+ \edef\next{\toksa{{%
+ \gidxentry{\termfsopstring}{#2}{}%
+ \ntt\the\toksa
+ }}}\next
+}
\def\stringify#1#2#3#4{\toksa{{\ntt@"#2"}}}
@@ -1244,10 +1514,13 @@
\edef\next{\toksc{\numname:}\toksd{\noexpand\lhs{\the\lastlastname}{\the\lastname}}}\next
}
-\def\rrhssep#1#2{}
-\def\arhssep#1#2{}
-\def\rarhssep#1#2{}
-\def\hspace#1#2{}
+% the macros below are defined in `parser-prototypes'
+%\def\rrhssep#1#2{}
+%\def\arhssep#1#2{}
+%\def\rarhssep#1#2{}
+%\def\hspace#1#2{}
+%\def\prodprodsep{}
+%\def\onesymbol#1#2#3{}
\def\rhs#1#2#3{%
\concat\toksa\toksc
@@ -1317,10 +1590,6 @@
\lastname{'#1'}%
}
-\def\prodprodsep{}
-
-\def\onesymbol#1#2#3{}
-
\def\anint#1#2#3{%
\lastname{#1}%
}
@@ -1342,119 +1611,72 @@
\savecslist{symbols}\yyunion
-% name parser macros
+%% token typesetting macros
-\def\idstr#1#2{%
- \toksb{#1}%
- \concat\toksa\toksb
- \toksb{#2}%
- \concat\toksc\toksb
-}
+\def\prettynamecs#1#2{'parser'#1#2} % naming convention for token typesetting alternatives
-\def\chstr#1#2{%
- \toksb{{\ntt@#1}}%
- \concat\toksa\toksb
- \toksb{#2}%
- \concat\toksc\toksb
+\def\setsafe#1{\def\saferword{#1}\setspecialcharsfrom\saferword}
+\def\prettyword#1{\setsafe{#1}\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{{\ntt\lowercase{\saferword}}}}
+\def\prettytoken#1{\setsafe{#1}\expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{{\ntt\saferword}}}
+\def\prettytoken@#1{\toksa{}\numberstocharsandspaces#1\end\expandafter\edef
+ \csname\prettynamecs\parsernamespace{\the\toksa}\endcsname##1{{\ntt\the\toksa}}%
}
-\def\sfxi#1#2{\toksb{\/${}_{\idxfont#1}$}\concat\toksa\toksb}
-
-\def\sfxn#1#2{\toksb{#1}\concat\toksa\toksb}
+\def\prettywordpair@@#1#2{\setsafe{#1}\expandafter\def\csname\prettynamecs\parsernamespace\saferword\endcsname##1{#2}}
-\def\dotsp{\futurelet\next\d@tsp}
+\def\prettywordpair#1#2{\prettywordpair@@{#1}{{\ntt@#2}}}
-\def\d@tsp{%
- \ifx\next\sfxi
- \else
- \ifx\next\qual
- \else
- \toksb{.}\concat\toksa\toksb
- \fi
- \fi
+\def\prettywordpair@#1#2{% a `weak symbol' version of the above macro, so that the new typesetting can be overriden
+ \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end\expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
+ \expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
+ \noexpand\ifdisplaytokenraw
+ {\ntt\saferword}% \ntt expands to \noexpand\ntt@
+ \noexpand\else
+ {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa}%
+ \noexpand\fi
+ }%
}
-\def\optstr#1#2{%
- \toksb\expandafter{\eatone#1}%
- \optstrextra
- \concat\toksa\toksb
- \toksb{#2}%
- \concat\toksc\toksb
+\def\prettywordpairwvis#1#2#3{%
+ \prettywordpair{#1}{#2}%
+ \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}
-\def\optstrextra{%
- \edef\next{\toksb{{$\nx\ulcorner$\ntt\the\toksb$\nx\urcorner$}}}\next
+\def\prettywordpair@wvis#1#2#3{%
+ \prettywordpair@{#1}{#2}%
+ \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}
-\def\optstrextraesc{%
- \edef\next{\toksb{{$\nx\langle$\ntt\the\toksb$\nx\rangle$}}}\next
+\def\prettywordpair@@wvis#1#2#3{%
+ \prettywordpair@@{#1}{#2}%
+ \expandafter\def\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{#3}%
}
-\def\qual#1#2{\toksb{\/${}_{\rm#2}$}\concat\toksd\toksb}
-
-\let\idxfont\relax
-
-\savecs{nameparser}{\idstr\chstr\sfxi\sfxn\qual\optstr\dotsp\optstringextra\idxfont}
+% a control sequence to reuse the token definition file for typesetting
-\newtoks\namechars
+\newif\iftraceprettytokens % display prettyfying control sequences
-\newif\iftracebadnames
-\newif\iftracenames
-
-\def\nameproc#1{%
- \ifbootstrapmode
- \toksa{}%
+\def\tokeneqpretty#1#2{%
+ \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end
+ \expandafter\nameproc\expandafter{\the\toksa}\with\parsebin
+ \ifyyparsefail
+ \toksc{{}$_{\rm m}${}}\toksd\toksa
\else
- \savecs{local-namespace}\getcurrentparser
- \tosmallparser
- \basicparserinit
- \yyparse#1\yyeof\yyeof\endparseinput\endparse
- \ifyyparsefail
- \iftracebadnames
- \ferrmessage{bad name: #1}%
- \fi
- \toksa{#1}\toksc{}%
- \else
- \toksa{}\toksb{}\toksc{}\toksd{}%
- \iftracenames
- \ferrmessage{parsed name: \the\namechars}%
- \fi
- \the\namechars\relax
- \iftracenames
- \ferrmessage{processed name: \the\toksa<identifier>, \the\toksb<suffices>, \the\toksc<hash name>, \the\toksd<options>}%
- \fi
- \expandafter\ifx\csname'parser'\hostparsernamespace\the\toksc \endcsname\relax
- \else
- \toksa\expandafter{\csname'parser'\hostparsernamespace\the\toksc \endcsname}%
- \appendr\toksa{{\currentrulecontext}}%
- \fi
- \concat\toksa\toksd
- \fi
- \restorecs{local-namespace}\getcurrentparser
- \getcurrentparser
+ \toksc{}\toksd\toksb % quotes should trigger the creation of a visible key
\fi
-}
-
-%% token typesetting macros
-
-\def\setsafe#1{\def\saferword{#1}\setspecialcharsfrom\saferword}
-\def\prettyword#1{\setsafe{#1}\expandafter\edef\csname'parser'\parsernamespace\saferword\endcsname##1{{\ntt\lowercase{\saferword}}}}
-\def\prettytoken#1{\setsafe{#1}\expandafter\edef\csname'parser'\parsernamespace\saferword\endcsname##1{{\ntt\saferword}}}
-\def\prettytoken@#1{\toksa{}\numberstocharsandspaces#1\end\expandafter\edef
- \csname'parser'\parsernamespace\the\toksa\endcsname##1{{\ntt\the\toksa}}%
-}
-\def\prettywordpair#1#2{\setsafe{#1}\expandafter\def\csname'parser'\parsernamespace\saferword\endcsname##1{{\ntt@#2}}}
-\def\prettywordpair@#1#2{%
- \setsafe{#1}\toksa{}\numberstocharsandspaces#2\end\expandafter\nameproc\expandafter{\the\toksa}%
- \expandafter\edef\csname'parser'\parsernamespace\saferword\endcsname##1{%
+ \expandafter\edef\csname\prettynamecs\parsernamespace\saferword\endcsname##1{%
\noexpand\ifdisplaytokenraw
{\ntt\saferword}%
\noexpand\else
- {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa}%
+ {\ntt\def\noexpand\_{\char`\noexpand\_}\the\toksa\the\toksc}%
\noexpand\fi
}%
+ \expandafter\edef\csname\viskeyref\parsernamespace{\saferword}\endcsname##1{\the\toksd}%
+ \iftraceprettytokens
+ \message{token: \expandafter\meaning\csname\prettynamecs\parsernamespace\saferword\endcsname}%
+ \message{visual key: \expandafter\meaning\csname\viskeyref\parsernamespace{\saferword}\endcsname}%
+ \fi
}
-\def\prettywordpair@@#1#2{\setsafe{#1}\expandafter\def\csname'parser'\parsernamespace\saferword\endcsname##1{{#2}}}
% the bootstrap macros
@@ -1471,7 +1693,7 @@
\def\idit#1#2#3#4{\toksb{#1}}
%\def\anint#1#2#3{\toksc{#1}}% this definition is not needed as all
- %information is extracted from yytname
+ % the information is extracted from yytname
% note that the definitions below do not handle cases such as
%token '{' "left brace" since the lexer knows how to return '{'
@@ -1509,15 +1731,923 @@
\savecslist{bootstrap:flex-state-declarations}\yyunion
-\def\charstonumberse#1\end{%
- \yystringempty{#1}{}%
- {\yystartsinspace{#1}{{\number`\ }\charstonumber@@#1\end}{\charstonumbers@#1\end}}%
+% correct a few macros for debugging
+
+\restorecslist{parser-debug}\yyunion
+
+\defc\actbraces{% action braces :: \actbraces{tex_string}{\idit|<nothing>}{fptr}{sptr}directive*\bdend
+ \noexpand\actbraces{#1}{#2}{#3}{#4}#5\noexpand\bdend
+}
+
+\defc\bpredicate{% predicate :: \bpredicate{tex_string}{<nothing>}{fptr}{sptr}directive*\bdend
+ \noexpand\bpredicate{#1}{}{#3}{#4}#5\noexpand\bdend
+}
+
+\defc\rhs{% a `right-hand side' :: \rhs{[\termname\hspace\rarhssep\arhssep\actbraces]+}{\arhssep|<nothing>}{\ifrhsfull}
+ \noexpand\rhs{#1}{#2}{}% omit setting the conditional
+}
+
+\toyyunion{parser-debug}
+
+% \romannumeral-expandable macros for character translation
+
+\def\charstonumbersre#1\end{\charstonumbersr@0\end#1\end}
+
+\def\charstonumbersr@#1\end#2\end{%
+ \yystringempty{#2}{#1}%
+ {\yystartsinspace{#2}{\expandafter\charstonumbers@@swap\expandafter{\number`\ }{#1}#2\end}{\charstonumb@rsr@#1\end#2\end}}%
}
-\def\charstonumbers@#1{%
- {\number`#1}\charstonumberse
+\def\charstonumb@rsr@#1\end#2{%
+ \expandafter\charstonumbersr@swap\expandafter{\number`#2}{#1}%
}
-\def\charstonumber@@#1 {\charstonumberse}
+\def\charstonumbersr@swap#1#2{\charstonumbersr@#2{#1}\end}
+
+\def\charstonumbers@@swap#1#2 {\charstonumbersr@#2{#1}\end}
+
+% variations: set \toksa to the expansion, \edef-expandable version
+
+\def\charstonumbers#1\end{\toksa\expandafter{\romannumeral\charstonumbersre#1\end}}
+\def\charstonumberse#1\end{\romannumeral\charstonumbersre#1\end}
+
+% name parser macros: currently only one suffix is handled, the very last one
+
+\def\yyuniontag{\yynpunion}
+\def\parserstrictnamespace{nameparser-strict}
+\def\parserprototypesnamespace{nameparser-strict:headers}
+\def\parserdebugnamespace{nameparser-debug}
+
+\def\yynpunion{\currentyyunionnamespace}
+
+\defp\idstr#1#2{} % idenifier ::
+\defp\bidstr#1#2{} % bison variable ::
+\defp\chstr#1#2{} % character ::
+\defp\sfxi#1#2{} % literal suffix ::
+\defp\sfxn#1#2{} % integer suffix ::
+\defp\dotsp{} % suffix separator ::
+\defp\optstr#1#2{} % option ::
+\defp\qual#1#2{} % qualifier ::
+\defp\visflag#1#2{} % visible key change flag ::
+
+\toyyunion{nameparser-prototypes}
+
+\defc\idstr{%
+ \appendrnx\toksa{#1}%
+ \appendrnx\toksc{#2}%
+ \toksb{}%
+}
+
+\defc\bidstr{%
+ \appendrnx\toksa{{}$\Upsilon${}}%
+ \appendrnx\toksc{#2}%
+ \toksb{}%
+}
+
+\defc\chstr{%
+ \appendrnx\toksa{{\ntt@#1}}%
+ \appendrnx\toksc{#2}%
+ \toksb{}%
+}
+
+% the \toksb, set in the next two macros is used by \optstr later
+
+\defc\sfxi{\toksb{\/${}_{\idxfont#1}$}\appendrnx\toksa{\/${}_{\idxfont#1}$}}
+
+\defc\sfxn{\toksb{#1}\appendrnx\toksa{#1}}
+
+\defc\dotsp{\futurelet\next\d@tsp}
+
+\def\d@tsp{%
+ \ifx\next\sfxi
+ \else
+ \ifx\next\qual
+ \else
+ \appendrnx\toksa{.}%
+ \fi
+ \fi
+}
+
+\defc\optstr{%
+ \toksb\expandafter{\eatone#1}%
+ \optstrextra
+ \concat\toksa\toksb
+ \appendrnx\toksc{#2}%
+ \toksb{}%
+}
+
+\def\optstrextra{%
+ \edef\next{\toksb{{$\nx\ulcorner$\ntt\the\toksb$\nx\urcorner$}}}\next
+}
+
+\def\optstrextraesc{%
+ \edef\next{\toksb{{$\nx\langle$\ntt\the\toksb$\nx\rangle$}}}\next
+}
+
+\defc\qual{\appendrnx\toksd{\/${}_{\rm#2}$}\toksb{}}
+
+\let\idxfont\relax
+
+\defc\visflag{\toksf{#1}\appendrnx\tokse{#1}}% flag visible key change
+ % \toksf contains the typesetting hint sequence
+
+\toyyunion{nameparser}
+
+\newtoks\namechars
+
+\newif\iftracebadnames
+\newif\iftracenames
+
+\def\viskeyref#1#2{% getting the possible visual key from the index key
+ % #1 is the namespace of the original key
+ % #2 is the key
+ \restorecsname{[nameparser:visual]}{.\prettynamecs{#1}{#2}}%
+}
+
+% ignore explicit (\ ) spaces while parsing names
+% the macro will be called whenever \tosmallparser is executed (see yyinit.sty);
+% this is neccessary due to a shortcut taken by the macros that display
+% \bison\ productions in text (see \beginprod macros in yyinit.sty).
+
+\def\ignoreexplicitspace{\yyinput}
+
+% the main name processing routine
+% does not have any side effects;
+% the parsed term is returned as #1 of the sequence in #2; #2 of #2 will hold a visual key if
+% one exists, otherwise it will be empty; #3 is \yyparsefailtrue or empty
+% #4 is the prettyfied version of the term if one exists; #5 is the typesetting hint from \visflag sequences;
+% the version below prettifies tokens only if the parsing was successful; if
+% needed, prettyfication may be applied after the parsing is complete
+
+\def\nameproc#1\with#2{%
+ {% keeping all changes local
+ \iftracenames
+ \toksa{#1}\ferrmessage{original name: \the\toksa}%
+ \fi
+ \tosmallparser\basicparserinit\d@parse{#1}%
+ \ifyyparsefail
+ \iftracebadnames
+ \toksa{#1}\ferrmessage{bad name: \the\toksa}%
+ \fi
+ \toksb{#2{#1}{}{\yyparsefailtrue}{}{}}%
+ \else
+ \npbuildnames{#2}%
+ \fi
+ \expandafter
+ }\the\toksb
+}
+
+\def\npbuildnames#1{%
+ \iftracenames
+ \ferrmessage{parsed name (namespace: \parsernamespace): \the\namechars}%
+ \fi
+ \toksa{}\toksb{}\toksc{}\toksd{}\tokse{}\toksf{}%
+ \restorecslist{nameparser}\yynpunion
+ \the\namechars\relax
+ \iftracenames
+ \ferrmessage{processed name: \the\toksa<identifier>, \the\toksb<suffices>, %
+ \the\toksc<hash name>, \the\toksd<options>, \the\tokse<visible key>}%
+ \fi
+ \expandafter\npsetkeys\expandafter{\the\toksc}%
+ \iftracenames
+ \ferrmessage{final name: \the\toksa\yytoksempty\toksc{}{, prettyfied name: \the\toksc}%
+ \yytoksempty\toksf{}{, hint: \the\toksf}%
+ \yytoksempty\tokse{}{, visual key: \the\tokse}, %
+ parsing \ifyyparsefail failed\else successful\fi
+ }%
+ \fi
+ \toksb{#1}%
+ \edef\next{%
+ \toksb{%
+ \the\toksb
+ {\the\toksa}% the parsed name
+ {\the\tokse}% the visual key
+ {}% parseflag
+ {\the\toksc}% the prettified name
+ {\the\toksf}% the typesetting hint
+ }%
+ }\next
+}
+
+\def\npsetkeys#1{%
+ \expandafter\ifx\csname\prettynamecs\hostparsernamespace{#1}\endcsname\relax
+ \toksc{}%
+ \else
+ \toksc\expandafter{\csname\prettynamecs\hostparsernamespace{#1}\endcsname}%
+ \appendr\toksc{{\currentrulecontext}\the\toksb\the\toksd}%
+ \toksf{\termidstring}%
+ \fi
+ \concat\toksa\toksd % changes \toksa and \toksc, no need for \toksb or \toksd after this
+ \expandafter\ifx\csname\viskeyref\hostparsernamespace{#1}\endcsname\relax
+ % no special visual key for this token
+ \yybreak{%
+ \yytoksempty\tokse{}{\makeviskeydirect}% if the visual key changed, output the key,
+ % attempt to compute it form the parsed output
+ }%
+ \else
+ \yybreak{\makeviskeyfrompretty{#1}}% modify the production of the visible key to use the
+ % prettified version
+ \yycontinue % changes \toksb, \toksd, and \tokse
+}
+
+% standard postprocessing function
+
+\def\parsebin#1#2#3#4#5{%
+ \yystringempty{#4}{%
+ \yystringempty{#3}{\toksa{\eatone{#5}#1}}{\toksa{#1}}%
+ }{%
+ \yystringempty{#3}{\toksa{\eatone{#5}#4}}{\toksa{#4}}%
+ }%
+ \toksb{#2}%
+ \yystringempty{#3}{\yyparsefailfalse}{\yyparsefailtrue}%
+}
+
+% creating a visual key from the stored visual key and the sufixes
+
+\restorecslist{nameparser-prototypes}\yynpunion % make most sequences do nothing
+
+\defc\qual{\toksb{#2}\concat\toksd\toksb}
+
+\toyyunion{nameparser:vispretty}
+
+\def\makeviskeyfrompretty#1{%
+ \edef\next{\tokse{\expandafter\expandafter\csname\viskeyref
+ \hostparsernamespace{#1}\endcsname\expandafter{\currentrulecontext}}}\next
+ \restorecslist{nameparser:vispretty}\yynpunion%
+ \toksd{}%
+ \the\namechars
+ \yytoksempty\toksd{}{\appendr\tokse.\concat\tokse\toksd}%
+}
+
+% creating a visual key directly from parsed output
+
+\restorecslist{nameparser-strict}\yynpunion
+
+\defc\idstr{\toksb{#2}\concat\tokse\toksb}
+
+\defc\bidstr{\toksb{#2}\concat\tokse\toksb}
+
+\defc\chstr{\toksb{#2}\concat\tokse\toksb}
+
+\defc\sfxi{\toksb{#2}\concat\tokse\toksb}
+
+\defc\sfxn{\toksb{#2}\concat\tokse\toksb}
+
+\defc\dotsp{\appendr\tokse.}
+
+\defc\optstr{\toksb{#2}\concat\tokse\toksb}
+
+\defc\qual{\toksb{#2}\concat\tokse\toksb}
+
+\defc\visflag{}
+
+\toyyunion{nameparser:visdirect}
+
+\def\makeviskeydirect{%
+ \tokse{}%
+ \restorecslist{nameparser:visdirect}\yynpunion
+ \the\namechars
+}
+
+\restorecslist{nameparser-strict}\yynpunion
+
+% \flex\ regex typesetting routine and \flex\ parser value stack types
+
+\def\frexproc#1\with#2{%
+ {% keeping all changes local
+ \iftracenames
+ \toksa{#1}\ferrmessage{original regex: \the\toksa}%
+ \fi
+ \toflexreparser
+ \basicparserinit
+ \flexreparserinit
+ \flexreparserdatainit
+ \d@parse{\ #1}% the parsed regex is put in the \table register
+ \ifyyparsefail
+ \iftracebadnames
+ \toksa{#1}\ferrmessage{bad regex: \the\toksa}%
+ \fi
+ \toksb{#2{#1}{}{\yyparsefailtrue}}%
+ \else
+ \toksd{#1}%
+ \frebuildx{#2}%
+ \fi
+ \expandafter
+ }\the\toksb
+}
+
+\def\frebuildx#1{%
+ \toksb{#1}\toksc{}%
+ \restorecslist{flexparser-re}\yyflunion
+ \the\table % typeset the regex (appears in \toksc)
+ \edef\next{%
+ \toksb{%
+ \the\toksb % the name of the processing routine
+ {\the\toksd}% the original regex
+ {\the\toksc}% processed regex (or empty)
+ {}% parse flag
+ }%
+ }\next
+}
+
+\def\ppregex#1#2#3{%
+ \toksc{#2}%
+}
+
+\def\setregextable{%
+ \restorecslist{flexparser-sect2}\yyflunion
+}
+
+\def\setregexdeftable{%
+ \restorecslist{flexparser-sect1}\yyflunion
+}
+
+% \flex\ parser indexing macros
+
+\def\flexidxdomain{F}
+\def\fstatedefidxrank{0}
+\def\fstateidxrank{1}
+\def\fregexidxrank{2}
+\def\ftextrefrank{5}
+
+\def\writeidxfsentry#1{% \flex\ state reference entry
+ \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fstateidxrank}#1}%
+}
+
+\def\writeidxfstextentry#1{% \flex\ state reference in text
+ \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\ftextrefrank}#1}%
+}
+
+\def\writeidxfsdentry#1{% \flex\ state definition entry
+ \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fstatedefidxrank}#1}%
+}
+
+\def\writeidxfsrdentry#1{% \flex\ regex name definition entry
+ \indxe\gindex{{\secno}{{}{\flexpseudorenamespace}}{\flexidxdomain}{\fstatedefidxrank}#1}%
+}
+
+\def\writeidxfsrnentry#1{% \flex\ regex name entry
+ \indxe\gindex{{\secno}{{}{\flexpseudorenamespace}}{\flexidxdomain}{\fregexidxrank}#1}%
+}
+
+\def\writeidxfscclentry#1{% \flex\ chracter class name entry
+ \indxe\gindex{{\secno}{{}{\flexpseudonamespace}}{\flexidxdomain}{\fregexidxrank}#1}%
+}
+
+% \flex\ parser typesetting macros and stack types
+
+\def\yyuniontag{\yyflunion}
+\def\parserstrictnamespace{flexparser-strict}
+\def\parserprototypesnamespace{flexparser-strict:headers}
+\def\parserdebugnamespace{flexparser-debug}
+
+\def\yyflunion{\currentyyunionnamespace}
+
+\defp\flnametok#1#2{} % named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
+%
+\defp\flcclexpr#1#2#3#4{} % character class expression :: \flcclexpr{matched text}{ccl token}{fptr}{sptr}
+\defp\flchar#1#2#3#4{} % character :: \flchar{char}{char12}{fptr}{sptr}
+\defp\flstring#1#2#3{} % string :: \flstring{{fptr}{sptr}}{\flchar{...}*}{{fptr}{sptr}}
+\defp\flcclrnge#1#2{} % character range :: \flcclrnge{/flchar{...}}{/flchar{...}}
+\defp\flbraceccl#1#2#3{} % character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
+\defp\flbracecclneg#1#2#3{} % negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
+\defp\flcclunion#1#2{} % union of character classes :: \flcclunion{ccl}{ccl}
+\defp\flccldiff#1#2{} % difference of character classes :: \flccldiff{ccl}{ccl}
+\defp\flrepeat#1{} % asterisk operation :: \flrepeat{re}
+\defp\flrepeatstrict#1{} % plus operation :: \flrepeatstrict{re}
+\defp\flrepeatonce#1{} % ? operation :: \flrepeatonce{re}
+\defp\flrepeatnm#1#2#3{} % {n, m} operation :: \flrepeatnm{re}{num}{num}
+\defp\flrepeatgen#1#2{} % {n,} operation :: \flrepeatgen{re}{num}
+\defp\flrepeatn#1#2{} % {n} operation :: \flrepeatn{re}{num}
+\defp\flparens#1#2#3#4{} % parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
+\defp\fldot#1#2{} % dot expression :: \fldot{fptr}{sptr}
+\defp\flor#1#2{} % `|' expression :: \flor{fptr}{sptr}
+\defp\fltrail#1#2{} % regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
+\defp\flretrail#1#2#3{} % regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
+\defp\flreateol#1#2#3{} % regular expression with the end of line as a trailing context :: \flreateol{re}{fptr}{sptr}
+\defp\flrule#1#2{} % full regular expression :: \flrule{re}{formatting command}
+\defp\flbolrule#1#2{} % full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
+\defp\fleof#1#2{} % end of file rule :: \fleof{fptr}{sptr}
+\defp\flsconlist#1#2#3{} % start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
+\defp\flsconuniv#1#2{} % universal start condition :: \flsconuniv{fptr}{sptr}
+\defp\flnamesep#1#2{} % separator between names :: \flnamesep{fptr}{sptr}
+\defp\flname#1#2#3#4{} % name :: \flname{name}{name12}{fptr}{sptr}
+\defp\flopt#1#2#3#4#5{} % \flex option :: \flopt{option}{text}{text12}{fptr}{sptr}
+\defp\floptions#1{} % \flex options :: \floptions{\flopt ...}
+\defp\flscondecl#1#2#3#4{} % start condition declarations :: \flscondecl{x|s}{fptr}{sptr}{\flname ...}
+\defp\flaction#1#2#3#4#5#6#7{} % \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+\defp\flactionc#1#2#3#4#5#6#7{} % \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+\defp\flactiongroup#1#2#3#4#5#6{} % a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
+\defp\flbareaction#1#2#3#4#5#6{} % bare action :: \flbareaction{fptr}{sptr}
+\defp\flptropt#1#2{} % %ponter directive :: \flptropt{fptr}{sptr}
+\defp\flarrayopt#1#2{} % %array directive :: \flarrayopt{fptr}{sptr}
+\defp\fltopopt#1#2#3#4{} % %top directive :: \fltopopt{fptr}{sptr}{fptr}{sptr}
+\defp\flredef#1#2#3#4#5#6#7#8{} % regular expression definition :: \flredef{name}{name12}{fptr}{sptr}{text}{text12}{fptr}{sptr}
+
+\toyyunion{flexparser-prototypes}
+
+\restorecslist{flexparser-strict}\yyflunion
+
+\defc\flrule{%#1#2% full regular expression :: \flrule{re}{formatting command}
+ {\toksc{}#1#2\expandafter}\expandafter\toksd\expandafter{\the\toksc}%
+ \concat\toksc\toksd
+}
+
+\newif\iftextre % check if this is a free standing regular expression or appears in the definitions table
+
+\defc\flbolrule{%#1#2% full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
+ {\toksc{}#1#2\appendlnx\toksc{\raise0.5pt%
+ \iftextre\yybreak{\hbox}\else\yybreak{\llap}\yycontinue{\sscmd$\dashv$\ }}\expandafter}\expandafter%
+ \toksd\expandafter{\the\toksc}\concat\toksc\toksd
+}
+
+\let\astformat@flrule\empty
+
+\def\inscomment#1{% insert a comment before the rule
+ \def\astformat@flrule{%
+ \appendlnx\toksc{\noalign{#1}}%
+ }%
+}
+
+\def\insrulealign#1#2{%
+ \def\astformat@flrule{%
+ \tokse{#1}\toksf{#2}%
+ \concatl\tokse\toksc
+ \concat\toksc\toksf
+ }%
+}
+
+\def\rulealigntemplate{% to hide the hash symbols from \\edef operators
+ \vbox\bgroup\halign\bgroup##\hfil&##\hfil\cr
+}
+
+% regular expression symbols
+
+\defc\flstring{%#1#2#3% string :: \flstring{{fptr}{sptr}}{\flchar{...}*}{{fptr}{sptr}}
+% \toksc\expandafter{\the\toksc\hbox{#2}}%
+ #2%
+}
+
+\defc\flnametok{%#1#2% named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
+ \let\termindex\writeidxfsrnentry
+ \fln@metok#1\end{#2}%
+ \let\termindex\eatone
+}
+
+\def\fln@metok\{#1\}#2\end#3{%
+ {%
+ \let\hostparsernamespace\flexpseudorenamespace
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \expandafter
+ }\the\toksd
+ \toksf{}\toksg{}%
+ #3%
+ \edef\next{%
+ \toksc{\the\toksc\the\toksf{}\nx\flexrendisplay{\the\toksd}%
+ \gidxentryxv{\termfsrestring}{#1}{\the\tokse}\the\toksg
+ }%
+ }\next
+}
+
+\def\insnamealign#1#2{%
+ \def\astformat@flnametok{%
+ \toksf{#1}%
+ \toksg{#2}%
+ }%
+}
+
+\let\astformat@flnametok\empty
+
+\defc\flchar{%#1#2#3#4% character :: \flchar{char}{char12}{fptr}{sptr}
+ \yystringempty{#2}{#1}{\flch@r#2\end}%
+}
+
+\def\flch@r#1#2\end{%
+ \ifnum`#1=`\\
+ \fl@h@r#2\end
+ \else
+ \toksc\expandafter{\the\toksc{\ntt#1#2}}%
+ \fi
+}
+
+\def\fl@h@r#1#2\end{% process the sequence after an escape
+ {%
+ \let\default\flescdefault
+ \tokse{#1}\toksd{#2}%
+ \switchon{\the\tokse}\in\flesccharswitch
+ \edef\next{\toksc{\toksc{\the\toksc\the\toksd}}}\next
+ \expandafter
+ }\the\toksc
+}
+
+\def\flesccharswitch{% recognizing different types of escape sequences
+01234567
+ {% octal constant
+ \edef\next{\toksd{\nx\.{\the\tokse\the\toksd}$_8${}}}\next
+ }
+nfrtabv
+ {% \Cee\ escape sequence
+ \edef\next{\toksd{{\nx\sscmd\nx\sscmd$\nx\langle$\nx\.{\the\tokse\the\toksd}$\nx\rangle$}}}\next
+ }
+x
+ {% hexadecimal constant
+ \edef\next{\toksd{\nx\.{\the\toksd}$_{16}${}}}\next
+ }
+}
+
+\def\flescdefault{%
+ \edef\next{\toksd{\nx\.{\the\tokse\the\toksd}}}\next
+}
+
+\setspecialcharsfrom\flesccharswitch
+
+\defc\fldot{%#1#2% dot expression :: \fldot{fptr}{sptr}
+ \toksc\expandafter{\the\toksc.}%
+}
+
+\defc\flparens{%#1#2#3#4% parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{format command}
+ {\toksc{}#2\expandafter}\expandafter\toksd\expandafter
+ {\the\toksc{\rm)}}%
+ \toksf{}\toksg{}#4%
+ \tokse\expandafter{\the\toksf{\rm(}}%
+ \concatl\tokse\toksd
+ \concat\toksc\toksd
+ \concat\toksc\toksg
+}
+
+\def\insparensalign#1#2{%
+ \def\astformat@flparens{%
+ \toksf{#1}%
+ \toksg{#2}%
+ }%
+}
+
+\let\astformat@flparens\empty
+
+\defc\flrepeat{%#1% asterisk operation :: \flrepeat{re}
+ {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
+ {\the\toksc{{}${}_{*}$}}%
+ \flselectiveseparator
+ \concat\toksc\toksd%
+}
+
+\defc\flrepeatn{%#1#2% {n} operation :: \flrepeatn{re}{num}
+ {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
+ {\the\toksc{{}${}_{\{\yyfirstoftwo#2\}}$}}%
+ \flselectiveseparator
+ \concat\toksc\toksd%
+}
+
+\defc\flrepeatnm{%#1#2#3% {n, m} operation :: \flrepeatnm{re}{num}{num}
+ {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
+ {\the\toksc{{}${}_{\{\yyfirstoftwo#2,\yyfirstoftwo#3\}}$}}%
+ \flselectiveseparator
+ \concat\toksc\toksd%
+}
+
+\defc\flrepeatonce{%#1% ? operation :: \flrepeatonce{re}
+ {\toksc{}#1\expandafter}\expandafter\toksd\expandafter
+ {\the\toksc{{}${}_{?}$}}%
+ \flselectiveseparator
+ \concat\toksc\toksd%
+}
+
+\defc\flrepeatstrict{%#1% plus operation :: \flrepeatstrict{re}
+ {\toksc{}#1\expandafter}\expandafter\toksd\expandafter{\the\toksc{{}${}_{+}$}}%
+ \flselectiveseparator
+ \concat\toksc\toksd%
+}
+
+\def\flselectiveseparator{%
+ \yytoksempty\toksc{}{%
+ \tokse{{{}$\,$}}%
+ \concatl\tokse\toksd
+ }%
+}
+
+\defc\flcclrnge{%#1#2% character range :: \flcclrnge{\flchar{...}}{\flchar{...}}
+% \toksd{#1}\getfirst\toksd\to\toksd
+% \tokse{#2}\getfirst\tokse\to\tokse
+ #1\appendr\toksc{{\nx\rm--}}#2%
+% \edef\next{\toksc{\the\toksc\the\toksd{\nx\rm--}\the\tokse}}\next
+}
+
+\defc\flcclexpr{%#1#2#3#4% character class expression :: \flcclexpr{matched text}{ccl token}{fptr}{sptr}
+ {%
+ \let\hostparsernamespace\flexpseudonamespace
+ \nameproc{#2}\with\parsebin
+ \edef\next{%
+ \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \expandafter
+ }\the\toksd
+ \let\termindex\writeidxfscclentry
+ \edef\next{\toksc{\the\toksc{\the\toksd}%
+ \gidxentryxv{\termidstring}{#2}{\the\tokse}%
+ }}\next
+ \let\termindex\eatone
+}
+
+\defc\flbraceccl{%#1#2#3% character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
+ \appendrnx\toksc{{}$[${}}%
+ #2%
+ \appendrnx\toksc{{}$]${}}%
+}
+
+\def\flbraceccldemo#1#2#3{% character class :: \flbraceccl{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
+ % a demo version without the beginning bracket
+ \appendrnx\toksc{{}$\ldots${}}%
+ #2%
+ \appendrnx\toksc{{}$]${}}%
+}
+
+\defc\flccldiff{%#1#2% difference of character classes :: \flccldiff{ccl}{ccl}
+ #1%
+ \appendr\toksc{${}\nx\setminus{}$}%
+ #2%
+}
+
+\defc\flbracecclneg{%#1#2#3% negative character class :: \flbracecclneg{{fptr}{sptr}}{ccl}{{fptr}{sptr}}
+ \appendr\toksc{{}$[${}}%
+ #2%
+ \appendr\toksc{{}$]^c${}}%
+}
+
+\defc\fleof{%#1#2% end of file rule :: \fleof{fptr}{sptr}
+ \let\termindex\writeidxfsrnentry
+ \edef\next{\toksc{\the\toksc{}\nx\flexrendisplay{EOF}%
+ \gidxentry{\termfsrestring}{EOF}{EOF}}}\next
+ \let\termindex\eatone
+}
+
+\defc\flor{%#1#2% `|' expression :: \flor{fptr}{sptr}
+ \appendr\toksc{{}${}\nx\mid{}${}}%
+}
+
+\defc\flsconuniv{%#1#2% universal start condition :: \flsconuniv{fptr}{sptr}
+ \let\termindex\writeidxfsentry
+ \edef\next{\toksb{\the\toksb{}$\langle*\rangle${}%
+ \gidxentryxv{\termvstring}{*}{*}}}\next % TODO: create a separate name space for state names
+ \let\termindex\eatone
+}
+
+\defc\fltrail{%#1#2% regular expression sans trailing suffix :: \fltrail{re}{{fptr}{sptr}}
+ \errmessage{\nx\fltrail\space macro has not been implemented (see yyunion.sty).}
+}
+
+\defc\flretrail{%#1#2#3% regular expression with a trailing context :: \flretrail{re}{{fptr}{sptr}}{re}
+ \errmessage{\nx\flretrail\space macro has not been implemented (see yyunion.sty).}
+}
+
+\defc\flreateol{%#1#2#3% regular expression with the end of line as a trailing context :: \flreateol{re}{fptr}{sptr}
+ {\toksc{}#1\expandafter}\expandafter\toksd\expandafter{\the\toksc{\raise0.5pt\rlap{\sscmd\ $\vdash$}}}%
+ \concat\toksc\toksd%
+}
+
+\toyyunion{flexparser-re}
+
+\def\flexsndisplay#1{% state name typesetting style
+ \hbox{\tt#1}%
+}
+
+\def\flexrendisplay#1{% regular expression name typesetting style
+ \penalty-100 \hbox{$\langle$\tt#1$\rangle$}%
+}
+
+\def\flexsnstyle#1{% typesetting \flex\ state names in text
+ {%
+ \let\parsernamespace\flexpseudonamespace
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksd{\the\toksa}\tokse{\the\toksb}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \let\termindex\writeidxfstextentry
+ \edef\next{\toksd{\nx\flexsndisplay{\the\toksd}%
+ \gidxentryxv{\termvstring}{#1}{\the\tokse}}}\next
+ \expandafter
+ }\the\toksd
+}
+
+\def\flexrenstyle#1{% typesetting \flex\ regular expression names in text
+ {%
+ \let\parsernamespace\flexpseudorenamespace
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksd{\the\toksa}\tokse{\the\toksb}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \let\termindex\writeidxfstextentry
+ \edef\next{\toksd{\nx\flexrendisplay{\the\toksd}%
+ \gidxentryxv{\termfsrestring}{#1}{\the\tokse}}}\next
+ \expandafter
+ }\the\toksd
+}
+
+\def\flexrestyle#1{% typesetting \flex\ regular expressions in text
+ {%
+ \frexproc{#1}\with\ppregex
+ \edef\next{\toksc{\toksc{\the\toksc}}}\next
+ \expandafter
+ }\the\toksc
+ \hbox{\textretrue\tt\the\toksc}%
+}
+
+\defc\flactiongroup{% #1#2#3#4#5#6% a group of \flex actions :: \flactiongroup{\flsconlist...}{fptr}{sptr}{\flaction...}{fptr}{sptr}
+ \toksb{}#1% collect all the state name information in \toksb
+ % gather the preceeding stash
+ \readstash{#3}% read stash late, after \flsconslist (ignore it)
+ \edef\next{%
+ \toksb{\noalign{\hbox to\nx\hsize{\hskip\the\rgindent
+ \nx\flexsndisplay{\the\toksb}:\toksa{\relax\the\toksg}\nx\makestashbox\nx\hfil}}}%
+ }\next
+ \yytoksempty\table{}{\appendrnx\table{\noalign{\medskip}}}%
+ \concat\table\toksb
+ \advance\rgindent by\parindent
+ #4%
+ \advance\rgindent by-\parindent
+}
+
+\defc\flsconlist{% #1#2#3% start condition list :: \flsconlist{{fptr}{sptr}}{\flname...}{{fptr}{sptr}}
+ \readstashwredirect#1\toksg % save the stash preceeding the conditions list to reinsert later
+ #2% collect the state names (and the stash in between)
+ \readstashwredirect#3\toksf
+ \appendr\toksb{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
+}
+
+\defc\flname{% #1#2#3#4% name :: \flname{name}{name12}{fptr}{sptr}
+ {%
+ \let\parsernamespace\flexpseudonamespace
+ \nameproc{#2}\with\parsebin
+ \edef\next{%
+ \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \expandafter
+ }\the\toksd
+ \let\termindex\writeidxfsentry
+ \readstashwredirect{}{#4}\toksf % do not touch \yystashlocal
+ \appendr\toksb{{\nx\rm\toksa{\the\toksf}\nx\makestashbox}%
+ \the\toksd\gidxentryxv{\termvstring}{#1}{\the\tokse}}%
+ \let\termindex\eatone
+}
+
+\defc\flnamesep{%#1#2% separator between names :: \flnamesep{fptr}{sptr}
+ \toksb\expandafter{\the\toksb\ }%
+}
+
+\newskip\rgindent
+
+\rgindent=\parindent
+
+\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+ \toksb{}#1%
+ \toksc{}#2%
+ #7% apply formatting
+ \readstash{#6}% read stash late, after \flsconlist and \fl...rule
+ \yytoksempty\toksb{%
+ \appendr\table{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
+ }{%
+ \appendr\table{%
+ \hskip\the\rgindent
+ \vbox{%
+ {\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
+ \halign{&####\hfil\cr
+ \hbox to\rgindent{\nx\sscmd\nx\tt\nx\strut\the\toksb\hss}\cr
+ \nx\tt\nx\strut\the\toksc\cr
+ }%
+ }%
+ }%
+ }%
+ \appendr\table{&\toksa{\relax\the\yystashlocal}\nx\cr}%
+}
+
+\defc\flactionc{%#1#2#3#4#5#6#7% \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+ \toksb{}#1%
+ \toksc{}#2%
+ \readstash{#6}% read stash late, after \flsconlist and \fl...rule
+ #7% apply formatting
+ \yytoksempty\toksb{%
+ \appendr\table{\hskip\the\rgindent\hbox{\nx\tt\nx\strut\the\toksc}}%
+ }{%
+ \appendr\table{%
+ \hskip\the\rgindent
+ \vbox{%
+ {\nx\sscmd\nx\expandafter}\nx\expandafter\baselineskip\nx\the\baselineskip
+ \halign{&####\hfil\cr
+ \hbox to\rgindent{\nx\sscmd\nx\tt\nx\strut\the\toksb\hss}\cr
+ \nx\tt\nx\strut\the\toksc\cr
+ }%
+ }%
+ }%
+ }%
+ \appendr\table{&\toksa{{}$\nx\hookleftarrow$\relax\the\yystashlocal}\nx\cr}% in the event \cr has been redefined
+}
+
+\def\inscomment#1{%
+ \def\astformat@flaction{%
+ \toksd{#1}%
+ \appendr\table{\noalign{\nx\smallskip\noindent\hskip\the\rgindent
+ $\nx\triangleright\;$\the\toksd\nx\smallskip}}%
+ }%
+}
+
+\let\astformat@flaction\empty
+
+\toyyunion{flexparser-sect2}
+
+% \flex\ setion~1 typesetting
+% currently, all \flex\state declarations are handled by an extended \bison\ parser
+% to collect the state data; we thus only implement regular expression definitions;
+
+\restorecslist{flexparser-strict}\yyflunion
+
+\defc\flredef{%#1#2#3#4#5#6#7#8% regular expression definition :: \flredef{name}{name12}{fptr}{sptr}{text}{text12}{fptr}{sptr}
+ \let\termindex\writeidxfsrdentry
+ {%
+ \let\hostparsernamespace\flexpseudorenamespace
+ \nameproc{#1}\with\parsebin
+ \edef\next{%
+ \toksd{\toksd{\the\toksa}\tokse{\the\toksb}}%
+ }\next % a trick to `reshuffle' the output of \nameproc:
+ % the parsed name goes to \toksd and the visual key is put in \tokse
+ \expandafter
+ }\the\toksd
+ \frexproc{#5}\with\ppregex
+ \appendr\table{{}$\nx\langle\hbox{\ntt\the\toksd}\nx\rangle${}%
+ \gidxentryxv{\termfsrestring}{#1}{\the\tokse}&\the\toksc\cr% TODO: make a special type processor for regex names
+ }%
+ \let\termindex\eatone
+}
+
+\defc\flopt{%#1#2#3#4#5% \flex option :: \flopt{option}{text}{text12}{fptr}{sptr}
+ \csname flex@ption_#1\endcsname{#2}{#3}%
+}
+
+\expandafter\def\csname flex@ption_deprecated\endcsname#1#2{%
+ \flex@opt@deprecated#2%
+}
+
+\def\flex@opt@deprecated#1#2#3^^J{%
+ \let\termindex\writeidxfsrnentry
+ \appendr\table{$\langle${\nx\tt #2}$\rangle_{\rm f}$\gidxentryxv{\termfsopstring}{#2}{#2}%
+ \the\toksg&\number\flexgetnumber#3.\nx\cr}%
+ \let\termindex\empty
+}
+
+\def\flexgetnumber#1{%
+ \ifnum`#1<"3A
+ \yybreak{%
+ \ifnum`#1>"2F
+ \yybreak{#1\flexgetnumber}%
+ \else
+ \yybreak{%
+ \ifnum`#1="20
+ \yybreak{\flexgetnumber}%
+ \else
+ \yybreak{}%
+ \yycontinue
+ }
+ \yycontinue
+ }%
+ \else
+ \yybreak{}%
+ \yycontinue
+}
+
+\toyyunion{flexparser-sect1}
+
+% correct a few sequences for debugging purposes
+
+\restorecslist{flexparser-debug}\yyflunion
+
+\defc\flnametok{%#1#2% named definition :: \flnametok{matched text}{formatting command}, the text is \{text\}\ *
+ \noexpand\flnametok{#1}{}%
+}
+
+\defc\flparens{%#1#2#3#4% parenthesized regular expression :: \flparens{{fptr}{sptr}}{re}{{fptr}{sptr}}{formatting command}
+ \noexpand\flparens{#1}{#2}{#3}{}%
+}
+
+\defc\flrule{%#1#2% full regular expression :: \flrule{re}{formatting command}
+ \noexpand\flrule{#1}{}%
+}
+
+\defc\flbolrule{%#1#2% full regular expression at the beginning of the line :: \flbolrule{re}{formatting command}
+ \noexpand\flbolrule{#1}{}%
+}
+
+\defc\flaction{%#1#2#3#4#5#6#7% \flex action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+ \noexpand\flaction{#1}{#2}{#3}{#4}{#5}{#6}{}%
+}
+
+\defc\flactionc{%#1#2#3#4#5#6#7% \flex continued action :: \flaction{\flsconlist...}{\fl..rule ...}{fptr}{sptr}{fptr}{sptr}{formatting command}
+ \noexpand\flactionc{#1}{#2}{#3}{#4}{#5}{#6}{}%
+}
-\def\charstonumbers#1\end{\edef\next{\toksa{\charstonumberse#1\end}}\next}
+\toyyunion{flexparser-debug}