diff options
Diffstat (limited to 'support/splint/cweb/lo.w')
-rw-r--r-- | support/splint/cweb/lo.w | 797 |
1 files changed, 797 insertions, 0 deletions
diff --git a/support/splint/cweb/lo.w b/support/splint/cweb/lo.w new file mode 100644 index 0000000000..b28711423c --- /dev/null +++ b/support/splint/cweb/lo.w @@ -0,0 +1,797 @@ +@q Copyright 2012-2014 Alexander Shibakov@> +@q Copyright 2002-2014 Free Software Foundation, Inc.@> +@q This file is part of SPLinT@> + +@q SPLinT is free software: you can redistribute it and/or modify@> +@q it under the terms of the GNU General Public License as published by@> +@q the Free Software Foundation, either version 3 of the License, or@> +@q (at your option) any later version.@> + +@q SPLinT is distributed in the hope that it will be useful,@> +@q but WITHOUT ANY WARRANTY; without even the implied warranty of@> +@q MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the@> +@q GNU General Public License for more details.@> + +@q You should have received a copy of the GNU General Public License@> +@q along with SPLinT. If not, see <http://www.gnu.org/licenses/>.@> + +@*1 The scanner for grammar syntax. +\ifx\parsernamespace\UNDEFINED + \input limbo.sty + \input grabstates.sty + \immediate\openout\stlist=lo_states.h +\fi +The fact that \bison\ has a relatively straightforward grammar is +due to the sophistication of its scanner. The primary reason for this +increased complexity is \bison's awareness +of syntax variations in its input files. In addition to the grammar +syntax, the parser has to be able to deal with extended \Cee\ syntax +inside \bison's actions. + +Since the names of the scanner {\it states\/} reside in the common +namespace with other variables, in order to make the \TeX\ version of +the scanner aware of the numerical values of the states, a special +procedure is required. It is executed as part of \flex's user +initialization code but the data for it has to be collected +separately. The procedure is declared in the preamble section of the scanner. + +Below, we follow the same convention (of italicizing the original +comments) as in the code for the parser. +@(lo.ll@>= +@G + @> @<Grammar lexer definitions@> @= +%{@> @<Grammar lexer \Cee\ preamble@> @=%} + @> @<Grammar lexer options@> @= +%% + @> @<Grammar token regular expressions@> @= +%% +@O +void define_all_states( void ) { + @<Collect state definitions for the grammar lexer@>@; +} +@o +@g + +@ It is convenient to abbreviate some commonly used subexpressions. +@<Grammar lexer definitions@>= + @<Grammar lexer states@>@; +@G +letter [.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_] +notletter [^.abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ_]{-}[%\{] +id {letter}({letter}|[-0-9])* +int [0-9]+ +@g + +@ {\it Zero or more instances of backslash-newline. Following \gcc, allow +white space between the backslash and the newline}. +@<Grammar lexer definitions@>= +@G +splice (\\[ \f\t\v]*\n)* +@g + +@ {\it An equal sign, with optional leading whitespaces. This is used in some +deprecated constructs}. +@<Grammar lexer definitions@>= +@G +eqopt ([[:space:]]*=)? +@g + +@ This is how the code for state value output is put inside the +routine mentioned above. The state information is collected by a +special small scanner that is coupled with the bootstrap parser. This +way, all the necessary token information comes `hardwired' in the +bootstrap parser, and the small scanner itself does not use any state +manipulation and thus can get away without any state setup. It can, +however, scan just enough of the \flex\ syntax to extract the state +information from it (only the state {\it names\/} are needed) and +output it in the form of a header file for the `real' lexer output +`driver' to use. +@<Collect state definitions for the grammar lexer@>= +#define _register_name( name ) @[Define_State( #name, name )@] +#include "lo_states.h" +#undef _register_name + +@ {\it A \Cee-like comment in directives/rules}. +@<Grammar lexer states@>= +@G +%x SC_YACC_COMMENT +@g + +@ {\it Strings and characters in directives/rules}. +@<Grammar lexer states@>= +@G +%x SC_ESCAPED_STRING SC_ESCAPED_CHARACTER +@g + +@ {\it A identifier was just read in directives/rules. Special state +to capture the sequence `\.{identifier:}'}. +@<Grammar lexer states@>= +@G +%x SC_AFTER_IDENTIFIER +@g + +@ {\it \POSIX\ says that a tag must be both an id and a \Cee\ union member, but +historically almost any character is allowed in a tag. We +disallow \prodstyle{NUL}, as this simplifies our implementation. We match +angle brackets in nested pairs: several languages use them for +generics/template types}. +@<Grammar lexer states@>= +@G +%x SC_TAG +@g + +@ {\it +\def\aterm{\item{\sqbullet}\ignorespaces}% +\setbox0=\hbox{\sqbullet\enspace}% +\parindent=0pt +\advance\parindent by \wd0 +Four types of user code: +\aterm prologue (code between \.{\%\{} \.{\%\}} in the first section, before \prodstyle{\%\%}); + +\aterm actions, printers, union, etc, (between braced in the middle section); + +\aterm epilogue (everything after the second \prodstyle{\%\%}); + +\aterm predicate (code between \.{\%?\{} and \.{\}} in middle section); +}% +@<Grammar lexer states@>= +@G +%x SC_PROLOGUE SC_BRACED_CODE SC_EPILOGUE SC_PREDICATE +@g + +@ {\it \Cee\ and \Cee++ comments in code}. +@<Grammar lexer states@>= +@G +%x SC_COMMENT SC_LINE_COMMENT +@g + +@ {\it Strings and characters in code}. +@<Grammar lexer states@>= +@G +%x SC_STRING SC_CHARACTER +@g + +@ Bracketed identifiers support. +@<Grammar lexer states@>= +@G +%x SC_BRACKETED_ID SC_RETURN_BRACKETED_ID +@g + +@ @<Grammar lexer \Cee\ preamble@>= + +#include <stdint.h> +#include <stdbool.h> + +@ The code for the generated scanner is highly dependent on the options +supplied. Most of the options below are essential for the scheme +adopted in this package to work. +@<Grammar lexer options@>= +@G +%option bison-bridge +%option noyywrap nounput noinput reentrant +%option noyy_top_state +%option debug +%option stack +%option outfile="lo.c" +@g + +@*2 Tokenizing with regular expressions. +Here is a full collection of regular expressions employed by the scanner. +@<Grammar token regular expressions@>= + @<Scan grammar white space@>@; + @<Scan \flex\ directives and options@>@; + @<Scan \bison\ directives@>@; + @<Do not support zero characters@>@; + @<Scan after an identifier, check whether a colon is next@>@; + @<Scan bracketed identifiers@>@; + @<Scan a Yacc comment@>@; + @<Scan a \Cee\ comment@>@; + @<Scan a line comment@>@; + @<Scan a \bison\ string@>@; + @<Scan a character literal@>@; + @<Scan a tag@>@; + @<Decode escaped characters@>@; + @<Scan user-code characters and strings@>@; + @<Strings, comments etc.\ found in user code@>@; + @<Scan code in braces@>@; + @<Scan prologue@>@; + @<Scan the epilogue@>@; + @<Add the scanned symbol to the current string@>@; + +@ @<Scan grammar white space@>= +@G +<INITIAL,SC_AFTER_IDENTIFIER,SC_BRACKETED_ID,SC_RETURN_BRACKETED_ID> +{ + /* {\it Comments and white space.} */ + "," {@> @[TeX_( "/yycomplain{stray `,' treated as white space}/yylexnext" );@]@=} + [ \f\n\t\v] | + "//".* {@> @[TeX_( "/yylexnext" );@]@=} +@= "/*" {@> @[TeX_( "/contextstate/YYSTART /yyBEGIN{SC_YACC_COMMENT}/yylexnext" );@]@=}@>@/ + /* {\it |@[#line@]| directives are not documented, and may be withdrawn or modified in future versions of \bison.} */ + ^"#line "{int}(" \"".*"\"")?"\n" {@> @[TeX_( "/yylexnext" );@]@=} +} +@g + +@ {\it For directives that are also command line options, the regex must be +\.{"\%..."} after \.{"[-\_]"}'s are removed, and the directive must match the \.{--long} +option name, with a single string argument. Otherwise, add exceptions +to \.{../build-aux/cross-options.pl}}. For most options the scanner +returns a pair of pointers as the value. + +@<Scan \bison\ directives@>= +@G +<INITIAL> +{ + "%binary" {@> @[TeX_( "/yylexreturnptr{PERCENT_NONASSOC}" );@]@=} + "%code" {@> @[TeX_( "/yylexreturnptr{PERCENT_CODE}" );@]@=} + "%debug" {@> @[@<Set \prodstyle{\%debug} flag@>@]@=} + "%default-prec" {@> @[TeX_( "/yylexreturnptr{PERCENT_DEFAULT_PREC}" );@]@=} + "%define" {@> @[TeX_( "/yylexreturnptr{PERCENT_DEFINE}" );@]@=} + "%defines" {@> @[TeX_( "/yylexreturnptr{PERCENT_DEFINES}" );@]@=} + "%destructor" {@> @[TeX_( "/yylexreturnptr{PERCENT_DESTRUCTOR}" );@]@=} + "%dprec" {@> @[TeX_( "/yylexreturnptr{PERCENT_DPREC}" );@]@=} + "%empty" {@> @[TeX_( "/yylexreturnptr{PERCENT_EMPTY}" );@]@=} + "%error-verbose" {@> @[TeX_( "/yylexreturnptr{PERCENT_ERROR_VERBOSE}" );@]@=} + "%expect" {@> @[TeX_( "/yylexreturnptr{PERCENT_EXPECT}" );@]@=} + "%expect-rr" {@> @[TeX_( "/yylexreturnptr{PERCENT_EXPECT_RR}" );@]@=} + "%file-prefix" {@> @[TeX_( "/yylexreturnptr{PERCENT_FILE_PREFIX}" );@]@=} + "%fixed-output-files" {@> @[TeX_( "/yylexreturnptr{PERCENT_YACC}" );@]@=} + "%initial-action" {@> @[TeX_( "/yylexreturnptr{PERCENT_INITIAL_ACTION}" );@]@=} + "%glr-parser" {@> @[TeX_( "/yylexreturnptr{PERCENT_GLR_PARSER}" );@]@=} + "%language" {@> @[TeX_( "/yylexreturnptr{PERCENT_LANGUAGE}" );@]@=} + "%left" {@> @[TeX_( "/yylexreturnptr{PERCENT_LEFT}" );@]@=} + "%lex-param" {@> @[@<Return lexer parameters@>@]@=} + "%locations" {@> @[@<Set \prodstyle{\%locations} flag@>@]@=} + "%merge" {@> @[TeX_( "/yylexreturnptr{PERCENT_MERGE}" );@]@=} + "%name-prefix" {@> @[TeX_( "/yylexreturnptr{PERCENT_NAME_PREFIX}" );@]@=} + "%no-default-prec" {@> @[TeX_( "/yylexreturnptr{PERCENT_NO_DEFAULT_PREC}" );@]@=} + "%no-lines" {@> @[TeX_( "/yylexreturnptr{PERCENT_NO_LINES}" );@]@=} + "%nonassoc" {@> @[TeX_( "/yylexreturnptr{PERCENT_NONASSOC}" );@]@=} + "%nondeterministic-parser" {@> @[TeX_( "/yylexreturnptr{PERCENT_NONDETERMINISTIC_PARSER}" );@]@=} + "%nterm" {@> @[TeX_( "/yylexreturnptr{PERCENT_NTERM}" );@]@=} + "%output" {@> @[TeX_( "/yylexreturnptr{PERCENT_OUTPUT}" );@]@=} + "%param" {@> @[@<Return lexer and parser parameters@>@]@=} + "%parse-param" {@> @[@<Return parser parameters@>@]@=} + "%prec" {@> @[TeX_( "/yylexreturnptr{PERCENT_PREC}" );@]@=} + "%precedence" {@> @[TeX_( "/yylexreturnptr{PERCENT_PRECEDENCE}" );@]@=} + "%printer" {@> @[TeX_( "/yylexreturnptr{PERCENT_PRINTER}" );@]@=} + "%pure-parser" {@> @[@<Set \prodstyle{\%pure-parser} flag@>@]@=} + "%require" {@> @[TeX_( "/yylexreturnptr{PERCENT_REQUIRE}" );@]@=} + "%right" {@> @[TeX_( "/yylexreturnptr{PERCENT_RIGHT}" );@]@=} + "%skeleton" {@> @[TeX_( "/yylexreturnptr{PERCENT_SKELETON}" );@]@=} + "%start" {@> @[TeX_( "/yylexreturnptr{PERCENT_START}" );@]@=} + "%term" {@> @[TeX_( "/yylexreturnptr{PERCENT_TOKEN}" );@]@=} + "%token" {@> @[TeX_( "/yylexreturnptr{PERCENT_TOKEN}" );@]@=} + "%token-table" {@> @[TeX_( "/yylexreturnptr{PERCENT_TOKEN_TABLE}" );@]@=} + "%type" {@> @[TeX_( "/yylexreturnptr{PERCENT_TYPE}" );@]@=} + "%union" {@> @[TeX_( "/yylexreturnptr{PERCENT_UNION}" );@]@=} + "%verbose" {@> @[TeX_( "/yylexreturnptr{PERCENT_VERBOSE}" );@]@=} + "%yacc" {@> @[TeX_( "/yylexreturnptr{PERCENT_YACC}" );@]@=} + + /* {\it deprecated} */ + "%default"[-_]"prec" {@> @[TeX_( "/yypdeprecated{\\%default-prec}" );@]@=} + "%error"[-_]"verbose" {@> @[TeX_( "/yypdeprecated{\\%define parse.error verbose}" );@]@=} + "%expect"[-_]"rr" {@> @[TeX_( "/yypdeprecated{\\%expect-rr}" );@]@=} + "%file-prefix"{eqopt} {@> @[TeX_( "/yypdeprecated{\\%file-prefix}" );@]@=} + "%fixed"[-_]"output"[-_]"files" {@> @[TeX_( "/yypdeprecated{\\%fixed-output-files}" );@]@=} + "%name"[-_]"prefix"{eqopt} {@> @[TeX_( "/yypdeprecated{\\%name-prefix}" );@]@=} + "%no"[-_]"default"[-_]"prec" {@> @[TeX_( "/yypdeprecated{\\%no-default-prec}" );@]@=} + "%no"[-_]"lines" {@> @[TeX_( "/yypdeprecated{\\%no-lines}" );@]@=} + "%output"{eqopt} {@> @[TeX_( "/yypdeprecated{\\%output}" );@]@=} + "%pure"[-_]"parser" {@> @[TeX_( "/yypdeprecated{\\%pure-parser}" );@]@=} + "%token"[-_]"table" {@> @[TeX_( "/yypdeprecated{\\%token-table}" );@]@=} + + /* {\it Semantic predicate.} */ + "%?"[ \f\n\t\v]*"{" {@> @[TeX_( "/yyBEGIN{SC_PREDICATE}/yylexnext" );@]@=} + + "%"{id}|"%"{notletter}([[:graph:]])+ {@> @[@<Possbly complain about a bad directive@>@]@=} + + "=" {@> @[TeX_( "/yylexreturnptr{EQUAL}" );@]@=} + "|" {@> @[TeX_( "/yylexreturnptr{PIPE}" );@]@=} + ";" {@> @[TeX_( "/yylexreturnptr{SEMICOLON}" );@]@=} + + {id} {@> @[@<Prepare an identifier@>@]@=} + {int} {@> @[TeX_( "/edef/next{/yylval{/nx/anint{/the/yytext}" );@]@; + @> @[TeX_( "{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @> @[TeX_( "/yylexreturn{INT}" );@]@=} + 0[xX][0-9abcdefABCDEF]+ {@> @[TeX_( "/edef/next{/yylval{/nx/hexint{/the/yytext}" );@]@; + @> @[TeX_( "{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @> @[TeX_( "/yylexreturn{INT}" );@]@=} + + /* {\it Identifiers may not start with a digit. Yet, don't silently accept \.{1FOO} as \.{1 FOO}.} */ + {int}{id} {@> @[TeX_( "/yycomplain{invalid identifier: /the/yytext}" );@] + @> @[TeX_( "/yyerrterminate" );@]@=} + + /* {\it Characters.} */ + "'" {@> @[TeX_( "/yyBEGIN{SC_ESCAPED_CHARACTER}/yylexnext" );@]@=} + + /* {\it Strings.} */ + "\"" {@> @[TeX_( "/yyBEGIN{SC_ESCAPED_STRING}/yylexnext" );@]@=} + + /* {\it Prologue.} */ + "%{" {@> @[@<Start assembling prologue code@>@]@=} + + /* {\it Code in between braces.} Originally preceded by \.{\\STRINGGROW} but it is omitted here. */ + "{" {@> @[TeX_( "/lonesting/z@@/yyBEGIN{SC_BRACED_CODE}/yylexnext" );@]@=} + + /* {\it A type.} */ + "<*>" {@> @[TeX_( "/yylexreturnptr{TAG_ANY}" );@]@=} + "<>" {@> @[TeX_( "/yylexreturnptr{TAG_NONE}" );@]@=} + "<" {@> @[TeX_( "/lonesting=/z@@/yyBEGIN{SC_TAG}/yylexnext" );@]@=} + + "%%" {@> @[@<Switch sections@>@]@=} + "[" {@> @[TeX_( "/let/bracketedidstr=/empty" );@]@; + @> @[TeX_( "/bracketedidcontextstate/YYSTART" );@] + @> @[TeX_( "/yyBEGIN{SC_BRACKETED_ID}/yylexnext" );@]@=} + + <<EOF>> {@> @[TeX_( "/yyterminate% EOF in INITIAL" );@]@=} + + [^\[%A-Za-z0-9_<>{}\"\'*;|=/, \f\n\t\v]+|. {@> @[@<Process a bad character@>@]@=} +} +@g + +@ Some additional constructs needed to typeset simple \flex\ +declarations. This is not part of the original \bison\ scanner. +@<Scan \flex\ directives and options@>= +@G +<INITIAL> +{ + "%option" {@> @[TeX_( "/yylexreturnptr{FLEX_OPTION}" );@]@=} + "%x" {@> @[TeX_( "/yylexreturnptr{FLEX_STATE_X}" );@]@=} + "%s" {@> @[TeX_( "/yylexreturnptr{FLEX_STATE_S}" );@]@=} +} +@g + +@ We present the `bad character' code first, before going into the details +of the character matching by the rest of the lexer. +@<Process a bad character@>= + @[TeX_( "/edef/next{/nx/csname lexspecial[/the/yytextpure]/nx/endcsname}" );@]@; + @[TeX_( "/expandafter/expandafter/expandafter/ifx/next/relax" );@]@; + @[TeX_( " /iftracebadchars" );@]@; + @[TeX_( " /yycomplain{invalid character(s): /the/yytext}" );@]@; + @[TeX_( " /fi" );@]@; + @[TeX_( " /yylexreturn{$undefined}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /expandafter/lexspecialchar/expandafter{/next}{/the/yyfmark}{/the/yysmark}/yylexnext" );@]@; + @[TeX_( "/fi" );@]@; + +@ @<Set \prodstyle{\%debug} flag@>= + @[TeX_( "/edef/next{/yylval{{parse.trace}{debug}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yylexreturn{PERCENT_FLAG}" );@]@; + +@ @<Return lexer parameters@>= + @[TeX_( "/edef/next{/yylval{{lex-param}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yylexreturn{PERCENT_PARAM}" );@]@; + +@ @<Set \prodstyle{\%locations} flag@>= + @[TeX_( "/edef/next{/yylval{{locations}{}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yylexreturn{PERCENT_FLAG}" );@]@; + +@ @<Return lexer and parser parameters@>= + @[TeX_( "/edef/next{/yylval{{both-param}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yylexreturn{PERCENT_PARAM}" );@]@; + +@ @<Return parser parameters@>= + @[TeX_( "/edef/next{/yylval{{parse-param}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yylexreturn{PERCENT_PARAM}" );@]@; + +@ @<Set \prodstyle{\%pure-parser} flag@>= + @[TeX_( "/edef/next{/yylval{{api.pure}{pure-parser}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yylexreturn{PERCENT_FLAG}" );@]@; + +@ @<Possbly complain about a bad directive@>= + @[TeX_( "/iftracebadchars" );@]@; + @[TeX_( " /yycomplain{invalid directive: /the/yytext}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/yylexnext" );@]@; + +@ @<Prepare an identifier@>= + @[TeX_( "/edef/next{/yylval{/nx/idit{/the/yytextpure}{/the/yytext}" );@]@; + @[TeX_( " {/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/let/bracketedidstr=/empty" );@]@; + @[TeX_( "/yyBEGIN{SC_AFTER_IDENTIFIER}/yylexnext" );@]@; + +@ @<Switch sections@>= + @[TeX_( "/advance/percentpercentcount/@@ne" );@]@; + @[TeX_( "/ifnum/percentpercentcount=/tw@@" );@]@; + @[TeX_( " /yyBEGIN{SC_EPILOGUE}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/yylexreturnptr{PERCENT_PERCENT}" );@]@; + +@ @<Start assembling prologue code@>= + @[TeX_( "/edef/next{/postoks{{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yyBEGIN{SC_PROLOGUE}/yylexnext" );@]@; + +@ {\it Supporting \.{\\0} complexifies our implementation for no expected added value}. + +@<Do not support zero characters@>= +@G +<SC_ESCAPED_CHARACTER,SC_ESCAPED_STRING,SC_TAG> +{ + \0 {@> @[TeX_( "/yycomplain{invalid null character}/yylexnext" );@]@=} +} +@g + +@ @<Scan after an identifier, check whether a colon is next@>= +@G +<SC_AFTER_IDENTIFIER> +{ + "[" {@> @[@<Process the bracketed part of an identifier@>@]@=} + ":" {@> @[@<Process a colon after an identifier@>@]@=} + <<EOF>> {@> @[@<End the scan with an identifier@>@]@=} + . {@> @[@<Process a character after an identifier@>@]@=} +} +@g + +@ @<Process the bracketed part of an identifier@>= + @[TeX_( "/ifx/bracketedidstr/empty" );@]@; + @[TeX_( " /bracketedidcontextstate/YYSTART /yyBEGIN{SC_BRACKETED_ID}" );@]@; + @[TeX_( " /let/next=/yylexnext" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /ROLLBACKCURRENTTOKEN" );@]@; + @[TeX_( " /yyBEGIN{SC_RETURN_BRACKETED_ID}" );@]@; + @[TeX_( " /def/next{/yylexreturn{ID}}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/next" );@]@; + +@ @<Process a colon after an identifier@>= + @[TeX_( "/ifx/bracketedidstr/empty" );@]@; + @[TeX_( " /yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /yyBEGIN{SC_RETURN_BRACKETED_ID}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/yylexreturn{ID_COLON}" );@]@; + +@ @<Process a character after an identifier@>= + @[TeX_( "/ROLLBACKCURRENTTOKEN" );@]@; + @[TeX_( "/ifx/bracketedidstr/empty" );@]@; + @[TeX_( " /yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /yyBEGIN{SC_RETURN_BRACKETED_ID}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/yylexreturn{ID}" );@]@; + +@ @<End the scan with an identifier@>= + @[TeX_( "/ifx/bracketedidstr/empty" );@]@; + @[TeX_( " /yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /yyBEGIN{SC_RETURN_BRACKETED_ID}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/ROLLBACKCURRENTTOKEN" );@]@; + @[TeX_( "/yylexreturn{ID}" );@]@; + +@ @<Scan bracketed identifiers@>= +@G +<SC_BRACKETED_ID> +{ + <<EOF>> {@> @[@<Complain about unexpected end of file inside brackets@>@]@=} + {id} {@> @[@<Process bracketed identifier@>@]@=} + "]" {@> @[@<Finish processing bracketed identifier@>@]@=} + [^\].A-Za-z0-9_/ \f\n\t\v]+|. {@> @[@<Complain about improper identifier characters@>@]@=} +} +@g + +@ @<Process bracketed identifier@>= + @[TeX_( "/ifx/bracketedidstr/empty" );@]@; + @[TeX_( " /edef/bracketedidstr{/nx/idit{/the/yytextpure}" );@]@; + @[TeX_( " {/the/yytext}{/the/yyfmark}{/the/yysmark}}" );@]@; + @[TeX_( " /let/next=/yylexnext" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /def/next{/yycomplain{unexpected " );@]@; + @[TeX_( " identifier in bracketed name: /the/yytext}/yylexnext}" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/next" );@]@; + +@ @<Finish processing bracketed identifier@>= + @[TeX_( "/yyBEGINr/bracketedidcontextstate" );@]@; + @[TeX_( "/ifx/bracketedidstr/empty" );@]@; + @[TeX_( " /def/next{/yycomplain{an identifier expected}/yylexnext}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /ifnum/bracketedidcontextstate=/yylexstate{INITIAL}/relax" );@]@; + @[TeX_( " /expandafter/yylval/expandafter{/bracketedidstr}" );@]@; + @[TeX_( " /let/bracketedidstr=/empty" );@]@; + @[TeX_( " /def/next{/yylexreturn{BRACKETED_ID}}" );@]@; + @[TeX_( " /else" );@]@; + @[TeX_( " /let/next=/yylexnext" );@]@; + @[TeX_( " /fi" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/next" );@]@; + +@ @<Complain about improper identifier characters@>= + @[TeX_( "/yycomplain{invalid character(s) in bracketed name: /the/yytext}/yyerrterminate" );@]@; + +@ @<Complain about unexpected end of file inside brackets@>= + @[TeX_( "/yyBEGINr/bracketedidcontextstate" );@]@; + @[TeX_( "/yycomplain{unexpected end of file inside brackets}/yyerrterminate" );@]@; + +@ @<Scan bracketed identifiers@>= +@G +<SC_RETURN_BRACKETED_ID> +{ + . {@> @[@<Return a bracketed identifier@>@]@=} +} +@g + +@ @<Return a bracketed identifier@>= + @[TeX_( "/ROLLBACKCURRENTTOKEN" );@]@; + @[TeX_( "/expandafter/yylval/expandafter{/bracketedidstr}" );@]@; + @[TeX_( "/let/bracketedidstr=/empty" );@]@; + @[TeX_( "/yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/yylexreturn{BRACKETED_ID}" );@]@; + +@ {\it Scanning a Yacc comment. The initial \.{/*} is already eaten}. +@<Scan a Yacc comment@>= +@G +<SC_YACC_COMMENT> +{ + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file in " );@] + @> @[TeX_( " a comment}/yyerrterminate" );@]@=} + "*/" {@> @[TeX_( "/yyBEGINr{/contextstate}/yylexnext" );@]@=} + .|\n {@> @[TeX_( "/yylexnext" );@]@=} +} +@g + +@ {\it Scanning a \Cee\ comment. The initial \.{/*} is already eaten}. +@<Scan a \Cee\ comment@>= +@G +<SC_COMMENT> +{ + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file in " );@] + @> @[TeX_( " a comment}/yyerrterminate" );@]@=} + "*"{splice}"/" {@> @[TeX_( "/STRINGGROW/yyBEGINr/contextstate/yylexnext" );@]@=} +} +@g + +@ {\it Scanning a line comment. The initial \.{//} is already eaten}. +@<Scan a line comment@>= +@G +<SC_LINE_COMMENT> +{ + <<EOF>> {@> @[TeX_( "/yyBEGINr/contextstate /ROLLBACKCURRENTTOKEN" );@] + @> @[TeX_( " /yylexnext" );@]@=} + "\n" {@> @[TeX_( "/STRINGGROW/yyBEGINr/contextstate /yylexnext" );@]@=} + {splice} {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} +} +@g + +@ {\it Scanning a \bison\ string, including its escapes. +The initial quote is already eaten}. +@<Scan a \bison\ string@>= +@G +<SC_ESCAPED_STRING> +{ + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file in " );@] + @> @[TeX_( " a string}/yyerrterminate" );@]@=} + "\"" {@> @[@<Finish a \bison\ string@>@]@=} + "\n" {@> @[TeX_( "/yycomplain{unexpected end of line in " );@] + @> @[TeX_( " a string}/yyerrterminate" );@]@=} +} +@g + +@ @<Finish a \bison\ string@>= + @[TeX_( "/STRINGFINISH" );@]@; + @[TeX_( "/edef/next{/yylval{/nx/stringify{/the/laststring}" );@]@; + @[TeX_( "{/the/laststringraw}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/yylexreturn{STRING}" );@]@; + +@ {\it Scanning a \bison\ character literal, decoding its escapes. +The initial quote is already eaten}. +@<Scan a character literal@>= +@G +<SC_ESCAPED_CHARACTER> +{ + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file in " );@] + @> @[TeX_( " a literal}/yyerrterminate" );@]@=} + "'" {@> @[@<Return an escaped character@>@]@=} + "\n" {@> @[TeX_( "/yycomplain{unexpected end of line in " );@] + @> @[TeX_( " a literal}/yyerrterminate" );@]@=} +} +@g + +@ @<Return an escaped character@>= + @[TeX_( "/STRINGFINISH" );@]@; + @[TeX_( "/edef/next{/yylval{/nx/charit{/the/laststring}{/the/laststringraw}" );@]@; + @[TeX_( " {/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/STRINGFREE" );@]@; + @[TeX_( "/yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/yylexreturn{CHAR}" );@]@; + +@ {\it Scanning a tag. The initial angle bracket is already eaten}. +@<Scan a tag@>= +@G +<SC_TAG> +{ + ">" {@> @[@<Finish a tag@>@]@=} + ([^<>]|->)+ {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + "<" {@> @[@<Raise nesting level@>@]@=} + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file in " );@] + @> @[TeX_( " a literal}/yyerrterminate" );@]@=} +} +@g + +@ @<Finish a tag@>= + @[TeX_( "/advance/lonesting/m@@ne" );@]@; + @[TeX_( "/ifnum/lonesting</z@@" );@]@; + @[TeX_( " /STRINGFINISH" );@]@; + @[TeX_( " /edef/next{/yylval{/nx/tagit{/the/laststring}{/the/laststringraw}" );@]@; + @[TeX_( " {/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( " /STRINGFREE" );@]@; + @[TeX_( " /yyBEGIN{INITIAL}" );@]@; + @[TeX_( " /def/next{/yylexreturn{TAG}}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /STRINGGROW/let/next=/yylexnext" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/next" );@]@; + +@ This is a slightly different rule from the original scanner. We do not perform |yyleng| computations, +so it makes sense to raise the nesting level one by one. +@<Raise nesting level@>= + @[TeX_( "/STRINGGROW" );@]@; + @[TeX_( "/advance/lonesting/@@ne" );@]@; + @[TeX_( "/yylexnext" );@]@; + +@ @<Decode escaped characters@>= +@G +<SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER> +{ + \\[0-7]{1,3} {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\x[0-9abcdefABCDEF]+ {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\a {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\b {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\f {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\n {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\r {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\t {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\v {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + + /* {\it \.{\\\\[\\"\\'?\\\\]} would be shorter, but it confuses |xgettext|.} */ + \\("\""|"'"|"?"|"\\") {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + + \\(u|U[0-9abcdefABCDEF]{4})[0-9abcdefABCDEF]{4} {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} + \\(.|\n) {@> @[TeX_( "/yycomplain{invalid character after " );@] + @> @[TeX_( " /\\-escape: /the/yytext}/yylexnext" );@]@=} +} +@g + +@ @<Scan user-code characters and strings@>= +@G +<SC_CHARACTER,SC_STRING> +{ + {splice}|\\{splice}[^\n\[\]] {@> @[TeX_( "/STRINGGROW/yylexnext" );@]@=} +} + +<SC_CHARACTER> +{ + "'" {@> @[TeX_( "/STRINGGROW /yyBEGINr{/contextstate}/yylexnext" );@]@=} + \n {@> @[TeX_( "/yycomplain{unexpected end of line instead of " );@] + @> @[TeX_( " a character}/yyerrterminate" );@]@=} + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file instead of " );@] + @> @[TeX_( " a character}/yyerrterminate" );@]@=} +} + +<SC_STRING> +{ + "\"" {@> @[TeX_( "/STRINGGROW /yyBEGINr{/contextstate}/yylexnext" );@]@=} + \n {@> @[TeX_( "/yycomplain{unexpected end of line instead of " );@] + @> @[TeX_( " a character}/yyerrterminate" );@]@=} + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file instead of " );@] + @> @[TeX_( " a character}/yyerrterminate" );@]@=} +} +@g + +@ @<Strings, comments etc.\ found in user code@>= +@G +<SC_BRACED_CODE,SC_PROLOGUE,SC_EPILOGUE,SC_PREDICATE> +{ + "'" {@> @[TeX_( "/STRINGGROW /contextstate/YYSTART" );@] + @> @[TeX_( " /yyBEGIN{SC_CHARACTER}/yylexnext" );@]@=} + "\"" {@> @[TeX_( "/STRINGGROW /contextstate/YYSTART" );@] + @> @[TeX_( " /yyBEGIN{SC_STRING}/yylexnext" );@]@=} + "/"{splice}"*" {@> @[TeX_( "/STRINGGROW /contextstate/YYSTART" );@] + @> @[TeX_( " /yyBEGIN{SC_COMMENT}/yylexnext" );@]@=} + "/"{splice}"/" {@> @[TeX_( "/STRINGGROW /contextstate/YYSTART" );@] + @> @[TeX_( " /yyBEGIN{SC_LINE_COMMENT}/yylexnext" );@]@=} +} +@g + +@ {\it Scanning some code in braces (actions, predicates). The +initial \.{\{} is already eaten}. +@<Scan code in braces@>= +@G +<SC_BRACED_CODE,SC_PREDICATE> +{ + "{"|"<"{splice}"%" {@> @[TeX_( "/STRINGGROW /advance/lonesting/@@ne /yylexnext" );@]@=} + "%"{splice}">" {@> @[TeX_( "/STRINGGROW /advance/lonesting/m@@ne /yylexnext" );@]@=} + + /* {\it Tokenize \.{<<\%} correctly (as \.{<<} \.{\%}) rather than incorrectly (as \.{<} \.{<\%}).} */ + "<"{splice}"<" {@> @[TeX_( "/STRINGGROW /yylexnext" );@]@=} + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of line " );@] + @> @[TeX_( " inside braced code}/yyerrterminate" );@]@=} +} + +<SC_BRACED_CODE> +{ + "}" {@> @[@<Add closing brace to the braced code@>@]@=} +} + +<SC_PREDICATE> +{ + "}" {@> @[@<Add closing brace to a predicate@>@]@=} +} +@g + +@ Unlike the original lexer, we do not return the closing brace as part of the +braced code. + +@<Add closing brace to the braced code@>= + @[TeX_( "/advance/lonesting/m@@ne" );@]@; + @[TeX_( "/ifnum/lonesting</z@@" );@]@; + @[TeX_( " /STRINGFINISH" );@]@; + @[TeX_( " /edef/next{/yylval{{/the/laststring}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( " /def/next{/yylexreturn{BRACED_CODE}}" );@]@; + @[TeX_( " /yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /STRINGGROW" );@]@; + @[TeX_( " /let/next=/yylexnext" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/next" );@]@; + +@ @<Add closing brace to a predicate@>= + @[TeX_( "/advance/lonesting/m@@ne" );@]@; + @[TeX_( "/ifnum/lonesting</z@@" );@]@; + @[TeX_( " /STRINGFINISH" );@]@; + @[TeX_( " /edef/next{/yylval{{/the/laststring}{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( " /yyBEGIN{INITIAL}" );@]@; + @[TeX_( " /def/next{/yylexreturn{BRACED_PREDICATE}}" );@]@; + @[TeX_( "/else" );@]@; + @[TeX_( " /STRINGGROW" );@]@; + @[TeX_( " /let/next=/yylexnext" );@]@; + @[TeX_( "/fi" );@]@; + @[TeX_( "/next" );@]@; + +@ {\it Scanning some prologue: from \.{\%\{} (already scanned) to \.{\%\}}}. +@<Scan prologue@>= +@G +<SC_PROLOGUE> +{ + "%}" {@> @[@<Finish braced code@>@]@=} + <<EOF>> {@> @[TeX_( "/yycomplain{unexpected end of file " );@] + @> @[TeX_( " inside prologue}/yyerrterminate" );@]@=} +} +@g + +@ @<Finish braced code@>= + @[TeX_( "/STRINGFINISH" );@]@; + @[TeX_( "/edef/next{/yylval{{/the/laststring}/the/postoks{/the/yyfmark}{/the/yysmark}}}/next" );@]@; + @[TeX_( "/yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/yylexreturn{PROLOGUE}" );@]@; + +@ {\it Scanning the epilogue (everything after the second \prodstyle{\%\%}, which +has already been eaten)}. +@<Scan the epilogue@>= +@G +<SC_EPILOGUE> +{ + <<EOF>> {@> @[@<Handle end of file in the epilogue@>@]@=} +} +@g + +@ @<Handle end of file in the epilogue@>= + @[TeX_( "/ROLLBACKCURRENTTOKEN" );@]@; + @[TeX_( "/STRINGFINISH" );@]@; + @[TeX_( "/yylval=/laststring" );@]@; + @[TeX_( "/yyBEGIN{INITIAL}" );@]@; + @[TeX_( "/yylexreturn{EPILOGUE}" );@]@; + +@ {\it By default, grow the string obstack with the input}. +\ifbootstrapmode % only if this file is used to extract state information + \immediate\closeout\stlist +\fi +@<Add the scanned symbol to the current string@>= +@G +<SC_COMMENT,SC_LINE_COMMENT,SC_BRACED_CODE,SC_PREDICATE,SC_PROLOGUE,SC_EPILOGUE, + SC_STRING,SC_CHARACTER,SC_ESCAPED_STRING,SC_ESCAPED_CHARACTER>. | + <SC_COMMENT,SC_LINE_COMMENT,SC_BRACED_CODE,SC_PREDICATE, + SC_PROLOGUE,SC_EPILOGUE>\n {@> @[TeX_( "/STRINGGROW /yylexnext" );@]@=} +@g |