summaryrefslogtreecommitdiff
path: root/support/splint/cweb/fo.w
blob: d34ca57d9673f008d16917ba92a4b42f7092035e (plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
@q  Copyright (c) 1990 The Regents of the University of California. @>
@q  All rights reserved. @>

@q  This code is derived from software contributed to Berkeley by @>
@q  Vern Paxson. @>

@q  The United States Government has rights in this work pursuant @>
@q  to contract no. DE-AC03-76SF00098 between the United States @>
@q  Department of Energy and the University of California. @>

@q  This file is part of SPLinT. @>

@q  Redistribution and use in source and binary forms, with or without @>
@q  modification, are permitted provided that the following conditions @>
@q  are met: @>

@q  1. Redistributions of source code must retain the above copyright @>
@q     notice, this list of conditions and the following disclaimer. @>
@q  2. Redistributions in binary form must reproduce the above copyright @>
@q     notice, this list of conditions and the following disclaimer in the @>
@q     documentation and/or other materials provided with the distribution. @>

@q  Neither the name of the University nor the names of its contributors @>
@q  may be used to endorse or promote products derived from this software @>
@q  without specific prior written permission. @>

@q  THIS SOFTWARE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR @>
@q  IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED @>
@q  WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR @>
@q  PURPOSE. @>

@**The \ifbootstrapmode flex \else\flex\ \fi parser stack.
\ifbootstrapmode % this is a bootstrap run to generate the token references
    \input limbo.sty
    \def\optimization{5}
    \input yy.sty
    \def\bstrapparser{dyytab.tex}% use the full prologue parser
    \def\bstraptokens{bo.tok}% for which we will need token equivalences
    \modebootstrap
\else % otherwise set up the pretty printing of tokens
    \let\hostparsernamespace\flexnamespace
\fi
% we can get away with the setup above because the minimal lexer
% used for bootstrapping does not know about the \prodstyle{\%left}
% token so in the declarations list below, the two tokens,
% \prodstyle{CCL_OP_DIFF} and \prodstyle{CCL_OP_UNION} become
% part of the previous \prodstyle{\%token} declaration which is a
% legal declaration syntax; see ldman.w and ldlex.w and ldgram.w for
% a more elaborate setup;
The scanner generator, \flex, uses \bison\ to produce a parser for its
input language. Its lexer is output by \flex\ itself so both are
reused to generate the parser and the scanner for pretty printing
\flex\ input.

This task is made somewhat complicated by the dependence of the \flex\
input scanner on the correctly placed whitespace\footnote{For example,
each regular expression definition in section~1 must start at the
beginning of the line.}, as well as the reliance of the said scanner
on rather involved state switching. Therefore, making subparsers for
different fragments of \flex\ input involves not only choosing an
appropriate subset of grammar rules to correctly process the
grammatic constructs but also setting up the correct lexer states.

The first subparser is designed to process a complete \flex\
file. This parser is not currently part of any parser stack and is
only used for testing. This is the only parser that does not rely on
any custom adjustments to the lexer state to operate correctly.
@(fip.yy@>=
@G
%{@> @<Preamble for the \flex\ parser@> @=%}
 @> @<Options for \flex\ parser@> @= 
%union {@> @=}
%{@> @<Postamble for \flex\ parser@> @=%} 
 @> @<Token definitions for \flex\ input parser@>@= 
%%
 @> @<Productions for \flex\ parser@> @= 
%%
@g

@ The selection of options for \bison\ parsers suitable for \splint\
has been discussed
\ifbootstrapmode\else\locallink{bison.options}earlier \endlink\fi so we
list them here without further comments.
@<Options for \flex\ parser@>=
@G
%token-table
%debug
%start goal
@g

@ A parser for section~1 (definitions and declarations). This parser requires a custom
lexer, as discussed above, to properly set up the state. Short of
this, the lexer may produce the wrong kind of tokens or even generate
an error.
@(ddp.yy@>=
@G
%{@> @<Preamble for the \flex\ parser@> @=%}
 @> @<Options for \flex\ parser@> @= 
%union {@> @=}
%{@> @<Postamble for \flex\ parser@> @=%} 
 @> @<Token definitions for \flex\ input parser@>@= 
%%
 @> @<Exclusive productions for \flex\ section~1 parser@> @= 
 @> @<Productions for \flex\ section~1 parser@> @= 
%%

@ A parser for section~2 (rules and actions). This subparser must also
use a custom set up for its lexer as discussed above.
@(rap.yy@>=
@G
%{@> @<Preamble for the \flex\ parser@> @=%}
 @> @<Options for \flex\ parser@> @= 
%union {@> @=}
%{@> @<Postamble for \flex\ parser@> @=%} 
 @> @<Token definitions for \flex\ input parser@>@= 
%%
 @> @<Special \flex\ section~2 parser productions@> @= 
 @> @<Productions for \flex\ section~2 parser@> @= 
%%
@g

@ A parser for just the regular expression syntax. A custom
lexer initialization must precede the use of this parser, as well.
@(rep.yy@>=
@G
%{@> @<Preamble for the \flex\ parser@> @=%}
 @> @<Options for \flex\ parser@> @= 
%union {@> @=}
%{@> @<Postamble for \flex\ parser@> @=%} 
 @> @<Token definitions for \flex\ input parser@>@= 
%%
 @> @<Special productions for regular expressions@> @= 
 @> @<Rules for \flex\ regular expressions@> @= 
%%

@*1 Token and state declarations for the \eatone{flex}\flex\ input scanner.
Needless to say, the original grammar used by \flex\ was not designed
with pretty printing in mind (and why would it be?). Instead, efficiency
was the goal which resulted in a number of lexical constructs being
processed `on the fly', as the lexer encounters them. Such syntax
fragments never reach the parser, and would not have a chance to be
displayed by our routines, unless some grammar extensions and
alterations were introduced.

To make the pretty printing possible, a number of new tokens have been
introduced below that are later used in a few altered or entirely new
grammar productions.
@<Token definitions for \flex\ input parser@>=
@G
%token CHAR NUMBER SECTEND SCDECL XSCDECL NAME PREVCCL EOF_OP
%token OPTION_OP OPT_OUTFILE OPT_PREFIX OPT_YYCLASS OPT_HEADER OPT_EXTRA_TYPE
%token OPT_TABLES

%token CCE_ALNUM CCE_ALPHA CCE_BLANK CCE_CNTRL CCE_DIGIT CCE_GRAPH
%token CCE_LOWER CCE_PRINT CCE_PUNCT CCE_SPACE CCE_UPPER CCE_XDIGIT

%token CCE_NEG_ALNUM CCE_NEG_ALPHA CCE_NEG_BLANK CCE_NEG_CNTRL CCE_NEG_DIGIT CCE_NEG_GRAPH
%token CCE_NEG_LOWER CCE_NEG_PRINT CCE_NEG_PUNCT CCE_NEG_SPACE CCE_NEG_UPPER CCE_NEG_XDIGIT

%left CCL_OP_DIFF CCL_OP_UNION

@ We introduce an additional option type to capture all the non-parametric options used
by the \flex\ lexer. The original lexer processes these options at the
point of recognition, while the typesetting parser needs to be aware of them.
@<Token definitions for \flex\ input parser@>=
@G
%token TOP_OP POINTER_OP ARRAY_OP DEF_OP RE_DEF OPT_OTHER OPT_DEPRECATED

@ \namedspot{flex braces}{\em \POSIX\ and \.{AT\&T} \lex\ place the
precedence of the repeat operator, \.{\{\}}, below that of concatenation.
Thus, \.{ab\{3\}} is\/ \.{ababab}.  Most other \POSIX\ utilities use an {\rm Extended
Regular Expression (ERE)} precedence that has the repeat operator
higher than concatenation. This causes \.{ab\{3\}} to yield\/ \.{abbb}.

In order to support the \POSIX\ and \.{AT\&T} precedence and the \flex\
precedence we define two token sets for the begin and end tokens of
the repeat operator, \prodstyle{BEGIN_REPEAT_POSIX} and\/ \prodstyle{END_REPEAT_POSIX}.  The lexical scanner chooses
which tokens to return based on whether {\let\it\itbold\prodstyle{posix_compat} or \prodstyle{lex_compat}
are specified. Specifying either \prodstyle{posix_compat} or \prodstyle{lex_compat}} will
cause \flex\ to parse scanner files as per the \.{AT\&T} and \POSIX-mandated behavior.}
@<Token definitions for \flex\ input parser@>=
@G
%token BEGIN_REPEAT_POSIX END_REPEAT_POSIX BEGIN_REPEAT_FLEX END_REPEAT_FLEX

@*1 The grammar for \eatone{flex}\flex\ input.
The original grammar has been carefully split into sections to
facilitate the assembly of various subparsers in the \flex's
stack. Neither the \flex\ parser nor its scanner are part of the
bootstrap procedure which simplifies both the input file organization,
as well as the macro design. Some amount of preprocessing is still
necessary, however, to extract the state names from the lexer file
(see \ifbootstrapmode\else\locallink{state.grabbing}above \endlink\fi for the
explanation). We can nevertheless get away with an empty \Cee\ preamble.
@<Preamble for the \flex\ parser@>=

@ @<Productions for \flex\ parser@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
@t}\vb{\inline\flatten}{@>
goal:
   initlex sect1 sect1end
     sect2 initforrule           {@> @<Assemble a \flex\ input file@> @=}
 ;

sect1end:
   SECTEND                       {@> @<Copy the value@> @=}
 ;

initlex:
                                 {@> @=}
 ;
@g

@ @<Assemble a \flex\ input file@>=
  @[TeX_( "/finishlist{/the/yy(4)}" );@]@;
  @[TeX_( "/yy0{/the/yy(2)/nx/executelist{/the/yy(4)}}" );@]@;

@ @<Productions for \flex\ parser@>=
@<Productions for \flex\ section~1 parser@>@;
@<Productions for \flex\ section~2 parser@>@;

@t}\endfoldedsections{@>

@ @<Exclusive productions for \flex\ section~1 parser@>=
@G
@t}\vb{\inline\flatten}{@>
goal:
   sect1                         {@> @<Assemble a \flex\ section~1 file@> @=}
 ;

@ @<Assemble a \flex\ section~1 file@>=
  @[TeX_( "/table/expandafter{/the/yy(1)}" );@]@;

@ @<Productions for \flex\ section~1 parser@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
sect1:
   sect1 startconddecl namelist1 {@> @<Add start condition declarations@> @=}
 | sect1 options                 {@> @<Add options to section~1@> @=}
 |                               {@> @<Create an empty section~1@> @=}
 | error                         {@> @<Report an error in section~1 and quit@> @=}
 ;
 
startconddecl:
   SCDECL                        {@> @<Prepare a state declaration@> @=}
 | XSCDECL                       {@> @<Prepare an exclusive state declaration@> @=}
 ;
 
namelist1:
   namelist1 NAME                {@> @<Add a name to a list@> @=}
 | NAME                          {@> @<Start a \prodstylens{namelist1}{\flexnamespace} with a name@> @=}
 | error                         {@> @<Report an error in \prodstylens{namelist1}{\flexnamespace} and quit@> @=}
 ;
@g

@ @<Add start condition declarations@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flscondecl/the/yy(2){/the/yy(3)}}" );@]@;

@ @<Add options to section~1@>=
  @[TeX_( "/yy0{/the/yy(1)/the/yy(2)}" );@]@;

@ @<Create an empty section~1@>=
  @[TeX_( "/yy0{}" );@]@;

@ @<Report an error in section~1 and quit@>=
  @[TeX_( "/yyerror" );@]@;

@ @<Prepare a state declaration@>=
  @[TeX_( "/yy0{{s}/the/yy(1)}" );@]@;

@ @<Prepare an exclusive state declaration@>=
  @[TeX_( "/yy0{{x}/the/yy(1)}" );@]@;

@ @<Add a name to a list@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flnamesep{}{}/nx/flname/the/yy(2)}" );@]@;

@ @<Start a \prodstylens{namelist1}{\flexnamespace} with a name@>=
  @[TeX_( "/yy0{/nx/flname/the/yy(1)}" );@]@;

@ @<Report an error in \prodstylens{namelist1}{\flexnamespace} and quit@>=
  @[TeX_( "/yyerror" );@]@;

@t}\endfoldedsections{@>

@ @<Productions for \flex\ section~1 parser@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
options:
   OPTION_OP optionlist          {@> @<Start an options list@> @=}
 | POINTER_OP                    {@> @<Add a pointer option@> @=}
 | ARRAY_OP                      {@> @<Add an array option@> @=}
 | TOP_OP '\n'                   {@> @<Add a \prodstylens{\%top}{\flexnamespace} directive@> @=}
 | DEF_OP RE_DEF                 {@> @<Add a regular expression definition@> @=}
 | OPT_DEPRECATED                {@> @<Output a deprecated option@> @=}
 ;
@t}\vb{\inline\flatten}{@>
optionlist:
   optionlist option             {@> @<Add an option to a list@> @=}
 |                               {@> @<Make an empty option list@> @=}
 ;
@t}\vb{\resetf}{@>
option:
   OPT_OUTFILE '=' NAME          {@> @<Record the name of the output file@> @=}
 | OPT_EXTRA_TYPE '=' NAME       {@> @<Declare an extra type@> @=}
 | OPT_PREFIX '=' NAME           {@> @<Declare a prefix@> @=}
 | OPT_YYCLASS '=' NAME          {@> @<Declare a class@> @=}
 | OPT_HEADER '=' NAME           {@> @<Declare the name of a header@> @=}
 | OPT_TABLES '=' NAME           {@> @<Declare the name for the tables@> @=}
 | OPT_OTHER                     {@> @<Output a non-parametric option@> @=}
 ;
@g

@ @<Start an options list@>=
  @[TeX_( "/yy0{/nx/floptions{/the/yy(2)}}" );@]@;

@ @<Add a pointer option@>=
  @[TeX_( "/yy0{/nx/flptropt/the/yy(1)}" );@]@;
  
@ @<Add an array option@>=
  @[TeX_( "/yy0{/nx/flarrayopt/the/yy(1)}" );@]@;

@ @<Add a \prodstylens{\%top}{\flexnamespace} directive@>=
  @[TeX_( "/yy0{/nx/fltopopt/the/yy(1)/the/yy(2)}" );@]@;

@ @<Add a regular expression definition@>=
  @[TeX_( "/yy0{/nx/flredef/the/yy(1)/the/yy(2)}" );@]@;
  
@ @<Add an option to a list@>=
  @[TeX_( "/yy0{/the/yy(1)/the/yy(2)}" );@]@;

@ @<Make an empty option list@>=
  @[TeX_( "/yy0{}" );@]@;

@ @<Record the name of the output file@>=
  @[TeX_( "/yy0{/nx/flopt{file}/the/yy(3)}" );@]@;

@ @<Declare an extra type@>=
  @[TeX_( "/yy0{/nx/flopt{xtype}/the/yy(3)}" );@]@;
  
@ @<Declare a prefix@>=
  @[TeX_( "/yy0{/nx/flopt{prefix}/the/yy(3)}" );@]@;
  
@ @<Declare a class@>=
  @[TeX_( "/yy0{/nx/flopt{yyclass}/the/yy(3)}" );@]@;
  
@ @<Declare the name of a header@>=
  @[TeX_( "/yy0{/nx/flopt{header}/the/yy(3)}" );@]@;
  
@ @<Declare the name for the tables@>=
  @[TeX_( "/yy0{/nx/flopt{tables}/the/yy(3)}" );@]@;

@ @<Output a non-parametric option@>=
  @[TeX_( "/yy0{/nx/flopt{other}/the/yy(1)}" );@]@;

@ @<Output a deprecated option@>=
  @[TeX_( "/yy0{/nx/flopt{deprecated}/the/yy(1)}" );@]@;

@t}\endfoldedsections{@>

@ @<Special \flex\ section~2 parser productions@>=
@G
@t}\vb{\inline\flatten}{@>
goal:
   sect2                                     {@> @<Output section~2@> @=} 
 ;
@g

@ @<Output section~2@>=
  @[TeX_( "/finishlist{/the/yy(1)}" );@]@;
  @[TeX_( "/table/expandafter{/expandafter/executelist/expandafter{/the/yy(1)}}" );@]@;

@ This portion of the grammar was changed to make it possible to read the
action code.
@<Productions for \flex\ section~2 parser@>=
@G
sect2:
   sect2 scon initforrule flexrule '\n' '\n' {@> @<Add a rule to section~2@> @=}
 | sect2 scon '{' sect2 '}'                  {@> @<Add a group of rules to section~2@> @=}
 |                                           {@> @<Start an empty section~2@> @=}
 | sect2 '\n'                                {@> @<Add a bare action@> @=}
 ;
@t}\vb{\inline\flatten}{@>
initforrule:
                                            {@> @[TeX_( "/flin@@ruletrue/yylexnext" );@] @=}
 ;
@g

@ The production below describes the most typical way a regular expression is
assigned an action. The redundant term \prodstyle{initforrule} is a standard
\bison\ trick to make sure that the appropriate initializations happen at
the right time.
\smallskip
\thisrulereference{\nx\inline\nx\flatten}
\smallskip\noindent
The original production has been modified so that the pretty printing parser has a chance
to consume the action code. The second \prodstyle{'\\n'} is output by the action
processing code.
@<Add a rule to section~2@>=
    @[TeX_( "/ifflcontinued@@action" );@]@;
    @[TeX_( "    /toksb{/flactionc}" );@]@;
    @[TeX_( "/else" );@]@;
    @[TeX_( "    /toksb{/flaction}" );@]@;
    @[TeX_( "/fi" );@]@;
    @[TeX_( "/toksa/expandafter{/astformat@@flaction}" );@]@; /* capture the formatting action */
    @[TeX_( "/yy0{/the/yy(1)}" );@]@;
    @[TeX_( "/appendtolistx{/the/yy(1)}{/the/toksb{/the/yy(2)}{/the/yy(4)}/the/yy(5)/the/yy(6){/the/toksa}}" );@]@;
    @[TeX_( "/let/astformat@@flaction/empty" );@]@; /* reset the format */

@ For convenience, rules that are active in the same set of states may be grouped together.
This pattern is the subject of the next production.
\smallskip
\thisrulereference{\nx\inline\nx\flatten}
\smallskip\noindent
The original parser ignores the braces while the pretty printing parser
uses the pointers associated with the braces to collect and process the accumulated
stash. This is how comments and \CWEB\ section references are typeset.
@<Add a group of rules to section~2@>=
  @[TeX_( "/yy0{/the/yy(1)}" );@]@;
  @[TeX_( "/finishlist{/the/yy(4)}" );@]@;
  @[TeX_( "/appendtolistx{/the/yy(1)}{/nx/flactiongroup{/the/yy(2)}/the/yy(3){/nx/executelist{/the/yy(4)}}/the/yy(5)}" );@]@;

@ Simple left recursive terms like~\prodstyle{sect2} are very suitable for being implemented as a list
(see the macros in \.{yycommon.sty} for the details on the list implementation). The `type' of~\prodstyle{sect2}
is a (symbolic pointer to a) list of items built up from an empty initial list. This production initializes
the list (with the name identical to the terminal on the left hand side of the production)
and updates the list name (rather the name's prefix) for future invocations of this action.

@<Start an empty section~2@>=
  @[TeX_( "/initlist{/secttwoprefix sect2}" );@]@;
  @[TeX_( "/yy0{/secttwoprefix sect2}" );@]@;
  @[TeX_( "/edef/secttwoprefix{/secttwoprefix.}" );@]@;

@ @<Add a bare action@>=
  @[TeX_( "/yy0{/the/yy(1)}" );@]@;
  @[TeX_( "/appendtolistx{/the/yy(1)}{/nx/flbareaction/the/yy(2)}" );@]@;

@ @<Productions for \flex\ section~2 parser@>=
@G
@t}\vb{\inline\flatten}{@>
scon_stk_ptr:
                                   {@> @=}
 ;
@t}\vb{\resetf}{@>
scon:
   '<' scon_stk_ptr namelist2 '>'  {@> @<Create a list of start conditions@> @=}
 | '<' '*' '>'                     {@> @<Create a universal start condition@> @=}
 |                                 {@> @<Create an empty start condition@> @=}
 ;

namelist2:
   namelist2 ',' sconname          {@> @<Add a start condition to a list@> @=}
 | sconname                        {@> @<Start a list with a start condition name@> @=}
 | error                           {@> @<Report an error in a start condition list@> @=}
 ;
@t}\vb{\inline\flatten}{@>
sconname:
   NAME                            {@> @<Make a \prodstylens{NAME}{\flexnamespace} into a start condition@> @=}
 ;

@ Start conditions are just names. The data structure that is output has
location pointers for the streams to enable interaction with \CWEB. These
pointers are in turn the values of the angle bracket tokens that enclose the
list of start conditions.

Start condition lists may be collected in their own sections, while the list itself
may be followed by a comment. The pointers mentioned above are used to typeset the comments
and section references.
\beginfoldedsectionshere
@<Create a list of start conditions@>=
  @[TeX_( "/yy0{/nx/flsconlist{/the/yy(1)}{/the/yy(3)}{/the/yy(4)}}" );@]@;

@ @<Create a universal start condition@>=
  @[TeX_( "/yy0{/nx/flsconuniv/the/yy(3)}" );@]@;

@ @<Create an empty start condition@>=
  @[TeX_( "/yy0{}" );@]@;

@ @<Add a start condition to a list@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flnamesep/the/yy(2)/the/yy(3)}" );@]@;

@ @<Start a list with a start condition name@>=
  @<Copy the value@>@;

@ @<Report an error in a start condition list@>=
  @[TeX_( "/yyerror" );@]@;

@ @<Make a \prodstylens{NAME}{\flexnamespace} into a start condition@>=
  @[TeX_( "/yy0{/nx/flname/the/yy(1)}" );@]@;

@t}\endfoldedsections{@>

@*1 The syntax of regular expressions. The productions in this section define the
syntax of \flex\ regular expressions in detail. The same productions are used for
parsing isolated regular expressions (e.g.\ to present example code). A few of these
productions have been modified to suit the needs of the pretty printing parser.
@<Productions for \flex\ section~2 parser@>=
    @<Rules for \flex\ regular expressions@>@;

@ @<Special productions for regular expressions@>=
@G
@t}\vb{\inline\flatten}{@>
goal:
   flexrule                        {@> @<Output a regular expression@> @=}
 ;
@g

@ The parsed regular expression is output in the \.{\\table}
register. It is important to ensure that whenever this parser is used
inside another parser that uses \.{\\table} for output, the changes to
this register stay local. The \.{\\frexproc} macro in \.{yyunion.sty}
ensures that all the changes are local to the parsing macro.
@<Output a regular expression@>=
  @[TeX_( "/table/yy(1)" );@]@;

@ Regular expressions are parsed using the following productions. There are
two major cases: rules active only at the beginning of the line, and the rest. From the
typesetting parser's point of view, there is not much difference between the two
(certainly not enough to justify singling out the rules at the beginning of the line into
their own production) but it was decided to keep the original grammar rules
for consistency.
@<Rules for \flex\ regular expressions@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
flexrule:
   '^' rule                        {@> @<Match a rule at the start of the line@> @=}
 | rule                            {@> @<Match an ordinary rule@> @=}
 | EOF_OP                          {@> @<Match an end of file@> @=}
 | error                           {@> @<Report an error and quit@> @=}
 ;

@ @<Match a rule at the start of the line@>=
    @[TeX_( "/toksa/expandafter{/astformat@@flrule}" );@]@;
    @[TeX_( "/let/astformat@@flrule/empty" );@]@;
    @[TeX_( "/yy0{/nx/flbolrule{/the/yy(2)}{/the/toksa}}" );@]@;

@ @<Match an end of file@>=
  @[TeX_( "/yy0{/nx/fleof/the/yy(1)}" );@]@;

@ @<Match an ordinary rule@>=
    @[TeX_( "/toksa/expandafter{/astformat@@flrule}" );@]@;
    @[TeX_( "/let/astformat@@flrule/empty" );@]@;
    @[TeX_( "/yy0{/nx/flrule{/the/yy(1)}{/the/toksa}}" );@]@;

@ @<Report an error and quit@>=
  @[TeX_( "/yyerror" );@]@;

@t}\endfoldedsections{@>

@ Another broad overview of regular expression types before diving into the details
of various operations. Note that the only trailing context that \splint\ output
lexer can process is the end of line (\prodstyle{'\$'}) due to the way the
scanner routine is written. It does not affect its ability to pretty print the appropriate
rules (for a lexer that is produced by \flex\ itself, for example).
@<Rules for \flex\ regular expressions@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
rule:
   re2 re                          {@> @<Match a regular expression with a trailing context@> @=}
 | re2 re '$'                      {@> @<Disallow a repeated trailing context@> @=}
 | re '$'                          {@> @<Match a regular expression at the end of the line@> @=}
 | re                              {@> @<Match an ordinary regular expression@> @=}
 ;

re:
   re '|' series                   {@> @<Match a sequence of alternatives@> @=}
 | series                          {@> @<Match a sequence of singletons@> @=}
 ;
@t}\vb{\inline\flatten}{@>
re2:
   re '/'                          {@> @<Prepare to match a trailing context@> @=}
 ;
@g

@ @<Match a regular expression with a trailing context@>=
  @[TeX_( "/getsecond{/yy(1)}/to/toksa/getthird{/yy(1)}/to/toksb" );@]@;
  @[TeX_( "/yy0{/nx/flretrail{/the/toksa}{/the/toksb}{/the/yy(2)}}" );@]@;
  
@ @<Disallow a repeated trailing context@>=
  @[TeX_( "/yyerror" );@]@;
  
@ @<Match a regular expression at the end of the line@>=
  @[TeX_( "/yy0{/nx/flreateol{/the/yy(1)}/the/yy(2)}" );@]@;

@ @<Match an ordinary regular expression@>=
  @<Copy the value@>@;
  
@ @<Match a sequence of alternatives@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flor/the/yy(2)/the/yy(3)}" );@]@;

@ @<Match a sequence of singletons@>=
  @<Copy the value@>@;

@ @<Prepare to match a trailing context@>=
  @[TeX_( "/yy0{/nx/fltrail{/the/yy(1)}{/the/yy(2)}}" );@]@;

  @t}\endfoldedsections{@>
  
@*2 Atoms. Every regular expression is assembled of atomic subexpressions, each
of which may be modified by an repetition operator that establishes how many
times a given pattern can repeat to stay part of the original atom. New atomic expressions
(or \prodstyle{singleton}s as they are called below) can be formed the usual way, by
enclosing a regular expression in parentheses.

As explained \locallink{flex braces}above\endlink, braced repetition operators may
have different binding strengths, depending on the options supplied to \flex. The pretty
printing in both cases is identical as only the application scopes of the operator differ, and not
its meaning.
@<Rules for \flex\ regular expressions@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
series:
   series singleton                     {@> @<Extend a series by a singleton@> @=}
 | singleton                            {@> @<Match a singleton@> @=}
 | series BEGIN_REPEAT_POSIX                     
     NUMBER ',' NUMBER END_REPEAT_POSIX {@> @<Match a series of specific length@> @=}                 
 | series BEGIN_REPEAT_POSIX
     NUMBER ',' END_REPEAT_POSIX        {@> @<Match a series of minimal length@> @=}
 | series BEGIN_REPEAT_POSIX
     NUMBER END_REPEAT_POSIX            {@> @<Match a series of exact length@> @=}
 ;                                               
@g

@ @<Extend a series by a singleton@>=
  @[TeX_( "/yy0{/the/yy(1)/the/yy(2)}" );@]@;
  
@ @<Match a singleton@>=
  @<Copy the value@>@;

@ @<Match a series of specific length@>=
  @<Create a series of specific length@>@;
  
@ @<Match a series of minimal length@>=
  @<Create a series of minimal length@>@;
  
@ @<Match a series of exact length@>=
  @<Create a series of exact length@>@;

@t}\endfoldedsections{@>

@ @<Rules for \flex\ regular expressions@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
singleton:                                       
   singleton '*'                        {@> @<Create a lazy series match@> @=}
 | singleton '+'                        {@> @<Create a nonempty series match@> @=}
 | singleton '?'                        {@> @<Create a possible single match@> @=}
 | singleton BEGIN_REPEAT_FLEX                   
     NUMBER ',' NUMBER END_REPEAT_FLEX  {@> @<Create a series of specific length@> @=}
 | singleton BEGIN_REPEAT_FLEX                   
     NUMBER ',' END_REPEAT_FLEX         {@> @<Create a series of minimal length@> @=}
 | singleton BEGIN_REPEAT_FLEX
     NUMBER END_REPEAT_FLEX             {@> @<Create a series of exact length@> @=}
 | '.'                                  {@> @<Match (almost) any character@> @=}
 | fullccl                              {@> @<Match a character class@> @=}
 | PREVCCL                              {@> @<Match a \prodstylens{PREVCCL}{\flexnamespace}@> @=}
 | '"' string '"'                       {@> @<Match a string@> @=}
 | '(' re ')'                           {@> @<Match an atom@> @=}
 | CHAR                                 {@> @<Match a specific character@> @=}
 ;
@g
                                                 
@ @<Create a lazy series match@>=
  @[TeX_( "/yy0{/nx/flrepeat{/the/yy(1)}}" );@]@;
  
@ @<Create a nonempty series match@>=
  @[TeX_( "/yy0{/nx/flrepeatstrict{/the/yy(1)}}" );@]@;
  
@ @<Create a possible single match@>=
  @[TeX_( "/yy0{/nx/flrepeatonce{/the/yy(1)}}" );@]@;
  
@ @<Create a series of specific length@>=
  @[TeX_( "/yy0{/nx/flrepeatnm{/the/yy(1)}{/the/yy(3)}{/the/yy(5)}}" );@]@;
  
@ @<Create a series of minimal length@>=
  @[TeX_( "/yy0{/nx/flrepeatgen{/the/yy(1)}{/the/yy(3)}}" );@]@;
  
@ @<Create a series of exact length@>=
  @[TeX_( "/yy0{/nx/flrepeatn{/the/yy(1)}{/the/yy(3)}}" );@]@;
  
@ @<Match (almost) any character@>=
  @[TeX_( "/yy0{/nx/fldot/the/yy(1)}" );@]@;

@ @<Match a character class@>=
  @<Copy the value@>@;
  
@ @<Match a \prodstylens{PREVCCL}{\flexnamespace}@>=
  @<Copy the value@>@;
  
@ @<Match a string@>=
  @[TeX_( "/yy0{/nx/flstring{/the/yy(1)}{/the/yy(2)}{/the/yy(3)}}" );@]@;
  
@ @<Match an atom@>=
    @[TeX_( "/toksa/expandafter{/astformat@@flparens}" );@]@;
    @[TeX_( "/let/astformat@@flparens/empty" );@]@;
    @[TeX_( "/yy0{/nx/flparens{/the/yy(1)}{/the/yy(2)}{/the/yy(3)}{/the/toksa}}" );@]@;
  
@ @<Match a specific character@>=
  @[TeX_( "/yy0{/nx/flchar/the/yy(1)}" );@]@;

@t}\endfoldedsections{@>

@*2 Characters. Several facilities are available to specify sets of characters, including
built-in characters classes such as {\em whitespace}, {\em printable characters},
{\em alphanumerics}, etc. Some simple boolean operaions are also supported to make specifying
character classes more efficient.
@<Rules for \flex\ regular expressions@>=
@G
@t}\vb{\insertraw{\beginfoldedsections}}{@>
fullccl:                                         
   fullccl CCL_OP_DIFF  braceccl        {@> @<Subtract a character class@> @=}
 | fullccl CCL_OP_UNION braceccl        {@> @<Create a union of character classes@> @=}
 | braceccl                             {@> @<Turn a basic character class into a character class@> @=}
 ;

braceccl:
   '[' ccl ']'        {@> @<Create a character class@> @=}
 | '[' '^' ccl ']'    {@> @<Complement a character class@> @=}
 ;                             
                               
ccl:                           
   ccl CHAR '-' CHAR  {@> @<Add a range to a character class@> @=}
 | ccl CHAR           {@> @<Add a character to a character class@> @=}
 | ccl ccl_expr       {@> @<Add an expression to a character class@> @=}
 |                    {@> @<Create an empty character class@> @=}
 ;
@g

@ @<Subtract a character class@>=
  @[TeX_( "/yy0{/nx/flccldiff{/the/yy(1)}{/the/yy(3)}}" );@]@;
  
@ @<Create a union of character classes@>=
  @[TeX_( "/yy0{/nx/flcclunion{/the/yy(1)}{/the/yy(3)}}" );@]@;

@ @<Turn a basic character class into a character class@>=
  @<Copy the value@>@;

@ @<Create a character class@>=
  @[TeX_( "/yy0{/nx/flbraceccl{/the/yy(1)}{/the/yy(2)}{/the/yy(3)}}" );@]@;
  
@ @<Complement a character class@>=
  @[TeXb( "/yy0{/nx/flbracecclneg" );@]@;
  @[TeXfo( "    {/the/yy(1)}{/the/yy(3)}{/the/yy(4)}}" );@]@;
  
@ @<Add a range to a character class@>=
  @[TeXb( "/yy0{/the/yy(1)/nx/flcclrnge" );@]@;
  @[TeXfo( "    {/nx/flchar/the/yy(2)}{/nx/flchar/the/yy(4)}}" );@]@;

@ @<Add a character to a character class@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flchar/the/yy(2)}" );@]@;

@ @<Add an expression to a character class@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flcclexpr/the/yy(2)}" );@]@;

@ @<Create an empty character class@>=
  @[TeX_( "/yy0{}" );@]@;

@t}\endfoldedsections{@>

@*3 Special character classes. Various character classes are predefined in \flex.
These include alphabetic and alphanumeric characters, digits, blank characters, upper and lower
case characters, etc.
@<Rules for \flex\ regular expressions@>=
@G
ccl_expr:
@t}\vb{\flatten}{@>
   CCE_ALNUM      {@> @<Copy the value@> @=}
 | CCE_ALPHA      {@> @<Copy the value@> @=}
 | CCE_BLANK      {@> @<Copy the value@> @=}
 | CCE_CNTRL      {@> @<Copy the value@> @=}
 | CCE_DIGIT      {@> @<Copy the value@> @=}
 | CCE_GRAPH      {@> @<Copy the value@> @=}
 @t}\vb{\fold\flatten}{@>
 | CCE_LOWER      {@> @<Copy the value@> @=}
 | CCE_PRINT      {@> @<Copy the value@> @=}
 | CCE_PUNCT      {@> @<Copy the value@> @=}
 | CCE_SPACE      {@> @<Copy the value@> @=}
 | CCE_XDIGIT     {@> @<Copy the value@> @=}
 | CCE_UPPER      {@> @<Copy the value@> @=}
@t}\vb{\fold\flatten}{@>
 | CCE_NEG_ALNUM  {@> @<Copy the value@> @=}
 | CCE_NEG_ALPHA  {@> @<Copy the value@> @=}
 | CCE_NEG_BLANK  {@> @<Copy the value@> @=}
 | CCE_NEG_CNTRL  {@> @<Copy the value@> @=}
 | CCE_NEG_DIGIT  {@> @<Copy the value@> @=}
 | CCE_NEG_GRAPH  {@> @<Copy the value@> @=}
 @t}\vb{\fold\flatten}{@>
 | CCE_NEG_PRINT  {@> @<Copy the value@> @=}
 | CCE_NEG_PUNCT  {@> @<Copy the value@> @=}
 | CCE_NEG_SPACE  {@> @<Copy the value@> @=}
 | CCE_NEG_XDIGIT {@> @<Copy the value@> @=}
 | CCE_NEG_LOWER  {@> @<Copy the value@> @=}
 | CCE_NEG_UPPER  {@> @<Copy the value@> @=}
 ;
@t}\vb{\inline}{@>
string:
   string CHAR    {@> @<Extend a \flex\ string by a character@> @=}
 |                {@> @<Make an empty regular expression string@> @=}
 ;
@g

@ @<Copy the value@>=
  @[TeX_( "/yy0{/the/yy(1)}" );@]@;

@ @<Extend a \flex\ string by a character@>=
  @[TeX_( "/yy0{/the/yy(1)/nx/flchar/the/yy(2)}" );@]@;

@ @<Make an empty regular expression string@>=
  @[TeX_( "/yy0{}" );@]@;

@ The postamble is empty for now.
@<Postamble for \flex\ parser@>=