summaryrefslogtreecommitdiff
path: root/Build/source/texk/web2c/luatexdir/tex/textoken.w
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/tex/textoken.w')
-rw-r--r--Build/source/texk/web2c/luatexdir/tex/textoken.w2410
1 files changed, 2410 insertions, 0 deletions
diff --git a/Build/source/texk/web2c/luatexdir/tex/textoken.w b/Build/source/texk/web2c/luatexdir/tex/textoken.w
new file mode 100644
index 00000000000..ac9ad1e1013
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/tex/textoken.w
@@ -0,0 +1,2410 @@
+% textoken.w
+%
+% Copyright 2006-2010 Taco Hoekwater <taco@@luatex.org>
+
+% This file is part of LuaTeX.
+
+% LuaTeX is free software; you can redistribute it and/or modify it under
+% the terms of the GNU General Public License as published by the Free
+% Software Foundation; either version 2 of the License, or (at your
+% option) any later version.
+
+% LuaTeX is distributed in the hope that it will be useful, but WITHOUT
+% ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+% FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+% License for more details.
+
+% You should have received a copy of the GNU General Public License along
+% with LuaTeX; if not, see <http://www.gnu.org/licenses/>.
+
+@ @c
+static const char _svn_version[] =
+ "$Id: textoken.w 3587 2010-04-03 14:32:25Z taco $"
+ "$URL: http://foundry.supelec.fr/svn/luatex/tags/beta-0.60.0/source/texk/web2c/luatexdir/tex/textoken.w $";
+
+#include "ptexlib.h"
+
+@ @c
+#define pausing int_par(pausing_code)
+#define cat_code_table int_par(cat_code_table_code)
+#define tracing_nesting int_par(tracing_nesting_code)
+#define end_line_char int_par(end_line_char_code)
+#define suppress_outer_error int_par(suppress_outer_error_code)
+
+#define every_eof equiv(every_eof_loc)
+#define box(A) equiv(box_base+(A))
+
+#define detokenized_line() (line_catcode_table==NO_CAT_TABLE)
+
+#define do_get_cat_code(a) do { \
+ if (line_catcode_table!=DEFAULT_CAT_TABLE) \
+ a=get_cat_code(line_catcode_table,cur_chr); \
+ else \
+ a=get_cat_code(cat_code_table,cur_chr); \
+ } while (0)
+
+
+@ The \TeX\ system does nearly all of its own memory allocation, so that it
+can readily be transported into environments that do not have automatic
+facilities for strings, garbage collection, etc., and so that it can be in
+control of what error messages the user receives. The dynamic storage
+requirements of \TeX\ are handled by providing two large arrays called
+|fixmem| and |varmem| in which consecutive blocks of words are used as
+nodes by the \TeX\ routines.
+
+Pointer variables are indices into this array, or into another array
+called |eqtb| that will be explained later. A pointer variable might
+also be a special flag that lies outside the bounds of |mem|, so we
+allow pointers to assume any |halfword| value. The minimum halfword
+value represents a null pointer. \TeX\ does not assume that |mem[null]| exists.
+
+
+
+@ Locations in |fixmem| are used for storing one-word records; a conventional
+\.{AVAIL} stack is used for allocation in this array.
+
+@c
+smemory_word *fixmem; /* the big dynamic storage area */
+unsigned fix_mem_min; /* the smallest location of one-word memory in use */
+unsigned fix_mem_max; /* the largest location of one-word memory in use */
+
+
+@ In order to study the memory requirements of particular applications, it
+is possible to prepare a version of \TeX\ that keeps track of current and
+maximum memory usage. When code between the delimiters |@!stat| $\ldots$
+|tats| is not ``commented out,'' \TeX\ will run a bit slower but it will
+report these statistics when |tracing_stats| is sufficiently large.
+
+@c
+int var_used, dyn_used; /* how much memory is in use */
+
+halfword avail; /* head of the list of available one-word nodes */
+unsigned fix_mem_end; /* the last one-word node used in |mem| */
+
+halfword garbage; /* head of a junk list, write only */
+halfword temp_token_head; /* head of a temporary list of some kind */
+halfword hold_token_head; /* head of a temporary list of another kind */
+halfword omit_template; /* a constant token list */
+halfword null_list; /* permanently empty list */
+halfword backup_head; /* head of token list built by |scan_keyword| */
+
+@ @c
+void initialize_tokens(void)
+{
+ halfword p;
+ avail = null;
+ fix_mem_end = 0;
+ p = get_avail();
+ temp_token_head = p;
+ set_token_info(temp_token_head, 0);
+ p = get_avail();
+ hold_token_head = p;
+ set_token_info(hold_token_head, 0);
+ p = get_avail();
+ omit_template = p;
+ set_token_info(omit_template, 0);
+ p = get_avail();
+ null_list = p;
+ set_token_info(null_list, 0);
+ p = get_avail();
+ backup_head = p;
+ set_token_info(backup_head, 0);
+ p = get_avail();
+ garbage = p;
+ set_token_info(garbage, 0);
+ dyn_used = 0; /* initialize statistics */
+}
+
+@ The function |get_avail| returns a pointer to a new one-word node whose
+|link| field is null. However, \TeX\ will halt if there is no more room left.
+@^inner loop@>
+
+If the available-space list is empty, i.e., if |avail=null|,
+we try first to increase |fix_mem_end|. If that cannot be done, i.e., if
+|fix_mem_end=fix_mem_max|, we try to reallocate array |fixmem|.
+If, that doesn't work, we have to quit.
+
+@c
+halfword get_avail(void)
+{ /* single-word node allocation */
+ unsigned p; /* the new node being got */
+ unsigned t;
+ p = (unsigned) avail; /* get top location in the |avail| stack */
+ if (p != null) {
+ avail = token_link(avail); /* and pop it off */
+ } else if (fix_mem_end < fix_mem_max) { /* or go into virgin territory */
+ incr(fix_mem_end);
+ p = fix_mem_end;
+ } else {
+ smemory_word *new_fixmem; /* the big dynamic storage area */
+ t = (fix_mem_max / 5);
+ new_fixmem =
+ fixmemcast(realloc
+ (fixmem, sizeof(smemory_word) * (fix_mem_max + t + 1)));
+ if (new_fixmem == NULL) {
+ runaway(); /* if memory is exhausted, display possible runaway text */
+ overflow("token memory size", fix_mem_max);
+ } else {
+ fixmem = new_fixmem;
+ }
+ memset(voidcast(fixmem + fix_mem_max + 1), 0, t * sizeof(smemory_word));
+ fix_mem_max += t;
+ p = ++fix_mem_end;
+ }
+ token_link(p) = null; /* provide an oft-desired initialization of the new node */
+ incr(dyn_used); /* maintain statistics */
+ return (halfword) p;
+}
+
+
+@ The procedure |flush_list(p)| frees an entire linked list of
+one-word nodes that starts at position |p|.
+@^inner loop@>
+
+@c
+void flush_list(halfword p)
+{ /* makes list of single-word nodes available */
+ halfword q, r; /* list traversers */
+ if (p != null) {
+ r = p;
+ do {
+ q = r;
+ r = token_link(r);
+ decr(dyn_used);
+ } while (r != null); /* now |q| is the last node on the list */
+ token_link(q) = avail;
+ avail = p;
+ }
+}
+
+@ A \TeX\ token is either a character or a control sequence, and it is
+@^token@>
+represented internally in one of two ways: (1)~A character whose ASCII
+code number is |c| and whose command code is |m| is represented as the
+number $2^{21}m+c$; the command code is in the range |1<=m<=14|. (2)~A control
+sequence whose |eqtb| address is |p| is represented as the number
+|cs_token_flag+p|. Here |cs_token_flag=@t$2^{25}-1$@>| is larger than
+$2^{21}m+c$, yet it is small enough that |cs_token_flag+p< max_halfword|;
+thus, a token fits comfortably in a halfword.
+
+A token |t| represents a |left_brace| command if and only if
+|t<left_brace_limit|; it represents a |right_brace| command if and only if
+we have |left_brace_limit<=t<right_brace_limit|; and it represents a |match| or
+|end_match| command if and only if |match_token<=t<=end_match_token|.
+The following definitions take care of these token-oriented constants
+and a few others.
+
+@ A token list is a singly linked list of one-word nodes in |mem|, where
+each word contains a token and a link. Macro definitions, output-routine
+definitions, marks, \.{\\write} texts, and a few other things
+are remembered by \TeX\ in the form
+of token lists, usually preceded by a node with a reference count in its
+|token_ref_count| field. The token stored in location |p| is called
+|info(p)|.
+
+Three special commands appear in the token lists of macro definitions.
+When |m=match|, it means that \TeX\ should scan a parameter
+for the current macro; when |m=end_match|, it means that parameter
+matching should end and \TeX\ should start reading the macro text; and
+when |m=out_param|, it means that \TeX\ should insert parameter
+number |c| into the text at this point.
+
+The enclosing \.{\char'173} and \.{\char'175} characters of a macro
+definition are omitted, but the final right brace of an output routine
+is included at the end of its token list.
+
+Here is an example macro definition that illustrates these conventions.
+After \TeX\ processes the text
+$$\.{\\def\\mac a\#1\#2 \\b \{\#1\\-a \#\#1\#2 \#2\}}$$
+the definition of \.{\\mac} is represented as a token list containing
+$$\def\,{\hskip2pt}
+\vbox{\halign{\hfil#\hfil\cr
+(reference count), |letter|\,\.a, |match|\,\#, |match|\,\#, |spacer|\,\.\ ,
+\.{\\b}, |end_match|,\cr
+|out_param|\,1, \.{\\-}, |letter|\,\.a, |spacer|\,\.\ , |mac_param|\,\#,
+|other_char|\,\.1,\cr
+|out_param|\,2, |spacer|\,\.\ , |out_param|\,2.\cr}}$$
+The procedure |scan_toks| builds such token lists, and |macro_call|
+does the parameter matching.
+@^reference counts@>
+
+Examples such as
+$$\.{\\def\\m\{\\def\\m\{a\}\ b\}}$$
+explain why reference counts would be needed even if \TeX\ had no \.{\\let}
+operation: When the token list for \.{\\m} is being read, the redefinition of
+\.{\\m} changes the |eqtb| entry before the token list has been fully
+consumed, so we dare not simply destroy a token list when its
+control sequence is being redefined.
+
+If the parameter-matching part of a definition ends with `\.{\#\{}',
+the corresponding token list will have `\.\{' just before the `|end_match|'
+and also at the very end. The first `\.\{' is used to delimit the parameter; the
+second one keeps the first from disappearing.
+
+The |print_meaning| subroutine displays |cur_cmd| and |cur_chr| in
+symbolic form, including the expansion of a macro or mark.
+
+@c
+void print_meaning(void)
+{
+ print_cmd_chr((quarterword) cur_cmd, cur_chr);
+ if (cur_cmd >= call_cmd) {
+ print_char(':');
+ print_ln();
+ token_show(cur_chr);
+ } else {
+ /* Show the meaning of a mark node */
+ if ((cur_cmd == top_bot_mark_cmd) && (cur_chr < marks_code)) {
+ print_char(':');
+ print_ln();
+ switch (cur_chr) {
+ case first_mark_code:
+ token_show(first_mark(0));
+ break;
+ case bot_mark_code:
+ token_show(bot_mark(0));
+ break;
+ case split_first_mark_code:
+ token_show(split_first_mark(0));
+ break;
+ case split_bot_mark_code:
+ token_show(split_bot_mark(0));
+ break;
+ default:
+ token_show(top_mark(0));
+ break;
+ }
+ }
+ }
+}
+
+
+@ The procedure |show_token_list|, which prints a symbolic form of
+the token list that starts at a given node |p|, illustrates these
+conventions. The token list being displayed should not begin with a reference
+count. However, the procedure is intended to be robust, so that if the
+memory links are awry or if |p| is not really a pointer to a token list,
+nothing catastrophic will happen.
+
+An additional parameter |q| is also given; this parameter is either null
+or it points to a node in the token list where a certain magic computation
+takes place that will be explained later. (Basically, |q| is non-null when
+we are printing the two-line context information at the time of an error
+message; |q| marks the place corresponding to where the second line
+should begin.)
+
+For example, if |p| points to the node containing the first \.a in the
+token list above, then |show_token_list| will print the string
+$$\hbox{`\.{a\#1\#2\ \\b\ ->\#1\\-a\ \#\#1\#2\ \#2}';}$$
+and if |q| points to the node containing the second \.a,
+the magic computation will be performed just before the second \.a is printed.
+
+The generation will stop, and `\.{\\ETC.}' will be printed, if the length
+of printing exceeds a given limit~|l|. Anomalous entries are printed in the
+form of control sequences that are not followed by a blank space, e.g.,
+`\.{\\BAD.}'; this cannot be confused with actual control sequences because
+a real control sequence named \.{BAD} would come out `\.{\\BAD\ }'.
+
+@c
+void show_token_list(int p, int q, int l)
+{
+ int m, c; /* pieces of a token */
+ ASCII_code match_chr; /* character used in a `|match|' */
+ ASCII_code n; /* the highest parameter number, as an ASCII digit */
+ match_chr = '#';
+ n = '0';
+ tally = 0;
+ if (l < 0)
+ l = 0x3FFFFFFF;
+ while ((p != null) && (tally < l)) {
+ if (p == q) {
+ /* Do magic computation */
+ set_trick_count();
+ }
+ /* Display token |p|, and |return| if there are problems */
+ if ((p < (int) fix_mem_min) || (p > (int) fix_mem_end)) {
+ tprint_esc("CLOBBERED.");
+ return;
+ }
+ if (token_info(p) >= cs_token_flag) {
+ if (!((inhibit_par_tokens) && (token_info(p) == par_token)))
+ print_cs(token_info(p) - cs_token_flag);
+ } else {
+ m = token_cmd(token_info(p));
+ c = token_chr(token_info(p));
+ if (token_info(p) < 0) {
+ tprint_esc("BAD.");
+ } else {
+ /* Display the token $(|m|,|c|)$ */
+ /* The procedure usually ``learns'' the character code used for macro
+ parameters by seeing one in a |match| command before it runs into any
+ |out_param| commands. */
+ switch (m) {
+ case left_brace_cmd:
+ case right_brace_cmd:
+ case math_shift_cmd:
+ case tab_mark_cmd:
+ case sup_mark_cmd:
+ case sub_mark_cmd:
+ case spacer_cmd:
+ case letter_cmd:
+ case other_char_cmd:
+ print(c);
+ break;
+ case mac_param_cmd:
+ if (!in_lua_escape)
+ print(c);
+ print(c);
+ break;
+ case out_param_cmd:
+ print(match_chr);
+ if (c <= 9) {
+ print_char(c + '0');
+ } else {
+ print_char('!');
+ return;
+ }
+ break;
+ case match_cmd:
+ match_chr = c;
+ print(c);
+ incr(n);
+ print_char(n);
+ if (n > '9')
+ return;
+ break;
+ case end_match_cmd:
+ if (c == 0)
+ tprint("->");
+ break;
+ default:
+ tprint_esc("BAD.");
+ break;
+ }
+ }
+ }
+ p = token_link(p);
+ }
+ if (p != null)
+ tprint_esc("ETC.");
+}
+
+@ @c
+#define do_buffer_to_unichar(a,b) do { \
+ a = (halfword)str2uni(buffer+b); \
+ b += utf8_size(a); \
+ } while (0)
+
+
+@ Here's the way we sometimes want to display a token list, given a pointer
+to its reference count; the pointer may be null.
+
+@c
+void token_show(halfword p)
+{
+ if (p != null)
+ show_token_list(token_link(p), null, 10000000);
+}
+
+
+
+@ |delete_token_ref|, is called when
+a pointer to a token list's reference count is being removed. This means
+that the token list should disappear if the reference count was |null|,
+otherwise the count should be decreased by one.
+@^reference counts@>
+
+@c
+void delete_token_ref(halfword p)
+{ /* |p| points to the reference count
+ of a token list that is losing one reference */
+ assert(token_ref_count(p) >= 0);
+ if (token_ref_count(p) == 0)
+ flush_list(p);
+ else
+ decr(token_ref_count(p));
+}
+
+@ @c
+int get_char_cat_code(int cur_chr)
+{
+ int a;
+ do_get_cat_code(a);
+ return a;
+}
+
+@ @c
+static void invalid_character_error(void)
+{
+ const char *hlp[] =
+ { "A funny symbol that I can't read has just been input.",
+ "Continue, and I'll forget that it ever happened.",
+ NULL
+ };
+ deletions_allowed = false;
+ tex_error("Text line contains an invalid character", hlp);
+ deletions_allowed = true;
+}
+
+@ @c
+static boolean process_sup_mark(void); /* below */
+
+static int scan_control_sequence(void); /* below */
+
+typedef enum { next_line_ok, next_line_return,
+ next_line_restart
+} next_line_retval;
+
+static next_line_retval next_line(void); /* below */
+
+
+@ In case you are getting bored, here is a slightly less trivial routine:
+ Given a string of lowercase letters, like `\.{pt}' or `\.{plus}' or
+ `\.{width}', the |scan_keyword| routine checks to see whether the next
+ tokens of input match this string. The match must be exact, except that
+ uppercase letters will match their lowercase counterparts; uppercase
+ equivalents are determined by subtracting |"a"-"A"|, rather than using the
+ |uc_code| table, since \TeX\ uses this routine only for its own limited
+ set of keywords.
+
+ If a match is found, the characters are effectively removed from the input
+ and |true| is returned. Otherwise |false| is returned, and the input
+ is left essentially unchanged (except for the fact that some macros
+ may have been expanded, etc.).
+ @^inner loop@>
+
+@c
+boolean scan_keyword(const char *s)
+{ /* look for a given string */
+ halfword p; /* tail of the backup list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ const char *k; /* index into |str_pool| */
+ halfword save_cur_cs = cur_cs;
+ if (strlen(s) == 1) {
+ /* Get the next non-blank non-call token; */
+ do {
+ get_x_token();
+ } while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd));
+ if ((cur_cs == 0) && ((cur_chr == *s) || (cur_chr == *s - 'a' + 'A'))) {
+ return true;
+ } else {
+ cur_cs = save_cur_cs;
+ back_input();
+ return false;
+ }
+ } else {
+ p = backup_head;
+ token_link(p) = null;
+ k = s;
+ while (*k) {
+ get_x_token(); /* recursion is possible here */
+ if ((cur_cs == 0) &&
+ ((cur_chr == *k) || (cur_chr == *k - 'a' + 'A'))) {
+ store_new_token(cur_tok);
+ k++;
+ } else if ((cur_cmd != spacer_cmd) || (p != backup_head)) {
+ if (p != backup_head) {
+ q = get_avail();
+ token_info(q) = cur_tok;
+ token_link(q) = null;
+ token_link(p) = q;
+ begin_token_list(token_link(backup_head), backed_up);
+ } else {
+ back_input();
+ }
+ cur_cs = save_cur_cs;
+ return false;
+ }
+ }
+ flush_list(token_link(backup_head));
+ }
+ cur_cs = save_cur_cs;
+ return true;
+}
+
+@ We can not return |undefined_control_sequence| under some conditions
+ (inside |shift_case|, for example). This needs thinking.
+
+@c
+halfword active_to_cs(int curchr, int force)
+{
+ halfword curcs;
+ char *a, *b;
+ char *utfbytes = xmalloc(10);
+ int nncs = no_new_control_sequence;
+ a = (char *) uni2str(0xFFFF);
+ utfbytes = strcpy(utfbytes, a);
+ if (force)
+ no_new_control_sequence = false;
+ if (curchr > 0) {
+ b = (char *) uni2str((unsigned) curchr);
+ utfbytes = strcat(utfbytes, b);
+ free(b);
+ curcs = string_lookup(utfbytes, strlen(utfbytes));
+ } else {
+ utfbytes[3] = '\0';
+ curcs = string_lookup(utfbytes, 4);
+ }
+ no_new_control_sequence = nncs;
+ free(a);
+ free(utfbytes);
+ return curcs;
+}
+
+@ TODO this function should listen to \.{\\escapechar}
+
+@c
+char *cs_to_string(halfword p)
+{ /* prints a control sequence */
+ const char *s;
+ char *sh;
+ int k = 0;
+ static char ret[256] = { 0 };
+ if (p == 0 || p == null_cs) {
+ ret[k++] = '\\';
+ s = "csname";
+ while (*s) {
+ ret[k++] = *s++;
+ }
+ ret[k++] = '\\';
+ s = "endcsname";
+ while (*s) {
+ ret[k++] = *s++;
+ }
+ ret[k] = 0;
+
+ } else {
+ str_number txt = cs_text(p);
+ sh = makecstring(txt);
+ s = sh;
+ if (is_active_cs(txt)) {
+ s = s + 3;
+ while (*s) {
+ ret[k++] = *s++;
+ }
+ ret[k] = 0;
+ } else {
+ ret[k++] = '\\';
+ while (*s) {
+ ret[k++] = *s++;
+ }
+ ret[k] = 0;
+ }
+ free(sh);
+ }
+ return (char *) ret;
+}
+
+@ TODO this is a quick hack, will be solved differently soon
+
+@c
+char *cmd_chr_to_string(int cmd, int chr)
+{
+ char *s;
+ str_number str;
+ int sel = selector;
+ selector = new_string;
+ print_cmd_chr((quarterword) cmd, chr);
+ str = make_string();
+ s = makecstring(str);
+ selector = sel;
+ flush_str(str);
+ return s;
+}
+
+@ The heart of \TeX's input mechanism is the |get_next| procedure, which
+we shall develop in the next few sections of the program. Perhaps we
+shouldn't actually call it the ``heart,'' however, because it really acts
+as \TeX's eyes and mouth, reading the source files and gobbling them up.
+And it also helps \TeX\ to regurgitate stored token lists that are to be
+processed again.
+@^eyes and mouth@>
+
+The main duty of |get_next| is to input one token and to set |cur_cmd|
+and |cur_chr| to that token's command code and modifier. Furthermore, if
+the input token is a control sequence, the |eqtb| location of that control
+sequence is stored in |cur_cs|; otherwise |cur_cs| is set to zero.
+
+Underlying this simple description is a certain amount of complexity
+because of all the cases that need to be handled.
+However, the inner loop of |get_next| is reasonably short and fast.
+
+When |get_next| is asked to get the next token of a \.{\\read} line,
+it sets |cur_cmd=cur_chr=cur_cs=0| in the case that no more tokens
+appear on that line. (There might not be any tokens at all, if the
+|end_line_char| has |ignore| as its catcode.)
+
+
+@ The value of |par_loc| is the |eqtb| address of `\.{\\par}'. This quantity
+is needed because a blank line of input is supposed to be exactly equivalent
+to the appearance of \.{\\par}; we must set |cur_cs:=par_loc|
+when detecting a blank line.
+
+@c
+halfword par_loc; /* location of `\.{\\par}' in |eqtb| */
+halfword par_token; /* token representing `\.{\\par}' */
+
+
+@ Parts |get_next| are executed more often than any other instructions of \TeX.
+@^mastication@>@^inner loop@>
+
+
+
+@ The global variable |force_eof| is normally |false|; it is set |true|
+by an \.{\\endinput} command. |luacstrings| is the number of lua print
+statements waiting to be input, it is changed by |luatokencall|.
+
+@c
+boolean force_eof; /* should the next \.{\\input} be aborted early? */
+int luacstrings; /* how many lua strings are waiting to be input? */
+
+
+@ If the user has set the |pausing| parameter to some positive value,
+and if nonstop mode has not been selected, each line of input is displayed
+on the terminal and the transcript file, followed by `\.{=>}'.
+\TeX\ waits for a response. If the response is simply |carriage_return|, the
+line is accepted as it stands, otherwise the line typed is
+used instead of the line in the file.
+
+@c
+void firm_up_the_line(void)
+{
+ int k; /* an index into |buffer| */
+ ilimit = last;
+ if (pausing > 0) {
+ if (interaction > nonstop_mode) {
+ wake_up_terminal();
+ print_ln();
+ if (istart < ilimit) {
+ for (k = istart; k <= ilimit - 1; k++)
+ print_char(buffer[k]);
+ }
+ first = ilimit;
+ prompt_input("=>"); /* wait for user response */
+ if (last > first) {
+ for (k = first; k < +last - 1; k++) /* move line down in buffer */
+ buffer[k + istart - first] = buffer[k];
+ ilimit = istart + last - first;
+ }
+ }
+ }
+}
+
+
+
+@ Before getting into |get_next|, let's consider the subroutine that
+ is called when an `\.{\\outer}' control sequence has been scanned or
+ when the end of a file has been reached. These two cases are distinguished
+ by |cur_cs|, which is zero at the end of a file.
+
+@c
+void check_outer_validity(void)
+{
+ halfword p; /* points to inserted token list */
+ halfword q; /* auxiliary pointer */
+ if (suppress_outer_error)
+ return;
+ if (scanner_status != normal) {
+ deletions_allowed = false;
+ /* Back up an outer control sequence so that it can be reread; */
+ /* An outer control sequence that occurs in a \.{\\read} will not be reread,
+ since the error recovery for \.{\\read} is not very powerful. */
+ if (cur_cs != 0) {
+ if ((istate == token_list) || (iname < 1) || (iname > 17)) {
+ p = get_avail();
+ token_info(p) = cs_token_flag + cur_cs;
+ begin_token_list(p, backed_up); /* prepare to read the control sequence again */
+ }
+ cur_cmd = spacer_cmd;
+ cur_chr = ' '; /* replace it by a space */
+ }
+ if (scanner_status > skipping) {
+ const char *errhlp[] =
+ { "I suspect you have forgotten a `}', causing me",
+ "to read past where you wanted me to stop.",
+ "I'll try to recover; but if the error is serious,",
+ "you'd better type `E' or `X' now and fix your file.",
+ NULL
+ };
+ char errmsg[256];
+ const char *startmsg;
+ const char *scannermsg;
+ /* Tell the user what has run away and try to recover */
+ runaway(); /* print a definition, argument, or preamble */
+ if (cur_cs == 0) {
+ startmsg = "File ended";
+ } else {
+ cur_cs = 0;
+ startmsg = "Forbidden control sequence found";
+ }
+ /* Print either `\.{definition}' or `\.{use}' or `\.{preamble}' or `\.{text}',
+ and insert tokens that should lead to recovery; */
+ /* The recovery procedure can't be fully understood without knowing more
+ about the \TeX\ routines that should be aborted, but we can sketch the
+ ideas here: For a runaway definition we will insert a right brace; for a
+ runaway preamble, we will insert a special \.{\\cr} token and a right
+ brace; and for a runaway argument, we will set |long_state| to
+ |outer_call| and insert \.{\\par}. */
+ p = get_avail();
+ switch (scanner_status) {
+ case defining:
+ scannermsg = "definition";
+ token_info(p) = right_brace_token + '}';
+ break;
+ case matching:
+ scannermsg = "use";
+ token_info(p) = par_token;
+ long_state = outer_call_cmd;
+ break;
+ case aligning:
+ scannermsg = "preamble";
+ token_info(p) = right_brace_token + '}';
+ q = p;
+ p = get_avail();
+ token_link(p) = q;
+ token_info(p) = cs_token_flag + frozen_cr;
+ align_state = -1000000;
+ break;
+ case absorbing:
+ scannermsg = "text";
+ token_info(p) = right_brace_token + '}';
+ break;
+ default: /* can't happen */
+ scannermsg = "unknown";
+ break;
+ } /*there are no other cases */
+ begin_token_list(p, inserted);
+ snprintf(errmsg, 255, "%s while scanning %s of %s",
+ startmsg, scannermsg, cs_to_string(warning_index));
+ tex_error(errmsg, errhlp);
+ } else {
+ char errmsg[256];
+ const char *errhlp_no[] =
+ { "The file ended while I was skipping conditional text.",
+ "This kind of error happens when you say `\\if...' and forget",
+ "the matching `\\fi'. I've inserted a `\\fi'; this might work.",
+ NULL
+ };
+ const char *errhlp_cs[] =
+ { "A forbidden control sequence occurred in skipped text.",
+ "This kind of error happens when you say `\\if...' and forget",
+ "the matching `\\fi'. I've inserted a `\\fi'; this might work.",
+ NULL
+ };
+ const char **errhlp = (const char **) errhlp_no;
+ char *ss;
+ if (cur_cs != 0) {
+ errhlp = errhlp_cs;
+ cur_cs = 0;
+ }
+ ss = cmd_chr_to_string(if_test_cmd, cur_if);
+ snprintf(errmsg, 255,
+ "Incomplete %s; all text was ignored after line %d",
+ ss, (int) skip_line);
+ free(ss);
+ /* Incomplete \\if... */
+ cur_tok = cs_token_flag + frozen_fi;
+ /* back up one inserted token and call |error| */
+ {
+ OK_to_interrupt = false;
+ back_input();
+ token_type = inserted;
+ OK_to_interrupt = true;
+ tex_error(errmsg, errhlp);
+ }
+ }
+ deletions_allowed = true;
+ }
+}
+
+@ @c
+static boolean get_next_file(void)
+{
+ SWITCH:
+ if (iloc <= ilimit) { /* current line not yet finished */
+ do_buffer_to_unichar(cur_chr, iloc);
+
+ RESWITCH:
+ if (detokenized_line()) {
+ cur_cmd = (cur_chr == ' ' ? 10 : 12);
+ } else {
+ do_get_cat_code(cur_cmd);
+ }
+ /*
+ Change state if necessary, and |goto switch| if the current
+ character should be ignored, or |goto reswitch| if the current
+ character changes to another;
+ */
+ /* The following 48-way switch accomplishes the scanning quickly, assuming
+ that a decent C compiler has translated the code. Note that the numeric
+ values for |mid_line|, |skip_blanks|, and |new_line| are spaced
+ apart from each other by |max_char_code+1|, so we can add a character's
+ command code to the state to get a single number that characterizes both.
+ */
+ switch (istate + cur_cmd) {
+ case mid_line + ignore_cmd:
+ case skip_blanks + ignore_cmd:
+ case new_line + ignore_cmd:
+ case skip_blanks + spacer_cmd:
+ case new_line + spacer_cmd: /* Cases where character is ignored */
+ goto SWITCH;
+ break;
+ case mid_line + escape_cmd:
+ case new_line + escape_cmd:
+ case skip_blanks + escape_cmd: /* Scan a control sequence ...; */
+ istate = (unsigned char) scan_control_sequence();
+ if (cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ break;
+ case mid_line + active_char_cmd:
+ case new_line + active_char_cmd:
+ case skip_blanks + active_char_cmd: /* Process an active-character */
+ cur_cs = active_to_cs(cur_chr, false);
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ istate = mid_line;
+ if (cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ break;
+ case mid_line + sup_mark_cmd:
+ case new_line + sup_mark_cmd:
+ case skip_blanks + sup_mark_cmd: /* If this |sup_mark| starts */
+ if (process_sup_mark())
+ goto RESWITCH;
+ else
+ istate = mid_line;
+ break;
+ case mid_line + invalid_char_cmd:
+ case new_line + invalid_char_cmd:
+ case skip_blanks + invalid_char_cmd: /* Decry the invalid character and |goto restart|; */
+ invalid_character_error();
+ return false; /* because state may be |token_list| now */
+ break;
+ case mid_line + spacer_cmd: /* Enter |skip_blanks| state, emit a space; */
+ istate = skip_blanks;
+ cur_chr = ' ';
+ break;
+ case mid_line + car_ret_cmd: /* Finish line, emit a space; */
+ /* When a character of type |spacer| gets through, its character code is
+ changed to $\.{"\ "}=040$. This means that the ASCII codes for tab and space,
+ and for the space inserted at the end of a line, will
+ be treated alike when macro parameters are being matched. We do this
+ since such characters are indistinguishable on most computer terminal displays.
+ */
+ iloc = ilimit + 1;
+ cur_cmd = spacer_cmd;
+ cur_chr = ' ';
+ break;
+ case skip_blanks + car_ret_cmd:
+ case mid_line + comment_cmd:
+ case new_line + comment_cmd:
+ case skip_blanks + comment_cmd: /* Finish line, |goto switch|; */
+ iloc = ilimit + 1;
+ goto SWITCH;
+ break;
+ case new_line + car_ret_cmd: /* Finish line, emit a \.{\\par}; */
+ iloc = ilimit + 1;
+ cur_cs = par_loc;
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ if (cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ break;
+ case skip_blanks + left_brace_cmd:
+ case new_line + left_brace_cmd:
+ istate = mid_line; /* fall through */
+ case mid_line + left_brace_cmd:
+ align_state++;
+ break;
+ case skip_blanks + right_brace_cmd:
+ case new_line + right_brace_cmd:
+ istate = mid_line; /* fall through */
+ case mid_line + right_brace_cmd:
+ align_state--;
+ break;
+ case mid_line + math_shift_cmd:
+ case mid_line + tab_mark_cmd:
+ case mid_line + mac_param_cmd:
+ case mid_line + sub_mark_cmd:
+ case mid_line + letter_cmd:
+ case mid_line + other_char_cmd:
+ break;
+#if 0
+ case skip_blanks + math_shift:
+ case skip_blanks + tab_mark:
+ case skip_blanks + mac_param:
+ case skip_blanks + sub_mark:
+ case skip_blanks + letter:
+ case skip_blanks + other_char:
+ case new_line + math_shift:
+ case new_line + tab_mark:
+ case new_line + mac_param:
+ case new_line + sub_mark:
+ case new_line + letter:
+ case new_line + other_char:
+#else
+ default:
+#endif
+ istate = mid_line;
+ break;
+ }
+ } else {
+ if (current_ocp_lstack > 0) {
+ pop_input();
+ return false;
+ }
+ if (iname != 21)
+ istate = new_line;
+
+ /*
+ Move to next line of file,
+ or |goto restart| if there is no next line,
+ or |return| if a \.{\\read} line has finished;
+ */
+ do {
+ next_line_retval r = next_line();
+ if (r == next_line_return) {
+ return true;
+ } else if (r == next_line_restart) {
+ return false;
+ }
+ } while (0);
+ check_interrupt();
+ goto SWITCH;
+ }
+ return true;
+}
+
+@ @c
+#define is_hex(a) ((a>='0'&&a<='9')||(a>='a'&&a<='f'))
+
+#define add_nybble(a) do { \
+ if (a<='9') cur_chr=(cur_chr<<4)+a-'0'; \
+ else cur_chr=(cur_chr<<4)+a-'a'+10; \
+ } while (0)
+
+#define hex_to_cur_chr do { \
+ if (c<='9') cur_chr=c-'0'; \
+ else cur_chr=c-'a'+10; \
+ add_nybble(cc); \
+ } while (0)
+
+#define four_hex_to_cur_chr do { \
+ hex_to_cur_chr; \
+ add_nybble(ccc); add_nybble(cccc); \
+ } while (0)
+
+#define five_hex_to_cur_chr do { \
+ four_hex_to_cur_chr; \
+ add_nybble(ccccc); \
+ } while (0)
+
+#define six_hex_to_cur_chr do { \
+ five_hex_to_cur_chr; \
+ add_nybble(cccccc); \
+ } while (0)
+
+
+@ Notice that a code like \.{\^\^8} becomes \.x if not followed by a hex digit.
+
+@c
+static boolean process_sup_mark(void)
+{
+ if (cur_chr == buffer[iloc]) {
+ int c, cc;
+ if (iloc < ilimit) {
+ if ((cur_chr == buffer[iloc + 1]) && (cur_chr == buffer[iloc + 2])
+ && (cur_chr == buffer[iloc + 3])
+ && (cur_chr == buffer[iloc + 4])
+ && ((iloc + 10) <= ilimit)) {
+ int ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */
+ c = buffer[iloc + 5];
+ cc = buffer[iloc + 6];
+ ccc = buffer[iloc + 7];
+ cccc = buffer[iloc + 8];
+ ccccc = buffer[iloc + 9];
+ cccccc = buffer[iloc + 10];
+ if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
+ && (is_hex(cccc))
+ && (is_hex(ccccc)) && (is_hex(cccccc))) {
+ iloc = iloc + 11;
+ six_hex_to_cur_chr;
+ return true;
+ }
+ }
+ if ((cur_chr == buffer[iloc + 1]) && (cur_chr == buffer[iloc + 2])
+ && (cur_chr == buffer[iloc + 3]) && ((iloc + 8) <= ilimit)) {
+ int ccc, cccc, ccccc; /* constituents of a possible expanded code */
+ c = buffer[iloc + 4];
+ cc = buffer[iloc + 5];
+ ccc = buffer[iloc + 6];
+ cccc = buffer[iloc + 7];
+ ccccc = buffer[iloc + 8];
+ if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
+ && (is_hex(cccc)) && (is_hex(ccccc))) {
+ iloc = iloc + 9;
+ five_hex_to_cur_chr;
+ return true;
+ }
+ }
+ if ((cur_chr == buffer[iloc + 1]) && (cur_chr == buffer[iloc + 2])
+ && ((iloc + 6) <= ilimit)) {
+ int ccc, cccc; /* constituents of a possible expanded code */
+ c = buffer[iloc + 3];
+ cc = buffer[iloc + 4];
+ ccc = buffer[iloc + 5];
+ cccc = buffer[iloc + 6];
+ if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
+ && (is_hex(cccc))) {
+ iloc = iloc + 7;
+ four_hex_to_cur_chr;
+ return true;
+ }
+ }
+ c = buffer[iloc + 1];
+ if (c < 0200) { /* yes we have an expanded char */
+ iloc = iloc + 2;
+ if (is_hex(c) && iloc <= ilimit) {
+ cc = buffer[iloc];
+ if (is_hex(cc)) {
+ incr(iloc);
+ hex_to_cur_chr;
+ return true;
+ }
+ }
+ cur_chr = (c < 0100 ? c + 0100 : c - 0100);
+ return true;
+ }
+ }
+ }
+ return false;
+}
+
+@ Control sequence names are scanned only when they appear in some line of
+ a file; once they have been scanned the first time, their |eqtb| location
+ serves as a unique identification, so \TeX\ doesn't need to refer to the
+ original name any more except when it prints the equivalent in symbolic form.
+
+ The program that scans a control sequence has been written carefully
+ in order to avoid the blowups that might otherwise occur if a malicious
+ user tried something like `\.{\\catcode\'15=0}'. The algorithm might
+ look at |buffer[ilimit+1]|, but it never looks at |buffer[ilimit+2]|.
+
+ If expanded characters like `\.{\^\^A}' or `\.{\^\^df}'
+ appear in or just following
+ a control sequence name, they are converted to single characters in the
+ buffer and the process is repeated, slowly but surely.
+
+@c
+static boolean check_expanded_code(int *kk); /* below */
+
+static int scan_control_sequence(void)
+{
+ int retval = mid_line;
+ if (iloc > ilimit) {
+ cur_cs = null_cs; /* |state| is irrelevant in this case */
+ } else {
+ register int cat; /* |cat_code(cur_chr)|, usually */
+ while (1) {
+ int k = iloc;
+ do_buffer_to_unichar(cur_chr, k);
+ do_get_cat_code(cat);
+ if (cat != letter_cmd || k > ilimit) {
+ retval = (cat == spacer_cmd ? skip_blanks : mid_line);
+ if (cat == sup_mark_cmd && check_expanded_code(&k)) /* If an expanded...; */
+ continue;
+ } else {
+ retval = skip_blanks;
+ do {
+ do_buffer_to_unichar(cur_chr, k);
+ do_get_cat_code(cat);
+ } while (cat == letter_cmd && k <= ilimit);
+
+ if (cat == sup_mark_cmd && check_expanded_code(&k)) /* If an expanded...; */
+ continue;
+ if (cat != letter_cmd) {
+ decr(k);
+ if (cur_chr > 0xFFFF)
+ decr(k);
+ if (cur_chr > 0x7FF)
+ decr(k);
+ if (cur_chr > 0x7F)
+ decr(k);
+ } /* now |k| points to first nonletter */
+ }
+ cur_cs = id_lookup(iloc, k - iloc);
+ iloc = k;
+ break;
+ }
+ }
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ return retval;
+}
+
+@ Whenever we reach the following piece of code, we will have
+ |cur_chr=buffer[k-1]| and |k<=ilimit+1| and |cat=get_cat_code(cat_code_table,cur_chr)|. If an
+ expanded code like \.{\^\^A} or \.{\^\^df} appears in |buffer[(k-1)..(k+1)]|
+ or |buffer[(k-1)..(k+2)]|, we
+ will store the corresponding code in |buffer[k-1]| and shift the rest of
+ the buffer left two or three places.
+
+@c
+static boolean check_expanded_code(int *kk)
+{
+ int l;
+ int k = *kk;
+ int d = 1; /* number of excess characters in an expanded code */
+ int c, cc, ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */
+ if (buffer[k] == cur_chr && k < ilimit) {
+ if ((cur_chr == buffer[k + 1]) && (cur_chr == buffer[k + 2])
+ && ((k + 6) <= ilimit)) {
+ d = 4;
+ if ((cur_chr == buffer[k + 3]) && ((k + 8) <= ilimit))
+ d = 5;
+ if ((cur_chr == buffer[k + 4]) && ((k + 10) <= ilimit))
+ d = 6;
+ c = buffer[k + d - 1];
+ cc = buffer[k + d];
+ ccc = buffer[k + d + 1];
+ cccc = buffer[k + d + 2];
+ if (d == 6) {
+ ccccc = buffer[k + d + 3];
+ cccccc = buffer[k + d + 4];
+ if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
+ && is_hex(ccccc) && is_hex(cccccc))
+ six_hex_to_cur_chr;
+ } else if (d == 5) {
+ ccccc = buffer[k + d + 3];
+ if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
+ && is_hex(ccccc))
+ five_hex_to_cur_chr;
+ } else {
+ if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc))
+ four_hex_to_cur_chr;
+ }
+ } else {
+ c = buffer[k + 1];
+ if (c < 0200) {
+ d = 1;
+ if (is_hex(c) && (k + 2) <= ilimit) {
+ cc = buffer[k + 2];
+ if (is_hex(c) && is_hex(cc)) {
+ d = 2;
+ hex_to_cur_chr;
+ }
+ } else if (c < 0100) {
+ cur_chr = c + 0100;
+ } else {
+ cur_chr = c - 0100;
+ }
+ }
+ }
+ if (d > 2)
+ d = 2 * d - 1;
+ else
+ d++;
+ if (cur_chr <= 0x7F) {
+ buffer[k - 1] = (packed_ASCII_code) cur_chr;
+ } else if (cur_chr <= 0x7FF) {
+ buffer[k - 1] = (packed_ASCII_code) (0xC0 + cur_chr / 0x40);
+ k++;
+ d--;
+ buffer[k - 1] = (packed_ASCII_code) (0x80 + cur_chr % 0x40);
+ } else if (cur_chr <= 0xFFFF) {
+ buffer[k - 1] = (packed_ASCII_code) (0xE0 + cur_chr / 0x1000);
+ k++;
+ d--;
+ buffer[k - 1] =
+ (packed_ASCII_code) (0x80 + (cur_chr % 0x1000) / 0x40);
+ k++;
+ d--;
+ buffer[k - 1] =
+ (packed_ASCII_code) (0x80 + (cur_chr % 0x1000) % 0x40);
+ } else {
+ buffer[k - 1] = (packed_ASCII_code) (0xF0 + cur_chr / 0x40000);
+ k++;
+ d--;
+ buffer[k - 1] =
+ (packed_ASCII_code) (0x80 + (cur_chr % 0x40000) / 0x1000);
+ k++;
+ d--;
+ buffer[k - 1] =
+ (packed_ASCII_code) (0x80 +
+ ((cur_chr % 0x40000) % 0x1000) / 0x40);
+ k++;
+ d--;
+ buffer[k - 1] =
+ (packed_ASCII_code) (0x80 +
+ ((cur_chr % 0x40000) % 0x1000) % 0x40);
+ }
+ l = k;
+ ilimit = ilimit - d;
+ while (l <= ilimit) {
+ buffer[l] = buffer[l + d];
+ l++;
+ }
+ *kk = k;
+ return true;
+ }
+ return false;
+}
+
+@ todo: this is a function because it is still used from the converted pascal.
+ once that is gone, it can be a \#define again
+
+@c
+boolean end_line_char_inactive(void)
+{
+ return ((end_line_char < 0) || (end_line_char > 127));
+}
+
+@ All of the easy branches of |get_next| have now been taken care of.
+ There is one more branch.
+
+@c
+static next_line_retval next_line(void)
+{
+ boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */
+ if (iname > 17) {
+ /* Read next line of file into |buffer|, or |goto restart| if the file has ended */
+ incr(line);
+ first = istart;
+ if (!force_eof) {
+ if (iname <= 20) {
+ if (pseudo_input()) { /* not end of file */
+ firm_up_the_line(); /* this sets |ilimit| */
+ line_catcode_table = DEFAULT_CAT_TABLE;
+ if ((iname == 19) && (pseudo_lines(pseudo_files) == null))
+ inhibit_eol = true;
+ } else if ((every_eof != null) && !eof_seen[iindex]) {
+ ilimit = first - 1;
+ eof_seen[iindex] = true; /* fake one empty line */
+ if (iname != 19)
+ begin_token_list(every_eof, every_eof_text);
+ return next_line_restart;
+ } else {
+ force_eof = true;
+ }
+ } else {
+ if (iname == 21) {
+ if (luacstring_input()) { /* not end of strings */
+ firm_up_the_line();
+ line_catcode_table = (short) luacstring_cattable();
+ line_partial = (signed char) luacstring_partial();
+ if (luacstring_final_line() || line_partial
+ || line_catcode_table == NO_CAT_TABLE)
+ inhibit_eol = true;
+ if (!line_partial)
+ istate = new_line;
+ } else {
+ force_eof = true;
+ }
+ } else {
+ if (lua_input_ln(cur_file, 0, true)) { /* not end of file */
+ firm_up_the_line(); /* this sets |ilimit| */
+ line_catcode_table = DEFAULT_CAT_TABLE;
+ } else if ((every_eof != null) && (!eof_seen[iindex])) {
+ ilimit = first - 1;
+ eof_seen[iindex] = true; /* fake one empty line */
+ begin_token_list(every_eof, every_eof_text);
+ return next_line_restart;
+ } else {
+ force_eof = true;
+ }
+ }
+ }
+ }
+ if (force_eof) {
+ if (tracing_nesting > 0)
+ if ((grp_stack[in_open] != cur_boundary)
+ || (if_stack[in_open] != cond_ptr))
+ if (!((iname == 19) || (iname == 21)))
+ file_warning(); /* give warning for some unfinished groups and/or conditionals */
+ if ((iname > 21) || (iname == 20)) {
+ if (tracefilenames)
+ print_char(')');
+ open_parens--;
+#if 0
+ update_terminal(); /* show user that file has been read */
+#endif
+ }
+ force_eof = false;
+ if (iname == 21 || /* lua input */
+ iname == 19) { /* \.{\\scantextokens} */
+ end_file_reading();
+ } else {
+ end_file_reading();
+ check_outer_validity();
+ }
+ return next_line_restart;
+ }
+ if (inhibit_eol || end_line_char_inactive())
+ ilimit--;
+ else
+ buffer[ilimit] = (packed_ASCII_code) end_line_char;
+ first = ilimit + 1;
+ iloc = istart; /* ready to read */
+ } else {
+ if (!terminal_input) { /* \.{\\read} line has ended */
+ cur_cmd = 0;
+ cur_chr = 0;
+ return next_line_return; /* OUTER */
+ }
+ if (input_ptr > 0) { /* text was inserted during error recovery */
+ end_file_reading();
+ return next_line_restart; /* resume previous level */
+ }
+ if (selector < log_only)
+ open_log_file();
+ if (interaction > nonstop_mode) {
+ if (end_line_char_inactive())
+ ilimit++;
+ if (ilimit == istart) { /* previous line was empty */
+ tprint_nl("(Please type a command or say `\\end')");
+ }
+ print_ln();
+ first = istart;
+ prompt_input("*"); /* input on-line into |buffer| */
+ ilimit = last;
+ if (end_line_char_inactive())
+ ilimit--;
+ else
+ buffer[ilimit] = (packed_ASCII_code) end_line_char;
+ first = ilimit + 1;
+ iloc = istart;
+ } else {
+ fatal_error("*** (job aborted, no legal \\end found)");
+ /* nonstop mode, which is intended for overnight batch processing,
+ never waits for on-line input */
+ }
+ }
+ return next_line_ok;
+}
+
+@ Let's consider now what happens when |get_next| is looking at a token list.
+
+@c
+static boolean get_next_tokenlist(void)
+{
+ register halfword t; /* a token */
+ t = token_info(iloc);
+ iloc = token_link(iloc); /* move to next */
+ if (t >= cs_token_flag) { /* a control sequence token */
+ cur_cs = t - cs_token_flag;
+ cur_cmd = eq_type(cur_cs);
+ if (cur_cmd >= outer_call_cmd) {
+ if (cur_cmd == dont_expand_cmd) { /* Get the next token, suppressing expansion */
+ /* The present point in the program is reached only when the |expand|
+ routine has inserted a special marker into the input. In this special
+ case, |token_info(iloc)| is known to be a control sequence token, and |token_link(iloc)=null|.
+ */
+ cur_cs = token_info(iloc) - cs_token_flag;
+ iloc = null;
+ cur_cmd = eq_type(cur_cs);
+ if (cur_cmd > max_command_cmd) {
+ cur_cmd = relax_cmd;
+ cur_chr = no_expand_flag;
+ return true;
+ }
+ } else {
+ check_outer_validity();
+ }
+ }
+ cur_chr = equiv(cur_cs);
+ } else {
+ cur_cmd = token_cmd(t);
+ cur_chr = token_chr(t);
+ switch (cur_cmd) {
+ case left_brace_cmd:
+ align_state++;
+ break;
+ case right_brace_cmd:
+ align_state--;
+ break;
+ case out_param_cmd: /* Insert macro parameter and |goto restart|; */
+ begin_token_list(param_stack[param_start + cur_chr - 1], parameter);
+ return false;
+ break;
+ }
+ }
+ return true;
+}
+
+@ Now we're ready to take the plunge into |get_next| itself. Parts of
+ this routine are executed more often than any other instructions of \TeX.
+ @^mastication@>@^inner loop@>
+
+@ sets |cur_cmd|, |cur_chr|, |cur_cs| to next token
+
+@c
+void get_next(void)
+{
+ RESTART:
+ cur_cs = 0;
+ if (istate != token_list) {
+ /* Input from external file, |goto restart| if no input found */
+ if (!get_next_file())
+ goto RESTART;
+ } else {
+ if (iloc == null) {
+ end_token_list();
+ goto RESTART; /* list exhausted, resume previous level */
+ } else if (!get_next_tokenlist()) {
+ goto RESTART; /* parameter needs to be expanded */
+ }
+ }
+ /* If an alignment entry has just ended, take appropriate action */
+ if ((cur_cmd == tab_mark_cmd || cur_cmd == car_ret_cmd) && align_state == 0) {
+ insert_vj_template();
+ goto RESTART;
+ }
+}
+
+
+@ Since |get_next| is used so frequently in \TeX, it is convenient
+to define three related procedures that do a little more:
+
+\yskip\hang|get_token| not only sets |cur_cmd| and |cur_chr|, it
+also sets |cur_tok|, a packed halfword version of the current token.
+
+\yskip\hang|get_x_token|, meaning ``get an expanded token,'' is like
+|get_token|, but if the current token turns out to be a user-defined
+control sequence (i.e., a macro call), or a conditional,
+or something like \.{\\topmark} or \.{\\expandafter} or \.{\\csname},
+it is eliminated from the input by beginning the expansion of the macro
+or the evaluation of the conditional.
+
+\yskip\hang|x_token| is like |get_x_token| except that it assumes that
+|get_next| has already been called.
+
+\yskip\noindent
+In fact, these three procedures account for almost every use of |get_next|.
+
+No new control sequences will be defined except during a call of
+|get_token|, or when \.{\\csname} compresses a token list, because
+|no_new_control_sequence| is always |true| at other times.
+
+@c
+void get_token(void)
+{ /* sets |cur_cmd|, |cur_chr|, |cur_tok| */
+ no_new_control_sequence = false;
+ get_token_lua();
+ no_new_control_sequence = true;
+ if (cur_cs == 0)
+ cur_tok = token_val(cur_cmd, cur_chr);
+ else
+ cur_tok = cs_token_flag + cur_cs;
+}
+
+@ @c
+void get_token_lua(void)
+{
+ register int callback_id;
+ callback_id = callback_defined(token_filter_callback);
+ if (callback_id > 0) {
+ while (istate == token_list && iloc == null && iindex != v_template)
+ end_token_list();
+ /* there is some stuff we don't want to see inside the callback */
+ if (!(istate == token_list &&
+ ((nofilter == true) || (iindex == backed_up && iloc != null)))) {
+ do_get_token_lua(callback_id);
+ return;
+ }
+ }
+ get_next();
+}
+
+
+@ changes the string |s| to a token list
+@c
+halfword string_to_toks(char *ss)
+{
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
+ char *s = ss, *se = ss + strlen(s);
+ p = temp_token_head;
+ set_token_link(p, null);
+ while (s < se) {
+ t = (halfword) str2uni((unsigned char *) s);
+ s += utf8_size(t);
+ if (t == ' ')
+ t = space_token;
+ else
+ t = other_token + t;
+ fast_store_new_token(t);
+ }
+ return token_link(temp_token_head);
+}
+
+@ The token lists for macros and for other things like \.{\\mark} and \.{\\output}
+and \.{\\write} are produced by a procedure called |scan_toks|.
+
+Before we get into the details of |scan_toks|, let's consider a much
+simpler task, that of converting the current string into a token list.
+The |str_toks| function does this; it classifies spaces as type |spacer|
+and everything else as type |other_char|.
+
+The token list created by |str_toks| begins at |link(temp_token_head)| and ends
+at the value |p| that is returned. (If |p=temp_token_head|, the list is empty.)
+
+|lua_str_toks| is almost identical, but it also escapes the three
+symbols that |lua| considers special while scanning a literal string
+
+@c
+static halfword lua_str_toks(lstring b)
+{ /* changes the string |str_pool[b..pool_ptr]| to a token list */
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
+ unsigned char *k; /* index into string */
+ p = temp_token_head;
+ set_token_link(p, null);
+ k = (unsigned char *) b.s;
+ while (k < (unsigned char *) b.s + b.l) {
+ t = pool_to_unichar(k);
+ k += utf8_size(t);
+ if (t == ' ') {
+ t = space_token;
+ } else {
+ if ((t == '\\') || (t == '"') || (t == '\'') || (t == 10)
+ || (t == 13))
+ fast_store_new_token(other_token + '\\');
+ if (t == 10)
+ t = 'n';
+ if (t == 13)
+ t = 'r';
+ t = other_token + t;
+ }
+ fast_store_new_token(t);
+ }
+ return p;
+}
+
+
+@ Incidentally, the main reason for wanting |str_toks| is the function |the_toks|,
+which has similar input/output characteristics.
+
+@c
+halfword str_toks(lstring s)
+{ /* changes the string |str_pool[b..pool_ptr]| to a token list */
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
+ unsigned char *k, *l; /* index into string */
+ p = temp_token_head;
+ set_token_link(p, null);
+ k = s.s;
+ l = k + s.l;
+ while (k < l) {
+ t = pool_to_unichar(k);
+ k += utf8_size(t);
+ if (t == ' ')
+ t = space_token;
+ else
+ t = other_token + t;
+ fast_store_new_token(t);
+ }
+ return p;
+}
+
+@ Here's part of the |expand| subroutine that we are now ready to complete:
+@c
+void ins_the_toks(void)
+{
+ (void) the_toks();
+ ins_list(token_link(temp_token_head));
+}
+
+
+@ Here is a routine that print the result of a convert command, using
+ the argument |i|. It returns |false | if it does not know to print
+ the code |c|. The function exists because lua code and tex code can
+ both call it to convert something.
+
+@c
+static boolean print_convert_string(halfword c, int i)
+{
+ int ff; /* for use with |set_ff| */
+ boolean ret = true;
+ switch (c) {
+ case number_code:
+ print_int(i);
+ break;
+ case roman_numeral_code:
+ print_roman_int(i);
+ break;
+ case etex_code:
+ tprint(eTeX_version_string);
+ break;
+ case omega_code:
+ tprint(Omega_version_string);
+ break;
+ case aleph_code:
+ tprint(Aleph_version_string);
+ break;
+ case pdftex_revision_code:
+ tprint(pdftex_revision);
+ break;
+ case luatex_revision_code:
+ print(get_luatexrevision());
+ break;
+ case luatex_date_code:
+ print_int(get_luatex_date_info());
+ break;
+ case pdftex_banner_code:
+ tprint(pdftex_banner);
+ break;
+ case uniform_deviate_code:
+ print_int(unif_rand(i));
+ break;
+ case normal_deviate_code:
+ print_int(norm_rand());
+ break;
+ case format_name_code:
+ print(format_name);
+ break;
+ case job_name_code:
+ print(job_name);
+ break;
+ case font_name_code:
+ append_string((unsigned char *) font_name(i),
+ (unsigned) strlen(font_name(i)));
+ if (font_size(i) != font_dsize(i)) {
+ tprint(" at ");
+ print_scaled(font_size(i));
+ tprint("pt");
+ }
+ break;
+ case math_style_code:
+ print_math_style();
+ break;
+ case pdf_font_name_code:
+ case pdf_font_objnum_code:
+ set_ff(i);
+ if (c == pdf_font_name_code)
+ print_int(obj_info(static_pdf, pdf_font_num(ff)));
+ else
+ print_int(pdf_font_num(ff));
+ break;
+ case pdf_font_size_code:
+ print_scaled(font_size(i));
+ tprint("pt");
+ break;
+ case pdf_page_ref_code:
+ print_int(get_obj(static_pdf, obj_type_page, i, false));
+ break;
+ case pdf_xform_name_code:
+ print_int(obj_info(static_pdf, i));
+ break;
+ case Aleph_revision_code:
+ tprint(Aleph_revision);
+ break;
+ case Omega_revision_code:
+ tprint(Omega_revision);
+ break;
+ case eTeX_revision_code:
+ tprint(eTeX_revision);
+ break;
+ default:
+ ret = false;
+ break;
+ }
+ return ret;
+}
+
+@ @c
+int scan_lua_state(void)
+{
+ int sn = 0;
+ if (scan_keyword("name")) {
+ scan_pdf_ext_toks();
+ sn = def_ref;
+ }
+ /* Parse optional lua state integer, or an instance name to be stored in |sn| */
+ /* Get the next non-blank non-relax non-call token */
+ do {
+ get_x_token();
+ } while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd));
+
+ back_input(); /* have to push it back, whatever it is */
+ if (cur_cmd != left_brace_cmd) {
+ scan_register_num();
+ if (get_lua_name(cur_val))
+ sn = (cur_val - 65536);
+ }
+ return sn;
+}
+
+
+@ The procedure |conv_toks| uses |str_toks| to insert the token list
+for |convert| functions into the scanner; `\.{\\outer}' control sequences
+are allowed to follow `\.{\\string}' and `\.{\\meaning}'.
+
+The extra temp string |u| is needed because |pdf_scan_ext_toks| incorporates
+any pending string in its output. In order to save such a pending string,
+we have to create a temporary string that is destroyed immediately after.
+
+@c
+void conv_toks(void)
+{
+ int old_setting; /* holds |selector| setting */
+ halfword p, q;
+ int save_scanner_status; /* |scanner_status| upon entry */
+ halfword save_def_ref; /* |def_ref| upon entry, important if inside `\.{\\message}' */
+ halfword save_warning_index;
+ boolean bool; /* temp boolean */
+ str_number s; /* first temp string */
+ int sn; /* lua chunk name */
+ str_number u = 0; /* third temp string, will become non-nil if a string is already being built */
+ int i = 0; /* first temp integer */
+ int j = 0; /* second temp integer */
+ int c = cur_chr; /* desired type of conversion */
+ str_number str;
+ /* Scan the argument for command |c| */
+ switch (c) {
+ case number_code:
+ case roman_numeral_code:
+ scan_int();
+ break;
+ case string_code:
+ case meaning_code:
+ save_scanner_status = scanner_status;
+ scanner_status = normal;
+ get_token();
+ scanner_status = save_scanner_status;
+ break;
+ case etex_code:
+ case omega_code:
+ case aleph_code:
+ break;
+ case font_name_code:
+ scan_font_ident();
+ break;
+ case pdftex_revision_code:
+ case luatex_revision_code:
+ case luatex_date_code:
+ case pdftex_banner_code:
+ break;
+ case pdf_font_name_code:
+ case pdf_font_objnum_code:
+ case pdf_font_size_code:
+ scan_font_ident();
+ if (cur_val == null_font)
+ pdf_error("font", "invalid font identifier");
+ if (c != pdf_font_size_code) {
+ pdf_check_vf(cur_val);
+ if (!font_used(cur_val))
+ pdf_init_font(static_pdf, cur_val);
+ }
+ break;
+ case pdf_page_ref_code:
+ scan_int();
+ if (cur_val <= 0)
+ pdf_error("pageref", "invalid page number");
+ break;
+ case left_margin_kern_code:
+ case right_margin_kern_code:
+ scan_int();
+ if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
+ pdf_error("marginkern", "a non-empty hbox expected");
+ break;
+ case pdf_xform_name_code:
+ scan_int();
+ check_obj_exists(static_pdf, obj_type_xform, cur_val);
+ break;
+ case pdf_creation_date_code:
+ ins_list(string_to_toks(getcreationdate(static_pdf)));
+ return;
+ break;
+ case format_name_code:
+ case job_name_code:
+ if (job_name == 0)
+ open_log_file();
+ break;
+ case pdf_colorstack_init_code:
+ bool = scan_keyword("page");
+ if (scan_keyword("direct"))
+ cur_val = direct_always;
+ else if (scan_keyword("page"))
+ cur_val = direct_page;
+ else
+ cur_val = set_origin;
+ save_scanner_status = scanner_status;
+ save_warning_index = warning_index;
+ save_def_ref = def_ref;
+ u = save_cur_string();
+ scan_pdf_ext_toks();
+ s = tokens_to_string(def_ref);
+ delete_token_ref(def_ref);
+ def_ref = save_def_ref;
+ warning_index = save_warning_index;
+ scanner_status = save_scanner_status;
+ cur_val = newcolorstack(s, cur_val, bool);
+ flush_str(s);
+ cur_val_level = int_val_level;
+ if (cur_val < 0) {
+ print_err("Too many color stacks");
+ help2("The number of color stacks is limited to 32768.",
+ "I'll use the default color stack 0 here.");
+ error();
+ cur_val = 0;
+ restore_cur_string(u);
+ }
+ break;
+ case uniform_deviate_code:
+ scan_int();
+ break;
+ case normal_deviate_code:
+ break;
+ case lua_escape_string_code:
+ {
+ lstring str;
+ int l = 0;
+ save_scanner_status = scanner_status;
+ save_def_ref = def_ref;
+ save_warning_index = warning_index;
+ scan_pdf_ext_toks();
+ bool = in_lua_escape;
+ in_lua_escape = true;
+ str.s = (unsigned char *) tokenlist_to_cstring(def_ref, false, &l);
+ str.l = (unsigned) l;
+ in_lua_escape = bool;
+ delete_token_ref(def_ref);
+ def_ref = save_def_ref;
+ warning_index = save_warning_index;
+ scanner_status = save_scanner_status;
+ (void) lua_str_toks(str);
+ ins_list(token_link(temp_token_head));
+ free(str.s);
+ return;
+ }
+ break;
+ case math_style_code:
+ break;
+ case expanded_code:
+ save_scanner_status = scanner_status;
+ save_warning_index = warning_index;
+ save_def_ref = def_ref;
+ u = save_cur_string();
+ scan_pdf_ext_toks();
+ warning_index = save_warning_index;
+ scanner_status = save_scanner_status;
+ ins_list(token_link(def_ref));
+ def_ref = save_def_ref;
+ restore_cur_string(u);
+ return;
+ break;
+ case lua_code:
+ u = save_cur_string();
+ save_scanner_status = scanner_status;
+ save_def_ref = def_ref;
+ save_warning_index = warning_index;
+ sn = scan_lua_state();
+ scan_pdf_ext_toks();
+ s = def_ref;
+ warning_index = save_warning_index;
+ def_ref = save_def_ref;
+ scanner_status = save_scanner_status;
+ luacstrings = 0;
+ luatokencall(s, sn);
+ delete_token_ref(s);
+ restore_cur_string(u); /* TODO: check this, was different */
+ if (luacstrings > 0)
+ lua_string_start();
+ return;
+ break;
+ case pdf_insert_ht_code:
+ scan_register_num();
+ break;
+ case pdf_ximage_bbox_code:
+ scan_int();
+ check_obj_exists(static_pdf, obj_type_ximage, cur_val);
+ i = obj_data_ptr(static_pdf, cur_val);
+ scan_int();
+ j = cur_val;
+ if ((j < 1) || (j > 4))
+ pdf_error("pdfximagebbox", "invalid parameter");
+ break;
+ /* Cases of 'Scan the argument for command |c|' */
+ case Aleph_revision_code:
+ case Omega_revision_code:
+ case eTeX_revision_code:
+ break;
+ default:
+ confusion("convert");
+ break;
+ }
+
+ old_setting = selector;
+ selector = new_string;
+
+ /* Print the result of command |c| */
+ if (!print_convert_string(c, cur_val)) {
+ switch (c) {
+ case string_code:
+ if (cur_cs != 0)
+ sprint_cs(cur_cs);
+ else
+ print(cur_chr);
+ break;
+ case meaning_code:
+ print_meaning();
+ break;
+ case left_margin_kern_code:
+ p = list_ptr(box(cur_val));
+ if ((p != null) && (!is_char_node(p)) &&
+ (type(p) == glue_node) && (subtype(p) == left_skip_code + 1))
+ p = vlink(p);
+ if ((p != null) && (!is_char_node(p)) &&
+ (type(p) == margin_kern_node) && (subtype(p) == left_side))
+ print_scaled(width(p));
+ else
+ print_char('0');
+ tprint("pt");
+ break;
+ case right_margin_kern_code:
+ q = list_ptr(box(cur_val));
+ p = null;
+ if (q != null) {
+ p = prev_rightmost(q, null);
+ if ((p != null) && (!is_char_node(p)) && (type(p) == glue_node)
+ && (subtype(p) == right_skip_code + 1))
+ p = prev_rightmost(q, p);
+ }
+ if ((p != null) && (!is_char_node(p)) &&
+ (type(p) == margin_kern_node) && (subtype(p) == right_side))
+ print_scaled(width(p));
+ else
+ print_char('0');
+ tprint("pt");
+ break;
+ case pdf_colorstack_init_code:
+ print_int(cur_val);
+ break;
+ case pdf_insert_ht_code:
+ i = cur_val;
+ p = page_ins_head;
+ while (i >= subtype(vlink(p)))
+ p = vlink(p);
+ if (subtype(p) == i)
+ print_scaled(height(p));
+ else
+ print_char('0');
+ tprint("pt");
+ break;
+ case pdf_ximage_bbox_code:
+ if (is_pdf_image(i)) {
+ switch (j) {
+ case 1:
+ print_scaled(epdf_orig_x(i));
+ break;
+ case 2:
+ print_scaled(epdf_orig_y(i));
+ break;
+ case 3:
+ print_scaled(epdf_orig_x(i) + epdf_xsize(i));
+ break;
+ case 4:
+ print_scaled(epdf_orig_y(i) + epdf_ysize(i));
+ break;
+ }
+ } else {
+ print_scaled(0);
+ }
+ tprint("pt");
+ break;
+ case pdf_creation_date_code:
+ case lua_escape_string_code:
+ case lua_code:
+ case expanded_code:
+ break;
+ default:
+ confusion("convert");
+ break;
+ }
+ }
+
+ selector = old_setting;
+ str = make_string();
+ (void) str_toks(str_lstring(str));
+ flush_str(str);
+ ins_list(token_link(temp_token_head));
+}
+
+@ This boolean is keeping track of the lua string escape state
+@c
+boolean in_lua_escape;
+
+@ probably not needed anymore
+@c
+boolean is_convert(halfword c)
+{
+ return (c == convert_cmd);
+}
+
+str_number the_convert_string(halfword c, int i)
+{
+ int old_setting; /* saved |selector| setting */
+ str_number ret = 0;
+ old_setting = selector;
+ selector = new_string;
+ if (print_convert_string(c, i)) {
+ ret = make_string();
+ } else if (c == font_identifier_code) {
+ print_font_identifier(i);
+ ret = make_string();
+ }
+ selector = old_setting;
+ return ret;
+}
+
+@ Another way to create a token list is via the \.{\\read} command. The
+sixteen files potentially usable for reading appear in the following
+global variables. The value of |read_open[n]| will be |closed| if
+stream number |n| has not been opened or if it has been fully read;
+|just_open| if an \.{\\openin} but not a \.{\\read} has been done;
+and |normal| if it is open and ready to read the next line.
+
+@c
+FILE *read_file[16]; /* used for \.{\\read} */
+int read_open[17]; /* state of |read_file[n]| */
+
+void initialize_read(void)
+{
+ int k;
+ for (k = 0; k <= 16; k++)
+ read_open[k] = closed;
+}
+
+@ The |read_toks| procedure constructs a token list like that for any
+macro definition, and makes |cur_val| point to it. Parameter |r| points
+to the control sequence that will receive this token list.
+
+@c
+void read_toks(int n, halfword r, halfword j)
+{
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ int s; /* saved value of |align_state| */
+ int m; /* stream number */
+ scanner_status = defining;
+ warning_index = r;
+ p = get_avail();
+ def_ref = p;
+ set_token_ref_count(def_ref, 0);
+ p = def_ref; /* the reference count */
+ store_new_token(end_match_token);
+ if ((n < 0) || (n > 15))
+ m = 16;
+ else
+ m = n;
+ s = align_state;
+ align_state = 1000000; /* disable tab marks, etc. */
+ do {
+ /* Input and store tokens from the next line of the file */
+ begin_file_reading();
+ iname = m + 1;
+ if (read_open[m] == closed) {
+ /* Input for \.{\\read} from the terminal */
+ /* Here we input on-line into the |buffer| array, prompting the user explicitly
+ if |n>=0|. The value of |n| is set negative so that additional prompts
+ will not be given in the case of multi-line input. */
+ if (interaction > nonstop_mode) {
+ if (n < 0) {
+ prompt_input("");
+ } else {
+ wake_up_terminal();
+ print_ln();
+ sprint_cs(r);
+ prompt_input(" =");
+ n = -1;
+ }
+ } else {
+ fatal_error
+ ("*** (cannot \\read from terminal in nonstop modes)");
+ }
+
+ } else if (read_open[m] == just_open) {
+ /* Input the first line of |read_file[m]| */
+ /* The first line of a file must be treated specially, since |lua_input_ln|
+ must be told not to start with |get|. */
+ if (lua_input_ln(read_file[m], (m + 1), false)) {
+ read_open[m] = normal;
+ } else {
+ lua_a_close_in(read_file[m], (m + 1));
+ read_open[m] = closed;
+ }
+
+ } else {
+ /* Input the next line of |read_file[m]| */
+ /* An empty line is appended at the end of a |read_file|. */
+ if (!lua_input_ln(read_file[m], (m + 1), true)) {
+ lua_a_close_in(read_file[m], (m + 1));
+ read_open[m] = closed;
+ if (align_state != 1000000) {
+ runaway();
+ print_err("File ended within \\read");
+ help1("This \\read has unbalanced braces.");
+ align_state = 1000000;
+ error();
+ }
+ }
+
+ }
+ ilimit = last;
+ if (end_line_char_inactive())
+ decr(ilimit);
+ else
+ buffer[ilimit] = (packed_ASCII_code) int_par(end_line_char_code);
+ first = ilimit + 1;
+ iloc = istart;
+ istate = new_line;
+ /* Handle \.{\\readline} and |goto done|; */
+ if (j == 1) {
+ while (iloc <= ilimit) { /* current line not yet finished */
+ cur_chr = buffer[iloc];
+ incr(iloc);
+ if (cur_chr == ' ')
+ cur_tok = space_token;
+ else
+ cur_tok = cur_chr + other_token;
+ store_new_token(cur_tok);
+ }
+ } else {
+ while (1) {
+ get_token();
+ if (cur_tok == 0)
+ break; /* |cur_cmd=cur_chr=0| will occur at the end of the line */
+ if (align_state < 1000000) { /* unmatched `\.\}' aborts the line */
+ do {
+ get_token();
+ } while (cur_tok != 0);
+ align_state = 1000000;
+ break;
+ }
+ store_new_token(cur_tok);
+ }
+ }
+ end_file_reading();
+
+ } while (align_state != 1000000);
+ cur_val = def_ref;
+ scanner_status = normal;
+ align_state = s;
+}
+
+@ @c
+str_number tokens_to_string(halfword p)
+{ /* return a string from tokens list */
+ int old_setting;
+ if (selector == new_string)
+ pdf_error("tokens",
+ "tokens_to_string() called while selector = new_string");
+ old_setting = selector;
+ selector = new_string;
+ show_token_list(token_link(p), null, -1);
+ selector = old_setting;
+ return make_string();
+}
+
+@ @c
+#define make_room(a) \
+ if ((unsigned)i+a+1>alloci) { \
+ ret = xrealloc(ret,(alloci+64)); \
+ alloci = alloci + 64; \
+ }
+
+
+#define append_i_byte(a) ret[i++] = (char)(a)
+
+#define Print_char(a) make_room(1); append_i_byte(a)
+
+#define Print_uchar(s) { \
+ make_room(4); \
+ if (s<=0x7F) { \
+ append_i_byte(s); \
+ } else if (s<=0x7FF) { \
+ append_i_byte(0xC0 + (s / 0x40)); \
+ append_i_byte(0x80 + (s % 0x40)); \
+ } else if (s<=0xFFFF) { \
+ append_i_byte(0xE0 + (s / 0x1000)); \
+ append_i_byte(0x80 + ((s % 0x1000) / 0x40)); \
+ append_i_byte(0x80 + ((s % 0x1000) % 0x40)); \
+ } else if (s>=0x110000) { \
+ append_i_byte(s-0x11000); \
+ } else { \
+ append_i_byte(0xF0 + (s / 0x40000)); \
+ append_i_byte(0x80 + ((s % 0x40000) / 0x1000)); \
+ append_i_byte(0x80 + (((s % 0x40000) % 0x1000) / 0x40)); \
+ append_i_byte(0x80 + (((s % 0x40000) % 0x1000) % 0x40)); \
+ } }
+
+
+#define Print_esc(b) { \
+ const char *v = b; \
+ if (e>0 && e<STRING_OFFSET) { \
+ Print_uchar (e); \
+ } \
+ make_room(strlen(v)); \
+ while (*v) { append_i_byte(*v); v++; } \
+ }
+
+#define is_cat_letter(a) \
+ (get_char_cat_code(pool_to_unichar(str_string((a)))) == 11)
+
+@ the actual token conversion in this function is now functionally
+ equivalent to |show_token_list|, except that it always prints the
+ whole token list.
+ TODO: check whether this causes problems in the lua library.
+
+@c
+char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
+{
+ register int p, c, m;
+ int q;
+ int infop;
+ char *s, *sh;
+ int e = 0;
+ char *ret;
+ int match_chr = '#';
+ int n = '0';
+ unsigned alloci = 1024;
+ int i = 0;
+ p = pp;
+ if (p == null) {
+ if (siz != NULL)
+ *siz = 0;
+ return NULL;
+ }
+ ret = xmalloc(alloci);
+ p = token_link(p); /* skip refcount */
+ if (p != null) {
+ e = int_par(escape_char_code);
+ }
+ while (p != null) {
+ if (p < (int) fix_mem_min || p > (int) fix_mem_end) {
+ Print_esc("CLOBBERED.");
+ break;
+ }
+ infop = token_info(p);
+ if (infop >= cs_token_flag) {
+ if (!(inhibit_par && infop == par_token)) {
+ q = infop - cs_token_flag;
+ if (q < hash_base) {
+ if (q == null_cs) {
+ Print_esc("csname");
+ Print_esc("endcsname");
+ } else {
+ Print_esc("IMPOSSIBLE.");
+ }
+ } else if ((q >= undefined_control_sequence)
+ && ((q <= eqtb_size)
+ || (q > eqtb_size + hash_extra))) {
+ Print_esc("IMPOSSIBLE.");
+ } else if ((cs_text(q) < 0) || (cs_text(q) >= str_ptr)) {
+ Print_esc("NONEXISTENT.");
+ } else {
+ str_number txt = cs_text(q);
+ sh = makecstring(txt);
+ s = sh;
+ if (is_active_cs(txt)) {
+ s = s + 3;
+ while (*s) {
+ Print_char(*s);
+ s++;
+ }
+ } else {
+ Print_uchar(e);
+ while (*s) {
+ Print_char(*s);
+ s++;
+ }
+ if ((!single_letter(txt)) || is_cat_letter(txt)) {
+ Print_char(' ');
+ }
+ }
+ free(sh);
+ }
+ }
+ } else {
+ if (infop < 0) {
+ Print_esc("BAD.");
+ } else {
+ m = token_cmd(infop);
+ c = token_chr(infop);
+ switch (m) {
+ case left_brace_cmd:
+ case right_brace_cmd:
+ case math_shift_cmd:
+ case tab_mark_cmd:
+ case sup_mark_cmd:
+ case sub_mark_cmd:
+ case spacer_cmd:
+ case letter_cmd:
+ case other_char_cmd:
+ Print_uchar(c);
+ break;
+ case mac_param_cmd:
+ if (!in_lua_escape)
+ Print_uchar(c);
+ Print_uchar(c);
+ break;
+ case out_param_cmd:
+ Print_uchar(match_chr);
+ if (c <= 9) {
+ Print_char(c + '0');
+ } else {
+ Print_char('!');
+ goto EXIT;
+ }
+ break;
+ case match_cmd:
+ match_chr = c;
+ Print_uchar(c);
+ n++;
+ Print_char(n);
+ if (n > '9')
+ goto EXIT;
+ break;
+ case end_match_cmd:
+ if (c == 0) {
+ Print_char('-');
+ Print_char('>');
+ }
+ break;
+ default:
+ Print_esc("BAD.");
+ break;
+ }
+ }
+ }
+ p = token_link(p);
+ }
+ EXIT:
+ ret[i] = '\0';
+ if (siz != NULL)
+ *siz = i;
+ return ret;
+}
+
+@ @c
+lstring *tokenlist_to_lstring(int pp, int inhibit_par)
+{
+ int siz;
+ lstring *ret = xmalloc(sizeof(lstring));
+ ret->s = (unsigned char *) tokenlist_to_cstring(pp, inhibit_par, &siz);
+ ret->l = (size_t) siz;
+ return ret;
+}
+
+@ @c
+void free_lstring(lstring * ls)
+{
+ if (ls == NULL)
+ return;
+ if (ls->s != NULL)
+ free(ls->s);
+ free(ls);
+}