summaryrefslogtreecommitdiff
path: root/Build/source/texk/web2c/luatexdir/tex/textoken.w
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/tex/textoken.w')
-rw-r--r--Build/source/texk/web2c/luatexdir/tex/textoken.w2614
1 files changed, 1579 insertions, 1035 deletions
diff --git a/Build/source/texk/web2c/luatexdir/tex/textoken.w b/Build/source/texk/web2c/luatexdir/tex/textoken.w
index 43445721435..0c11bb0f290 100644
--- a/Build/source/texk/web2c/luatexdir/tex/textoken.w
+++ b/Build/source/texk/web2c/luatexdir/tex/textoken.w
@@ -31,6 +31,7 @@
#define every_eof equiv(every_eof_loc)
#define box(A) equiv(box_base+(A))
+#define toks(A) equiv(toks_base+(A))
#define detokenized_line() (line_catcode_table==NO_CAT_TABLE)
@@ -42,19 +43,18 @@
} while (0)
-@ The \TeX\ system does nearly all of its own memory allocation, so that it
-can readily be transported into environments that do not have automatic
-facilities for strings, garbage collection, etc., and so that it can be in
-control of what error messages the user receives. The dynamic storage
-requirements of \TeX\ are handled by providing two large arrays called
-|fixmem| and |varmem| in which consecutive blocks of words are used as
-nodes by the \TeX\ routines.
+@ The \TeX\ system does nearly all of its own memory allocation, so that it can
+readily be transported into environments that do not have automatic facilities
+for strings, garbage collection, etc., and so that it can be in control of what
+error messages the user receives. The dynamic storage requirements of \TeX\ are
+handled by providing two large arrays called |fixmem| and |varmem| in which
+consecutive blocks of words are used as nodes by the \TeX\ routines.
-Pointer variables are indices into this array, or into another array
-called |eqtb| that will be explained later. A pointer variable might
-also be a special flag that lies outside the bounds of |mem|, so we
-allow pointers to assume any |halfword| value. The minimum halfword
-value represents a null pointer. \TeX\ does not assume that |mem[null]| exists.
+Pointer variables are indices into this array, or into another array called
+|eqtb| that will be explained later. A pointer variable might also be a special
+flag that lies outside the bounds of |mem|, so we allow pointers to assume any
+|halfword| value. The minimum halfword value represents a null pointer. \TeX\
+does not assume that |mem[null]| exists.
@ Locations in |fixmem| are used for storing one-word records; a conventional
\.{AVAIL} stack is used for allocation in this array.
@@ -64,11 +64,11 @@ smemory_word *fixmem; /* the big dynamic storage area */
unsigned fix_mem_min; /* the smallest location of one-word memory in use */
unsigned fix_mem_max; /* the largest location of one-word memory in use */
-@ In order to study the memory requirements of particular applications, it
-is possible to prepare a version of \TeX\ that keeps track of current and
-maximum memory usage. When code between the delimiters |@!stat| $\ldots$
-|tats| is not ``commented out,'' \TeX\ will run a bit slower but it will
-report these statistics when |tracing_stats| is sufficiently large.
+@ In order to study the memory requirements of particular applications, it is
+possible to prepare a version of \TeX\ that keeps track of current and maximum
+memory usage. When code between the delimiters |@!stat| $\ldots$ |tats| is not
+commented out, \TeX\ will run a bit slower but it will report these statistics
+when |tracing_stats| is sufficiently large.
@c
int var_used, dyn_used; /* how much memory is in use */
@@ -110,14 +110,14 @@ void initialize_tokens(void)
dyn_used = 0; /* initialize statistics */
}
-@ The function |get_avail| returns a pointer to a new one-word node whose
-|link| field is null. However, \TeX\ will halt if there is no more room left.
+@ The function |get_avail| returns a pointer to a new one-word node whose |link|
+field is null. However, \TeX\ will halt if there is no more room left.
@^inner loop@>
-If the available-space list is empty, i.e., if |avail=null|,
-we try first to increase |fix_mem_end|. If that cannot be done, i.e., if
-|fix_mem_end=fix_mem_max|, we try to reallocate array |fixmem|.
-If, that doesn't work, we have to quit.
+If the available-space list is empty, i.e., if |avail=null|, we try first to
+increase |fix_mem_end|. If that cannot be done, i.e., if
+|fix_mem_end=fix_mem_max|, we try to reallocate array |fixmem|. If, that doesn't
+work, we have to quit.
@c
halfword get_avail(void)
@@ -151,8 +151,8 @@ halfword get_avail(void)
return (halfword) p;
}
-@ The procedure |flush_list(p)| frees an entire linked list of
-one-word nodes that starts at position |p|.
+@ The procedure |flush_list(p)| frees an entire linked list of one-word nodes
+that starts at position |p|.
@^inner loop@>
@c
@@ -171,46 +171,45 @@ void flush_list(halfword p)
}
}
-@ A \TeX\ token is either a character or a control sequence, and it is
-@^token@>
-represented internally in one of two ways: (1)~A character whose ASCII
-code number is |c| and whose command code is |m| is represented as the
-number $2^{21}m+c$; the command code is in the range |1<=m<=14|. (2)~A control
-sequence whose |eqtb| address is |p| is represented as the number
-|cs_token_flag+p|. Here |cs_token_flag=@t$2^{25}-1$@>| is larger than
-$2^{21}m+c$, yet it is small enough that |cs_token_flag+p< max_halfword|;
-thus, a token fits comfortably in a halfword.
+@ A \TeX\ token is either a character or a control sequence, and it is @^token@>
+represented internally in one of two ways: (1)~A character whose ASCII code
+number is |c| and whose command code is |m| is represented as the number
+$2^{21}m+c$; the command code is in the range |1<=m<=14|. (2)~A control sequence
+whose |eqtb| address is |p| is represented as the number |cs_token_flag+p|. Here
+|cs_token_flag=@t$2^{25}-1$@>| is larger than $2^{21}m+c$, yet it is small enough
+that |cs_token_flag+p< max_halfword|; thus, a token fits comfortably in a
+halfword.
A token |t| represents a |left_brace| command if and only if
-|t<left_brace_limit|; it represents a |right_brace| command if and only if
-we have |left_brace_limit<=t<right_brace_limit|; and it represents a |match| or
-|end_match| command if and only if |match_token<=t<=end_match_token|.
-The following definitions take care of these token-oriented constants
-and a few others.
-
-@ A token list is a singly linked list of one-word nodes in |mem|, where
-each word contains a token and a link. Macro definitions, output-routine
-definitions, marks, \.{\\write} texts, and a few other things
-are remembered by \TeX\ in the form
-of token lists, usually preceded by a node with a reference count in its
-|token_ref_count| field. The token stored in location |p| is called
-|info(p)|.
-
-Three special commands appear in the token lists of macro definitions.
-When |m=match|, it means that \TeX\ should scan a parameter
-for the current macro; when |m=end_match|, it means that parameter
-matching should end and \TeX\ should start reading the macro text; and
-when |m=out_param|, it means that \TeX\ should insert parameter
-number |c| into the text at this point.
-
-The enclosing \.{\char'173} and \.{\char'175} characters of a macro
-definition are omitted, but the final right brace of an output routine
-is included at the end of its token list.
-
-Here is an example macro definition that illustrates these conventions.
-After \TeX\ processes the text
+|t<left_brace_limit|; it represents a |right_brace| command if and only if we
+have |left_brace_limit<=t<right_brace_limit|; and it represents a |match| or
+|end_match| command if and only if |match_token<=t<=end_match_token|. The
+following definitions take care of these token-oriented constants and a few
+others.
+
+@ A token list is a singly linked list of one-word nodes in |mem|, where each
+word contains a token and a link. Macro definitions, output-routine definitions,
+marks, \.{\\write} texts, and a few other things are remembered by \TeX\ in the
+form of token lists, usually preceded by a node with a reference count in its
+|token_ref_count| field. The token stored in location |p| is called |info(p)|.
+
+Three special commands appear in the token lists of macro definitions. When
+|m=match|, it means that \TeX\ should scan a parameter for the current macro;
+when |m=end_match|, it means that parameter matching should end and \TeX\ should
+start reading the macro text; and when |m=out_param|, it means that \TeX\ should
+insert parameter number |c| into the text at this point.
+
+The enclosing \.{\char'173} and \.{\char'175} characters of a macro definition
+are omitted, but the final right brace of an output routine is included at the
+end of its token list.
+
+Here is an example macro definition that illustrates these conventions. After
+\TeX\ processes the text
+
$$\.{\\def\\mac a\#1\#2 \\b \{\#1\\-a \#\#1\#2 \#2\}}$$
+
the definition of \.{\\mac} is represented as a token list containing
+
$$\def\,{\hskip2pt}
\vbox{\halign{\hfil#\hfil\cr
(reference count), |letter|\,\.a, |match|\,\#, |match|\,\#, |spacer|\,\.\ ,
@@ -218,25 +217,23 @@ $$\def\,{\hskip2pt}
|out_param|\,1, \.{\\-}, |letter|\,\.a, |spacer|\,\.\ , |mac_param|\,\#,
|other_char|\,\.1,\cr
|out_param|\,2, |spacer|\,\.\ , |out_param|\,2.\cr}}$$
-The procedure |scan_toks| builds such token lists, and |macro_call|
-does the parameter matching.
-@^reference counts@>
-Examples such as
-$$\.{\\def\\m\{\\def\\m\{a\}\ b\}}$$
-explain why reference counts would be needed even if \TeX\ had no \.{\\let}
-operation: When the token list for \.{\\m} is being read, the redefinition of
-\.{\\m} changes the |eqtb| entry before the token list has been fully
-consumed, so we dare not simply destroy a token list when its
-control sequence is being redefined.
+The procedure |scan_toks| builds such token lists, and |macro_call| does the
+parameter matching. @^reference counts@>
+
+Examples such as $$\.{\\def\\m\{\\def\\m\{a\}\ b\}}$$ explain why reference
+counts would be needed even if \TeX\ had no \.{\\let} operation: When the token
+list for \.{\\m} is being read, the redefinition of \.{\\m} changes the |eqtb|
+entry before the token list has been fully consumed, so we dare not simply
+destroy a token list when its control sequence is being redefined.
-If the parameter-matching part of a definition ends with `\.{\#\{}',
-the corresponding token list will have `\.\{' just before the `|end_match|'
-and also at the very end. The first `\.\{' is used to delimit the parameter; the
-second one keeps the first from disappearing.
+If the parameter-matching part of a definition ends with `\.{\#\{}', the
+corresponding token list will have `\.\{' just before the `|end_match|' and also
+at the very end. The first `\.\{' is used to delimit the parameter; the second
+one keeps the first from disappearing.
-The |print_meaning| subroutine displays |cur_cmd| and |cur_chr| in
-symbolic form, including the expansion of a macro or mark.
+The |print_meaning| subroutine displays |cur_cmd| and |cur_chr| in symbolic form,
+including the expansion of a macro or mark.
@c
void print_meaning(void)
@@ -272,31 +269,29 @@ void print_meaning(void)
}
}
-@ The procedure |show_token_list|, which prints a symbolic form of
-the token list that starts at a given node |p|, illustrates these
-conventions. The token list being displayed should not begin with a reference
-count. However, the procedure is intended to be robust, so that if the
-memory links are awry or if |p| is not really a pointer to a token list,
-nothing catastrophic will happen.
-
-An additional parameter |q| is also given; this parameter is either null
-or it points to a node in the token list where a certain magic computation
-takes place that will be explained later. (Basically, |q| is non-null when
-we are printing the two-line context information at the time of an error
-message; |q| marks the place corresponding to where the second line
-should begin.)
-
-For example, if |p| points to the node containing the first \.a in the
-token list above, then |show_token_list| will print the string
-$$\hbox{`\.{a\#1\#2\ \\b\ ->\#1\\-a\ \#\#1\#2\ \#2}';}$$
-and if |q| points to the node containing the second \.a,
-the magic computation will be performed just before the second \.a is printed.
-
-The generation will stop, and `\.{\\ETC.}' will be printed, if the length
-of printing exceeds a given limit~|l|. Anomalous entries are printed in the
-form of control sequences that are not followed by a blank space, e.g.,
-`\.{\\BAD.}'; this cannot be confused with actual control sequences because
-a real control sequence named \.{BAD} would come out `\.{\\BAD\ }'.
+@ The procedure |show_token_list|, which prints a symbolic form of the token list
+that starts at a given node |p|, illustrates these conventions. The token list
+being displayed should not begin with a reference count. However, the procedure
+is intended to be robust, so that if the memory links are awry or if |p| is not
+really a pointer to a token list, nothing catastrophic will happen.
+
+An additional parameter |q| is also given; this parameter is either null or it
+points to a node in the token list where a certain magic computation takes place
+that will be explained later. (Basically, |q| is non-null when we are printing
+the two-line context information at the time of an error message; |q| marks the
+place corresponding to where the second line should begin.)
+
+For example, if |p| points to the node containing the first \.a in the token list
+above, then |show_token_list| will print the string $$\hbox{`\.{a\#1\#2\ \\b\
+->\#1\\-a\ \#\#1\#2\ \#2}';}$$ and if |q| points to the node containing the
+second \.a, the magic computation will be performed just before the second \.a is
+printed.
+
+The generation will stop, and `\.{\\ETC.}' will be printed, if the length of
+printing exceeds a given limit~|l|. Anomalous entries are printed in the form of
+control sequences that are not followed by a blank space, e.g., `\.{\\BAD.}';
+this cannot be confused with actual control sequences because a real control
+sequence named \.{BAD} would come out `\.{\\BAD\ }'.
@c
#define not_so_bad(p) \
@@ -320,11 +315,9 @@ a real control sequence named \.{BAD} would come out `\.{\\BAD\ }'.
void show_token_list(int p, int q, int l)
{
- int m, c; /* pieces of a token */
- ASCII_code match_chr; /* character used in a `|match|' */
- ASCII_code n; /* the highest parameter number, as an ASCII digit */
- match_chr = '#';
- n = '0';
+ int m, c; /* pieces of a token */
+ ASCII_code match_chr = '#'; /* character used in a `|match|' */
+ ASCII_code n = '0'; /* the highest parameter number, as an ASCII digit */
tally = 0;
if (l < 0)
l = 0x3FFFFFFF;
@@ -347,51 +340,54 @@ void show_token_list(int p, int q, int l)
if (token_info(p) < 0) {
tprint_esc("BAD");
} else {
- /* Display the token $(|m|,|c|)$ */
- /* The procedure usually ``learns'' the character code used for macro
- parameters by seeing one in a |match| command before it runs into any
- |out_param| commands. */
+ /*
+ Display the token $(|m|,|c|)$
+
+ The procedure usually ``learns'' the character code used for macro
+ parameters by seeing one in a |match| command before it runs into any
+ |out_param| commands.
+ */
switch (m) {
- case left_brace_cmd:
- case right_brace_cmd:
- case math_shift_cmd:
- case tab_mark_cmd:
- case sup_mark_cmd:
- case sub_mark_cmd:
- case spacer_cmd:
- case letter_cmd:
- case other_char_cmd:
- print(c);
- break;
- case mac_param_cmd:
- if (!in_lua_escape && (is_in_csname==0))
+ case left_brace_cmd:
+ case right_brace_cmd:
+ case math_shift_cmd:
+ case tab_mark_cmd:
+ case sup_mark_cmd:
+ case sub_mark_cmd:
+ case spacer_cmd:
+ case letter_cmd:
+ case other_char_cmd:
print(c);
- print(c);
- break;
- case out_param_cmd:
- print(match_chr);
- if (c <= 9) {
- print_char(c + '0');
- } else {
- print_char('!');
- return;
- }
- break;
- case match_cmd:
- match_chr = c;
- print(c);
- incr(n);
- print_char(n);
- if (n > '9')
- return;
- break;
- case end_match_cmd:
- if (c == 0)
- tprint("->");
- break;
- default:
- not_so_bad(tprint);
- break;
+ break;
+ case mac_param_cmd:
+ if (!in_lua_escape && (is_in_csname==0))
+ print(c);
+ print(c);
+ break;
+ case out_param_cmd:
+ print(match_chr);
+ if (c <= 9) {
+ print_char(c + '0');
+ } else {
+ print_char('!');
+ return;
+ }
+ break;
+ case match_cmd:
+ match_chr = c;
+ print(c);
+ incr(n);
+ print_char(n);
+ if (n > '9')
+ return;
+ break;
+ case end_match_cmd:
+ if (c == 0)
+ tprint("->");
+ break;
+ default:
+ not_so_bad(tprint);
+ break;
}
}
}
@@ -402,14 +398,13 @@ void show_token_list(int p, int q, int l)
}
@ @c
-#define do_buffer_to_unichar(a,b) do { \
- a = (halfword)str2uni(buffer+b); \
- b += utf8_size(a); \
- } while (0)
-
+#define do_buffer_to_unichar(a,b) do { \
+ a = (halfword)str2uni(buffer+b); \
+ b += utf8_size(a); \
+} while (0)
-@ Here's the way we sometimes want to display a token list, given a pointer
-to its reference count; the pointer may be null.
+@ Here's the way we sometimes want to display a token list, given a pointer to
+its reference count; the pointer may be null.
@c
void token_show(halfword p)
@@ -418,17 +413,17 @@ void token_show(halfword p)
show_token_list(token_link(p), null, 10000000);
}
-@ |delete_token_ref|, is called when
-a pointer to a token list's reference count is being removed. This means
-that the token list should disappear if the reference count was |null|,
-otherwise the count should be decreased by one.
+@ |delete_token_ref|, is called when a pointer to a token list's reference count
+is being removed. This means that the token list should disappear if the
+reference count was |null|, otherwise the count should be decreased by one.
@^reference counts@>
+@ |p| points to the reference count of a token list that is losing one
+reference.
+
@c
void delete_token_ref(halfword p)
-{ /* |p| points to the reference count
- of a token list that is losing one reference */
- assert(token_ref_count(p) >= 0);
+{
if (token_ref_count(p) == 0)
flush_list(p);
else
@@ -446,8 +441,8 @@ int get_char_cat_code(int curchr)
@ @c
static void invalid_character_error(void)
{
- const char *hlp[] =
- { "A funny symbol that I can't read has just been input.",
+ const char *hlp[] = {
+ "A funny symbol that I can't read has just been input.",
"Continue, and I'll forget that it ever happened.",
NULL
};
@@ -461,26 +456,26 @@ static boolean process_sup_mark(void); /* below */
static int scan_control_sequence(void); /* below */
-typedef enum { next_line_ok, next_line_return,
+typedef enum {
+ next_line_ok,
+ next_line_return,
next_line_restart
} next_line_retval;
-static next_line_retval next_line(void); /* below */
+static next_line_retval next_line(void); /* below */
-@ In case you are getting bored, here is a slightly less trivial routine:
- Given a string of lowercase letters, like `\.{pt}' or `\.{plus}' or
- `\.{width}', the |scan_keyword| routine checks to see whether the next
- tokens of input match this string. The match must be exact, except that
- uppercase letters will match their lowercase counterparts; uppercase
- equivalents are determined by subtracting |"a"-"A"|, rather than using the
- |uc_code| table, since \TeX\ uses this routine only for its own limited
- set of keywords.
+@ In case you are getting bored, here is a slightly less trivial routine: Given a
+string of lowercase letters, like `\.{pt}' or `\.{plus}' or `\.{width}', the
+|scan_keyword| routine checks to see whether the next tokens of input match this
+string. The match must be exact, except that uppercase letters will match their
+lowercase counterparts; uppercase equivalents are determined by subtracting
+|"a"-"A"|, rather than using the |uc_code| table, since \TeX\ uses this routine
+only for its own limited set of keywords.
- If a match is found, the characters are effectively removed from the input
- and |true| is returned. Otherwise |false| is returned, and the input
- is left essentially unchanged (except for the fact that some macros
- may have been expanded, etc.).
- @^inner loop@>
+If a match is found, the characters are effectively removed from the input and
+|true| is returned. Otherwise |false| is returned, and the input is left
+essentially unchanged (except for the fact that some macros may have been
+expanded, etc.). @^inner loop@>
@c
boolean scan_keyword(const char *s)
@@ -509,7 +504,7 @@ boolean scan_keyword(const char *s)
token_link(p) = q;
begin_token_list(token_link(backup_head), backed_up);
if (cur_cmd != endv_cmd)
- align_state = saved_align_state;
+ align_state = saved_align_state;
} else {
back_input();
}
@@ -517,7 +512,8 @@ boolean scan_keyword(const char *s)
return false;
}
}
- flush_list(token_link(backup_head));
+ if (token_link(backup_head) != null)
+ flush_list(token_link(backup_head));
cur_cs = save_cur_cs;
if (cur_cmd != endv_cmd)
align_state = saved_align_state;
@@ -528,36 +524,119 @@ boolean scan_keyword(const char *s)
(inside |shift_case|, for example). This needs thinking.
@c
+
+/*
+ halfword active_to_cs(int curchr, int force)
+ {
+ halfword curcs;
+ char *a, *b;
+ char *utfbytes = xmalloc(8);
+ int nncs = no_new_control_sequence;
+ a = (char *) uni2str(0xFFFF);
+ utfbytes = strcpy(utfbytes, a);
+ if (force)
+ no_new_control_sequence = false;
+ if (curchr > 0) {
+ b = (char *) uni2str((unsigned) curchr);
+ utfbytes = strcat(utfbytes, b);
+ free(b);
+ curcs = string_lookup(utfbytes, strlen(utfbytes));
+ } else {
+ utfbytes[3] = '\0';
+ curcs = string_lookup(utfbytes, 4);
+ }
+ no_new_control_sequence = nncs;
+ free(a);
+ free(utfbytes);
+ return curcs;
+ }
+*/
+
+/*static char * FFFF = "\xEF\xBF\xBF";*/ /* 0xFFFF */
+
halfword active_to_cs(int curchr, int force)
{
halfword curcs;
- char *a, *b;
- char *utfbytes = xmalloc(10);
int nncs = no_new_control_sequence;
- a = (char *) uni2str(0xFFFF);
- utfbytes = strcpy(utfbytes, a);
- if (force)
+ if (force) {
no_new_control_sequence = false;
+ }
if (curchr > 0) {
- b = (char *) uni2str((unsigned) curchr);
+ char *b = (char *) uni2str((unsigned) curchr);
+ char *utfbytes = xmalloc(8);
+ utfbytes = strcpy(utfbytes, "\xEF\xBF\xBF");
utfbytes = strcat(utfbytes, b);
free(b);
- curcs = string_lookup(utfbytes, strlen(utfbytes));
+ curcs = string_lookup(utfbytes, utf8_size(curchr)+3);
+ free(utfbytes);
} else {
- utfbytes[3] = '\0';
- curcs = string_lookup(utfbytes, 4);
+ curcs = string_lookup("\xEF\xBF\xBF", 4); /* 0xFFFF ... why not 3 ? */
}
no_new_control_sequence = nncs;
- free(a);
- free(utfbytes);
return curcs;
}
+/*
+
+ static unsigned char *uni2csstr(unsigned unic)
+ {
+ unsigned char *buf = xmalloc(8);
+ unsigned char *pt = buf;
+ *pt++ = 239; *pt++ = 191; *pt++ = 191; // 0xFFFF
+ if (unic < 0x80)
+ *pt++ = (unsigned char) unic;
+ else if (unic < 0x800) {
+ *pt++ = (unsigned char) (0xc0 | (unic >> 6));
+ *pt++ = (unsigned char) (0x80 | (unic & 0x3f));
+ } else if (unic >= 0x110000) {
+ *pt++ = (unsigned char) (unic - 0x110000);
+ } else if (unic < 0x10000) {
+ *pt++ = (unsigned char) (0xe0 | (unic >> 12));
+ *pt++ = (unsigned char) (0x80 | ((unic >> 6) & 0x3f));
+ *pt++ = (unsigned char) (0x80 | (unic & 0x3f));
+ } else {
+ int u, z, y, x;
+ unsigned val = unic - 0x10000;
+ u = (int) (((val & 0xf0000) >> 16) + 1);
+ z = (int) ((val & 0x0f000) >> 12);
+ y = (int) ((val & 0x00fc0) >> 6);
+ x = (int) (val & 0x0003f);
+ *pt++ = (unsigned char) (0xf0 | (u >> 2));
+ *pt++ = (unsigned char) (0x80 | ((u & 3) << 4) | z);
+ *pt++ = (unsigned char) (0x80 | y);
+ *pt++ = (unsigned char) (0x80 | x);
+ }
+ *pt = '\0';
+ return buf;
+ }
+
+ halfword active_to_cs(int curchr, int force)
+ {
+ halfword curcs;
+ int nncs = no_new_control_sequence;
+ if (force) {
+ no_new_control_sequence = false;
+ }
+ if (curchr > 0) {
+ char * utfbytes = (char *) uni2csstr((unsigned) curchr);
+ curcs = string_lookup(utfbytes, utf8_size(curchr)+3);
+ free(utfbytes);
+ } else {
+ curcs = string_lookup(FFFF, 4); // 0xFFFF ... why not 3 ?
+ }
+ no_new_control_sequence = nncs;
+ return curcs;
+ }
+
+*/
+
@ TODO this function should listen to \.{\\escapechar}
+@ prints a control sequence
+
@c
static char *cs_to_string(halfword p)
-{ /* prints a control sequence */
+{
const char *s;
char *sh;
int k = 0;
@@ -614,57 +693,51 @@ static char *cmd_chr_to_string(int cmd, int chr)
return s;
}
-@ The heart of \TeX's input mechanism is the |get_next| procedure, which
-we shall develop in the next few sections of the program. Perhaps we
-shouldn't actually call it the ``heart,'' however, because it really acts
-as \TeX's eyes and mouth, reading the source files and gobbling them up.
-And it also helps \TeX\ to regurgitate stored token lists that are to be
-processed again.
-@^eyes and mouth@>
+@ The heart of \TeX's input mechanism is the |get_next| procedure, which we shall
+develop in the next few sections of the program. Perhaps we shouldn't actually
+call it the ``heart,'' however, because it really acts as \TeX's eyes and mouth,
+reading the source files and gobbling them up. And it also helps \TeX\ to
+regurgitate stored token lists that are to be processed again. @^eyes and mouth@>
-The main duty of |get_next| is to input one token and to set |cur_cmd|
-and |cur_chr| to that token's command code and modifier. Furthermore, if
-the input token is a control sequence, the |eqtb| location of that control
-sequence is stored in |cur_cs|; otherwise |cur_cs| is set to zero.
+The main duty of |get_next| is to input one token and to set |cur_cmd| and
+|cur_chr| to that token's command code and modifier. Furthermore, if the input
+token is a control sequence, the |eqtb| location of that control sequence is
+stored in |cur_cs|; otherwise |cur_cs| is set to zero.
-Underlying this simple description is a certain amount of complexity
-because of all the cases that need to be handled.
-However, the inner loop of |get_next| is reasonably short and fast.
+Underlying this simple description is a certain amount of complexity because of
+all the cases that need to be handled. However, the inner loop of |get_next| is
+reasonably short and fast.
When |get_next| is asked to get the next token of a \.{\\read} line,
it sets |cur_cmd=cur_chr=cur_cs=0| in the case that no more tokens
appear on that line. (There might not be any tokens at all, if the
|end_line_char| has |ignore| as its catcode.)
-@ The value of |par_loc| is the |eqtb| address of `\.{\\par}'. This quantity
-is needed because a blank line of input is supposed to be exactly equivalent
-to the appearance of \.{\\par}; we must set |cur_cs:=par_loc|
-when detecting a blank line.
+The value of |par_loc| is the |eqtb| address of `\.{\\par}'. This quantity is
+needed because a blank line of input is supposed to be exactly equivalent to the
+appearance of \.{\\par}; we must set |cur_cs:=par_loc| when detecting a blank
+line.
@c
-halfword par_loc; /* location of `\.{\\par}' in |eqtb| */
-halfword par_token; /* token representing `\.{\\par}' */
-
+halfword par_loc; /* location of `\.{\\par}' in |eqtb| */
+halfword par_token; /* token representing `\.{\\par}' */
@ Parts |get_next| are executed more often than any other instructions of \TeX.
@^mastication@>@^inner loop@>
-
-
-@ The global variable |force_eof| is normally |false|; it is set |true|
-by an \.{\\endinput} command. |luacstrings| is the number of lua print
-statements waiting to be input, it is changed by |luatokencall|.
+The global variable |force_eof| is normally |false|; it is set |true| by an
+\.{\\endinput} command. |luacstrings| is the number of lua print statements
+waiting to be input, it is changed by |luatokencall|.
@c
-boolean force_eof; /* should the next \.{\\input} be aborted early? */
-int luacstrings; /* how many lua strings are waiting to be input? */
+boolean force_eof; /* should the next \.{\\input} be aborted early? */
+int luacstrings; /* how many lua strings are waiting to be input? */
-@ If the user has set the |pausing| parameter to some positive value,
-and if nonstop mode has not been selected, each line of input is displayed
-on the terminal and the transcript file, followed by `\.{=>}'.
-\TeX\ waits for a response. If the response is simply |carriage_return|, the
-line is accepted as it stands, otherwise the line typed is
-used instead of the line in the file.
+@ If the user has set the |pausing| parameter to some positive value, and if
+nonstop mode has not been selected, each line of input is displayed on the
+terminal and the transcript file, followed by `\.{=>}'. \TeX\ waits for a
+response. If the response is simply |carriage_return|, the line is accepted as it
+stands, otherwise the line typed is used instead of the line in the file.
@c
void firm_up_the_line(void)
@@ -690,10 +763,10 @@ void firm_up_the_line(void)
}
}
-@ Before getting into |get_next|, let's consider the subroutine that
- is called when an `\.{\\outer}' control sequence has been scanned or
- when the end of a file has been reached. These two cases are distinguished
- by |cur_cs|, which is zero at the end of a file.
+@ Before getting into |get_next|, let's consider the subroutine that is called
+when an `\.{\\outer}' control sequence has been scanned or when the end of a file
+has been reached. These two cases are distinguished by |cur_cs|, which is zero at
+the end of a file.
@c
void check_outer_validity(void)
@@ -717,8 +790,8 @@ void check_outer_validity(void)
cur_chr = ' '; /* replace it by a space */
}
if (scanner_status > skipping) {
- const char *errhlp[] =
- { "I suspect you have forgotten a `}', causing me",
+ const char *errhlp[] = {
+ "I suspect you have forgotten a `}', causing me",
"to read past where you wanted me to stop.",
"I'll try to recover; but if the error is serious,",
"you'd better type `E' or `X' now and fix your file.",
@@ -777,14 +850,14 @@ void check_outer_validity(void)
tex_error(errmsg, errhlp);
} else {
char errmsg[256];
- const char *errhlp_no[] =
- { "The file ended while I was skipping conditional text.",
+ const char *errhlp_no[] = {
+ "The file ended while I was skipping conditional text.",
"This kind of error happens when you say `\\if...' and forget",
"the matching `\\fi'. I've inserted a `\\fi'; this might work.",
NULL
};
- const char *errhlp_cs[] =
- { "A forbidden control sequence occurred in skipped text.",
+ const char *errhlp_cs[] = {
+ "A forbidden control sequence occurred in skipped text.",
"This kind of error happens when you say `\\if...' and forget",
"the matching `\\fi'. I've inserted a `\\fi'; this might work.",
NULL
@@ -796,9 +869,8 @@ void check_outer_validity(void)
cur_cs = 0;
}
ss = cmd_chr_to_string(if_test_cmd, cur_if);
- snprintf(errmsg, 255,
- "Incomplete %s; all text was ignored after line %d",
- ss, (int) skip_line);
+ snprintf(errmsg, 255, "Incomplete %s; all text was ignored after line %d",
+ ss, (int) skip_line);
free(ss);
/* Incomplete \\if... */
cur_tok = cs_token_flag + frozen_fi;
@@ -816,10 +888,19 @@ void check_outer_validity(void)
}
@ @c
+
+#if 0
+
+/*
+ The other variant gives less clutter in tracing cache usage when profiling and for
+ some files (like the manual) also a bit of a speedup.
+*/
+
static boolean get_next_file(void)
{
SWITCH:
- if (iloc <= ilimit) { /* current line not yet finished */
+ if (iloc <= ilimit) {
+ /* current line not yet finished */
do_buffer_to_unichar(cur_chr, iloc);
RESWITCH:
@@ -829,127 +910,160 @@ static boolean get_next_file(void)
do_get_cat_code(cur_cmd, cur_chr);
}
/*
- Change state if necessary, and |goto switch| if the current
- character should be ignored, or |goto reswitch| if the current
- character changes to another;
- */
- /* The following 48-way switch accomplishes the scanning quickly, assuming
- that a decent C compiler has translated the code. Note that the numeric
- values for |mid_line|, |skip_blanks|, and |new_line| are spaced
- apart from each other by |max_char_code+1|, so we can add a character's
- command code to the state to get a single number that characterizes both.
- */
+ Change state if necessary, and |goto switch| if the current
+ character should be ignored, or |goto reswitch| if the current
+ character changes to another;
+
+ The following 48-way switch accomplishes the scanning quickly, assuming
+ that a decent C compiler has translated the code. Note that the numeric
+ values for |mid_line|, |skip_blanks|, and |new_line| are spaced
+ apart from each other by |max_char_code+1|, so we can add a character's
+ command code to the state to get a single number that characterizes both.
+
+ Remark [ls/hh]: checking performance indicated that this switch was the
+ cause of many branch prediction errors but changing it to:
+
+ c = istate + cur_cmd;
+ if (c == (mid_line + letter_cmd) || c == (mid_line + other_char_cmd)) {
+ return true;
+ } else if (c >= new_line) {
+ switch (c) {
+ }
+ } else if (c >= skip_blanks) {
+ switch (c) {
+ }
+ } else if (c >= mid_line) {
+ switch (c) {
+ }
+ } else {
+ istate = mid_line;
+ return true;
+ }
+
+ gives as many prediction errors. So, we can indeed assume that the compiler
+ does the right job, or that there is simply no other way.
+ */
+
switch (istate + cur_cmd) {
- case mid_line + ignore_cmd:
- case skip_blanks + ignore_cmd:
- case new_line + ignore_cmd:
- case skip_blanks + spacer_cmd:
- case new_line + spacer_cmd: /* Cases where character is ignored */
- goto SWITCH;
- break;
- case mid_line + escape_cmd:
- case new_line + escape_cmd:
- case skip_blanks + escape_cmd: /* Scan a control sequence ...; */
- istate = (unsigned char) scan_control_sequence();
- if (cur_cmd >= outer_call_cmd)
- check_outer_validity();
- break;
- case mid_line + active_char_cmd:
- case new_line + active_char_cmd:
- case skip_blanks + active_char_cmd: /* Process an active-character */
- cur_cs = active_to_cs(cur_chr, false);
- cur_cmd = eq_type(cur_cs);
- cur_chr = equiv(cur_cs);
- istate = mid_line;
- if (cur_cmd >= outer_call_cmd)
- check_outer_validity();
- break;
- case mid_line + sup_mark_cmd:
- case new_line + sup_mark_cmd:
- case skip_blanks + sup_mark_cmd: /* If this |sup_mark| starts */
- if (process_sup_mark())
- goto RESWITCH;
- else
+ case mid_line + ignore_cmd:
+ case skip_blanks + ignore_cmd:
+ case new_line + ignore_cmd:
+ case skip_blanks + spacer_cmd:
+ case new_line + spacer_cmd:
+ /* Cases where character is ignored */
+ goto SWITCH;
+ break;
+ case mid_line + escape_cmd:
+ case new_line + escape_cmd:
+ case skip_blanks + escape_cmd:
+ /* Scan a control sequence ...; */
+ istate = (unsigned char) scan_control_sequence();
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ break;
+ case mid_line + active_char_cmd:
+ case new_line + active_char_cmd:
+ case skip_blanks + active_char_cmd:
+ /* Process an active-character */
+ cur_cs = active_to_cs(cur_chr, false);
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
istate = mid_line;
- break;
- case mid_line + invalid_char_cmd:
- case new_line + invalid_char_cmd:
- case skip_blanks + invalid_char_cmd: /* Decry the invalid character and |goto restart|; */
- invalid_character_error();
- return false; /* because state may be |token_list| now */
- break;
- case mid_line + spacer_cmd: /* Enter |skip_blanks| state, emit a space; */
- istate = skip_blanks;
- cur_chr = ' ';
- break;
- case mid_line + car_ret_cmd: /* Finish line, emit a space; */
- /* When a character of type |spacer| gets through, its character code is
- changed to $\.{"\ "}=040$. This means that the ASCII codes for tab and space,
- and for the space inserted at the end of a line, will
- be treated alike when macro parameters are being matched. We do this
- since such characters are indistinguishable on most computer terminal displays.
- */
- iloc = ilimit + 1;
- cur_cmd = spacer_cmd;
- cur_chr = ' ';
- break;
- case skip_blanks + car_ret_cmd:
- case mid_line + comment_cmd:
- case new_line + comment_cmd:
- case skip_blanks + comment_cmd: /* Finish line, |goto switch|; */
- iloc = ilimit + 1;
- goto SWITCH;
- break;
- case new_line + car_ret_cmd: /* Finish line, emit a \.{\\par}; */
- iloc = ilimit + 1;
- cur_cs = par_loc;
- cur_cmd = eq_type(cur_cs);
- cur_chr = equiv(cur_cs);
- if (cur_cmd >= outer_call_cmd)
- check_outer_validity();
- break;
- case skip_blanks + left_brace_cmd:
- case new_line + left_brace_cmd:
- istate = mid_line; /* fall through */
- case mid_line + left_brace_cmd:
- align_state++;
- break;
- case skip_blanks + right_brace_cmd:
- case new_line + right_brace_cmd:
- istate = mid_line; /* fall through */
- case mid_line + right_brace_cmd:
- align_state--;
- break;
- case mid_line + math_shift_cmd:
- case mid_line + tab_mark_cmd:
- case mid_line + mac_param_cmd:
- case mid_line + sub_mark_cmd:
- case mid_line + letter_cmd:
- case mid_line + other_char_cmd:
- break;
-#if 0
- case skip_blanks + math_shift:
- case skip_blanks + tab_mark:
- case skip_blanks + mac_param:
- case skip_blanks + sub_mark:
- case skip_blanks + letter:
- case skip_blanks + other_char:
- case new_line + math_shift:
- case new_line + tab_mark:
- case new_line + mac_param:
- case new_line + sub_mark:
- case new_line + letter:
- case new_line + other_char:
-#else
- default:
-#endif
- istate = mid_line;
- break;
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ break;
+ case mid_line + sup_mark_cmd:
+ case new_line + sup_mark_cmd:
+ case skip_blanks + sup_mark_cmd:
+ /* If this |sup_mark| starts */
+ if (process_sup_mark())
+ goto RESWITCH;
+ else
+ istate = mid_line;
+ break;
+ case mid_line + invalid_char_cmd:
+ case new_line + invalid_char_cmd:
+ case skip_blanks + invalid_char_cmd:
+ /* Decry the invalid character and |goto restart|; */
+ invalid_character_error();
+ return false; /* because state may be |token_list| now */
+ break;
+ case mid_line + spacer_cmd:
+ /* Enter |skip_blanks| state, emit a space; */
+ istate = skip_blanks;
+ cur_chr = ' ';
+ break;
+ case mid_line + car_ret_cmd:
+ /*
+ Finish line, emit a space. When a character of type |spacer| gets through, its
+ character code is changed to $\.{"\ "}=040$. This means that the ASCII codes
+ for tab and space, and for the space inserted at the end of a line, will be
+ treated alike when macro parameters are being matched. We do this since such
+ characters are indistinguishable on most computer terminal displays.
+ */
+ iloc = ilimit + 1;
+ cur_cmd = spacer_cmd;
+ cur_chr = ' ';
+ break;
+ case skip_blanks + car_ret_cmd:
+ case mid_line + comment_cmd:
+ case new_line + comment_cmd:
+ case skip_blanks + comment_cmd:
+ /* Finish line, |goto switch|; */
+ iloc = ilimit + 1;
+ goto SWITCH;
+ break;
+ case new_line + car_ret_cmd:
+ /* Finish line, emit a \.{\\par}; */
+ iloc = ilimit + 1;
+ cur_cs = par_loc;
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ break;
+ case skip_blanks + left_brace_cmd:
+ case new_line + left_brace_cmd:
+ istate = mid_line;
+ /* fall through */
+ case mid_line + left_brace_cmd:
+ align_state++;
+ break;
+ case skip_blanks + right_brace_cmd:
+ case new_line + right_brace_cmd:
+ istate = mid_line;
+ /* fall through */
+ case mid_line + right_brace_cmd:
+ align_state--;
+ break;
+ case mid_line + math_shift_cmd:
+ case mid_line + tab_mark_cmd:
+ case mid_line + mac_param_cmd:
+ case mid_line + sub_mark_cmd:
+ case mid_line + letter_cmd:
+ case mid_line + other_char_cmd:
+ break;
+ /*
+ case skip_blanks + math_shift:
+ case skip_blanks + tab_mark:
+ case skip_blanks + mac_param:
+ case skip_blanks + sub_mark:
+ case skip_blanks + letter:
+ case skip_blanks + other_char:
+ case new_line + math_shift:
+ case new_line + tab_mark:
+ case new_line + mac_param:
+ case new_line + sub_mark:
+ case new_line + letter:
+ case new_line + other_char:
+ */
+ default:
+ istate = mid_line;
+ break;
}
} else {
if (iname != 21)
istate = new_line;
-
/*
Move to next line of file,
or |goto restart| if there is no next line,
@@ -969,108 +1083,389 @@ static boolean get_next_file(void)
return true;
}
-@ @c
-#define is_hex(a) ((a>='0'&&a<='9')||(a>='a'&&a<='f'))
-
-#define add_nybble(a) do { \
- if (a<='9') cur_chr=(cur_chr<<4)+a-'0'; \
- else cur_chr=(cur_chr<<4)+a-'a'+10; \
- } while (0)
-
-#define hex_to_cur_chr do { \
- if (c<='9') cur_chr=c-'0'; \
- else cur_chr=c-'a'+10; \
- add_nybble(cc); \
- } while (0)
+#else
-#define four_hex_to_cur_chr do { \
- hex_to_cur_chr; \
- add_nybble(ccc); add_nybble(cccc); \
- } while (0)
+/* 10 times less Bim in callgrind */
-#define five_hex_to_cur_chr do { \
- four_hex_to_cur_chr; \
- add_nybble(ccccc); \
- } while (0)
+/*
+ escape_cmd left_brace_cmd right_brace_cmd math_shift_cmd
+ tab_mark_cmd car_ret_cmd mac_param_cmd sup_mark_cmd
+ sub_mark_cmd ignore_cmd spacer_cmd letter_cmd
+ other_char_cmd active_char_cmd comment_cmd invalid_char_cmd
+*/
-#define six_hex_to_cur_chr do { \
- five_hex_to_cur_chr; \
- add_nybble(cccccc); \
- } while (0)
+static boolean get_next_file(void)
+{
+ int c = 0;
+ SWITCH:
+ if (iloc <= ilimit) {
+ /* current line not yet finished */
+ do_buffer_to_unichar(cur_chr, iloc);
+ RESWITCH:
+ if (detokenized_line()) {
+ cur_cmd = (cur_chr == ' ' ? 10 : 12);
+ } else {
+ do_get_cat_code(cur_cmd, cur_chr);
+ }
+ /*
+ Change state if necessary, and |goto switch| if the current
+ character should be ignored, or |goto reswitch| if the current
+ character changes to another;
+ */
+ c = istate + cur_cmd;
+ if (c == (mid_line + letter_cmd) || c == (mid_line + other_char_cmd)) {
+ return true;
+ } else if (c >= new_line) {
+ switch (c-new_line) {
+ case escape_cmd:
+ istate = (unsigned char) scan_control_sequence();
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case left_brace_cmd:
+ istate = mid_line;
+ align_state++;
+ return true;
+ case right_brace_cmd:
+ istate = mid_line;
+ align_state--;
+ return true;
+ case math_shift_cmd:
+ istate = mid_line;
+ return true;
+ case tab_mark_cmd:
+ istate = mid_line;
+ return true;
+ case car_ret_cmd:
+ /* Finish line, emit a \.{\\par}; */
+ iloc = ilimit + 1;
+ cur_cs = par_loc;
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case mac_param_cmd:
+ istate = mid_line;
+ return true;
+ case sup_mark_cmd:
+ if (process_sup_mark())
+ goto RESWITCH;
+ else
+ istate = mid_line;
+ return true;
+ case sub_mark_cmd:
+ istate = mid_line;
+ return true;
+ case ignore_cmd:
+ goto SWITCH;
+ return true;
+ case spacer_cmd:
+ /* Cases where character is ignored */
+ goto SWITCH;
+ case letter_cmd:
+ istate = mid_line;
+ return true;
+ case other_char_cmd:
+ istate = mid_line;
+ return true;
+ case active_char_cmd:
+ cur_cs = active_to_cs(cur_chr, false);
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ istate = mid_line;
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case comment_cmd:
+ iloc = ilimit + 1;
+ goto SWITCH;
+ case invalid_char_cmd:
+ invalid_character_error();
+ return false; /* because state may be |token_list| now */
+ default:
+ istate = mid_line;
+ return true;
+ }
+ } else if (c >= skip_blanks) {
+ switch (c-skip_blanks) {
+ case escape_cmd:
+ /* Scan a control sequence ...; */
+ istate = (unsigned char) scan_control_sequence();
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case left_brace_cmd:
+ istate = mid_line;
+ align_state++;
+ return true;
+ case right_brace_cmd:
+ istate = mid_line;
+ align_state--;
+ return true;
+ case math_shift_cmd:
+ istate = mid_line;
+ return true;
+ case tab_mark_cmd:
+ istate = mid_line;
+ return true;
+ case car_ret_cmd:
+ iloc = ilimit + 1;
+ goto SWITCH;
+ case mac_param_cmd:
+ istate = mid_line;
+ return true;
+ case sup_mark_cmd:
+ /* If this |sup_mark| starts */
+ if (process_sup_mark())
+ goto RESWITCH;
+ else
+ istate = mid_line;
+ return true;
+ case sub_mark_cmd:
+ istate = mid_line;
+ return true;
+ case ignore_cmd:
+ goto SWITCH;
+ case spacer_cmd:
+ goto SWITCH;
+ case letter_cmd:
+ istate = mid_line;
+ return true;
+ case other_char_cmd:
+ istate = mid_line;
+ return true;
+ case active_char_cmd:
+ cur_cs = active_to_cs(cur_chr, false);
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ istate = mid_line;
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case comment_cmd:
+ /* Finish line, |goto switch|; */
+ iloc = ilimit + 1;
+ goto SWITCH;
+ case invalid_char_cmd:
+ /* Decry the invalid character and |goto restart|; */
+ invalid_character_error();
+ return false; /* because state may be |token_list| now */
+ default:
+ istate = mid_line;
+ return true;
+ }
+ } else if (c >= mid_line) {
+ switch (c-mid_line) {
+ case escape_cmd:
+ istate = (unsigned char) scan_control_sequence();
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case left_brace_cmd:
+ align_state++;
+ return true;
+ case right_brace_cmd:
+ align_state--;
+ return true;
+ case math_shift_cmd:
+ return true;
+ case tab_mark_cmd:
+ return true;
+ case car_ret_cmd:
+ /*
+ Finish line, emit a space. When a character of type |spacer| gets through, its
+ character code is changed to $\.{"\ "}=040$. This means that the ASCII codes
+ for tab and space, and for the space inserted at the end of a line, will be
+ treated alike when macro parameters are being matched. We do this since such
+ characters are indistinguishable on most computer terminal displays.
+ */
+ iloc = ilimit + 1;
+ cur_cmd = spacer_cmd;
+ cur_chr = ' ';
+ return true;
+ case mac_param_cmd:
+ return true;
+ case sup_mark_cmd:
+ if (process_sup_mark())
+ goto RESWITCH;
+ else
+ istate = mid_line;
+ return true;
+ case sub_mark_cmd:
+ return true;
+ case ignore_cmd:
+ goto SWITCH;
+ case spacer_cmd:
+ /* Enter |skip_blanks| state, emit a space; */
+ istate = skip_blanks;
+ cur_chr = ' ';
+ return true;
+ case letter_cmd:
+ istate = mid_line;
+ return true;
+ case other_char_cmd:
+ istate = mid_line;
+ return true;
+ case active_char_cmd:
+ cur_cs = active_to_cs(cur_chr, false);
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ istate = mid_line;
+ if (! suppress_outer_error && cur_cmd >= outer_call_cmd)
+ check_outer_validity();
+ return true;
+ case comment_cmd:
+ iloc = ilimit + 1;
+ goto SWITCH;
+ case invalid_char_cmd:
+ invalid_character_error();
+ return false; /* because state may be |token_list| now */
+ default:
+ istate = mid_line;
+ return true;
+ }
+ } else {
+ istate = mid_line;
+ return true;
+ }
+ } else {
+ if (iname != 21) {
+ istate = new_line;
+ }
+ /*
+ Move to next line of file, or |goto restart| if there is no next line,
+ or |return| if a \.{\\read} line has finished;
+ */
+ do {
+ next_line_retval r = next_line();
+ if (r == next_line_return) {
+ return true;
+ } else if (r == next_line_restart) {
+ return false;
+ }
+ } while (0);
+ check_interrupt();
+ goto SWITCH;
+ }
+ return true;
+}
+#endif
@ Notice that a code like \.{\^\^8} becomes \.x if not followed by a hex digit.
+We only support a limited set:
+
+^^^^^^XXXXXX
+^^^^XXXXXX
+^^XX ^^<char>
@c
+
+#define is_hex(a) ((a>='0'&&a<='9')||(a>='a'&&a<='f'))
+
+#define add_nybble(c) \
+ if (c<='9') { \
+ cur_chr=(cur_chr<<4)+c-'0'; \
+ } else { \
+ cur_chr=(cur_chr<<4)+c-'a'+10; \
+ }
+
+#define set_nybble(c) \
+ if (c<='9') { \
+ cur_chr=c-'0'; \
+ } else { \
+ cur_chr=c-'a'+10; \
+ }
+
+#define one_hex_to_cur_chr(c1) \
+ set_nybble(c1);
+
+#define two_hex_to_cur_chr(c1,c2) \
+ set_nybble(c1); \
+ add_nybble(c2);
+
+#define four_hex_to_cur_chr(c1,c2,c3,c4) \
+ two_hex_to_cur_chr(c1,c2); \
+ add_nybble(c3); \
+ add_nybble(c4);
+
+#define six_hex_to_cur_chr(c1,c2,c3,c4,c5,c6) \
+ four_hex_to_cur_chr(c1,c2,c3,c4); \
+ add_nybble(c5); \
+ add_nybble(c6);
+
static boolean process_sup_mark(void)
{
if (cur_chr == buffer[iloc]) {
- int c, cc;
if (iloc < ilimit) {
- if ( (cur_chr == buffer[iloc + 1])
- && (cur_chr == buffer[iloc + 2])
- && (cur_chr == buffer[iloc + 3])
- && (cur_chr == buffer[iloc + 4])
- && ((iloc + 10) <= ilimit)) {
- int ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */
- c = buffer[iloc + 5];
- cc = buffer[iloc + 6];
- ccc = buffer[iloc + 7];
- cccc = buffer[iloc + 8];
- ccccc = buffer[iloc + 9];
- cccccc = buffer[iloc + 10];
- if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
- && (is_hex(cccc))
- && (is_hex(ccccc)) && (is_hex(cccccc))) {
- iloc = iloc + 11;
- six_hex_to_cur_chr;
- return true;
- }
- }
- if ( (cur_chr == buffer[iloc + 1])
- && (cur_chr == buffer[iloc + 2])
- && (cur_chr == buffer[iloc + 3])
- && ((iloc + 8) <= ilimit)) {
- int ccc, cccc, ccccc; /* constituents of a possible expanded code */
- c = buffer[iloc + 4];
- cc = buffer[iloc + 5];
- ccc = buffer[iloc + 6];
- cccc = buffer[iloc + 7];
- ccccc = buffer[iloc + 8];
- if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
- && (is_hex(cccc)) && (is_hex(ccccc))) {
- iloc = iloc + 9;
- five_hex_to_cur_chr;
- return true;
+ if ((cur_chr == buffer[iloc + 1]) && (cur_chr == buffer[iloc + 2])) {
+ if ((cur_chr == buffer[iloc + 3]) && (cur_chr == buffer[iloc + 4])) {
+ /* ^^^^^^XXXXXX */
+ if ((iloc + 10) <= ilimit) {
+ int c1 = buffer[iloc + 5];
+ int c2 = buffer[iloc + 6];
+ int c3 = buffer[iloc + 7];
+ int c4 = buffer[iloc + 8];
+ int c5 = buffer[iloc + 9];
+ int c6 = buffer[iloc + 10];
+ if (is_hex(c1) && is_hex(c2) && is_hex(c3) &&
+ is_hex(c4) && is_hex(c5) && is_hex(c6)) {
+ iloc = iloc + 11;
+ six_hex_to_cur_chr(c1,c2,c3,c4,c5,c6);
+ return true;
+ } else {
+ tex_error("^^^^^^ needs six hex digits", NULL);
+ }
+ } else {
+ tex_error("^^^^^^ needs six hex digits, end of input", NULL);
+ }
+ } else {
+ /* ^^^^XXXX */
+ if ((iloc + 6) <= ilimit) {
+ int c1 = buffer[iloc + 3];
+ int c2 = buffer[iloc + 4];
+ int c3 = buffer[iloc + 5];
+ int c4 = buffer[iloc + 6];
+ if (is_hex(c1) && is_hex(c2) && is_hex(c3) && is_hex(c4)) {
+ iloc = iloc + 7;
+ four_hex_to_cur_chr(c1,c2,c3,c4);
+ return true;
+ } else {
+ tex_error("^^^^ needs four hex digits", NULL);
+ }
+ } else {
+ tex_error("^^^^ needs four hex digits, end of input", NULL);
+ }
}
- }
- if ( (cur_chr == buffer[iloc + 1])
- && (cur_chr == buffer[iloc + 2])
- && ((iloc + 6) <= ilimit)) {
- int ccc, cccc; /* constituents of a possible expanded code */
- c = buffer[iloc + 3];
- cc = buffer[iloc + 4];
- ccc = buffer[iloc + 5];
- cccc = buffer[iloc + 6];
- if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
- && (is_hex(cccc))) {
- iloc = iloc + 7;
- four_hex_to_cur_chr;
- return true;
+ } else {
+ /* ^^XX */
+ if ((iloc + 2) <= ilimit) {
+ int c1 = buffer[iloc + 1];
+ int c2 = buffer[iloc + 2];
+ if (is_hex(c1) && is_hex(c2)) {
+ iloc = iloc + 3;
+ two_hex_to_cur_chr(c1,c2);
+ return true;
+ }
}
+ /* go on, no error, good old tex */
}
- c = buffer[iloc + 1];
- if (c < 0200) { /* yes we have an expanded char */
+ }
+ /* the rest */
+ {
+ int c1 = buffer[iloc + 1];
+ if (c1 < 0200) {
iloc = iloc + 2;
- if (is_hex(c) && iloc <= ilimit) {
- cc = buffer[iloc];
- if (is_hex(cc)) {
+ if (is_hex(c1) && (iloc <= ilimit)) {
+ int c2 = buffer[iloc];
+ if (is_hex(c2)) {
incr(iloc);
- hex_to_cur_chr;
+ two_hex_to_cur_chr(c1,c2);
return true;
}
}
- cur_chr = (c < 0100 ? c + 0100 : c - 0100);
+ cur_chr = (c1 < 0100 ? c1 + 0100 : c1 - 0100);
return true;
}
}
@@ -1078,20 +1473,19 @@ static boolean process_sup_mark(void)
return false;
}
-@ Control sequence names are scanned only when they appear in some line of
- a file; once they have been scanned the first time, their |eqtb| location
- serves as a unique identification, so \TeX\ doesn't need to refer to the
- original name any more except when it prints the equivalent in symbolic form.
+@ Control sequence names are scanned only when they appear in some line of a
+file; once they have been scanned the first time, their |eqtb| location serves as
+a unique identification, so \TeX\ doesn't need to refer to the original name any
+more except when it prints the equivalent in symbolic form.
- The program that scans a control sequence has been written carefully
- in order to avoid the blowups that might otherwise occur if a malicious
- user tried something like `\.{\\catcode\'15=0}'. The algorithm might
- look at |buffer[ilimit+1]|, but it never looks at |buffer[ilimit+2]|.
+The program that scans a control sequence has been written carefully in order to
+avoid the blowups that might otherwise occur if a malicious user tried something
+like `\.{\\catcode\'15=0}'. The algorithm might look at |buffer[ilimit+1]|, but
+it never looks at |buffer[ilimit+2]|.
- If expanded characters like `\.{\^\^A}' or `\.{\^\^df}'
- appear in or just following
- a control sequence name, they are converted to single characters in the
- buffer and the process is repeated, slowly but surely.
+If expanded characters like `\.{\^\^A}' or `\.{\^\^df}' appear in or just
+following a control sequence name, they are converted to single characters in the
+buffer and the process is repeated, slowly but surely.
@c
static boolean check_expanded_code(int *kk); /* below */
@@ -1121,6 +1515,8 @@ static int scan_control_sequence(void)
if (cat == sup_mark_cmd && check_expanded_code(&k)) /* If an expanded...; */
continue;
if (cat != letter_cmd) {
+ /* backtrack one character which can be utf */
+ /*
decr(k);
if (cur_chr > 0xFFFF)
decr(k);
@@ -1128,7 +1524,18 @@ static int scan_control_sequence(void)
decr(k);
if (cur_chr > 0x7F)
decr(k);
- } /* now |k| points to first nonletter */
+ */
+ if (cur_chr <= 0x7F) {
+ k -= 1; /* in most cases */
+ } else if (cur_chr > 0xFFFF) {
+ k -= 4;
+ } else if (cur_chr > 0x7FF) {
+ k -= 3;
+ } else /* if (cur_chr > 0x7F) */ {
+ k -= 2;
+ }
+ /* now |k| points to first nonletter */
+ }
}
cur_cs = id_lookup(iloc, k - iloc);
iloc = k;
@@ -1141,60 +1548,67 @@ static int scan_control_sequence(void)
}
@ Whenever we reach the following piece of code, we will have
- |cur_chr=buffer[k-1]| and |k<=ilimit+1| and |cat=get_cat_code(cat_code_table,cur_chr)|. If an
- expanded code like \.{\^\^A} or \.{\^\^df} appears in |buffer[(k-1)..(k+1)]|
- or |buffer[(k-1)..(k+2)]|, we
- will store the corresponding code in |buffer[k-1]| and shift the rest of
- the buffer left two or three places.
+|cur_chr=buffer[k-1]| and |k<=ilimit+1| and
+|cat=get_cat_code(cat_code_table,cur_chr)|. If an expanded code like \.{\^\^A} or
+\.{\^\^df} appears in |buffer[(k-1)..(k+1)]| or |buffer[(k-1)..(k+2)]|, we will
+store the corresponding code in |buffer[k-1]| and shift the rest of the buffer
+left two or three places.
@c
static boolean check_expanded_code(int *kk)
{
int l;
int k = *kk;
- int d = 1; /* number of excess characters in an expanded code */
- int c, cc, ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */
+ int d = 1;
if (buffer[k] == cur_chr && k < ilimit) {
- if ((cur_chr == buffer[k + 1]) && (cur_chr == buffer[k + 2])
- && ((k + 6) <= ilimit)) {
- d = 4;
- if ((cur_chr == buffer[k + 3]) && ((k + 8) <= ilimit))
- d = 5;
- if ((cur_chr == buffer[k + 4]) && ((k + 10) <= ilimit))
- d = 6;
- c = buffer[k + d - 1];
- cc = buffer[k + d];
- ccc = buffer[k + d + 1];
- cccc = buffer[k + d + 2];
- if (d == 6) {
- ccccc = buffer[k + d + 3];
- cccccc = buffer[k + d + 4];
- if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
- && is_hex(ccccc) && is_hex(cccccc))
- six_hex_to_cur_chr;
- } else if (d == 5) {
- ccccc = buffer[k + d + 3];
- if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
- && is_hex(ccccc))
- five_hex_to_cur_chr;
+ if ((cur_chr == buffer[k + 1]) && (cur_chr == buffer[k + 2])) {
+ if ((cur_chr == buffer[k + 3]) && (cur_chr == buffer[k + 4])) {
+ if ((k + 10) <= ilimit) {
+ int c1 = buffer[k + 6 - 1];
+ int c2 = buffer[k + 6];
+ int c3 = buffer[k + 6 + 1];
+ int c4 = buffer[k + 6 + 2];
+ int c5 = buffer[k + 6 + 3];
+ int c6 = buffer[k + 6 + 4];
+ if (is_hex(c1) && is_hex(c2) && is_hex(c3) && is_hex(c4) && is_hex(c5) && is_hex(c6)) {
+ d = 6;
+ six_hex_to_cur_chr(c1,c2,c3,c4,c5,c6);
+ } else {
+ tex_error("^^^^^^ needs six hex digits", NULL);
+ }
+ } else {
+ tex_error("^^^^^^ needs six hex digits, end of input", NULL);
+ }
} else {
- if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc))
- four_hex_to_cur_chr;
+ if ((k + 6) <= ilimit) {
+ int c1 = buffer[k + 4 - 1];
+ int c2 = buffer[k + 4];
+ int c3 = buffer[k + 4 + 1];
+ int c4 = buffer[k + 4 + 2];
+ if (is_hex(c1) && is_hex(c2) && is_hex(c3) && is_hex(c4)) {
+ d = 4;
+ four_hex_to_cur_chr(c1,c2,c3,c4);
+ } else {
+ tex_error("^^^^ needs four hex digits", NULL);
+ }
+ } else {
+ tex_error("^^^^ needs four hex digits, end of input", NULL);
+ }
}
} else {
- c = buffer[k + 1];
- if (c < 0200) {
+ int c1 = buffer[k + 1];
+ if (c1 < 0200) {
d = 1;
- if (is_hex(c) && (k + 2) <= ilimit) {
- cc = buffer[k + 2];
- if (is_hex(c) && is_hex(cc)) {
+ if (is_hex(c1) && (k + 2) <= ilimit) {
+ int c2 = buffer[k + 2];
+ if (is_hex(c2)) {
d = 2;
- hex_to_cur_chr;
+ two_hex_to_cur_chr(c1,c2);
+ } else {
+ cur_chr = (c1 < 0100 ? c1 + 0100 : c1 - 0100);
}
- } else if (c < 0100) {
- cur_chr = c + 0100;
} else {
- cur_chr = c - 0100;
+ cur_chr = (c1 < 0100 ? c1 + 0100 : c1 - 0100);
}
}
}
@@ -1213,28 +1627,21 @@ static boolean check_expanded_code(int *kk)
buffer[k - 1] = (packed_ASCII_code) (0xE0 + cur_chr / 0x1000);
k++;
d--;
- buffer[k - 1] =
- (packed_ASCII_code) (0x80 + (cur_chr % 0x1000) / 0x40);
+ buffer[k - 1] = (packed_ASCII_code) (0x80 + (cur_chr % 0x1000) / 0x40);
k++;
d--;
- buffer[k - 1] =
- (packed_ASCII_code) (0x80 + (cur_chr % 0x1000) % 0x40);
+ buffer[k - 1] = (packed_ASCII_code) (0x80 + (cur_chr % 0x1000) % 0x40);
} else {
buffer[k - 1] = (packed_ASCII_code) (0xF0 + cur_chr / 0x40000);
k++;
d--;
- buffer[k - 1] =
- (packed_ASCII_code) (0x80 + (cur_chr % 0x40000) / 0x1000);
+ buffer[k - 1] = (packed_ASCII_code) (0x80 + (cur_chr % 0x40000) / 0x1000);
k++;
d--;
- buffer[k - 1] =
- (packed_ASCII_code) (0x80 +
- ((cur_chr % 0x40000) % 0x1000) / 0x40);
+ buffer[k - 1] = (packed_ASCII_code) (0x80 + ((cur_chr % 0x40000) % 0x1000) / 0x40);
k++;
d--;
- buffer[k - 1] =
- (packed_ASCII_code) (0x80 +
- ((cur_chr % 0x40000) % 0x1000) % 0x40);
+ buffer[k - 1] = (packed_ASCII_code) (0x80 + ((cur_chr % 0x40000) % 0x1000) % 0x40);
}
l = k;
ilimit = ilimit - d;
@@ -1248,13 +1655,12 @@ static boolean check_expanded_code(int *kk)
return false;
}
-@ All of the easy branches of |get_next| have now been taken care of.
- There is one more branch.
+@ All of the easy branches of |get_next| have now been taken care of. There is
+one more branch.
-@c
-static next_line_retval next_line(void)
+@c static next_line_retval next_line(void)
{
- boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */
+ boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */
if (iname > 17) {
/* Read next line of file into |buffer|, or |goto restart| if the file has ended */
incr(line);
@@ -1268,7 +1674,7 @@ static next_line_retval next_line(void)
inhibit_eol = true;
} else if ((every_eof != null) && !eof_seen[iindex]) {
ilimit = first - 1;
- eof_seen[iindex] = true; /* fake one empty line */
+ eof_seen[iindex] = true; /* fake one empty line */
if (iname != 19)
begin_token_list(every_eof, every_eof_text);
return next_line_restart;
@@ -1277,7 +1683,7 @@ static next_line_retval next_line(void)
}
} else {
if (iname == 21) {
- if (luacstring_input()) { /* not end of strings */
+ if (luacstring_input()) { /* not end of strings */
firm_up_the_line();
line_catcode_table = (short) luacstring_cattable();
line_partial = (signed char) luacstring_partial();
@@ -1290,12 +1696,12 @@ static next_line_retval next_line(void)
force_eof = true;
}
} else {
- if (lua_input_ln(cur_file, 0, true)) { /* not end of file */
- firm_up_the_line(); /* this sets |ilimit| */
+ if (lua_input_ln(cur_file, 0, true)) { /* not end of file */
+ firm_up_the_line(); /* this sets |ilimit| */
line_catcode_table = DEFAULT_CAT_TABLE;
} else if ((every_eof != null) && (!eof_seen[iindex])) {
ilimit = first - 1;
- eof_seen[iindex] = true; /* fake one empty line */
+ eof_seen[iindex] = true; /* fake one empty line */
begin_token_list(every_eof, every_eof_text);
return next_line_restart;
} else {
@@ -1306,24 +1712,23 @@ static next_line_retval next_line(void)
}
if (force_eof) {
if (tracing_nesting > 0)
- if ((grp_stack[in_open] != cur_boundary)
- || (if_stack[in_open] != cond_ptr))
- if (!((iname == 19) || (iname == 21)))
- file_warning(); /* give warning for some unfinished groups and/or conditionals */
+ if ((grp_stack[in_open] != cur_boundary) || (if_stack[in_open] != cond_ptr))
+ if (!((iname == 19) || (iname == 21))) {
+ /* give warning for some unfinished groups and/or conditionals */
+ file_warning();
+ }
if ((iname > 21) || (iname == 20)) {
report_stop_file(filetype_tex);
decr(open_parens);
-#if 0
- update_terminal(); /* show user that file has been read */
-#endif
}
force_eof = false;
- if (iname == 21 || /* lua input */
- iname == 19) { /* \.{\\scantextokens} */
+ /* lua input or \.{\\scantextokens} */
+ if (iname == 21 || iname == 19) {
end_file_reading();
} else {
end_file_reading();
- check_outer_validity();
+ if (! suppress_outer_error)
+ check_outer_validity();
}
return next_line_restart;
}
@@ -1332,28 +1737,31 @@ static next_line_retval next_line(void)
else
buffer[ilimit] = (packed_ASCII_code) end_line_char;
first = ilimit + 1;
- iloc = istart; /* ready to read */
+ iloc = istart; /* ready to read */
} else {
- if (!terminal_input) { /* \.{\\read} line has ended */
+ if (!terminal_input) {
+ /* \.{\\read} line has ended */
cur_cmd = 0;
cur_chr = 0;
return next_line_return; /* OUTER */
}
- if (input_ptr > 0) { /* text was inserted during error recovery */
+ if (input_ptr > 0) {
+ /* text was inserted during error recovery */
end_file_reading();
- return next_line_restart; /* resume previous level */
+ return next_line_restart; /* resume previous level */
}
if (selector < log_only)
open_log_file();
if (interaction > nonstop_mode) {
if (end_line_char_inactive)
ilimit++;
- if (ilimit == istart) { /* previous line was empty */
+ if (ilimit == istart) {
+ /* previous line was empty */
tprint_nl("(Please type a command or say `\\end')");
}
print_ln();
first = istart;
- prompt_input("*"); /* input on-line into |buffer| */
+ prompt_input("*"); /* input on-line into |buffer| */
ilimit = last;
if (end_line_char_inactive)
ilimit--;
@@ -1362,9 +1770,11 @@ static next_line_retval next_line(void)
first = ilimit + 1;
iloc = istart;
} else {
+ /*
+ Nonstop mode, which is intended for overnight batch processing,
+ never waits for on-line input.
+ */
fatal_error("*** (job aborted, no legal \\end found)");
- /* nonstop mode, which is intended for overnight batch processing,
- never waits for on-line input */
}
}
return next_line_ok;
@@ -1375,18 +1785,20 @@ static next_line_retval next_line(void)
@c
static boolean get_next_tokenlist(void)
{
- register halfword t; /* a token */
- t = token_info(iloc);
- iloc = token_link(iloc); /* move to next */
- if (t >= cs_token_flag) { /* a control sequence token */
+ register halfword t = token_info(iloc);
+ iloc = token_link(iloc); /* move to next */
+ if (t >= cs_token_flag) {
+ /* a control sequence token */
cur_cs = t - cs_token_flag;
cur_cmd = eq_type(cur_cs);
if (cur_cmd >= outer_call_cmd) {
- if (cur_cmd == dont_expand_cmd) { /* Get the next token, suppressing expansion */
- /* The present point in the program is reached only when the |expand|
- routine has inserted a special marker into the input. In this special
- case, |token_info(iloc)| is known to be a control sequence token, and |token_link(iloc)=null|.
- */
+ if (cur_cmd == dont_expand_cmd) {
+ /*
+ Get the next token, suppressing expansion. The present point in the program
+ is reached only when the |expand| routine has inserted a special marker into
+ the input. In this special case, |token_info(iloc)| is known to be a control
+ sequence token, and |token_link(iloc)=null|.
+ */
cur_cs = token_info(iloc) - cs_token_flag;
iloc = null;
cur_cmd = eq_type(cur_cs);
@@ -1395,7 +1807,7 @@ static boolean get_next_tokenlist(void)
cur_chr = no_expand_flag;
return true;
}
- } else {
+ } else if (! suppress_outer_error) {
check_outer_validity();
}
}
@@ -1404,24 +1816,25 @@ static boolean get_next_tokenlist(void)
cur_cmd = token_cmd(t);
cur_chr = token_chr(t);
switch (cur_cmd) {
- case left_brace_cmd:
- align_state++;
- break;
- case right_brace_cmd:
- align_state--;
- break;
- case out_param_cmd: /* Insert macro parameter and |goto restart|; */
- begin_token_list(param_stack[param_start + cur_chr - 1], parameter);
- return false;
- break;
+ case left_brace_cmd:
+ align_state++;
+ break;
+ case right_brace_cmd:
+ align_state--;
+ break;
+ case out_param_cmd:
+ /* Insert macro parameter and |goto restart|; */
+ begin_token_list(param_stack[param_start + cur_chr - 1], parameter);
+ return false;
+ break;
}
}
return true;
}
-@ Now we're ready to take the plunge into |get_next| itself. Parts of
- this routine are executed more often than any other instructions of \TeX.
- @^mastication@>@^inner loop@>
+@ Now we're ready to take the plunge into |get_next| itself. Parts of this
+routine are executed more often than any other instructions of \TeX.
+@^mastication@>@^inner loop@>
@ sets |cur_cmd|, |cur_chr|, |cur_cs| to next token
@@ -1449,34 +1862,35 @@ void get_next(void)
}
}
-@ Since |get_next| is used so frequently in \TeX, it is convenient
-to define three related procedures that do a little more:
+@ Since |get_next| is used so frequently in \TeX, it is convenient to define
+three related procedures that do a little more:
+
+\yskip\hang|get_token| not only sets |cur_cmd| and |cur_chr|, it also sets
+|cur_tok|, a packed halfword version of the current token.
-\yskip\hang|get_token| not only sets |cur_cmd| and |cur_chr|, it
-also sets |cur_tok|, a packed halfword version of the current token.
+\yskip\hang|get_x_token|, meaning ``get an expanded token,'' is like |get_token|,
+but if the current token turns out to be a user-defined control sequence (i.e., a
+macro call), or a conditional, or something like \.{\\topmark} or
+\.{\\expandafter} or \.{\\csname}, it is eliminated from the input by beginning
+the expansion of the macro or the evaluation of the conditional.
-\yskip\hang|get_x_token|, meaning ``get an expanded token,'' is like
-|get_token|, but if the current token turns out to be a user-defined
-control sequence (i.e., a macro call), or a conditional,
-or something like \.{\\topmark} or \.{\\expandafter} or \.{\\csname},
-it is eliminated from the input by beginning the expansion of the macro
-or the evaluation of the conditional.
+\yskip\hang|x_token| is like |get_x_token| except that it assumes that |get_next|
+has already been called.
-\yskip\hang|x_token| is like |get_x_token| except that it assumes that
-|get_next| has already been called.
+\yskip\noindent In fact, these three procedures account for almost every use of
+|get_next|.
-\yskip\noindent
-In fact, these three procedures account for almost every use of |get_next|.
+No new control sequences will be defined except during a call of |get_token|, or
+when \.{\\csname} compresses a token list, because |no_new_control_sequence| is
+always |true| at other times.
-No new control sequences will be defined except during a call of
-|get_token|, or when \.{\\csname} compresses a token list, because
-|no_new_control_sequence| is always |true| at other times.
+@ sets |cur_cmd|, |cur_chr|, |cur_tok|
@c
void get_token(void)
-{ /* sets |cur_cmd|, |cur_chr|, |cur_tok| */
+{
no_new_control_sequence = false;
- get_next(); /* get_token_lua(); */
+ get_next();
no_new_control_sequence = true;
if (cur_cs == 0)
cur_tok = token_val(cur_cmd, cur_chr);
@@ -1484,32 +1898,14 @@ void get_token(void)
cur_tok = cs_token_flag + cur_cs;
}
-@ @c
-void get_token_lua(void)
-{
- register int callback_id;
- callback_id = callback_defined(token_filter_callback);
- if (callback_id > 0) {
- while (istate == token_list && iloc == null && iindex != v_template)
- end_token_list();
- /* there is some stuff we don't want to see inside the callback */
- if (!(istate == token_list &&
- ((nofilter == true) || (iindex == backed_up && iloc != null)))) {
- do_get_token_lua(callback_id);
- return;
- }
- }
- get_next();
-}
-
-
@ changes the string |s| to a token list
+
@c
halfword string_to_toks(const char *ss)
{
- halfword p; /* tail of the token list */
- halfword q; /* new node being added to the token list via |store_new_token| */
- halfword t; /* token being appended */
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
const char *s = ss;
const char *se = ss + strlen(s);
p = temp_token_head;
@@ -1526,27 +1922,29 @@ halfword string_to_toks(const char *ss)
return token_link(temp_token_head);
}
-@ The token lists for macros and for other things like \.{\\mark} and \.{\\output}
-and \.{\\write} are produced by a procedure called |scan_toks|.
+@ The token lists for macros and for other things like \.{\\mark} and
+\.{\\output} and \.{\\write} are produced by a procedure called |scan_toks|.
-Before we get into the details of |scan_toks|, let's consider a much
-simpler task, that of converting the current string into a token list.
-The |str_toks| function does this; it classifies spaces as type |spacer|
-and everything else as type |other_char|.
+Before we get into the details of |scan_toks|, let's consider a much simpler
+task, that of converting the current string into a token list. The |str_toks|
+function does this; it classifies spaces as type |spacer| and everything else as
+type |other_char|.
The token list created by |str_toks| begins at |link(temp_token_head)| and ends
at the value |p| that is returned. (If |p=temp_token_head|, the list is empty.)
-|lua_str_toks| is almost identical, but it also escapes the three
-symbols that |lua| considers special while scanning a literal string
+|lua_str_toks| is almost identical, but it also escapes the three symbols that
+|lua| considers special while scanning a literal string
+
+@ changes the string |str_pool[b..pool_ptr]| to a token list
@c
halfword lua_str_toks(lstring b)
-{ /* changes the string |str_pool[b..pool_ptr]| to a token list */
- halfword p; /* tail of the token list */
- halfword q; /* new node being added to the token list via |store_new_token| */
- halfword t; /* token being appended */
- unsigned char *k; /* index into string */
+{
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
+ unsigned char *k; /* index into string */
p = temp_token_head;
set_token_link(p, null);
k = (unsigned char *) b.s;
@@ -1569,16 +1967,18 @@ halfword lua_str_toks(lstring b)
return p;
}
-@ Incidentally, the main reason for wanting |str_toks| is the function |the_toks|,
-which has similar input/output characteristics.
+@ Incidentally, the main reason for wanting |str_toks| is the function
+|the_toks|, which has similar input/output characteristics.
+
+@ changes the string |str_pool[b..pool_ptr]| to a token list
@c
halfword str_toks(lstring s)
-{ /* changes the string |str_pool[b..pool_ptr]| to a token list */
- halfword p; /* tail of the token list */
- halfword q; /* new node being added to the token list via |store_new_token| */
- halfword t; /* token being appended */
- unsigned char *k, *l; /* index into string */
+{
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
+ unsigned char *k, *l; /* index into string */
p = temp_token_head;
set_token_link(p, null);
k = s.s;
@@ -1602,11 +2002,11 @@ halfword str_toks(lstring s)
*/
halfword str_scan_toks(int ct, lstring s)
-{ /* changes the string |str_pool[b..pool_ptr]| to a token list */
- halfword p; /* tail of the token list */
- halfword q; /* new node being added to the token list via |store_new_token| */
- halfword t; /* token being appended */
- unsigned char *k, *l; /* index into string */
+{ /* changes the string |str_pool[b..pool_ptr]| to a token list */
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ halfword t; /* token being appended */
+ unsigned char *k, *l; /* index into string */
int cc;
p = temp_token_head;
set_token_link(p, null);
@@ -1667,6 +2067,7 @@ halfword str_scan_toks(int ct, lstring s)
}
@ Here's part of the |expand| subroutine that we are now ready to complete:
+
@c
void ins_the_toks(void)
{
@@ -1674,8 +2075,133 @@ void ins_the_toks(void)
ins_list(token_link(temp_token_head));
}
-@ This routine, used in the next one, prints the job name, possibly
-modified by the |process_jobname| callback.
+#define set_toks_register(n,t,g) { \
+ int a = (g>0) ? 4 : 0; \
+ halfword ref = get_avail(); \
+ set_token_ref_count(ref, 0); \
+ set_token_link(ref, token_link(t)); \
+ define(n + toks_base, call_cmd, ref); \
+}
+
+void combine_the_toks(int how)
+{
+ halfword nt;
+ get_x_token();
+ /* target */
+ if (cur_cmd == assign_toks_cmd) {
+ nt = equiv(cur_cs) - toks_base;
+ /* check range */
+ } else {
+ back_input();
+ scan_int();
+ nt = cur_val;
+ }
+ /* source */
+ do {
+ get_x_token();
+ } while (cur_cmd == spacer_cmd);
+ if (cur_cmd == left_brace_cmd) {
+ halfword x, source;
+ back_input();
+ x = scan_toks(false,how > 1); /* expanded or not */
+ source = def_ref;
+ /* action */
+ if (source != null) {
+ halfword target = toks(nt);
+ if (target == null) {
+ set_toks_register(nt,source,0);
+ } else {
+ halfword s = token_link(source);
+ if (s != null) {
+ halfword t = token_link(target);
+ if (t == null) {
+ /* can this happen ? */
+ set_token_link(target, s);
+ } else if (odd(how)) {
+ /* prepend */
+ if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
+ halfword p = temp_token_head;
+ halfword q;
+ set_token_link(p, s); /* s = head, x = tail */
+ p = x;
+ while (t != null) {
+ fast_store_new_token(token_info(t));
+ t = token_link(t);
+ }
+ set_toks_register(nt,temp_token_head,0);
+ } else {
+ set_token_link(x,t);
+ set_token_link(target,s);
+ }
+ } else {
+ /* append */
+ if (cur_level != eq_level_field(eqtb[toks_base+nt])) {
+ halfword p = temp_token_head;
+ halfword q;
+ set_token_link(p, null);
+ while (t != null) {
+ fast_store_new_token(token_info(t));
+ t = token_link(t);
+ }
+ set_token_link(p,s);
+ set_toks_register(nt,temp_token_head,0);
+ } else {
+ while (token_link(t) != null) {
+ t = token_link(t);
+ }
+ set_token_link(t,s);
+ }
+ }
+ }
+ }
+ }
+ } else {
+ halfword source, ns;
+ if (cur_cmd == assign_toks_cmd) {
+ ns = equiv(cur_cs) - toks_base;
+ /* check range */
+ } else {
+ back_input();
+ scan_int();
+ ns = cur_val;
+ }
+ /* action */
+ source = toks(ns);
+ if (source != null) {
+ halfword target = toks(nt);
+ if (target == null) {
+ equiv(toks_base+nt) = source;
+ equiv(toks_base+ns) = null;
+ } else {
+ halfword s = token_link(source);
+ if (s != null) {
+ halfword t = token_link(target);
+ if (t == null) {
+ set_token_link(target, s);
+ } else if (odd(how)) {
+ /* prepend */
+ halfword x = s;
+ while (token_link(x) != null) {
+ x = token_link(x);
+ }
+ set_token_link(x,t);
+ set_token_link(target,s);
+ } else {
+ /* append */
+ while (token_link(t) != null) {
+ t = token_link(t);
+ }
+ set_token_link(t,s);
+ }
+ equiv(toks_base+ns) = null;
+ }
+ }
+ }
+ }
+}
+
+@ This routine, used in the next one, prints the job name, possibly modified by
+the |process_jobname| callback.
@c
static void print_job_name(void)
@@ -1696,21 +2222,23 @@ static void print_job_name(void)
}
}
-@ Here is a routine that print the result of a convert command, using
- the argument |i|. It returns |false | if it does not know to print
- the code |c|. The function exists because lua code and tex code can
- both call it to convert something.
+@ Here is a routine that print the result of a convert command, using the
+argument |i|. It returns |false | if it does not know to print the code |c|. The
+function exists because lua code and tex code can both call it to convert
+something.
-@ @c
-int scan_lua_state(void) /* hh-ls: optional name or number (not optional name optional number) */
+@ Parse optional lua state integer, or an instance name to be stored in |sn| and
+get the next non-blank non-relax non-call token.
+
+@c
+
+int scan_lua_state(void)
{
- /* Parse optional lua state integer, or an instance name to be stored in |sn| */
- /* Get the next non-blank non-relax non-call token */
int sn = 0;
do {
get_x_token();
} while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd));
- back_input(); /* have to push it back, whatever it is */
+ back_input();
if (cur_cmd != left_brace_cmd) {
if (scan_keyword("name")) {
(void) scan_toks(false, true);
@@ -1724,13 +2252,13 @@ int scan_lua_state(void) /* hh-ls: optional name or number (not optional name op
return sn;
}
-@ The procedure |conv_toks| uses |str_toks| to insert the token list
-for |convert| functions into the scanner; `\.{\\outer}' control sequences
-are allowed to follow `\.{\\string}' and `\.{\\meaning}'.
+@ The procedure |conv_toks| uses |str_toks| to insert the token list for
+|convert| functions into the scanner; `\.{\\outer}' control sequences are allowed
+to follow `\.{\\string}' and `\.{\\meaning}'.
-The extra temp string |u| is needed because |pdf_scan_ext_toks| incorporates
-any pending string in its output. In order to save such a pending string,
-we have to create a temporary string that is destroyed immediately after.
+The extra temp string |u| is needed because |pdf_scan_ext_toks| incorporates any
+pending string in its output. In order to save such a pending string, we have to
+create a temporary string that is destroyed immediately after.
@c
#define push_selector { \
@@ -1783,7 +2311,6 @@ static int do_variable_pdf(halfword c)
else if (scan_keyword("objcompresslevel")) { do_variable_backend_int(c_pdf_objcompresslevel); }
else if (scan_keyword("inclusioncopyfonts")) { do_variable_backend_int(c_pdf_inclusion_copy_font); }
else if (scan_keyword("gentounicode")) { do_variable_backend_int(c_pdf_gen_tounicode); }
- else if (scan_keyword("replacefont")) { do_variable_backend_int(c_pdf_replace_font); }
else if (scan_keyword("horigin")) { do_variable_backend_dimen(d_pdf_h_origin); }
else if (scan_keyword("vorigin")) { do_variable_backend_dimen(d_pdf_v_origin); }
@@ -1900,7 +2427,7 @@ static int do_feedback_pdf(halfword c)
save_warning_index = warning_index;
save_def_ref = def_ref;
u = save_cur_string();
- scan_toks(false, true); /*hh-ls was scan_pdf_ext_toks();*/
+ scan_toks(false, true);
s = tokens_to_string(def_ref);
delete_token_ref(def_ref);
def_ref = save_def_ref;
@@ -1951,313 +2478,306 @@ void conv_toks(void)
int i = 0;
/* Scan the argument for command |c| */
switch (c) {
- case uchar_code:
- scan_char_num();
- push_selector;
- print(cur_val);
- pop_selector;
- break;
- case number_code:
- scan_int();
- push_selector;
- print_int(cur_val);
- pop_selector;
- break;
- case roman_numeral_code:
- scan_int();
- push_selector;
- print_roman_int(cur_val);
- pop_selector;
- break;
- case string_code:
- save_scanner_status = scanner_status;
- scanner_status = normal;
- get_token();
- scanner_status = save_scanner_status;
- push_selector;
- if (cur_cs != 0)
- sprint_cs(cur_cs);
- else
- print(cur_chr);
- pop_selector;
- break;
- case meaning_code:
- save_scanner_status = scanner_status;
- scanner_status = normal;
- get_token();
- scanner_status = save_scanner_status;
- push_selector;
- print_meaning();
- pop_selector;
- break;
- case etex_code:
- push_selector;
- tprint(eTeX_version_string);
- pop_selector;
- break;
- case font_name_code:
- scan_font_ident();
- push_selector;
- append_string((unsigned char *) font_name(cur_val),(unsigned) strlen(font_name(cur_val)));
- if (font_size(cur_val) != font_dsize(cur_val)) {
- tprint(" at ");
- print_scaled(font_size(cur_val));
- tprint("pt");
- }
- pop_selector;
- break;
- case font_id_code:
- scan_font_ident();
- push_selector;
- print_int(cur_val);
- pop_selector;
- break;
- case luatex_revision_code:
- push_selector;
- print(get_luatexrevision());
- pop_selector;
- break;
- case luatex_date_code:
- push_selector;
- print_int(get_luatex_date_info());
- pop_selector;
- break;
- case luatex_banner_code:
- push_selector;
- tprint(luatex_banner);
- pop_selector;
- break;
- case left_margin_kern_code:
- scan_int();
- if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
- normal_error("marginkern", "a non-empty hbox expected");
- push_selector;
- p = list_ptr(box(cur_val));
- while ((p != null) && (type(p) == glue_node)) {
- p = vlink(p);
- }
- if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == left_side))
- print_scaled(width(p));
- else
- print_char('0');
- tprint("pt");
- pop_selector;
- break;
- case right_margin_kern_code:
- scan_int();
- if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
- normal_error("marginkern", "a non-empty hbox expected");
- push_selector;
- p = list_ptr(box(cur_val));
- if (p != null) {
- p = tail_of_list(p);
- /*
- there can be a leftskip, rightskip, penalty and yes, also a disc node with a nesting
- node that points to glue spec ... and we don't want to analyze that messy lot
- */
- while ((p != null) && (type(p) == glue_node)) {
- p = alink(p);
- }
- if ((p != null) && ! ((type(p) == margin_kern_node) && (subtype(p) == right_side))) {
- if (type(p) == disc_node) {
- q = alink(p);
- if ((q != null) && ((type(q) == margin_kern_node) && (subtype(q) == right_side))) {
- p = q;
- } else {
- /*
- officially we should look in the replace but currently protrusion doesn't
- work anyway with "foo\discretionary{}{}{bar-} " (no following char) so we
- don't need it now
- */
- }
- }
+ case number_code:
+ scan_int();
+ push_selector;
+ print_int(cur_val);
+ pop_selector;
+ break;
+ case lua_function_code:
+ scan_int();
+ if (cur_val <= 0) {
+ normal_error("luafunction", "invalid number");
+ } else {
+ u = save_cur_string();
+ luacstrings = 0;
+ luafunctioncall(cur_val);
+ restore_cur_string(u);
+ if (luacstrings > 0)
+ lua_string_start();
}
- }
- if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == right_side))
- print_scaled(width(p));
- else
- print_char('0');
- tprint("pt");
- pop_selector;
- break;
- case format_name_code:
- if (job_name == 0)
- open_log_file();
- push_selector;
- print(format_name);
- pop_selector;
- break;
- case job_name_code:
- if (job_name == 0)
- open_log_file();
- push_selector;
- print_job_name();
- pop_selector;
- break;
- case uniform_deviate_code:
- scan_int();
- push_selector;
- print_int(unif_rand(cur_val));
- pop_selector;
- break;
- case normal_deviate_code:
- scan_int();
- push_selector;
- print_int(norm_rand());
- pop_selector;
- break;
- case lua_escape_string_code:
- {
- lstring escstr;
- int l = 0;
+ /* no further action */
+ return;
+ break;
+ case lua_code:
+ u = save_cur_string();
save_scanner_status = scanner_status;
save_def_ref = def_ref;
save_warning_index = warning_index;
- scan_toks(false, true); /*hh-ls was scan_pdf_ext_toks();*/
- bool = in_lua_escape;
- in_lua_escape = true;
- escstr.s = (unsigned char *) tokenlist_to_cstring(def_ref, false, &l);
- escstr.l = (unsigned) l;
- in_lua_escape = bool;
- delete_token_ref(def_ref);
+ sn = scan_lua_state();
+ scan_toks(false, true);
+ s = def_ref;
+ warning_index = save_warning_index;
def_ref = save_def_ref;
+ scanner_status = save_scanner_status;
+ luacstrings = 0;
+ luatokencall(s, sn);
+ delete_token_ref(s);
+ restore_cur_string(u); /* TODO: check this, was different */
+ if (luacstrings > 0)
+ lua_string_start();
+ /* no further action */
+ return;
+ break;
+ case expanded_code:
+ save_scanner_status = scanner_status;
+ save_warning_index = warning_index;
+ save_def_ref = def_ref;
+ u = save_cur_string();
+ scan_toks(false, true);
warning_index = save_warning_index;
scanner_status = save_scanner_status;
- (void) lua_str_toks(escstr);
- ins_list(token_link(temp_token_head));
- free(escstr.s);
+ ins_list(token_link(def_ref));
+ def_ref = save_def_ref;
+ restore_cur_string(u);
+ /* no further action */
return;
- }
- /* no further action */
- break;
- case math_style_code:
- push_selector;
- print_math_style();
- pop_selector;
- break;
- case math_char_class_code:
- {
- mathcodeval mval;
+ break;
+ case math_style_code:
+ push_selector;
+ print_math_style();
+ pop_selector;
+ break;
+ case string_code:
+ save_scanner_status = scanner_status;
+ scanner_status = normal;
+ get_token();
+ scanner_status = save_scanner_status;
+ push_selector;
+ if (cur_cs != 0)
+ sprint_cs(cur_cs);
+ else
+ print(cur_chr);
+ pop_selector;
+ break;
+ case cs_string_code:
+ save_scanner_status = scanner_status;
+ scanner_status = normal;
+ get_token();
+ scanner_status = save_scanner_status;
+ push_selector;
+ if (cur_cs != 0)
+ sprint_cs_name(cur_cs);
+ else
+ print(cur_chr);
+ pop_selector;
+ break;
+ case roman_numeral_code:
scan_int();
- mval = get_math_code(cur_val);
push_selector;
- print_int(mval.class_value);
+ print_roman_int(cur_val);
pop_selector;
- }
- break;
- case math_char_fam_code:
- {
- mathcodeval mval;
+ break;
+ case meaning_code:
+ save_scanner_status = scanner_status;
+ scanner_status = normal;
+ get_token();
+ scanner_status = save_scanner_status;
+ push_selector;
+ print_meaning();
+ pop_selector;
+ break;
+ case uchar_code:
+ scan_char_num();
+ push_selector;
+ print(cur_val);
+ pop_selector;
+ break;
+ case lua_escape_string_code:
+ {
+ lstring escstr;
+ int l = 0;
+ save_scanner_status = scanner_status;
+ save_def_ref = def_ref;
+ save_warning_index = warning_index;
+ scan_toks(false, true);
+ bool = in_lua_escape;
+ in_lua_escape = true;
+ escstr.s = (unsigned char *) tokenlist_to_cstring(def_ref, false, &l);
+ escstr.l = (unsigned) l;
+ in_lua_escape = bool;
+ delete_token_ref(def_ref);
+ def_ref = save_def_ref;
+ warning_index = save_warning_index;
+ scanner_status = save_scanner_status;
+ (void) lua_str_toks(escstr);
+ ins_list(token_link(temp_token_head));
+ free(escstr.s);
+ return;
+ }
+ /* no further action */
+ break;
+ case font_id_code:
+ scan_font_ident();
+ push_selector;
+ print_int(cur_val);
+ pop_selector;
+ break;
+ case font_name_code:
+ scan_font_ident();
+ push_selector;
+ append_string((unsigned char *) font_name(cur_val),(unsigned) strlen(font_name(cur_val)));
+ if (font_size(cur_val) != font_dsize(cur_val)) {
+ tprint(" at ");
+ print_scaled(font_size(cur_val));
+ tprint("pt");
+ }
+ pop_selector;
+ break;
+ case left_margin_kern_code:
scan_int();
- mval = get_math_code(cur_val);
+ if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
+ normal_error("marginkern", "a non-empty hbox expected");
push_selector;
- print_int(mval.family_value);
+ p = list_ptr(box(cur_val));
+ while ((p != null) && (type(p) == glue_node)) {
+ p = vlink(p);
+ }
+ if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == left_side))
+ print_scaled(width(p));
+ else
+ print_char('0');
+ tprint("pt");
pop_selector;
- }
- break;
- case math_char_slot_code:
- {
- mathcodeval mval;
+ break;
+ case right_margin_kern_code:
scan_int();
- mval = get_math_code(cur_val);
+ if ((box(cur_val) == null) || (type(box(cur_val)) != hlist_node))
+ normal_error("marginkern", "a non-empty hbox expected");
push_selector;
- print_int(mval.character_value);
+ p = list_ptr(box(cur_val));
+ if (p != null) {
+ p = tail_of_list(p);
+ /*
+ there can be a leftskip, rightskip, penalty and yes, also a disc node with a nesting
+ node that points to glue spec ... and we don't want to analyze that messy lot
+ */
+ while ((p != null) && (type(p) == glue_node)) {
+ p = alink(p);
+ }
+ if ((p != null) && ! ((type(p) == margin_kern_node) && (subtype(p) == right_side))) {
+ if (type(p) == disc_node) {
+ q = alink(p);
+ if ((q != null) && ((type(q) == margin_kern_node) && (subtype(q) == right_side))) {
+ p = q;
+ } else {
+ /*
+ officially we should look in the replace but currently protrusion doesn't
+ work anyway with "foo\discretionary{}{}{bar-} " (no following char) so we
+ don't need it now
+ */
+ }
+ }
+ }
+ }
+ if ((p != null) && (type(p) == margin_kern_node) && (subtype(p) == right_side))
+ print_scaled(width(p));
+ else
+ print_char('0');
+ tprint("pt");
pop_selector;
- }
- break;
- case expanded_code:
- save_scanner_status = scanner_status;
- save_warning_index = warning_index;
- save_def_ref = def_ref;
- u = save_cur_string();
- scan_toks(false, true); /*hh-ls was scan_pdf_ext_toks();*/
- warning_index = save_warning_index;
- scanner_status = save_scanner_status;
- ins_list(token_link(def_ref));
- def_ref = save_def_ref;
- restore_cur_string(u);
- /* no further action */
- return;
- break;
- case lua_code:
- u = save_cur_string();
- save_scanner_status = scanner_status;
- save_def_ref = def_ref;
- save_warning_index = warning_index;
- sn = scan_lua_state();
- scan_toks(false, true); /*hh-ls was scan_pdf_ext_toks();*/
- s = def_ref;
- warning_index = save_warning_index;
- def_ref = save_def_ref;
- scanner_status = save_scanner_status;
- luacstrings = 0;
- luatokencall(s, sn);
- delete_token_ref(s);
- restore_cur_string(u); /* TODO: check this, was different */
- if (luacstrings > 0)
- lua_string_start();
- /* no further action */
- return;
- break;
- case lua_function_code:
- scan_int();
- if (cur_val <= 0) {
- normal_error("luafunction", "invalid number");
- } else {
- u = save_cur_string();
- luacstrings = 0;
- luafunctioncall(cur_val);
- restore_cur_string(u);
- if (luacstrings > 0)
- lua_string_start();
- }
- /* no further action */
- return;
- break;
- case eTeX_revision_code:
- push_selector;
- tprint(eTeX_revision);
- pop_selector;
- break;
- case insert_ht_code:
- scan_register_num();
- push_selector;
- i = cur_val;
- p = page_ins_head;
- while (i >= subtype(vlink(p)))
- p = vlink(p);
- if (subtype(p) == i)
- print_scaled(height(p));
- else
- print_char('0');
- tprint("pt");
- pop_selector;
- break;
- /*
- case dvi_feedback_code:
- if (get_o_mode() == OMODE_DVI)
- done = do_feedback_dvi(c);
- else
- done = 0;
- if (done==0)
- tex_error("unexpected use of \\dvifeedback",null);
- else if (done==2)
- return;
- break;
- case pdf_feedback_code:
- if (get_o_mode() == OMODE_PDF)
- done = conv_toks_pdf(c);
- else
- done = 0;
- if (done==0)
- tex_error("unexpected use of \\pdffeedback",null);
- else if (done==2)
- return;
- break;
- */
- default:
- confusion("convert");
- break;
+ break;
+ case uniform_deviate_code:
+ scan_int();
+ push_selector;
+ print_int(unif_rand(cur_val));
+ pop_selector;
+ break;
+ case normal_deviate_code:
+ scan_int();
+ push_selector;
+ print_int(norm_rand());
+ pop_selector;
+ break;
+ case math_char_class_code:
+ {
+ mathcodeval mval;
+ scan_int();
+ mval = get_math_code(cur_val);
+ push_selector;
+ print_int(mval.class_value);
+ pop_selector;
+ }
+ break;
+ case math_char_fam_code:
+ {
+ mathcodeval mval;
+ scan_int();
+ mval = get_math_code(cur_val);
+ push_selector;
+ print_int(mval.family_value);
+ pop_selector;
+ }
+ break;
+ case math_char_slot_code:
+ {
+ mathcodeval mval;
+ scan_int();
+ mval = get_math_code(cur_val);
+ push_selector;
+ print_int(mval.character_value);
+ pop_selector;
+ }
+ break;
+ case insert_ht_code:
+ scan_register_num();
+ push_selector;
+ i = cur_val;
+ p = page_ins_head;
+ while (i >= subtype(vlink(p)))
+ p = vlink(p);
+ if (subtype(p) == i)
+ print_scaled(height(p));
+ else
+ print_char('0');
+ tprint("pt");
+ pop_selector;
+ break;
+ case job_name_code:
+ if (job_name == 0)
+ open_log_file();
+ push_selector;
+ print_job_name();
+ pop_selector;
+ break;
+ case format_name_code:
+ if (job_name == 0)
+ open_log_file();
+ push_selector;
+ print(format_name);
+ pop_selector;
+ break;
+ case luatex_banner_code:
+ push_selector;
+ tprint(luatex_banner);
+ pop_selector;
+ break;
+ case luatex_revision_code:
+ push_selector;
+ print(get_luatexrevision());
+ pop_selector;
+ break;
+ case luatex_date_code:
+ push_selector;
+ print_int(get_luatex_date_info());
+ pop_selector;
+ break;
+ case etex_code:
+ push_selector;
+ tprint(eTeX_version_string);
+ pop_selector;
+ break;
+ case eTeX_revision_code:
+ push_selector;
+ tprint(eTeX_revision);
+ pop_selector;
+ break;
+ case font_identifier_code:
+ confusion("convert");
+ break;
+ default:
+ confusion("convert");
+ break;
}
str = make_string();
(void) str_toks(str_lstring(str));
@@ -2380,49 +2900,62 @@ str_number the_convert_string(halfword c, int i)
case number_code:
print_int(i);
break;
- case uchar_code:
- print(i);
+ /* case lua_function_code: */
+ /* case lua_code: */
+ /* case expanded_code: */
+ case math_style_code:
+ print_math_style();
break;
+ /* case string_code: */
+ /* case cs_string_code: */
case roman_numeral_code:
print_roman_int(i);
break;
- case etex_code:
- tprint(eTeX_version_string);
- break;
- case luatex_revision_code:
- print(get_luatexrevision());
+ /* case meaning_code: */
+ case uchar_code:
+ print(i);
break;
- case luatex_date_code:
- print_int(get_luatex_date_info());
+ /* lua_escape_string_code: */
+ case font_id_code:
+ print_int(i);
break;
- case luatex_banner_code:
- tprint(luatex_banner);
+ case font_name_code:
+ append_string((unsigned char *) font_name(i),(unsigned) strlen(font_name(i)));
+ if (font_size(i) != font_dsize(i)) {
+ tprint(" at ");
+ print_scaled(font_size(i));
+ tprint("pt");
+ }
break;
+ /* left_margin_kern_code: */
+ /* right_margin_kern_code: */
case uniform_deviate_code:
print_int(unif_rand(i));
break;
case normal_deviate_code:
print_int(norm_rand());
break;
+ /* math_char_class_code: */
+ /* math_char_fam_code: */
+ /* math_char_slot_code: */
+ /* insert_ht_code: */
+ case job_name_code:
+ print_job_name();
+ break;
case format_name_code:
print(format_name);
break;
- case job_name_code:
- print_job_name();
+ case luatex_banner_code:
+ tprint(luatex_banner);
break;
- case font_name_code:
- append_string((unsigned char *) font_name(i),(unsigned) strlen(font_name(i)));
- if (font_size(i) != font_dsize(i)) {
- tprint(" at ");
- print_scaled(font_size(i));
- tprint("pt");
- }
+ case luatex_revision_code:
+ print(get_luatexrevision());
break;
- case font_id_code:
- print_int(i);
+ case luatex_date_code:
+ print_int(get_luatex_date_info());
break;
- case math_style_code:
- print_math_style();
+ case etex_code:
+ tprint(eTeX_version_string);
break;
case eTeX_revision_code:
tprint(eTeX_revision);
@@ -2449,16 +2982,16 @@ str_number the_convert_string(halfword c, int i)
return ret;
}
-@ Another way to create a token list is via the \.{\\read} command. The
-sixteen files potentially usable for reading appear in the following
-global variables. The value of |read_open[n]| will be |closed| if
-stream number |n| has not been opened or if it has been fully read;
-|just_open| if an \.{\\openin} but not a \.{\\read} has been done;
-and |normal| if it is open and ready to read the next line.
+@ Another way to create a token list is via the \.{\\read} command. The sixteen
+files potentially usable for reading appear in the following global variables.
+The value of |read_open[n]| will be |closed| if stream number |n| has not been
+opened or if it has been fully read; |just_open| if an \.{\\openin} but not a
+\.{\\read} has been done; and |normal| if it is open and ready to read the next
+line.
@c
-FILE *read_file[16]; /* used for \.{\\read} */
-int read_open[17]; /* state of |read_file[n]| */
+FILE *read_file[16]; /* used for \.{\\read} */
+int read_open[17]; /* state of |read_file[n]| */
void initialize_read(void)
{
@@ -2467,17 +3000,17 @@ void initialize_read(void)
read_open[k] = closed;
}
-@ The |read_toks| procedure constructs a token list like that for any
-macro definition, and makes |cur_val| point to it. Parameter |r| points
-to the control sequence that will receive this token list.
+@ The |read_toks| procedure constructs a token list like that for any macro
+definition, and makes |cur_val| point to it. Parameter |r| points to the control
+sequence that will receive this token list.
@c
void read_toks(int n, halfword r, halfword j)
{
- halfword p; /* tail of the token list */
- halfword q; /* new node being added to the token list via |store_new_token| */
- int s; /* saved value of |align_state| */
- int m; /* stream number */
+ halfword p; /* tail of the token list */
+ halfword q; /* new node being added to the token list via |store_new_token| */
+ int s; /* saved value of |align_state| */
+ int m; /* stream number */
scanner_status = defining;
warning_index = r;
p = get_avail();
@@ -2496,10 +3029,13 @@ void read_toks(int n, halfword r, halfword j)
begin_file_reading();
iname = m + 1;
if (read_open[m] == closed) {
- /* Input for \.{\\read} from the terminal */
- /* Here we input on-line into the |buffer| array, prompting the user explicitly
- if |n>=0|. The value of |n| is set negative so that additional prompts
- will not be given in the case of multi-line input. */
+ /*
+ Input for \.{\\read} from the terminal
+
+ Here we input on-line into the |buffer| array, prompting the user explicitly
+ if |n>=0|. The value of |n| is set negative so that additional prompts
+ will not be given in the case of multi-line input.
+ */
if (interaction > nonstop_mode) {
if (n < 0) {
prompt_input("");
@@ -2516,9 +3052,12 @@ void read_toks(int n, halfword r, halfword j)
}
} else if (read_open[m] == just_open) {
- /* Input the first line of |read_file[m]| */
- /* The first line of a file must be treated specially, since |lua_input_ln|
- must be told not to start with |get|. */
+ /*
+ Input the first line of |read_file[m]|
+
+ The first line of a file must be treated specially, since |lua_input_ln|
+ must be told not to start with |get|.
+ */
if (lua_input_ln(read_file[m], (m + 1), false)) {
read_open[m] = normal;
} else {
@@ -2527,8 +3066,11 @@ void read_toks(int n, halfword r, halfword j)
}
} else {
- /* Input the next line of |read_file[m]| */
- /* An empty line is appended at the end of a |read_file|. */
+ /*
+ Input the next line of |read_file[m]|
+
+ An empty line is appended at the end of a |read_file|.
+ */
if (!lua_input_ln(read_file[m], (m + 1), true)) {
lua_a_close_in(read_file[m], (m + 1));
read_open[m] = closed;
@@ -2552,8 +3094,9 @@ void read_toks(int n, halfword r, halfword j)
istate = new_line;
/* Handle \.{\\readline} and |goto done|; */
if (j == 1) {
- while (iloc <= ilimit) { /* current line not yet finished */
- do_buffer_to_unichar(cur_chr, iloc);
+ while (iloc <= ilimit) {
+ /* current line not yet finished */
+ do_buffer_to_unichar(cur_chr, iloc);
if (cur_chr == ' ')
cur_tok = space_token;
else
@@ -2563,9 +3106,12 @@ void read_toks(int n, halfword r, halfword j)
} else {
while (1) {
get_token();
- if (cur_tok == 0)
- break; /* |cur_cmd=cur_chr=0| will occur at the end of the line */
- if (align_state < 1000000) { /* unmatched `\.\}' aborts the line */
+ if (cur_tok == 0) {
+ /* |cur_cmd=cur_chr=0| will occur at the end of the line */
+ break;
+ }
+ if (align_state < 1000000) {
+ /* unmatched `\.\}' aborts the line */
do {
get_token();
} while (cur_tok != 0);
@@ -2583,9 +3129,11 @@ void read_toks(int n, halfword r, halfword j)
align_state = s;
}
-@ @c
+@ return a string from tokens list
+
+@c
str_number tokens_to_string(halfword p)
-{ /* return a string from tokens list */
+{
int old_setting;
if (selector == new_string)
normal_error("tokens","tokens_to_string() called while selector = new_string");
@@ -2627,7 +3175,6 @@ str_number tokens_to_string(halfword p)
append_i_byte(0x80 + (((s % 0x40000) % 0x1000) % 0x40)); \
} }
-
#define Print_esc(b) { \
const char *v = b; \
if (e>0 && e<STRING_OFFSET) { \
@@ -2646,10 +3193,9 @@ str_number tokens_to_string(halfword p)
#define is_cat_letter(a) \
(get_char_cat_code(pool_to_unichar(str_string((a)))) == 11)
-@ the actual token conversion in this function is now functionally
- equivalent to |show_token_list|, except that it always prints the
- whole token list.
- TODO: check whether this causes problems in the lua library.
+@ the actual token conversion in this function is now functionally equivalent to
+|show_token_list|, except that it always prints the whole token list. TODO: check
+whether this causes problems in the lua library.
@c
char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
@@ -2691,9 +3237,7 @@ char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
} else {
Print_esc("IMPOSSIBLE.");
}
- } else if ((q >= undefined_control_sequence)
- && ((q <= eqtb_size)
- || (q > eqtb_size + hash_extra))) {
+ } else if ((q >= undefined_control_sequence) && ((q <= eqtb_size) || (q > eqtb_size + hash_extra))) {
Print_esc("IMPOSSIBLE.");
} else if ((cs_text(q) < 0) || (cs_text(q) >= str_ptr)) {
Print_esc("NONEXISTENT.");
@@ -2727,48 +3271,48 @@ char *tokenlist_to_cstring(int pp, int inhibit_par, int *siz)
m = token_cmd(infop);
c = token_chr(infop);
switch (m) {
- case left_brace_cmd:
- case right_brace_cmd:
- case math_shift_cmd:
- case tab_mark_cmd:
- case sup_mark_cmd:
- case sub_mark_cmd:
- case spacer_cmd:
- case letter_cmd:
- case other_char_cmd:
- Print_uchar(c);
- break;
- case mac_param_cmd:
- if (!in_lua_escape && (is_in_csname==0))
+ case left_brace_cmd:
+ case right_brace_cmd:
+ case math_shift_cmd:
+ case tab_mark_cmd:
+ case sup_mark_cmd:
+ case sub_mark_cmd:
+ case spacer_cmd:
+ case letter_cmd:
+ case other_char_cmd:
Print_uchar(c);
- Print_uchar(c);
- break;
- case out_param_cmd:
- Print_uchar(match_chr);
- if (c <= 9) {
- Print_char(c + '0');
- } else {
- Print_char('!');
- goto EXIT;
- }
- break;
- case match_cmd:
- match_chr = c;
- Print_uchar(c);
- n++;
- Print_char(n);
- if (n > '9')
- goto EXIT;
- break;
- case end_match_cmd:
- if (c == 0) {
- Print_char('-');
- Print_char('>');
- }
- break;
- default:
- not_so_bad(Print_esc);
- break;
+ break;
+ case mac_param_cmd:
+ if (!in_lua_escape && (is_in_csname==0))
+ Print_uchar(c);
+ Print_uchar(c);
+ break;
+ case out_param_cmd:
+ Print_uchar(match_chr);
+ if (c <= 9) {
+ Print_char(c + '0');
+ } else {
+ Print_char('!');
+ goto EXIT;
+ }
+ break;
+ case match_cmd:
+ match_chr = c;
+ Print_uchar(c);
+ n++;
+ Print_char(n);
+ if (n > '9')
+ goto EXIT;
+ break;
+ case end_match_cmd:
+ if (c == 0) {
+ Print_char('-');
+ Print_char('>');
+ }
+ break;
+ default:
+ not_so_bad(Print_esc);
+ break;
}
}
}