summaryrefslogtreecommitdiff
path: root/Build/source/texk/web2c/luatexdir/tex/textoken.c
diff options
context:
space:
mode:
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/tex/textoken.c')
-rw-r--r--Build/source/texk/web2c/luatexdir/tex/textoken.c1190
1 files changed, 607 insertions, 583 deletions
diff --git a/Build/source/texk/web2c/luatexdir/tex/textoken.c b/Build/source/texk/web2c/luatexdir/tex/textoken.c
index 20c57b9479b..366fa798791 100644
--- a/Build/source/texk/web2c/luatexdir/tex/textoken.c
+++ b/Build/source/texk/web2c/luatexdir/tex/textoken.c
@@ -1,4 +1,4 @@
-/* $Id: textoken.c 1013 2008-02-14 00:09:02Z oneiros $ */
+/* $Id: textoken.c 1155 2008-04-14 07:53:21Z oneiros $ */
#include "luatex-api.h"
#include <ptexlib.h>
@@ -7,18 +7,18 @@
/* Integer parameters and other command-related defines. This needs it's own header file! */
-#define end_line_char_code 48 /* character placed at the right end of the buffer */
+#define end_line_char_code 48 /* character placed at the right end of the buffer */
#define cat_code_table_code 63
-#define tex_int_pars 66 /* total number of \.{\\TeX} + Aleph integer parameters */
+#define tex_int_pars 66 /* total number of \.{\\TeX} + Aleph integer parameters */
/* this is not what happens in the pascal code! there the values shift from bare numbers to offsets ! */
#define dir_base tex_int_pars
#define dir_pars 5
-#define pdftex_first_integer_code dir_base+dir_pars /* base for \pdfTeX's integer parameters */
-#define pdf_int_pars pdftex_first_integer_code+27 /*total number of \pdfTeX's integer parameters*/
-#define etex_int_base pdf_int_pars /*base for \eTeX's integer parameters*/
-#define tracing_nesting_code etex_int_base+4 /*show incomplete groups and ifs within files*/
+#define pdftex_first_integer_code dir_base+dir_pars /* base for \pdfTeX's integer parameters */
+#define pdf_int_pars pdftex_first_integer_code+27 /*total number of \pdfTeX's integer parameters */
+#define etex_int_base pdf_int_pars /*base for \eTeX's integer parameters */
+#define tracing_nesting_code etex_int_base+4 /*show incomplete groups and ifs within files */
-#define int_par(a) zeqtb[static_int_base+a].cint /* an integer parameter */
+#define int_par(a) zeqtb[static_int_base+a].cint /* an integer parameter */
#define cat_code_table int_par(cat_code_table_code)
#define tracing_nesting int_par(tracing_nesting_code)
#define end_line_char int_par(end_line_char_code)
@@ -26,7 +26,7 @@
#define every_eof get_every_eof()
#define number_active_chars 65536+65536
-#define active_base 1 /* beginning of region 1, for active character equivalents */
+#define active_base 1 /* beginning of region 1, for active character equivalents */
#define null_cs active_base+number_active_chars /* equivalent of \.{\\csname\\endcsname} */
@@ -40,18 +40,18 @@
#define nonstop_mode 1
/* command codes */
-#define relax 0
+#define relax 0
#define out_param 5
-#define max_command 117 /* fetched from C output */
-#define dont_expand 133 /* fetched from C output */
+#define max_command 117 /* fetched from C output */
+#define dont_expand 133 /* fetched from C output */
-#define terminal_input (name==0) /* are we reading from the terminal? */
-#define special_char 1114113 /* |biggest_char+2| */
-#define cur_file input_file[index] /* the current |alpha_file| variable */
+#define terminal_input (name==0) /* are we reading from the terminal? */
+#define special_char 1114113 /* |biggest_char+2| */
+#define cur_file input_file[index] /* the current |alpha_file| variable */
-#define no_expand_flag special_char /*this characterizes a special variant of |relax|*/
+#define no_expand_flag special_char /*this characterizes a special variant of |relax| */
-extern void insert_vj_template (void);
+extern void insert_vj_template(void);
#define do_get_cat_code(a) do { \
if (local_catcode_table) \
@@ -61,219 +61,223 @@ extern void insert_vj_template (void);
} while (0)
-static void
-invalid_character_error (void) {
- char *hlp[] = {"A funny symbol that I can't read has just been input.",
- "Continue, and I'll forget that it ever happened.",
- NULL};
- deletions_allowed=false;
- tex_error("Text line contains an invalid character", hlp);
- deletions_allowed=true;
+static void invalid_character_error(void)
+{
+ char *hlp[] = { "A funny symbol that I can't read has just been input.",
+ "Continue, and I'll forget that it ever happened.",
+ NULL
+ };
+ deletions_allowed = false;
+ tex_error("Text line contains an invalid character", hlp);
+ deletions_allowed = true;
}
/* no longer done */
-static boolean process_sup_mark(void); /* below */
+static boolean process_sup_mark(void); /* below */
static int scan_control_sequence(void); /* below */
-typedef enum {next_line_ok, next_line_return, next_line_restart } next_line_retval ;
+typedef enum { next_line_ok, next_line_return,
+ next_line_restart } next_line_retval;
-static next_line_retval next_line(void); /* below */
+static next_line_retval next_line(void); /* below */
/* @^inner loop@>*/
-static void
-utf_error (void) {
- char *hlp[] = {"A funny symbol that I can't read has just been input.",
- "Just continue, I'll change it to 0xFFFD.",
- NULL };
- deletions_allowed=false;
- tex_error("Text line contains an invalid utf-8 sequence", hlp);
- deletions_allowed=true;
+static void utf_error(void)
+{
+ char *hlp[] = { "A funny symbol that I can't read has just been input.",
+ "Just continue, I'll change it to 0xFFFD.",
+ NULL
+ };
+ deletions_allowed = false;
+ tex_error("Text line contains an invalid utf-8 sequence", hlp);
+ deletions_allowed = true;
}
-static integer
-qbuffer_to_unichar (integer *k) {
- register int ch;
- int val = 0xFFFD;
- unsigned char *text = buffer+*k;
- if ( (ch = *text++)<0x80 ) {
- val = ch;
- *k+=1;
- } else if ( ch<=0xbf ) { /* error */
- *k+=1;
- } else if ( ch<=0xdf ) {
- if ( *text>=0x80 && *text<0xc0 )
- val = ((ch&0x1f)<<6) | (*text++&0x3f);
- *k+=2;
- } else if ( ch<=0xef ) {
- if ( *text>=0x80 && *text<0xc0 && text[1]>=0x80 && text[1]<0xc0 ) {
- val = ((ch&0xf)<<12) | ((text[0]&0x3f)<<6) | (text[1]&0x3f);
- *k+=3;
+static integer qbuffer_to_unichar(integer * k)
+{
+ register int ch;
+ int val = 0xFFFD;
+ unsigned char *text = buffer + *k;
+ if ((ch = *text++) < 0x80) {
+ val = ch;
+ *k += 1;
+ } else if (ch <= 0xbf) { /* error */
+ *k += 1;
+ } else if (ch <= 0xdf) {
+ if (*text >= 0x80 && *text < 0xc0)
+ val = ((ch & 0x1f) << 6) | (*text++ & 0x3f);
+ *k += 2;
+ } else if (ch <= 0xef) {
+ if (*text >= 0x80 && *text < 0xc0 && text[1] >= 0x80 && text[1] < 0xc0) {
+ val =
+ ((ch & 0xf) << 12) | ((text[0] & 0x3f) << 6) | (text[1] & 0x3f);
+ *k += 3;
+ }
+ } else {
+ int w = (((ch & 0x7) << 2) | ((text[0] & 0x30) >> 4)) - 1, w2;
+ w = (w << 6) | ((text[0] & 0xf) << 2) | ((text[1] & 0x30) >> 4);
+ w2 = ((text[1] & 0xf) << 6) | (text[2] & 0x3f);
+ val = w * 0x400 + w2 + 0x10000;
+ if (*text < 0x80 || text[1] < 0x80 || text[2] < 0x80 ||
+ *text >= 0xc0 || text[1] >= 0xc0 || text[2] >= 0xc0)
+ val = 0xFFFD;
+ *k += 4;
}
- } else {
- int w = ( ((ch&0x7)<<2) | ((text[0]&0x30)>>4) )-1, w2;
- w = (w<<6) | ((text[0]&0xf)<<2) | ((text[1]&0x30)>>4);
- w2 = ((text[1]&0xf)<<6) | (text[2]&0x3f);
- val = w*0x400 + w2 + 0x10000;
- if ( *text<0x80 || text[1]<0x80 || text[2]<0x80 ||
- *text>=0xc0 || text[1]>=0xc0 || text[2]>=0xc0 )
- val = 0xFFFD;
- *k+=4;
- }
- if (val==0xFFFD)
- utf_error();
- return( val );
+ if (val == 0xFFFD)
+ utf_error();
+ return (val);
}
-static boolean
-get_next_file (void) {
- SWITCH:
- if (loc<=limit) { /* current line not yet finished */
- cur_chr = qbuffer_to_unichar(&loc);
- RESWITCH:
- if (detokenized_line) {
- cur_cmd = (cur_chr==' ' ? 10 : 12) ;
+static boolean get_next_file(void)
+{
+ SWITCH:
+ if (loc <= limit) { /* current line not yet finished */
+ cur_chr = qbuffer_to_unichar(&loc);
+ RESWITCH:
+ if (detokenized_line) {
+ cur_cmd = (cur_chr == ' ' ? 10 : 12);
+ } else {
+ do_get_cat_code(cur_cmd);
+ }
+ /*
+ @<Change state if necessary, and |goto switch| if the current
+ character should be ignored, or |goto reswitch| if the current
+ character changes to another@>;
+ */
+ /* The following 48-way switch accomplishes the scanning quickly, assuming
+ that a decent \PASCAL\ compiler has translated the code. Note that the numeric
+ values for |mid_line|, |skip_blanks|, and |new_line| are spaced
+ apart from each other by |max_char_code+1|, so we can add a character's
+ command code to the state to get a single number that characterizes both.
+ */
+ switch (state + cur_cmd) {
+ case mid_line + ignore:
+ case skip_blanks + ignore:
+ case new_line + ignore:
+ case skip_blanks + spacer:
+ case new_line + spacer: /* @<Cases where character is ignored@> */
+ goto SWITCH;
+ break;
+ case mid_line + escape:
+ case new_line + escape:
+ case skip_blanks + escape: /* @<Scan a control sequence ...@>; */
+ state = scan_control_sequence();
+ break;
+ case mid_line + active_char:
+ case new_line + active_char:
+ case skip_blanks + active_char: /* @<Process an active-character */
+ cur_cs = cur_chr + active_base;
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ state = mid_line;
+ break;
+ case mid_line + sup_mark:
+ case new_line + sup_mark:
+ case skip_blanks + sup_mark: /* @<If this |sup_mark| starts */
+ if (process_sup_mark())
+ goto RESWITCH;
+ else
+ state = mid_line;
+ break;
+ case mid_line + invalid_char:
+ case new_line + invalid_char:
+ case skip_blanks + invalid_char: /* @<Decry the invalid character and |goto restart|@>; */
+ invalid_character_error();
+ return false; /* because state may be token_list now */
+ break;
+ case mid_line + spacer: /* @<Enter |skip_blanks| state, emit a space@>; */
+ state = skip_blanks;
+ cur_chr = ' ';
+ break;
+ case mid_line + car_ret: /* @<Finish line, emit a space@>; */
+ /* When a character of type |spacer| gets through, its character code is
+ changed to $\.{"\ "}=@'40$. This means that the ASCII codes for tab and space,
+ and for the space inserted at the end of a line, will
+ be treated alike when macro parameters are being matched. We do this
+ since such characters are indistinguishable on most computer terminal displays.
+ */
+ loc = limit + 1;
+ cur_cmd = spacer;
+ cur_chr = ' ';
+ break;
+ case skip_blanks + car_ret:
+ case mid_line + comment:
+ case new_line + comment:
+ case skip_blanks + comment: /* @<Finish line, |goto switch|@>; */
+ loc = limit + 1;
+ goto SWITCH;
+ break;
+ case new_line + car_ret: /* @<Finish line, emit a \.{\\par}@>; */
+ loc = limit + 1;
+ cur_cs = par_loc;
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ break;
+ case skip_blanks + left_brace:
+ case new_line + left_brace:
+ state = mid_line; /* fall through */
+ case mid_line + left_brace:
+ align_state++;
+ break;
+ case skip_blanks + right_brace:
+ case new_line + right_brace:
+ state = mid_line; /* fall through */
+ case mid_line + right_brace:
+ align_state--;
+ break;
+ case mid_line + math_shift:
+ case mid_line + tab_mark:
+ case mid_line + mac_param:
+ case mid_line + sub_mark:
+ case mid_line + letter:
+ case mid_line + other_char:
+ break;
+ default:
+ /*
+ case skip_blanks + math_shift:
+ case skip_blanks + tab_mark:
+ case skip_blanks + mac_param:
+ case skip_blanks + sub_mark:
+ case skip_blanks + letter:
+ case skip_blanks + other_char:
+ case new_line + math_shift:
+ case new_line + tab_mark:
+ case new_line + mac_param:
+ case new_line + sub_mark:
+ case new_line + letter:
+ case new_line + other_char:
+ */
+ state = mid_line;
+ break;
+ }
} else {
- do_get_cat_code(cur_cmd);
+ if (current_ocp_lstack > 0) {
+ pop_input();
+ return false;
+ }
+ if (name != 21)
+ state = new_line;
+
+ /*
+ @<Move to next line of file,
+ or |goto restart| if there is no next line,
+ or |return| if a \.{\\read} line has finished@>;
+ */
+ do {
+ next_line_retval r = next_line();
+ if (r == next_line_return) {
+ return true;
+ } else if (r == next_line_restart) {
+ return false;
+ }
+ } while (0);
+ check_interrupt();
+ goto SWITCH;
}
- /*
- @<Change state if necessary, and |goto switch| if the current
- character should be ignored, or |goto reswitch| if the current
- character changes to another@>;
- */
- /* The following 48-way switch accomplishes the scanning quickly, assuming
- that a decent \PASCAL\ compiler has translated the code. Note that the numeric
- values for |mid_line|, |skip_blanks|, and |new_line| are spaced
- apart from each other by |max_char_code+1|, so we can add a character's
- command code to the state to get a single number that characterizes both.
- */
- switch(state+cur_cmd) {
- case mid_line + ignore:
- case skip_blanks + ignore:
- case new_line + ignore:
- case skip_blanks + spacer:
- case new_line + spacer: /* @<Cases where character is ignored@> */
- goto SWITCH;
- break;
- case mid_line + escape:
- case new_line + escape:
- case skip_blanks + escape: /* @<Scan a control sequence ...@>;*/
- state = scan_control_sequence();
- break;
- case mid_line + active_char:
- case new_line + active_char:
- case skip_blanks + active_char: /* @<Process an active-character */
- cur_cs=cur_chr+active_base;
- cur_cmd=eq_type(cur_cs);
- cur_chr=equiv(cur_cs);
- state=mid_line;
- break;
- case mid_line + sup_mark:
- case new_line + sup_mark:
- case skip_blanks + sup_mark: /* @<If this |sup_mark| starts */
- if (process_sup_mark())
- goto RESWITCH;
- else
- state=mid_line;
- break;
- case mid_line + invalid_char:
- case new_line + invalid_char:
- case skip_blanks + invalid_char: /* @<Decry the invalid character and |goto restart|@>;*/
- invalid_character_error();
- return false; /* because state may be token_list now */
- break;
- case mid_line + spacer: /* @<Enter |skip_blanks| state, emit a space@>;*/
- state=skip_blanks;
- cur_chr=' ';
- break;
- case mid_line + car_ret: /* @<Finish line, emit a space@>;*/
- /* When a character of type |spacer| gets through, its character code is
- changed to $\.{"\ "}=@'40$. This means that the ASCII codes for tab and space,
- and for the space inserted at the end of a line, will
- be treated alike when macro parameters are being matched. We do this
- since such characters are indistinguishable on most computer terminal displays.
- */
- loc=limit+1;
- cur_cmd=spacer;
- cur_chr=' ';
- break;
- case skip_blanks + car_ret:
- case mid_line + comment:
- case new_line + comment:
- case skip_blanks + comment: /* @<Finish line, |goto switch|@>;*/
- loc=limit+1;
- goto SWITCH;
- break;
- case new_line + car_ret: /* @<Finish line, emit a \.{\\par}@>;*/
- loc=limit+1;
- cur_cs=par_loc;
- cur_cmd=eq_type(cur_cs);
- cur_chr=equiv(cur_cs);
- break;
- case skip_blanks + left_brace:
- case new_line + left_brace:
- state=mid_line; /* fall through */
- case mid_line + left_brace:
- align_state++;
- break;
- case skip_blanks + right_brace:
- case new_line + right_brace:
- state=mid_line; /* fall through */
- case mid_line + right_brace:
- align_state--;
- break;
- case mid_line + math_shift:
- case mid_line + tab_mark:
- case mid_line + mac_param:
- case mid_line + sub_mark:
- case mid_line + letter:
- case mid_line + other_char:
- break;
- default:
- /*
- case skip_blanks + math_shift:
- case skip_blanks + tab_mark:
- case skip_blanks + mac_param:
- case skip_blanks + sub_mark:
- case skip_blanks + letter:
- case skip_blanks + other_char:
- case new_line + math_shift:
- case new_line + tab_mark:
- case new_line + mac_param:
- case new_line + sub_mark:
- case new_line + letter:
- case new_line + other_char:
- */
- state=mid_line;
- break;
- }
- } else {
- if (current_ocp_lstack>0) {
- pop_input();
- return false;
- }
- if (name!=21)
- state=new_line;
-
- /*
- @<Move to next line of file,
- or |goto restart| if there is no next line,
- or |return| if a \.{\\read} line has finished@>;
- */
- do {
- next_line_retval r = next_line();
- if (r==next_line_return) {
- return true;
- } else if (r==next_line_restart) {
- return false;
- }
- } while (0);
- check_interrupt();
- goto SWITCH;
- }
- return true;
+ return true;
}
#define is_hex(a) ((a>='0'&&a<='9')||(a>='a'&&a<='f'))
@@ -307,71 +311,75 @@ get_next_file (void) {
/* Notice that a code like \.{\^\^8} becomes \.x if not followed by a hex digit.*/
-static boolean
-process_sup_mark(void) {
- if (cur_chr==buffer[loc]) {
- int c,cc;
- if (loc<limit) {
- if ((cur_chr==buffer[loc+1]) && (cur_chr==buffer[loc+2])
- && (cur_chr==buffer[loc+3]) && (cur_chr==buffer[loc+4])
- && ((loc+10)<=limit)) {
- int ccc,cccc,ccccc,cccccc; /* constituents of a possible expanded code */
- c=buffer[loc+5];
- cc=buffer[loc+6];
- ccc=buffer[loc+7];
- cccc=buffer[loc+8];
- ccccc=buffer[loc+9];
- cccccc=buffer[loc+10];
- if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc))
- && (is_hex(ccccc)) && (is_hex(cccccc))) {
- loc=loc+11;
- six_hex_to_cur_chr;
- return true;
- }
- }
- if ((cur_chr==buffer[loc+1]) && (cur_chr==buffer[loc+2])
- && (cur_chr==buffer[loc+3]) && ((loc+8)<=limit)) {
- int ccc,cccc,ccccc; /* constituents of a possible expanded code */
- c=buffer[loc+4];
- cc=buffer[loc+5];
- ccc=buffer[loc+6];
- cccc=buffer[loc+7];
- ccccc=buffer[loc+8];
- if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc)) && (is_hex(ccccc))) {
- loc=loc+9;
- five_hex_to_cur_chr;
- return true;
- }
- }
- if ((cur_chr==buffer[loc+1]) && (cur_chr==buffer[loc+2]) && ((loc+6)<=limit)) {
- int ccc,cccc; /* constituents of a possible expanded code */
- c=buffer[loc+3];
- cc=buffer[loc+4];
- ccc=buffer[loc+5];
- cccc=buffer[loc+6];
- if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc))) {
- loc=loc+7;
- four_hex_to_cur_chr;
- return true;
- }
- }
- c = buffer[loc+1];
- if (c<0200) { /* yes we have an expanded char */
- loc=loc+2;
- if (is_hex(c) && loc<=limit) {
- cc=buffer[loc];
- if (is_hex(cc)){
- incr(loc);
- hex_to_cur_chr;
- return true;
- }
+static boolean process_sup_mark(void)
+{
+ if (cur_chr == buffer[loc]) {
+ int c, cc;
+ if (loc < limit) {
+ if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2])
+ && (cur_chr == buffer[loc + 3]) && (cur_chr == buffer[loc + 4])
+ && ((loc + 10) <= limit)) {
+ int ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */
+ c = buffer[loc + 5];
+ cc = buffer[loc + 6];
+ ccc = buffer[loc + 7];
+ cccc = buffer[loc + 8];
+ ccccc = buffer[loc + 9];
+ cccccc = buffer[loc + 10];
+ if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
+ && (is_hex(cccc))
+ && (is_hex(ccccc)) && (is_hex(cccccc))) {
+ loc = loc + 11;
+ six_hex_to_cur_chr;
+ return true;
+ }
+ }
+ if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2])
+ && (cur_chr == buffer[loc + 3]) && ((loc + 8) <= limit)) {
+ int ccc, cccc, ccccc; /* constituents of a possible expanded code */
+ c = buffer[loc + 4];
+ cc = buffer[loc + 5];
+ ccc = buffer[loc + 6];
+ cccc = buffer[loc + 7];
+ ccccc = buffer[loc + 8];
+ if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
+ && (is_hex(cccc)) && (is_hex(ccccc))) {
+ loc = loc + 9;
+ five_hex_to_cur_chr;
+ return true;
+ }
+ }
+ if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2])
+ && ((loc + 6) <= limit)) {
+ int ccc, cccc; /* constituents of a possible expanded code */
+ c = buffer[loc + 3];
+ cc = buffer[loc + 4];
+ ccc = buffer[loc + 5];
+ cccc = buffer[loc + 6];
+ if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc))
+ && (is_hex(cccc))) {
+ loc = loc + 7;
+ four_hex_to_cur_chr;
+ return true;
+ }
+ }
+ c = buffer[loc + 1];
+ if (c < 0200) { /* yes we have an expanded char */
+ loc = loc + 2;
+ if (is_hex(c) && loc <= limit) {
+ cc = buffer[loc];
+ if (is_hex(cc)) {
+ incr(loc);
+ hex_to_cur_chr;
+ return true;
+ }
+ }
+ cur_chr = (c < 0100 ? c + 0100 : c - 0100);
+ return true;
+ }
}
- cur_chr = (c<0100 ? c+0100 : c-0100);
- return true;
- }
}
- }
- return false;
+ return false;
}
/* Control sequence names are scanned only when they appear in some line of
@@ -390,46 +398,50 @@ process_sup_mark(void) {
buffer and the process is repeated, slowly but surely.
*/
-static boolean check_expanded_code (integer *kk) ; /* below */
-
-static int scan_control_sequence (void ) {
- int retval = mid_line;
- if (loc>limit) {
- cur_cs=null_cs; /* |state| is irrelevant in this case */
- } else {
- register int cat; /* |cat_code(cur_chr)|, usually */
- while (1) {
- integer k=loc;
- cur_chr = qbuffer_to_unichar(&k);
- do_get_cat_code(cat);
- if (cat!=letter||k>limit) {
- retval = (cat==spacer ? skip_blanks : mid_line);
- if (cat==sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */
- continue;
- } else {
- retval=skip_blanks;
- do {
- cur_chr = qbuffer_to_unichar(&k);
- do_get_cat_code(cat);
- } while (cat==letter&&k<=limit);
-
- if (cat==sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */
- continue;
- if (cat!=letter) {
- decr(k);
- if (cur_chr>0xFFFF) decr(k);
- if (cur_chr>0x7FF) decr(k);
- if (cur_chr>0x7F) decr(k);
- } /* now |k| points to first nonletter */
- }
- cur_cs=id_lookup(loc,k-loc);
- loc=k;
- break;
+static boolean check_expanded_code(integer * kk); /* below */
+
+static int scan_control_sequence(void)
+{
+ int retval = mid_line;
+ if (loc > limit) {
+ cur_cs = null_cs; /* |state| is irrelevant in this case */
+ } else {
+ register int cat; /* |cat_code(cur_chr)|, usually */
+ while (1) {
+ integer k = loc;
+ cur_chr = qbuffer_to_unichar(&k);
+ do_get_cat_code(cat);
+ if (cat != letter || k > limit) {
+ retval = (cat == spacer ? skip_blanks : mid_line);
+ if (cat == sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */
+ continue;
+ } else {
+ retval = skip_blanks;
+ do {
+ cur_chr = qbuffer_to_unichar(&k);
+ do_get_cat_code(cat);
+ } while (cat == letter && k <= limit);
+
+ if (cat == sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */
+ continue;
+ if (cat != letter) {
+ decr(k);
+ if (cur_chr > 0xFFFF)
+ decr(k);
+ if (cur_chr > 0x7FF)
+ decr(k);
+ if (cur_chr > 0x7F)
+ decr(k);
+ } /* now |k| points to first nonletter */
+ }
+ cur_cs = id_lookup(loc, k - loc);
+ loc = k;
+ break;
+ }
}
- }
- cur_cmd=eq_type(cur_cs);
- cur_chr=equiv(cur_cs);
- return retval;
+ cur_cmd = eq_type(cur_cs);
+ cur_chr = equiv(cur_cs);
+ return retval;
}
/* Whenever we reach the following piece of code, we will have
@@ -440,89 +452,97 @@ static int scan_control_sequence (void ) {
the buffer left two or three places.
*/
-static boolean
-check_expanded_code (integer *kk) {
- int l;
- int k = *kk;
- int d =1; /* number of excess characters in an expanded code */
- int c,cc,ccc,cccc,ccccc,cccccc; /* constituents of a possible expanded code */
- if (buffer[k]==cur_chr && k<limit) {
- if ((cur_chr==buffer[k+1]) && (cur_chr==buffer[k+2]) && ((k+6)<=limit)) {
- d = 4;
- if ((cur_chr==buffer[k+3]) && ((k+8)<=limit))
- d = 5;
- if ((cur_chr==buffer[k+4]) && ((k+10)<=limit))
- d = 6;
- c = buffer[k+d-1];
- cc = buffer[k+d];
- ccc = buffer[k+d+1];
- cccc= buffer[k+d+2];
- if (d==6) {
- ccccc =buffer[k+d+3];
- cccccc=buffer[k+d+4];
- if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
- && is_hex(ccccc) && is_hex(cccccc))
- six_hex_to_cur_chr;
- } else if (d==5) {
- ccccc=buffer[k+d+3];
- if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) && is_hex(ccccc))
- five_hex_to_cur_chr;
- } else {
- if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc))
- four_hex_to_cur_chr;
- }
- } else {
- c=buffer[k+1];
- if (c<0200) {
- d=1;
- if (is_hex(c) && (k+2)<=limit) {
- cc=buffer[k+2];
- if (is_hex(c) && is_hex(cc)) {
- d=2;
- hex_to_cur_chr;
- }
- } else if (c<0100) {
- cur_chr=c+0100;
- } else {
- cur_chr=c-0100;
- }
- }
- }
- if (d>2)
- d=2*d-1;
- else
- d++;
- if (cur_chr <= 0x7F) {
- buffer[k-1]= cur_chr;
- } else if (cur_chr<=0x7FF) {
- buffer[k-1]=0xC0 + cur_chr / 0x40;
- k++; d--;
- buffer[k-1]=0x80 + cur_chr % 0x40;
- } else if (cur_chr<=0xFFFF){
- buffer[k-1]=0xE0 + cur_chr / 0x1000;
- k++; d--;
- buffer[k-1]=0x80 + (cur_chr % 0x1000) / 0x40;
- k++; d--;
- buffer[k-1]=0x80 + (cur_chr % 0x1000) % 0x40;
- } else {
- buffer[k-1]=0xF0 + cur_chr / 0x40000;
- k++; d--;
- buffer[k-1]=0x80 + (cur_chr % 0x40000) / 0x1000;
- k++; d--;
- buffer[k-1]=0x80 + ((cur_chr % 0x40000) % 0x1000) / 0x40;
- k++; d--;
- buffer[k-1]=0x80 + ((cur_chr % 0x40000) % 0x1000) % 0x40;
- }
- l = k;
- limit = limit - d;
- while (l<=limit) {
- buffer[l]=buffer[l+d];
- l++;
+static boolean check_expanded_code(integer * kk)
+{
+ int l;
+ int k = *kk;
+ int d = 1; /* number of excess characters in an expanded code */
+ int c, cc, ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */
+ if (buffer[k] == cur_chr && k < limit) {
+ if ((cur_chr == buffer[k + 1]) && (cur_chr == buffer[k + 2])
+ && ((k + 6) <= limit)) {
+ d = 4;
+ if ((cur_chr == buffer[k + 3]) && ((k + 8) <= limit))
+ d = 5;
+ if ((cur_chr == buffer[k + 4]) && ((k + 10) <= limit))
+ d = 6;
+ c = buffer[k + d - 1];
+ cc = buffer[k + d];
+ ccc = buffer[k + d + 1];
+ cccc = buffer[k + d + 2];
+ if (d == 6) {
+ ccccc = buffer[k + d + 3];
+ cccccc = buffer[k + d + 4];
+ if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
+ && is_hex(ccccc) && is_hex(cccccc))
+ six_hex_to_cur_chr;
+ } else if (d == 5) {
+ ccccc = buffer[k + d + 3];
+ if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)
+ && is_hex(ccccc))
+ five_hex_to_cur_chr;
+ } else {
+ if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc))
+ four_hex_to_cur_chr;
+ }
+ } else {
+ c = buffer[k + 1];
+ if (c < 0200) {
+ d = 1;
+ if (is_hex(c) && (k + 2) <= limit) {
+ cc = buffer[k + 2];
+ if (is_hex(c) && is_hex(cc)) {
+ d = 2;
+ hex_to_cur_chr;
+ }
+ } else if (c < 0100) {
+ cur_chr = c + 0100;
+ } else {
+ cur_chr = c - 0100;
+ }
+ }
+ }
+ if (d > 2)
+ d = 2 * d - 1;
+ else
+ d++;
+ if (cur_chr <= 0x7F) {
+ buffer[k - 1] = cur_chr;
+ } else if (cur_chr <= 0x7FF) {
+ buffer[k - 1] = 0xC0 + cur_chr / 0x40;
+ k++;
+ d--;
+ buffer[k - 1] = 0x80 + cur_chr % 0x40;
+ } else if (cur_chr <= 0xFFFF) {
+ buffer[k - 1] = 0xE0 + cur_chr / 0x1000;
+ k++;
+ d--;
+ buffer[k - 1] = 0x80 + (cur_chr % 0x1000) / 0x40;
+ k++;
+ d--;
+ buffer[k - 1] = 0x80 + (cur_chr % 0x1000) % 0x40;
+ } else {
+ buffer[k - 1] = 0xF0 + cur_chr / 0x40000;
+ k++;
+ d--;
+ buffer[k - 1] = 0x80 + (cur_chr % 0x40000) / 0x1000;
+ k++;
+ d--;
+ buffer[k - 1] = 0x80 + ((cur_chr % 0x40000) % 0x1000) / 0x40;
+ k++;
+ d--;
+ buffer[k - 1] = 0x80 + ((cur_chr % 0x40000) % 0x1000) % 0x40;
+ }
+ l = k;
+ limit = limit - d;
+ while (l <= limit) {
+ buffer[l] = buffer[l + d];
+ l++;
+ }
+ *kk = k;
+ return true;
}
- *kk =k;
- return true;
- }
- return false;
+ return false;
}
#define end_line_char_inactive ((end_line_char<0)||(end_line_char>127))
@@ -540,122 +560,126 @@ check_expanded_code (integer *kk) {
There is one more branch.
*/
-static next_line_retval
-next_line (void) {
- integer tab; /* a category table */
- boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */
- detokenized_line=false;
- local_catcode_table=false;
- if (name>17) {
- /* @<Read next line of file into |buffer|, or |goto restart| if the file has ended@> */
- incr(line);
- first=start;
- if (!force_eof) {
- if (name<=20) {
- if (pseudo_input()) { /* not end of file */
- firm_up_the_line(); /* this sets |limit| */
- if ((name==19)&&(pseudo_lines(pseudo_files)==null))
- inhibit_eol=true;
- } else if ((every_eof!=null)&&!eof_seen[index]) {
- limit=first-1; eof_seen[index]=true; /* fake one empty line */
- if (name!=19)
- begin_token_list(every_eof,every_eof_text);
- return next_line_restart;
- } else {
- force_eof=true;
- }
- } else {
- if (name==21) {
- if (luacstring_input()) { /* not end of strings */
- firm_up_the_line();
- if ((luacstring_penultimate())||(luacstring_simple()))
- inhibit_eol=true;
- if (!luacstring_simple())
- state=new_line;
- if (luacstring_detokenized()) {
- inhibit_eol=true;
- detokenized_line=true;
- } else {
- if (! luacstring_defaultcattable()) {
- tab=luacstring_cattable();
- if (valid_catcode_table(tab)) {
- local_catcode_table=true;
- line_catcode_table=tab;
- }
- }
- }
- } else {
- force_eof=true;
- }
- } else {
- if (lua_input_ln(cur_file,0,true)) { /* not end of file */
- firm_up_the_line(); /* this sets |limit| */
- } else if ((every_eof!=null)&& (!eof_seen[index])) {
- limit=first-1; eof_seen[index]=true; /* fake one empty line */
- begin_token_list(every_eof,every_eof_text);
- return next_line_restart;
- } else {
- force_eof=true;
- }
- }
- }
- }
- if (force_eof) {
- if (tracing_nesting>0)
- if ((grp_stack[in_open]!=cur_boundary)||(if_stack[in_open]!=cond_ptr))
- if (!((name==19)||(name=21)))
- file_warning(); /* give warning for some unfinished groups and/or conditionals */
- if ((name>21)||(name==20)) {
- if (tracefilenames)
- print_char(')');
- open_parens--;
- /* update_terminal();*/ /* show user that file has been read */
- }
- force_eof=false;
- end_file_reading();
- return next_line_restart;
- }
- if (inhibit_eol||end_line_char_inactive)
- limit--;
- else
- buffer[limit]=end_line_char;
- first=limit+1;
- loc=start; /* ready to read */
- } else {
- if (!terminal_input) { /* \.{\\read} line has ended */
- cur_cmd=0;
- cur_chr=0;
- return next_line_return; /* OUTER */
- }
- if (input_ptr>0) { /* text was inserted during error recovery */
- end_file_reading();
- return next_line_restart; /* resume previous level */
- }
- if (selector<log_only)
- open_log_file();
- if (interaction>nonstop_mode) {
- if (end_line_char_inactive)
- limit++;
- if (limit==start) { /* previous line was empty */
- tprint_nl("(Please type a command or say `\\end')");
- }
- print_ln();
- first=start;
- prompt_input((str_number)'*'); /* input on-line into |buffer| */
- limit=last;
- if (end_line_char_inactive)
- limit--;
- else
- buffer[limit]=end_line_char;
- first=limit+1;
- loc=start;
+static next_line_retval next_line(void)
+{
+ integer tab; /* a category table */
+ boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */
+ detokenized_line = false;
+ local_catcode_table = false;
+ if (name > 17) {
+ /* @<Read next line of file into |buffer|, or |goto restart| if the file has ended@> */
+ incr(line);
+ first = start;
+ if (!force_eof) {
+ if (name <= 20) {
+ if (pseudo_input()) { /* not end of file */
+ firm_up_the_line(); /* this sets |limit| */
+ if ((name == 19) && (pseudo_lines(pseudo_files) == null))
+ inhibit_eol = true;
+ } else if ((every_eof != null) && !eof_seen[index]) {
+ limit = first - 1;
+ eof_seen[index] = true; /* fake one empty line */
+ if (name != 19)
+ begin_token_list(every_eof, every_eof_text);
+ return next_line_restart;
+ } else {
+ force_eof = true;
+ }
+ } else {
+ if (name == 21) {
+ if (luacstring_input()) { /* not end of strings */
+ firm_up_the_line();
+ if ((luacstring_penultimate()) || (luacstring_simple()))
+ inhibit_eol = true;
+ if (!luacstring_simple())
+ state = new_line;
+ if (luacstring_detokenized()) {
+ inhibit_eol = true;
+ detokenized_line = true;
+ } else {
+ if (!luacstring_defaultcattable()) {
+ tab = luacstring_cattable();
+ if (valid_catcode_table(tab)) {
+ local_catcode_table = true;
+ line_catcode_table = tab;
+ }
+ }
+ }
+ } else {
+ force_eof = true;
+ }
+ } else {
+ if (lua_input_ln(cur_file, 0, true)) { /* not end of file */
+ firm_up_the_line(); /* this sets |limit| */
+ } else if ((every_eof != null) && (!eof_seen[index])) {
+ limit = first - 1;
+ eof_seen[index] = true; /* fake one empty line */
+ begin_token_list(every_eof, every_eof_text);
+ return next_line_restart;
+ } else {
+ force_eof = true;
+ }
+ }
+ }
+ }
+ if (force_eof) {
+ if (tracing_nesting > 0)
+ if ((grp_stack[in_open] != cur_boundary)
+ || (if_stack[in_open] != cond_ptr))
+ if (!((name == 19) || (name = 21)))
+ file_warning(); /* give warning for some unfinished groups and/or conditionals */
+ if ((name > 21) || (name == 20)) {
+ if (tracefilenames)
+ print_char(')');
+ open_parens--;
+ /* update_terminal(); *//* show user that file has been read */
+ }
+ force_eof = false;
+ end_file_reading();
+ return next_line_restart;
+ }
+ if (inhibit_eol || end_line_char_inactive)
+ limit--;
+ else
+ buffer[limit] = end_line_char;
+ first = limit + 1;
+ loc = start; /* ready to read */
} else {
- fatal_error(maketexstring("*** (job aborted, no legal \\end found)"));
- /* nonstop mode, which is intended for overnight batch processing,
- never waits for on-line input */
+ if (!terminal_input) { /* \.{\\read} line has ended */
+ cur_cmd = 0;
+ cur_chr = 0;
+ return next_line_return; /* OUTER */
+ }
+ if (input_ptr > 0) { /* text was inserted during error recovery */
+ end_file_reading();
+ return next_line_restart; /* resume previous level */
+ }
+ if (selector < log_only)
+ open_log_file();
+ if (interaction > nonstop_mode) {
+ if (end_line_char_inactive)
+ limit++;
+ if (limit == start) { /* previous line was empty */
+ tprint_nl("(Please type a command or say `\\end')");
+ }
+ print_ln();
+ first = start;
+ prompt_input((str_number) '*'); /* input on-line into |buffer| */
+ limit = last;
+ if (end_line_char_inactive)
+ limit--;
+ else
+ buffer[limit] = end_line_char;
+ first = limit + 1;
+ loc = start;
+ } else {
+ fatal_error(maketexstring
+ ("*** (job aborted, no legal \\end found)"));
+ /* nonstop mode, which is intended for overnight batch processing,
+ never waits for on-line input */
+ }
}
- }
- return next_line_ok;
+ return next_line_ok;
}
@@ -663,46 +687,46 @@ next_line (void) {
/* Let's consider now what happens when |get_next| is looking at a token list. */
-static boolean
-get_next_tokenlist (void) {
- register halfword t; /* a token*/
- t=info(loc);
- loc=link(loc); /* move to next */
- if (t>=cs_token_flag) { /* a control sequence token */
- cur_cs=t-cs_token_flag;
- cur_cmd=eq_type(cur_cs);
- if (cur_cmd==dont_expand) { /* @<Get the next token, suppressing expansion@>*/
- /* The present point in the program is reached only when the |expand|
- routine has inserted a special marker into the input. In this special
- case, |info(loc)| is known to be a control sequence token, and |link(loc)=null|.
- */
- cur_cs=info(loc)-cs_token_flag;
- loc=null;
- cur_cmd=eq_type(cur_cs);
- if (cur_cmd>max_command) {
- cur_cmd=relax;
- cur_chr=no_expand_flag;
- return true;
- }
- }
- cur_chr=equiv(cur_cs);
- } else {
- cur_cmd=t >> string_offset_bits; /* cur_cmd=t / string_offset; */
- cur_chr=t & (string_offset-1); /* cur_chr=t % string_offset; */
- switch (cur_cmd) {
- case left_brace:
- align_state++;
- break;
- case right_brace:
- align_state--;
- break;
- case out_param: /* @<Insert macro parameter and |goto restart|@>; */
- begin_token_list(param_stack[param_start+cur_chr-1],parameter);
- return false;
- break;
+static boolean get_next_tokenlist(void)
+{
+ register halfword t; /* a token */
+ t = info(loc);
+ loc = link(loc); /* move to next */
+ if (t >= cs_token_flag) { /* a control sequence token */
+ cur_cs = t - cs_token_flag;
+ cur_cmd = eq_type(cur_cs);
+ if (cur_cmd == dont_expand) { /* @<Get the next token, suppressing expansion@> */
+ /* The present point in the program is reached only when the |expand|
+ routine has inserted a special marker into the input. In this special
+ case, |info(loc)| is known to be a control sequence token, and |link(loc)=null|.
+ */
+ cur_cs = info(loc) - cs_token_flag;
+ loc = null;
+ cur_cmd = eq_type(cur_cs);
+ if (cur_cmd > max_command) {
+ cur_cmd = relax;
+ cur_chr = no_expand_flag;
+ return true;
+ }
+ }
+ cur_chr = equiv(cur_cs);
+ } else {
+ cur_cmd = t >> string_offset_bits; /* cur_cmd=t / string_offset; */
+ cur_chr = t & (string_offset - 1); /* cur_chr=t % string_offset; */
+ switch (cur_cmd) {
+ case left_brace:
+ align_state++;
+ break;
+ case right_brace:
+ align_state--;
+ break;
+ case out_param: /* @<Insert macro parameter and |goto restart|@>; */
+ begin_token_list(param_stack[param_start + cur_chr - 1], parameter);
+ return false;
+ break;
+ }
}
- }
- return true;
+ return true;
}
/* Now we're ready to take the plunge into |get_next| itself. Parts of
@@ -712,42 +736,42 @@ get_next_tokenlist (void) {
/* sets |cur_cmd|, |cur_chr|, |cur_cs| to next token */
-void
-get_next (void) {
- RESTART:
- cur_cs=0;
- if (state!=token_list) {
- /* Input from external file, |goto restart| if no input found */
- if (!get_next_file())
- goto RESTART;
- } else {
- if (loc==null) {
- end_token_list();
- goto RESTART; /* list exhausted, resume previous level */
- } else if (!get_next_tokenlist()) {
- goto RESTART; /* parameter needs to be expanded */
+void get_next(void)
+{
+ RESTART:
+ cur_cs = 0;
+ if (state != token_list) {
+ /* Input from external file, |goto restart| if no input found */
+ if (!get_next_file())
+ goto RESTART;
+ } else {
+ if (loc == null) {
+ end_token_list();
+ goto RESTART; /* list exhausted, resume previous level */
+ } else if (!get_next_tokenlist()) {
+ goto RESTART; /* parameter needs to be expanded */
+ }
+ }
+ /* @<If an alignment entry has just ended, take appropriate action@> */
+ if ((cur_cmd == tab_mark || cur_cmd == car_ret) && align_state == 0) {
+ insert_vj_template();
+ goto RESTART;
}
- }
- /* @<If an alignment entry has just ended, take appropriate action@> */
- if ((cur_cmd==tab_mark || cur_cmd==car_ret) && align_state==0) {
- insert_vj_template();
- goto RESTART;
- }
}
-void
-get_token_lua (void) {
- register int callback_id ;
- callback_id = callback_defined(token_filter_callback);
- if (callback_id!=0) {
- while (state==token_list && loc==null && index!=v_template)
- end_token_list();
- /* there is some stuff we don't want to see inside the callback */
- if (!(state==token_list &&
- ((nofilter==true) || (index==backed_up && loc!=null)))) {
- do_get_token_lua(callback_id);
- return;
+void get_token_lua(void)
+{
+ register int callback_id;
+ callback_id = callback_defined(token_filter_callback);
+ if (callback_id != 0) {
+ while (state == token_list && loc == null && index != v_template)
+ end_token_list();
+ /* there is some stuff we don't want to see inside the callback */
+ if (!(state == token_list &&
+ ((nofilter == true) || (index == backed_up && loc != null)))) {
+ do_get_token_lua(callback_id);
+ return;
+ }
}
- }
- get_next();
+ get_next();
}