diff options
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/tex/textoken.c')
-rw-r--r-- | Build/source/texk/web2c/luatexdir/tex/textoken.c | 1190 |
1 files changed, 607 insertions, 583 deletions
diff --git a/Build/source/texk/web2c/luatexdir/tex/textoken.c b/Build/source/texk/web2c/luatexdir/tex/textoken.c index 20c57b9479b..366fa798791 100644 --- a/Build/source/texk/web2c/luatexdir/tex/textoken.c +++ b/Build/source/texk/web2c/luatexdir/tex/textoken.c @@ -1,4 +1,4 @@ -/* $Id: textoken.c 1013 2008-02-14 00:09:02Z oneiros $ */ +/* $Id: textoken.c 1155 2008-04-14 07:53:21Z oneiros $ */ #include "luatex-api.h" #include <ptexlib.h> @@ -7,18 +7,18 @@ /* Integer parameters and other command-related defines. This needs it's own header file! */ -#define end_line_char_code 48 /* character placed at the right end of the buffer */ +#define end_line_char_code 48 /* character placed at the right end of the buffer */ #define cat_code_table_code 63 -#define tex_int_pars 66 /* total number of \.{\\TeX} + Aleph integer parameters */ +#define tex_int_pars 66 /* total number of \.{\\TeX} + Aleph integer parameters */ /* this is not what happens in the pascal code! there the values shift from bare numbers to offsets ! */ #define dir_base tex_int_pars #define dir_pars 5 -#define pdftex_first_integer_code dir_base+dir_pars /* base for \pdfTeX's integer parameters */ -#define pdf_int_pars pdftex_first_integer_code+27 /*total number of \pdfTeX's integer parameters*/ -#define etex_int_base pdf_int_pars /*base for \eTeX's integer parameters*/ -#define tracing_nesting_code etex_int_base+4 /*show incomplete groups and ifs within files*/ +#define pdftex_first_integer_code dir_base+dir_pars /* base for \pdfTeX's integer parameters */ +#define pdf_int_pars pdftex_first_integer_code+27 /*total number of \pdfTeX's integer parameters */ +#define etex_int_base pdf_int_pars /*base for \eTeX's integer parameters */ +#define tracing_nesting_code etex_int_base+4 /*show incomplete groups and ifs within files */ -#define int_par(a) zeqtb[static_int_base+a].cint /* an integer parameter */ +#define int_par(a) zeqtb[static_int_base+a].cint /* an integer parameter */ #define cat_code_table int_par(cat_code_table_code) #define tracing_nesting int_par(tracing_nesting_code) #define end_line_char int_par(end_line_char_code) @@ -26,7 +26,7 @@ #define every_eof get_every_eof() #define number_active_chars 65536+65536 -#define active_base 1 /* beginning of region 1, for active character equivalents */ +#define active_base 1 /* beginning of region 1, for active character equivalents */ #define null_cs active_base+number_active_chars /* equivalent of \.{\\csname\\endcsname} */ @@ -40,18 +40,18 @@ #define nonstop_mode 1 /* command codes */ -#define relax 0 +#define relax 0 #define out_param 5 -#define max_command 117 /* fetched from C output */ -#define dont_expand 133 /* fetched from C output */ +#define max_command 117 /* fetched from C output */ +#define dont_expand 133 /* fetched from C output */ -#define terminal_input (name==0) /* are we reading from the terminal? */ -#define special_char 1114113 /* |biggest_char+2| */ -#define cur_file input_file[index] /* the current |alpha_file| variable */ +#define terminal_input (name==0) /* are we reading from the terminal? */ +#define special_char 1114113 /* |biggest_char+2| */ +#define cur_file input_file[index] /* the current |alpha_file| variable */ -#define no_expand_flag special_char /*this characterizes a special variant of |relax|*/ +#define no_expand_flag special_char /*this characterizes a special variant of |relax| */ -extern void insert_vj_template (void); +extern void insert_vj_template(void); #define do_get_cat_code(a) do { \ if (local_catcode_table) \ @@ -61,219 +61,223 @@ extern void insert_vj_template (void); } while (0) -static void -invalid_character_error (void) { - char *hlp[] = {"A funny symbol that I can't read has just been input.", - "Continue, and I'll forget that it ever happened.", - NULL}; - deletions_allowed=false; - tex_error("Text line contains an invalid character", hlp); - deletions_allowed=true; +static void invalid_character_error(void) +{ + char *hlp[] = { "A funny symbol that I can't read has just been input.", + "Continue, and I'll forget that it ever happened.", + NULL + }; + deletions_allowed = false; + tex_error("Text line contains an invalid character", hlp); + deletions_allowed = true; } /* no longer done */ -static boolean process_sup_mark(void); /* below */ +static boolean process_sup_mark(void); /* below */ static int scan_control_sequence(void); /* below */ -typedef enum {next_line_ok, next_line_return, next_line_restart } next_line_retval ; +typedef enum { next_line_ok, next_line_return, + next_line_restart } next_line_retval; -static next_line_retval next_line(void); /* below */ +static next_line_retval next_line(void); /* below */ /* @^inner loop@>*/ -static void -utf_error (void) { - char *hlp[] = {"A funny symbol that I can't read has just been input.", - "Just continue, I'll change it to 0xFFFD.", - NULL }; - deletions_allowed=false; - tex_error("Text line contains an invalid utf-8 sequence", hlp); - deletions_allowed=true; +static void utf_error(void) +{ + char *hlp[] = { "A funny symbol that I can't read has just been input.", + "Just continue, I'll change it to 0xFFFD.", + NULL + }; + deletions_allowed = false; + tex_error("Text line contains an invalid utf-8 sequence", hlp); + deletions_allowed = true; } -static integer -qbuffer_to_unichar (integer *k) { - register int ch; - int val = 0xFFFD; - unsigned char *text = buffer+*k; - if ( (ch = *text++)<0x80 ) { - val = ch; - *k+=1; - } else if ( ch<=0xbf ) { /* error */ - *k+=1; - } else if ( ch<=0xdf ) { - if ( *text>=0x80 && *text<0xc0 ) - val = ((ch&0x1f)<<6) | (*text++&0x3f); - *k+=2; - } else if ( ch<=0xef ) { - if ( *text>=0x80 && *text<0xc0 && text[1]>=0x80 && text[1]<0xc0 ) { - val = ((ch&0xf)<<12) | ((text[0]&0x3f)<<6) | (text[1]&0x3f); - *k+=3; +static integer qbuffer_to_unichar(integer * k) +{ + register int ch; + int val = 0xFFFD; + unsigned char *text = buffer + *k; + if ((ch = *text++) < 0x80) { + val = ch; + *k += 1; + } else if (ch <= 0xbf) { /* error */ + *k += 1; + } else if (ch <= 0xdf) { + if (*text >= 0x80 && *text < 0xc0) + val = ((ch & 0x1f) << 6) | (*text++ & 0x3f); + *k += 2; + } else if (ch <= 0xef) { + if (*text >= 0x80 && *text < 0xc0 && text[1] >= 0x80 && text[1] < 0xc0) { + val = + ((ch & 0xf) << 12) | ((text[0] & 0x3f) << 6) | (text[1] & 0x3f); + *k += 3; + } + } else { + int w = (((ch & 0x7) << 2) | ((text[0] & 0x30) >> 4)) - 1, w2; + w = (w << 6) | ((text[0] & 0xf) << 2) | ((text[1] & 0x30) >> 4); + w2 = ((text[1] & 0xf) << 6) | (text[2] & 0x3f); + val = w * 0x400 + w2 + 0x10000; + if (*text < 0x80 || text[1] < 0x80 || text[2] < 0x80 || + *text >= 0xc0 || text[1] >= 0xc0 || text[2] >= 0xc0) + val = 0xFFFD; + *k += 4; } - } else { - int w = ( ((ch&0x7)<<2) | ((text[0]&0x30)>>4) )-1, w2; - w = (w<<6) | ((text[0]&0xf)<<2) | ((text[1]&0x30)>>4); - w2 = ((text[1]&0xf)<<6) | (text[2]&0x3f); - val = w*0x400 + w2 + 0x10000; - if ( *text<0x80 || text[1]<0x80 || text[2]<0x80 || - *text>=0xc0 || text[1]>=0xc0 || text[2]>=0xc0 ) - val = 0xFFFD; - *k+=4; - } - if (val==0xFFFD) - utf_error(); - return( val ); + if (val == 0xFFFD) + utf_error(); + return (val); } -static boolean -get_next_file (void) { - SWITCH: - if (loc<=limit) { /* current line not yet finished */ - cur_chr = qbuffer_to_unichar(&loc); - RESWITCH: - if (detokenized_line) { - cur_cmd = (cur_chr==' ' ? 10 : 12) ; +static boolean get_next_file(void) +{ + SWITCH: + if (loc <= limit) { /* current line not yet finished */ + cur_chr = qbuffer_to_unichar(&loc); + RESWITCH: + if (detokenized_line) { + cur_cmd = (cur_chr == ' ' ? 10 : 12); + } else { + do_get_cat_code(cur_cmd); + } + /* + @<Change state if necessary, and |goto switch| if the current + character should be ignored, or |goto reswitch| if the current + character changes to another@>; + */ + /* The following 48-way switch accomplishes the scanning quickly, assuming + that a decent \PASCAL\ compiler has translated the code. Note that the numeric + values for |mid_line|, |skip_blanks|, and |new_line| are spaced + apart from each other by |max_char_code+1|, so we can add a character's + command code to the state to get a single number that characterizes both. + */ + switch (state + cur_cmd) { + case mid_line + ignore: + case skip_blanks + ignore: + case new_line + ignore: + case skip_blanks + spacer: + case new_line + spacer: /* @<Cases where character is ignored@> */ + goto SWITCH; + break; + case mid_line + escape: + case new_line + escape: + case skip_blanks + escape: /* @<Scan a control sequence ...@>; */ + state = scan_control_sequence(); + break; + case mid_line + active_char: + case new_line + active_char: + case skip_blanks + active_char: /* @<Process an active-character */ + cur_cs = cur_chr + active_base; + cur_cmd = eq_type(cur_cs); + cur_chr = equiv(cur_cs); + state = mid_line; + break; + case mid_line + sup_mark: + case new_line + sup_mark: + case skip_blanks + sup_mark: /* @<If this |sup_mark| starts */ + if (process_sup_mark()) + goto RESWITCH; + else + state = mid_line; + break; + case mid_line + invalid_char: + case new_line + invalid_char: + case skip_blanks + invalid_char: /* @<Decry the invalid character and |goto restart|@>; */ + invalid_character_error(); + return false; /* because state may be token_list now */ + break; + case mid_line + spacer: /* @<Enter |skip_blanks| state, emit a space@>; */ + state = skip_blanks; + cur_chr = ' '; + break; + case mid_line + car_ret: /* @<Finish line, emit a space@>; */ + /* When a character of type |spacer| gets through, its character code is + changed to $\.{"\ "}=@'40$. This means that the ASCII codes for tab and space, + and for the space inserted at the end of a line, will + be treated alike when macro parameters are being matched. We do this + since such characters are indistinguishable on most computer terminal displays. + */ + loc = limit + 1; + cur_cmd = spacer; + cur_chr = ' '; + break; + case skip_blanks + car_ret: + case mid_line + comment: + case new_line + comment: + case skip_blanks + comment: /* @<Finish line, |goto switch|@>; */ + loc = limit + 1; + goto SWITCH; + break; + case new_line + car_ret: /* @<Finish line, emit a \.{\\par}@>; */ + loc = limit + 1; + cur_cs = par_loc; + cur_cmd = eq_type(cur_cs); + cur_chr = equiv(cur_cs); + break; + case skip_blanks + left_brace: + case new_line + left_brace: + state = mid_line; /* fall through */ + case mid_line + left_brace: + align_state++; + break; + case skip_blanks + right_brace: + case new_line + right_brace: + state = mid_line; /* fall through */ + case mid_line + right_brace: + align_state--; + break; + case mid_line + math_shift: + case mid_line + tab_mark: + case mid_line + mac_param: + case mid_line + sub_mark: + case mid_line + letter: + case mid_line + other_char: + break; + default: + /* + case skip_blanks + math_shift: + case skip_blanks + tab_mark: + case skip_blanks + mac_param: + case skip_blanks + sub_mark: + case skip_blanks + letter: + case skip_blanks + other_char: + case new_line + math_shift: + case new_line + tab_mark: + case new_line + mac_param: + case new_line + sub_mark: + case new_line + letter: + case new_line + other_char: + */ + state = mid_line; + break; + } } else { - do_get_cat_code(cur_cmd); + if (current_ocp_lstack > 0) { + pop_input(); + return false; + } + if (name != 21) + state = new_line; + + /* + @<Move to next line of file, + or |goto restart| if there is no next line, + or |return| if a \.{\\read} line has finished@>; + */ + do { + next_line_retval r = next_line(); + if (r == next_line_return) { + return true; + } else if (r == next_line_restart) { + return false; + } + } while (0); + check_interrupt(); + goto SWITCH; } - /* - @<Change state if necessary, and |goto switch| if the current - character should be ignored, or |goto reswitch| if the current - character changes to another@>; - */ - /* The following 48-way switch accomplishes the scanning quickly, assuming - that a decent \PASCAL\ compiler has translated the code. Note that the numeric - values for |mid_line|, |skip_blanks|, and |new_line| are spaced - apart from each other by |max_char_code+1|, so we can add a character's - command code to the state to get a single number that characterizes both. - */ - switch(state+cur_cmd) { - case mid_line + ignore: - case skip_blanks + ignore: - case new_line + ignore: - case skip_blanks + spacer: - case new_line + spacer: /* @<Cases where character is ignored@> */ - goto SWITCH; - break; - case mid_line + escape: - case new_line + escape: - case skip_blanks + escape: /* @<Scan a control sequence ...@>;*/ - state = scan_control_sequence(); - break; - case mid_line + active_char: - case new_line + active_char: - case skip_blanks + active_char: /* @<Process an active-character */ - cur_cs=cur_chr+active_base; - cur_cmd=eq_type(cur_cs); - cur_chr=equiv(cur_cs); - state=mid_line; - break; - case mid_line + sup_mark: - case new_line + sup_mark: - case skip_blanks + sup_mark: /* @<If this |sup_mark| starts */ - if (process_sup_mark()) - goto RESWITCH; - else - state=mid_line; - break; - case mid_line + invalid_char: - case new_line + invalid_char: - case skip_blanks + invalid_char: /* @<Decry the invalid character and |goto restart|@>;*/ - invalid_character_error(); - return false; /* because state may be token_list now */ - break; - case mid_line + spacer: /* @<Enter |skip_blanks| state, emit a space@>;*/ - state=skip_blanks; - cur_chr=' '; - break; - case mid_line + car_ret: /* @<Finish line, emit a space@>;*/ - /* When a character of type |spacer| gets through, its character code is - changed to $\.{"\ "}=@'40$. This means that the ASCII codes for tab and space, - and for the space inserted at the end of a line, will - be treated alike when macro parameters are being matched. We do this - since such characters are indistinguishable on most computer terminal displays. - */ - loc=limit+1; - cur_cmd=spacer; - cur_chr=' '; - break; - case skip_blanks + car_ret: - case mid_line + comment: - case new_line + comment: - case skip_blanks + comment: /* @<Finish line, |goto switch|@>;*/ - loc=limit+1; - goto SWITCH; - break; - case new_line + car_ret: /* @<Finish line, emit a \.{\\par}@>;*/ - loc=limit+1; - cur_cs=par_loc; - cur_cmd=eq_type(cur_cs); - cur_chr=equiv(cur_cs); - break; - case skip_blanks + left_brace: - case new_line + left_brace: - state=mid_line; /* fall through */ - case mid_line + left_brace: - align_state++; - break; - case skip_blanks + right_brace: - case new_line + right_brace: - state=mid_line; /* fall through */ - case mid_line + right_brace: - align_state--; - break; - case mid_line + math_shift: - case mid_line + tab_mark: - case mid_line + mac_param: - case mid_line + sub_mark: - case mid_line + letter: - case mid_line + other_char: - break; - default: - /* - case skip_blanks + math_shift: - case skip_blanks + tab_mark: - case skip_blanks + mac_param: - case skip_blanks + sub_mark: - case skip_blanks + letter: - case skip_blanks + other_char: - case new_line + math_shift: - case new_line + tab_mark: - case new_line + mac_param: - case new_line + sub_mark: - case new_line + letter: - case new_line + other_char: - */ - state=mid_line; - break; - } - } else { - if (current_ocp_lstack>0) { - pop_input(); - return false; - } - if (name!=21) - state=new_line; - - /* - @<Move to next line of file, - or |goto restart| if there is no next line, - or |return| if a \.{\\read} line has finished@>; - */ - do { - next_line_retval r = next_line(); - if (r==next_line_return) { - return true; - } else if (r==next_line_restart) { - return false; - } - } while (0); - check_interrupt(); - goto SWITCH; - } - return true; + return true; } #define is_hex(a) ((a>='0'&&a<='9')||(a>='a'&&a<='f')) @@ -307,71 +311,75 @@ get_next_file (void) { /* Notice that a code like \.{\^\^8} becomes \.x if not followed by a hex digit.*/ -static boolean -process_sup_mark(void) { - if (cur_chr==buffer[loc]) { - int c,cc; - if (loc<limit) { - if ((cur_chr==buffer[loc+1]) && (cur_chr==buffer[loc+2]) - && (cur_chr==buffer[loc+3]) && (cur_chr==buffer[loc+4]) - && ((loc+10)<=limit)) { - int ccc,cccc,ccccc,cccccc; /* constituents of a possible expanded code */ - c=buffer[loc+5]; - cc=buffer[loc+6]; - ccc=buffer[loc+7]; - cccc=buffer[loc+8]; - ccccc=buffer[loc+9]; - cccccc=buffer[loc+10]; - if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc)) - && (is_hex(ccccc)) && (is_hex(cccccc))) { - loc=loc+11; - six_hex_to_cur_chr; - return true; - } - } - if ((cur_chr==buffer[loc+1]) && (cur_chr==buffer[loc+2]) - && (cur_chr==buffer[loc+3]) && ((loc+8)<=limit)) { - int ccc,cccc,ccccc; /* constituents of a possible expanded code */ - c=buffer[loc+4]; - cc=buffer[loc+5]; - ccc=buffer[loc+6]; - cccc=buffer[loc+7]; - ccccc=buffer[loc+8]; - if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc)) && (is_hex(ccccc))) { - loc=loc+9; - five_hex_to_cur_chr; - return true; - } - } - if ((cur_chr==buffer[loc+1]) && (cur_chr==buffer[loc+2]) && ((loc+6)<=limit)) { - int ccc,cccc; /* constituents of a possible expanded code */ - c=buffer[loc+3]; - cc=buffer[loc+4]; - ccc=buffer[loc+5]; - cccc=buffer[loc+6]; - if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc))) { - loc=loc+7; - four_hex_to_cur_chr; - return true; - } - } - c = buffer[loc+1]; - if (c<0200) { /* yes we have an expanded char */ - loc=loc+2; - if (is_hex(c) && loc<=limit) { - cc=buffer[loc]; - if (is_hex(cc)){ - incr(loc); - hex_to_cur_chr; - return true; - } +static boolean process_sup_mark(void) +{ + if (cur_chr == buffer[loc]) { + int c, cc; + if (loc < limit) { + if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2]) + && (cur_chr == buffer[loc + 3]) && (cur_chr == buffer[loc + 4]) + && ((loc + 10) <= limit)) { + int ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */ + c = buffer[loc + 5]; + cc = buffer[loc + 6]; + ccc = buffer[loc + 7]; + cccc = buffer[loc + 8]; + ccccc = buffer[loc + 9]; + cccccc = buffer[loc + 10]; + if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) + && (is_hex(cccc)) + && (is_hex(ccccc)) && (is_hex(cccccc))) { + loc = loc + 11; + six_hex_to_cur_chr; + return true; + } + } + if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2]) + && (cur_chr == buffer[loc + 3]) && ((loc + 8) <= limit)) { + int ccc, cccc, ccccc; /* constituents of a possible expanded code */ + c = buffer[loc + 4]; + cc = buffer[loc + 5]; + ccc = buffer[loc + 6]; + cccc = buffer[loc + 7]; + ccccc = buffer[loc + 8]; + if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) + && (is_hex(cccc)) && (is_hex(ccccc))) { + loc = loc + 9; + five_hex_to_cur_chr; + return true; + } + } + if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2]) + && ((loc + 6) <= limit)) { + int ccc, cccc; /* constituents of a possible expanded code */ + c = buffer[loc + 3]; + cc = buffer[loc + 4]; + ccc = buffer[loc + 5]; + cccc = buffer[loc + 6]; + if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) + && (is_hex(cccc))) { + loc = loc + 7; + four_hex_to_cur_chr; + return true; + } + } + c = buffer[loc + 1]; + if (c < 0200) { /* yes we have an expanded char */ + loc = loc + 2; + if (is_hex(c) && loc <= limit) { + cc = buffer[loc]; + if (is_hex(cc)) { + incr(loc); + hex_to_cur_chr; + return true; + } + } + cur_chr = (c < 0100 ? c + 0100 : c - 0100); + return true; + } } - cur_chr = (c<0100 ? c+0100 : c-0100); - return true; - } } - } - return false; + return false; } /* Control sequence names are scanned only when they appear in some line of @@ -390,46 +398,50 @@ process_sup_mark(void) { buffer and the process is repeated, slowly but surely. */ -static boolean check_expanded_code (integer *kk) ; /* below */ - -static int scan_control_sequence (void ) { - int retval = mid_line; - if (loc>limit) { - cur_cs=null_cs; /* |state| is irrelevant in this case */ - } else { - register int cat; /* |cat_code(cur_chr)|, usually */ - while (1) { - integer k=loc; - cur_chr = qbuffer_to_unichar(&k); - do_get_cat_code(cat); - if (cat!=letter||k>limit) { - retval = (cat==spacer ? skip_blanks : mid_line); - if (cat==sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */ - continue; - } else { - retval=skip_blanks; - do { - cur_chr = qbuffer_to_unichar(&k); - do_get_cat_code(cat); - } while (cat==letter&&k<=limit); - - if (cat==sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */ - continue; - if (cat!=letter) { - decr(k); - if (cur_chr>0xFFFF) decr(k); - if (cur_chr>0x7FF) decr(k); - if (cur_chr>0x7F) decr(k); - } /* now |k| points to first nonletter */ - } - cur_cs=id_lookup(loc,k-loc); - loc=k; - break; +static boolean check_expanded_code(integer * kk); /* below */ + +static int scan_control_sequence(void) +{ + int retval = mid_line; + if (loc > limit) { + cur_cs = null_cs; /* |state| is irrelevant in this case */ + } else { + register int cat; /* |cat_code(cur_chr)|, usually */ + while (1) { + integer k = loc; + cur_chr = qbuffer_to_unichar(&k); + do_get_cat_code(cat); + if (cat != letter || k > limit) { + retval = (cat == spacer ? skip_blanks : mid_line); + if (cat == sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */ + continue; + } else { + retval = skip_blanks; + do { + cur_chr = qbuffer_to_unichar(&k); + do_get_cat_code(cat); + } while (cat == letter && k <= limit); + + if (cat == sup_mark && check_expanded_code(&k)) /* @<If an expanded...@>; */ + continue; + if (cat != letter) { + decr(k); + if (cur_chr > 0xFFFF) + decr(k); + if (cur_chr > 0x7FF) + decr(k); + if (cur_chr > 0x7F) + decr(k); + } /* now |k| points to first nonletter */ + } + cur_cs = id_lookup(loc, k - loc); + loc = k; + break; + } } - } - cur_cmd=eq_type(cur_cs); - cur_chr=equiv(cur_cs); - return retval; + cur_cmd = eq_type(cur_cs); + cur_chr = equiv(cur_cs); + return retval; } /* Whenever we reach the following piece of code, we will have @@ -440,89 +452,97 @@ static int scan_control_sequence (void ) { the buffer left two or three places. */ -static boolean -check_expanded_code (integer *kk) { - int l; - int k = *kk; - int d =1; /* number of excess characters in an expanded code */ - int c,cc,ccc,cccc,ccccc,cccccc; /* constituents of a possible expanded code */ - if (buffer[k]==cur_chr && k<limit) { - if ((cur_chr==buffer[k+1]) && (cur_chr==buffer[k+2]) && ((k+6)<=limit)) { - d = 4; - if ((cur_chr==buffer[k+3]) && ((k+8)<=limit)) - d = 5; - if ((cur_chr==buffer[k+4]) && ((k+10)<=limit)) - d = 6; - c = buffer[k+d-1]; - cc = buffer[k+d]; - ccc = buffer[k+d+1]; - cccc= buffer[k+d+2]; - if (d==6) { - ccccc =buffer[k+d+3]; - cccccc=buffer[k+d+4]; - if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) - && is_hex(ccccc) && is_hex(cccccc)) - six_hex_to_cur_chr; - } else if (d==5) { - ccccc=buffer[k+d+3]; - if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) && is_hex(ccccc)) - five_hex_to_cur_chr; - } else { - if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)) - four_hex_to_cur_chr; - } - } else { - c=buffer[k+1]; - if (c<0200) { - d=1; - if (is_hex(c) && (k+2)<=limit) { - cc=buffer[k+2]; - if (is_hex(c) && is_hex(cc)) { - d=2; - hex_to_cur_chr; - } - } else if (c<0100) { - cur_chr=c+0100; - } else { - cur_chr=c-0100; - } - } - } - if (d>2) - d=2*d-1; - else - d++; - if (cur_chr <= 0x7F) { - buffer[k-1]= cur_chr; - } else if (cur_chr<=0x7FF) { - buffer[k-1]=0xC0 + cur_chr / 0x40; - k++; d--; - buffer[k-1]=0x80 + cur_chr % 0x40; - } else if (cur_chr<=0xFFFF){ - buffer[k-1]=0xE0 + cur_chr / 0x1000; - k++; d--; - buffer[k-1]=0x80 + (cur_chr % 0x1000) / 0x40; - k++; d--; - buffer[k-1]=0x80 + (cur_chr % 0x1000) % 0x40; - } else { - buffer[k-1]=0xF0 + cur_chr / 0x40000; - k++; d--; - buffer[k-1]=0x80 + (cur_chr % 0x40000) / 0x1000; - k++; d--; - buffer[k-1]=0x80 + ((cur_chr % 0x40000) % 0x1000) / 0x40; - k++; d--; - buffer[k-1]=0x80 + ((cur_chr % 0x40000) % 0x1000) % 0x40; - } - l = k; - limit = limit - d; - while (l<=limit) { - buffer[l]=buffer[l+d]; - l++; +static boolean check_expanded_code(integer * kk) +{ + int l; + int k = *kk; + int d = 1; /* number of excess characters in an expanded code */ + int c, cc, ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */ + if (buffer[k] == cur_chr && k < limit) { + if ((cur_chr == buffer[k + 1]) && (cur_chr == buffer[k + 2]) + && ((k + 6) <= limit)) { + d = 4; + if ((cur_chr == buffer[k + 3]) && ((k + 8) <= limit)) + d = 5; + if ((cur_chr == buffer[k + 4]) && ((k + 10) <= limit)) + d = 6; + c = buffer[k + d - 1]; + cc = buffer[k + d]; + ccc = buffer[k + d + 1]; + cccc = buffer[k + d + 2]; + if (d == 6) { + ccccc = buffer[k + d + 3]; + cccccc = buffer[k + d + 4]; + if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) + && is_hex(ccccc) && is_hex(cccccc)) + six_hex_to_cur_chr; + } else if (d == 5) { + ccccc = buffer[k + d + 3]; + if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) + && is_hex(ccccc)) + five_hex_to_cur_chr; + } else { + if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)) + four_hex_to_cur_chr; + } + } else { + c = buffer[k + 1]; + if (c < 0200) { + d = 1; + if (is_hex(c) && (k + 2) <= limit) { + cc = buffer[k + 2]; + if (is_hex(c) && is_hex(cc)) { + d = 2; + hex_to_cur_chr; + } + } else if (c < 0100) { + cur_chr = c + 0100; + } else { + cur_chr = c - 0100; + } + } + } + if (d > 2) + d = 2 * d - 1; + else + d++; + if (cur_chr <= 0x7F) { + buffer[k - 1] = cur_chr; + } else if (cur_chr <= 0x7FF) { + buffer[k - 1] = 0xC0 + cur_chr / 0x40; + k++; + d--; + buffer[k - 1] = 0x80 + cur_chr % 0x40; + } else if (cur_chr <= 0xFFFF) { + buffer[k - 1] = 0xE0 + cur_chr / 0x1000; + k++; + d--; + buffer[k - 1] = 0x80 + (cur_chr % 0x1000) / 0x40; + k++; + d--; + buffer[k - 1] = 0x80 + (cur_chr % 0x1000) % 0x40; + } else { + buffer[k - 1] = 0xF0 + cur_chr / 0x40000; + k++; + d--; + buffer[k - 1] = 0x80 + (cur_chr % 0x40000) / 0x1000; + k++; + d--; + buffer[k - 1] = 0x80 + ((cur_chr % 0x40000) % 0x1000) / 0x40; + k++; + d--; + buffer[k - 1] = 0x80 + ((cur_chr % 0x40000) % 0x1000) % 0x40; + } + l = k; + limit = limit - d; + while (l <= limit) { + buffer[l] = buffer[l + d]; + l++; + } + *kk = k; + return true; } - *kk =k; - return true; - } - return false; + return false; } #define end_line_char_inactive ((end_line_char<0)||(end_line_char>127)) @@ -540,122 +560,126 @@ check_expanded_code (integer *kk) { There is one more branch. */ -static next_line_retval -next_line (void) { - integer tab; /* a category table */ - boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */ - detokenized_line=false; - local_catcode_table=false; - if (name>17) { - /* @<Read next line of file into |buffer|, or |goto restart| if the file has ended@> */ - incr(line); - first=start; - if (!force_eof) { - if (name<=20) { - if (pseudo_input()) { /* not end of file */ - firm_up_the_line(); /* this sets |limit| */ - if ((name==19)&&(pseudo_lines(pseudo_files)==null)) - inhibit_eol=true; - } else if ((every_eof!=null)&&!eof_seen[index]) { - limit=first-1; eof_seen[index]=true; /* fake one empty line */ - if (name!=19) - begin_token_list(every_eof,every_eof_text); - return next_line_restart; - } else { - force_eof=true; - } - } else { - if (name==21) { - if (luacstring_input()) { /* not end of strings */ - firm_up_the_line(); - if ((luacstring_penultimate())||(luacstring_simple())) - inhibit_eol=true; - if (!luacstring_simple()) - state=new_line; - if (luacstring_detokenized()) { - inhibit_eol=true; - detokenized_line=true; - } else { - if (! luacstring_defaultcattable()) { - tab=luacstring_cattable(); - if (valid_catcode_table(tab)) { - local_catcode_table=true; - line_catcode_table=tab; - } - } - } - } else { - force_eof=true; - } - } else { - if (lua_input_ln(cur_file,0,true)) { /* not end of file */ - firm_up_the_line(); /* this sets |limit| */ - } else if ((every_eof!=null)&& (!eof_seen[index])) { - limit=first-1; eof_seen[index]=true; /* fake one empty line */ - begin_token_list(every_eof,every_eof_text); - return next_line_restart; - } else { - force_eof=true; - } - } - } - } - if (force_eof) { - if (tracing_nesting>0) - if ((grp_stack[in_open]!=cur_boundary)||(if_stack[in_open]!=cond_ptr)) - if (!((name==19)||(name=21))) - file_warning(); /* give warning for some unfinished groups and/or conditionals */ - if ((name>21)||(name==20)) { - if (tracefilenames) - print_char(')'); - open_parens--; - /* update_terminal();*/ /* show user that file has been read */ - } - force_eof=false; - end_file_reading(); - return next_line_restart; - } - if (inhibit_eol||end_line_char_inactive) - limit--; - else - buffer[limit]=end_line_char; - first=limit+1; - loc=start; /* ready to read */ - } else { - if (!terminal_input) { /* \.{\\read} line has ended */ - cur_cmd=0; - cur_chr=0; - return next_line_return; /* OUTER */ - } - if (input_ptr>0) { /* text was inserted during error recovery */ - end_file_reading(); - return next_line_restart; /* resume previous level */ - } - if (selector<log_only) - open_log_file(); - if (interaction>nonstop_mode) { - if (end_line_char_inactive) - limit++; - if (limit==start) { /* previous line was empty */ - tprint_nl("(Please type a command or say `\\end')"); - } - print_ln(); - first=start; - prompt_input((str_number)'*'); /* input on-line into |buffer| */ - limit=last; - if (end_line_char_inactive) - limit--; - else - buffer[limit]=end_line_char; - first=limit+1; - loc=start; +static next_line_retval next_line(void) +{ + integer tab; /* a category table */ + boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */ + detokenized_line = false; + local_catcode_table = false; + if (name > 17) { + /* @<Read next line of file into |buffer|, or |goto restart| if the file has ended@> */ + incr(line); + first = start; + if (!force_eof) { + if (name <= 20) { + if (pseudo_input()) { /* not end of file */ + firm_up_the_line(); /* this sets |limit| */ + if ((name == 19) && (pseudo_lines(pseudo_files) == null)) + inhibit_eol = true; + } else if ((every_eof != null) && !eof_seen[index]) { + limit = first - 1; + eof_seen[index] = true; /* fake one empty line */ + if (name != 19) + begin_token_list(every_eof, every_eof_text); + return next_line_restart; + } else { + force_eof = true; + } + } else { + if (name == 21) { + if (luacstring_input()) { /* not end of strings */ + firm_up_the_line(); + if ((luacstring_penultimate()) || (luacstring_simple())) + inhibit_eol = true; + if (!luacstring_simple()) + state = new_line; + if (luacstring_detokenized()) { + inhibit_eol = true; + detokenized_line = true; + } else { + if (!luacstring_defaultcattable()) { + tab = luacstring_cattable(); + if (valid_catcode_table(tab)) { + local_catcode_table = true; + line_catcode_table = tab; + } + } + } + } else { + force_eof = true; + } + } else { + if (lua_input_ln(cur_file, 0, true)) { /* not end of file */ + firm_up_the_line(); /* this sets |limit| */ + } else if ((every_eof != null) && (!eof_seen[index])) { + limit = first - 1; + eof_seen[index] = true; /* fake one empty line */ + begin_token_list(every_eof, every_eof_text); + return next_line_restart; + } else { + force_eof = true; + } + } + } + } + if (force_eof) { + if (tracing_nesting > 0) + if ((grp_stack[in_open] != cur_boundary) + || (if_stack[in_open] != cond_ptr)) + if (!((name == 19) || (name = 21))) + file_warning(); /* give warning for some unfinished groups and/or conditionals */ + if ((name > 21) || (name == 20)) { + if (tracefilenames) + print_char(')'); + open_parens--; + /* update_terminal(); *//* show user that file has been read */ + } + force_eof = false; + end_file_reading(); + return next_line_restart; + } + if (inhibit_eol || end_line_char_inactive) + limit--; + else + buffer[limit] = end_line_char; + first = limit + 1; + loc = start; /* ready to read */ } else { - fatal_error(maketexstring("*** (job aborted, no legal \\end found)")); - /* nonstop mode, which is intended for overnight batch processing, - never waits for on-line input */ + if (!terminal_input) { /* \.{\\read} line has ended */ + cur_cmd = 0; + cur_chr = 0; + return next_line_return; /* OUTER */ + } + if (input_ptr > 0) { /* text was inserted during error recovery */ + end_file_reading(); + return next_line_restart; /* resume previous level */ + } + if (selector < log_only) + open_log_file(); + if (interaction > nonstop_mode) { + if (end_line_char_inactive) + limit++; + if (limit == start) { /* previous line was empty */ + tprint_nl("(Please type a command or say `\\end')"); + } + print_ln(); + first = start; + prompt_input((str_number) '*'); /* input on-line into |buffer| */ + limit = last; + if (end_line_char_inactive) + limit--; + else + buffer[limit] = end_line_char; + first = limit + 1; + loc = start; + } else { + fatal_error(maketexstring + ("*** (job aborted, no legal \\end found)")); + /* nonstop mode, which is intended for overnight batch processing, + never waits for on-line input */ + } } - } - return next_line_ok; + return next_line_ok; } @@ -663,46 +687,46 @@ next_line (void) { /* Let's consider now what happens when |get_next| is looking at a token list. */ -static boolean -get_next_tokenlist (void) { - register halfword t; /* a token*/ - t=info(loc); - loc=link(loc); /* move to next */ - if (t>=cs_token_flag) { /* a control sequence token */ - cur_cs=t-cs_token_flag; - cur_cmd=eq_type(cur_cs); - if (cur_cmd==dont_expand) { /* @<Get the next token, suppressing expansion@>*/ - /* The present point in the program is reached only when the |expand| - routine has inserted a special marker into the input. In this special - case, |info(loc)| is known to be a control sequence token, and |link(loc)=null|. - */ - cur_cs=info(loc)-cs_token_flag; - loc=null; - cur_cmd=eq_type(cur_cs); - if (cur_cmd>max_command) { - cur_cmd=relax; - cur_chr=no_expand_flag; - return true; - } - } - cur_chr=equiv(cur_cs); - } else { - cur_cmd=t >> string_offset_bits; /* cur_cmd=t / string_offset; */ - cur_chr=t & (string_offset-1); /* cur_chr=t % string_offset; */ - switch (cur_cmd) { - case left_brace: - align_state++; - break; - case right_brace: - align_state--; - break; - case out_param: /* @<Insert macro parameter and |goto restart|@>; */ - begin_token_list(param_stack[param_start+cur_chr-1],parameter); - return false; - break; +static boolean get_next_tokenlist(void) +{ + register halfword t; /* a token */ + t = info(loc); + loc = link(loc); /* move to next */ + if (t >= cs_token_flag) { /* a control sequence token */ + cur_cs = t - cs_token_flag; + cur_cmd = eq_type(cur_cs); + if (cur_cmd == dont_expand) { /* @<Get the next token, suppressing expansion@> */ + /* The present point in the program is reached only when the |expand| + routine has inserted a special marker into the input. In this special + case, |info(loc)| is known to be a control sequence token, and |link(loc)=null|. + */ + cur_cs = info(loc) - cs_token_flag; + loc = null; + cur_cmd = eq_type(cur_cs); + if (cur_cmd > max_command) { + cur_cmd = relax; + cur_chr = no_expand_flag; + return true; + } + } + cur_chr = equiv(cur_cs); + } else { + cur_cmd = t >> string_offset_bits; /* cur_cmd=t / string_offset; */ + cur_chr = t & (string_offset - 1); /* cur_chr=t % string_offset; */ + switch (cur_cmd) { + case left_brace: + align_state++; + break; + case right_brace: + align_state--; + break; + case out_param: /* @<Insert macro parameter and |goto restart|@>; */ + begin_token_list(param_stack[param_start + cur_chr - 1], parameter); + return false; + break; + } } - } - return true; + return true; } /* Now we're ready to take the plunge into |get_next| itself. Parts of @@ -712,42 +736,42 @@ get_next_tokenlist (void) { /* sets |cur_cmd|, |cur_chr|, |cur_cs| to next token */ -void -get_next (void) { - RESTART: - cur_cs=0; - if (state!=token_list) { - /* Input from external file, |goto restart| if no input found */ - if (!get_next_file()) - goto RESTART; - } else { - if (loc==null) { - end_token_list(); - goto RESTART; /* list exhausted, resume previous level */ - } else if (!get_next_tokenlist()) { - goto RESTART; /* parameter needs to be expanded */ +void get_next(void) +{ + RESTART: + cur_cs = 0; + if (state != token_list) { + /* Input from external file, |goto restart| if no input found */ + if (!get_next_file()) + goto RESTART; + } else { + if (loc == null) { + end_token_list(); + goto RESTART; /* list exhausted, resume previous level */ + } else if (!get_next_tokenlist()) { + goto RESTART; /* parameter needs to be expanded */ + } + } + /* @<If an alignment entry has just ended, take appropriate action@> */ + if ((cur_cmd == tab_mark || cur_cmd == car_ret) && align_state == 0) { + insert_vj_template(); + goto RESTART; } - } - /* @<If an alignment entry has just ended, take appropriate action@> */ - if ((cur_cmd==tab_mark || cur_cmd==car_ret) && align_state==0) { - insert_vj_template(); - goto RESTART; - } } -void -get_token_lua (void) { - register int callback_id ; - callback_id = callback_defined(token_filter_callback); - if (callback_id!=0) { - while (state==token_list && loc==null && index!=v_template) - end_token_list(); - /* there is some stuff we don't want to see inside the callback */ - if (!(state==token_list && - ((nofilter==true) || (index==backed_up && loc!=null)))) { - do_get_token_lua(callback_id); - return; +void get_token_lua(void) +{ + register int callback_id; + callback_id = callback_defined(token_filter_callback); + if (callback_id != 0) { + while (state == token_list && loc == null && index != v_template) + end_token_list(); + /* there is some stuff we don't want to see inside the callback */ + if (!(state == token_list && + ((nofilter == true) || (index == backed_up && loc != null)))) { + do_get_token_lua(callback_id); + return; + } } - } - get_next(); + get_next(); } |