/* textoken.c Copyright 2006-2008 Taco Hoekwater This file is part of LuaTeX. LuaTeX is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 2 of the License, or (at your option) any later version. LuaTeX is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. You should have received a copy of the GNU General Public License along with LuaTeX; if not, see . */ #include "luatex-api.h" #include #include "tokens.h" #include "commands.h" static const char _svn_version[] = "$Id: textoken.c 2448 2009-06-08 07:43:50Z taco $ $URL: http://foundry.supelec.fr/svn/luatex/tags/beta-0.40.6/source/texk/web2c/luatexdir/tex/textoken.c $"; #define skipping 1 /* |scanner_status| when passing conditional text */ #define defining 2 /* |scanner_status| when reading a macro definition */ #define matching 3 /* |scanner_status| when reading macro arguments */ #define aligning 4 /* |scanner_status| when reading an alignment preamble */ #define absorbing 5 /* |scanner_status| when reading a balanced text */ #define right_brace_token 0x400000 /* $2^{21}\cdot|right_brace|$ */ #define cat_code_table int_par(param_cat_code_table_code) #define tracing_nesting int_par(param_tracing_nesting_code) #define end_line_char int_par(param_end_line_char_code) #define suppress_outer_error int_par(param_suppress_outer_error_code) #define every_eof get_every_eof() #define null_cs 1 /* equivalent of \.{\\csname\\endcsname} */ #define eq_level(a) zeqtb[a].hh.u.B1 #define eq_type(a) zeqtb[a].hh.u.B0 #define equiv(a) zeqtb[a].hh.v.RH /* leave an input level, re-enter the old */ #define pop_input() cur_input=input_stack[--input_ptr] #define nonstop_mode 1 #define terminal_input (name==0) /* are we reading from the terminal? */ #define special_char 1114113 /* |biggest_char+2| */ #define cur_file input_file[index] /* the current |alpha_file| variable */ #define no_expand_flag special_char /*this characterizes a special variant of |relax| */ #define detokenized_line() (line_catcode_table==NO_CAT_TABLE) extern void insert_vj_template(void); #define do_get_cat_code(a) do { \ if (line_catcode_table!=DEFAULT_CAT_TABLE) \ a=get_cat_code(line_catcode_table,cur_chr); \ else \ a=get_cat_code(cat_code_table,cur_chr); \ } while (0) /* string compare */ boolean str_eq_cstr(str_number r, const char *s, size_t l) { if (l != (size_t) length(r)) return false; return (strncmp((const char *) (str_pool + str_start_macro(r)), s, l) == 0); } int get_char_cat_code(int cur_chr) { int a; do_get_cat_code(a); return a; } static void invalid_character_error(void) { const char *hlp[] = { "A funny symbol that I can't read has just been input.", "Continue, and I'll forget that it ever happened.", NULL }; deletions_allowed = false; tex_error("Text line contains an invalid character", hlp); deletions_allowed = true; } static boolean process_sup_mark(void); /* below */ static int scan_control_sequence(void); /* below */ typedef enum { next_line_ok, next_line_return, next_line_restart } next_line_retval; static next_line_retval next_line(void); /* below */ /* @^inner loop@>*/ static void utf_error(void) { const char *hlp[] = { "A funny symbol that I can't read has just been input.", "Just continue, I'll change it to 0xFFFD.", NULL }; deletions_allowed = false; tex_error("Text line contains an invalid utf-8 sequence", hlp); deletions_allowed = true; } #define do_buffer_to_unichar(a,b) a = buffer[b] < 0x80 ? buffer[b++] : qbuffer_to_unichar(&b) static integer qbuffer_to_unichar(integer * k) { register int ch; int val = 0xFFFD; unsigned char *text = buffer + *k; if ((ch = *text++) < 0x80) { val = ch; *k += 1; } else if (ch <= 0xbf) { /* error */ *k += 1; } else if (ch <= 0xdf) { if (*text >= 0x80 && *text < 0xc0) val = ((ch & 0x1f) << 6) | (*text++ & 0x3f); *k += 2; } else if (ch <= 0xef) { if (*text >= 0x80 && *text < 0xc0 && text[1] >= 0x80 && text[1] < 0xc0) { val = ((ch & 0xf) << 12) | ((text[0] & 0x3f) << 6) | (text[1] & 0x3f); *k += 3; } } else { int w = (((ch & 0x7) << 2) | ((text[0] & 0x30) >> 4)) - 1, w2; w = (w << 6) | ((text[0] & 0xf) << 2) | ((text[1] & 0x30) >> 4); w2 = ((text[1] & 0xf) << 6) | (text[2] & 0x3f); val = w * 0x400 + w2 + 0x10000; if (*text < 0x80 || text[1] < 0x80 || text[2] < 0x80 || *text >= 0xc0 || text[1] >= 0xc0 || text[2] >= 0xc0) val = 0xFFFD; *k += 4; } if (val == 0xFFFD) utf_error(); return (val); } /* This is a very basic helper */ static char *u2s(unsigned unic) { char *buf = xmalloc(5); char *pt = buf; if (unic < 0x80) *pt++ = unic; else if (unic < 0x800) { *pt++ = 0xc0 | (unic >> 6); *pt++ = 0x80 | (unic & 0x3f); } else if (unic >= 0x110000) { *pt++ = unic - 0x110000; } else if (unic < 0x10000) { *pt++ = 0xe0 | (unic >> 12); *pt++ = 0x80 | ((unic >> 6) & 0x3f); *pt++ = 0x80 | (unic & 0x3f); } else { int u, z, y, x; unsigned val = unic - 0x10000; u = ((val & 0xf0000) >> 16) + 1; z = (val & 0x0f000) >> 12; y = (val & 0x00fc0) >> 6; x = val & 0x0003f; *pt++ = 0xf0 | (u >> 2); *pt++ = 0x80 | ((u & 3) << 4) | z; *pt++ = 0x80 | y; *pt++ = 0x80 | x; } *pt = '\0'; return buf; } /* In case you are getting bored, here is a slightly less trivial routine: Given a string of lowercase letters, like `\.{pt}' or `\.{plus}' or `\.{width}', the |scan_keyword| routine checks to see whether the next tokens of input match this string. The match must be exact, except that uppercase letters will match their lowercase counterparts; uppercase equivalents are determined by subtracting |"a"-"A"|, rather than using the |uc_code| table, since \TeX\ uses this routine only for its own limited set of keywords. If a match is found, the characters are effectively removed from the input and |true| is returned. Otherwise |false| is returned, and the input is left essentially unchanged (except for the fact that some macros may have been expanded, etc.). @^inner loop@> */ boolean scan_keyword(const char *s) { /* look for a given string */ pointer p; /* tail of the backup list */ pointer q; /* new node being added to the token list via |store_new_token| */ const char *k; /* index into |str_pool| */ pointer save_cur_cs = cur_cs; if (strlen(s) == 1) { /* @; */ do { get_x_token(); } while ((cur_cmd == spacer_cmd) || (cur_cmd == relax_cmd)); if ((cur_cs == 0) && ((cur_chr == *s) || (cur_chr == *s - 'a' + 'A'))) { return true; } else { cur_cs = save_cur_cs; back_input(); return false; } } else { p = backup_head; link(p) = null; k = s; while (*k) { get_x_token(); /* recursion is possible here */ if ((cur_cs == 0) && ((cur_chr == *k) || (cur_chr == *k - 'a' + 'A'))) { store_new_token(cur_tok); k++; } else if ((cur_cmd != spacer_cmd) || (p != backup_head)) { if (p != backup_head) { q = get_avail(); info(q) = cur_tok; link(q) = null; link(p) = q; begin_token_list(link(backup_head), backed_up); } else { back_input(); } cur_cs = save_cur_cs; return false; } } flush_list(link(backup_head)); } return true; } /* |scan_direction| has to be defined here because luatangle will output a character constant when it sees a string literal of length 1 */ #define dir_T 0 #define dir_L 1 #define dir_B 2 #define dir_R 3 #define scan_single_dir(A) do { \ if (scan_keyword("T")) A=dir_T; \ else if (scan_keyword("L")) A=dir_L; \ else if (scan_keyword("B")) A=dir_B; \ else if (scan_keyword("R")) A=dir_R; \ else { \ tex_error("Bad direction", NULL); \ cur_val=0; \ return; \ } \ } while (0) void scan_direction(void) { integer d1, d2, d3; get_x_token(); if (cur_cmd == assign_dir_cmd) { cur_val = zeqtb[cur_chr].cint; return; } else { back_input(); } scan_single_dir(d1); scan_single_dir(d2); if (dir_parallel(d1, d2)) { tex_error("Bad direction", NULL); cur_val = 0; return; } scan_single_dir(d3); get_x_token(); if (cur_cmd != spacer_cmd) back_input(); cur_val = d1 * 8 + dir_rearrange[d2] * 4 + d3; } /* We can not return |undefined_control_sequence| under some conditions * (inside |shift_case|, for example). This needs thinking. */ halfword active_to_cs(int curchr, int force) { halfword curcs; char *a, *b; char *utfbytes = xmalloc(10); int nncs = no_new_control_sequence; a = u2s(0xFFFF); utfbytes = strcpy(utfbytes, a); if (force) no_new_control_sequence = false; if (curchr > 0) { b = u2s(curchr); utfbytes = strcat(utfbytes, b); free(b); curcs = string_lookup(utfbytes, strlen(utfbytes)); } else { utfbytes[3] = '\0'; curcs = string_lookup(utfbytes, 4); } no_new_control_sequence = nncs; free(a); free(utfbytes); return curcs; } /* TODO this function should listen to \.{\\escapechar} */ #define is_active_cs(a) (length(a)>3 && \ (str_pool[str_start_macro(a)] == 0xEF) && \ (str_pool[str_start_macro(a)+1] == 0xBF) && \ (str_pool[str_start_macro(a)+2] == 0xBF)) static char *cs_to_string(pointer p) { /* prints a control sequence */ const char *s; int k = 0; static char ret[256] = { 0 }; if (p == null_cs) { ret[k++] = '\\'; s = "csname"; while (*s) { ret[k++] = *s++; } ret[k++] = '\\'; s = "endcsname"; while (*s) { ret[k++] = *s++; } ret[k] = 0; } else { str_number txt = zget_cs_text(p); s = makecstring(txt); if (is_active_cs(txt)) { s = s + 3; while (*s) { ret[k++] = *s++; } ret[k] = 0; } else { ret[k++] = '\\'; while (*s) { ret[k++] = *s++; } ret[k] = 0; } } return (char *) ret; } /* TODO this is a quick hack, will be solved differently soon */ static char *cmd_chr_to_string(int cmd, int chr) { char *s; str_number str; int sel = selector; selector = new_string; print_cmd_chr(cmd, chr); str = make_string(); s = makecstring(str); selector = sel; flush_str(str); return s; } /* Before getting into |get_next|, let's consider the subroutine that is called when an `\.{\\outer}' control sequence has been scanned or when the end of a file has been reached. These two cases are distinguished by |cur_cs|, which is zero at the end of a file. */ static int frozen_control_sequence = 0; #define frozen_cr (frozen_control_sequence+1) /* permanent `\.{\\cr}' */ #define frozen_fi (frozen_control_sequence+4) /* permanent `\.{\\fi}' */ void check_outer_validity(void) { pointer p; /* points to inserted token list */ pointer q; /* auxiliary pointer */ if (suppress_outer_error) return; if (frozen_control_sequence == 0) { frozen_control_sequence = get_nullcs() + 1 + get_hash_size(); /* hashbase=nullcs+1 */ } if (scanner_status != normal) { deletions_allowed = false; /* @; */ /* An outer control sequence that occurs in a \.{\\read} will not be reread, since the error recovery for \.{\\read} is not very powerful. */ if (cur_cs != 0) { if ((state == token_list) || (name < 1) || (name > 17)) { p = get_avail(); info(p) = cs_token_flag + cur_cs; begin_token_list(p, backed_up); /* prepare to read the control sequence again */ } cur_cmd = spacer_cmd; cur_chr = ' '; /* replace it by a space */ } if (scanner_status > skipping) { const char *errhlp[] = { "I suspect you have forgotten a `}', causing me", "to read past where you wanted me to stop.", "I'll try to recover; but if the error is serious,", "you'd better type `E' or `X' now and fix your file.", NULL }; char errmsg[256]; const char *startmsg, *scannermsg; /* @ */ runaway(); /* print a definition, argument, or preamble */ if (cur_cs == 0) { startmsg = "File ended"; } else { cur_cs = 0; startmsg = "Forbidden control sequence found"; } /* @; */ /* The recovery procedure can't be fully understood without knowing more about the \TeX\ routines that should be aborted, but we can sketch the ideas here: For a runaway definition we will insert a right brace; for a runaway preamble, we will insert a special \.{\\cr} token and a right brace; and for a runaway argument, we will set |long_state| to |outer_call| and insert \.{\\par}. */ p = get_avail(); switch (scanner_status) { case defining: scannermsg = "definition"; info(p) = right_brace_token + '}'; break; case matching: scannermsg = "use"; info(p) = par_token; long_state = outer_call_cmd; break; case aligning: scannermsg = "preamble"; info(p) = right_brace_token + '}'; q = p; p = get_avail(); link(p) = q; info(p) = cs_token_flag + frozen_cr; align_state = -1000000; break; case absorbing: scannermsg = "text"; info(p) = right_brace_token + '}'; break; default: /* can't happen */ scannermsg = "unknown"; break; } /*there are no other cases */ begin_token_list(p, inserted); snprintf(errmsg, 255, "%s while scanning %s of %s", startmsg, scannermsg, cs_to_string(warning_index)); tex_error(errmsg, errhlp); } else { char errmsg[256]; const char *errhlp_no[] = { "The file ended while I was skipping conditional text.", "This kind of error happens when you say `\\if...' and forget", "the matching `\\fi'. I've inserted a `\\fi'; this might work.", NULL }; const char *errhlp_cs[] = { "A forbidden control sequence occurred in skipped text.", "This kind of error happens when you say `\\if...' and forget", "the matching `\\fi'. I've inserted a `\\fi'; this might work.", NULL }; const char **errhlp = (const char **) errhlp_no; if (cur_cs != 0) { errhlp = errhlp_cs; cur_cs = 0; } snprintf(errmsg, 255, "Incomplete %s; all text was ignored after line %d", cmd_chr_to_string(if_test_cmd, cur_if), (int) skip_line); /* @.Incomplete \\if...@> */ cur_tok = cs_token_flag + frozen_fi; /* back up one inserted token and call |error| */ { OK_to_interrupt = false; back_input(); token_type = inserted; OK_to_interrupt = true; tex_error(errmsg, errhlp); } } deletions_allowed = true; } } static boolean get_next_file(void) { SWITCH: if (loc <= limit) { /* current line not yet finished */ do_buffer_to_unichar(cur_chr, loc); RESWITCH: if (detokenized_line()) { cur_cmd = (cur_chr == ' ' ? 10 : 12); } else { do_get_cat_code(cur_cmd); } /* @; */ /* The following 48-way switch accomplishes the scanning quickly, assuming that a decent \PASCAL\ compiler has translated the code. Note that the numeric values for |mid_line|, |skip_blanks|, and |new_line| are spaced apart from each other by |max_char_code+1|, so we can add a character's command code to the state to get a single number that characterizes both. */ switch (state + cur_cmd) { case mid_line + ignore_cmd: case skip_blanks + ignore_cmd: case new_line + ignore_cmd: case skip_blanks + spacer_cmd: case new_line + spacer_cmd: /* @ */ goto SWITCH; break; case mid_line + escape_cmd: case new_line + escape_cmd: case skip_blanks + escape_cmd: /* @; */ state = scan_control_sequence(); if (cur_cmd >= outer_call_cmd) check_outer_validity(); break; case mid_line + active_char_cmd: case new_line + active_char_cmd: case skip_blanks + active_char_cmd: /* @= outer_call_cmd) check_outer_validity(); break; case mid_line + sup_mark_cmd: case new_line + sup_mark_cmd: case skip_blanks + sup_mark_cmd: /* @; */ invalid_character_error(); return false; /* because state may be token_list now */ break; case mid_line + spacer_cmd: /* @; */ state = skip_blanks; cur_chr = ' '; break; case mid_line + car_ret_cmd: /* @; */ /* When a character of type |spacer| gets through, its character code is changed to $\.{"\ "}=@'40$. This means that the ASCII codes for tab and space, and for the space inserted at the end of a line, will be treated alike when macro parameters are being matched. We do this since such characters are indistinguishable on most computer terminal displays. */ loc = limit + 1; cur_cmd = spacer_cmd; cur_chr = ' '; break; case skip_blanks + car_ret_cmd: case mid_line + comment_cmd: case new_line + comment_cmd: case skip_blanks + comment_cmd: /* @; */ loc = limit + 1; goto SWITCH; break; case new_line + car_ret_cmd: /* @; */ loc = limit + 1; cur_cs = par_loc; cur_cmd = eq_type(cur_cs); cur_chr = equiv(cur_cs); if (cur_cmd >= outer_call_cmd) check_outer_validity(); break; case skip_blanks + left_brace_cmd: case new_line + left_brace_cmd: state = mid_line; /* fall through */ case mid_line + left_brace_cmd: align_state++; break; case skip_blanks + right_brace_cmd: case new_line + right_brace_cmd: state = mid_line; /* fall through */ case mid_line + right_brace_cmd: align_state--; break; case mid_line + math_shift_cmd: case mid_line + tab_mark_cmd: case mid_line + mac_param_cmd: case mid_line + sub_mark_cmd: case mid_line + letter_cmd: case mid_line + other_char_cmd: break; default: /* case skip_blanks + math_shift: case skip_blanks + tab_mark: case skip_blanks + mac_param: case skip_blanks + sub_mark: case skip_blanks + letter: case skip_blanks + other_char: case new_line + math_shift: case new_line + tab_mark: case new_line + mac_param: case new_line + sub_mark: case new_line + letter: case new_line + other_char: */ state = mid_line; break; } } else { if (current_ocp_lstack > 0) { pop_input(); return false; } if (name != 21) state = new_line; /* @; */ do { next_line_retval r = next_line(); if (r == next_line_return) { return true; } else if (r == next_line_restart) { return false; } } while (0); check_interrupt(); goto SWITCH; } return true; } #define is_hex(a) ((a>='0'&&a<='9')||(a>='a'&&a<='f')) #define add_nybble(a) do { \ if (a<='9') cur_chr=(cur_chr<<4)+a-'0'; \ else cur_chr=(cur_chr<<4)+a-'a'+10; \ } while (0) #define hex_to_cur_chr do { \ if (c<='9') cur_chr=c-'0'; \ else cur_chr=c-'a'+10; \ add_nybble(cc); \ } while (0) #define four_hex_to_cur_chr do { \ hex_to_cur_chr; \ add_nybble(ccc); add_nybble(cccc); \ } while (0) #define five_hex_to_cur_chr do { \ four_hex_to_cur_chr; \ add_nybble(ccccc); \ } while (0) #define six_hex_to_cur_chr do { \ five_hex_to_cur_chr; \ add_nybble(cccccc); \ } while (0) /* Notice that a code like \.{\^\^8} becomes \.x if not followed by a hex digit.*/ static boolean process_sup_mark(void) { if (cur_chr == buffer[loc]) { int c, cc; if (loc < limit) { if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2]) && (cur_chr == buffer[loc + 3]) && (cur_chr == buffer[loc + 4]) && ((loc + 10) <= limit)) { int ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */ c = buffer[loc + 5]; cc = buffer[loc + 6]; ccc = buffer[loc + 7]; cccc = buffer[loc + 8]; ccccc = buffer[loc + 9]; cccccc = buffer[loc + 10]; if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc)) && (is_hex(ccccc)) && (is_hex(cccccc))) { loc = loc + 11; six_hex_to_cur_chr; return true; } } if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2]) && (cur_chr == buffer[loc + 3]) && ((loc + 8) <= limit)) { int ccc, cccc, ccccc; /* constituents of a possible expanded code */ c = buffer[loc + 4]; cc = buffer[loc + 5]; ccc = buffer[loc + 6]; cccc = buffer[loc + 7]; ccccc = buffer[loc + 8]; if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc)) && (is_hex(ccccc))) { loc = loc + 9; five_hex_to_cur_chr; return true; } } if ((cur_chr == buffer[loc + 1]) && (cur_chr == buffer[loc + 2]) && ((loc + 6) <= limit)) { int ccc, cccc; /* constituents of a possible expanded code */ c = buffer[loc + 3]; cc = buffer[loc + 4]; ccc = buffer[loc + 5]; cccc = buffer[loc + 6]; if ((is_hex(c)) && (is_hex(cc)) && (is_hex(ccc)) && (is_hex(cccc))) { loc = loc + 7; four_hex_to_cur_chr; return true; } } c = buffer[loc + 1]; if (c < 0200) { /* yes we have an expanded char */ loc = loc + 2; if (is_hex(c) && loc <= limit) { cc = buffer[loc]; if (is_hex(cc)) { incr(loc); hex_to_cur_chr; return true; } } cur_chr = (c < 0100 ? c + 0100 : c - 0100); return true; } } } return false; } /* Control sequence names are scanned only when they appear in some line of a file; once they have been scanned the first time, their |eqtb| location serves as a unique identification, so \TeX\ doesn't need to refer to the original name any more except when it prints the equivalent in symbolic form. The program that scans a control sequence has been written carefully in order to avoid the blowups that might otherwise occur if a malicious user tried something like `\.{\\catcode\'15=0}'. The algorithm might look at |buffer[limit+1]|, but it never looks at |buffer[limit+2]|. If expanded characters like `\.{\^\^A}' or `\.{\^\^df}' appear in or just following a control sequence name, they are converted to single characters in the buffer and the process is repeated, slowly but surely. */ static boolean check_expanded_code(integer * kk); /* below */ static int scan_control_sequence(void) { int retval = mid_line; if (loc > limit) { cur_cs = null_cs; /* |state| is irrelevant in this case */ } else { register int cat; /* |cat_code(cur_chr)|, usually */ while (1) { integer k = loc; do_buffer_to_unichar(cur_chr, k); do_get_cat_code(cat); if (cat != letter_cmd || k > limit) { retval = (cat == spacer_cmd ? skip_blanks : mid_line); if (cat == sup_mark_cmd && check_expanded_code(&k)) /* @; */ continue; } else { retval = skip_blanks; do { do_buffer_to_unichar(cur_chr, k); do_get_cat_code(cat); } while (cat == letter_cmd && k <= limit); if (cat == sup_mark_cmd && check_expanded_code(&k)) /* @; */ continue; if (cat != letter_cmd) { decr(k); if (cur_chr > 0xFFFF) decr(k); if (cur_chr > 0x7FF) decr(k); if (cur_chr > 0x7F) decr(k); } /* now |k| points to first nonletter */ } cur_cs = id_lookup(loc, k - loc); loc = k; break; } } cur_cmd = eq_type(cur_cs); cur_chr = equiv(cur_cs); return retval; } /* Whenever we reach the following piece of code, we will have |cur_chr=buffer[k-1]| and |k<=limit+1| and |cat=get_cat_code(cat_code_table,cur_chr)|. If an expanded code like \.{\^\^A} or \.{\^\^df} appears in |buffer[(k-1)..(k+1)]| or |buffer[(k-1)..(k+2)]|, we will store the corresponding code in |buffer[k-1]| and shift the rest of the buffer left two or three places. */ static boolean check_expanded_code(integer * kk) { int l; int k = *kk; int d = 1; /* number of excess characters in an expanded code */ int c, cc, ccc, cccc, ccccc, cccccc; /* constituents of a possible expanded code */ if (buffer[k] == cur_chr && k < limit) { if ((cur_chr == buffer[k + 1]) && (cur_chr == buffer[k + 2]) && ((k + 6) <= limit)) { d = 4; if ((cur_chr == buffer[k + 3]) && ((k + 8) <= limit)) d = 5; if ((cur_chr == buffer[k + 4]) && ((k + 10) <= limit)) d = 6; c = buffer[k + d - 1]; cc = buffer[k + d]; ccc = buffer[k + d + 1]; cccc = buffer[k + d + 2]; if (d == 6) { ccccc = buffer[k + d + 3]; cccccc = buffer[k + d + 4]; if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) && is_hex(ccccc) && is_hex(cccccc)) six_hex_to_cur_chr; } else if (d == 5) { ccccc = buffer[k + d + 3]; if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc) && is_hex(ccccc)) five_hex_to_cur_chr; } else { if (is_hex(c) && is_hex(cc) && is_hex(ccc) && is_hex(cccc)) four_hex_to_cur_chr; } } else { c = buffer[k + 1]; if (c < 0200) { d = 1; if (is_hex(c) && (k + 2) <= limit) { cc = buffer[k + 2]; if (is_hex(c) && is_hex(cc)) { d = 2; hex_to_cur_chr; } } else if (c < 0100) { cur_chr = c + 0100; } else { cur_chr = c - 0100; } } } if (d > 2) d = 2 * d - 1; else d++; if (cur_chr <= 0x7F) { buffer[k - 1] = cur_chr; } else if (cur_chr <= 0x7FF) { buffer[k - 1] = 0xC0 + cur_chr / 0x40; k++; d--; buffer[k - 1] = 0x80 + cur_chr % 0x40; } else if (cur_chr <= 0xFFFF) { buffer[k - 1] = 0xE0 + cur_chr / 0x1000; k++; d--; buffer[k - 1] = 0x80 + (cur_chr % 0x1000) / 0x40; k++; d--; buffer[k - 1] = 0x80 + (cur_chr % 0x1000) % 0x40; } else { buffer[k - 1] = 0xF0 + cur_chr / 0x40000; k++; d--; buffer[k - 1] = 0x80 + (cur_chr % 0x40000) / 0x1000; k++; d--; buffer[k - 1] = 0x80 + ((cur_chr % 0x40000) % 0x1000) / 0x40; k++; d--; buffer[k - 1] = 0x80 + ((cur_chr % 0x40000) % 0x1000) % 0x40; } l = k; limit = limit - d; while (l <= limit) { buffer[l] = buffer[l + d]; l++; } *kk = k; return true; } return false; } #define end_line_char_inactive ((end_line_char<0)||(end_line_char>127)) /* The global variable |force_eof| is normally |false|; it is set |true| by an \.{\\endinput} command. @= @!force_eof:boolean; {should the next \.{\\input} be aborted early?} */ /* All of the easy branches of |get_next| have now been taken care of. There is one more branch. */ static next_line_retval next_line(void) { boolean inhibit_eol = false; /* a way to end a pseudo file without trailing space */ if (name > 17) { /* @ */ incr(line); first = start; if (!force_eof) { if (name <= 20) { if (pseudo_input()) { /* not end of file */ firm_up_the_line(); /* this sets |limit| */ line_catcode_table = DEFAULT_CAT_TABLE; if ((name == 19) && (pseudo_lines(pseudo_files) == null)) inhibit_eol = true; } else if ((every_eof != null) && !eof_seen[index]) { limit = first - 1; eof_seen[index] = true; /* fake one empty line */ if (name != 19) begin_token_list(every_eof, every_eof_text); return next_line_restart; } else { force_eof = true; } } else { if (name == 21) { if (luacstring_input()) { /* not end of strings */ firm_up_the_line(); line_catcode_table = luacstring_cattable(); line_partial = luacstring_partial(); if (luacstring_final_line() || line_partial || line_catcode_table == NO_CAT_TABLE) inhibit_eol = true; if (!line_partial) state = new_line; } else { force_eof = true; } } else { if (lua_input_ln(cur_file, 0, true)) { /* not end of file */ firm_up_the_line(); /* this sets |limit| */ line_catcode_table = DEFAULT_CAT_TABLE; } else if ((every_eof != null) && (!eof_seen[index])) { limit = first - 1; eof_seen[index] = true; /* fake one empty line */ begin_token_list(every_eof, every_eof_text); return next_line_restart; } else { force_eof = true; } } } } if (force_eof) { if (tracing_nesting > 0) if ((grp_stack[in_open] != cur_boundary) || (if_stack[in_open] != cond_ptr)) if (!((name == 19) || (name = 21))) file_warning(); /* give warning for some unfinished groups and/or conditionals */ if ((name > 21) || (name == 20)) { if (tracefilenames) print_char(')'); open_parens--; /* update_terminal(); *//* show user that file has been read */ } force_eof = false; if (name == 21 || /* lua input */ name == 19) { /* \scantextokens */ end_file_reading(); } else { end_file_reading(); check_outer_validity(); } return next_line_restart; } if (inhibit_eol || end_line_char_inactive) limit--; else buffer[limit] = end_line_char; first = limit + 1; loc = start; /* ready to read */ } else { if (!terminal_input) { /* \.{\\read} line has ended */ cur_cmd = 0; cur_chr = 0; return next_line_return; /* OUTER */ } if (input_ptr > 0) { /* text was inserted during error recovery */ end_file_reading(); return next_line_restart; /* resume previous level */ } if (selector < log_only) open_log_file(); if (interaction > nonstop_mode) { if (end_line_char_inactive) limit++; if (limit == start) { /* previous line was empty */ tprint_nl("(Please type a command or say `\\end')"); } print_ln(); first = start; prompt_input((str_number) '*'); /* input on-line into |buffer| */ limit = last; if (end_line_char_inactive) limit--; else buffer[limit] = end_line_char; first = limit + 1; loc = start; } else { fatal_error(maketexstring ("*** (job aborted, no legal \\end found)")); /* nonstop mode, which is intended for overnight batch processing, never waits for on-line input */ } } return next_line_ok; } /* Let's consider now what happens when |get_next| is looking at a token list. */ static boolean get_next_tokenlist(void) { register halfword t; /* a token */ t = info(loc); loc = link(loc); /* move to next */ if (t >= cs_token_flag) { /* a control sequence token */ cur_cs = t - cs_token_flag; cur_cmd = eq_type(cur_cs); if (cur_cmd >= outer_call_cmd) { if (cur_cmd == dont_expand_cmd) { /* @ */ /* The present point in the program is reached only when the |expand| routine has inserted a special marker into the input. In this special case, |info(loc)| is known to be a control sequence token, and |link(loc)=null|. */ cur_cs = info(loc) - cs_token_flag; loc = null; cur_cmd = eq_type(cur_cs); if (cur_cmd > max_command_cmd) { cur_cmd = relax_cmd; cur_chr = no_expand_flag; return true; } } else { check_outer_validity(); } } cur_chr = equiv(cur_cs); } else { cur_cmd = t >> string_offset_bits; /* cur_cmd=t / string_offset; */ cur_chr = t & (string_offset - 1); /* cur_chr=t % string_offset; */ switch (cur_cmd) { case left_brace_cmd: align_state++; break; case right_brace_cmd: align_state--; break; case out_param_cmd: /* @; */ begin_token_list(param_stack[param_start + cur_chr - 1], parameter); return false; break; } } return true; } /* Now we're ready to take the plunge into |get_next| itself. Parts of this routine are executed more often than any other instructions of \TeX. @^mastication@>@^inner loop@> */ /* sets |cur_cmd|, |cur_chr|, |cur_cs| to next token */ void get_next(void) { RESTART: cur_cs = 0; if (state != token_list) { /* Input from external file, |goto restart| if no input found */ if (!get_next_file()) goto RESTART; } else { if (loc == null) { end_token_list(); goto RESTART; /* list exhausted, resume previous level */ } else if (!get_next_tokenlist()) { goto RESTART; /* parameter needs to be expanded */ } } /* @ */ if ((cur_cmd == tab_mark_cmd || cur_cmd == car_ret_cmd) && align_state == 0) { insert_vj_template(); goto RESTART; } } void get_token_lua(void) { register int callback_id; callback_id = callback_defined(token_filter_callback); if (callback_id > 0) { while (state == token_list && loc == null && index != v_template) end_token_list(); /* there is some stuff we don't want to see inside the callback */ if (!(state == token_list && ((nofilter == true) || (index == backed_up && loc != null)))) { do_get_token_lua(callback_id); return; } } get_next(); }