From 76e833603f3eb350549d8b472d5b0c1ebae21713 Mon Sep 17 00:00:00 2001 From: Hironobu Yamashita Date: Sat, 22 Jan 2022 17:03:22 +0000 Subject: [e][u]ptex: Distinguish 8-bit characters and Japanese characters (H. Kitagawa et al.) For better support of LaTeX3 (expl3). More details in TUGboat 41(2):329--334, 2020. git-svn-id: svn://tug.org/texlive/trunk@61692 c570f23f-e606-0410-a88d-b1316a301751 --- Build/source/texk/mendexk/ChangeLog | 4 + Build/source/texk/mendexk/fread.c | 2 +- Build/source/texk/ptexenc/ChangeLog | 7 + Build/source/texk/ptexenc/configure | 26 +- Build/source/texk/ptexenc/ptexenc.c | 140 +++++-- Build/source/texk/ptexenc/ptexenc/ptexenc.h | 10 +- Build/source/texk/ptexenc/ptexenc/unicode.h | 1 + Build/source/texk/ptexenc/unicode.c | 25 +- Build/source/texk/ptexenc/version.ac | 2 +- Build/source/texk/web2c/ChangeLog | 5 + Build/source/texk/web2c/eptexdir/ChangeLog | 9 + Build/source/texk/web2c/eptexdir/eptex.defines | 3 + Build/source/texk/web2c/eptexdir/eptex.ech | 22 +- Build/source/texk/web2c/eptexdir/etex.ch0 | 6 + Build/source/texk/web2c/eptexdir/etex.ch1 | 8 + Build/source/texk/web2c/eptexdir/pdfutils.ch | 29 +- .../texk/web2c/eptexdir/tests/printkanji-eptex.tex | 68 ++++ Build/source/texk/web2c/euptexdir/ChangeLog | 8 + Build/source/texk/web2c/euptexdir/euptex.ch1 | 18 +- Build/source/texk/web2c/euptexdir/euptex.defines | 3 + .../texk/web2c/euptexdir/pdfstrcmp-eup-post.ch | 26 +- .../texk/web2c/euptexdir/pdfstrcmp-eup-pre.ch | 12 +- Build/source/texk/web2c/lib/ChangeLog | 6 + Build/source/texk/web2c/lib/openclose.c | 7 + Build/source/texk/web2c/lib/texmfmp.c | 73 +++- Build/source/texk/web2c/pmpostdir/ChangeLog | 4 + Build/source/texk/web2c/pmpostdir/pmp.ch | 2 +- Build/source/texk/web2c/ptexdir/ChangeLog | 11 + Build/source/texk/web2c/ptexdir/kanji.h | 4 +- Build/source/texk/web2c/ptexdir/ptex-base.ch | 424 +++++++++++++++++++-- Build/source/texk/web2c/ptexdir/ptex.defines | 3 + Build/source/texk/web2c/ptexdir/ptex_version.h | 2 +- .../texk/web2c/ptexdir/tests/filename_test.sh | 45 +++ .../source/texk/web2c/ptexdir/tests/printkanji.tex | 168 ++++++++ Build/source/texk/web2c/synctexdir/ChangeLog | 4 + Build/source/texk/web2c/synctexdir/synctex.c | 21 + Build/source/texk/web2c/texmfmem.h | 6 +- Build/source/texk/web2c/uptexdir/ChangeLog | 9 + Build/source/texk/web2c/uptexdir/kanji.h | 4 +- Build/source/texk/web2c/uptexdir/uptex-m.ch | 79 ++-- Build/source/texk/web2c/uptexdir/uptex.defines | 3 + 41 files changed, 1100 insertions(+), 209 deletions(-) create mode 100644 Build/source/texk/web2c/eptexdir/tests/printkanji-eptex.tex create mode 100644 Build/source/texk/web2c/ptexdir/tests/filename_test.sh create mode 100644 Build/source/texk/web2c/ptexdir/tests/printkanji.tex (limited to 'Build/source') diff --git a/Build/source/texk/mendexk/ChangeLog b/Build/source/texk/mendexk/ChangeLog index bdcbc5c03b4..da3b5bf7624 100644 --- a/Build/source/texk/mendexk/ChangeLog +++ b/Build/source/texk/mendexk/ChangeLog @@ -1,3 +1,7 @@ +2022-01-22 Hironori Kitagawa + + * fread.c: Adapt to arguments of input_line2() in ptexenc-1.4.0. + 2022-01-18 Karl Berry * COPYRIGHT, diff --git a/Build/source/texk/mendexk/fread.c b/Build/source/texk/mendexk/fread.c index daffe7d523c..0dfad459896 100644 --- a/Build/source/texk/mendexk/fread.c +++ b/Build/source/texk/mendexk/fread.c @@ -567,7 +567,7 @@ char *mfgets(char *buf, int size, FILE *fp) { int c, len; - if ((len = input_line2(fp, (unsigned char *) buf, 0, size, &c)) == 0 + if ((len = input_line2(fp, (unsigned char *) buf, NULL, 0, size, &c)) == 0 && c != '\r' && c != '\n') return NULL; if (c == '\n' || c == '\r') { if (len+1 < size) strcat(buf+len, "\n"); diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog index 6f40a869e27..bc121f89771 100644 --- a/Build/source/texk/ptexenc/ChangeLog +++ b/Build/source/texk/ptexenc/ChangeLog @@ -1,3 +1,10 @@ +2022-01-22 Hironori Kitagawa + + * ptexenc.c, ptexenc/ptexenc.h, unicode.c, ptexenc/unicode.h: + Distinguish 8-bit characters and Japanese characters by using + flag 0x100 for Japanese char. + * version.ac: Bump to 1.4.0. + 2022-01-18 Karl Berry * version.ac: remove /dev, for TL22. diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure index 191a62891da..6464350d9e6 100755 --- a/Build/source/texk/ptexenc/configure +++ b/Build/source/texk/ptexenc/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for ptexenc 1.3.11. +# Generated by GNU Autoconf 2.71 for ptexenc 1.4.0. # # Report bugs to . # @@ -629,8 +629,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='ptexenc' PACKAGE_TARNAME='ptexenc' -PACKAGE_VERSION='1.3.11' -PACKAGE_STRING='ptexenc 1.3.11' +PACKAGE_VERSION='1.4.0' +PACKAGE_STRING='ptexenc 1.4.0' PACKAGE_BUGREPORT='tex-k@tug.org' PACKAGE_URL='' @@ -1375,7 +1375,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures ptexenc 1.3.11 to adapt to many kinds of systems. +\`configure' configures ptexenc 1.4.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1446,7 +1446,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of ptexenc 1.3.11:";; + short | recursive ) echo "Configuration of ptexenc 1.4.0:";; esac cat <<\_ACEOF @@ -1566,7 +1566,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -ptexenc configure 1.3.11 +ptexenc configure 1.4.0 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2090,7 +2090,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ptexenc $as_me 1.3.11, which was +It was created by ptexenc $as_me 1.4.0, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2850,10 +2850,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu -PTEXENCVERSION=1.3.11 +PTEXENCVERSION=1.4.0 -PTEXENC_LT_VERSINFO=4:11:3 +PTEXENC_LT_VERSINFO=5:0:4 am__api_version='1.16' @@ -8418,7 +8418,7 @@ fi # Define the identity of the package. PACKAGE='ptexenc' - VERSION='1.3.11' + VERSION='1.4.0' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -14528,7 +14528,7 @@ Usage: $0 [OPTIONS] Report bugs to ." lt_cl_version="\ -ptexenc config.lt 1.3.11 +ptexenc config.lt 1.4.0 configured by $0, generated by GNU Autoconf 2.71. Copyright (C) 2011 Free Software Foundation, Inc. @@ -16255,7 +16255,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by ptexenc $as_me 1.3.11, which was +This file was extended by ptexenc $as_me 1.4.0, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -16323,7 +16323,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -ptexenc config.status 1.3.11 +ptexenc config.status 1.4.0 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c index 91cb4fa6f5a..a5c67dc82f7 100644 --- a/Build/source/texk/ptexenc/ptexenc.c +++ b/Build/source/texk/ptexenc/ptexenc.c @@ -26,6 +26,7 @@ static int default_kanji_enc; static boolean UPTEX_enabled; +static boolean ptex_mode = false; static boolean prior_file_enc = false; #define ESC '\033' @@ -145,6 +146,13 @@ static int get_terminal_enc(void) return terminal_enc; } +/* enable ptex mode (use flag 0x100 for Japanese char) */ +void ptenc_ptex_mode (const boolean enable) +{ + //fprintf(stderr, "ptenc_ptex_mode is called! (%d)\n", enable); + ptex_mode = enable; +} + /* enable/disable UPTEX */ void enable_UPTEX (boolean enable) { @@ -253,16 +261,37 @@ boolean iskanji2(int c) } /* multi-byte char length in s[pos] */ -int multistrlen(unsigned char *s, int len, int pos) +#define DEFINE_MULTISTRLEN(SUFF,TYPE) \ +int multistrlen ## SUFF(TYPE *s, int len, int pos) \ +{ \ + s += pos; len -= pos; \ + if (is_internalUPTEX()) { \ + int ret = UTF8Slength ## SUFF(s, len); \ + if (ret < 0) return 1; \ + return ret; \ + } \ + if (len < 2) return 1; \ + if (is_internalSJIS()) { \ + if (isSJISkanji1(s[0]) && isSJISkanji2(s[1])) return 2; \ + } else { /* EUC */ \ + if (isEUCkanji1(s[0]) && isEUCkanji2(s[1])) return 2; \ + } \ + return 1; \ +} +DEFINE_MULTISTRLEN(,unsigned char); +DEFINE_MULTISTRLEN(short,unsigned short); + +/* for outputting filename (*s) to the terminal */ +int multistrlenfilename(unsigned short *s, int len, int pos) { s += pos; len -= pos; - if (is_internalUPTEX()) { - int ret = UTF8Slength(s, len); + if (terminal_enc == ENC_UTF8) { + int ret = UTF8Slengthshort(s, len); if (ret < 0) return 1; return ret; } if (len < 2) return 1; - if (is_internalSJIS()) { + if (terminal_enc == ENC_SJIS) { if (isSJISkanji1(s[0]) && isSJISkanji2(s[1])) return 2; } else { /* EUC */ if (isEUCkanji1(s[0]) && isEUCkanji2(s[1])) return 2; @@ -300,6 +329,14 @@ long fromBUFF(unsigned char *s, int len, int pos) return s[0]; } +long fromBUFFshort(unsigned short *s, int len, int pos) +{ + unsigned char sc[6]; + s += pos; len -= pos; + for (int i=0;i<(len<6 ? len : 6);i++) sc[i]=0xFF&s[i]; + return fromBUFF(sc, (len<6 ? len : 6), 0); +} + /* internal (EUC/SJIS/UPTEX) to buffer (EUC/SJIS/UTF-8) code conversion */ long toBUFF(long kcode) { @@ -473,6 +510,10 @@ static int flush (unsigned char *buff, int num, FILE *fp) /* putc() with code conversion */ int putc2(int c, FILE *fp) +/* + c in [0,255]: writes the character c, without code conversion + c in [256,511]: writes the character c-256, with code conversion +*/ { static int num[NOFILE]; /* 0 : not in Kanji @@ -481,7 +522,6 @@ int putc2(int c, FILE *fp) static unsigned char store[NOFILE][4]; const int fd = fileno(fp); int ret = c, output_enc; - #ifdef WIN32 if ((fp == stdout || fp == stderr) && (_isatty(fd) || !prior_file_enc)) { output_enc = ENC_UTF8; @@ -493,34 +533,43 @@ int putc2(int c, FILE *fp) } else output_enc = get_file_enc(); #endif - if (num[fd] > 0) { /* multi-byte char */ - if (is_internalUPTEX() && iskanji1(c)) { /* error */ - ret = flush(store[fd], num[fd], fp); - num[fd] = 0; - } - store[fd][num[fd]] = c; - num[fd]++; - if (multistrlen(store[fd], num[fd], 0) == num[fd]) { - long i = fromBUFF(store[fd], num[fd], 0); - ret = put_multibyte(toENC(i, output_enc), fp); - num[fd] = -1; - } else if ((is_internalUPTEX() && num[fd] == 4) || - (!is_internalUPTEX() && num[fd] == 2)) { /* error */ - ret = flush(store[fd], num[fd], fp); - num[fd] = -1; - } - } else if (iskanji1(c)) { /* first multi-byte char */ - if (num[fd] == 0 && output_enc == ENC_JIS) { - ret = put_multibyte(KANJI_IN, fp); - } - store[fd][0] = c; - num[fd] = 1; - } else { /* ASCII */ + if (ptex_mode && (c<256)) { if (num[fd] < 0 && output_enc == ENC_JIS) { put_multibyte(KANJI_OUT, fp); } ret = putc(c, fp); num[fd] = 0; + } else { + c &= 0xFF; + if (num[fd] > 0) { /* multi-byte char */ + if (is_internalUPTEX() && iskanji1(c)) { /* error */ + ret = flush(store[fd], num[fd], fp); + num[fd] = 0; + } + store[fd][num[fd]] = c; + num[fd]++; + if (multistrlen(store[fd], num[fd], 0) == num[fd]) { + long i = fromBUFF(store[fd], num[fd], 0); + ret = put_multibyte(toENC(i, output_enc), fp); + num[fd] = -1; + } else if ((is_internalUPTEX() && num[fd] == 4) || + (!is_internalUPTEX() && num[fd] == 2)) { /* error */ + ret = flush(store[fd], num[fd], fp); + num[fd] = -1; + } + } else if (iskanji1(c)) { /* first multi-byte char */ + if (num[fd] == 0 && output_enc == ENC_JIS) { + ret = put_multibyte(KANJI_IN, fp); + } + store[fd][0] = c; + num[fd] = 1; + } else { /* ASCII */ + if (num[fd] < 0 && output_enc == ENC_JIS) { + put_multibyte(KANJI_OUT, fp); + } + ret = putc(c, fp); + num[fd] = 0; + } } return ret; } @@ -745,8 +794,8 @@ static int infile_enc[NOFILE]; /* ENC_UNKNOWN (=0): not determined other: determined */ /* input line with encoding conversion */ -long input_line2(FILE *fp, unsigned char *buff, long pos, - const long buffsize, int *lastchar) +long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, + long pos, const long buffsize, int *lastchar) { long i = 0; static boolean injis = false; @@ -826,6 +875,9 @@ long input_line2(FILE *fp, unsigned char *buff, long pos, if (i == EOF || i == '\n' || i == '\r') injis = false; if (lastchar != NULL) *lastchar = i; + if (buff2!= NULL) for (i=pos; i<=last; i++) buff2[i] = 0; + /* buff2 is initialized */ + return last; } @@ -925,6 +977,8 @@ int nkf_close(FILE *fp) { } #define break_if_bad_utf8_second(k) if ((k<0x80)||(k>0xBF)) { i--; k='\0'; break; } +#define write_hex_if_not_ascii(c) \ + if ((c>=0x20)&&(c<=0x7E)) buffer[last++]=c; else write_hex(c); unsigned char *ptenc_from_utf8_string_to_internal_enc(const unsigned char *is) { int i; @@ -975,10 +1029,10 @@ unsigned char *ptenc_from_utf8_string_to_internal_enc(const unsigned char *is) j = (u != 0) ? toBUFF(fromUCS(u)) : 0; if (j == 0) { /* can't represent in EUC/SJIS */ if (last+16>=len) buffer = buf = xrealloc(buffer, len=last+64); - write_hex(i1); - if (i2 != '\0') write_hex(i2); - if (i3 != '\0') write_hex(i3); - if (i4 != '\0') write_hex(i4); + write_hex_if_not_ascii(i1); + if (i2 != '\0') write_hex_if_not_ascii(i2); + if (i3 != '\0') write_hex_if_not_ascii(i3); + if (i4 != '\0') write_hex_if_not_ascii(i4); } else { write_multibyte(j); } @@ -1005,7 +1059,7 @@ unsigned char *ptenc_from_internal_enc_string_to_utf8(const unsigned char *is) last_bak = last; len = strlen(is)+1; - buffer = buf = xmalloc(len*1.5); + buffer = buf = xmalloc(len*4); first = last = 0; for (i=0; i buff_len) return -3; /* overflow */ - for (i=0; i buff_len) return -3; /* overflow */ \ + for (i=0; i + + * texmfmem.h: Change type of str_pool to unsigned short for pTeX. + We use only 0--511, and use flag 0x100 for Japanese char. + 2022-01-21 Karl Berry * ac/web2c.ac (kpse_tex_progs): enable hitex by default. diff --git a/Build/source/texk/web2c/eptexdir/ChangeLog b/Build/source/texk/web2c/eptexdir/ChangeLog index 38863aa20d2..1a80cdacd32 100644 --- a/Build/source/texk/web2c/eptexdir/ChangeLog +++ b/Build/source/texk/web2c/eptexdir/ChangeLog @@ -1,3 +1,12 @@ +2022-01-22 Hironori Kitagawa + + * eptex.ech, etex.ch0, etex.ch1, pdfutils.ch: + Change type of str_pool to unsigned short. + We use only 0--511, and use flag 0x100 for Japanese char. + More details in TUGboat 41(2):329--334, 2020. + * eptex.defines: Add multistrlen{short,filename}, fromBUFFshort. + * tests/printkanji-eptex.tex: Added. + 2022-01-10 Karl Berry * am/eptex.am: do not silence tangle-sh. diff --git a/Build/source/texk/web2c/eptexdir/eptex.defines b/Build/source/texk/web2c/eptexdir/eptex.defines index d6f6c96131b..fc154904830 100644 --- a/Build/source/texk/web2c/eptexdir/eptex.defines +++ b/Build/source/texk/web2c/eptexdir/eptex.defines @@ -11,7 +11,10 @@ @define function iskanji1 (); @define function multistrlen (); +@define function multistrlenshort (); +@define function multistrlenfilename (); @define function fromBUFF (); +@define function fromBUFFshort (); @define function toBUFF (); @define function fromDVI (); diff --git a/Build/source/texk/web2c/eptexdir/eptex.ech b/Build/source/texk/web2c/eptexdir/eptex.ech index 516fecf682c..ac90a5b3ab9 100644 --- a/Build/source/texk/web2c/eptexdir/eptex.ech +++ b/Build/source/texk/web2c/eptexdir/eptex.ech @@ -472,12 +472,7 @@ begin if k = false then begin begin_diagnostic; print_nl("Unknown encoding `"); - case selector of - term_and_log: begin wterm(stringcast(name_of_file + 1)); - wlog(stringcast(name_of_file + 1)); end; - log_only: wlog(stringcast(name_of_file + 1)); - term_only: wterm(stringcast(name_of_file + 1)); - endcases; + slow_print(cur_area); slow_print(cur_name); slow_print(cur_ext); print("'"); end_diagnostic(false); end end @@ -649,6 +644,16 @@ othercases goto next_p add_glue_ref(space_ptr(r)); add_glue_ref(xspace_ptr(r)); @z +@x e-pTeX: pseudo file + buffer[last]:=w.b0; buffer[last+1]:=w.b1; + buffer[last+2]:=w.b2; buffer[last+3]:=w.b3; +@y + buffer[last]:=w.b0 mod @"100; buffer[last+1]:=w.b1 mod @"100; + buffer[last+2]:=w.b2 mod @"100; buffer[last+3]:=w.b3 mod @"100;@/ + buffer2[last]:=0; buffer2[last+1]:=0; + buffer2[last+2]:=0; buffer2[last+3]:=0; +@z + @x e-pTeX: \readline @ @= if j=1 then @@ -672,8 +677,9 @@ if j=1 then buffer[m]:=info(p) mod @'400; incr(m); p:=link(p); @y if check_kanji(info(p)) then {|wchar_token|} - begin buffer[m]:=Hi(info(p)); incr(m); - end; + begin buffer[m]:=Hi(info(p)); buffer2[m]:=1; incr(m); buffer2[m]:=1; + end + else buffer2[m]:=0; buffer[m]:=Lo(info(p)); incr(m); p:=link(p); @z diff --git a/Build/source/texk/web2c/eptexdir/etex.ch0 b/Build/source/texk/web2c/eptexdir/etex.ch0 index 43981fb0c17..2f8b79aa112 100644 --- a/Build/source/texk/web2c/eptexdir/etex.ch0 +++ b/Build/source/texk/web2c/eptexdir/etex.ch0 @@ -15,6 +15,12 @@ @d int_pars=web2c_int_pars {total number of integer parameters} @z +@x [18.???] pTeX: ensure buffer2[]=0 in primitive + for j:=0 to l-1 do buffer[first+j]:=so(str_pool[k+j]); +@y + for j:=0 to l-1 do buffer[j]:=so(str_pool[k+j]); +@z + @x [26.413] {fetch an internal parameter} label exit; diff --git a/Build/source/texk/web2c/eptexdir/etex.ch1 b/Build/source/texk/web2c/eptexdir/etex.ch1 index aa1d33c3752..5a708657ab7 100644 --- a/Build/source/texk/web2c/eptexdir/etex.ch1 +++ b/Build/source/texk/web2c/eptexdir/etex.ch1 @@ -21,6 +21,14 @@ last_node_type:=-1; last_node_subtype:=-1; @d etex_int_base=web2c_int_pars {base for \eTeX's integer parameters} @z +@x [18] + for j:=0 to l-1 do begin + buffer[j]:=Lo(so(str_pool[k+j])); buffer2[j]:=Hi(so(str_pool[k+j])); end; +@y + for j:=0 to l-1 do begin + buffer[first+j]:=Lo(so(str_pool[k+j])); buffer2[first+j]:=Hi(so(str_pool[k+j])); end; +@z + @x @d eTeX_int=badness_code+1 {first of \eTeX\ codes for integers} @y diff --git a/Build/source/texk/web2c/eptexdir/pdfutils.ch b/Build/source/texk/web2c/eptexdir/pdfutils.ch index 71493201cb9..eb567d3583b 100644 --- a/Build/source/texk/web2c/eptexdir/pdfutils.ch +++ b/Build/source/texk/web2c/eptexdir/pdfutils.ch @@ -1783,22 +1783,22 @@ if_font_char_code:begin scan_font_ident; n:=cur_val; @x procedure print_kanji(@!s:KANJI_code); {prints a single character} begin -if s>255 then - begin print_char(Hi(s)); print_char(Lo(s)); +if s>@"FF then + begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end else print_char(s); end; @y procedure print_kanji(@!s:integer); {prints a single character} begin -if s>255 then begin +if s>@"FF then begin if isprint_utf8 then begin s:=UCStoUTF8(toUCS(s)); - if BYTE1(s)<>0 then print_char(BYTE1(s)); - if BYTE2(s)<>0 then print_char(BYTE2(s)); - if BYTE3(s)<>0 then print_char(BYTE3(s)); - print_char(BYTE4(s)); + if BYTE1(s)<>0 then print_char(@"100+BYTE1(s)); + if BYTE2(s)<>0 then print_char(@"100+BYTE2(s)); + if BYTE3(s)<>0 then print_char(@"100+BYTE3(s)); + print_char(@"100+BYTE4(s)); end - else begin print_char(Hi(s)); print_char(Lo(s)); end; + else begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end; end else print_char(s); end; @@ -1853,6 +1853,7 @@ procedure compare_strings; {to implement \.{\\pdfstrcmp}} label done; var s1, s2: str_number; i1, i2, j1, j2: pool_pointer; + c1, c2: integer; save_cur_cs: pointer; begin save_cur_cs:=cur_cs; call_func(scan_toks(false, true)); @@ -1866,14 +1867,10 @@ begin i2 := str_start[s2]; j2 := str_start[s2 + 1]; while (i1 < j1) and (i2 < j2) do begin - if str_pool[i1] < str_pool[i2] then begin - cur_val := -1; - goto done; - end; - if str_pool[i1] > str_pool[i2] then begin - cur_val := 1; - goto done; - end; + if str_pool[i1]>=@"100 then c1:=str_pool[i1]-@"100 else c1:=str_pool[i1]; + if str_pool[i2]>=@"100 then c2:=str_pool[i2]-@"100 else c2:=str_pool[i2]; + if c1c2 then begin cur_val := 1; goto done; end; incr(i1); incr(i2); end; diff --git a/Build/source/texk/web2c/eptexdir/tests/printkanji-eptex.tex b/Build/source/texk/web2c/eptexdir/tests/printkanji-eptex.tex new file mode 100644 index 00000000000..67c795e7d80 --- /dev/null +++ b/Build/source/texk/web2c/eptexdir/tests/printkanji-eptex.tex @@ -0,0 +1,68 @@ +%あaª +\tracingstats=1000 +\noautoxspacing\scrollmode +\tracingscantokens=1 +\everyeof{\noexpand} +\font\x=ec-lmtt10 \x +\immediate\openout1=\jobname.out +\def\MSG#1{% + \message{\string{MSG #1\string}}% + \immediate\write17{\string{TOT #1\string}}% + \immediate\write1{#1}% +} +\def\head#1{\message{■#1.}\par\noindent\hbox{■\null}#1.\par} +\message{^^J} +\count255="80 +\loop\ifnum\count255<"100 + \catcode\count255=11\relax + \advance\count255 1\relax +\repeat + +%======== +\head{detokenize} +\def\A{^^c5^^bf ſ 顛 } + +\detokenize\expandafter{\meaning\A} +\MSG{\detokenize\expandafter{\meaning\A}} + +%======== +\head{scantokens} +\def\c{捉^^c2^^aaª} + +\scantokens{\noexpand\^^a4^^a2あ^^e3^^81^^82.^^a4^^a2あ^^e3^^81^^82.\c} +\def\a{\scantokens{\noexpand\^^a4^^a2あ^^e3^^81^^82.^^a4^^a2あ^^e3^^81^^82.\c}} + +\edef\b{\scantokens{\noexpand\^^a4^^a2あ^^e3^^81^^82.^^a4^^a2あ^^e3^^81^^82.\c}} +\MSG{\a:\b:\meaning\b} + +\def\あ{a} +\MSG{ + \ifcsname あ\endcsname Y\csname あ\endcsname\else n\fi, + \ifcsname ^^e3^^81^^82\endcsname Y\csname ^^e3^^81^^82\endcsname\else n\fi, + \ifcsname ^^a4^^a2\endcsname Y\csname ^^a4^^a2\endcsname\else n\fi} + +%======== +\head{pdfstrcmp} + +\MSG{あ \pdfstrcmp{あ}{^^e3^^81^^82} ^^e3^^81^^82} +\MSG{あ \pdfstrcmp{あ}{^^a4^^a2} ^^a4^^a2} +\MSG{捉 \pdfstrcmp{捉}{^^c2^^aa} ^^c2^^aa} +\MSG{捉 \pdfstrcmp{捉}{ª} ^^c2^^aa} +\MSG{ª \pdfstrcmp{ª}{^^c2^^aa} ^^c2^^aa} + +%======== +\head{other pdf* util.} + +\MSG{\detokenize\expandafter{\pdffiledump length 20{\jobname.tex}}} + +\MSG{\pdfmdfivesum{}} +\MSG{\pdfmdfivesum{あ}} +\MSG{\pdfmdfivesum{^^e3^^81^^82}} +\MSG{\pdfcreationdate} +\MSG{\pdffilemoddate{\jobname.tex}} +\MSG{\pdffilesize{\jobname.tex}} +\MSG{\the\pdfelapsedtime} +\MSG{\expanded{^^c2^^aaª捉.^^e3^^81^^82あ^^a4^^a2}} + +\immediate\closeout1 +\bye diff --git a/Build/source/texk/web2c/euptexdir/ChangeLog b/Build/source/texk/web2c/euptexdir/ChangeLog index 97c6e6f9500..72a0056a658 100644 --- a/Build/source/texk/web2c/euptexdir/ChangeLog +++ b/Build/source/texk/web2c/euptexdir/ChangeLog @@ -1,3 +1,11 @@ +2022-01-22 Hironori Kitagawa + + * euptex.ch1, pdfstrcmp-eup-post.ch, pdfstrcmp-eup-pre.ch: + Change type of str_pool to unsigned short. + We use only 0--511, and use flag 0x100 for Japanese char. + More details in TUGboat 41(2):329--334, 2020. + * euptex.defines: Add multistrlen{short,filename}, fromBUFFshort. + 2022-01-10 Karl Berry * am/euptex.am: do not silence tangle-sh. diff --git a/Build/source/texk/web2c/euptexdir/euptex.ch1 b/Build/source/texk/web2c/euptexdir/euptex.ch1 index 35af4241760..84f8be6cf28 100644 --- a/Build/source/texk/web2c/euptexdir/euptex.ch1 +++ b/Build/source/texk/web2c/euptexdir/euptex.ch1 @@ -67,19 +67,23 @@ if j=1 then @z @x e-pTeX: ifcsname l.28620 - begin buffer[m]:=Hi(info(p)); incr(m); - end; + begin buffer[m]:=Hi(info(p)); buffer2[m]:=1; incr(m); buffer2[m]:=1; + end + else buffer2[m]:=0; buffer[m]:=Lo(info(p)); incr(m); p:=link(p); @y begin - if BYTE1(toBUFF(info(p) mod max_cjk_val))<>0 then begin buffer[m]:=BYTE1(toBUFF(info(p) mod max_cjk_val)); incr(m); end; - if BYTE2(toBUFF(info(p) mod max_cjk_val))<>0 then begin buffer[m]:=BYTE2(toBUFF(info(p) mod max_cjk_val)); incr(m); end; - if BYTE3(toBUFF(info(p) mod max_cjk_val))<>0 then begin buffer[m]:=BYTE3(toBUFF(info(p) mod max_cjk_val)); incr(m); end; - buffer[m]:=BYTE4(toBUFF(info(p) mod max_cjk_val)); incr(m); + if BYTE1(toBUFF(info(p) mod max_cjk_val))<>0 then + begin buffer[m]:=BYTE1(toBUFF(info(p) mod max_cjk_val)); buffer2[m]:=1; incr(m); end; + if BYTE2(toBUFF(info(p) mod max_cjk_val))<>0 then + begin buffer[m]:=BYTE2(toBUFF(info(p) mod max_cjk_val)); buffer2[m]:=1; incr(m); end; + if BYTE3(toBUFF(info(p) mod max_cjk_val))<>0 then + begin buffer[m]:=BYTE3(toBUFF(info(p) mod max_cjk_val)); buffer2[m]:=1; incr(m); end; + buffer[m]:=BYTE4(toBUFF(info(p) mod max_cjk_val)); buffer2[m]:=1; incr(m); p:=link(p); end else - begin buffer[m]:=info(p) mod max_char_val; incr(m); p:=link(p); + begin buffer[m]:=info(p) mod max_char_val; buffer2[m]:=0; incr(m); p:=link(p); end; @z diff --git a/Build/source/texk/web2c/euptexdir/euptex.defines b/Build/source/texk/web2c/euptexdir/euptex.defines index 613c70e7f14..0fb52bbdd86 100644 --- a/Build/source/texk/web2c/euptexdir/euptex.defines +++ b/Build/source/texk/web2c/euptexdir/euptex.defines @@ -16,7 +16,10 @@ @define function ismultichr (); @define function multistrlen (); +@define function multistrlenshort (); +@define function multistrlenfilename (); @define function fromBUFF (); +@define function fromBUFFshort (); @define function toBUFF (); @define function fromDVI (); diff --git a/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-post.ch b/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-post.ch index 10becd15815..8c2a909e95a 100644 --- a/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-post.ch +++ b/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-post.ch @@ -17,11 +17,9 @@ @z @x - check_kcat_code(cc) then - begin if (cc=not_cjk) then cc:=other_kchar; + if (cc=not_cjk) then cc:=other_kchar; @y - ((cat>=kanji)or check_kcat_code(cc)) then - begin if cat>=kanji then cc:=cat else if (cc=not_cjk) then cc:=other_kchar; + if cat>=kanji then cc:=cat else if (cc=not_cjk) then cc:=other_kchar; @z @x @@ -97,15 +95,15 @@ Ucharcat_convert_code: @x procedure print_kanji(@!s:integer); {prints a single character} begin -if s>255 then begin +if s>@"FF then begin if isprint_utf8 then begin s:=UCStoUTF8(toUCS(s)); - if BYTE1(s)<>0 then print_char(BYTE1(s)); - if BYTE2(s)<>0 then print_char(BYTE2(s)); - if BYTE3(s)<>0 then print_char(BYTE3(s)); - print_char(BYTE4(s)); + if BYTE1(s)<>0 then print_char(@"100+BYTE1(s)); + if BYTE2(s)<>0 then print_char(@"100+BYTE2(s)); + if BYTE3(s)<>0 then print_char(@"100+BYTE3(s)); + print_char(@"100+BYTE4(s)); end - else begin print_char(Hi(s)); print_char(Lo(s)); end; + else begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end; end else print_char(s); end; @@ -114,9 +112,9 @@ procedure print_kanji(@!s:KANJI_code); {prints a single character} begin if isprint_utf8 then s:=UCStoUTF8(toUCS(s mod max_cjk_val)) else s:=toBUFF(s mod max_cjk_val); -if BYTE1(s)<>0 then print_char(BYTE1(s)); -if BYTE2(s)<>0 then print_char(BYTE2(s)); -if BYTE3(s)<>0 then print_char(BYTE3(s)); - print_char(BYTE4(s)); +if BYTE1(s)<>0 then print_char(@"100+BYTE1(s)); +if BYTE2(s)<>0 then print_char(@"100+BYTE2(s)); +if BYTE3(s)<>0 then print_char(@"100+BYTE3(s)); + print_char(@"100+BYTE4(s)); end; @z diff --git a/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-pre.ch b/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-pre.ch index b583095744c..00b3864b571 100644 --- a/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-pre.ch +++ b/Build/source/texk/web2c/euptexdir/pdfstrcmp-eup-pre.ch @@ -10,16 +10,16 @@ procedure print_kanji(@!s:KANJI_code); {prints a single character} begin s:=toBUFF(s mod max_cjk_val); -if BYTE1(s)<>0 then print_char(BYTE1(s)); -if BYTE2(s)<>0 then print_char(BYTE2(s)); -if BYTE3(s)<>0 then print_char(BYTE3(s)); - print_char(BYTE4(s)); +if BYTE1(s)<>0 then print_char(@"100+BYTE1(s)); +if BYTE2(s)<>0 then print_char(@"100+BYTE2(s)); +if BYTE3(s)<>0 then print_char(@"100+BYTE3(s)); + print_char(@"100+BYTE4(s)); end; @y procedure print_kanji(@!s:KANJI_code); {prints a single character} begin -if s>255 then - begin print_char(Hi(s)); print_char(Lo(s)); +if s>@"FF then + begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end else print_char(s); end; @z diff --git a/Build/source/texk/web2c/lib/ChangeLog b/Build/source/texk/web2c/lib/ChangeLog index da207633e79..a7a08288a07 100644 --- a/Build/source/texk/web2c/lib/ChangeLog +++ b/Build/source/texk/web2c/lib/ChangeLog @@ -1,3 +1,9 @@ +2022-01-22 Hironori Kitagawa + + * openclose.c (open_input): Reencode nameoffile to utf8 for pTeX. + * texmfmp.c: Change type of str_pool to unsigned short for pTeX. + We use only 0--511, and use flag 0x100 for Japanese char. + 2022-01-01 Akira Kakuto * printversion.c: Update copyright year. diff --git a/Build/source/texk/web2c/lib/openclose.c b/Build/source/texk/web2c/lib/openclose.c index 63fa0393528..20c3a47de73 100644 --- a/Build/source/texk/web2c/lib/openclose.c +++ b/Build/source/texk/web2c/lib/openclose.c @@ -281,6 +281,13 @@ open_input (FILE **f_ptr, int filefmt, const_string fopen_mode) must_exist); if (fname) { fullnameoffile = xstrdup(fname); +#if defined(PTEX) && !defined(WIN32) + fname0 = ptenc_from_utf8_string_to_internal_enc(fullnameoffile); + if (fname0) { + free (fullnameoffile); + fullnameoffile = fname0; + } +#endif /* If we found the file in the current directory, don't leave the `./' at the beginning of `nameoffile', since it looks dumb when `tex foo' says `(./foo.tex ... )'. On the other diff --git a/Build/source/texk/web2c/lib/texmfmp.c b/Build/source/texk/web2c/lib/texmfmp.c index 20e46c6a7f3..39de9870fd7 100644 --- a/Build/source/texk/web2c/lib/texmfmp.c +++ b/Build/source/texk/web2c/lib/texmfmp.c @@ -596,7 +596,14 @@ runsystem (const char *cmd) int allow = 0; char *safecmd = NULL; char *cmdname = NULL; +#if IS_pTeX && !defined(WIN32) + char *cmd2; +#endif int status = 0; +#if IS_pTeX && !defined(WIN32) + cmd2 = (char *)ptenc_from_internal_enc_string_to_utf8((unsigned char *)cmd); + if (!cmd2) cmd2=cmd; +#endif if (shellenabledp <= 0) { return 0; @@ -606,10 +613,18 @@ runsystem (const char *cmd) if (restrictedshell == 0) allow = 1; else +#if IS_pTeX && !defined(WIN32) + allow = shell_cmd_is_allowed (cmd2, &safecmd, &cmdname); +#else allow = shell_cmd_is_allowed (cmd, &safecmd, &cmdname); +#endif if (allow == 1) +#if IS_pTeX && !defined(WIN32) + status = system (cmd2); +#else status = system (cmd); +#endif else if (allow == 2) { /* command including a character '|' is not allowed in @@ -627,6 +642,9 @@ runsystem (const char *cmd) if (status != 0) fprintf(stderr,"system returned with code %d\n", status); +#if IS_pTeX && !defined(WIN32) + if (cmd!=cmd2) free(cmd2); +#endif if (safecmd) free (safecmd); if (cmdname) @@ -779,7 +797,7 @@ maininit (int ac, string *av) #if IS_pTeX kpse_set_program_name (argv[0], NULL); - initkanji (); + initkanji (); ptenc_ptex_mode(true); #endif #if (defined(XeTeX) || defined(pdfTeX)) && defined(WIN32) kpse_set_program_name (argv[0], NULL); @@ -1457,7 +1475,11 @@ ipcpage (int is_eof) { unsigned i; for (i=0; i 0) - strpool[poolptr++] = *s++; + strpool[poolptr++] = 0xFF&(*s++); #endif /* ! XeTeX */ return makestring(); @@ -3053,9 +3076,19 @@ makefullnamestring(void) strnumber getjobname(strnumber name) { - strnumber ret = name; + strnumber ret = name; int i, l, p; if (c_job_name != NULL) ret = maketexstring(c_job_name); +#if IS_pTeX + i = strstart[ret]; l = strstart[ret+1]; + while (i1) + for (int j=i+p; i + + * pmp.ch: Adapt to arguments of input_line2() in ptexenc-1.4.0. + 2022-01-18 Karl Berry * pmp.ch: Sync with the version 2.02 mp.w (three times). diff --git a/Build/source/texk/web2c/pmpostdir/pmp.ch b/Build/source/texk/web2c/pmpostdir/pmp.ch index 166e9b760b2..15e35fb64b1 100644 --- a/Build/source/texk/web2c/pmpostdir/pmp.ch +++ b/Build/source/texk/web2c/pmpostdir/pmp.ch @@ -100,7 +100,7 @@ static boolean mp_input_ln (MP mp, void *f) { @y static boolean mp_input_ln (MP mp, void *f ) { int i = EOF; - mp->last = input_line2((FILE *)f, mp->buffer, mp->first, mp->buf_size, &i); + mp->last = input_line2((FILE *)f, mp->buffer, NULL, mp->first, mp->buf_size, &i); if (i == EOF && errno != EINTR && mp->last == mp->first) return false; if (i != EOF && i != '\n' && i != '\r') { diff --git a/Build/source/texk/web2c/ptexdir/ChangeLog b/Build/source/texk/web2c/ptexdir/ChangeLog index c77ab5bbd3f..ce2506f302e 100644 --- a/Build/source/texk/web2c/ptexdir/ChangeLog +++ b/Build/source/texk/web2c/ptexdir/ChangeLog @@ -1,3 +1,14 @@ +2022-01-22 Hironori Kitagawa + + * ptex-base.ch: Change type of str_pool to unsigned short. + We use only 0--511, and use flag 0x100 for Japanese char. + More details in TUGboat 41(2):329--334, 2020. + * kanji.h: Adapt to arguments of input_line2() in ptexenc-1.4.0 + (for pBibTeX). + * ptex.defines: Add multistrlen{short,filename}, fromBUFFshort. + * ptex_version.h: Version p4.0.0. + * tests/filename_test.sh, tests/printkanji.tex: Added. + 2022-01-10 Karl Berry * am/ptex.am: do not silence tangle-sh. diff --git a/Build/source/texk/web2c/ptexdir/kanji.h b/Build/source/texk/web2c/ptexdir/kanji.h index 5be133ecad4..e0edeb2ed70 100644 --- a/Build/source/texk/web2c/ptexdir/kanji.h +++ b/Build/source/texk/web2c/ptexdir/kanji.h @@ -65,9 +65,9 @@ extern void init_default_kanji (const_string file_str, const_string internal_str #endif /* !PRESERVE_FPUTS */ #ifdef PBIBTEX -#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,pos,size,ptr) +#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,NULL,pos,size,ptr) #else -#define inputline2(fp,buff,pos,size) input_line2(fp,buff,pos,size,NULL) +#define inputline2(fp,buff,pos,size) input_line2(fp,buff,NULL,pos,size,NULL) #endif extern void dump_kanji (gzFile fp); diff --git a/Build/source/texk/web2c/ptexdir/ptex-base.ch b/Build/source/texk/web2c/ptexdir/ptex-base.ch index e3b15e30d58..15a76eaeca6 100644 --- a/Build/source/texk/web2c/ptexdir/ptex-base.ch +++ b/Build/source/texk/web2c/ptexdir/ptex-base.ch @@ -67,6 +67,10 @@ % based on TeX 3.141592653 (for TL21). % (2021-06-25) HY pTeX p3.9.1 Various fixes. % (2021-06-20) HK pTeX p3.10.0 Add \ucs and \toucs. +% (2022-01-22) HK pTeX p4.0.0 Distinguish 8-bit characters and Japanese characters +% for better support of LaTeX3 (expl3). +% Requires ptexenc version 1.4.0. +% More details in TUGboat 41(2):329--334, 2020. @x % Here is TeX material that gets inserted after \input webmac @@ -79,10 +83,10 @@ @d banner==TeX_banner @d banner_k==TeX_banner_k @y -@d pTeX_version=3 -@d pTeX_minor_version=10 +@d pTeX_version=4 +@d pTeX_minor_version=0 @d pTeX_revision==".0" -@d pTeX_version_string=='-p3.10.0' {current \pTeX\ version} +@d pTeX_version_string=='-p4.0.0' {current \pTeX\ version} @# @d pTeX_banner=='This is pTeX, Version 3.141592653',pTeX_version_string @d pTeX_banner_k==pTeX_banner @@ -97,6 +101,21 @@ @y @!ASCII_code=0..255; {eight-bit numbers} @!KANJI_code=0..65535; {sixteen-bit numbers} +@!ext_ASCII_code=0..32768; { only use 0--511 } +@z + +@x pTeX: xchr +xchr: array [ASCII_code] of text_char; + { specifies conversion of output characters } +@y +xchr: array [ext_ASCII_code] of ext_ASCII_code; + { specifies conversion of output characters } +@z + +@x pTeX: xchr +for i:=@'177 to @'377 do xchr[i]:=i; +@y +for i:=@'177 to @'777 do xchr[i]:=i; @z @x [3.??] l.870 - pTeX: @@ -114,6 +133,36 @@ @ Kanji code handling. @z +@x [3.??] pTeX +@= +@!buffer:^ASCII_code; {lines of characters being read} +@y +In \pTeX, we use another array |buffer2[]| to indicate which byte +is a part of a Japanese character. +|buffer2[]| is initialized to zero in reading one line from a file +(|input_ln|). |buffer2[i]| is set to one when |buffer[i]| is known +to be a part of a Japanese character, in |get_next| routine. + +@= +@!buffer:^ASCII_code; {lines of characters being read} +@!buffer2:^ASCII_code; +@z + +@x [4] +@!packed_ASCII_code = 0..255; {elements of |str_pool| array} +@y +@!packed_ASCII_code = 0..32768; {elements of |str_pool| array} + { 256..511 are used by Japanese characters } +@z + +@x [4] pTeX: str_eq_buf +while jbuffer[k] then +@y +while jbuffer2[k]*@"100+buffer[k] then +@z + @x [4.47] l.1325 - pTeX: @!init function get_strings_started:boolean; {initializes the string pool, but returns |false| if something goes wrong} @@ -126,19 +175,11 @@ label done,exit; var k,@!l:KANJI_code; {small indices or counters} @z -@x [4.49] l.1384 - pTeX: -@= - (k<" ")or(k>"~") -@y -@= - not (ismultiprn(k) or xprn[k]) -@z - @x [5.54] l.1514 - pTeX: Global variables @!trick_buf:array[0..ssup_error_line] of ASCII_code; {circular buffer for pseudoprinting} @y -@!trick_buf:array[0..ssup_error_line] of ASCII_code; {circular buffer for +@!trick_buf:array[0..ssup_error_line] of ext_ASCII_code; {circular buffer for pseudoprinting} @!trick_buf2:array[0..ssup_error_line] of 0..2; {pTeX: buffer for KANJI} @!kcode_pos: 0..2; {pTeX: denotes whether first byte or second byte of KANJI} @@ -212,14 +253,16 @@ term_only: begin wterm(xchr[s]); incr(term_offset); no_print: do_nothing; pseudo: if tally then if selector@"1FF then s:=s mod 256; +if s<256 then kcode_pos:=0 +else if kcode_pos=1 then kcode_pos:=2 +else if iskanji1(xchr[s-256]) then begin kcode_pos:=1; if (selector=term_and_log)or(selector=log_only) then if file_offset>=max_print_line-1 then @@ -299,6 +342,7 @@ else if s<256 then if selector=str_ptr) or (s<256) then print(s) +else begin j:=str_start[s]; + while j=str_ptr) or (s<256) then print(s) +else begin j:=str_start[s]; + while j=@"100 then print_char(c) else print(c); incr(j); + end; + end; +end; + +procedure slow_print_filename(@!s:integer); + {prints string |s| which represents filename, without code conversion} +var i,j,l:pool_pointer; p:integer; +begin if (s>=str_ptr) or (s<256) then print(s) +else begin i:=str_start[s]; l:=str_start[s+1]; + while i1 then + begin for j:=i to i+p-1 do print_char(@"100+(so(str_pool[j]) mod @"100)); + i:=i+p; end + else begin print(so(str_pool[i]) mod @"100); incr(i); end; + end; + end; +end; + +procedure print_quoted(@!s:integer); + {prints string |s| which represents filename, + omitting quotes and with code conversion} +var i,l:pool_pointer; j,p:integer; +begin if s<>0 then begin + i:=str_start[s]; l:=str_start[s+1]; + while i1 then begin + for j:=i to i+p-1 do print_char(@"100+(so(str_pool[j]) mod @"100)); + i:=i+p; end + else begin + if so(str_pool[i])<>"""" then print(so(str_pool[i]) mod @"100); + incr(i); end; + end; + end; +end; + +@z + @x [5.61] l.1656 - pTeX: @= if src_specials_p or file_line_error_style_p or parse_first_line_p then @@ -341,6 +443,18 @@ else @ Old versions of \TeX\ needed a procedure called |print_ASCII| whose function @z +@x [5.??] - pTeX: term_input +@p procedure term_input; {gets a line from the terminal} +@y +@p procedure@?print_unread_buffer_with_ptenc; forward;@t\2@>@/ +procedure term_input; {gets a line from the terminal} +@z +@x [5.??] - pTeX: term_input +if last<>first then for k:=first to last-1 do print(buffer[k]); +@y +if last<>first then print_unread_buffer_with_ptenc(first,last); +@z + @x @d max_halfword==@"FFFFFFF {largest allowable value in a |halfword|} @y @@ -1452,14 +1566,20 @@ end; tats @z -@x [17.???] l.???? - pTeX multibyte control symbol +@x [18] buffer2 +for k:=j to j+l-1 do append_char(buffer[k]); +@y +for k:=j to j+l-1 do append_char(buffer2[k]*@"100+buffer[k]); +@z + +@x [18.???] l.???? - pTeX multibyte control symbol procedure print_cs(@!p:integer); {prints a purported control sequence} @y procedure print_cs(@!p:integer); {prints a purported control sequence} var j, l:pool_pointer; @!cat:0..max_char_code; @z -@x +@x [18.???] else begin print_esc(text(p)); print_char(" "); end; @@ -1467,8 +1587,8 @@ else begin print_esc(text(p)); else begin l:=text(p); print_esc(l); j:=str_start[l]; l:=str_start[l+1]; if l>j+1 then begin - if l-j=multistrlen(ustringcast(str_pool), l, j) then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(str_pool), l, j))); + if (str_pool[j]>=@"100)and(l-j=multistrlenshort(str_pool, l, j)) then + begin cat:=kcat_code(kcatcodekey(fromBUFFshort(str_pool, l, j))); if (cat<>other_kchar) then print_char(" "); end else print_char(" "); end @@ -1476,6 +1596,13 @@ else begin l:=text(p); end; @z +@x [18.???] pTeX: ensure buffer2[]=0 in primitive + for j:=0 to l-1 do buffer[j]:=so(str_pool[k+j]); +@y + for j:=0 to l-1 do begin + buffer[j]:=Lo(so(str_pool[k+j])); buffer2[j]:=Hi(so(str_pool[k+j])); end; +@z + @x [18.265] l.5903 - pTeX: \jfont \tfont primitive("font",def_font,0);@/ @!@:font_}{\.{\\font} primitive@> @@ -1634,6 +1761,27 @@ if m+n<=error_line then p:=first_count+m else p:=first_count+(error_line-n-3); if trick_buf2[(p-1) mod error_line]=1 then p:=p-1; @z +@x pTeX: buffer +if j>0 then for i:=start to j-1 do + begin if i=loc then set_trick_count; + print(buffer[i]); + end +@y +if j>0 then begin + i:=start; + if (loc<=j-1)and(start<=loc) then begin + for i:=start to loc-1 do + if buffer2[i]>0 then + print_char(@"100*buffer2[i]+buffer[i]) else print(buffer[i]); + set_trick_count; print_unread_buffer_with_ptenc(loc,j); + end + else + for i:=start to j-1 do + if buffer2[i]>0 then + print_char(@"100*buffer2[i]+buffer[i]) else print(buffer[i]); + end +@z + @x [22.319] l.7157 - pTeX: adjust kanji code token @ @= begin_pseudoprint; @@ -1660,6 +1808,13 @@ else show_token_list(link(start),loc,100000); {avoid reference count} done1: @z +@x [23.???] pTeX: init the input routines +first:=buf_size; repeat buffer[first]:=0; decr(first); until first=0; +@y +first:=buf_size; repeat buffer[first]:=0; buffer2[first]:=0; decr(first); until first=0; +@z + + @x [24.341] l.7479 - pTeX: set last_chr @!cat:0..max_char_code; {|cat_code(cur_chr)|, usually} @y @@ -1681,6 +1836,8 @@ begin switch: if loc<=limit then {current line not yet finished} if multistrlen(ustringcast(buffer), limit+1, loc-1)=2 then begin cur_chr:=fromBUFF(ustringcast(buffer), limit+1, loc-1); cur_cmd:=kcat_code(kcatcodekey(cur_chr)); + for l:=loc-1 to loc-2+multistrlen(ustringcast(buffer), limit+1, loc-1) do + buffer2[l]:=1; incr(loc); end else reswitch: cur_cmd:=cat_code(cur_chr); @@ -1769,7 +1926,10 @@ end begin if loc>limit then cur_cs:=null_cs {|state| is irrelevant in this case} else begin k:=loc; cur_chr:=buffer[k]; incr(k); if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k); + begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); + for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do + buffer2[l]:=1; + incr(k); end else cat:=cat_code(cur_chr); start_cs: @@ -1832,10 +1992,10 @@ begin if buffer[k]=cur_chr then @+if cat=sup_mark then @+if k= begin repeat cur_chr:=buffer[k]; incr(k); if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k); + begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); + for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do + buffer2[l]:=1; + incr(k); end else cat:=cat_code(cur_chr); while (buffer[k]=cur_chr)and(cat=sup_mark)and(k"); {wait for user response} +@.=>@> + if last>first then + begin for k:=first to last-1 do {move line down in buffer} + buffer[k+start-first]:=buffer[k]; +@y + if start=@"100 then print_char(buffer[k]) else print(buffer[k]); + first:=limit; prompt_input("=>"); {wait for user response} +@.=>@> + if last>first then + begin for k:=first to last-1 do {move line down in buffer} + begin buffer[k+start-first]:=buffer[k]; buffer2[k+start-first]:=buffer2[k]; end; +@z + @x [24.365] l.7935 - pTeX: get_token @p procedure get_token; {sets |cur_cmd|, |cur_chr|, |cur_tok|} begin no_new_control_sequence:=false; get_next; no_new_control_sequence:=true; @@ -1982,6 +2162,12 @@ while p<>null do end; buffer[j]:=info(p) mod @'400; incr(j); p:=link(p); end; +if j>first+1 then + begin no_new_control_sequence:=false; cur_cs:=id_lookup(first,j-first); + no_new_control_sequence:=true; + end +else if j=first then cur_cs:=null_cs {the list is empty} +else cur_cs:=single_base+buffer[first] {the list has length one} @y @ @= j:=first; p:=link(r); @@ -1993,10 +2179,17 @@ while p<>null do @:TeX capacity exceeded buffer size}{\quad buffer size@> end; if check_kanji(info(p)) then {|wchar_token|} - begin buffer[j]:=Hi(info(p)); incr(j); - end; + begin buffer[j]:=Hi(info(p)); buffer2[j]:=1; incr(j); buffer2[j]:=1; + end + else buffer2[j]:=0; buffer[j]:=Lo(info(p)); incr(j); p:=link(p); end; +if j>first+1 then + begin no_new_control_sequence:=false; cur_cs:=id_lookup(first,j-first); + no_new_control_sequence:=true; + end +else if j=first then cur_cs:=null_cs {the list is empty} +else cur_cs:=single_base+buffer[first] {the list has length one} @z @x [25.380] l.8221 - pTeX: get_x_token @@ -2363,8 +2556,8 @@ help6("Dimensions can be in units of em, ex, zw, zh, in, pt, pc,")@/ if t=" " then t:=space_token else t:=other_token+t; @y - if multistrlen(ustringcast(str_pool), pool_ptr, k)=2 then - begin t:=fromBUFF(ustringcast(str_pool), pool_ptr, k); incr(k); + if t>=@"100 then + begin t:=fromBUFFshort(str_pool, pool_ptr, k); incr(k); end else if t=" " then t:=space_token else t:=other_token+t; @@ -2651,6 +2844,28 @@ else begin str_room(1); append_char(c); {contribute |c| to the current string} end; @z +@x [29.518] - print_quoted in pTeX is already defined +@d print_quoted(#) == {print string |#|, omitting quotes} +if #<>0 then + for j:=str_start[#] to str_start[#+1]-1 do + if so(str_pool[j])<>"""" then + print(so(str_pool[j])) + +@y +@z + +@x +@d append_to_name(#)==begin c:=#; if not (c="""") then begin incr(k); + if k<=file_name_size then name_of_file[k]:=xchr[c]; + end end +@y +@d append_to_name(#)==begin if (#)>=@"100 then c:=(#)-@"100 else c:=#; + { Since the type of |c| is |ASCII_code|, above if-statement might not be needed } + if not (c="""") then begin incr(k); + if k<=file_name_size then name_of_file[k]:=xchr[c]; + end end +@z + @x [29.526] l.10668 - pTeX: scan file name loop@+begin if (cur_cmd>other_char)or(cur_chr>255) then {not a character} begin back_input; goto done; @@ -2669,8 +2884,8 @@ skip_mode:=false; loop@+begin if (cur_cmd=kanji)or(cur_cmd=kana)or(cur_cmd=other_kchar) then {is kanji} begin str_room(2); - append_char(Hi(cur_chr)); {kanji upper byte} - append_char(Lo(cur_chr)); {kanji lower byte} + append_char(@"100+Hi(cur_chr)); {kanji upper byte} + append_char(@"100+Lo(cur_chr)); {kanji lower byte} end else if (cur_cmd>other_char)or(cur_chr>255) then {not an alphabet} begin back_input; goto done; @@ -2686,6 +2901,15 @@ done: end_name; name_in_progress:=false; skip_mode:=true; @z +@x [29.???] open_log_file +if buffer[l]=end_line_char then decr(l); +for k:=1 to l do print(buffer[k]); +print_ln; {now the transcript file contains the first line of input} +@y +if buffer[l]=end_line_char then decr(l); print_unread_buffer_with_ptenc(1,l+1); +print_ln; {now the transcript file contains the first line of input} +@z + @x [29.536] l.10834 - pTeX: begin if src_specials_p or file_line_error_style_p or parse_first_line_p @@ -2705,6 +2929,14 @@ else wlog(')'); @z +@x [29.???] pTeX - start_input +print_char("("); incr(open_parens); +slow_print(full_source_filename_stack[in_open]); update_terminal; +@y +print_char("("); incr(open_parens); +slow_print_filename(full_source_filename_stack[in_open]); update_terminal; +@z + @x [30.560] l.10968 - pTeX: This is called BigEndian order. @!@^BigEndian order@> @@ -6268,6 +6500,24 @@ show_code: @; show_mode: @; @z +@x dump +@!format_engine: ^text_char; +@y +@!w: four_quarters; {four ASCII codes} +@!format_engine: ^text_char; +@z + +@x undump +@!format_engine: ^text_char; +@!dummy_xord: ASCII_code; +@!dummy_xchr: text_char; +@y +@!w: four_quarters; {four ASCII codes} +@!format_engine: ^text_char; +@!dummy_xord: ASCII_code; +@!dummy_xchr: ext_ASCII_code; +@z + @x libc_free(format_engine);@/ @y @@ -6282,6 +6532,28 @@ libc_free(format_engine); undump_kanji(fmt_file); @z +@x +dump_things(str_pool[0], pool_ptr); +@y +for k:=0 to str_ptr do dump_int(str_start[k]); +k:=0; +while k+4=@"100 then + print_char(so(str_pool[str_start[str_ptr]+d])) + else print(so(str_pool[str_start[str_ptr]+d])); {N.B.: not |print_char|} + end; +@z + +@x [53.????] ignore "flag bit" in str_pool for system(3) + runsystem_ret := runsystem(conststringcast(addressof( + str_pool[str_start[str_ptr]]))); +@y + if name_of_file then libc_free(name_of_file); + name_of_file := xmalloc(cur_length * 3 + 2); + k := 0; + for d:=0 to cur_length-1 do + append_to_name(str_pool[str_start[str_ptr]+d]); + name_of_file[k+1] := 0; + runsystem_ret := runsystem(conststringcast(name_of_file+1)); +@z + @x [53.????] Implement \immediate, inhibit_glue_flag begin p:=tail; do_extension; {append a whatsit node} out_what(tail); {do the action immediately} @@ -6439,6 +6752,31 @@ else begin inhibit_glue_flag:=false; @!debug debug_format_file:=true; @+gubed; @z +@x pTeX: xchr + if eight_bit_p then + for k:=0 to 255 do + xprn[k]:=1; +end; +@y + if eight_bit_p then + for k:=0 to 255 do + xprn[k]:=1; +end; +for k:=256 to 511 do xchr[k]:=k; +@z + +@x [54/web2c.???] scan_file_name_braced + for i:=str_start[s] to str_start[s+1]-1 do + dummy := more_name(str_pool[i]); {add each read character to the current file name} +@y + for i:=str_start[s] to str_start[s+1]-1 do + if str_pool[i]>=@"100 then + begin str_room(1); append_char(str_pool[i]); + end + else + dummy := more_name(str_pool[i]); {add each read character to the current file name} +@z + @x l.26984 - pTeX @* \[54] System-dependent changes. @y @@ -7501,8 +7839,8 @@ skip_loop: do_nothing; @ @= procedure print_kanji(@!s:KANJI_code); {prints a single character} begin -if s>255 then - begin print_char(Hi(s)); print_char(Lo(s)); +if s>@"FF then + begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end else print_char(s); end; @@ -7536,5 +7874,25 @@ else begin end; end; +@ This procedure is used in printing the second line in showing contexts. +This part is not read by |get_next| yet, so we don't know which bytes +are part of Japaense characters when the procedure is called. + +@= +procedure print_unread_buffer_with_ptenc(@!f, @!l: integer); +{ print |buffer[f..l-1]| with code conversion } +var @!i,@!j: pool_pointer; @!p: integer; +begin + i:=f; + while i1 then + begin for j:=i to i+p-1 do + print_char(@"100+buffer[j]); + i:=i+p; end + else begin print(buffer[i]); incr(i); end; + end; +end; + @* \[56] System-dependent changes. @z diff --git a/Build/source/texk/web2c/ptexdir/ptex.defines b/Build/source/texk/web2c/ptexdir/ptex.defines index e087b545741..462ee6227ea 100644 --- a/Build/source/texk/web2c/ptexdir/ptex.defines +++ b/Build/source/texk/web2c/ptexdir/ptex.defines @@ -7,7 +7,10 @@ @define function iskanji1 (); @define function multistrlen (); +@define function multistrlenshort (); +@define function multistrlenfilename (); @define function fromBUFF (); +@define function fromBUFFshort (); @define function toBUFF (); @define function fromDVI (); diff --git a/Build/source/texk/web2c/ptexdir/ptex_version.h b/Build/source/texk/web2c/ptexdir/ptex_version.h index 91d996f48a6..e451d4fcd80 100644 --- a/Build/source/texk/web2c/ptexdir/ptex_version.h +++ b/Build/source/texk/web2c/ptexdir/ptex_version.h @@ -1 +1 @@ -#define PTEX_VERSION "p3.10.0" +#define PTEX_VERSION "p4.0.0" diff --git a/Build/source/texk/web2c/ptexdir/tests/filename_test.sh b/Build/source/texk/web2c/ptexdir/tests/filename_test.sh new file mode 100644 index 00000000000..4c04623462e --- /dev/null +++ b/Build/source/texk/web2c/ptexdir/tests/filename_test.sh @@ -0,0 +1,45 @@ +#!/bin/bash +cat <<'EOF' > filename_日本語.tex +% UTF-8 encoding +\def\TEST#1{\ifx#1\relax\else\immediate\write16{[\meaning#1]}\expandafter\TEST\fi} +\expandafter\TEST\jobname\relax + +\immediate\openout0=\jobname.txt +\immediate\write0{日本語} +\immediate\closeout0 + +\openin0=\jobname.txt +\ifeof0 + \immediate\write16{File `\jobname.txt' not found.} +\else + \read0 to \TEXT + \immediate\write16{TEXT=\TEXT} +\fi +\closein0 + +\input \jobname.txt + +\end + +EOF + +ln -s filename_日本語.tex "filename_日本a.tex" +eptex -synctex=1 "\input {filename_日本a}" +ls *.synctex.gz; rm *.synctex.gz +euptex -synctex=1 "\input {filename_日本a}" +ls *.synctex.gz; rm *.synctex.gz +ls "filename_日本a".* +rm "filename_日本a".* +ln -s filename_日本語.tex filename_日本ßſa.tex +euptex -synctex=1 filename_日本ßſa +ls *.synctex.gz; rm *.synctex.gz +ls filename_日本ßſa.* +rm filename_日本ßſa.* + +eptex -synctex=1 "-jobname=漢ßſa" filename_日本語 +ls *.synctex.gz; rm *.synctex.gz +ls 漢*; rm 漢* +euptex -synctex=1 "-jobname=漢ßſa" filename_日本語 +ls *.synctex.gz; rm *.synctex.gz +ls 漢*; rm 漢* +rm filename_日本語.tex diff --git a/Build/source/texk/web2c/ptexdir/tests/printkanji.tex b/Build/source/texk/web2c/ptexdir/tests/printkanji.tex new file mode 100644 index 00000000000..a8f1d4cb5a0 --- /dev/null +++ b/Build/source/texk/web2c/ptexdir/tests/printkanji.tex @@ -0,0 +1,168 @@ +%\scrollmode +\tracingstats=1000 +\noautoxspacing +\newlinechar=`\^^J +\font\x=ec-lmtt10 \x +\immediate\openout1=\jobname.out +\def\MSG#1{% + \message{\string{MSG #1\string}}% + \immediate\write17{\string{TOT #1\string}}% + \immediate\write1{#1}% +} +\def\head#1{\message{■#1.}\par\noindent\hbox{■\null}#1.\par} +\message{^^J} + +%================ +\def\A{^^c5^^bf ſ 顛 } +A: \A + +\MSG{\A} +\show\A + +%================ +\par +\head{\string\meaning} + +\edef\B{\meaning\A} +\meaning\B + +\B +\MSG{\meaning\B} +\show\B + +%================ +\head{\string\jobname} +\edef\C{:\jobname:} +\MSG{*あ*\jobname *\C*\meaning\C*} +\message{^^J} + +%================ +\catcode`\^^c5=11 +\catcode`\^^bf=11 +\catcode`\^^e1=11 +\catcode`\^^e3=11 +\catcode`\^^81=11 +\catcode`\^^82=11 + +\head{oneletter} + +\string\^^c4.\string\^^c5.\string\^^ff. +\count0=`\^^c5% +\MSG{\string\catcode`\string\^^c5 = \the\count0}% +\count0=`^^c5% +\MSG{\string\catcode`^^c5 = \the\count0}% +\count0=`^^c5^^bf% +\MSG{\string\catcode`^^c5^^bf = \the\count0}% +\count0=`顛% +\MSG{\string\catcode`顛 = \the\count0}% + +%================ +\head{csname1} + +{\def\顛{hoge}\def\^^c5^^bf{piyo} +\show\顛 +\show\^^c5^^bf +\expandafter\show\csname ^^c5^^bf\endcsname +\MSG{\string\顛=>\meaning\顛} +\MSG{\string\^^c5^^bf=>\meaning\^^c5^^bf} +\MSG{\expandafter\string\csname ^^c5^^bf\endcsname + =>\expandafter\meaning\csname ^^c5^^bf\endcsname}} + +\string\^^c5^^bf +\expandafter\string\csname ^^c5^^bf\endcsname, +\expandafter\string\csname ſ\endcsname, +\expandafter\string\csname 顛\endcsname +\MSG{\string\^^c5^^bf, + \expandafter\string\csname ^^c5^^bf\endcsname, + \expandafter\string\csname ſ\endcsname, + \expandafter\string\csname 顛\endcsname.} + +%================ + +\def\あ{hoge} +\message{^^J} +\def\TEST#1#2{% + \expandafter\def\csname#2\endcsname{piyo} + \par\toks0={#2} + \expandafter\string\csname #1\endcsname => \csname #1\endcsname,\par + \expandafter\string\csname #2\endcsname => \csname #2\endcsname,\par + \expandafter\string\csname \the\toks0\endcsname => \csname \the\toks0\endcsname. + \MSG{% + \expandafter\string\csname #1\endcsname => \csname #1\endcsname, + \expandafter\string\csname #2\endcsname => \csname #2\endcsname. + \expandafter\string\csname \the\toks0\endcsname => \csname \the\toks0\endcsname. + }% +} +\ifnum\euc"A4A2="A4A2\relax + \TEST{あ}{^^a4^^a2} +\else + \TEST{あ}{^^e3^^81^^82} +\fi + +%================ +\head{0xFF} +\catcode"FF=11 +\message{^^J} + +\def\^^ff^^c5^^ff^^bf{あ} +\edef\E{(\string\^^ff^^c5^^ff^^bf:\meaning\^^ff^^c5^^ff^^bf)} +\^^ff^^c5^^ff^^bf, \E, \string\^^ff^^c5^^ff^^bf, \meaning\E. +\MSG{\^^ff^^c5^^ff^^bf, \A, \string\^^ff^^c5^^ff^^bf, \meaning\E.} + +\def\^^ff{い} +\edef\E{(\string\^^ff :\meaning\^^ff)} +\^^ff, \E, \string\^^ff, \meaning\E. +\MSG{\^^ff, \E, \string\^^ff, \meaning\E.} + +\immediate\closeout1 + +%======== +\head{contexts} + +\errorcontextlines=10000 +\def\@{\A\undefined} +\edef\a{^^c5^^bf a\A\noexpand\@ ſ 顛 } +\edef\+#1{\noexpand\^^c5^^bf\noexpand\顛\noexpand\^^ff^^c5^^ff^^bf#1 + \noexpand\^^ff\meaning\A\noexpand\^^c5^^bf\noexpand\顛} +\+\a + +\^^ff^^c5^^ff^^bf ^^c5^^bf \^^c5^^bf ſ顛\a \^^ff ^^ff\^^c5^^bf + +\^^c5\^^ff\^^c4\^^fe + +\catcode`\^^c5=11 +\catcode`\^^be=11 +\catcode`\^^bf=11 +\catcode`\^^bd=11 +\catcode`\^^ff=11 +\font\x=ec-lmtt10 \x\scrollmode + +\def\^^c5^^bf{a} +\def\顛{b} +\def\転{c} +\message{\string\^^c5^^bf \string\^^ff^^c5} + +\def\b{\ž.\ſ.\Ž..\転.\顛.\貼.} +\b + +\ž.\ſ.\Ž..\転.\顛.\貼. + + +%======== +\head{csname2} + +SHOW \show\ſ.\show\顛.% +\expandafter\show\csname ſ\endcsname.% +\expandafter\show\csname 顛\endcsname + +STRING \string\ſ.\string\顛.% +\expandafter\string\csname ſ\endcsname.% +\expandafter\string\csname 顛\endcsname + +MEANING \meaning\ſ.\meaning\顛.% +\expandafter\meaning\csname ſ\endcsname.% +\expandafter\meaning\csname 顛\endcsname.% + +\message{\string\^^e3^^81^^82 \string\あ} + +\end diff --git a/Build/source/texk/web2c/synctexdir/ChangeLog b/Build/source/texk/web2c/synctexdir/ChangeLog index c9b70a2bb0c..67f4da43e3c 100644 --- a/Build/source/texk/web2c/synctexdir/ChangeLog +++ b/Build/source/texk/web2c/synctexdir/ChangeLog @@ -1,3 +1,7 @@ +2022-01-22 Hironori Kitagawa + + * synctex.c: Reencode nameoffile to utf8 for pTeX. + 2021-12-16 Luigi Scarso * synctex_main.c [WIN32]: #include diff --git a/Build/source/texk/web2c/synctexdir/synctex.c b/Build/source/texk/web2c/synctexdir/synctex.c index ae1334d61e2..4ec54a4494e 100644 --- a/Build/source/texk/web2c/synctexdir/synctex.c +++ b/Build/source/texk/web2c/synctexdir/synctex.c @@ -301,13 +301,34 @@ static int fsyscp_remove(char *name); #endif /* pdfTeX ... */ #endif /* _WIN32 */ +#if defined(pTeX) || defined(upTeX) || defined(epTeX) || defined(eupTeX) +#define IS_pTeX 1 +#else +#define IS_pTeX 0 +#endif + /* This macro layer was added to take luatex into account as suggested by T. Hoekwater. */ +# if IS_pTeX && !defined(_WIN32) +char *SYNCTEX_GET_JOB_NAME() +{ + char *tmp = gettexstring(jobname); + char *tmpa = ptenc_from_internal_enc_string_to_utf8(tmp); + if (tmpa) { SYNCTEX_FREE(tmp); return tmpa; } else return tmp; +} +char *SYNCTEX_GET_LOG_NAME() +{ + char *tmp = gettexstring(texmflogname); + char *tmpa = ptenc_from_internal_enc_string_to_utf8(tmp); + if (tmpa) { SYNCTEX_FREE(tmp); return tmpa; } else return tmp; +} +# else # if !defined(SYNCTEX_GET_JOB_NAME) # define SYNCTEX_GET_JOB_NAME() (gettexstring(jobname)) # endif # if !defined(SYNCTEX_GET_LOG_NAME) # define SYNCTEX_GET_LOG_NAME() (gettexstring(texmflogname)) # endif +# endif # if !defined(SYNCTEX_CURRENT_TAG) # define SYNCTEX_CURRENT_TAG (curinput.synctextagfield) # endif diff --git a/Build/source/texk/web2c/texmfmem.h b/Build/source/texk/web2c/texmfmem.h index cbe0ae95470..8957d2469e6 100644 --- a/Build/source/texk/web2c/texmfmem.h +++ b/Build/source/texk/web2c/texmfmem.h @@ -63,7 +63,7 @@ */ /* Aleph is sufficiently different to separate the definition. */ -#if !defined(Aleph) && !defined(epTeX) && !defined(eupTeX) && !defined(upTeX) +#if !defined(Aleph) && !defined(pTeX) && !defined(epTeX) && !defined(eupTeX) && !defined(upTeX) typedef union { @@ -181,7 +181,7 @@ typedef union #define qqqq v.QQQQ #endif -#else /* Aleph || epTeX || eupTeX || upTeX */ +#else /* Aleph || pTeX || epTeX || eupTeX || upTeX */ typedef union { @@ -257,4 +257,4 @@ typedef union #define gr gg.GLUE -#endif /* Aleph || epTeX || eupTeX || upTeX */ +#endif /* Aleph || pTeX || epTeX || eupTeX || upTeX */ diff --git a/Build/source/texk/web2c/uptexdir/ChangeLog b/Build/source/texk/web2c/uptexdir/ChangeLog index 34a49bf8842..3f455af6ce3 100644 --- a/Build/source/texk/web2c/uptexdir/ChangeLog +++ b/Build/source/texk/web2c/uptexdir/ChangeLog @@ -1,3 +1,12 @@ +2022-01-22 Hironori Kitagawa + + * uptex-m.ch: Change type of str_pool to unsigned short. + We use only 0--511, and use flag 0x100 for Japanese char. + More details in TUGboat 41(2):329--334, 2020. + * kanji.h: Adapt to arguments of input_line2() in ptexenc-1.4.0 + (for upBibTeX). + * uptex.defines: Add multistrlen{short,filename}, fromBUFFshort. + 2022-01-10 Karl Berry * am/uptex.am: do not silence tangle-sh. diff --git a/Build/source/texk/web2c/uptexdir/kanji.h b/Build/source/texk/web2c/uptexdir/kanji.h index 601e88d81f6..7a23e31a0b9 100644 --- a/Build/source/texk/web2c/uptexdir/kanji.h +++ b/Build/source/texk/web2c/uptexdir/kanji.h @@ -59,9 +59,9 @@ extern void init_default_kanji_select (void); #endif /* !PRESERVE_FPUTS */ #ifdef UPBIBTEX -#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,pos,size,ptr) +#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,NULL,pos,size,ptr) #else -#define inputline2(fp,buff,pos,size) input_line2(fp,buff,pos,size,NULL) +#define inputline2(fp,buff,pos,size) input_line2(fp,buff,NULL,pos,size,NULL) #endif extern void init_kanji (const_string file_str, const_string internal_str); diff --git a/Build/source/texk/web2c/uptexdir/uptex-m.ch b/Build/source/texk/web2c/uptexdir/uptex-m.ch index ab6accfbfc1..10a0cac21b5 100644 --- a/Build/source/texk/web2c/uptexdir/uptex-m.ch +++ b/Build/source/texk/web2c/uptexdir/uptex-m.ch @@ -115,9 +115,9 @@ term_only: begin @z @x -if kcode_pos=1 then kcode_pos:=2 +else if kcode_pos=1 then kcode_pos:=2 @y -if (kcode_pos=1)or((kcode_pos>=@'11)and(kcode_pos<=@'12)) +else if (kcode_pos=1)or((kcode_pos>=@'11)and(kcode_pos<=@'12)) or((kcode_pos>=@'21)and(kcode_pos<=@'23)) then incr(kcode_pos) @z @@ -388,6 +388,8 @@ if ((kcp mod @'10)>0)and(nrestmultichr(kcp)>0) then p:=p-(kcp mod @'10); if multistrlen(ustringcast(buffer), limit+1, loc-1)=2 then begin cur_chr:=fromBUFF(ustringcast(buffer), limit+1, loc-1); cur_cmd:=kcat_code(kcatcodekey(cur_chr)); + for l:=loc-1 to loc-2+multistrlen(ustringcast(buffer), limit+1, loc-1) do + buffer2[l]:=1; incr(loc); end else reswitch: cur_cmd:=cat_code(cur_chr); @@ -397,6 +399,8 @@ if ((kcp mod @'10)>0)and(nrestmultichr(kcp)>0) then p:=p-(kcp mod @'10); cur_cmd:=kcat_code(kcatcodekey(cur_chr)); if (multistrlen(ustringcast(buffer), limit+1, loc)>1) and check_kcat_code(cur_cmd) then begin if (cur_cmd=not_cjk) then cur_cmd:=other_kchar; + for l:=loc to loc-1+multistrlen(ustringcast(buffer), limit+1, loc) do + buffer2[l]:=1; loc:=loc+multistrlen(ustringcast(buffer), limit+1, loc) end else begin cur_chr:=buffer[loc]; incr(loc); @@ -428,7 +432,10 @@ hangul_code(skip_blanks),hangul_code(new_line),hangul_code(mid_kanji): @x else begin k:=loc; cur_chr:=buffer[k]; incr(k); if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k); + begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); + for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do + buffer2[l]:=1; + incr(k); end else cat:=cat_code(cur_chr); start_cs: @@ -439,6 +446,8 @@ else begin k:=loc; cat:=kcat_code(kcatcodekey(cur_chr)); if (multistrlen(ustringcast(buffer), limit+1, k)>1) and check_kcat_code(cat) then begin if (cat=not_cjk) then cat:=other_kchar; + for l:=k to k-1+multistrlen(ustringcast(buffer), limit+1, k) do + buffer2[l]:=1; k:=k+multistrlen(ustringcast(buffer), limit+1, k) end else begin {not multi-byte char} cur_chr:=buffer[k]; @@ -464,7 +473,10 @@ start_cs: @x begin repeat cur_chr:=buffer[k]; incr(k); if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k); + begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); + for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do + buffer2[l]:=1; + incr(k); end else cat:=cat_code(cur_chr); @y @@ -473,6 +485,8 @@ begin repeat cat:=kcat_code(kcatcodekey(cur_chr)); if (multistrlen(ustringcast(buffer), limit+1, k)>1) and check_kcat_code(cat) then begin if (cat=not_cjk) then cat:=other_kchar; + for l:=k to k-1+multistrlen(ustringcast(buffer), limit+1, k) do + buffer2[l]:=1; k:=k+multistrlen(ustringcast(buffer), limit+1, k) end else begin {not multi-byte char} cur_chr:=buffer[k]; @@ -518,20 +532,21 @@ if cat=other_kchar then k:=k-multilenbuffchar(cur_chr)+1; {now |k| points to fir @x if check_kanji(info(p)) then {|wchar_token|} - begin buffer[j]:=Hi(info(p)); incr(j); - end; + begin buffer[j]:=Hi(info(p)); buffer2[j]:=1; incr(j); buffer2[j]:=1; + end + else buffer2[j]:=0; buffer[j]:=Lo(info(p)); incr(j); p:=link(p); @y if check_kanji(info(p)) then {|wchar_token|} begin t:=toBUFF(info(p) mod max_cjk_val); - if BYTE1(t)<>0 then begin buffer[j]:=BYTE1(t); incr(j); end; - if BYTE2(t)<>0 then begin buffer[j]:=BYTE2(t); incr(j); end; - if BYTE3(t)<>0 then begin buffer[j]:=BYTE3(t); incr(j); end; - buffer[j]:=BYTE4(t); incr(j); + if BYTE1(t)<>0 then begin buffer[j]:=BYTE1(t); buffer2[j]:=1; incr(j); end; + if BYTE2(t)<>0 then begin buffer[j]:=BYTE2(t); buffer2[j]:=1; incr(j); end; + if BYTE3(t)<>0 then begin buffer[j]:=BYTE3(t); buffer2[j]:=1; incr(j); end; + buffer[j]:=BYTE4(t); buffer2[j]:=1; incr(j); p:=link(p); end else - begin buffer[j]:=info(p) mod max_char_val; incr(j); p:=link(p); + begin buffer[j]:=info(p) mod max_char_val; buffer2[j]:=0; incr(j); p:=link(p); end; @z @@ -606,8 +621,8 @@ begin str_room(1); p:=temp_head; link(p):=null; k:=b; while k=@"100 then + begin t:=fromBUFFshort(str_pool, pool_ptr, k); incr(k); end else if t=" " then t:=space_token else t:=other_token+t; @@ -618,15 +633,15 @@ while k1)and - check_kcat_code(cc) then - begin if (cc=not_cjk) then cc:=other_kchar; - t:=t+cc*max_cjk_val; - k:=k+multistrlen(ustringcast(str_pool), pool_ptr, k)-1; + begin t:=so(str_pool[k]); + if t>=@"180 then { there is no |wchar_token| whose code is 0--127. } + begin t:=fromBUFFshort(str_pool, pool_ptr, k); cc:=kcat_code(kcatcodekey(t)); + if (cc=not_cjk) then cc:=other_kchar; + t:=t+cc*max_cjk_val; + k:=k+multistrlenshort(str_pool, pool_ptr, k)-1; end else begin t:=so(str_pool[k]); + if t>=@"100 then t:=t-@"100; if t=" " then t:=space_token else t:=other_token+t; end; @@ -709,16 +724,16 @@ if (cur_cmd>=kanji)and(cur_cmd<=hangul) then @x if (cur_cmd=kanji)or(cur_cmd=kana)or(cur_cmd=other_kchar) then {is kanji} begin str_room(2); - append_char(Hi(cur_chr)); {kanji upper byte} - append_char(Lo(cur_chr)); {kanji lower byte} + append_char(@"100+Hi(cur_chr)); {kanji upper byte} + append_char(@"100+Lo(cur_chr)); {kanji lower byte} @y if (cur_cmd>=kanji)and(cur_cmd<=hangul) then {|wchar_token|} begin str_room(4); {4 is maximum} cur_chr:=toBUFF(cur_chr); - if BYTE1(cur_chr)<>0 then append_char(BYTE1(cur_chr)); - if BYTE2(cur_chr)<>0 then append_char(BYTE2(cur_chr)); - if BYTE3(cur_chr)<>0 then append_char(BYTE3(cur_chr)); - append_char(BYTE4(cur_chr)); + if BYTE1(cur_chr)<>0 then append_char(@"100+BYTE1(cur_chr)); + if BYTE2(cur_chr)<>0 then append_char(@"100+BYTE2(cur_chr)); + if BYTE3(cur_chr)<>0 then append_char(@"100+BYTE3(cur_chr)); + append_char(@"100+BYTE4(cur_chr)); @z @x @@ -1220,17 +1235,17 @@ begin if is_char_node(link(p)) then @x procedure print_kanji(@!s:KANJI_code); {prints a single character} begin -if s>255 then - begin print_char(Hi(s)); print_char(Lo(s)); +if s>@"FF then + begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end else print_char(s); @y procedure print_kanji(@!s:KANJI_code); {prints a single character} begin s:=toBUFF(s mod max_cjk_val); -if BYTE1(s)<>0 then print_char(BYTE1(s)); -if BYTE2(s)<>0 then print_char(BYTE2(s)); -if BYTE3(s)<>0 then print_char(BYTE3(s)); - print_char(BYTE4(s)); +if BYTE1(s)<>0 then print_char(@"100+BYTE1(s)); +if BYTE2(s)<>0 then print_char(@"100+BYTE2(s)); +if BYTE3(s)<>0 then print_char(@"100+BYTE3(s)); + print_char(@"100+BYTE4(s)); end; function check_kcat_code(@!ct:integer):integer; diff --git a/Build/source/texk/web2c/uptexdir/uptex.defines b/Build/source/texk/web2c/uptexdir/uptex.defines index 9a59b7b0a1a..8afdb7e109a 100644 --- a/Build/source/texk/web2c/uptexdir/uptex.defines +++ b/Build/source/texk/web2c/uptexdir/uptex.defines @@ -17,8 +17,11 @@ @define function ismultichr (); @define function multistrlen (); +@define function multistrlenshort (); +@define function multistrlenfilename (); @define function multibytelen (); @define function fromBUFF (); +@define function fromBUFFshort (); @define function toBUFF (); @define function fromDVI (); -- cgit v1.2.3