diff options
author | Hironobu Yamashita <h.y.acetaminophen@gmail.com> | 2022-01-22 17:03:22 +0000 |
---|---|---|
committer | Hironobu Yamashita <h.y.acetaminophen@gmail.com> | 2022-01-22 17:03:22 +0000 |
commit | 76e833603f3eb350549d8b472d5b0c1ebae21713 (patch) | |
tree | 0adce490f86927e7a81a2f05330b9590bf80663d /Build/source/texk/ptexenc | |
parent | 87e77a4b4b35459b0bc1aa06f48f2cf0db19ae9e (diff) |
[e][u]ptex: Distinguish 8-bit characters and Japanese characters (H. Kitagawa et al.)
For better support of LaTeX3 (expl3).
More details in TUGboat 41(2):329--334, 2020.
git-svn-id: svn://tug.org/texlive/trunk@61692 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/ptexenc')
-rw-r--r-- | Build/source/texk/ptexenc/ChangeLog | 7 | ||||
-rwxr-xr-x | Build/source/texk/ptexenc/configure | 26 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/ptexenc.c | 140 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/ptexenc/ptexenc.h | 10 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/ptexenc/unicode.h | 1 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/unicode.c | 25 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/version.ac | 2 |
7 files changed, 144 insertions, 67 deletions
diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog index 6f40a869e27..bc121f89771 100644 --- a/Build/source/texk/ptexenc/ChangeLog +++ b/Build/source/texk/ptexenc/ChangeLog @@ -1,3 +1,10 @@ +2022-01-22 Hironori Kitagawa <h_kitagawa2001@yahoo.co.jp> + + * ptexenc.c, ptexenc/ptexenc.h, unicode.c, ptexenc/unicode.h: + Distinguish 8-bit characters and Japanese characters by using + flag 0x100 for Japanese char. + * version.ac: Bump to 1.4.0. + 2022-01-18 Karl Berry <karl@tug.org> * version.ac: remove /dev, for TL22. diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure index 191a62891da..6464350d9e6 100755 --- a/Build/source/texk/ptexenc/configure +++ b/Build/source/texk/ptexenc/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for ptexenc 1.3.11. +# Generated by GNU Autoconf 2.71 for ptexenc 1.4.0. # # Report bugs to <tex-k@tug.org>. # @@ -629,8 +629,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='ptexenc' PACKAGE_TARNAME='ptexenc' -PACKAGE_VERSION='1.3.11' -PACKAGE_STRING='ptexenc 1.3.11' +PACKAGE_VERSION='1.4.0' +PACKAGE_STRING='ptexenc 1.4.0' PACKAGE_BUGREPORT='tex-k@tug.org' PACKAGE_URL='' @@ -1375,7 +1375,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures ptexenc 1.3.11 to adapt to many kinds of systems. +\`configure' configures ptexenc 1.4.0 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1446,7 +1446,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of ptexenc 1.3.11:";; + short | recursive ) echo "Configuration of ptexenc 1.4.0:";; esac cat <<\_ACEOF @@ -1566,7 +1566,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -ptexenc configure 1.3.11 +ptexenc configure 1.4.0 generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2090,7 +2090,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ptexenc $as_me 1.3.11, which was +It was created by ptexenc $as_me 1.4.0, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2850,10 +2850,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu -PTEXENCVERSION=1.3.11 +PTEXENCVERSION=1.4.0 -PTEXENC_LT_VERSINFO=4:11:3 +PTEXENC_LT_VERSINFO=5:0:4 am__api_version='1.16' @@ -8418,7 +8418,7 @@ fi # Define the identity of the package. PACKAGE='ptexenc' - VERSION='1.3.11' + VERSION='1.4.0' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -14528,7 +14528,7 @@ Usage: $0 [OPTIONS] Report bugs to <bug-libtool@gnu.org>." lt_cl_version="\ -ptexenc config.lt 1.3.11 +ptexenc config.lt 1.4.0 configured by $0, generated by GNU Autoconf 2.71. Copyright (C) 2011 Free Software Foundation, Inc. @@ -16255,7 +16255,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by ptexenc $as_me 1.3.11, which was +This file was extended by ptexenc $as_me 1.4.0, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -16323,7 +16323,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -ptexenc config.status 1.3.11 +ptexenc config.status 1.4.0 configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c index 91cb4fa6f5a..a5c67dc82f7 100644 --- a/Build/source/texk/ptexenc/ptexenc.c +++ b/Build/source/texk/ptexenc/ptexenc.c @@ -26,6 +26,7 @@ static int default_kanji_enc; static boolean UPTEX_enabled; +static boolean ptex_mode = false; static boolean prior_file_enc = false; #define ESC '\033' @@ -145,6 +146,13 @@ static int get_terminal_enc(void) return terminal_enc; } +/* enable ptex mode (use flag 0x100 for Japanese char) */ +void ptenc_ptex_mode (const boolean enable) +{ + //fprintf(stderr, "ptenc_ptex_mode is called! (%d)\n", enable); + ptex_mode = enable; +} + /* enable/disable UPTEX */ void enable_UPTEX (boolean enable) { @@ -253,16 +261,37 @@ boolean iskanji2(int c) } /* multi-byte char length in s[pos] */ -int multistrlen(unsigned char *s, int len, int pos) +#define DEFINE_MULTISTRLEN(SUFF,TYPE) \ +int multistrlen ## SUFF(TYPE *s, int len, int pos) \ +{ \ + s += pos; len -= pos; \ + if (is_internalUPTEX()) { \ + int ret = UTF8Slength ## SUFF(s, len); \ + if (ret < 0) return 1; \ + return ret; \ + } \ + if (len < 2) return 1; \ + if (is_internalSJIS()) { \ + if (isSJISkanji1(s[0]) && isSJISkanji2(s[1])) return 2; \ + } else { /* EUC */ \ + if (isEUCkanji1(s[0]) && isEUCkanji2(s[1])) return 2; \ + } \ + return 1; \ +} +DEFINE_MULTISTRLEN(,unsigned char); +DEFINE_MULTISTRLEN(short,unsigned short); + +/* for outputting filename (*s) to the terminal */ +int multistrlenfilename(unsigned short *s, int len, int pos) { s += pos; len -= pos; - if (is_internalUPTEX()) { - int ret = UTF8Slength(s, len); + if (terminal_enc == ENC_UTF8) { + int ret = UTF8Slengthshort(s, len); if (ret < 0) return 1; return ret; } if (len < 2) return 1; - if (is_internalSJIS()) { + if (terminal_enc == ENC_SJIS) { if (isSJISkanji1(s[0]) && isSJISkanji2(s[1])) return 2; } else { /* EUC */ if (isEUCkanji1(s[0]) && isEUCkanji2(s[1])) return 2; @@ -300,6 +329,14 @@ long fromBUFF(unsigned char *s, int len, int pos) return s[0]; } +long fromBUFFshort(unsigned short *s, int len, int pos) +{ + unsigned char sc[6]; + s += pos; len -= pos; + for (int i=0;i<(len<6 ? len : 6);i++) sc[i]=0xFF&s[i]; + return fromBUFF(sc, (len<6 ? len : 6), 0); +} + /* internal (EUC/SJIS/UPTEX) to buffer (EUC/SJIS/UTF-8) code conversion */ long toBUFF(long kcode) { @@ -473,6 +510,10 @@ static int flush (unsigned char *buff, int num, FILE *fp) /* putc() with code conversion */ int putc2(int c, FILE *fp) +/* + c in [0,255]: writes the character c, without code conversion + c in [256,511]: writes the character c-256, with code conversion +*/ { static int num[NOFILE]; /* 0 : not in Kanji @@ -481,7 +522,6 @@ int putc2(int c, FILE *fp) static unsigned char store[NOFILE][4]; const int fd = fileno(fp); int ret = c, output_enc; - #ifdef WIN32 if ((fp == stdout || fp == stderr) && (_isatty(fd) || !prior_file_enc)) { output_enc = ENC_UTF8; @@ -493,34 +533,43 @@ int putc2(int c, FILE *fp) } else output_enc = get_file_enc(); #endif - if (num[fd] > 0) { /* multi-byte char */ - if (is_internalUPTEX() && iskanji1(c)) { /* error */ - ret = flush(store[fd], num[fd], fp); - num[fd] = 0; - } - store[fd][num[fd]] = c; - num[fd]++; - if (multistrlen(store[fd], num[fd], 0) == num[fd]) { - long i = fromBUFF(store[fd], num[fd], 0); - ret = put_multibyte(toENC(i, output_enc), fp); - num[fd] = -1; - } else if ((is_internalUPTEX() && num[fd] == 4) || - (!is_internalUPTEX() && num[fd] == 2)) { /* error */ - ret = flush(store[fd], num[fd], fp); - num[fd] = -1; - } - } else if (iskanji1(c)) { /* first multi-byte char */ - if (num[fd] == 0 && output_enc == ENC_JIS) { - ret = put_multibyte(KANJI_IN, fp); - } - store[fd][0] = c; - num[fd] = 1; - } else { /* ASCII */ + if (ptex_mode && (c<256)) { if (num[fd] < 0 && output_enc == ENC_JIS) { put_multibyte(KANJI_OUT, fp); } ret = putc(c, fp); num[fd] = 0; + } else { + c &= 0xFF; + if (num[fd] > 0) { /* multi-byte char */ + if (is_internalUPTEX() && iskanji1(c)) { /* error */ + ret = flush(store[fd], num[fd], fp); + num[fd] = 0; + } + store[fd][num[fd]] = c; + num[fd]++; + if (multistrlen(store[fd], num[fd], 0) == num[fd]) { + long i = fromBUFF(store[fd], num[fd], 0); + ret = put_multibyte(toENC(i, output_enc), fp); + num[fd] = -1; + } else if ((is_internalUPTEX() && num[fd] == 4) || + (!is_internalUPTEX() && num[fd] == 2)) { /* error */ + ret = flush(store[fd], num[fd], fp); + num[fd] = -1; + } + } else if (iskanji1(c)) { /* first multi-byte char */ + if (num[fd] == 0 && output_enc == ENC_JIS) { + ret = put_multibyte(KANJI_IN, fp); + } + store[fd][0] = c; + num[fd] = 1; + } else { /* ASCII */ + if (num[fd] < 0 && output_enc == ENC_JIS) { + put_multibyte(KANJI_OUT, fp); + } + ret = putc(c, fp); + num[fd] = 0; + } } return ret; } @@ -745,8 +794,8 @@ static int infile_enc[NOFILE]; /* ENC_UNKNOWN (=0): not determined other: determined */ /* input line with encoding conversion */ -long input_line2(FILE *fp, unsigned char *buff, long pos, - const long buffsize, int *lastchar) +long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, + long pos, const long buffsize, int *lastchar) { long i = 0; static boolean injis = false; @@ -826,6 +875,9 @@ long input_line2(FILE *fp, unsigned char *buff, long pos, if (i == EOF || i == '\n' || i == '\r') injis = false; if (lastchar != NULL) *lastchar = i; + if (buff2!= NULL) for (i=pos; i<=last; i++) buff2[i] = 0; + /* buff2 is initialized */ + return last; } @@ -925,6 +977,8 @@ int nkf_close(FILE *fp) { } #define break_if_bad_utf8_second(k) if ((k<0x80)||(k>0xBF)) { i--; k='\0'; break; } +#define write_hex_if_not_ascii(c) \ + if ((c>=0x20)&&(c<=0x7E)) buffer[last++]=c; else write_hex(c); unsigned char *ptenc_from_utf8_string_to_internal_enc(const unsigned char *is) { int i; @@ -975,10 +1029,10 @@ unsigned char *ptenc_from_utf8_string_to_internal_enc(const unsigned char *is) j = (u != 0) ? toBUFF(fromUCS(u)) : 0; if (j == 0) { /* can't represent in EUC/SJIS */ if (last+16>=len) buffer = buf = xrealloc(buffer, len=last+64); - write_hex(i1); - if (i2 != '\0') write_hex(i2); - if (i3 != '\0') write_hex(i3); - if (i4 != '\0') write_hex(i4); + write_hex_if_not_ascii(i1); + if (i2 != '\0') write_hex_if_not_ascii(i2); + if (i3 != '\0') write_hex_if_not_ascii(i3); + if (i4 != '\0') write_hex_if_not_ascii(i4); } else { write_multibyte(j); } @@ -1005,7 +1059,7 @@ unsigned char *ptenc_from_internal_enc_string_to_utf8(const unsigned char *is) last_bak = last; len = strlen(is)+1; - buffer = buf = xmalloc(len*1.5); + buffer = buf = xmalloc(len*4); first = last = 0; for (i=0; i<strlen(is); i++) { @@ -1015,15 +1069,21 @@ unsigned char *ptenc_from_internal_enc_string_to_utf8(const unsigned char *is) buffer[last++] = i1; /* ASCII */ if (i1 == '\0') goto end; continue; - case 2: - i2 = is[++i]; if (i2 == '\0') break; - u = JIStoUCS2(toJIS(HILO(i1,i2))); + case 2: /* i1: not ASCII */ + i2 = is[++i]; + if (i2 == '\0') { + write_hex(i1); continue; + } else { + u = JIStoUCS2(toJIS(HILO(i1,i2))); + if (u==0) { + write_hex(i1); write_hex_if_not_ascii(i2); continue; + } + } break; - default: + default: /* reachable only if internal code is uptex */ u = U_REPLACEMENT_CHARACTER; break; } - write_multibyte(UCStoUTF8(u)); } buffer[last] = '\0'; diff --git a/Build/source/texk/ptexenc/ptexenc/ptexenc.h b/Build/source/texk/ptexenc/ptexenc/ptexenc.h index 93004162390..e1641978237 100644 --- a/Build/source/texk/ptexenc/ptexenc/ptexenc.h +++ b/Build/source/texk/ptexenc/ptexenc/ptexenc.h @@ -32,6 +32,7 @@ extern PTENCDLL int infile_enc_auto; /* enable/disable UPTEX */ extern PTENCDLL void enable_UPTEX (boolean enable); extern PTENCDLL void set_prior_file_enc(void); +extern PTENCDLL void ptenc_ptex_mode(const boolean enable); /* get/set Kanji encoding by string */ extern PTENCDLL const_string get_enc_string(void); @@ -58,10 +59,15 @@ extern PTENCDLL boolean iskanji2(int c); /* internal (EUC/SJIS/UPTEX) from/to buffer (EUC/SJIS/UTF-8) code conversion */ extern PTENCDLL int multistrlen(unsigned char *s, int len, int pos); +extern PTENCDLL int multistrlenshort(unsigned short *s, int len, int pos); extern PTENCDLL int multibytelen (int first_byte); extern PTENCDLL long fromBUFF(unsigned char *s, int len, int pos); +extern PTENCDLL long fromBUFFshort(unsigned short *s, int len, int pos); extern PTENCDLL long toBUFF(long inter); +/* for outputting filename (*s) to the terminal */ +extern PTENCDLL int multistrlenfilename(unsigned short *s, int len, int pos); + /* internal (EUC/SJIS/UPTEX) from/to DVI (JIS/UCS) code conversion */ extern PTENCDLL long toDVI (long kcode); extern PTENCDLL long fromDVI (long kcode); @@ -81,8 +87,8 @@ extern PTENCDLL int putc2(int c, FILE *fp); extern PTENCDLL int fputs2(const char *s, FILE *fp); /* input line with encoding conversion */ -extern PTENCDLL long input_line2(FILE *fp, unsigned char *buff, long pos, - const long buffsize, int *lastchar); +extern PTENCDLL long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, + long pos, const long buffsize, int *lastchar); /* set current encoding */ extern PTENCDLL boolean setinfileenc(FILE *fp, const char *str); diff --git a/Build/source/texk/ptexenc/ptexenc/unicode.h b/Build/source/texk/ptexenc/ptexenc/unicode.h index 9e4c91ff3fb..61552fd6251 100644 --- a/Build/source/texk/ptexenc/ptexenc/unicode.h +++ b/Build/source/texk/ptexenc/ptexenc/unicode.h @@ -14,6 +14,7 @@ extern boolean isUTF8(int length, int nth, int c); extern int UTF8length(int first_byte); extern int UTF8Slength(unsigned char *buff, int buff_len); +extern int UTF8Slengthshort(unsigned short *buff, int buff_len); extern long UTF8StoUCS(unsigned char *s); extern long PTENCDLL UCStoUTF8(long ucs); diff --git a/Build/source/texk/ptexenc/unicode.c b/Build/source/texk/ptexenc/unicode.c index 3fb90da913a..a3ed05baf3b 100644 --- a/Build/source/texk/ptexenc/unicode.c +++ b/Build/source/texk/ptexenc/unicode.c @@ -43,19 +43,22 @@ int UTF8length(int first_byte) /* with strict range check */ -int UTF8Slength(unsigned char *buff, int buff_len) -{ - int i, len; - - len = UTF8length(buff[0]); - if (len < 0) return -2; /* illegal */ - if (len > buff_len) return -3; /* overflow */ - for (i=0; i<len; i++) { - if (!isUTF8(len, 1+i, buff[i])) return -1; /* not UTF-8 */ - } - return len; +#define DEFINE_UTF8SLENGTH(SUFF,TYPE) \ +int UTF8Slength ## SUFF(TYPE *buff, int buff_len) \ +{ \ + int i, len; \ + len = UTF8length(buff[0]); \ + if (len < 0) return -2; /* illegal */ \ + if (len > buff_len) return -3; /* overflow */ \ + for (i=0; i<len; i++) { \ + if (!isUTF8(len, 1+i, buff[i])) return -1; /* not UTF-8 */ \ + } \ + return len; \ } +DEFINE_UTF8SLENGTH(, unsigned char) +DEFINE_UTF8SLENGTH(short, unsigned short) + /* WITHOUT strict range check */ long UTF8StoUCS(unsigned char *s) diff --git a/Build/source/texk/ptexenc/version.ac b/Build/source/texk/ptexenc/version.ac index 7c02f883de3..9b8e183f2dd 100644 --- a/Build/source/texk/ptexenc/version.ac +++ b/Build/source/texk/ptexenc/version.ac @@ -10,4 +10,4 @@ dnl Keep package and shared library versions the same; dnl see kpathsea/version.ac. dnl dnl This file is m4-included from configure.ac. -m4_define([ptexenc_version], [1.3.11]) +m4_define([ptexenc_version], [1.4.0]) |