From ab299733b38a3196a337723507b86a22a19589f3 Mon Sep 17 00:00:00 2001 From: Takuji Tanaka Date: Sun, 12 Jun 2022 08:18:08 +0000 Subject: ptexenc: support guessing input file encodings git-svn-id: svn://tug.org/texlive/trunk@63556 c570f23f-e606-0410-a88d-b1316a301751 --- Build/source/texk/ptexenc/ChangeLog | 8 + Build/source/texk/ptexenc/c-auto.in | 2 +- Build/source/texk/ptexenc/configure | 26 ++-- Build/source/texk/ptexenc/ptexenc.c | 219 ++++++++++++++++++++++++++-- Build/source/texk/ptexenc/ptexenc/ptexenc.h | 7 +- Build/source/texk/ptexenc/ptexenc/unicode.h | 7 +- Build/source/texk/ptexenc/unicode.c | 5 +- Build/source/texk/ptexenc/version.ac | 2 +- 8 files changed, 246 insertions(+), 30 deletions(-) (limited to 'Build/source') diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog index 6f916d7b319..03e9be4beca 100644 --- a/Build/source/texk/ptexenc/ChangeLog +++ b/Build/source/texk/ptexenc/ChangeLog @@ -1,3 +1,11 @@ +2022-06-12 TANAKA Takuji + + * ptexenc.c, unicode.c, ptexenc/ptexenc.h, ptexenc/unicode.h: + Add new functions ptenc_guess_enc() and set_guess_file_enc() + to support guessing input file encodings. + https://github.com/texjporg/tex-jp-build/issues/142 + * version.ac: Bump to 1.4.2/dev. + 2022-03-21 Karl Berry * TL'22 release. diff --git a/Build/source/texk/ptexenc/c-auto.in b/Build/source/texk/ptexenc/c-auto.in index d27c2f238d3..40e4044f9e7 100644 --- a/Build/source/texk/ptexenc/c-auto.in +++ b/Build/source/texk/ptexenc/c-auto.in @@ -6,7 +6,7 @@ #define PTEXENC_C_AUTO_H /* ptexenc: the version string. */ -#define PTEXENCVERSION "ptexenc version 1.4.1/dev" +#define PTEXENCVERSION "ptexenc version 1.4.2/dev" /* Define to 1 if the `closedir' function returns void instead of int. */ #undef CLOSEDIR_VOID diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure index 10eed0384b0..838e8791635 100755 --- a/Build/source/texk/ptexenc/configure +++ b/Build/source/texk/ptexenc/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.71 for ptexenc 1.4.1/dev. +# Generated by GNU Autoconf 2.71 for ptexenc 1.4.2/dev. # # Report bugs to . # @@ -629,8 +629,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='ptexenc' PACKAGE_TARNAME='ptexenc' -PACKAGE_VERSION='1.4.1/dev' -PACKAGE_STRING='ptexenc 1.4.1/dev' +PACKAGE_VERSION='1.4.2/dev' +PACKAGE_STRING='ptexenc 1.4.2/dev' PACKAGE_BUGREPORT='tex-k@tug.org' PACKAGE_URL='' @@ -1375,7 +1375,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures ptexenc 1.4.1/dev to adapt to many kinds of systems. +\`configure' configures ptexenc 1.4.2/dev to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1446,7 +1446,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of ptexenc 1.4.1/dev:";; + short | recursive ) echo "Configuration of ptexenc 1.4.2/dev:";; esac cat <<\_ACEOF @@ -1566,7 +1566,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -ptexenc configure 1.4.1/dev +ptexenc configure 1.4.2/dev generated by GNU Autoconf 2.71 Copyright (C) 2021 Free Software Foundation, Inc. @@ -2090,7 +2090,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ptexenc $as_me 1.4.1/dev, which was +It was created by ptexenc $as_me 1.4.2/dev, which was generated by GNU Autoconf 2.71. Invocation command line was $ $0$ac_configure_args_raw @@ -2850,10 +2850,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu -PTEXENCVERSION=1.4.1/dev +PTEXENCVERSION=1.4.2/dev -PTEXENC_LT_VERSINFO=5:1:4 +PTEXENC_LT_VERSINFO=5:2:4 am__api_version='1.16' @@ -8418,7 +8418,7 @@ fi # Define the identity of the package. PACKAGE='ptexenc' - VERSION='1.4.1/dev' + VERSION='1.4.2/dev' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -14528,7 +14528,7 @@ Usage: $0 [OPTIONS] Report bugs to ." lt_cl_version="\ -ptexenc config.lt 1.4.1/dev +ptexenc config.lt 1.4.2/dev configured by $0, generated by GNU Autoconf 2.71. Copyright (C) 2011 Free Software Foundation, Inc. @@ -16255,7 +16255,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by ptexenc $as_me 1.4.1/dev, which was +This file was extended by ptexenc $as_me 1.4.2/dev, which was generated by GNU Autoconf 2.71. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -16323,7 +16323,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -ptexenc config.status 1.4.1/dev +ptexenc config.status 1.4.2/dev configured by $0, generated by GNU Autoconf 2.71, with options \\"\$ac_cs_config\\" diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c index 35f0fe9b7f0..e5f5e0f6cc4 100644 --- a/Build/source/texk/ptexenc/ptexenc.c +++ b/Build/source/texk/ptexenc/ptexenc.c @@ -41,10 +41,8 @@ static boolean prior_file_enc = false; const char *ptexenc_version_string = PTEXENCVERSION; #if defined(WIN32) FILE *Poptr; -int infile_enc_auto; -#else -static int infile_enc_auto = 1; #endif +int infile_enc_auto = 0; static int file_enc = ENC_UNKNOWN; static int internal_enc = ENC_UNKNOWN; @@ -72,6 +70,8 @@ static int string_to_enc(const_string str) if (strcasecmp(str, "utf8") == 0) return ENC_UTF8; if (UPTEX_enabled && strcasecmp(str, "uptex") == 0) return ENC_UPTEX; + if (strcasecmp(str, "ASCII")== 0) return file_enc; + if (strncasecmp(str, "AMBIGUOUS", 9) == 0) return file_enc; if (strcasecmp(str, "BINARY") == 0) return ENC_JIS; if (strcasecmp(str, "ISO-2022-JP") == 0) return ENC_JIS; if (strcasecmp(str, "EUC-JP") == 0) return ENC_EUC; @@ -146,6 +146,11 @@ static int get_terminal_enc(void) return terminal_enc; } +void set_guess_file_enc(boolean enable) +{ + infile_enc_auto = (int) enable; +} + /* enable ptex mode (use flag 0x100 for Japanese char) */ void ptenc_ptex_mode (const boolean enable) { @@ -199,7 +204,6 @@ boolean set_enc_string(const_string file_str, const_string internal_str) if (file != ENC_UNKNOWN) { set_file_enc(file); #if !defined(WIN32) - infile_enc_auto = 0; nkf_disable(); #endif } @@ -753,7 +757,7 @@ static void get_euc(int i, FILE *fp) buffer[last++] = i; ungetc4(j, fp); } -} +} static void get_sjis(int i, FILE *fp) { @@ -765,7 +769,7 @@ static void get_sjis(int i, FILE *fp) buffer[last++] = i; ungetc4(j, fp); } -} +} static boolean is_tail(long *c, FILE *fp) { @@ -787,7 +791,7 @@ static boolean isUTF8Nstream(FILE *fp) int i; int c[MARK_LEN]; int bom_u[MARK_LEN] = { 0xEF, 0xBB, 0xBF, 0x7E }; - int bom_l[MARK_LEN] = { 0xEF, 0xBB, 0xBF, 0 }; + int bom_l[MARK_LEN] = { 0xEF, 0xBB, 0xBF, 0x01 }; for (i=0; i1) { + lbyte++; + if (k0==ESC) { + k0 = fgetc(fp); + if (k0=='$') { + k0 = fgetc(fp); + if (k0=='@' || k0=='B') { + strcpy(enc,"ISO-2022-JP"); + goto post_process; + } + } + if (k0>0x7F) { + strcpy(enc,"BINARY"); + goto post_process; + } else if (k0==EOF) { + break; + } + continue; + } else if (k0==0x00) { + strcpy(enc,"BINARY"); + goto post_process; + } else if (k0<0x80) { + if (pos_utf8>0) { + maybe_utf8 = 0; + pos_utf8 = 0; + } + if (pos_db==1) { + maybe_euc = 0; + pos_db = 0; + if (maybe_sjis) { + cdb[1] = k0; + k1 = JIStoUCS2(SJIStoJIS(HILO(cdb[0],cdb[1]))); + if (k1) { +#ifdef DEBUG + i = UCStoUTF8S(k1, str0); + str0[i] = '\0'; + fprintf(stderr, "Character for guess encoding: 0x%02X%02X", cdb[0], cdb[1]); + fprintf(stderr, " sjis (%s)\n", str0); +#endif /* DEBUG */ + continue; + } + } + maybe_sjis = 0; + } + if (is_ascii && lbyte>10000) { + /* guess ASCII if we did not find 8bit chars in head 10000 bytes */ + strcpy(enc,"ASCII"); + goto post_process; + } + continue; + } + is_ascii = 0; + if (pos_db==0) { + cdb[0] = k0; + cdb[1] = 0; + pos_db = 1; + } + else if (pos_db==1 && (maybe_sjis || maybe_euc)) { + cdb[1] = k0; + k1 = JIStoUCS2(SJIStoJIS(HILO(cdb[0],cdb[1]))); + k2 = JIStoUCS2(EUCtoJIS(HILO(cdb[0],cdb[1]))); + if (maybe_sjis) { + if (!k1) + maybe_sjis = 0; + } + if (maybe_euc) { + if (!k2) + maybe_euc = 0; + } + pos_db = 0; +#ifdef DEBUG + if (maybe_sjis || maybe_euc) { + fprintf(stderr, "Character for guess encoding: 0x%02X%02X", cdb[0], cdb[1]); + if (maybe_sjis) { + i = UCStoUTF8S(k1, str0); + str0[i] = '\0'; + fprintf(stderr, " sjis (%s)", str0); + } + if (maybe_euc) { + i = UCStoUTF8S(k2, str0); + str0[i] = '\0'; + fprintf(stderr, " euc (%s)", str0); + } + fprintf(stderr, "\n"); + } +#endif /* DEBUG */ + } + if (pos_utf8==0) { + len_utf8 = UTF8length(k0); + if (len_utf8<2) { + maybe_utf8 = 0; + pos_utf8 = 0; + continue; + } + cu8[0] = k0; + pos_utf8 = 1; + } + else if (pos_utf8>0 && maybe_utf8) { + if (k0>0xBF) { + maybe_utf8 = 0; + pos_utf8 = 0; + continue; + } + cu8[pos_utf8] = k0; + pos_utf8++; + if (pos_utf8==len_utf8) { + if ((cu8[0]==0xE0 && cu8[1]<0xA0) || + (cu8[0]==0xED && cu8[1]>0x9F) || + (cu8[0]==0xF0 && cu8[1]<0x90)) { /* illegal combination in UTF-8 */ + maybe_utf8 = 0; + pos_utf8 = 0; + continue; + } +#ifdef DEBUG + for (i=0; i1) { + strcpy(enc,"AMBIGUOUS("); + if (maybe_sjis) + enc = strcat(enc,"s"); + if (maybe_euc) + enc = strcat(enc, maybe_sjis ? ",e" : "e"); + if (maybe_utf8) + enc = strcat(enc, ",u"); + enc = strcat(enc,")"); + } + else if (maybe_sjis) + strcpy(enc,"Shift_JIS"); + else if (maybe_euc) + strcpy(enc,"EUC-JP"); + else if (maybe_utf8) + strcpy(enc,"UTF-8"); + else + strcpy(enc,"BINARY"); + post_process: + rewind (fp); + return enc; +} + /* input line with encoding conversion */ long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, long pos, const long buffsize, int *lastchar) @@ -811,13 +986,35 @@ long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, static boolean injis = false; const int fd = fileno(fp); + buffer = buff; + first = last = pos; + if (infile_enc[fd] == ENC_UNKNOWN) { /* just after opened */ ungetbuff[fd].size = 0; - if (isUTF8Nstream(fp)) infile_enc[fd] = ENC_UTF8; - else infile_enc[fd] = get_file_enc(); + if (isUTF8Nstream(fp)) { + infile_enc[fd] = ENC_UTF8; +#ifdef DEBUG + fprintf(stderr, "Detect UTF-8 with BOM #%d\n", fd); +#endif /* DEBUG */ + } + else if (infile_enc_auto && fd != fileno(stdin)) { + char *enc; + getc4(fp); + getc4(fp); + getc4(fp); + getc4(fp); + rewind(fp); + enc = ptenc_guess_enc(fp); + if (string_to_enc(enc) > 0) { + infile_enc[fd] = string_to_enc(enc); + fprintf(stderr, "(guessed encoding #%d: %s = %s)", fd, enc, enc_to_string(infile_enc[fd])); + } else { + infile_enc[fd] = get_file_enc(); + } + if (enc) free(enc); + } + else infile_enc[fd] = get_file_enc(); } - buffer = buff; - first = last = pos; while (last < buffsize-30 && (i=getc4(fp)) != EOF && i!='\n' && i!='\r') { /* 30 is enough large size for one char */ diff --git a/Build/source/texk/ptexenc/ptexenc/ptexenc.h b/Build/source/texk/ptexenc/ptexenc/ptexenc.h index 5ed63e8c93b..b93e5e7e5e9 100644 --- a/Build/source/texk/ptexenc/ptexenc/ptexenc.h +++ b/Build/source/texk/ptexenc/ptexenc/ptexenc.h @@ -5,6 +5,9 @@ #ifndef PTEXENC_PTEXENC_H #define PTEXENC_PTEXENC_H +#ifdef WIN32 +#include +#endif #include #if defined (WIN32) && !defined (__MINGW32__) && !defined (NO_PTENC_DLL) @@ -22,9 +25,10 @@ #endif extern PTENCDLL const char *ptexenc_version_string; +extern PTENCDLL int infile_enc_auto; #if defined(WIN32) extern PTENCDLL FILE *Poptr; -extern PTENCDLL int infile_enc_auto; +extern PTENCDLL char * ptexenc_nkf(FILE *f); #endif #define KANJI_OPTS "{jis|euc|sjis|utf8}" @@ -32,6 +36,7 @@ extern PTENCDLL int infile_enc_auto; /* enable/disable UPTEX */ extern PTENCDLL void enable_UPTEX (boolean enable); extern PTENCDLL void set_prior_file_enc(void); +extern PTENCDLL void set_guess_file_enc(boolean enable); extern PTENCDLL void ptenc_ptex_mode(const boolean enable); /* get/set Kanji encoding by string */ diff --git a/Build/source/texk/ptexenc/ptexenc/unicode.h b/Build/source/texk/ptexenc/ptexenc/unicode.h index 61552fd6251..09db49475aa 100644 --- a/Build/source/texk/ptexenc/ptexenc/unicode.h +++ b/Build/source/texk/ptexenc/ptexenc/unicode.h @@ -26,7 +26,12 @@ extern long UPTEXtoUCS(long uptex); #define BYTE2(x) (((x)>>16) & 0xff) #define BYTE3(x) (((x)>> 8) & 0xff) #define BYTE4(x) ( (x) & 0xff) - + +/* used only for debug */ +#ifdef DEBUG +extern int UCStoUTF8S(long ucs, unsigned char *s); +#endif + /* UTF-8 -> UCS */ #define UTF8BtoUCS(a,b) ((((a)&0x1f) << 6) | ((b)&0x3f)) diff --git a/Build/source/texk/ptexenc/unicode.c b/Build/source/texk/ptexenc/unicode.c index a3ed05baf3b..e2d654aeb9b 100644 --- a/Build/source/texk/ptexenc/unicode.c +++ b/Build/source/texk/ptexenc/unicode.c @@ -72,7 +72,8 @@ long UTF8StoUCS(unsigned char *s) } } -#if 0 /* not used */ +/* used only for debug */ +#ifdef DEBUG int UCStoUTF8S(long ucs, unsigned char *s) { if (ucs < 0x80) { @@ -96,7 +97,7 @@ int UCStoUTF8S(long ucs, unsigned char *s) } return 0; /* unsupported */ } -#endif /* 0 */ +#endif /* DEBUG */ long UCStoUTF8(long ucs) diff --git a/Build/source/texk/ptexenc/version.ac b/Build/source/texk/ptexenc/version.ac index 8a3d3febcb5..7f85d778652 100644 --- a/Build/source/texk/ptexenc/version.ac +++ b/Build/source/texk/ptexenc/version.ac @@ -10,4 +10,4 @@ dnl Keep package and shared library versions the same; dnl see kpathsea/version.ac. dnl dnl This file is m4-included from configure.ac. -m4_define([ptexenc_version], [1.4.1/dev]) +m4_define([ptexenc_version], [1.4.2/dev]) -- cgit v1.2.3