ptexenc: support guessing input file encodings

git-svn-id: svn://tug.org/texlive/trunk@63556 c570f23f-e606-0410-a88d-b1316a301751

8 files changed, 246 insertions, 30 deletions

diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog
index 6f916d7b319..03e9be4beca 100644
--- a/Build/source/texk/ptexenc/ChangeLog
+++ b/Build/source/texk/ptexenc/ChangeLog
@@ -1,3 +1,11 @@
+2022-06-12  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
+
+	* ptexenc.c, unicode.c, ptexenc/ptexenc.h, ptexenc/unicode.h:
+	Add new functions ptenc_guess_enc() and set_guess_file_enc()
+	to support guessing input file encodings.
+	https://github.com/texjporg/tex-jp-build/issues/142
+	* version.ac: Bump to 1.4.2/dev.
+
 2022-03-21  Karl Berry  <karl@tug.org>
 
 	* TL'22 release.
diff --git a/Build/source/texk/ptexenc/c-auto.in b/Build/source/texk/ptexenc/c-auto.in
index d27c2f238d3..40e4044f9e7 100644
--- a/Build/source/texk/ptexenc/c-auto.in
+++ b/Build/source/texk/ptexenc/c-auto.in
@@ -6,7 +6,7 @@
 #define PTEXENC_C_AUTO_H
 
 /* ptexenc: the version string. */
-#define PTEXENCVERSION "ptexenc version 1.4.1/dev"
+#define PTEXENCVERSION "ptexenc version 1.4.2/dev"
 
 /* Define to 1 if the `closedir' function returns void instead of int. */
 #undef CLOSEDIR_VOID
diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure
index 10eed0384b0..838e8791635 100755
--- a/Build/source/texk/ptexenc/configure
+++ b/Build/source/texk/ptexenc/configure
@@ -1,6 +1,6 @@
 #! /bin/sh
 # Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for ptexenc 1.4.1/dev.
+# Generated by GNU Autoconf 2.71 for ptexenc 1.4.2/dev.
 #
 # Report bugs to <tex-k@tug.org>.
 #
@@ -629,8 +629,8 @@ MAKEFLAGS=
 # Identity of this package.
 PACKAGE_NAME='ptexenc'
 PACKAGE_TARNAME='ptexenc'
-PACKAGE_VERSION='1.4.1/dev'
-PACKAGE_STRING='ptexenc 1.4.1/dev'
+PACKAGE_VERSION='1.4.2/dev'
+PACKAGE_STRING='ptexenc 1.4.2/dev'
 PACKAGE_BUGREPORT='tex-k@tug.org'
 PACKAGE_URL=''
 
@@ -1375,7 +1375,7 @@ if test "$ac_init_help" = "long"; then
   # Omit some internal or obsolete options to make the list less imposing.
   # This message is too long to be a string in the A/UX 3.1 sh.
   cat <<_ACEOF
-\`configure' configures ptexenc 1.4.1/dev to adapt to many kinds of systems.
+\`configure' configures ptexenc 1.4.2/dev to adapt to many kinds of systems.
 
 Usage: $0 [OPTION]... [VAR=VALUE]...
 
@@ -1446,7 +1446,7 @@ fi
 
 if test -n "$ac_init_help"; then
   case $ac_init_help in
-     short | recursive ) echo "Configuration of ptexenc 1.4.1/dev:";;
+     short | recursive ) echo "Configuration of ptexenc 1.4.2/dev:";;
    esac
   cat <<\_ACEOF
 
@@ -1566,7 +1566,7 @@ fi
 test -n "$ac_init_help" && exit $ac_status
 if $ac_init_version; then
   cat <<\_ACEOF
-ptexenc configure 1.4.1/dev
+ptexenc configure 1.4.2/dev
 generated by GNU Autoconf 2.71
 
 Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2090,7 +2090,7 @@ cat >config.log <<_ACEOF
 This file contains any messages produced by compilers while
 running configure, to aid debugging if configure makes a mistake.
 
-It was created by ptexenc $as_me 1.4.1/dev, which was
+It was created by ptexenc $as_me 1.4.2/dev, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   $ $0$ac_configure_args_raw
@@ -2850,10 +2850,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
 
 
 
-PTEXENCVERSION=1.4.1/dev
+PTEXENCVERSION=1.4.2/dev
 
 
-PTEXENC_LT_VERSINFO=5:1:4
+PTEXENC_LT_VERSINFO=5:2:4
 
 
 am__api_version='1.16'
@@ -8418,7 +8418,7 @@ fi
 
 # Define the identity of the package.
  PACKAGE='ptexenc'
- VERSION='1.4.1/dev'
+ VERSION='1.4.2/dev'
 
 
 printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h
@@ -14528,7 +14528,7 @@ Usage: $0 [OPTIONS]
 Report bugs to <bug-libtool@gnu.org>."
 
 lt_cl_version="\
-ptexenc config.lt 1.4.1/dev
+ptexenc config.lt 1.4.2/dev
 configured by $0, generated by GNU Autoconf 2.71.
 
 Copyright (C) 2011 Free Software Foundation, Inc.
@@ -16255,7 +16255,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
 # report actual input values of CONFIG_FILES etc. instead of their
 # values after options handling.
 ac_log="
-This file was extended by ptexenc $as_me 1.4.1/dev, which was
+This file was extended by ptexenc $as_me 1.4.2/dev, which was
 generated by GNU Autoconf 2.71.  Invocation command line was
 
   CONFIG_FILES    = $CONFIG_FILES
@@ -16323,7 +16323,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
 cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
 ac_cs_config='$ac_cs_config_escaped'
 ac_cs_version="\\
-ptexenc config.status 1.4.1/dev
+ptexenc config.status 1.4.2/dev
 configured by $0, generated by GNU Autoconf 2.71,
   with options \\"\$ac_cs_config\\"
 
diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c
index 35f0fe9b7f0..e5f5e0f6cc4 100644
--- a/Build/source/texk/ptexenc/ptexenc.c
+++ b/Build/source/texk/ptexenc/ptexenc.c
@@ -41,10 +41,8 @@ static boolean prior_file_enc = false;
 const char *ptexenc_version_string = PTEXENCVERSION;
 #if defined(WIN32)
 FILE *Poptr;
-int infile_enc_auto;
-#else
-static int infile_enc_auto = 1;
 #endif
+int infile_enc_auto = 0;
 
 static int     file_enc = ENC_UNKNOWN;
 static int internal_enc = ENC_UNKNOWN;
@@ -72,6 +70,8 @@ static int string_to_enc(const_string str)
     if (strcasecmp(str, "utf8")   == 0) return ENC_UTF8;
     if (UPTEX_enabled && strcasecmp(str, "uptex")  == 0) return ENC_UPTEX;
 
+    if (strcasecmp(str, "ASCII")== 0)        return file_enc;
+    if (strncasecmp(str, "AMBIGUOUS", 9) == 0) return file_enc;
     if (strcasecmp(str, "BINARY") == 0)      return ENC_JIS;
     if (strcasecmp(str, "ISO-2022-JP") == 0) return ENC_JIS;
     if (strcasecmp(str, "EUC-JP") == 0)      return ENC_EUC;
@@ -146,6 +146,11 @@ static int get_terminal_enc(void)
     return terminal_enc;
 }
 
+void set_guess_file_enc(boolean enable)
+{
+    infile_enc_auto = (int) enable;
+}
+
 /* enable ptex mode (use flag 0x100 for Japanese char) */
 void ptenc_ptex_mode (const boolean enable)
 {
@@ -199,7 +204,6 @@ boolean set_enc_string(const_string file_str, const_string internal_str)
     if (file     != ENC_UNKNOWN) {
         set_file_enc(file);
 #if !defined(WIN32)
-        infile_enc_auto = 0;
         nkf_disable();
 #endif
     }
@@ -753,7 +757,7 @@ static void get_euc(int i, FILE *fp)
         buffer[last++] = i;
         ungetc4(j, fp);
     }
-}        
+}
 
 static void get_sjis(int i, FILE *fp)
 {
@@ -765,7 +769,7 @@ static void get_sjis(int i, FILE *fp)
         buffer[last++] = i;
         ungetc4(j, fp);
     }
-}        
+}
 
 static boolean is_tail(long *c, FILE *fp)
 {
@@ -787,7 +791,7 @@ static boolean isUTF8Nstream(FILE *fp)
     int i;
     int c[MARK_LEN];
     int bom_u[MARK_LEN] = { 0xEF, 0xBB, 0xBF, 0x7E };
-    int bom_l[MARK_LEN] = { 0xEF, 0xBB, 0xBF, 0 };
+    int bom_l[MARK_LEN] = { 0xEF, 0xBB, 0xBF, 0x01 };
 
     for (i=0; i<MARK_LEN; i++) {
         c[i] = getc4(fp);
@@ -803,6 +807,177 @@ static boolean isUTF8Nstream(FILE *fp)
 static int infile_enc[NOFILE]; /* ENC_UNKNOWN (=0): not determined
                                   other: determined */
 
+/* guess file encoding */
+/*
+    asumption:
+      No halfwidth katakana in Shift_JIS
+      No SS2 nor SS3 in EUC-JP
+      JIS X 0208 only and no platform dependent characters in Shift_JIS, EUC-JP
+*/
+char *ptenc_guess_enc(FILE *fp)
+{
+    char *enc;
+    int k0, k1, k2, cdb[2], cu8[4], len_utf8;
+    int is_ascii=1, lbyte=0;
+    int maybe_sjis=1, maybe_euc=1, maybe_utf8=1, pos_db=0, pos_utf8=0;
+#ifdef DEBUG
+    int i;
+    unsigned char str0[5];
+#endif /* DEBUG */
+    enc = xmalloc(sizeof(char)*18);
+
+    while ((k0 = fgetc(fp)) != EOF && maybe_sjis+maybe_euc+maybe_utf8>1) {
+        lbyte++;
+        if (k0==ESC) {
+            k0 = fgetc(fp);
+            if (k0=='$') {
+                k0 = fgetc(fp);
+                if (k0=='@' || k0=='B') {
+                    strcpy(enc,"ISO-2022-JP");
+                    goto post_process;
+                }
+            }
+            if (k0>0x7F) {
+                strcpy(enc,"BINARY");
+                goto post_process;
+            } else if (k0==EOF) {
+                break;
+            }
+            continue;
+        } else if (k0==0x00) {
+            strcpy(enc,"BINARY");
+            goto post_process;
+        } else if (k0<0x80) {
+            if (pos_utf8>0) {
+                maybe_utf8 = 0;
+                pos_utf8 = 0;
+            }
+            if (pos_db==1) {
+                maybe_euc = 0;
+                pos_db = 0;
+                if (maybe_sjis) {
+                    cdb[1] = k0;
+                    k1 = JIStoUCS2(SJIStoJIS(HILO(cdb[0],cdb[1])));
+                    if (k1) {
+#ifdef DEBUG
+                        i = UCStoUTF8S(k1, str0);
+                        str0[i] = '\0';
+                        fprintf(stderr, "Character for guess encoding: 0x%02X%02X", cdb[0], cdb[1]);
+                        fprintf(stderr, " sjis (%s)\n", str0);
+#endif /* DEBUG */
+                        continue;
+                    }
+                }
+                maybe_sjis = 0;
+            }
+            if (is_ascii && lbyte>10000) {
+                /* guess ASCII if we did not find 8bit chars in head 10000 bytes */
+                strcpy(enc,"ASCII");
+                goto post_process;
+            }
+            continue;
+        }
+        is_ascii = 0;
+        if (pos_db==0) {
+            cdb[0] = k0;
+            cdb[1] = 0;
+            pos_db = 1;
+        }
+        else if (pos_db==1 && (maybe_sjis || maybe_euc)) {
+            cdb[1] = k0;
+            k1 = JIStoUCS2(SJIStoJIS(HILO(cdb[0],cdb[1])));
+            k2 = JIStoUCS2(EUCtoJIS(HILO(cdb[0],cdb[1])));
+            if (maybe_sjis) {
+                if (!k1)
+                    maybe_sjis = 0;
+            }
+            if (maybe_euc) {
+                if (!k2)
+                    maybe_euc = 0;
+            }
+            pos_db = 0;
+#ifdef DEBUG
+            if (maybe_sjis || maybe_euc) {
+                fprintf(stderr, "Character for guess encoding: 0x%02X%02X", cdb[0], cdb[1]);
+                if (maybe_sjis) {
+                    i = UCStoUTF8S(k1, str0);
+                    str0[i] = '\0';
+                    fprintf(stderr, " sjis (%s)", str0);
+                }
+                if (maybe_euc) {
+                    i = UCStoUTF8S(k2, str0);
+                    str0[i] = '\0';
+                    fprintf(stderr, " euc (%s)", str0);
+                }
+                fprintf(stderr, "\n");
+            }
+#endif /* DEBUG */
+        }
+        if (pos_utf8==0) {
+            len_utf8 = UTF8length(k0);
+            if (len_utf8<2) {
+                maybe_utf8 = 0;
+                pos_utf8 = 0;
+                continue;
+            }
+            cu8[0] = k0;
+            pos_utf8 = 1;
+        }
+        else if (pos_utf8>0 && maybe_utf8) {
+            if (k0>0xBF) {
+                maybe_utf8 = 0;
+                pos_utf8 = 0;
+                continue;
+            }
+            cu8[pos_utf8] = k0;
+            pos_utf8++;
+            if (pos_utf8==len_utf8) {
+                if ((cu8[0]==0xE0 && cu8[1]<0xA0) ||
+                    (cu8[0]==0xED && cu8[1]>0x9F) ||
+                    (cu8[0]==0xF0 && cu8[1]<0x90)) { /* illegal combination in UTF-8 */
+                    maybe_utf8 = 0;
+                    pos_utf8 = 0;
+                    continue;
+                }
+#ifdef DEBUG
+                for (i=0; i<len_utf8; i++) str0[i] = cu8[i];
+                str0[i] = '\0';
+                fprintf(stderr, "Character for guess encoding: 0x");
+                for (i=0; i<len_utf8; i++) fprintf(stderr, "%02X", cu8[i]);
+                fprintf(stderr, " U+%06lX (%s)\n", UTF8StoUCS(str0), str0);
+#endif /* DEBUG */
+                len_utf8 = 0;
+                pos_utf8 = 0;
+                cu8[0]=cu8[1]=cu8[2]=cu8[3]=0;
+            }
+        }
+    }
+
+    if (is_ascii)
+        strcpy(enc,"ASCII");
+    else if (maybe_sjis+maybe_euc+maybe_utf8>1) {
+        strcpy(enc,"AMBIGUOUS(");
+        if (maybe_sjis)
+            enc = strcat(enc,"s");
+        if (maybe_euc)
+            enc = strcat(enc, maybe_sjis ? ",e" : "e");
+        if (maybe_utf8)
+            enc = strcat(enc, ",u");
+        enc = strcat(enc,")");
+    }
+    else if (maybe_sjis)
+        strcpy(enc,"Shift_JIS");
+    else if (maybe_euc)
+        strcpy(enc,"EUC-JP");
+    else if (maybe_utf8)
+        strcpy(enc,"UTF-8");
+    else
+        strcpy(enc,"BINARY");
+  post_process:
+    rewind (fp);
+    return enc;
+}
+
 /* input line with encoding conversion */
 long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2,
                  long pos, const long buffsize, int *lastchar)
@@ -811,13 +986,35 @@ long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2,
     static boolean injis = false;
     const int fd = fileno(fp);
 
+    buffer = buff;
+    first = last = pos;
+
     if (infile_enc[fd] == ENC_UNKNOWN) { /* just after opened */
         ungetbuff[fd].size = 0;
-        if (isUTF8Nstream(fp)) infile_enc[fd] = ENC_UTF8;
-        else                   infile_enc[fd] = get_file_enc();
+        if (isUTF8Nstream(fp)) {
+            infile_enc[fd] = ENC_UTF8;
+#ifdef DEBUG
+            fprintf(stderr, "Detect UTF-8 with BOM #%d\n", fd);
+#endif /* DEBUG */
+        }
+        else if (infile_enc_auto && fd != fileno(stdin)) {
+            char *enc;
+            getc4(fp);
+            getc4(fp);
+            getc4(fp);
+            getc4(fp);
+            rewind(fp);
+            enc = ptenc_guess_enc(fp);
+            if (string_to_enc(enc) > 0) {
+                infile_enc[fd] = string_to_enc(enc);
+                fprintf(stderr, "(guessed encoding #%d: %s = %s)", fd, enc, enc_to_string(infile_enc[fd]));
+            } else {
+                infile_enc[fd] = get_file_enc();
+            }
+            if (enc) free(enc);
+        }
+        else infile_enc[fd] = get_file_enc();
     }
-    buffer = buff;
-    first = last = pos;
 
     while (last < buffsize-30 && (i=getc4(fp)) != EOF && i!='\n' && i!='\r') {
         /* 30 is enough large size for one char */
diff --git a/Build/source/texk/ptexenc/ptexenc/ptexenc.h b/Build/source/texk/ptexenc/ptexenc/ptexenc.h
index 5ed63e8c93b..b93e5e7e5e9 100644
--- a/Build/source/texk/ptexenc/ptexenc/ptexenc.h
+++ b/Build/source/texk/ptexenc/ptexenc/ptexenc.h
@@ -5,6 +5,9 @@
 #ifndef PTEXENC_PTEXENC_H
 #define PTEXENC_PTEXENC_H
 
+#ifdef WIN32
+#include <ptexenc/c-auto.h>
+#endif
 #include <kpathsea/types.h>
 
 #if defined (WIN32) && !defined (__MINGW32__) && !defined (NO_PTENC_DLL)
@@ -22,9 +25,10 @@
 #endif
 
 extern PTENCDLL const char *ptexenc_version_string;
+extern PTENCDLL int infile_enc_auto;
 #if defined(WIN32)
 extern PTENCDLL FILE *Poptr;
-extern PTENCDLL int infile_enc_auto;
+extern PTENCDLL char * ptexenc_nkf(FILE *f);
 #endif
 
 #define KANJI_OPTS "{jis|euc|sjis|utf8}"
@@ -32,6 +36,7 @@ extern PTENCDLL int infile_enc_auto;
 /* enable/disable UPTEX */
 extern PTENCDLL void enable_UPTEX (boolean enable);
 extern PTENCDLL void set_prior_file_enc(void);
+extern PTENCDLL void set_guess_file_enc(boolean enable);
 extern PTENCDLL void ptenc_ptex_mode(const boolean enable);
 
 /* get/set Kanji encoding by string */
diff --git a/Build/source/texk/ptexenc/ptexenc/unicode.h b/Build/source/texk/ptexenc/ptexenc/unicode.h
index 61552fd6251..09db49475aa 100644
--- a/Build/source/texk/ptexenc/ptexenc/unicode.h
+++ b/Build/source/texk/ptexenc/ptexenc/unicode.h
@@ -26,7 +26,12 @@ extern long UPTEXtoUCS(long uptex);
 #define BYTE2(x) (((x)>>16) & 0xff)
 #define BYTE3(x) (((x)>> 8) & 0xff)
 #define BYTE4(x) ( (x)      & 0xff)
- 
+
+/* used only for debug */
+#ifdef DEBUG
+extern int UCStoUTF8S(long ucs, unsigned char *s);
+#endif
+
 
 /* UTF-8 -> UCS */
 #define UTF8BtoUCS(a,b)   ((((a)&0x1f) << 6) | ((b)&0x3f))
diff --git a/Build/source/texk/ptexenc/unicode.c b/Build/source/texk/ptexenc/unicode.c
index a3ed05baf3b..e2d654aeb9b 100644
--- a/Build/source/texk/ptexenc/unicode.c
+++ b/Build/source/texk/ptexenc/unicode.c
@@ -72,7 +72,8 @@ long UTF8StoUCS(unsigned char *s)
     }
 }
 
-#if 0 /* not used */
+/* used only for debug */
+#ifdef DEBUG
 int UCStoUTF8S(long ucs, unsigned char *s)
 {
     if (ucs < 0x80) {
@@ -96,7 +97,7 @@ int UCStoUTF8S(long ucs, unsigned char *s)
     }
     return 0; /* unsupported */
 }
-#endif /* 0 */
+#endif /* DEBUG */
 
 
 long UCStoUTF8(long ucs)
diff --git a/Build/source/texk/ptexenc/version.ac b/Build/source/texk/ptexenc/version.ac
index 8a3d3febcb5..7f85d778652 100644
--- a/Build/source/texk/ptexenc/version.ac
+++ b/Build/source/texk/ptexenc/version.ac
@@ -10,4 +10,4 @@ dnl Keep package and shared library versions the same;
 dnl see kpathsea/version.ac.
 dnl
 dnl This file is m4-included from configure.ac.
-m4_define([ptexenc_version], [1.4.1/dev])
+m4_define([ptexenc_version], [1.4.2/dev])