summaryrefslogtreecommitdiff
path: root/Build/source/texk/ptexenc
diff options
context:
space:
mode:
authorHironobu Yamashita <h.y.acetaminophen@gmail.com>2022-01-22 17:03:22 +0000
committerHironobu Yamashita <h.y.acetaminophen@gmail.com>2022-01-22 17:03:22 +0000
commit76e833603f3eb350549d8b472d5b0c1ebae21713 (patch)
tree0adce490f86927e7a81a2f05330b9590bf80663d /Build/source/texk/ptexenc
parent87e77a4b4b35459b0bc1aa06f48f2cf0db19ae9e (diff)
[e][u]ptex: Distinguish 8-bit characters and Japanese characters (H. Kitagawa et al.)
For better support of LaTeX3 (expl3). More details in TUGboat 41(2):329--334, 2020. git-svn-id: svn://tug.org/texlive/trunk@61692 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/ptexenc')
-rw-r--r--Build/source/texk/ptexenc/ChangeLog7
-rwxr-xr-xBuild/source/texk/ptexenc/configure26
-rw-r--r--Build/source/texk/ptexenc/ptexenc.c140
-rw-r--r--Build/source/texk/ptexenc/ptexenc/ptexenc.h10
-rw-r--r--Build/source/texk/ptexenc/ptexenc/unicode.h1
-rw-r--r--Build/source/texk/ptexenc/unicode.c25
-rw-r--r--Build/source/texk/ptexenc/version.ac2
7 files changed, 144 insertions, 67 deletions
diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog
index 6f40a869e27..bc121f89771 100644
--- a/Build/source/texk/ptexenc/ChangeLog
+++ b/Build/source/texk/ptexenc/ChangeLog
@@ -1,3 +1,10 @@
+2022-01-22 Hironori Kitagawa <h_kitagawa2001@yahoo.co.jp>
+
+ * ptexenc.c, ptexenc/ptexenc.h, unicode.c, ptexenc/unicode.h:
+ Distinguish 8-bit characters and Japanese characters by using
+ flag 0x100 for Japanese char.
+ * version.ac: Bump to 1.4.0.
+
2022-01-18 Karl Berry <karl@tug.org>
* version.ac: remove /dev, for TL22.
diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure
index 191a62891da..6464350d9e6 100755
--- a/Build/source/texk/ptexenc/configure
+++ b/Build/source/texk/ptexenc/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.71 for ptexenc 1.3.11.
+# Generated by GNU Autoconf 2.71 for ptexenc 1.4.0.
#
# Report bugs to <tex-k@tug.org>.
#
@@ -629,8 +629,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='ptexenc'
PACKAGE_TARNAME='ptexenc'
-PACKAGE_VERSION='1.3.11'
-PACKAGE_STRING='ptexenc 1.3.11'
+PACKAGE_VERSION='1.4.0'
+PACKAGE_STRING='ptexenc 1.4.0'
PACKAGE_BUGREPORT='tex-k@tug.org'
PACKAGE_URL=''
@@ -1375,7 +1375,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-\`configure' configures ptexenc 1.3.11 to adapt to many kinds of systems.
+\`configure' configures ptexenc 1.4.0 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1446,7 +1446,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of ptexenc 1.3.11:";;
+ short | recursive ) echo "Configuration of ptexenc 1.4.0:";;
esac
cat <<\_ACEOF
@@ -1566,7 +1566,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-ptexenc configure 1.3.11
+ptexenc configure 1.4.0
generated by GNU Autoconf 2.71
Copyright (C) 2021 Free Software Foundation, Inc.
@@ -2090,7 +2090,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by ptexenc $as_me 1.3.11, which was
+It was created by ptexenc $as_me 1.4.0, which was
generated by GNU Autoconf 2.71. Invocation command line was
$ $0$ac_configure_args_raw
@@ -2850,10 +2850,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
-PTEXENCVERSION=1.3.11
+PTEXENCVERSION=1.4.0
-PTEXENC_LT_VERSINFO=4:11:3
+PTEXENC_LT_VERSINFO=5:0:4
am__api_version='1.16'
@@ -8418,7 +8418,7 @@ fi
# Define the identity of the package.
PACKAGE='ptexenc'
- VERSION='1.3.11'
+ VERSION='1.4.0'
printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h
@@ -14528,7 +14528,7 @@ Usage: $0 [OPTIONS]
Report bugs to <bug-libtool@gnu.org>."
lt_cl_version="\
-ptexenc config.lt 1.3.11
+ptexenc config.lt 1.4.0
configured by $0, generated by GNU Autoconf 2.71.
Copyright (C) 2011 Free Software Foundation, Inc.
@@ -16255,7 +16255,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by ptexenc $as_me 1.3.11, which was
+This file was extended by ptexenc $as_me 1.4.0, which was
generated by GNU Autoconf 2.71. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -16323,7 +16323,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config='$ac_cs_config_escaped'
ac_cs_version="\\
-ptexenc config.status 1.3.11
+ptexenc config.status 1.4.0
configured by $0, generated by GNU Autoconf 2.71,
with options \\"\$ac_cs_config\\"
diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c
index 91cb4fa6f5a..a5c67dc82f7 100644
--- a/Build/source/texk/ptexenc/ptexenc.c
+++ b/Build/source/texk/ptexenc/ptexenc.c
@@ -26,6 +26,7 @@
static int default_kanji_enc;
static boolean UPTEX_enabled;
+static boolean ptex_mode = false;
static boolean prior_file_enc = false;
#define ESC '\033'
@@ -145,6 +146,13 @@ static int get_terminal_enc(void)
return terminal_enc;
}
+/* enable ptex mode (use flag 0x100 for Japanese char) */
+void ptenc_ptex_mode (const boolean enable)
+{
+ //fprintf(stderr, "ptenc_ptex_mode is called! (%d)\n", enable);
+ ptex_mode = enable;
+}
+
/* enable/disable UPTEX */
void enable_UPTEX (boolean enable)
{
@@ -253,16 +261,37 @@ boolean iskanji2(int c)
}
/* multi-byte char length in s[pos] */
-int multistrlen(unsigned char *s, int len, int pos)
+#define DEFINE_MULTISTRLEN(SUFF,TYPE) \
+int multistrlen ## SUFF(TYPE *s, int len, int pos) \
+{ \
+ s += pos; len -= pos; \
+ if (is_internalUPTEX()) { \
+ int ret = UTF8Slength ## SUFF(s, len); \
+ if (ret < 0) return 1; \
+ return ret; \
+ } \
+ if (len < 2) return 1; \
+ if (is_internalSJIS()) { \
+ if (isSJISkanji1(s[0]) && isSJISkanji2(s[1])) return 2; \
+ } else { /* EUC */ \
+ if (isEUCkanji1(s[0]) && isEUCkanji2(s[1])) return 2; \
+ } \
+ return 1; \
+}
+DEFINE_MULTISTRLEN(,unsigned char);
+DEFINE_MULTISTRLEN(short,unsigned short);
+
+/* for outputting filename (*s) to the terminal */
+int multistrlenfilename(unsigned short *s, int len, int pos)
{
s += pos; len -= pos;
- if (is_internalUPTEX()) {
- int ret = UTF8Slength(s, len);
+ if (terminal_enc == ENC_UTF8) {
+ int ret = UTF8Slengthshort(s, len);
if (ret < 0) return 1;
return ret;
}
if (len < 2) return 1;
- if (is_internalSJIS()) {
+ if (terminal_enc == ENC_SJIS) {
if (isSJISkanji1(s[0]) && isSJISkanji2(s[1])) return 2;
} else { /* EUC */
if (isEUCkanji1(s[0]) && isEUCkanji2(s[1])) return 2;
@@ -300,6 +329,14 @@ long fromBUFF(unsigned char *s, int len, int pos)
return s[0];
}
+long fromBUFFshort(unsigned short *s, int len, int pos)
+{
+ unsigned char sc[6];
+ s += pos; len -= pos;
+ for (int i=0;i<(len<6 ? len : 6);i++) sc[i]=0xFF&s[i];
+ return fromBUFF(sc, (len<6 ? len : 6), 0);
+}
+
/* internal (EUC/SJIS/UPTEX) to buffer (EUC/SJIS/UTF-8) code conversion */
long toBUFF(long kcode)
{
@@ -473,6 +510,10 @@ static int flush (unsigned char *buff, int num, FILE *fp)
/* putc() with code conversion */
int putc2(int c, FILE *fp)
+/*
+ c in [0,255]: writes the character c, without code conversion
+ c in [256,511]: writes the character c-256, with code conversion
+*/
{
static int num[NOFILE];
/* 0 : not in Kanji
@@ -481,7 +522,6 @@ int putc2(int c, FILE *fp)
static unsigned char store[NOFILE][4];
const int fd = fileno(fp);
int ret = c, output_enc;
-
#ifdef WIN32
if ((fp == stdout || fp == stderr) && (_isatty(fd) || !prior_file_enc)) {
output_enc = ENC_UTF8;
@@ -493,34 +533,43 @@ int putc2(int c, FILE *fp)
} else
output_enc = get_file_enc();
#endif
- if (num[fd] > 0) { /* multi-byte char */
- if (is_internalUPTEX() && iskanji1(c)) { /* error */
- ret = flush(store[fd], num[fd], fp);
- num[fd] = 0;
- }
- store[fd][num[fd]] = c;
- num[fd]++;
- if (multistrlen(store[fd], num[fd], 0) == num[fd]) {
- long i = fromBUFF(store[fd], num[fd], 0);
- ret = put_multibyte(toENC(i, output_enc), fp);
- num[fd] = -1;
- } else if ((is_internalUPTEX() && num[fd] == 4) ||
- (!is_internalUPTEX() && num[fd] == 2)) { /* error */
- ret = flush(store[fd], num[fd], fp);
- num[fd] = -1;
- }
- } else if (iskanji1(c)) { /* first multi-byte char */
- if (num[fd] == 0 && output_enc == ENC_JIS) {
- ret = put_multibyte(KANJI_IN, fp);
- }
- store[fd][0] = c;
- num[fd] = 1;
- } else { /* ASCII */
+ if (ptex_mode && (c<256)) {
if (num[fd] < 0 && output_enc == ENC_JIS) {
put_multibyte(KANJI_OUT, fp);
}
ret = putc(c, fp);
num[fd] = 0;
+ } else {
+ c &= 0xFF;
+ if (num[fd] > 0) { /* multi-byte char */
+ if (is_internalUPTEX() && iskanji1(c)) { /* error */
+ ret = flush(store[fd], num[fd], fp);
+ num[fd] = 0;
+ }
+ store[fd][num[fd]] = c;
+ num[fd]++;
+ if (multistrlen(store[fd], num[fd], 0) == num[fd]) {
+ long i = fromBUFF(store[fd], num[fd], 0);
+ ret = put_multibyte(toENC(i, output_enc), fp);
+ num[fd] = -1;
+ } else if ((is_internalUPTEX() && num[fd] == 4) ||
+ (!is_internalUPTEX() && num[fd] == 2)) { /* error */
+ ret = flush(store[fd], num[fd], fp);
+ num[fd] = -1;
+ }
+ } else if (iskanji1(c)) { /* first multi-byte char */
+ if (num[fd] == 0 && output_enc == ENC_JIS) {
+ ret = put_multibyte(KANJI_IN, fp);
+ }
+ store[fd][0] = c;
+ num[fd] = 1;
+ } else { /* ASCII */
+ if (num[fd] < 0 && output_enc == ENC_JIS) {
+ put_multibyte(KANJI_OUT, fp);
+ }
+ ret = putc(c, fp);
+ num[fd] = 0;
+ }
}
return ret;
}
@@ -745,8 +794,8 @@ static int infile_enc[NOFILE]; /* ENC_UNKNOWN (=0): not determined
other: determined */
/* input line with encoding conversion */
-long input_line2(FILE *fp, unsigned char *buff, long pos,
- const long buffsize, int *lastchar)
+long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2,
+ long pos, const long buffsize, int *lastchar)
{
long i = 0;
static boolean injis = false;
@@ -826,6 +875,9 @@ long input_line2(FILE *fp, unsigned char *buff, long pos,
if (i == EOF || i == '\n' || i == '\r') injis = false;
if (lastchar != NULL) *lastchar = i;
+ if (buff2!= NULL) for (i=pos; i<=last; i++) buff2[i] = 0;
+ /* buff2 is initialized */
+
return last;
}
@@ -925,6 +977,8 @@ int nkf_close(FILE *fp) {
}
#define break_if_bad_utf8_second(k) if ((k<0x80)||(k>0xBF)) { i--; k='\0'; break; }
+#define write_hex_if_not_ascii(c) \
+ if ((c>=0x20)&&(c<=0x7E)) buffer[last++]=c; else write_hex(c);
unsigned char *ptenc_from_utf8_string_to_internal_enc(const unsigned char *is)
{
int i;
@@ -975,10 +1029,10 @@ unsigned char *ptenc_from_utf8_string_to_internal_enc(const unsigned char *is)
j = (u != 0) ? toBUFF(fromUCS(u)) : 0;
if (j == 0) { /* can't represent in EUC/SJIS */
if (last+16>=len) buffer = buf = xrealloc(buffer, len=last+64);
- write_hex(i1);
- if (i2 != '\0') write_hex(i2);
- if (i3 != '\0') write_hex(i3);
- if (i4 != '\0') write_hex(i4);
+ write_hex_if_not_ascii(i1);
+ if (i2 != '\0') write_hex_if_not_ascii(i2);
+ if (i3 != '\0') write_hex_if_not_ascii(i3);
+ if (i4 != '\0') write_hex_if_not_ascii(i4);
} else {
write_multibyte(j);
}
@@ -1005,7 +1059,7 @@ unsigned char *ptenc_from_internal_enc_string_to_utf8(const unsigned char *is)
last_bak = last;
len = strlen(is)+1;
- buffer = buf = xmalloc(len*1.5);
+ buffer = buf = xmalloc(len*4);
first = last = 0;
for (i=0; i<strlen(is); i++) {
@@ -1015,15 +1069,21 @@ unsigned char *ptenc_from_internal_enc_string_to_utf8(const unsigned char *is)
buffer[last++] = i1; /* ASCII */
if (i1 == '\0') goto end;
continue;
- case 2:
- i2 = is[++i]; if (i2 == '\0') break;
- u = JIStoUCS2(toJIS(HILO(i1,i2)));
+ case 2: /* i1: not ASCII */
+ i2 = is[++i];
+ if (i2 == '\0') {
+ write_hex(i1); continue;
+ } else {
+ u = JIStoUCS2(toJIS(HILO(i1,i2)));
+ if (u==0) {
+ write_hex(i1); write_hex_if_not_ascii(i2); continue;
+ }
+ }
break;
- default:
+ default: /* reachable only if internal code is uptex */
u = U_REPLACEMENT_CHARACTER;
break;
}
-
write_multibyte(UCStoUTF8(u));
}
buffer[last] = '\0';
diff --git a/Build/source/texk/ptexenc/ptexenc/ptexenc.h b/Build/source/texk/ptexenc/ptexenc/ptexenc.h
index 93004162390..e1641978237 100644
--- a/Build/source/texk/ptexenc/ptexenc/ptexenc.h
+++ b/Build/source/texk/ptexenc/ptexenc/ptexenc.h
@@ -32,6 +32,7 @@ extern PTENCDLL int infile_enc_auto;
/* enable/disable UPTEX */
extern PTENCDLL void enable_UPTEX (boolean enable);
extern PTENCDLL void set_prior_file_enc(void);
+extern PTENCDLL void ptenc_ptex_mode(const boolean enable);
/* get/set Kanji encoding by string */
extern PTENCDLL const_string get_enc_string(void);
@@ -58,10 +59,15 @@ extern PTENCDLL boolean iskanji2(int c);
/* internal (EUC/SJIS/UPTEX) from/to buffer (EUC/SJIS/UTF-8) code conversion */
extern PTENCDLL int multistrlen(unsigned char *s, int len, int pos);
+extern PTENCDLL int multistrlenshort(unsigned short *s, int len, int pos);
extern PTENCDLL int multibytelen (int first_byte);
extern PTENCDLL long fromBUFF(unsigned char *s, int len, int pos);
+extern PTENCDLL long fromBUFFshort(unsigned short *s, int len, int pos);
extern PTENCDLL long toBUFF(long inter);
+/* for outputting filename (*s) to the terminal */
+extern PTENCDLL int multistrlenfilename(unsigned short *s, int len, int pos);
+
/* internal (EUC/SJIS/UPTEX) from/to DVI (JIS/UCS) code conversion */
extern PTENCDLL long toDVI (long kcode);
extern PTENCDLL long fromDVI (long kcode);
@@ -81,8 +87,8 @@ extern PTENCDLL int putc2(int c, FILE *fp);
extern PTENCDLL int fputs2(const char *s, FILE *fp);
/* input line with encoding conversion */
-extern PTENCDLL long input_line2(FILE *fp, unsigned char *buff, long pos,
- const long buffsize, int *lastchar);
+extern PTENCDLL long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2,
+ long pos, const long buffsize, int *lastchar);
/* set current encoding */
extern PTENCDLL boolean setinfileenc(FILE *fp, const char *str);
diff --git a/Build/source/texk/ptexenc/ptexenc/unicode.h b/Build/source/texk/ptexenc/ptexenc/unicode.h
index 9e4c91ff3fb..61552fd6251 100644
--- a/Build/source/texk/ptexenc/ptexenc/unicode.h
+++ b/Build/source/texk/ptexenc/ptexenc/unicode.h
@@ -14,6 +14,7 @@ extern boolean isUTF8(int length, int nth, int c);
extern int UTF8length(int first_byte);
extern int UTF8Slength(unsigned char *buff, int buff_len);
+extern int UTF8Slengthshort(unsigned short *buff, int buff_len);
extern long UTF8StoUCS(unsigned char *s);
extern long PTENCDLL UCStoUTF8(long ucs);
diff --git a/Build/source/texk/ptexenc/unicode.c b/Build/source/texk/ptexenc/unicode.c
index 3fb90da913a..a3ed05baf3b 100644
--- a/Build/source/texk/ptexenc/unicode.c
+++ b/Build/source/texk/ptexenc/unicode.c
@@ -43,19 +43,22 @@ int UTF8length(int first_byte)
/* with strict range check */
-int UTF8Slength(unsigned char *buff, int buff_len)
-{
- int i, len;
-
- len = UTF8length(buff[0]);
- if (len < 0) return -2; /* illegal */
- if (len > buff_len) return -3; /* overflow */
- for (i=0; i<len; i++) {
- if (!isUTF8(len, 1+i, buff[i])) return -1; /* not UTF-8 */
- }
- return len;
+#define DEFINE_UTF8SLENGTH(SUFF,TYPE) \
+int UTF8Slength ## SUFF(TYPE *buff, int buff_len) \
+{ \
+ int i, len; \
+ len = UTF8length(buff[0]); \
+ if (len < 0) return -2; /* illegal */ \
+ if (len > buff_len) return -3; /* overflow */ \
+ for (i=0; i<len; i++) { \
+ if (!isUTF8(len, 1+i, buff[i])) return -1; /* not UTF-8 */ \
+ } \
+ return len; \
}
+DEFINE_UTF8SLENGTH(, unsigned char)
+DEFINE_UTF8SLENGTH(short, unsigned short)
+
/* WITHOUT strict range check */
long UTF8StoUCS(unsigned char *s)
diff --git a/Build/source/texk/ptexenc/version.ac b/Build/source/texk/ptexenc/version.ac
index 7c02f883de3..9b8e183f2dd 100644
--- a/Build/source/texk/ptexenc/version.ac
+++ b/Build/source/texk/ptexenc/version.ac
@@ -10,4 +10,4 @@ dnl Keep package and shared library versions the same;
dnl see kpathsea/version.ac.
dnl
dnl This file is m4-included from configure.ac.
-m4_define([ptexenc_version], [1.3.11])
+m4_define([ptexenc_version], [1.4.0])