summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorTakuji Tanaka <ttk@t-lab.opal.ne.jp>2024-02-11 08:19:52 +0000
committerTakuji Tanaka <ttk@t-lab.opal.ne.jp>2024-02-11 08:19:52 +0000
commitcd617e1f4eb9af88c114a2f7983182108fcf57f1 (patch)
tree2bfa4479729452673755fcf5be3d666225affe6a
parenta4be270706ba9d1b678be1ccc72ac97ed72fe18d (diff)
ptexenc: Add function guessing end-of-line code
git-svn-id: svn://tug.org/texlive/trunk@69798 c570f23f-e606-0410-a88d-b1316a301751
-rw-r--r--Build/source/texk/ptexenc/ChangeLog10
-rw-r--r--Build/source/texk/ptexenc/Makefile.am6
-rw-r--r--Build/source/texk/ptexenc/Makefile.in6
-rw-r--r--Build/source/texk/ptexenc/c-auto.in2
-rwxr-xr-xBuild/source/texk/ptexenc/configure26
-rw-r--r--Build/source/texk/ptexenc/ptekf.c11
-rw-r--r--Build/source/texk/ptexenc/ptexenc.c40
-rw-r--r--Build/source/texk/ptexenc/ptexenc/ptexenc.h2
-rwxr-xr-xBuild/source/texk/ptexenc/tests/ptekf-eol.test58
-rw-r--r--Build/source/texk/ptexenc/version.ac2
10 files changed, 126 insertions, 37 deletions
diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog
index 940aa29eb2f..a0084ef3bee 100644
--- a/Build/source/texk/ptexenc/ChangeLog
+++ b/Build/source/texk/ptexenc/ChangeLog
@@ -1,3 +1,13 @@
+2024-02-11 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
+
+ * ptexenc.c, ptexenc/ptexenc.h:
+ Add function guessing end-of-line code.
+ * ptekf.c: Better treatment of end-of-line.
+ * tests/ptekf-eol.test: New tests for end-of-line.
+ * Makefile.am: Adjust.
+ * version.ac: Bump to 1.4.6.
+ https://github.com/texjporg/tex-jp-build/issues/142
+
2024-02-10 TANAKA Takuji <ttk@t-lab.opal.ne.jp>
* ptekf.1: Add command line manual.
diff --git a/Build/source/texk/ptexenc/Makefile.am b/Build/source/texk/ptexenc/Makefile.am
index 70f0206f93f..fe35a6b7b24 100644
--- a/Build/source/texk/ptexenc/Makefile.am
+++ b/Build/source/texk/ptexenc/Makefile.am
@@ -49,9 +49,9 @@ LDADD = $(KPATHSEA_LIBS)
## Tests
#
-TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test
+TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test tests/ptekf-eol.test
#
-tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log \
+tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log tests/ptekf-eol.log \
: ptekf$(EXEEXT)
#
EXTRA_DIST += $(TESTS)
@@ -67,7 +67,7 @@ EXTRA_DIST += \
tests/enc-utf8.bib-euc tests/enc-utf8.bib-jis tests/enc-utf8.bib-sjis tests/enc-utf8.bib-utf8 \
tests/enc-utf8a.bib-euc tests/enc-utf8a.bib-jis tests/enc-utf8a.bib-sjis tests/enc-utf8a.bib-utf8 \
tests/enc-utf8b.bib-euc tests/enc-utf8b.bib-jis tests/enc-utf8b.bib-sjis tests/enc-utf8b.bib-utf8
-DISTCLEANFILES = enc-*.bib*
+DISTCLEANFILES = enc-*.bib* saza-*.txt*
# Rebuild
diff --git a/Build/source/texk/ptexenc/Makefile.in b/Build/source/texk/ptexenc/Makefile.in
index 787b066ebdb..37c6a8840fb 100644
--- a/Build/source/texk/ptexenc/Makefile.in
+++ b/Build/source/texk/ptexenc/Makefile.in
@@ -617,8 +617,8 @@ man1_MANS = ptekf.1
LDADD = $(KPATHSEA_LIBS)
#
-TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test
-DISTCLEANFILES = enc-*.bib*
+TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test tests/ptekf-eol.test
+DISTCLEANFILES = enc-*.bib* saza-*.txt*
# Rebuild
rebuild_prereq =
@@ -1486,7 +1486,7 @@ $(libptexenc_la_OBJECTS): $(KPATHSEA_DEPEND)
@KPATHSEA_RULE@
#
-tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log \
+tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log tests/ptekf-eol.log \
: ptekf$(EXEEXT)
rebuild.stamp: $(rebuild_target)
echo timestamp >$@
diff --git a/Build/source/texk/ptexenc/c-auto.in b/Build/source/texk/ptexenc/c-auto.in
index c21e2d2173b..de910e60e1f 100644
--- a/Build/source/texk/ptexenc/c-auto.in
+++ b/Build/source/texk/ptexenc/c-auto.in
@@ -6,7 +6,7 @@
#define PTEXENC_C_AUTO_H
/* ptexenc: the version string. */
-#define PTEXENCVERSION "ptexenc version 1.4.5"
+#define PTEXENCVERSION "ptexenc version 1.4.6"
/* Define to 1 if the 'closedir' function returns void instead of int. */
#undef CLOSEDIR_VOID
diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure
index d88902cb48e..9df80f95e7d 100755
--- a/Build/source/texk/ptexenc/configure
+++ b/Build/source/texk/ptexenc/configure
@@ -1,6 +1,6 @@
#! /bin/sh
# Guess values for system-dependent variables and create Makefiles.
-# Generated by GNU Autoconf 2.72 for ptexenc 1.4.5.
+# Generated by GNU Autoconf 2.72 for ptexenc 1.4.6.
#
# Report bugs to <tex-k@tug.org>.
#
@@ -614,8 +614,8 @@ MAKEFLAGS=
# Identity of this package.
PACKAGE_NAME='ptexenc'
PACKAGE_TARNAME='ptexenc'
-PACKAGE_VERSION='1.4.5'
-PACKAGE_STRING='ptexenc 1.4.5'
+PACKAGE_VERSION='1.4.6'
+PACKAGE_STRING='ptexenc 1.4.6'
PACKAGE_BUGREPORT='tex-k@tug.org'
PACKAGE_URL=''
@@ -1363,7 +1363,7 @@ if test "$ac_init_help" = "long"; then
# Omit some internal or obsolete options to make the list less imposing.
# This message is too long to be a string in the A/UX 3.1 sh.
cat <<_ACEOF
-'configure' configures ptexenc 1.4.5 to adapt to many kinds of systems.
+'configure' configures ptexenc 1.4.6 to adapt to many kinds of systems.
Usage: $0 [OPTION]... [VAR=VALUE]...
@@ -1434,7 +1434,7 @@ fi
if test -n "$ac_init_help"; then
case $ac_init_help in
- short | recursive ) echo "Configuration of ptexenc 1.4.5:";;
+ short | recursive ) echo "Configuration of ptexenc 1.4.6:";;
esac
cat <<\_ACEOF
@@ -1555,7 +1555,7 @@ fi
test -n "$ac_init_help" && exit $ac_status
if $ac_init_version; then
cat <<\_ACEOF
-ptexenc configure 1.4.5
+ptexenc configure 1.4.6
generated by GNU Autoconf 2.72
Copyright (C) 2023 Free Software Foundation, Inc.
@@ -2097,7 +2097,7 @@ cat >config.log <<_ACEOF
This file contains any messages produced by compilers while
running configure, to aid debugging if configure makes a mistake.
-It was created by ptexenc $as_me 1.4.5, which was
+It was created by ptexenc $as_me 1.4.6, which was
generated by GNU Autoconf 2.72. Invocation command line was
$ $0$ac_configure_args_raw
@@ -2875,10 +2875,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu
-PTEXENCVERSION=1.4.5
+PTEXENCVERSION=1.4.6
-PTEXENC_LT_VERSINFO=5:5:4
+PTEXENC_LT_VERSINFO=5:6:4
am__api_version='1.16'
@@ -8664,7 +8664,7 @@ fi
# Define the identity of the package.
PACKAGE='ptexenc'
- VERSION='1.4.5'
+ VERSION='1.4.6'
printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h
@@ -15005,7 +15005,7 @@ Usage: $0 [OPTIONS]
Report bugs to <bug-libtool@gnu.org>."
lt_cl_version="\
-ptexenc config.lt 1.4.5
+ptexenc config.lt 1.4.6
configured by $0, generated by GNU Autoconf 2.72.
Copyright (C) 2011 Free Software Foundation, Inc.
@@ -16777,7 +16777,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1
# report actual input values of CONFIG_FILES etc. instead of their
# values after options handling.
ac_log="
-This file was extended by ptexenc $as_me 1.4.5, which was
+This file was extended by ptexenc $as_me 1.4.6, which was
generated by GNU Autoconf 2.72. Invocation command line was
CONFIG_FILES = $CONFIG_FILES
@@ -16845,7 +16845,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\
cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1
ac_cs_config='$ac_cs_config_escaped'
ac_cs_version="\\
-ptexenc config.status 1.4.5
+ptexenc config.status 1.4.6
configured by $0, generated by GNU Autoconf 2.72,
with options \\"\$ac_cs_config\\"
diff --git a/Build/source/texk/ptexenc/ptekf.c b/Build/source/texk/ptexenc/ptekf.c
index 4c52009f62c..4f17155ea28 100644
--- a/Build/source/texk/ptexenc/ptekf.c
+++ b/Build/source/texk/ptexenc/ptekf.c
@@ -45,7 +45,7 @@ static struct option long_options[] = {
{0, 0, 0, 0}
};
-#define MY_VERSION "20240201"
+#define MY_VERSION "20240211"
#define BUG_ADDRESS "issue@texjp.org"
static void show_version(void)
@@ -87,7 +87,7 @@ static char *mfgets(char *buff, int size, FILE *fp)
if ((len = input_line2(fp, (unsigned char *)buff, NULL, 0, size, &c)) == 0
&& c != '\r' && c != '\n') return NULL;
if (c == '\n' || c == '\r') {
- if (len+1 < size) strcat(buff+len, "\n");
+ if (len+1 < size) { buff[len]=(unsigned char)c; buff[len+1]='\0'; }
else ungetc(c, fp);
}
if (c == EOF) return NULL;
@@ -183,7 +183,7 @@ main (int argc, char **argv)
exit(32);
}
if (flg_guess_enc) {
- genc = ptenc_guess_enc(infp, 1);
+ genc = ptenc_guess_enc(infp, 1, 1);
printf("%s: %s\n", infname, genc);
setinfileenc(infp, genc);
free(genc);
@@ -212,6 +212,11 @@ main (int argc, char **argv)
while ((ret = mfgets(buff, BUFFERLEN, infp)) != NULL) {
(*fputs__)(buff, outfp);
}
+ if (ret == NULL && feof(infp)) {
+ c = buff[strlen(buff)-1];
+ if (c != '\n' && c != '\r')
+ (*fputs__)(buff, outfp);
+ }
if (fclose(infp)) {
fprintf(stderr, "ERROR: fail to close input file [%s].", infname);
exit(32);
diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c
index adb853971a9..9531c2a6933 100644
--- a/Build/source/texk/ptexenc/ptexenc.c
+++ b/Build/source/texk/ptexenc/ptexenc.c
@@ -73,14 +73,14 @@ static int string_to_enc(const_string str)
if (strcasecmp(str, "utf8") == 0) return ENC_UTF8;
if (UPTEX_enabled && strcasecmp(str, "uptex") == 0) return ENC_UPTEX;
- if (strcasecmp(str, "ASCII")== 0) return file_enc;
+ if (strncasecmp(str, "ASCII", 5)== 0) return file_enc;
if (strncasecmp(str, "AMBIGUOUS", 9) == 0) return guess_enc;
- if (strcasecmp(str, "BINARY") == 0) return ENC_JIS;
- if (strcasecmp(str, "ISO-2022-JP") == 0) return ENC_JIS;
- if (strcasecmp(str, "EUC-JP") == 0) return ENC_EUC;
- if (strcasecmp(str, "Shift_JIS") == 0) return ENC_SJIS;
- if (strncasecmp(str, "UTF-8", 5) == 0) return ENC_UTF8;
- if (strcasecmp(str, "ISO-8859") == 0) return ENC_JIS;
+ if (strncasecmp(str, "BINARY", 6) == 0) return ENC_JIS;
+ if (strncasecmp(str, "ISO-2022-JP", 11) == 0) return ENC_JIS;
+ if (strncasecmp(str, "EUC-JP", 6) == 0) return ENC_EUC;
+ if (strncasecmp(str, "Shift_JIS", 9) == 0) return ENC_SJIS;
+ if (strncasecmp(str, "UTF-8", 5) == 0) return ENC_UTF8;
+ if (strncasecmp(str, "ISO-8859", 8) == 0) return ENC_JIS;
return -1; /* error */
}
@@ -820,10 +820,11 @@ static int infile_enc[NOFILE]; /* ENC_UNKNOWN (=0): not determined
JIS X 0208 only and no platform dependent characters in Shift_JIS, EUC-JP
ISO-8859 may have 0xA0..0xFF, may not have 0x80..0x9F
*/
-char *ptenc_guess_enc(FILE *fp, boolean chk_bom)
+char *ptenc_guess_enc(FILE *fp, boolean chk_bom, boolean chk_nl)
{
char *enc;
int k0, k1, k2, cdb[2], cu8[4], len_utf8;
+ int nl0=0, nl_cr=0, nl_lf=0, nl_crlf=0;
int is_ascii=1, lbyte=0;
int maybe_sjis=1, maybe_euc=1, maybe_utf8=1, maybe_iso8859=1, pos_db=0, pos_utf8=0;
int ch_sjis=0, ch_euc=0, ch_utf8=0, ch_iso8859=0, bom=0;
@@ -831,14 +832,23 @@ char *ptenc_guess_enc(FILE *fp, boolean chk_bom)
int i;
unsigned char str0[5];
#endif /* DEBUG */
- enc = xmalloc(sizeof(char)*20);
+ enc = xmalloc(sizeof(char)*32);
while ((k0 = fgetc(fp)) != EOF &&
- (maybe_sjis+maybe_euc+maybe_utf8+maybe_iso8859>1 || pos_db || pos_utf8 || lbyte<200)) {
+ (maybe_sjis+maybe_euc+maybe_utf8+maybe_iso8859>1 || pos_db || pos_utf8
+ || lbyte<320 || k0=='\r')) {
+ if (chk_nl) {
+ if (k0 == '\r') nl_cr++;
+ if (k0 == '\n') {
+ if (nl0 == '\r') { nl_cr--; nl_crlf++; }
+ else { nl_lf++; }
+ }
+ }
if (maybe_iso8859 && maybe_sjis+maybe_euc+maybe_utf8==1 && !pos_db && !pos_utf8
- && ch_iso8859>=2000) {
+ && ch_iso8859>=2000 && k0!='\r') {
break;
}
+ nl0 = k0;
if (chk_bom && lbyte<4 && bom_l[lbyte] <= k0 && k0 <= bom_u[lbyte]) bom++;
lbyte++;
if (k0==ESC) {
@@ -1049,6 +1059,12 @@ char *ptenc_guess_enc(FILE *fp, boolean chk_bom)
else
strcpy(enc,"BINARY");
post_process:
+ if (chk_nl && (nl_cr || nl_lf || nl_crlf)) {
+ if (nl_lf+nl_crlf==0) strcat(enc," (CR)");
+ else if (nl_cr+nl_crlf==0) strcat(enc," (LF)");
+ else if (nl_cr+nl_lf ==0) strcat(enc," (CRLF)");
+ else strcat(enc," (MIXED NL)");
+ }
rewind (fp);
return enc;
}
@@ -1102,7 +1118,7 @@ long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2,
getc4(fp);
getc4(fp);
rewind(fp);
- enc = ptenc_guess_enc(fp, 0);
+ enc = ptenc_guess_enc(fp, 0, 0);
if (string_to_enc(enc) > 0) {
infile_enc[fd] = string_to_enc(enc);
fprintf(stderr, "(guessed encoding #%d: %s = %s)", fd, enc, enc_to_string(infile_enc[fd]));
diff --git a/Build/source/texk/ptexenc/ptexenc/ptexenc.h b/Build/source/texk/ptexenc/ptexenc/ptexenc.h
index 35a4425a80e..2d6d21e5333 100644
--- a/Build/source/texk/ptexenc/ptexenc/ptexenc.h
+++ b/Build/source/texk/ptexenc/ptexenc/ptexenc.h
@@ -101,7 +101,7 @@ extern PTENCDLL long input_line2(FILE *fp, unsigned char *buff, unsigned char *b
extern PTENCDLL boolean setinfileenc(FILE *fp, const char *str);
extern PTENCDLL boolean setstdinenc(const char *str);
extern PTENCDLL boolean setfileenc(const char *str);
-extern PTENCDLL char *ptenc_guess_enc(FILE *fp, boolean chk_bom);
+extern PTENCDLL char *ptenc_guess_enc(FILE *fp, boolean chk_bom, boolean chk_nl);
#ifdef WIN32
extern PTENCDLL void clear_infile_enc(FILE *fp);
diff --git a/Build/source/texk/ptexenc/tests/ptekf-eol.test b/Build/source/texk/ptexenc/tests/ptekf-eol.test
new file mode 100755
index 00000000000..028a5ab3619
--- /dev/null
+++ b/Build/source/texk/ptexenc/tests/ptekf-eol.test
@@ -0,0 +1,58 @@
+#! /bin/sh -vx
+# $Id$
+# Copyright 2024 Japanese TeX Development Community <issue@texjp.org>
+# You may freely use, modify and/or distribute this file.
+
+# Not really a test, just making sure the program executes.
+
+BinDir=${BinDir:-.}
+ExeExt=${ExeExt:-}
+_ptekf=$BinDir/ptekf$ExeExt
+
+TEXMFCNF=$srcdir/../kpathsea
+web2cdir=$srcdir/../web2c
+
+export TEXMFCNF
+
+rm -f ./saza*.txt*
+
+# "さざ波" -> \343\201\225\343\201\226\346\263\242
+jstr='\343\201\225\343\201\226\346\263\242'
+astr='Ripples'
+cr='\015'
+lf='\012'
+crlf='\015\012'
+
+printf "$astr$cr$jstr$cr$astr $jstr$cr$jstr" > saza-utf8-cr0.txt
+printf "$astr$lf$jstr$lf$astr $jstr$lf$jstr" > saza-utf8-lf0.txt
+printf "$astr$crlf$jstr$crlf$astr $jstr$crlf$jstr" > saza-utf8-crlf0.txt
+printf "$astr$cr$jstr$lf$astr $jstr$crlf$jstr" > saza-utf8-mix.txt
+printf "$astr$cr$jstr$cr$astr $jstr$cr$jstr$cr" > saza-utf8-cr1.txt
+printf "$astr$lf$jstr$lf$astr $jstr$lf$jstr$lf" > saza-utf8-lf1.txt
+printf "$astr$crlf$jstr$crlf$astr $jstr$crlf$jstr$crlf" > saza-utf8-crlf1.txt
+
+for nl in cr0 lf0 crlf0 cr1 lf1 crlf1 mix; do
+
+ $_ptekf --guess saza-utf8-$nl.txt || exit 1
+ $_ptekf -Gj saza-utf8-$nl.txt || exit 2
+ mv saza-utf8-$nl.txt.out saza-utf8-$nl-jis.txt
+ $_ptekf -Ge saza-utf8-$nl.txt || exit 3
+ mv saza-utf8-$nl.txt.out saza-utf8-$nl-euc.txt
+ $_ptekf -Gs saza-utf8-$nl.txt || exit 4
+ mv saza-utf8-$nl.txt.out saza-utf8-$nl-sjis.txt
+
+
+ for enc in jis euc sjis; do
+
+ $_ptekf -Gu saza-utf8-$nl-$enc.txt || exit 21
+ mv saza-utf8-$nl-$enc.txt.out saza-utf8-$nl-$enc-utf8.txt
+ diff saza-utf8-$nl.txt saza-utf8-$nl-$enc-utf8.txt || exit 22
+
+ done
+
+done
+
+$_ptekf --guess saza-utf8-*-jis.txt || exit 51
+$_ptekf --guess saza-utf8-*-euc.txt || exit 52
+$_ptekf --guess saza-utf8-*-sjis.txt || exit 53
+$_ptekf --guess saza-utf8-*-*-utf8.txt || exit 54
diff --git a/Build/source/texk/ptexenc/version.ac b/Build/source/texk/ptexenc/version.ac
index 1243b354401..53662598bcc 100644
--- a/Build/source/texk/ptexenc/version.ac
+++ b/Build/source/texk/ptexenc/version.ac
@@ -11,4 +11,4 @@ dnl Keep package and shared library versions the same;
dnl see kpathsea/version.ac.
dnl
dnl This file is m4-included from configure.ac.
-m4_define([ptexenc_version], [1.4.5])
+m4_define([ptexenc_version], [1.4.6])