From 18ad6385908e8e5ae37cd3f3acbb3d85d9f183d4 Mon Sep 17 00:00:00 2001 From: Takuji Tanaka Date: Sat, 24 Sep 2022 02:01:36 +0000 Subject: ptexenc: skip guessing file encoding if from pipe git-svn-id: svn://tug.org/texlive/trunk@64494 c570f23f-e606-0410-a88d-b1316a301751 --- Build/source/texk/mendexk/ChangeLog | 6 ++++++ Build/source/texk/mendexk/Makefile.am | 8 +++++--- Build/source/texk/mendexk/Makefile.in | 7 ++++--- Build/source/texk/mendexk/tests/enc-e.idx | 1 + Build/source/texk/mendexk/tests/enc-j.idx | 1 + Build/source/texk/mendexk/tests/enc-s.idx | 1 + Build/source/texk/mendexk/tests/enc-u.idx | 1 + Build/source/texk/mendexk/tests/enc-x.ind | 6 ++++++ Build/source/texk/mendexk/tests/enc.dic | 1 + Build/source/texk/mendexk/tests/jis.ist | 10 ++++++++++ Build/source/texk/mendexk/tests/mendex.test | 31 ++++++++++++++++++++++++++++- Build/source/texk/ptexenc/ChangeLog | 5 +++++ Build/source/texk/ptexenc/ptexenc.c | 11 +++++++++- 13 files changed, 81 insertions(+), 8 deletions(-) create mode 100644 Build/source/texk/mendexk/tests/enc-e.idx create mode 100644 Build/source/texk/mendexk/tests/enc-j.idx create mode 100644 Build/source/texk/mendexk/tests/enc-s.idx create mode 100644 Build/source/texk/mendexk/tests/enc-u.idx create mode 100644 Build/source/texk/mendexk/tests/enc-x.ind create mode 100644 Build/source/texk/mendexk/tests/enc.dic create mode 100644 Build/source/texk/mendexk/tests/jis.ist (limited to 'Build/source') diff --git a/Build/source/texk/mendexk/ChangeLog b/Build/source/texk/mendexk/ChangeLog index 686bbb16802..09e51c57401 100644 --- a/Build/source/texk/mendexk/ChangeLog +++ b/Build/source/texk/mendexk/ChangeLog @@ -1,3 +1,9 @@ +2022-09-24 TANAKA Takuji + + * tests/mendex.test, tests/enc-{e,j,s,u}.idx, tests/enc.dic, + tests/jis.ist, tests/enc-x.ind, Makefile.am: + Add tests for guessing file encodings. + 2022-09-01 Hironori Kitagawa * main.c: diff --git a/Build/source/texk/mendexk/Makefile.am b/Build/source/texk/mendexk/Makefile.am index 2fd2b1b750d..7a495270923 100644 --- a/Build/source/texk/mendexk/Makefile.am +++ b/Build/source/texk/mendexk/Makefile.am @@ -59,7 +59,7 @@ EXTRA_DIST += $(TESTS) EXTRA_DIST += tests/foo.idx tests/foo.ind \ tests/uni.idx tests/uni.dict tests/uni.ist \ tests/uni1.ist tests/uni2.ist tests/uni.ind \ - tests/sjis.ist tests/euc.ist \ + tests/sjis.ist tests/euc.ist tests/jis.ist \ tests/range.idx tests/range2.ist tests/range2.ist tests/range3.ist \ tests/range1.ind tests/range2.ind tests/range3.ind \ tests/pprec0.ist tests/pprec1.ist tests/pprec2.ist tests/pprec3.ist \ @@ -69,6 +69,8 @@ EXTRA_DIST += tests/foo.idx tests/foo.ind \ tests/romalpA.idx tests/romalpA-5.ind tests/romalpA-6.ind \ tests/romalpB.idx tests/romalpB-5.ind tests/romalpB-6.ind tests/romalpB-7.ind \ tests/romalpC.idx tests/romalpC-5.ind \ - tests/romalpD.idx tests/romalpD-5.ind tests/romalpD-6.ind tests/romalpD-7.ind -DISTCLEANFILES = foo.* uni.* range*.* pprec*.* romalp*.* + tests/romalpD.idx tests/romalpD-5.ind tests/romalpD-6.ind tests/romalpD-7.ind \ + tests/enc-e.idx tests/enc-j.idx tests/enc-s.idx tests/enc-u.idx \ + tests/enc-x.ind tests/enc.dic +DISTCLEANFILES = foo.* uni.* range*.* pprec*.* romalp*.* enc*.* diff --git a/Build/source/texk/mendexk/Makefile.in b/Build/source/texk/mendexk/Makefile.in index 2cbcdab4525..0dbf1a495d2 100644 --- a/Build/source/texk/mendexk/Makefile.in +++ b/Build/source/texk/mendexk/Makefile.in @@ -583,7 +583,7 @@ EXTRA_DIST = COPYRIGHT COPYRIGHT.ja ChangeLog.ja indexsty \ mendex.ja.txt mendex.1 mendex.1.ja $(TESTS) tests/foo.idx \ tests/foo.ind tests/uni.idx tests/uni.dict tests/uni.ist \ tests/uni1.ist tests/uni2.ist tests/uni.ind tests/sjis.ist \ - tests/euc.ist tests/range.idx tests/range2.ist \ + tests/euc.ist tests/jis.ist tests/range.idx tests/range2.ist \ tests/range2.ist tests/range3.ist tests/range1.ind \ tests/range2.ind tests/range3.ind tests/pprec0.ist \ tests/pprec1.ist tests/pprec2.ist tests/pprec3.ist \ @@ -595,9 +595,10 @@ EXTRA_DIST = COPYRIGHT COPYRIGHT.ja ChangeLog.ja indexsty \ tests/romalpB.idx tests/romalpB-5.ind tests/romalpB-6.ind \ tests/romalpB-7.ind tests/romalpC.idx tests/romalpC-5.ind \ tests/romalpD.idx tests/romalpD-5.ind tests/romalpD-6.ind \ - tests/romalpD-7.ind + tests/romalpD-7.ind tests/enc-e.idx tests/enc-j.idx \ + tests/enc-s.idx tests/enc-u.idx tests/enc-x.ind tests/enc.dic TESTS = tests/mendex.test -DISTCLEANFILES = foo.* uni.* range*.* pprec*.* romalp*.* +DISTCLEANFILES = foo.* uni.* range*.* pprec*.* romalp*.* enc*.* all: c-auto.h $(MAKE) $(AM_MAKEFLAGS) all-am diff --git a/Build/source/texk/mendexk/tests/enc-e.idx b/Build/source/texk/mendexk/tests/enc-e.idx new file mode 100644 index 00000000000..2e051d95758 --- /dev/null +++ b/Build/source/texk/mendexk/tests/enc-e.idx @@ -0,0 +1 @@ +\indexentry{}{1} diff --git a/Build/source/texk/mendexk/tests/enc-j.idx b/Build/source/texk/mendexk/tests/enc-j.idx new file mode 100644 index 00000000000..d7d2f7fd20f --- /dev/null +++ b/Build/source/texk/mendexk/tests/enc-j.idx @@ -0,0 +1 @@ +\indexentry{$B@c7n2V(B}{1} diff --git a/Build/source/texk/mendexk/tests/enc-s.idx b/Build/source/texk/mendexk/tests/enc-s.idx new file mode 100644 index 00000000000..c2c496fb9e4 --- /dev/null +++ b/Build/source/texk/mendexk/tests/enc-s.idx @@ -0,0 +1 @@ +\indexentry{ጎ}{1} diff --git a/Build/source/texk/mendexk/tests/enc-u.idx b/Build/source/texk/mendexk/tests/enc-u.idx new file mode 100644 index 00000000000..cd73308ccb0 --- /dev/null +++ b/Build/source/texk/mendexk/tests/enc-u.idx @@ -0,0 +1 @@ +\indexentry{雪月花}{1} diff --git a/Build/source/texk/mendexk/tests/enc-x.ind b/Build/source/texk/mendexk/tests/enc-x.ind new file mode 100644 index 00000000000..1312760cafa --- /dev/null +++ b/Build/source/texk/mendexk/tests/enc-x.ind @@ -0,0 +1,6 @@ +\begin{theindex} + +\centerline{\bfseries ◆◆◆ セ ◆◆◆}\par\nobreak + \item 雪月花\leaders\hbox{$\cdot$}\hfill 1 + +\end{theindex} diff --git a/Build/source/texk/mendexk/tests/enc.dic b/Build/source/texk/mendexk/tests/enc.dic new file mode 100644 index 00000000000..db01b9875ee --- /dev/null +++ b/Build/source/texk/mendexk/tests/enc.dic @@ -0,0 +1 @@ +$B@c7n2V(B $B$;$D$2$D$+(B diff --git a/Build/source/texk/mendexk/tests/jis.ist b/Build/source/texk/mendexk/tests/jis.ist new file mode 100644 index 00000000000..76a5f16752f --- /dev/null +++ b/Build/source/texk/mendexk/tests/jis.ist @@ -0,0 +1,10 @@ +% -*- coding: iso-2022-jp -*- + +delim_0 "\\leaders\\hbox{$\\cdot$}\\hfill " +delim_1 "\\leaders\\hbox{$\\cdot$}\\hfill " +delim_2 "\\leaders\\hbox{$\\cdot$}\\hfill " + +lethead_flag 1 +lethead_prefix "\n\\centerline{\\bfseries $B"!"!"!(B " +lethead_suffix " $B"!"!"!(B}\\par\\nobreak" + diff --git a/Build/source/texk/mendexk/tests/mendex.test b/Build/source/texk/mendexk/tests/mendex.test index 81ae944748c..b1fe4eab828 100755 --- a/Build/source/texk/mendexk/tests/mendex.test +++ b/Build/source/texk/mendexk/tests/mendex.test @@ -7,7 +7,7 @@ TEXMFCNF=$srcdir/../kpathsea export TEXMFCNF -rm -f foo.* uni.* range*.* pprec*.* romalp*.* sjis.* euc.* +rm -f foo.* uni.* range*.* pprec*.* romalp*.* sjis.* euc.* enc*.* rc=0 ./mendex $srcdir/tests/foo.idx -o foo.ind1 -t foo.ilg1 \ @@ -89,5 +89,34 @@ for num in 5 6 7; do && diff $srcdir/tests/romalp$sfx-$num.ind romalp$sfx-$num.ind1 || rc=12 done +# guess file encoding +# input from file +for enc in j e s u; do +./mendex --guess-input-enc -s $srcdir/tests/jis.ist -d $srcdir/tests/enc.dic $srcdir/tests/enc-$enc.idx \ + -o enc-$enc.ind1 -t enc-$enc.ilg \ + && diff $srcdir/tests/enc-x.ind enc-$enc.ind1 || rc=101 +done + +for enc in j u; do +./mendex --no-guess-input-enc -s $srcdir/tests/uni.ist -d $srcdir/tests/enc.dic $srcdir/tests/enc-$enc.idx \ + -o enc-$enc.ind1 -t enc-$enc.ilg \ + && diff $srcdir/tests/enc-x.ind enc-$enc.ind1 || rc=102 +done + +# input from pipe +for enc in j u; do +cat $srcdir/tests/enc-$enc.idx | \ + ./mendex --guess-input-enc -i -s $srcdir/tests/euc.ist -d $srcdir/tests/enc.dic \ + -o enc-p-$enc.ind1 -t enc-p-$enc.ilg \ + && diff $srcdir/tests/enc-x.ind enc-p-$enc.ind1 || rc=103 +done + +# imput from redirect +for enc in j u; do +./mendex --guess-input-enc -i -s $srcdir/tests/sjis.ist -d $srcdir/tests/enc.dic < $srcdir/tests/enc-$enc.idx \ + -o enc-r-$enc.ind1 -t enc-r-$enc.ilg \ + && diff $srcdir/tests/enc-x.ind enc-r-$enc.ind1 || rc=104 +done + exit $rc diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog index 5128834aab8..193debcd6ee 100644 --- a/Build/source/texk/ptexenc/ChangeLog +++ b/Build/source/texk/ptexenc/ChangeLog @@ -1,3 +1,8 @@ +2022-09-24 TANAKA Takuji + + * ptexenc.c: Skip guessing file encoding if input is from pipe. + https://github.com/texjporg/tex-jp-build/issues/148 + 2022-09-01 Hironori Kitagawa * ptexenc.c: Initialize infile_enc_auto only when this variable diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c index 42858d47c0d..6d74a71cecb 100644 --- a/Build/source/texk/ptexenc/ptexenc.c +++ b/Build/source/texk/ptexenc/ptexenc.c @@ -16,6 +16,7 @@ #include #include +#include #define ENC_UNKNOWN 0 #define ENC_JIS 1 @@ -1024,8 +1025,16 @@ long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, #endif /* DEBUG */ } else { + struct stat st; if (infile_enc_auto == 2) ptenc_set_infile_enc_auto(); - if (infile_enc_auto && fd != fileno(stdin)) { +#ifdef DEBUG + if (infile_enc_auto) { + fprintf(stderr, "\nInput fd: %d, stdin?: %d, pipe?: %d\n", fd, + fd==fileno(stdin), (fstat(fd, &st)==0 && S_ISFIFO(st.st_mode))); + } +#endif /* DEBUG */ + if (infile_enc_auto && fd != fileno(stdin) + && !(fstat(fd, &st)==0 && S_ISFIFO(st.st_mode))) { char *enc; getc4(fp); getc4(fp); -- cgit v1.2.3