diff options
author | Takuji Tanaka <ttk@t-lab.opal.ne.jp> | 2024-02-11 08:19:52 +0000 |
---|---|---|
committer | Takuji Tanaka <ttk@t-lab.opal.ne.jp> | 2024-02-11 08:19:52 +0000 |
commit | cd617e1f4eb9af88c114a2f7983182108fcf57f1 (patch) | |
tree | 2bfa4479729452673755fcf5be3d666225affe6a | |
parent | a4be270706ba9d1b678be1ccc72ac97ed72fe18d (diff) |
ptexenc: Add function guessing end-of-line code
git-svn-id: svn://tug.org/texlive/trunk@69798 c570f23f-e606-0410-a88d-b1316a301751
-rw-r--r-- | Build/source/texk/ptexenc/ChangeLog | 10 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/Makefile.am | 6 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/Makefile.in | 6 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/c-auto.in | 2 | ||||
-rwxr-xr-x | Build/source/texk/ptexenc/configure | 26 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/ptekf.c | 11 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/ptexenc.c | 40 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/ptexenc/ptexenc.h | 2 | ||||
-rwxr-xr-x | Build/source/texk/ptexenc/tests/ptekf-eol.test | 58 | ||||
-rw-r--r-- | Build/source/texk/ptexenc/version.ac | 2 |
10 files changed, 126 insertions, 37 deletions
diff --git a/Build/source/texk/ptexenc/ChangeLog b/Build/source/texk/ptexenc/ChangeLog index 940aa29eb2f..a0084ef3bee 100644 --- a/Build/source/texk/ptexenc/ChangeLog +++ b/Build/source/texk/ptexenc/ChangeLog @@ -1,3 +1,13 @@ +2024-02-11 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * ptexenc.c, ptexenc/ptexenc.h: + Add function guessing end-of-line code. + * ptekf.c: Better treatment of end-of-line. + * tests/ptekf-eol.test: New tests for end-of-line. + * Makefile.am: Adjust. + * version.ac: Bump to 1.4.6. + https://github.com/texjporg/tex-jp-build/issues/142 + 2024-02-10 TANAKA Takuji <ttk@t-lab.opal.ne.jp> * ptekf.1: Add command line manual. diff --git a/Build/source/texk/ptexenc/Makefile.am b/Build/source/texk/ptexenc/Makefile.am index 70f0206f93f..fe35a6b7b24 100644 --- a/Build/source/texk/ptexenc/Makefile.am +++ b/Build/source/texk/ptexenc/Makefile.am @@ -49,9 +49,9 @@ LDADD = $(KPATHSEA_LIBS) ## Tests # -TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test +TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test tests/ptekf-eol.test # -tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log \ +tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log tests/ptekf-eol.log \ : ptekf$(EXEEXT) # EXTRA_DIST += $(TESTS) @@ -67,7 +67,7 @@ EXTRA_DIST += \ tests/enc-utf8.bib-euc tests/enc-utf8.bib-jis tests/enc-utf8.bib-sjis tests/enc-utf8.bib-utf8 \ tests/enc-utf8a.bib-euc tests/enc-utf8a.bib-jis tests/enc-utf8a.bib-sjis tests/enc-utf8a.bib-utf8 \ tests/enc-utf8b.bib-euc tests/enc-utf8b.bib-jis tests/enc-utf8b.bib-sjis tests/enc-utf8b.bib-utf8 -DISTCLEANFILES = enc-*.bib* +DISTCLEANFILES = enc-*.bib* saza-*.txt* # Rebuild diff --git a/Build/source/texk/ptexenc/Makefile.in b/Build/source/texk/ptexenc/Makefile.in index 787b066ebdb..37c6a8840fb 100644 --- a/Build/source/texk/ptexenc/Makefile.in +++ b/Build/source/texk/ptexenc/Makefile.in @@ -617,8 +617,8 @@ man1_MANS = ptekf.1 LDADD = $(KPATHSEA_LIBS) # -TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test -DISTCLEANFILES = enc-*.bib* +TESTS = tests/ptekf-smoke.test tests/ptekf-guess.test tests/ptekf-conv.test tests/ptekf-eol.test +DISTCLEANFILES = enc-*.bib* saza-*.txt* # Rebuild rebuild_prereq = @@ -1486,7 +1486,7 @@ $(libptexenc_la_OBJECTS): $(KPATHSEA_DEPEND) @KPATHSEA_RULE@ # -tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log \ +tests/ptekf-smoke.log tests/ptekf-guess.log tests/ptekf-conv.log tests/ptekf-eol.log \ : ptekf$(EXEEXT) rebuild.stamp: $(rebuild_target) echo timestamp >$@ diff --git a/Build/source/texk/ptexenc/c-auto.in b/Build/source/texk/ptexenc/c-auto.in index c21e2d2173b..de910e60e1f 100644 --- a/Build/source/texk/ptexenc/c-auto.in +++ b/Build/source/texk/ptexenc/c-auto.in @@ -6,7 +6,7 @@ #define PTEXENC_C_AUTO_H /* ptexenc: the version string. */ -#define PTEXENCVERSION "ptexenc version 1.4.5" +#define PTEXENCVERSION "ptexenc version 1.4.6" /* Define to 1 if the 'closedir' function returns void instead of int. */ #undef CLOSEDIR_VOID diff --git a/Build/source/texk/ptexenc/configure b/Build/source/texk/ptexenc/configure index d88902cb48e..9df80f95e7d 100755 --- a/Build/source/texk/ptexenc/configure +++ b/Build/source/texk/ptexenc/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.72 for ptexenc 1.4.5. +# Generated by GNU Autoconf 2.72 for ptexenc 1.4.6. # # Report bugs to <tex-k@tug.org>. # @@ -614,8 +614,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='ptexenc' PACKAGE_TARNAME='ptexenc' -PACKAGE_VERSION='1.4.5' -PACKAGE_STRING='ptexenc 1.4.5' +PACKAGE_VERSION='1.4.6' +PACKAGE_STRING='ptexenc 1.4.6' PACKAGE_BUGREPORT='tex-k@tug.org' PACKAGE_URL='' @@ -1363,7 +1363,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -'configure' configures ptexenc 1.4.5 to adapt to many kinds of systems. +'configure' configures ptexenc 1.4.6 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1434,7 +1434,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of ptexenc 1.4.5:";; + short | recursive ) echo "Configuration of ptexenc 1.4.6:";; esac cat <<\_ACEOF @@ -1555,7 +1555,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -ptexenc configure 1.4.5 +ptexenc configure 1.4.6 generated by GNU Autoconf 2.72 Copyright (C) 2023 Free Software Foundation, Inc. @@ -2097,7 +2097,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by ptexenc $as_me 1.4.5, which was +It was created by ptexenc $as_me 1.4.6, which was generated by GNU Autoconf 2.72. Invocation command line was $ $0$ac_configure_args_raw @@ -2875,10 +2875,10 @@ ac_compiler_gnu=$ac_cv_c_compiler_gnu -PTEXENCVERSION=1.4.5 +PTEXENCVERSION=1.4.6 -PTEXENC_LT_VERSINFO=5:5:4 +PTEXENC_LT_VERSINFO=5:6:4 am__api_version='1.16' @@ -8664,7 +8664,7 @@ fi # Define the identity of the package. PACKAGE='ptexenc' - VERSION='1.4.5' + VERSION='1.4.6' printf "%s\n" "#define PACKAGE \"$PACKAGE\"" >>confdefs.h @@ -15005,7 +15005,7 @@ Usage: $0 [OPTIONS] Report bugs to <bug-libtool@gnu.org>." lt_cl_version="\ -ptexenc config.lt 1.4.5 +ptexenc config.lt 1.4.6 configured by $0, generated by GNU Autoconf 2.72. Copyright (C) 2011 Free Software Foundation, Inc. @@ -16777,7 +16777,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by ptexenc $as_me 1.4.5, which was +This file was extended by ptexenc $as_me 1.4.6, which was generated by GNU Autoconf 2.72. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -16845,7 +16845,7 @@ ac_cs_config_escaped=`printf "%s\n" "$ac_cs_config" | sed "s/^ //; s/'/'\\\\\\\\ cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config='$ac_cs_config_escaped' ac_cs_version="\\ -ptexenc config.status 1.4.5 +ptexenc config.status 1.4.6 configured by $0, generated by GNU Autoconf 2.72, with options \\"\$ac_cs_config\\" diff --git a/Build/source/texk/ptexenc/ptekf.c b/Build/source/texk/ptexenc/ptekf.c index 4c52009f62c..4f17155ea28 100644 --- a/Build/source/texk/ptexenc/ptekf.c +++ b/Build/source/texk/ptexenc/ptekf.c @@ -45,7 +45,7 @@ static struct option long_options[] = { {0, 0, 0, 0} }; -#define MY_VERSION "20240201" +#define MY_VERSION "20240211" #define BUG_ADDRESS "issue@texjp.org" static void show_version(void) @@ -87,7 +87,7 @@ static char *mfgets(char *buff, int size, FILE *fp) if ((len = input_line2(fp, (unsigned char *)buff, NULL, 0, size, &c)) == 0 && c != '\r' && c != '\n') return NULL; if (c == '\n' || c == '\r') { - if (len+1 < size) strcat(buff+len, "\n"); + if (len+1 < size) { buff[len]=(unsigned char)c; buff[len+1]='\0'; } else ungetc(c, fp); } if (c == EOF) return NULL; @@ -183,7 +183,7 @@ main (int argc, char **argv) exit(32); } if (flg_guess_enc) { - genc = ptenc_guess_enc(infp, 1); + genc = ptenc_guess_enc(infp, 1, 1); printf("%s: %s\n", infname, genc); setinfileenc(infp, genc); free(genc); @@ -212,6 +212,11 @@ main (int argc, char **argv) while ((ret = mfgets(buff, BUFFERLEN, infp)) != NULL) { (*fputs__)(buff, outfp); } + if (ret == NULL && feof(infp)) { + c = buff[strlen(buff)-1]; + if (c != '\n' && c != '\r') + (*fputs__)(buff, outfp); + } if (fclose(infp)) { fprintf(stderr, "ERROR: fail to close input file [%s].", infname); exit(32); diff --git a/Build/source/texk/ptexenc/ptexenc.c b/Build/source/texk/ptexenc/ptexenc.c index adb853971a9..9531c2a6933 100644 --- a/Build/source/texk/ptexenc/ptexenc.c +++ b/Build/source/texk/ptexenc/ptexenc.c @@ -73,14 +73,14 @@ static int string_to_enc(const_string str) if (strcasecmp(str, "utf8") == 0) return ENC_UTF8; if (UPTEX_enabled && strcasecmp(str, "uptex") == 0) return ENC_UPTEX; - if (strcasecmp(str, "ASCII")== 0) return file_enc; + if (strncasecmp(str, "ASCII", 5)== 0) return file_enc; if (strncasecmp(str, "AMBIGUOUS", 9) == 0) return guess_enc; - if (strcasecmp(str, "BINARY") == 0) return ENC_JIS; - if (strcasecmp(str, "ISO-2022-JP") == 0) return ENC_JIS; - if (strcasecmp(str, "EUC-JP") == 0) return ENC_EUC; - if (strcasecmp(str, "Shift_JIS") == 0) return ENC_SJIS; - if (strncasecmp(str, "UTF-8", 5) == 0) return ENC_UTF8; - if (strcasecmp(str, "ISO-8859") == 0) return ENC_JIS; + if (strncasecmp(str, "BINARY", 6) == 0) return ENC_JIS; + if (strncasecmp(str, "ISO-2022-JP", 11) == 0) return ENC_JIS; + if (strncasecmp(str, "EUC-JP", 6) == 0) return ENC_EUC; + if (strncasecmp(str, "Shift_JIS", 9) == 0) return ENC_SJIS; + if (strncasecmp(str, "UTF-8", 5) == 0) return ENC_UTF8; + if (strncasecmp(str, "ISO-8859", 8) == 0) return ENC_JIS; return -1; /* error */ } @@ -820,10 +820,11 @@ static int infile_enc[NOFILE]; /* ENC_UNKNOWN (=0): not determined JIS X 0208 only and no platform dependent characters in Shift_JIS, EUC-JP ISO-8859 may have 0xA0..0xFF, may not have 0x80..0x9F */ -char *ptenc_guess_enc(FILE *fp, boolean chk_bom) +char *ptenc_guess_enc(FILE *fp, boolean chk_bom, boolean chk_nl) { char *enc; int k0, k1, k2, cdb[2], cu8[4], len_utf8; + int nl0=0, nl_cr=0, nl_lf=0, nl_crlf=0; int is_ascii=1, lbyte=0; int maybe_sjis=1, maybe_euc=1, maybe_utf8=1, maybe_iso8859=1, pos_db=0, pos_utf8=0; int ch_sjis=0, ch_euc=0, ch_utf8=0, ch_iso8859=0, bom=0; @@ -831,14 +832,23 @@ char *ptenc_guess_enc(FILE *fp, boolean chk_bom) int i; unsigned char str0[5]; #endif /* DEBUG */ - enc = xmalloc(sizeof(char)*20); + enc = xmalloc(sizeof(char)*32); while ((k0 = fgetc(fp)) != EOF && - (maybe_sjis+maybe_euc+maybe_utf8+maybe_iso8859>1 || pos_db || pos_utf8 || lbyte<200)) { + (maybe_sjis+maybe_euc+maybe_utf8+maybe_iso8859>1 || pos_db || pos_utf8 + || lbyte<320 || k0=='\r')) { + if (chk_nl) { + if (k0 == '\r') nl_cr++; + if (k0 == '\n') { + if (nl0 == '\r') { nl_cr--; nl_crlf++; } + else { nl_lf++; } + } + } if (maybe_iso8859 && maybe_sjis+maybe_euc+maybe_utf8==1 && !pos_db && !pos_utf8 - && ch_iso8859>=2000) { + && ch_iso8859>=2000 && k0!='\r') { break; } + nl0 = k0; if (chk_bom && lbyte<4 && bom_l[lbyte] <= k0 && k0 <= bom_u[lbyte]) bom++; lbyte++; if (k0==ESC) { @@ -1049,6 +1059,12 @@ char *ptenc_guess_enc(FILE *fp, boolean chk_bom) else strcpy(enc,"BINARY"); post_process: + if (chk_nl && (nl_cr || nl_lf || nl_crlf)) { + if (nl_lf+nl_crlf==0) strcat(enc," (CR)"); + else if (nl_cr+nl_crlf==0) strcat(enc," (LF)"); + else if (nl_cr+nl_lf ==0) strcat(enc," (CRLF)"); + else strcat(enc," (MIXED NL)"); + } rewind (fp); return enc; } @@ -1102,7 +1118,7 @@ long input_line2(FILE *fp, unsigned char *buff, unsigned char *buff2, getc4(fp); getc4(fp); rewind(fp); - enc = ptenc_guess_enc(fp, 0); + enc = ptenc_guess_enc(fp, 0, 0); if (string_to_enc(enc) > 0) { infile_enc[fd] = string_to_enc(enc); fprintf(stderr, "(guessed encoding #%d: %s = %s)", fd, enc, enc_to_string(infile_enc[fd])); diff --git a/Build/source/texk/ptexenc/ptexenc/ptexenc.h b/Build/source/texk/ptexenc/ptexenc/ptexenc.h index 35a4425a80e..2d6d21e5333 100644 --- a/Build/source/texk/ptexenc/ptexenc/ptexenc.h +++ b/Build/source/texk/ptexenc/ptexenc/ptexenc.h @@ -101,7 +101,7 @@ extern PTENCDLL long input_line2(FILE *fp, unsigned char *buff, unsigned char *b extern PTENCDLL boolean setinfileenc(FILE *fp, const char *str); extern PTENCDLL boolean setstdinenc(const char *str); extern PTENCDLL boolean setfileenc(const char *str); -extern PTENCDLL char *ptenc_guess_enc(FILE *fp, boolean chk_bom); +extern PTENCDLL char *ptenc_guess_enc(FILE *fp, boolean chk_bom, boolean chk_nl); #ifdef WIN32 extern PTENCDLL void clear_infile_enc(FILE *fp); diff --git a/Build/source/texk/ptexenc/tests/ptekf-eol.test b/Build/source/texk/ptexenc/tests/ptekf-eol.test new file mode 100755 index 00000000000..028a5ab3619 --- /dev/null +++ b/Build/source/texk/ptexenc/tests/ptekf-eol.test @@ -0,0 +1,58 @@ +#! /bin/sh -vx +# $Id$ +# Copyright 2024 Japanese TeX Development Community <issue@texjp.org> +# You may freely use, modify and/or distribute this file. + +# Not really a test, just making sure the program executes. + +BinDir=${BinDir:-.} +ExeExt=${ExeExt:-} +_ptekf=$BinDir/ptekf$ExeExt + +TEXMFCNF=$srcdir/../kpathsea +web2cdir=$srcdir/../web2c + +export TEXMFCNF + +rm -f ./saza*.txt* + +# "さざ波" -> \343\201\225\343\201\226\346\263\242 +jstr='\343\201\225\343\201\226\346\263\242' +astr='Ripples' +cr='\015' +lf='\012' +crlf='\015\012' + +printf "$astr$cr$jstr$cr$astr $jstr$cr$jstr" > saza-utf8-cr0.txt +printf "$astr$lf$jstr$lf$astr $jstr$lf$jstr" > saza-utf8-lf0.txt +printf "$astr$crlf$jstr$crlf$astr $jstr$crlf$jstr" > saza-utf8-crlf0.txt +printf "$astr$cr$jstr$lf$astr $jstr$crlf$jstr" > saza-utf8-mix.txt +printf "$astr$cr$jstr$cr$astr $jstr$cr$jstr$cr" > saza-utf8-cr1.txt +printf "$astr$lf$jstr$lf$astr $jstr$lf$jstr$lf" > saza-utf8-lf1.txt +printf "$astr$crlf$jstr$crlf$astr $jstr$crlf$jstr$crlf" > saza-utf8-crlf1.txt + +for nl in cr0 lf0 crlf0 cr1 lf1 crlf1 mix; do + + $_ptekf --guess saza-utf8-$nl.txt || exit 1 + $_ptekf -Gj saza-utf8-$nl.txt || exit 2 + mv saza-utf8-$nl.txt.out saza-utf8-$nl-jis.txt + $_ptekf -Ge saza-utf8-$nl.txt || exit 3 + mv saza-utf8-$nl.txt.out saza-utf8-$nl-euc.txt + $_ptekf -Gs saza-utf8-$nl.txt || exit 4 + mv saza-utf8-$nl.txt.out saza-utf8-$nl-sjis.txt + + + for enc in jis euc sjis; do + + $_ptekf -Gu saza-utf8-$nl-$enc.txt || exit 21 + mv saza-utf8-$nl-$enc.txt.out saza-utf8-$nl-$enc-utf8.txt + diff saza-utf8-$nl.txt saza-utf8-$nl-$enc-utf8.txt || exit 22 + + done + +done + +$_ptekf --guess saza-utf8-*-jis.txt || exit 51 +$_ptekf --guess saza-utf8-*-euc.txt || exit 52 +$_ptekf --guess saza-utf8-*-sjis.txt || exit 53 +$_ptekf --guess saza-utf8-*-*-utf8.txt || exit 54 diff --git a/Build/source/texk/ptexenc/version.ac b/Build/source/texk/ptexenc/version.ac index 1243b354401..53662598bcc 100644 --- a/Build/source/texk/ptexenc/version.ac +++ b/Build/source/texk/ptexenc/version.ac @@ -11,4 +11,4 @@ dnl Keep package and shared library versions the same; dnl see kpathsea/version.ac. dnl dnl This file is m4-included from configure.ac. -m4_define([ptexenc_version], [1.4.5]) +m4_define([ptexenc_version], [1.4.6]) |