diff options
Diffstat (limited to 'indexing/upmendex/source')
-rw-r--r-- | indexing/upmendex/source/ChangeLog | 43 | ||||
-rw-r--r-- | indexing/upmendex/source/Makefile.am | 13 | ||||
-rw-r--r-- | indexing/upmendex/source/Makefile.in | 9 | ||||
-rw-r--r-- | indexing/upmendex/source/aclocal.m4 | 1 | ||||
-rwxr-xr-x | indexing/upmendex/source/configure | 92 | ||||
-rw-r--r-- | indexing/upmendex/source/configure.ac | 20 | ||||
-rw-r--r-- | indexing/upmendex/source/convert.c | 3 | ||||
-rw-r--r-- | indexing/upmendex/source/fwrite.c | 19 | ||||
-rw-r--r-- | indexing/upmendex/source/main.c | 4 | ||||
-rw-r--r-- | indexing/upmendex/source/sort.c | 14 | ||||
-rwxr-xr-x | indexing/upmendex/source/tests/upmendex.test | 2 |
11 files changed, 186 insertions, 34 deletions
diff --git a/indexing/upmendex/source/ChangeLog b/indexing/upmendex/source/ChangeLog index 6e9e19eaa1..6c0e2aaef7 100644 --- a/indexing/upmendex/source/ChangeLog +++ b/indexing/upmendex/source/ChangeLog @@ -1,3 +1,46 @@ +2020-02-22 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * version 0.54 Beta version. + * configure.ac, main.c, COPYRIGHT: Bump version + and copyright year. + * {convert,fwrite,sort}.c: + Support Small Kana Extension in Unicode 12.0. + Add CJK Unified Ideographs Extension G. + Update samples. + +2020-01-16 Karl Berry <karl@freefriends.org> + + * configure.ac, + * COPYRIGHT, + * main.c (VERSION): version 0.53 for TL'20. + +2019-12-12 Karl Berry <karl@freefriends.org> + + * configure.ac: look for a pthread library cntaining + pthread_create (using KPSE_SEARCH_LIBS) and set PTHREAD_LIB if found. + * Makefile.am (LDADD): include $(PTHREAD_LIB). + +As of ICU 65.1, upmendex requires -lpthread. Otherwise, it crashes +at sort.c:82, which merely creates the icu_collator: + icu_collator = ucol_open(icu_locale, &status); + +About 10 stack frames deep into the ICU initialization, this winds up calling +(Build/source/libs/icu/icu-src/source/common/umutex.cpp:146): + std::call_once(*pInitFlag, umtx_init); + +which results in a fatal error from stdc++: + terminate called after throwing an instance of 'std::system_error' + what(): Unknown error -1 + +Web searching indicated that that opaque error message is a result of not +linking with -lpthread. So now we do so. So far our other ICU programs +(bibtexu, xetex) don't seem to require it. + +2019-03-16 TANAKA Takuji <ttk@t-lab.opal.ne.jp> + + * upmendex.ja.txt: Rename from upmendex.doc, + minor update. + 2019-03-06 Hironobu Yamashita <h.y.acetaminophen@gmail.com> Fix a bug in multibytelen(). diff --git a/indexing/upmendex/source/Makefile.am b/indexing/upmendex/source/Makefile.am index 28e452e447..fc188887f4 100644 --- a/indexing/upmendex/source/Makefile.am +++ b/indexing/upmendex/source/Makefile.am @@ -1,8 +1,8 @@ -## $Id: Makefile.am 44086 2017-04-27 21:38:29Z karl $ +## $Id$ ## Makefile.am for the TeX Live subdirectory texk/upmendex/ ## -## Copyright 2017 Karl Berry <tex-live@tug.org> -## Copyright 2010-2014 Peter Breitenlohner <tex-live@tug.org> +## Copyright 2015-2019 Karl Berry <tex-live@tug.org> +## Copyright 2010-2015 Peter Breitenlohner <tex-live@tug.org> ## You may freely use, modify and/or distribute this file. ## AM_CPPFLAGS = $(KPATHSEA_INCLUDES) $(ICU_INCLUDES) @@ -29,15 +29,12 @@ upmendex_SOURCES = \ styfile.c \ var.h - - # Force Automake to use CXXLD for linking nodist_EXTRA_upmendex_SOURCES = dummy.cxx - $(upmendex_OBJECTS): $(KPATHSEA_DEPEND) $(ICU_DEPEND) -LDADD = $(KPATHSEA_LIBS) $(ICU_LIBS) $(ICU_LIBS_EXTRA) +LDADD = $(KPATHSEA_LIBS) $(ICU_LIBS) $(PTHREAD_LIB) $(ICU_LIBS_EXTRA) ## Rebuild libkpathsea @KPATHSEA_RULE@ @@ -51,7 +48,7 @@ EXTRA_DIST = \ ## EXTRA_DIST += \ indexsty \ - upmendex.doc + upmendex.ja.txt ## Tests ## diff --git a/indexing/upmendex/source/Makefile.in b/indexing/upmendex/source/Makefile.in index 81311dda78..08c9fae74a 100644 --- a/indexing/upmendex/source/Makefile.in +++ b/indexing/upmendex/source/Makefile.in @@ -95,6 +95,7 @@ am__aclocal_m4_deps = $(top_srcdir)/../../m4/kpse-common.m4 \ $(top_srcdir)/../../m4/kpse-cxx-hack.m4 \ $(top_srcdir)/../../m4/kpse-icu-flags.m4 \ $(top_srcdir)/../../m4/kpse-kpathsea-flags.m4 \ + $(top_srcdir)/../../m4/kpse-search-libs.m4 \ $(top_srcdir)/../../m4/kpse-warnings.m4 \ $(top_srcdir)/../../m4/libtool.m4 \ $(top_srcdir)/../../m4/ltoptions.m4 \ @@ -121,7 +122,8 @@ am_upmendex_OBJECTS = convert.$(OBJEXT) fread.$(OBJEXT) \ upmendex_OBJECTS = $(am_upmendex_OBJECTS) upmendex_LDADD = $(LDADD) am__DEPENDENCIES_1 = -upmendex_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) +upmendex_DEPENDENCIES = $(am__DEPENDENCIES_1) $(am__DEPENDENCIES_1) \ + $(am__DEPENDENCIES_1) AM_V_lt = $(am__v_lt_@AM_V@) am__v_lt_ = $(am__v_lt_@AM_DEFAULT_V@) am__v_lt_0 = --silent @@ -515,6 +517,7 @@ PACKAGE_URL = @PACKAGE_URL@ PACKAGE_VERSION = @PACKAGE_VERSION@ PATH_SEPARATOR = @PATH_SEPARATOR@ PKG_CONFIG = @PKG_CONFIG@ +PTHREAD_LIB = @PTHREAD_LIB@ RANLIB = @RANLIB@ SED = @SED@ SET_MAKE = @SET_MAKE@ @@ -600,8 +603,8 @@ upmendex_SOURCES = \ # Force Automake to use CXXLD for linking nodist_EXTRA_upmendex_SOURCES = dummy.cxx -LDADD = $(KPATHSEA_LIBS) $(ICU_LIBS) $(ICU_LIBS_EXTRA) -EXTRA_DIST = COPYRIGHT indexsty upmendex.doc $(TESTS) tests/foo.idx \ +LDADD = $(KPATHSEA_LIBS) $(ICU_LIBS) $(PTHREAD_LIB) $(ICU_LIBS_EXTRA) +EXTRA_DIST = COPYRIGHT indexsty upmendex.ja.txt $(TESTS) tests/foo.idx \ tests/foo.ind tests/uni.idx tests/uni.dict tests/uni.ist \ tests/uni.ind TESTS = tests/upmendex.test diff --git a/indexing/upmendex/source/aclocal.m4 b/indexing/upmendex/source/aclocal.m4 index 5dc224d318..5392efefa9 100644 --- a/indexing/upmendex/source/aclocal.m4 +++ b/indexing/upmendex/source/aclocal.m4 @@ -1173,6 +1173,7 @@ m4_include([../../m4/kpse-common.m4]) m4_include([../../m4/kpse-cxx-hack.m4]) m4_include([../../m4/kpse-icu-flags.m4]) m4_include([../../m4/kpse-kpathsea-flags.m4]) +m4_include([../../m4/kpse-search-libs.m4]) m4_include([../../m4/kpse-warnings.m4]) m4_include([../../m4/libtool.m4]) m4_include([../../m4/ltoptions.m4]) diff --git a/indexing/upmendex/source/configure b/indexing/upmendex/source/configure index f3c0ab6482..55a19d1649 100755 --- a/indexing/upmendex/source/configure +++ b/indexing/upmendex/source/configure @@ -1,6 +1,6 @@ #! /bin/sh # Guess values for system-dependent variables and create Makefiles. -# Generated by GNU Autoconf 2.69 for upmendex (TeX Live) 0.52. +# Generated by GNU Autoconf 2.69 for upmendex (TeX Live) 0.54. # # # Copyright (C) 1992-1996, 1998-2012 Free Software Foundation, Inc. @@ -587,8 +587,8 @@ MAKEFLAGS= # Identity of this package. PACKAGE_NAME='upmendex (TeX Live)' PACKAGE_TARNAME='upmendex--tex-live-' -PACKAGE_VERSION='0.52' -PACKAGE_STRING='upmendex (TeX Live) 0.52' +PACKAGE_VERSION='0.54' +PACKAGE_STRING='upmendex (TeX Live) 0.54' PACKAGE_BUGREPORT='' PACKAGE_URL='' @@ -638,6 +638,7 @@ ICU_DEPEND ICU_LIBS ICU_INCLUDES ICU_CONFIG +PTHREAD_LIB KPATHSEA_RULE KPATHSEA_DEPEND KPATHSEA_LIBS @@ -1344,7 +1345,7 @@ if test "$ac_init_help" = "long"; then # Omit some internal or obsolete options to make the list less imposing. # This message is too long to be a string in the A/UX 3.1 sh. cat <<_ACEOF -\`configure' configures upmendex (TeX Live) 0.52 to adapt to many kinds of systems. +\`configure' configures upmendex (TeX Live) 0.54 to adapt to many kinds of systems. Usage: $0 [OPTION]... [VAR=VALUE]... @@ -1415,7 +1416,7 @@ fi if test -n "$ac_init_help"; then case $ac_init_help in - short | recursive ) echo "Configuration of upmendex (TeX Live) 0.52:";; + short | recursive ) echo "Configuration of upmendex (TeX Live) 0.54:";; esac cat <<\_ACEOF @@ -1540,7 +1541,7 @@ fi test -n "$ac_init_help" && exit $ac_status if $ac_init_version; then cat <<\_ACEOF -upmendex (TeX Live) configure 0.52 +upmendex (TeX Live) configure 0.54 generated by GNU Autoconf 2.69 Copyright (C) 2012 Free Software Foundation, Inc. @@ -2473,7 +2474,7 @@ cat >config.log <<_ACEOF This file contains any messages produced by compilers while running configure, to aid debugging if configure makes a mistake. -It was created by upmendex (TeX Live) $as_me 0.52, which was +It was created by upmendex (TeX Live) $as_me 0.54, which was generated by GNU Autoconf 2.69. Invocation command line was $ $0 $@ @@ -8672,7 +8673,7 @@ fi # Define the identity of the package. PACKAGE='upmendex--tex-live-' - VERSION='0.52' + VERSION='0.54' # Some tools Automake needs. @@ -17749,7 +17750,7 @@ ac_config_headers="$ac_config_headers c-auto.h:c-auto.in" -$as_echo "#define TL_VERSION \"TeX Live 2019\"" >>confdefs.h +$as_echo "#define TL_VERSION \"TeX Live 2020\"" >>confdefs.h { $as_echo "$as_me:${as_lineno-$LINENO}: checking if C99 variadic macros are supported" >&5 @@ -18201,7 +18202,7 @@ Usage: $0 [OPTIONS] Report bugs to <bug-libtool@gnu.org>." lt_cl_version="\ -upmendex (TeX Live) config.lt 0.52 +upmendex (TeX Live) config.lt 0.54 configured by $0, generated by GNU Autoconf 2.69. Copyright (C) 2011 Free Software Foundation, Inc. @@ -19513,6 +19514,73 @@ if test "x$kpse_cv_kpse_debug" != xyes; then : KPATHSEA_INCLUDES="$KPATHSEA_INCLUDES -DNO_DEBUG" fi +kpse_search_save_LIBS=$LIBS +{ $as_echo "$as_me:${as_lineno-$LINENO}: checking for library containing pthread_create" >&5 +$as_echo_n "checking for library containing pthread_create... " >&6; } +if ${ac_cv_search_pthread_create+:} false; then : + $as_echo_n "(cached) " >&6 +else + ac_func_search_save_LIBS=$LIBS +cat confdefs.h - <<_ACEOF >conftest.$ac_ext +/* end confdefs.h. */ + +/* Override any GCC internal prototype to avoid an error. + Use char because int might match the return type of a GCC + builtin and then its argument prototype would still apply. */ +#ifdef __cplusplus +extern "C" +#endif +char pthread_create (); +int +main () +{ +return pthread_create (); + ; + return 0; +} +_ACEOF +for ac_lib in '' pthread; do + if test -z "$ac_lib"; then + ac_res="none required" + else + ac_res=-l$ac_lib + LIBS="-l$ac_lib $ac_func_search_save_LIBS" + fi + if ac_fn_c_try_link "$LINENO"; then : + ac_cv_search_pthread_create=$ac_res +fi +rm -f core conftest.err conftest.$ac_objext \ + conftest$ac_exeext + if ${ac_cv_search_pthread_create+:} false; then : + break +fi +done +if ${ac_cv_search_pthread_create+:} false; then : + +else + ac_cv_search_pthread_create=no +fi +rm conftest.$ac_ext +LIBS=$ac_func_search_save_LIBS +fi +{ $as_echo "$as_me:${as_lineno-$LINENO}: result: $ac_cv_search_pthread_create" >&5 +$as_echo "$ac_cv_search_pthread_create" >&6; } +ac_res=$ac_cv_search_pthread_create +if test "$ac_res" != no; then : + test "$ac_res" = "none required" || LIBS="$ac_res $LIBS" + +fi + +LIBS=$kpse_search_save_LIBS +case $ac_cv_search_pthread_create in #( + "none required") : + ;; #( + no) : + as_fn_error $? "Sorry, did not find pthread_create()" "$LINENO" 5 ;; #( + *) : + PTHREAD_LIB=$ac_cv_search_pthread_create ;; +esac + echo 'tldbg:KPSE_ICU_FLAGS called: more-icu-libs=icuio icui18n, icu-config-args=--ldflags-icuio.' >&5 ##tldbg _KPSE_LIB_FLAGS: Setup icu (-licuxxx) flags. echo 'tldbg:_KPSE_LIB_FLAGS called: libdir=icu, libname=icuxxx, options=, tlincl=-DU_STATIC_IMPLEMENTATION -IBLD/libs/icu/include, tllib=BLD/libs/icu/icu-build/lib/libicuio.a BLD/libs/icu/icu-build/lib/libicui18n.a BLD/libs/icu/icu-build/lib/libicuuc.a BLD/libs/icu/icu-build/lib/libicudata.a, tlextra=, rebuildsrcdeps=, rebuildblddeps=${top_builddir}/../../libs/icu/include/unicode/uversion.h.' >&5 @@ -20274,7 +20342,7 @@ cat >>$CONFIG_STATUS <<\_ACEOF || ac_write_fail=1 # report actual input values of CONFIG_FILES etc. instead of their # values after options handling. ac_log=" -This file was extended by upmendex (TeX Live) $as_me 0.52, which was +This file was extended by upmendex (TeX Live) $as_me 0.54, which was generated by GNU Autoconf 2.69. Invocation command line was CONFIG_FILES = $CONFIG_FILES @@ -20340,7 +20408,7 @@ _ACEOF cat >>$CONFIG_STATUS <<_ACEOF || ac_write_fail=1 ac_cs_config="`$as_echo "$ac_configure_args" | sed 's/^ //; s/[\\""\`\$]/\\\\&/g'`" ac_cs_version="\\ -upmendex (TeX Live) config.status 0.52 +upmendex (TeX Live) config.status 0.54 configured by $0, generated by GNU Autoconf 2.69, with options \\"\$ac_cs_config\\" diff --git a/indexing/upmendex/source/configure.ac b/indexing/upmendex/source/configure.ac index 9a6cbfa555..c95d31520b 100644 --- a/indexing/upmendex/source/configure.ac +++ b/indexing/upmendex/source/configure.ac @@ -1,13 +1,14 @@ +dnl $Id$ dnl Process this file with autoconf to produce a configure script. dnl -dnl Copyright 2016 Karl Berry <tex-live@tug.org> -dnl Copyright 2010-2014 Peter Breitenlohner <tex-live@tug.org> +dnl Copyright 2015-2020 Karl Berry <tex-live@tug.org> +dnl Copyright 2010-2015 Peter Breitenlohner <tex-live@tug.org> dnl dnl This file is free software; the copyright holder dnl gives unlimited permission to copy and/or distribute it, dnl with or without modifications, as long as this notice is preserved. dnl -AC_INIT([upmendex (TeX Live)], [0.52]) +AC_INIT([upmendex (TeX Live)], [0.54]) AC_PREREQ([2.63]) AC_CONFIG_SRCDIR([main.c]) AC_CONFIG_AUX_DIR([../../build-aux]) @@ -36,7 +37,20 @@ AS_CASE([$kpse_cv_have___VA_ARGS__], [yes], [AC_DEFINE([HAVE___VA_ARGS__], 1, [Define to 1 if you have C99 variadic macros.])]) +dnl As of ICU 65.1, ICU requires -lpthread on x86_64-linux and, most +dnl likely, on other platforms. Otherwise there is a crash in +dnl std::call_once, as shown in ChangeLog. OpenBSD already has to link +dnl with -lpthread on all platforms (see m4/kpse-icu-flags.m4), but so +dnl far it seems only upmendex requires it elsewhere. The double +dnl inclusion of -lpthread on obsd should be harmless. +dnl +dnl The autoconf-archive ax_pthread macro (used in asymptote) goes to +dnl much greater lengths to find the best thread library and support all +dnl platforms. We can switch to that if necessary, but hopefully this +dnl simple test will be enough. +dnl KPSE_KPATHSEA_FLAGS +KPSE_SEARCH_LIBS([PTHREAD_LIB], [pthread_create], [pthread]) KPSE_ICU_FLAGS([icuio icui18n], [--ldflags-icuio]) AC_CONFIG_FILES([Makefile]) diff --git a/indexing/upmendex/source/convert.c b/indexing/upmendex/source/convert.c index 72e7cb5b78..d02b05bf4d 100644 --- a/indexing/upmendex/source/convert.c +++ b/indexing/upmendex/source/convert.c @@ -217,7 +217,8 @@ int convert(UChar *buff1, UChar *buff2) ||is_jpn_kana(buff3)||is_kor_hngl(buff3)||is_zhuyin(buff3) ||is_comb_diacritical_mark(buff3)) { buff2[j]=buff3[0]; - i++; + if (wclen==2) buff2[j+1]=buff3[1]; + i+=wclen; j+=wclen; } diff --git a/indexing/upmendex/source/fwrite.c b/indexing/upmendex/source/fwrite.c index 6736f56e5a..380e1cbf37 100644 --- a/indexing/upmendex/source/fwrite.c +++ b/indexing/upmendex/source/fwrite.c @@ -649,6 +649,7 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) { int k, len, hi, lo, mi; UChar ch,src[2],dest[8],strX[4],strY[4],strZ[4]; + UChar32 c32; UErrorCode perr; UCollationResult order; static int hanzi_mode=0; @@ -668,8 +669,22 @@ static void index_normalize(UChar *istr, UChar *ini, int *chset) ini[0]=extkanatable[ch-EXKANATOP]; return; } - else if (ch==0x309F) { ini[0]=0x30E8; return; } /* HIRAGANA YORI */ - else if (ch==0x30FF) { ini[0]=0x30B3; return; } /* KATAKANA KOTO */ + else if (ch==0x309F) { ini[0]=0x30E8; return; } /* HIRAGANA YORI -> ヨ */ + else if (ch==0x30FF) { ini[0]=0x30B3; return; } /* KATAKANA KOTO -> コ */ + else if (is_jpn_kana(istr)==2) { + c32=U16_GET_SUPPLEMENTARY(istr[0],istr[1]); + switch (c32) { + case 0x1B150: case 0x1B164: + ini[0]=0x30F0; break; /* ヰ */ + case 0x1B151: case 0x1B165: + ini[0]=0x30F1; break; /* ヱ */ + case 0x1B152: case 0x1B166: + ini[0]=0x30F2; break; /* ヲ */ + case 0x1B167: default: + ini[0]=0x30F3; break; /* ン */ + } + return; + } else if (is_kor_hngl(&ch)) { if ((ch>=0xAC00)&&(ch<=0xD7AF)) { /* Hangul Syllables */ ch=(ch-0xAC00)/(21*28)+CHOSEONG_KIYEOK; /* convert to Hangul Jamo, Initial consonants */ diff --git a/indexing/upmendex/source/main.c b/indexing/upmendex/source/main.c index e65ac3108d..c7d8106b53 100644 --- a/indexing/upmendex/source/main.c +++ b/indexing/upmendex/source/main.c @@ -19,7 +19,7 @@ char *styfile,*idxfile[256],*indfile,*dicfile,*logfile; #endif KpathseaSupportInfo kp_ist,kp_dict; -#define VERSION "version 0.52" +#define VERSION "version 0.54" int main(int argc, char **argv) { @@ -156,7 +156,7 @@ int main(int argc, char **argv) u_getVersion(icuVersion); u_versionToString(icuVersion, icu_version); fprintf(stderr,"upmendex - index processor, %s (%s).\n",VERSION, TL_VERSION); - fprintf(stderr," Copyright 2009 ASCII MEDIA WORKS, 2015-2019 TANAKA Takuji\n"); + fprintf(stderr," Copyright 2009 ASCII MEDIA WORKS, 2015-2020 TANAKA Takuji\n"); fprintf(stderr," using ICU version %s\n",icu_version); fprintf(stderr,"usage:\n"); fprintf(stderr,"%% upmendex [-ilqrcgf] [-s sty] [-d dic] [-o ind] [-t log] [-p no] [--] [idx0 idx1 ...]\n"); diff --git a/indexing/upmendex/source/sort.c b/indexing/upmendex/source/sort.c index 5d11a0727c..15272dc859 100644 --- a/indexing/upmendex/source/sort.c +++ b/indexing/upmendex/source/sort.c @@ -366,9 +366,18 @@ int is_numeric(UChar *c) int is_jpn_kana(UChar *c) { + UChar32 c32; + if ((*c>=0x3040)&&(*c<=0x30FF)) return 1; /* Hiragana, Katakana */ else if ((*c>=0x31F0)&&(*c<=0x31FF)) return 1; /* Katakana Phonetic Extensions */ - else return 0; + + if (is_surrogate_pair(c)) { + c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); + if ((c32>=0x1B130) && (c32<=0x1B16F)) return 2; /* Small Kana Extensions */ + } + return 0; + /* ICU 65 does not seem to support + "Kana Supplement" and "Kana Extended-A" yet. (2020/02/16) */ } int is_kor_hngl(UChar *c) @@ -399,7 +408,8 @@ int is_hanzi(UChar *c) if (is_surrogate_pair(c)) { c32=U16_GET_SUPPLEMENTARY(*c,*(c+1)); if ((c32>=0x20000) && /* CJK Unified Ideographs Extension B,C,D,E,F */ - (c32<=0x2FA1F)) return 2; /* CJK Compatibility Ideographs Supplement */ + /* CJK Compatibility Ideographs Supplement */ + (c32<=0x3134F)) return 2; /* CJK Unified Ideographs Extension G */ } return 0; } diff --git a/indexing/upmendex/source/tests/upmendex.test b/indexing/upmendex/source/tests/upmendex.test index 5b1f6b7db8..da75821d10 100755 --- a/indexing/upmendex/source/tests/upmendex.test +++ b/indexing/upmendex/source/tests/upmendex.test @@ -1,5 +1,5 @@ #! /bin/sh -vx -# $Id: upmendex.test 45809 2017-11-15 00:36:56Z karl $ +# $Id$ # Copyright 2017 Karl Berry <tex-live@tug.org> # Copyright 2013 Peter Breitenlohner <tex-live@tug.org> # You may freely use, modify and/or distribute this file. |