cjkutils: bug fixes; add simple test cases

git-svn-id: svn://tug.org/texlive/trunk@24133 c570f23f-e606-0410-a88d-b1316a301751

33 files changed, 549 insertions, 21 deletions

diff --git a/Build/source/texk/cjkutils/ChangeLog b/Build/source/texk/cjkutils/ChangeLog
index 5510635e5e3..3c3935b2df9 100644
--- a/Build/source/texk/cjkutils/ChangeLog
+++ b/Build/source/texk/cjkutils/ChangeLog
@@ -1,3 +1,11 @@
+2011-09-16  Peter Breitenlohner  <peb@mppmu.mpg.de>
+
+	Add simple CJK conversion tests.
+	* {bg5,cef,cef5,cefs,ext,sjis}conv.test: Test shell scripts.
+	* tests/{bg5,cef,cef5,cefs,ext,sjis}conv.{tex,cjk}: Test input
+	and expected output.
+	* Makefile.am: Add the tests and their data.
+
 2011-05-30  Peter Breitenlohner  <peb@mppmu.mpg.de>
 
 	* Makefile.am: Use ../am/bin_links.am for $(bindir) links.
diff --git a/Build/source/texk/cjkutils/Makefile.am b/Build/source/texk/cjkutils/Makefile.am
index efdc603188e..d247fe4648f 100644
--- a/Build/source/texk/cjkutils/Makefile.am
+++ b/Build/source/texk/cjkutils/Makefile.am
@@ -181,3 +181,27 @@ hbf2gf.c: $(CWEB_SRC)
 	$(CTANGLE) $<
 endif do_cweb
 
+## Tests
+##
+TESTS = bg5conv.test cefconv.test cef5conv.test cefsconv.test extconv.test sjisconv.test
+EXTRA_DIST += $(TESTS)
+
+## bg5conv.test
+EXTRA_DIST += tests/bg5conv.tex tests/bg5conv.cjk
+DISTCLEANFILES += bg5conv.cjk
+## cefconv.test
+EXTRA_DIST += tests/cefconv.tex tests/cefconv.cjk
+DISTCLEANFILES += cefconv.cjk
+## cef5conv.test
+EXTRA_DIST += tests/cef5conv.tex tests/cef5conv.cjk
+DISTCLEANFILES += cef5conv.cjk
+## cefsconv.test
+EXTRA_DIST += tests/cefsconv.tex tests/cefsconv.cjk
+DISTCLEANFILES += cefsconv.cjk
+## extconv.test
+EXTRA_DIST += tests/extconv.tex tests/extconv.cjk
+DISTCLEANFILES += extconv.cjk
+## sjisconv.test
+EXTRA_DIST += tests/sjisconv.tex tests/sjisconv.cjk
+DISTCLEANFILES += sjisconv.cjk
+
diff --git a/Build/source/texk/cjkutils/Makefile.in b/Build/source/texk/cjkutils/Makefile.in
index 03870948156..d21872c99a0 100644
--- a/Build/source/texk/cjkutils/Makefile.in
+++ b/Build/source/texk/cjkutils/Makefile.in
@@ -154,6 +154,8 @@ MANS = $(man1_MANS)
 DATA = $(hbf2gf_cfg_DATA)
 ETAGS = etags
 CTAGS = ctags
+am__tty_colors = \
+red=; grn=; lgn=; blu=; std=
 DISTFILES = $(DIST_COMMON) $(DIST_SOURCES) $(TEXINFOS) $(EXTRA_DIST)
 distdir = $(PACKAGE)-$(VERSION)
 top_distdir = $(distdir)
@@ -292,8 +294,13 @@ top_builddir = @top_builddir@
 top_srcdir = @top_srcdir@
 ACLOCAL_AMFLAGS = -I ../../m4
 EXTRA_DIST = $(CJKUTILS_TREE) $(CJKUTILS_TREE)-PATCHES $(CWEB_TEX) \
-	$(CWEB_DVI)
-DISTCLEANFILES = config.force
+	$(CWEB_DVI) $(TESTS) tests/bg5conv.tex tests/bg5conv.cjk \
+	tests/cefconv.tex tests/cefconv.cjk tests/cef5conv.tex \
+	tests/cef5conv.cjk tests/cefsconv.tex tests/cefsconv.cjk \
+	tests/extconv.tex tests/extconv.cjk tests/sjisconv.tex \
+	tests/sjisconv.cjk
+DISTCLEANFILES = config.force bg5conv.cjk cefconv.cjk cef5conv.cjk \
+	cefsconv.cjk extconv.cjk sjisconv.cjk
 INCLUDES = $(KPATHSEA_INCLUDES)
 AM_CPPFLAGS = $(XX_CPPFLAGS)
 AM_CFLAGS = $(XX_CFLAGS)
@@ -363,6 +370,7 @@ CWEB_SRC = $(CJKUTILS_TREE)/hbf2gf/hbf2gf.w
 CWEB_TEX = hbf2gf.tex hbf2gf.idx hbf2gf.scn
 CWEB_DVI = hbf2gf.dvi
 CLEANFILES = hbf2gf.log hbf2gf.toc
+TESTS = bg5conv.test cefconv.test cef5conv.test cefsconv.test extconv.test sjisconv.test
 all: c-auto.h
 	$(MAKE) $(AM_MAKEFLAGS) all-am
 
@@ -733,6 +741,98 @@ GTAGS:
 distclean-tags:
 	-rm -f TAGS ID GTAGS GRTAGS GSYMS GPATH tags
 
+check-TESTS: $(TESTS)
+	@failed=0; all=0; xfail=0; xpass=0; skip=0; \
+	srcdir=$(srcdir); export srcdir; \
+	list=' $(TESTS) '; \
+	$(am__tty_colors); \
+	if test -n "$$list"; then \
+	  for tst in $$list; do \
+	    if test -f ./$$tst; then dir=./; \
+	    elif test -f $$tst; then dir=; \
+	    else dir="$(srcdir)/"; fi; \
+	    if $(TESTS_ENVIRONMENT) $${dir}$$tst; then \
+	      all=`expr $$all + 1`; \
+	      case " $(XFAIL_TESTS) " in \
+	      *[\ \	]$$tst[\ \	]*) \
+		xpass=`expr $$xpass + 1`; \
+		failed=`expr $$failed + 1`; \
+		col=$$red; res=XPASS; \
+	      ;; \
+	      *) \
+		col=$$grn; res=PASS; \
+	      ;; \
+	      esac; \
+	    elif test $$? -ne 77; then \
+	      all=`expr $$all + 1`; \
+	      case " $(XFAIL_TESTS) " in \
+	      *[\ \	]$$tst[\ \	]*) \
+		xfail=`expr $$xfail + 1`; \
+		col=$$lgn; res=XFAIL; \
+	      ;; \
+	      *) \
+		failed=`expr $$failed + 1`; \
+		col=$$red; res=FAIL; \
+	      ;; \
+	      esac; \
+	    else \
+	      skip=`expr $$skip + 1`; \
+	      col=$$blu; res=SKIP; \
+	    fi; \
+	    echo "$${col}$$res$${std}: $$tst"; \
+	  done; \
+	  if test "$$all" -eq 1; then \
+	    tests="test"; \
+	    All=""; \
+	  else \
+	    tests="tests"; \
+	    All="All "; \
+	  fi; \
+	  if test "$$failed" -eq 0; then \
+	    if test "$$xfail" -eq 0; then \
+	      banner="$$All$$all $$tests passed"; \
+	    else \
+	      if test "$$xfail" -eq 1; then failures=failure; else failures=failures; fi; \
+	      banner="$$All$$all $$tests behaved as expected ($$xfail expected $$failures)"; \
+	    fi; \
+	  else \
+	    if test "$$xpass" -eq 0; then \
+	      banner="$$failed of $$all $$tests failed"; \
+	    else \
+	      if test "$$xpass" -eq 1; then passes=pass; else passes=passes; fi; \
+	      banner="$$failed of $$all $$tests did not behave as expected ($$xpass unexpected $$passes)"; \
+	    fi; \
+	  fi; \
+	  dashes="$$banner"; \
+	  skipped=""; \
+	  if test "$$skip" -ne 0; then \
+	    if test "$$skip" -eq 1; then \
+	      skipped="($$skip test was not run)"; \
+	    else \
+	      skipped="($$skip tests were not run)"; \
+	    fi; \
+	    test `echo "$$skipped" | wc -c` -le `echo "$$banner" | wc -c` || \
+	      dashes="$$skipped"; \
+	  fi; \
+	  report=""; \
+	  if test "$$failed" -ne 0 && test -n "$(PACKAGE_BUGREPORT)"; then \
+	    report="Please report to $(PACKAGE_BUGREPORT)"; \
+	    test `echo "$$report" | wc -c` -le `echo "$$banner" | wc -c` || \
+	      dashes="$$report"; \
+	  fi; \
+	  dashes=`echo "$$dashes" | sed s/./=/g`; \
+	  if test "$$failed" -eq 0; then \
+	    echo "$$grn$$dashes"; \
+	  else \
+	    echo "$$red$$dashes"; \
+	  fi; \
+	  echo "$$banner"; \
+	  test -z "$$skipped" || echo "$$skipped"; \
+	  test -z "$$report" || echo "$$report"; \
+	  echo "$$dashes$$std"; \
+	  test "$$failed" -eq 0; \
+	else :; fi
+
 distdir: $(DISTFILES)
 	@list='$(MANS)'; if test -n "$$list"; then \
 	  list=`for p in $$list; do \
@@ -897,6 +997,7 @@ distcleancheck: distclean
 	       $(distcleancheck_listfiles) ; \
 	       exit 1; } >&2
 check-am: all-am
+	$(MAKE) $(AM_MAKEFLAGS) check-TESTS
 check: check-am
 all-am: Makefile $(PROGRAMS) $(SCRIPTS) $(MANS) $(DATA) c-auto.h \
 		all-local
@@ -1009,12 +1110,13 @@ uninstall-am: uninstall-binPROGRAMS uninstall-binSCRIPTS \
 	$(MAKE) $(AM_MAKEFLAGS) uninstall-hook
 uninstall-man: uninstall-man1
 
-.MAKE: all install-am install-exec-am install-strip uninstall-am
+.MAKE: all check-am install-am install-exec-am install-strip \
+	uninstall-am
 
-.PHONY: CTAGS GTAGS all all-am all-local am--refresh check check-am \
-	clean clean-binPROGRAMS clean-generic clean-libtool ctags dist \
-	dist-all dist-bzip2 dist-gzip dist-hook dist-lzma dist-shar \
-	dist-tarZ dist-xz dist-zip distcheck distclean \
+.PHONY: CTAGS GTAGS all all-am all-local am--refresh check check-TESTS \
+	check-am clean clean-binPROGRAMS clean-generic clean-libtool \
+	ctags dist dist-all dist-bzip2 dist-gzip dist-hook dist-lzma \
+	dist-shar dist-tarZ dist-xz dist-zip distcheck distclean \
 	distclean-compile distclean-generic distclean-hdr \
 	distclean-libtool distclean-tags distcleancheck distdir \
 	distuninstallcheck dvi dvi-am html html-am info info-am \
diff --git a/Build/source/texk/cjkutils/bg5conv.test b/Build/source/texk/cjkutils/bg5conv.test
new file mode 100755
index 00000000000..c24e680a265
--- /dev/null
+++ b/Build/source/texk/cjkutils/bg5conv.test
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+# Copyright (C) 2011 Peter Breitenlohner <tex-live@tug.org>
+# You may freely use, modify and/or distribute this file.
+
+./bg5conv <$srcdir/tests/bg5conv.tex >bg5conv.cjk \
+	&& diff bg5conv.cjk $srcdir/tests/bg5conv.cjk \
+	|| exit 1
+
diff --git a/Build/source/texk/cjkutils/cef5conv.c b/Build/source/texk/cjkutils/cef5conv.c
index 1d648ef49a1..a521ad3fc20 100644
--- a/Build/source/texk/cjkutils/cef5conv.c
+++ b/Build/source/texk/cjkutils/cef5conv.c
@@ -70,6 +70,7 @@ if(*inp!='-'||feof(stdin))
 goto no_macro;
 
 *(outp++)= '\177';
+*(outp++)= '\177';
 *(outp++)= '\"';
 *(outp++)= '0';
 
@@ -86,7 +87,6 @@ else
 goto no_macro;
 
 *(outp++)= '\177';
-*(outp++)= '\177';
 *(outp++)= '\"';
 *(outp++)= '0';
 
diff --git a/Build/source/texk/cjkutils/cef5conv.test b/Build/source/texk/cjkutils/cef5conv.test
new file mode 100755
index 00000000000..ba35a21c08e
--- /dev/null
+++ b/Build/source/texk/cjkutils/cef5conv.test
@@ -0,0 +1,19 @@
+#! /bin/sh
+
+# Copyright (C) 2011 Peter Breitenlohner <tex-live@tug.org>
+# You may freely use, modify and/or distribute this file.
+
+./cef5conv <$srcdir/tests/cef5conv.tex >cef5conv.cjk \
+	&& diff cef5conv.cjk $srcdir/tests/cef5conv.cjk \
+	|| exit 1
+
+./bg5conv <$srcdir/tests/cef5conv.tex | ./cefconv \
+	| sed 's/cefconv.*bg5conv/cef5conv/' >cef5conv.cjk \
+	&& diff cef5conv.cjk $srcdir/tests/cef5conv.cjk \
+	|| exit 1
+
+./cefconv <$srcdir/tests/cef5conv.tex | ./bg5conv \
+	| sed 's/CJKpreproc.*CNSpreproc/CNSpreproc/;s/cefconv/cef5conv/' >cef5conv.cjk \
+	&& diff cef5conv.cjk $srcdir/tests/cef5conv.cjk \
+	|| exit 1
+
diff --git a/Build/source/texk/cjkutils/cefconv.test b/Build/source/texk/cjkutils/cefconv.test
new file mode 100755
index 00000000000..fe6aded53d6
--- /dev/null
+++ b/Build/source/texk/cjkutils/cefconv.test
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+# Copyright (C) 2011 Peter Breitenlohner <tex-live@tug.org>
+# You may freely use, modify and/or distribute this file.
+
+./cefconv <$srcdir/tests/cefconv.tex >cefconv.cjk \
+	&& diff cefconv.cjk $srcdir/tests/cefconv.cjk \
+	|| exit 1
+
diff --git a/Build/source/texk/cjkutils/cefsconv.c b/Build/source/texk/cjkutils/cefsconv.c
index b359e12533e..a276f9aefe6 100644
--- a/Build/source/texk/cjkutils/cefsconv.c
+++ b/Build/source/texk/cjkutils/cefsconv.c
@@ -20,7 +20,7 @@ fprintf(stdout,"\\def\\CNSpreproc{%s}",banner);
 ch= fgetc(stdin);
 
 while(!feof(stdin))
-{if((ch>=0x81&&ch<=0x9F)||(ch>=0xE0&&ch<=0xEF))
+{if((ch>=0x81&&ch<=0x9F)||(ch>=0xE0&&ch<=0xFC))
 {fprintf(stdout,"\177%c\177",ch);
 
 ch= fgetc(stdin);
@@ -70,6 +70,7 @@ if(*inp!='-'||feof(stdin))
 goto no_macro;
 
 *(outp++)= '\177';
+*(outp++)= '\177';
 *(outp++)= '\"';
 *(outp++)= '0';
 
@@ -86,7 +87,6 @@ else
 goto no_macro;
 
 *(outp++)= '\177';
-*(outp++)= '\177';
 *(outp++)= '\"';
 *(outp++)= '0';
 
diff --git a/Build/source/texk/cjkutils/cefsconv.test b/Build/source/texk/cjkutils/cefsconv.test
new file mode 100755
index 00000000000..c005f3d2b00
--- /dev/null
+++ b/Build/source/texk/cjkutils/cefsconv.test
@@ -0,0 +1,19 @@
+#! /bin/sh
+
+# Copyright (C) 2011 Peter Breitenlohner <tex-live@tug.org>
+# You may freely use, modify and/or distribute this file.
+
+./cefsconv <$srcdir/tests/cefsconv.tex >cefsconv.cjk \
+	&& diff cefsconv.cjk $srcdir/tests/cefsconv.cjk \
+	|| exit 1
+
+./sjisconv <$srcdir/tests/cefsconv.tex | ./cefconv \
+	| sed 's/cefconv.*sjisconv/cefsconv/' >cefsconv.cjk \
+	&& diff cefsconv.cjk $srcdir/tests/cefsconv.cjk \
+	|| exit 1
+
+./cefconv <$srcdir/tests/cefsconv.tex | ./sjisconv \
+	| sed 's/CJKpreproc.*CNSpreproc/CNSpreproc/;s/cefconv/cefsconv/' >cefsconv.cjk \
+	&& diff cefsconv.cjk $srcdir/tests/cefsconv.cjk \
+	|| exit 1
+
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/ChangeLog b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/ChangeLog
index 257f4f6c06f..317380cf7da 100644
--- a/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/ChangeLog
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/ChangeLog
@@ -1,3 +1,15 @@
+2011-09-17  Peter Breitenlohner  <peb@mppmu.mpg.de>
+
+	* patch-07-CEFconv (new): Make sure that, as advertised,
+	cef5conv acts as bg5conv + cefconv and cefsconv as sjisconv +
+	cefconv; make TeX description of cefconv agree with the code.
+	* patch-08-SJISconv (new): Correct upper limit for first byte of
+	Kanji SJIS (CP 292) codes.
+
+2011-09-12  Peter Breitenlohner  <peb@mppmu.mpg.de>
+
+	* patch-06-cjklatex (new): Drop #include <kpathsea/win32lib.h>.
+
 2010-12-19  Peter Breitenlohner  <peb@mppmu.mpg.de>
 
 	Adapt to cjk-4.8.2/utils.
@@ -5,7 +17,7 @@
 	* patch-02-const (new): Declare numerous strings as const.
 	* patch-03-kpathsea (new): Require recent kpathsea version.
 
-	* patch-04-cjklatexn (new): Merge upstream versions with changes
+	* patch-04-cjklatex (new): Merge upstream versions with changes
 	  from Akira (mail 18 Dec 2010 09:12:01).
 	  Return type of main is int. ANSI C function declarations.
 	  Declare functions as static. Declare various strings as const.
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-06-cjklatex b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-06-cjklatex
new file mode 100644
index 00000000000..74e4930d4f7
--- /dev/null
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-06-cjklatex
@@ -0,0 +1,17 @@
+	No need to directly #include <kpathsea/win32lib.h> (already done
+	from <kpathsea/config.h>).
+
+diff -ur cjkutils-4.8.2.orig/cjklatex/cjklatex.c cjkutils-4.8.2/cjklatex/cjklatex.c
+--- cjkutils-4.8.2.orig/cjklatex/cjklatex.c	2010-12-19 16:25:43.000000000 +0100
++++ cjkutils-4.8.2/cjklatex/cjklatex.c	2011-09-12 13:44:29.000000000 +0200
+@@ -36,10 +36,6 @@
+ #include <kpathsea/lib.h>
+ #include <kpathsea/getopt.h>
+ 
+-#if defined(WIN32) && !defined(__MINGW32__)
+-#include <kpathsea/win32lib.h>
+-#endif
+-
+ static const char *cjklatex_version_string = "1.0";
+ 
+ static const char *usage_str[] = {
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-07-CEFconv b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-07-CEFconv
new file mode 100644
index 00000000000..12f9d60eb5a
--- /dev/null
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-07-CEFconv
@@ -0,0 +1,55 @@
+	Make sure that, as advertised, cef5conv acts as bg5conv + cefconv
+	and cefsconv as sjisconv + cefconv.
+
+	Make the TeX description of cefconv agree with the actual code.
+
+diff -ur cjkutils-4.8.2.orig/CEFconv/cef5conv.w cjkutils-4.8.2/CEFconv/cef5conv.w
+--- cjkutils-4.8.2.orig/CEFconv/cef5conv.w	2010-12-17 20:41:33.000000000 +0100
++++ cjkutils-4.8.2/CEFconv/cef5conv.w	2011-09-16 21:49:49.000000000 +0200
+@@ -172,6 +172,7 @@
+                 goto no_macro;@#
+ 
+             *(outp++) = '\177';
++            *(outp++) = '\177';
+             *(outp++) = '\"';
+             *(outp++) = '0';@#
+ 
+@@ -188,7 +189,6 @@
+                 goto no_macro;@#
+ 
+             *(outp++) = '\177';
+-            *(outp++) = '\177';
+             *(outp++) = '\"';
+             *(outp++) = '0';@#
+ 
+diff -ur cjkutils-4.8.2.orig/CEFconv/cefconv.w cjkutils-4.8.2/CEFconv/cefconv.w
+--- cjkutils-4.8.2.orig/CEFconv/cefconv.w	2010-12-17 20:41:34.000000000 +0100
++++ cjkutils-4.8.2/CEFconv/cefconv.w	2011-09-17 14:19:53.000000000 +0200
+@@ -80,7 +80,7 @@
+ IRIZ, a private encoding CY, and U for Unicode encoding; \.{yyzz} is a
+ hexadecimal representation of the code point in this plane) with
+ \medskip
+-\quad\.{\char94 \char94 7f72\char94 \char94 ffXX\char94 \char94 7f%
++\quad\.{\char94 \char94 7f72\char94 \char94 7fXX\char94 \char94 7f%
+ \char94 \char94 7f"0yy\char94 \char94 7f"0zz\char94 \char94 7f}\quad.
+ \medskip
+ \.{XX} is the corresponding CJK encoding of \.{xx}; the number `72'
+diff -ur cjkutils-4.8.2.orig/CEFconv/cefsconv.w cjkutils-4.8.2/CEFconv/cefsconv.w
+--- cjkutils-4.8.2.orig/CEFconv/cefsconv.w	2010-12-17 20:41:35.000000000 +0100
++++ cjkutils-4.8.2/CEFconv/cefsconv.w	2011-09-16 21:49:46.000000000 +0200
+@@ -172,6 +172,7 @@
+                 goto no_macro;@#
+ 
+             *(outp++) = '\177';
++            *(outp++) = '\177';
+             *(outp++) = '\"';
+             *(outp++) = '0';@#
+ 
+@@ -188,7 +189,6 @@
+                 goto no_macro;@#
+ 
+             *(outp++) = '\177';
+-            *(outp++) = '\177';
+             *(outp++) = '\"';
+             *(outp++) = '0';@#
+ 
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-08-SJISconv b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-08-SJISconv
new file mode 100644
index 00000000000..3d30e3792e3
--- /dev/null
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2-PATCHES/patch-08-SJISconv
@@ -0,0 +1,26 @@
+	Correct upper limit for first byte of Kanji SJIS (CP 292) codes.
+
+diff -ur cjkutils-4.8.2.orig/CEFconv/cefsconv.w cjkutils-4.8.2/CEFconv/cefsconv.w
+--- cjkutils-4.8.2.orig/CEFconv/cefsconv.w	2011-09-16 21:49:46.000000000 +0200
++++ cjkutils-4.8.2/CEFconv/cefsconv.w	2011-09-16 21:59:48.000000000 +0200
+@@ -122,7 +122,7 @@
+     ch = fgetc(stdin);@#
+ 
+     while(!feof(stdin))
+-       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xEF))
++       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xFC))
+            {fprintf(stdout, "\177%c\177", ch);@#
+ 
+             ch = fgetc(stdin);
+diff -ur cjkutils-4.8.2.orig/SJISconv/sjisconv.w cjkutils-4.8.2/SJISconv/sjisconv.w
+--- cjkutils-4.8.2.orig/SJISconv/sjisconv.w	2010-12-17 20:41:37.000000000 +0100
++++ cjkutils-4.8.2/SJISconv/sjisconv.w	2011-09-16 21:59:15.000000000 +0200
+@@ -103,7 +103,7 @@
+     ch = fgetc(stdin);@#
+ 
+     while(!feof(stdin))
+-       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xEF))
++       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xFC))
+            {fprintf(stdout, "\177%c\177", ch);@#
+ 
+             ch = fgetc(stdin);
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cef5conv.w b/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cef5conv.w
index 6c7618eeb74..5b81ea33c63 100644
--- a/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cef5conv.w
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cef5conv.w
@@ -172,6 +172,7 @@ int main(int argc, char *argv[])
                 goto no_macro;@#
 
             *(outp++) = '\177';
+            *(outp++) = '\177';
             *(outp++) = '\"';
             *(outp++) = '0';@#
 
@@ -188,7 +189,6 @@ int main(int argc, char *argv[])
                 goto no_macro;@#
 
             *(outp++) = '\177';
-            *(outp++) = '\177';
             *(outp++) = '\"';
             *(outp++) = '0';@#
 
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefconv.w b/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefconv.w
index 00f6a2b5917..4027dc5f368 100644
--- a/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefconv.w
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefconv.w
@@ -80,7 +80,7 @@ the CNS planes 1--7, C0 for Big~5 encoding, an encoding CX reserved for
 IRIZ, a private encoding CY, and U for Unicode encoding; \.{yyzz} is a
 hexadecimal representation of the code point in this plane) with
 \medskip
-\quad\.{\char94 \char94 7f72\char94 \char94 ffXX\char94 \char94 7f%
+\quad\.{\char94 \char94 7f72\char94 \char94 7fXX\char94 \char94 7f%
 \char94 \char94 7f"0yy\char94 \char94 7f"0zz\char94 \char94 7f}\quad.
 \medskip
 \.{XX} is the corresponding CJK encoding of \.{xx}; the number `72'
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefsconv.w b/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefsconv.w
index 06cd823f2de..802aff4cd91 100644
--- a/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefsconv.w
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2/CEFconv/cefsconv.w
@@ -122,7 +122,7 @@ int main(int argc, char *argv[])
     ch = fgetc(stdin);@#
 
     while(!feof(stdin))
-       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xEF))
+       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xFC))
            {fprintf(stdout, "\177%c\177", ch);@#
 
             ch = fgetc(stdin);
@@ -172,6 +172,7 @@ int main(int argc, char *argv[])
                 goto no_macro;@#
 
             *(outp++) = '\177';
+            *(outp++) = '\177';
             *(outp++) = '\"';
             *(outp++) = '0';@#
 
@@ -188,7 +189,6 @@ int main(int argc, char *argv[])
                 goto no_macro;@#
 
             *(outp++) = '\177';
-            *(outp++) = '\177';
             *(outp++) = '\"';
             *(outp++) = '0';@#
 
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2/SJISconv/sjisconv.w b/Build/source/texk/cjkutils/cjkutils-4.8.2/SJISconv/sjisconv.w
index 26469af5c57..fac5beb4825 100644
--- a/Build/source/texk/cjkutils/cjkutils-4.8.2/SJISconv/sjisconv.w
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2/SJISconv/sjisconv.w
@@ -103,7 +103,7 @@ int main(int argc, char *argv[])
     ch = fgetc(stdin);@#
 
     while(!feof(stdin))
-       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xEF))
+       {if((ch >= 0x81 && ch <= 0x9F) || (ch >= 0xE0 && ch <= 0xFC))
            {fprintf(stdout, "\177%c\177", ch);@#
 
             ch = fgetc(stdin);
diff --git a/Build/source/texk/cjkutils/cjkutils-4.8.2/cjklatex/cjklatex.c b/Build/source/texk/cjkutils/cjkutils-4.8.2/cjklatex/cjklatex.c
index 6518cc63491..342258968f2 100644
--- a/Build/source/texk/cjkutils/cjkutils-4.8.2/cjklatex/cjklatex.c
+++ b/Build/source/texk/cjkutils/cjkutils-4.8.2/cjklatex/cjklatex.c
@@ -36,10 +36,6 @@
 #include <kpathsea/lib.h>
 #include <kpathsea/getopt.h>
 
-#if defined(WIN32) && !defined(__MINGW32__)
-#include <kpathsea/win32lib.h>
-#endif
-
 static const char *cjklatex_version_string = "1.0";
 
 static const char *usage_str[] = {
diff --git a/Build/source/texk/cjkutils/extconv.test b/Build/source/texk/cjkutils/extconv.test
new file mode 100755
index 00000000000..08368b02f81
--- /dev/null
+++ b/Build/source/texk/cjkutils/extconv.test
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+# Copyright (C) 2011 Peter Breitenlohner <tex-live@tug.org>
+# You may freely use, modify and/or distribute this file.
+
+./extconv <$srcdir/tests/extconv.tex >extconv.cjk \
+	&& diff extconv.cjk $srcdir/tests/extconv.cjk \
+	|| exit 1
+
diff --git a/Build/source/texk/cjkutils/sjisconv.c b/Build/source/texk/cjkutils/sjisconv.c
index 3fcfbe1ae33..a844dd101eb 100644
--- a/Build/source/texk/cjkutils/sjisconv.c
+++ b/Build/source/texk/cjkutils/sjisconv.c
@@ -16,7 +16,7 @@ fprintf(stdout,"\\def\\CJKpreproc{%s}",banner);
 ch= fgetc(stdin);
 
 while(!feof(stdin))
-{if((ch>=0x81&&ch<=0x9F)||(ch>=0xE0&&ch<=0xEF))
+{if((ch>=0x81&&ch<=0x9F)||(ch>=0xE0&&ch<=0xFC))
 {fprintf(stdout,"\177%c\177",ch);
 
 ch= fgetc(stdin);
diff --git a/Build/source/texk/cjkutils/sjisconv.test b/Build/source/texk/cjkutils/sjisconv.test
new file mode 100755
index 00000000000..2e6e867d5a1
--- /dev/null
+++ b/Build/source/texk/cjkutils/sjisconv.test
@@ -0,0 +1,9 @@
+#! /bin/sh
+
+# Copyright (C) 2011 Peter Breitenlohner <tex-live@tug.org>
+# You may freely use, modify and/or distribute this file.
+
+./sjisconv <$srcdir/tests/sjisconv.tex >sjisconv.cjk \
+	&& diff sjisconv.cjk $srcdir/tests/sjisconv.cjk \
+	|| exit 1
+
diff --git a/Build/source/texk/cjkutils/tests/bg5conv.cjk b/Build/source/texk/cjkutils/tests/bg5conv.cjk
new file mode 100644
index 00000000000..90f9acef8e3
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/bg5conv.cjk
@@ -0,0 +1,7 @@
+\def\CJKpreproc{bg5conv (CJK ver. 4.8.2)}
+0x81 0x41:�A
+0xa0 0x5a:�Z
+0xa1 0x7b:�123
+0xfe 0x5c:�92
+0xc3 0x7d:�125
+==========
diff --git a/Build/source/texk/cjkutils/tests/bg5conv.tex b/Build/source/texk/cjkutils/tests/bg5conv.tex
new file mode 100644
index 00000000000..ab5bc0d66bd
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/bg5conv.tex
@@ -0,0 +1,7 @@
+
+0x81 0x41:�A
+0xa0 0x5a:�Z
+0xa1 0x7b:�{
+0xfe 0x5c:�\
+0xc3 0x7d:�}
+==========
diff --git a/Build/source/texk/cjkutils/tests/cef5conv.cjk b/Build/source/texk/cjkutils/tests/cef5conv.cjk
new file mode 100644
index 00000000000..a9d6a780711
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/cef5conv.cjk
@@ -0,0 +1,29 @@
+\def\CNSpreproc{cef5conv (CJK ver. 4.8.2)}
+0x81 0x41:�A
+0xa0 0x5a:�Z
+0xa1 0x7b:�123
+0xfe 0x5c:�92
+0xc3 0x7d:�125
+==========
+AC0-0f0f; 72Bg5"00F"00F
+AC1-0F0F; 72CNS1"00F"00F
+AC7-f0f0; 72CNS7"0F0"0F0
+ACX-F0F0; 72CEFX"0F0"0F0
+ACY-aBcD; 72CEFY"0AB"0CD
+AU-0123;  72UTF8"001"023
+=========
+AC0-0f0f;AU-0123;  72Bg5"00F"00F72UTF8"001"023
+AU-0123;AC0-0f0f;  72UTF8"001"02372Bg5"00F"00F
+==================
+AAC0-0f0f;         &72Bg5"00F"00F
+ACAC0-0f0f;        &C72Bg5"00F"00F
+AC1AC0-0f0f;       &C172Bg5"00F"00F
+AUAC0-0f0f;        &U72Bg5"00F"00F
+AC2-AC0-0f0f;      &C2-72Bg5"00F"00F
+AC3-1AC0-0f0f;     &C3-172Bg5"00F"00F
+AC4-12AC0-0f0f;    &C4-1272Bg5"00F"00F
+AC5-123AC0-0f0f;   &C5-12372Bg5"00F"00F
+AC6-1234AC0-0f0f;  &C6-123472Bg5"00F"00F
+AC7-12345AC0-0f0f; &C7-1234572Bg5"00F"00F
+A 0xe1 0x64:       &�100
+==================
diff --git a/Build/source/texk/cjkutils/tests/cef5conv.tex b/Build/source/texk/cjkutils/tests/cef5conv.tex
new file mode 100644
index 00000000000..0bfbeaaf3c1
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/cef5conv.tex
@@ -0,0 +1,29 @@
+
+0x81 0x41:�A
+0xa0 0x5a:�Z
+0xa1 0x7b:�{
+0xfe 0x5c:�\
+0xc3 0x7d:�}
+==========
+AC0-0f0f; &C0-0f0f;
+AC1-0F0F; &C1-0F0F;
+AC7-f0f0; &C7-f0f0;
+ACX-F0F0; &CX-F0F0;
+ACY-aBcD; &CY-aBcD;
+AU-0123;  &U-0123;
+=========
+AC0-0f0f;AU-0123;  &C0-0f0f;&U-0123;
+AU-0123;AC0-0f0f;  &U-0123;&C0-0f0f;
+==================
+AAC0-0f0f;         &&C0-0f0f;
+ACAC0-0f0f;        &C&C0-0f0f;
+AC1AC0-0f0f;       &C1&C0-0f0f;
+AUAC0-0f0f;        &U&C0-0f0f;
+AC2-AC0-0f0f;      &C2-&C0-0f0f;
+AC3-1AC0-0f0f;     &C3-1&C0-0f0f;
+AC4-12AC0-0f0f;    &C4-12&C0-0f0f;
+AC5-123AC0-0f0f;   &C5-123&C0-0f0f;
+AC6-1234AC0-0f0f;  &C6-1234&C0-0f0f;
+AC7-12345AC0-0f0f; &C7-12345&C0-0f0f;
+A 0xe1 0x64:       &�d
+==================
diff --git a/Build/source/texk/cjkutils/tests/cefconv.cjk b/Build/source/texk/cjkutils/tests/cefconv.cjk
new file mode 100644
index 00000000000..b7ff0f69ed6
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/cefconv.cjk
@@ -0,0 +1,22 @@
+\def\CNSpreproc{cefconv (CJK ver. 4.8.2)}
+AC0-0f0f; 72Bg5"00F"00F
+AC1-0F0F; 72CNS1"00F"00F
+AC7-f0f0; 72CNS7"0F0"0F0
+ACX-F0F0; 72CEFX"0F0"0F0
+ACY-aBcD; 72CEFY"0AB"0CD
+AU-0123;  72UTF8"001"023
+=========
+AC0-0f0f;AU-0123;  72Bg5"00F"00F72UTF8"001"023
+AU-0123;AC0-0f0f;  72UTF8"001"02372Bg5"00F"00F
+==================
+AAC0-0f0f;         &72Bg5"00F"00F
+ACAC0-0f0f;        &C72Bg5"00F"00F
+AC1AC0-0f0f;       &C172Bg5"00F"00F
+AUAC0-0f0f;        &U72Bg5"00F"00F
+AC2-AC0-0f0f;      &C2-72Bg5"00F"00F
+AC3-1AC0-0f0f;     &C3-172Bg5"00F"00F
+AC4-12AC0-0f0f;    &C4-1272Bg5"00F"00F
+AC5-123AC0-0f0f;   &C5-12372Bg5"00F"00F
+AC6-1234AC0-0f0f;  &C6-123472Bg5"00F"00F
+AC7-12345AC0-0f0f; &C7-1234572Bg5"00F"00F
+==================
diff --git a/Build/source/texk/cjkutils/tests/cefconv.tex b/Build/source/texk/cjkutils/tests/cefconv.tex
new file mode 100644
index 00000000000..5fcf9db255e
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/cefconv.tex
@@ -0,0 +1,22 @@
+
+AC0-0f0f; &C0-0f0f;
+AC1-0F0F; &C1-0F0F;
+AC7-f0f0; &C7-f0f0;
+ACX-F0F0; &CX-F0F0;
+ACY-aBcD; &CY-aBcD;
+AU-0123;  &U-0123;
+=========
+AC0-0f0f;AU-0123;  &C0-0f0f;&U-0123;
+AU-0123;AC0-0f0f;  &U-0123;&C0-0f0f;
+==================
+AAC0-0f0f;         &&C0-0f0f;
+ACAC0-0f0f;        &C&C0-0f0f;
+AC1AC0-0f0f;       &C1&C0-0f0f;
+AUAC0-0f0f;        &U&C0-0f0f;
+AC2-AC0-0f0f;      &C2-&C0-0f0f;
+AC3-1AC0-0f0f;     &C3-1&C0-0f0f;
+AC4-12AC0-0f0f;    &C4-12&C0-0f0f;
+AC5-123AC0-0f0f;   &C5-123&C0-0f0f;
+AC6-1234AC0-0f0f;  &C6-1234&C0-0f0f;
+AC7-12345AC0-0f0f; &C7-12345&C0-0f0f;
+==================
diff --git a/Build/source/texk/cjkutils/tests/cefsconv.cjk b/Build/source/texk/cjkutils/tests/cefsconv.cjk
new file mode 100644
index 00000000000..b4e06bdfbe9
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/cefsconv.cjk
@@ -0,0 +1,33 @@
+\def\CNSpreproc{cefsconv (CJK ver. 4.8.2)}
+0x81 0x7b:�123
+0x9f 0x5c:�92
+0xe0 0x7d:�125
+0xef 0x64:�100
+0xf0 0x64:�100
+0xfc 0x64:�100
+0xfd 0x64:�d
+0xa0 0x64:�d
+0xdf 0x64:�d
+==========
+AC0-0f0f; 72Bg5"00F"00F
+AC1-0F0F; 72CNS1"00F"00F
+AC7-f0f0; 72CNS7"0F0"0F0
+ACX-F0F0; 72CEFX"0F0"0F0
+ACY-aBcD; 72CEFY"0AB"0CD
+AU-0123;  72UTF8"001"023
+=========
+AC0-0f0f;AU-0123;  72Bg5"00F"00F72UTF8"001"023
+AU-0123;AC0-0f0f;  72UTF8"001"02372Bg5"00F"00F
+==================
+AAC0-0f0f;         &72Bg5"00F"00F
+ACAC0-0f0f;        &C72Bg5"00F"00F
+AC1AC0-0f0f;       &C172Bg5"00F"00F
+AUAC0-0f0f;        &U72Bg5"00F"00F
+AC2-AC0-0f0f;      &C2-72Bg5"00F"00F
+AC3-1AC0-0f0f;     &C3-172Bg5"00F"00F
+AC4-12AC0-0f0f;    &C4-1272Bg5"00F"00F
+AC5-123AC0-0f0f;   &C5-12372Bg5"00F"00F
+AC6-1234AC0-0f0f;  &C6-123472Bg5"00F"00F
+AC7-12345AC0-0f0f; &C7-1234572Bg5"00F"00F
+A 0xe1 0x64:       &�100
+==================
diff --git a/Build/source/texk/cjkutils/tests/cefsconv.tex b/Build/source/texk/cjkutils/tests/cefsconv.tex
new file mode 100644
index 00000000000..21a6546ae68
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/cefsconv.tex
@@ -0,0 +1,33 @@
+
+0x81 0x7b:�{
+0x9f 0x5c:�\
+0xe0 0x7d:�}
+0xef 0x64:�d
+0xf0 0x64:�d
+0xfc 0x64:�d
+0xfd 0x64:�d
+0xa0 0x64:�d
+0xdf 0x64:�d
+==========
+AC0-0f0f; &C0-0f0f;
+AC1-0F0F; &C1-0F0F;
+AC7-f0f0; &C7-f0f0;
+ACX-F0F0; &CX-F0F0;
+ACY-aBcD; &CY-aBcD;
+AU-0123;  &U-0123;
+=========
+AC0-0f0f;AU-0123;  &C0-0f0f;&U-0123;
+AU-0123;AC0-0f0f;  &U-0123;&C0-0f0f;
+==================
+AAC0-0f0f;         &&C0-0f0f;
+ACAC0-0f0f;        &C&C0-0f0f;
+AC1AC0-0f0f;       &C1&C0-0f0f;
+AUAC0-0f0f;        &U&C0-0f0f;
+AC2-AC0-0f0f;      &C2-&C0-0f0f;
+AC3-1AC0-0f0f;     &C3-1&C0-0f0f;
+AC4-12AC0-0f0f;    &C4-12&C0-0f0f;
+AC5-123AC0-0f0f;   &C5-123&C0-0f0f;
+AC6-1234AC0-0f0f;  &C6-1234&C0-0f0f;
+AC7-12345AC0-0f0f; &C7-12345&C0-0f0f;
+A 0xe1 0x64:       &�d
+==================
diff --git a/Build/source/texk/cjkutils/tests/extconv.cjk b/Build/source/texk/cjkutils/tests/extconv.cjk
new file mode 100644
index 00000000000..f64b086b61a
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/extconv.cjk
@@ -0,0 +1,5 @@
+\def\CJKpreproc{extconv (CJK ver. 4.8.2)}
+0x81 0x7b:�123
+0xfe 0x5c:�92
+0xc3 0x7d:�125
+==========
diff --git a/Build/source/texk/cjkutils/tests/extconv.tex b/Build/source/texk/cjkutils/tests/extconv.tex
new file mode 100644
index 00000000000..1c2a113add4
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/extconv.tex
@@ -0,0 +1,5 @@
+
+0x81 0x7b:�{
+0xfe 0x5c:�\
+0xc3 0x7d:�}
+==========
diff --git a/Build/source/texk/cjkutils/tests/sjisconv.cjk b/Build/source/texk/cjkutils/tests/sjisconv.cjk
new file mode 100644
index 00000000000..f40b0944f04
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/sjisconv.cjk
@@ -0,0 +1,11 @@
+\def\CJKpreproc{sjisconv (CJK ver. 4.8.2)}
+0x81 0x7b:�123
+0x9f 0x5c:�92
+0xe0 0x7d:�125
+0xef 0x64:�100
+0xf0 0x64:�100
+0xfc 0x64:�100
+0xfd 0x64:�d
+0xa0 0x64:�d
+0xdf 0x64:�d
+==========
diff --git a/Build/source/texk/cjkutils/tests/sjisconv.tex b/Build/source/texk/cjkutils/tests/sjisconv.tex
new file mode 100644
index 00000000000..24cb3da56f4
--- /dev/null
+++ b/Build/source/texk/cjkutils/tests/sjisconv.tex
@@ -0,0 +1,11 @@
+
+0x81 0x7b:�{
+0x9f 0x5c:�\
+0xe0 0x7d:�}
+0xef 0x64:�d
+0xf0 0x64:�d
+0xfc 0x64:�d
+0xfd 0x64:�d
+0xa0 0x64:�d
+0xdf 0x64:�d
+==========