\tracinglostchars>=3 now causes an error; missing character code always reported in hex

git-svn-id: svn://tug.org/texlive/trunk@55874 c570f23f-e606-0410-a88d-b1316a301751

15 files changed, 270 insertions, 11 deletions

diff --git a/Build/source/texk/web2c/ChangeLog b/Build/source/texk/web2c/ChangeLog
index 92ff686754d..1b7b5a98ca9 100644
--- a/Build/source/texk/web2c/ChangeLog
+++ b/Build/source/texk/web2c/ChangeLog
@@ -1,3 +1,8 @@
+2020-07-19  Karl Berry  <karl@freefriends.org>
+
+	* NEWS: mention new \tracinglostchars>=3 causing an error.
+	See pdftexdir/ChangeLog for details.
+
 2020-07-17  Andreas Scherer  <https://ascherer.github.io>
 
 	* Makefile.am,
diff --git a/Build/source/texk/web2c/Makefile.in b/Build/source/texk/web2c/Makefile.in
index 56a690b5f6a..4c9dc16d85f 100644
--- a/Build/source/texk/web2c/Makefile.in
+++ b/Build/source/texk/web2c/Makefile.in
@@ -3322,9 +3322,9 @@ DISTCLEANFILES = CXXLD.sh tangle.c tangle.h tangle.p tangle-web2c \
 	cnfline.log postV3.afm postV7.afm test-13.pdf test-13.xref \
 	test-15.pdf test-15.xref $(nodist_libluatex_sources) \
 	luaimage.* luajitimage.* $(nodist_xetex_SOURCES) xetex.web \
-	xetex-final.ch xetex-web2c xetex.p xetex.pool xetex-tangle bug73.fmt \
-	bug73.log bug73.out bug73.tex filedump.log filedump.out \
-	filedump.tex $(omegaware_programs:=.c) \
+	xetex-final.ch xetex-web2c xetex.p xetex.pool xetex-tangle \
+	bug73.fmt bug73.log bug73.out bug73.tex filedump.log \
+	filedump.out filedump.tex $(omegaware_programs:=.c) \
 	$(omegaware_programs:=.h) $(omegaware_programs:=.p) \
 	$(omegaware_programs:=-web2c) ofm2opl.web opl2ofm.web \
 	ovf2ovp.web ovp2ovf.web omegaware/bad*.* \
@@ -3818,6 +3818,7 @@ eptex_ch_srcs = \
 	$(eptex_ch_synctex) \
 	eptexdir/fam256.ch \
 	eptexdir/pdfutils.ch \
+	eptexdir/char-warning-eptex.ch \
 	tex-binpool.ch
 
 
@@ -3929,6 +3930,7 @@ euptex_ch_srcs = \
 	euptexdir/pdfstrcmp-eup-pre.ch \
 	eptexdir/pdfutils.ch \
 	euptexdir/pdfstrcmp-eup-post.ch \
+	eptexdir/char-warning-eptex.ch \
 	tex-binpool.ch
 
 
@@ -3986,6 +3988,7 @@ pdftex_ch_srcs = \
 	enctex.ch \
 	$(pdftex_ch_synctex) \
 	pdftexdir/pdftex.ch \
+	pdftexdir/char-warning-pdftex.ch \
 	tex-binpool.ch
 
 
@@ -4673,6 +4676,7 @@ xetex_ch_srcs = \
 	$(xetex_ch_synctex) \
 	xetexdir/xetex.ch \
 	$(xetex_post_ch_synctex) \
+	xetexdir/char-warning-xetex.ch \
 	tex-binpool.ch
 
 libxetex_a_CPPFLAGS = $(xetex_cppflags)
@@ -18783,7 +18787,6 @@ $(srcdir)/xetexdir/xetex_version.h: @MAINTAINER_MODE_TRUE@ xetexdir/xetex.web
 	$(AM_V_GEN)grep '^@d XeTeX_version_string==' $(srcdir)/xetexdir/xetex.web \
 	  | sed "s/^.*'-/#define XETEX_VERSION \"/;s/'.*$$/\"/" >$@
 
-# Generate xetex-final.ch
 xetex-final.ch: tie$(EXEEXT) $(xetex_ch_srcs)
 	$(tie_c) $(xetex_ch_srcs)
 $(libxetex_a_OBJECTS): $(libxetex_prereq)
diff --git a/Build/source/texk/web2c/NEWS b/Build/source/texk/web2c/NEWS
index 4ec1756deed..1e7b70cfd82 100644
--- a/Build/source/texk/web2c/NEWS
+++ b/Build/source/texk/web2c/NEWS
@@ -1,5 +1,9 @@
 This file records noteworthy changes.  (Public domain.)
 
+* pdftex, xetex, e(u)ptex: if \tracinglostchars >= 3,
+make missing characters an error (not just a log message),
+and always report the character code in hex.
+
 * obscure interaction sequences try to avoid crashing.
 
 
diff --git a/Build/source/texk/web2c/eptexdir/ChangeLog b/Build/source/texk/web2c/eptexdir/ChangeLog
index 58f0987ff03..ddb5dc477df 100644
--- a/Build/source/texk/web2c/eptexdir/ChangeLog
+++ b/Build/source/texk/web2c/eptexdir/ChangeLog
@@ -1,3 +1,8 @@
+2020-07-19  Karl Berry  <karl@freefriends.org>
+
+	* char-warning-eptex.ch: new file (see ../pdftexdir/ChangeLog).
+	* am/eptex.am (eptex_ch_srcs): add it.
+
 2020-05-05  Hironori Kitagawa  <h_kitagawa2001@yahoo.co.jp>
 
 	* pdfutils.ch: Sync with inhibit_glue_flag improvement.
diff --git a/Build/source/texk/web2c/eptexdir/am/eptex.am b/Build/source/texk/web2c/eptexdir/am/eptex.am
index 0bb060024bd..596d7a42d7e 100644
--- a/Build/source/texk/web2c/eptexdir/am/eptex.am
+++ b/Build/source/texk/web2c/eptexdir/am/eptex.am
@@ -1,6 +1,7 @@
+## $Id$
 ## texk/web2c/eptexdir/am/eptex.am: Makefile fragment for e-pTeX.
 ##
-## Copyright 2015-2019 Karl Berry <tex-live@tug.org>
+## Copyright 2015-2020 Karl Berry <tex-live@tug.org>
 ## Copyright 2011-2015 Peter Breitenlohner <tex-live@tug.org>
 ## You may freely use, modify and/or distribute this file.
 
@@ -68,6 +69,7 @@ eptex_ch_srcs = \
 	$(eptex_ch_synctex) \
 	eptexdir/fam256.ch \
 	eptexdir/pdfutils.ch \
+	eptexdir/char-warning-eptex.ch \
 	tex-binpool.ch
 
 EXTRA_DIST += $(eptex_web_srcs) $(eptex_ch_srcs) eptexdir/eptex.defines
diff --git a/Build/source/texk/web2c/eptexdir/char-warning-eptex.ch b/Build/source/texk/web2c/eptexdir/char-warning-eptex.ch
new file mode 100644
index 00000000000..f6593ad1e9d
--- /dev/null
+++ b/Build/source/texk/web2c/eptexdir/char-warning-eptex.ch
@@ -0,0 +1,63 @@
+% $Id$
+% Public domain. See ../pdftexdir/char-warning-pdftex.ch for info.
+
+@x [30] If \tracinglostchars > 2, then promote full errors.
+@p procedure char_warning(@!f:internal_font_number;@!c:eight_bits);
+var @!l:0..255; {small indices or counters}
+old_setting: integer; {saved value of |tracing_online|}
+begin if tracing_lost_chars>0 then
+ begin old_setting:=tracing_online;
+ if eTeX_ex and(tracing_lost_chars>1) then tracing_online:=1;
+  begin begin_diagnostic;
+  print_nl("Missing character: There is no ");
+@.Missing character@>
+  if (c<" ")or(c>"~") then
+    begin print_char("^"); print_char("^");
+    if c<64 then print_char(c+64)
+    else if c<128 then print_char(c-64)
+    else begin print_lc_hex(c div 16);  print_lc_hex(c mod 16); end
+    end
+  else print_ASCII(c);
+  print(" in font ");
+  slow_print(font_name[f]); print_char("!"); end_diagnostic(false);
+  end;
+ tracing_online:=old_setting;
+ end;
+end;
+@y
+@p procedure char_warning(@!f:internal_font_number;@!c:eight_bits);
+var @!l:0..255; {small indices or counters}
+old_setting: integer; {saved value of |tracing_online|}
+begin if tracing_lost_chars>0 then
+ begin old_setting:=tracing_online;
+ if eTeX_ex and(tracing_lost_chars>1) then tracing_online:=1;
+ if tracing_lost_chars > 2 then
+   print_err("Missing character: There is no ")
+ else begin
+   begin_diagnostic;
+   print_nl("Missing character: There is no ")
+ end;
+@.Missing character@>
+ if (c<" ")or(c>"~") then begin
+   print_char("^"); print_char("^");
+   if c<64 then print_char(c+64)
+   else if c<128 then print_char(c-64)
+        else begin print_lc_hex(c div 16);  print_lc_hex(c mod 16); end
+             end
+ else print_ASCII(c);
+ if tracing_lost_chars > 2 then begin
+   print(" (");
+   print_hex(c);
+   print(")");
+ end;
+ print(" in font ");
+ slow_print(font_name[f]);
+ if tracing_lost_chars < 3 then print_char("!");
+ tracing_online:=old_setting;
+ if tracing_lost_chars > 2 then begin
+   help0;
+   error;
+ end else end_diagnostic(false);
+ end; {of |tracing_lost_chars>0|}
+end;  {of procedure}
+@z
diff --git a/Build/source/texk/web2c/euptexdir/ChangeLog b/Build/source/texk/web2c/euptexdir/ChangeLog
index 56a65423fb9..4e8e11a3d7f 100644
--- a/Build/source/texk/web2c/euptexdir/ChangeLog
+++ b/Build/source/texk/web2c/euptexdir/ChangeLog
@@ -1,3 +1,7 @@
+2020-07-19  Karl Berry  <karl@freefriends.org>
+
+	* am/euptex.am (euptex_ch_srcs): add eptexdir/char-warning-eptex.ch.
+
 2019-12-10  Hironori Kitagawa  <h_kitagawa2001@yahoo.co.jp>
 
 	* am/euptex.am (euptex_CPPFLAGS): append $(ZLIB_INCLUDES).
diff --git a/Build/source/texk/web2c/euptexdir/am/euptex.am b/Build/source/texk/web2c/euptexdir/am/euptex.am
index 07ef89d91b0..cd65b7c618c 100644
--- a/Build/source/texk/web2c/euptexdir/am/euptex.am
+++ b/Build/source/texk/web2c/euptexdir/am/euptex.am
@@ -1,6 +1,7 @@
+## $Id$
 ## texk/web2c/euptexdir/am/euptex.am: Makefile fragment for e-upTeX.
 ##
-## Copyright 2015-2019 Karl Berry <tex-live@tug.org>
+## Copyright 2015-2020 Karl Berry <tex-live@tug.org>
 ## Copyright 2011-2015 Peter Breitenlohner <tex-live@tug.org>
 ## You may freely use, modify and/or distribute this file.
 
@@ -68,6 +69,7 @@ euptex_ch_srcs = \
 	euptexdir/pdfstrcmp-eup-pre.ch \
 	eptexdir/pdfutils.ch \
 	euptexdir/pdfstrcmp-eup-post.ch \
+	eptexdir/char-warning-eptex.ch \
 	tex-binpool.ch
 
 EXTRA_DIST += $(euptex_web_srcs) $(euptex_ch_srcs) euptexdir/euptex.defines
diff --git a/Build/source/texk/web2c/pdftexdir/ChangeLog b/Build/source/texk/web2c/pdftexdir/ChangeLog
index d99bbc8826b..400996e8206 100644
--- a/Build/source/texk/web2c/pdftexdir/ChangeLog
+++ b/Build/source/texk/web2c/pdftexdir/ChangeLog
@@ -1,3 +1,12 @@
+2020-07-19  Karl Berry  <karl@freefriends.org>
+
+	* char-warning-pdftex.ch: new file to implement new feature;
+	if \tracinglostchars>=3, missing characters cause an error and not
+	just a diagnostic message. Also, always report the hex code of the
+	missing character. From David Jones, tex-implements 30jun2020.
+	* am/pdftex.am (pdftex_ch_srcs): add it.
+	* change-files.txt: update.
+
 2020-07-17  Andreas Scherer  <https://ascherer.github.io>
 
 	* am/pdftex.am: Avoid changefile ambiguity.
diff --git a/Build/source/texk/web2c/pdftexdir/am/pdftex.am b/Build/source/texk/web2c/pdftexdir/am/pdftex.am
index 1eb82160dcc..3bd5280521a 100644
--- a/Build/source/texk/web2c/pdftexdir/am/pdftex.am
+++ b/Build/source/texk/web2c/pdftexdir/am/pdftex.am
@@ -1,6 +1,7 @@
+## $Id$
 ## texk/web2c/pdftexdir/am/pdftex.am: Makefile fragment for pdfTeX.
 ##
-## Copyright 2016-2019 Karl Berry <tex-live@tug.org>
+## Copyright 2016-2020 Karl Berry <tex-live@tug.org>
 ## Copyright 2009-2015 Peter Breitenlohner <tex-live@tug.org>
 ## You may freely use, modify and/or distribute this file.
 
@@ -73,6 +74,7 @@ pdftex_ch_srcs = \
 	enctex.ch \
 	$(pdftex_ch_synctex) \
 	pdftexdir/pdftex.ch \
+	pdftexdir/char-warning-pdftex.ch \
 	tex-binpool.ch
 
 # Convenience target for making the typeset output; not invoked automatically
diff --git a/Build/source/texk/web2c/pdftexdir/change-files.txt b/Build/source/texk/web2c/pdftexdir/change-files.txt
index c6a3387f41d..a435369d052 100644
--- a/Build/source/texk/web2c/pdftexdir/change-files.txt
+++ b/Build/source/texk/web2c/pdftexdir/change-files.txt
@@ -1,14 +1,20 @@
+$Id$
+Public domain.
 How change files are applied for pdftex/pdfetex?
 
 ============================= pdftex =============================
 
-2) pdftex.ch is generated from
+2) pdftex.ch is essentially generated from (see pdftexdir/am/pdftex.am)
     ./tie -c pdftex.ch pdftex.web \
     $(srcdir)/pdftexdir/tex.ch0 \
     $(srcdir)/tex.ch \
     $(srcdir)/pdftexdir/tex.ch1 \
     $(srcdir)/pdftexdir/tex.pch
 
+There are more feature-specific change files, for synctex, including the
+pool in the binary, etc., many of which are shared between engines. They
+don't alter the basic process described here.
+
 Input:
     pdftex.web:
         pdftex without system-dependent changes
@@ -31,5 +37,5 @@ Input:
         change the tex banner to pdftex banner (reverse tex.ch0)
 
 Output:
-    pdftex.ch (not pdftexdir/pdftex.ch):
+    pdftex.ch (in the build dir, not pdftexdir/pdftex.ch):
         additional & system-dependent changes
diff --git a/Build/source/texk/web2c/pdftexdir/char-warning-pdftex.ch b/Build/source/texk/web2c/pdftexdir/char-warning-pdftex.ch
new file mode 100644
index 00000000000..828ebfcc3cf
--- /dev/null
+++ b/Build/source/texk/web2c/pdftexdir/char-warning-pdftex.ch
@@ -0,0 +1,62 @@
+% $Id$
+% Public domain (from David Jones, tex-implementors 30jun2020).
+% Unfortunately this cannot be in pdftex.ch since the ordering
+% of changes gets messed up.
+%
+% If \tracinglostchars >= 3, then making missing characters from a font
+% be an error, not just a log message, and always report the character
+% code in hex as well as a character.
+% 
+% Simple example test:
+% pdftex -ini
+%   \font\x=/usr/local/texlive/2020/texmf-dist/fonts/tfm/public/knuth-lib/logo10.tfm
+%   \tracinglostchars=3 \x \char99\end
+% should get:
+%   ! Missing character: There is no c ("63) in font logo10.
+
+@x [30] If \tracinglostchars > 2, then promote full errors.
+@p procedure char_warning(@!f:internal_font_number;@!c:eight_bits);
+var old_setting: integer; {saved value of |tracing_online|}
+begin if tracing_lost_chars>0 then
+ begin old_setting:=tracing_online;
+ if eTeX_ex and(tracing_lost_chars>1) then tracing_online:=1;
+  begin begin_diagnostic;
+  print_nl("Missing character: There is no ");
+@.Missing character@>
+  print_ASCII(c); print(" in font ");
+  slow_print(font_name[f]); print_char("!"); end_diagnostic(false);
+  end;
+ tracing_online:=old_setting;
+ end;
+end;
+@y
+@p procedure char_warning(@!f:internal_font_number;@!c:eight_bits);
+var old_setting: integer; {saved value of |tracing_online|}
+begin if tracing_lost_chars>0 then
+ begin old_setting:=tracing_online;
+ if eTeX_ex and(tracing_lost_chars>1) then tracing_online:=1;
+ if tracing_lost_chars > 2 then
+   print_err("Missing character: There is no ")
+ else begin
+   begin_diagnostic;
+   print_nl("Missing character: There is no ")
+ end;
+@.Missing character@>
+ print_ASCII(c);
+ if tracing_lost_chars > 2 then begin
+   print(" (");
+   print_hex(c);
+   print(")");
+ end;
+ print(" in font ");
+ slow_print(font_name[f]);
+ if tracing_lost_chars < 3 then print_char("!");
+ tracing_online:=old_setting;
+ if tracing_lost_chars > 2 then begin
+   help0;
+   error;
+ end else end_diagnostic(false);
+ end; {of |tracing_lost_chars>0|}
+end;  {of procedure}
+@z
+
diff --git a/Build/source/texk/web2c/xetexdir/ChangeLog b/Build/source/texk/web2c/xetexdir/ChangeLog
index 81b02b0fcfc..279ddbbbffe 100644
--- a/Build/source/texk/web2c/xetexdir/ChangeLog
+++ b/Build/source/texk/web2c/xetexdir/ChangeLog
@@ -1,3 +1,10 @@
+2020-07-19  Karl Berry  <karl@freefriends.org>
+
+	* char-warning-xetex.ch: new file; see ../pdftexdir/ChangeLog.
+	Also define new procedure print_ucs_code to print U+nnnn
+	for system fonts.
+	* am/xetex.am: add it.
+
 2020-07-17  Andreas Scherer  <https://ascherer.github.io>
 
 	* am/xetex.am: Avoid changefile ambiguity.
diff --git a/Build/source/texk/web2c/xetexdir/am/xetex.am b/Build/source/texk/web2c/xetexdir/am/xetex.am
index ddba5f9f10b..1d3cb7062eb 100644
--- a/Build/source/texk/web2c/xetexdir/am/xetex.am
+++ b/Build/source/texk/web2c/xetexdir/am/xetex.am
@@ -116,13 +116,14 @@ xetex_ch_srcs = \
 	$(xetex_ch_synctex) \
 	xetexdir/xetex.ch \
 	$(xetex_post_ch_synctex) \
+	xetexdir/char-warning-xetex.ch \
 	tex-binpool.ch
 
 ##
 EXTRA_DIST += $(xetex_web_srcs) $(xetex_ch_srcs) xetexdir/xetex.defines
 
-DISTCLEANFILES += $(nodist_xetex_SOURCES) xetex.web xetex-final.ch xetex-web2c \
-	xetex.p xetex.pool xetex-tangle
+DISTCLEANFILES += $(nodist_xetex_SOURCES) xetex.web xetex-final.ch \
+  xetex-web2c xetex.p xetex.pool xetex-tangle
 
 
 ## libxetex
diff --git a/Build/source/texk/web2c/xetexdir/char-warning-xetex.ch b/Build/source/texk/web2c/xetexdir/char-warning-xetex.ch
new file mode 100644
index 00000000000..246c0489ce5
--- /dev/null
+++ b/Build/source/texk/web2c/xetexdir/char-warning-xetex.ch
@@ -0,0 +1,84 @@
+% $Id$
+% Public domain. See ../pdftexdir/char-warning-pdftex.ch for basic info.
+% In addition to the basic change, we also define a new procedure
+% print_ucs_code to print in the format U+.... for system fonts.
+%
+% To test:
+% xetex -ini
+%   \font\x=/usr/local/texlive/2020/texmf-dist/fonts/tfm/public/knuth-lib/logo10.tfm
+%   \tracinglostchars=3 \x \char99\end
+% should get:
+%   ! Missing character: There is no c ("63) in font logo10.
+% And:
+%   \font\y="Nimbus Mono" \tracinglostchars=3 \y \char43981\end
+% should get:
+%   ! Missing character: There is no ꯍ (U+ABCD) in font Nimbus Mono Regular!.
+% Assuming that font is findable as a system font, e.g., shown by
+%   fc-list :scalable=true family
+
+@x [30] If \tracinglostchars > 2, then promote full errors.
+procedure char_warning(@!f:internal_font_number;@!c:integer);
+var old_setting: integer; {saved value of |tracing_online|}
+begin if tracing_lost_chars>0 then
+ begin old_setting:=tracing_online;
+ if eTeX_ex and(tracing_lost_chars>1) then tracing_online:=1;
+  begin begin_diagnostic;
+  print_nl("Missing character: There is no ");
+@.Missing character@>
+  if c < @"10000 then print_ASCII(c)
+  else print_char(c); {non-Plane 0 Unicodes can't be sent through |print_ASCII|}
+  print(" in font ");
+  slow_print(font_name[f]); print_char("!"); end_diagnostic(false);
+  end;
+ tracing_online:=old_setting;
+ end;
+end;
+@y
+procedure print_ucs_code(@!n: UnicodeScalar); { cf. print_hex }
+    var k: 0..22; {index to current digit; we assume that $0\L n<16^{22}$}
+    begin
+        k := 0;
+        print("U+"); { prefix with U+ instead of " }
+        repeat
+            dig[k] := n mod 16;
+            n      := n div 16;
+            incr(k);
+        until n = 0;
+        { pad to at least 4 hex digits }
+        while k < 4 do
+        begin
+            dig[k] := 0;
+            incr(k);
+        end;
+        print_the_digs(k);
+end;
+
+procedure char_warning(@!f:internal_font_number;@!c:integer);
+var old_setting: integer; {saved value of |tracing_online|}
+begin if tracing_lost_chars>0 then
+ begin old_setting:=tracing_online;
+ if eTeX_ex and(tracing_lost_chars>1) then tracing_online:=1;
+ if tracing_lost_chars > 2 then
+   print_err("Missing character: There is no ")
+ else begin
+   begin_diagnostic;
+   print_nl("Missing character: There is no ")
+ end;
+@.Missing character@>
+ if c < @"10000 then print_ASCII(c)
+ else print_char(c); {non-Plane 0 Unicodes can't be sent through |print_ASCII|}
+ print(" (");
+ if is_native_font(f) then print_ucs_code(c)
+ else print_hex(c);
+ print(")");
+ print(" in font ");
+ slow_print(font_name[f]);
+ if tracing_lost_chars < 3 then print_char("!");
+ tracing_online:=old_setting;
+ if tracing_lost_chars > 2 then begin
+   help0;
+   error;
+ end else end_diagnostic(false);
+ end; {of |tracing_lost_chars>0|}
+end;  {of procedure}
+@z