{,e}{,u}ptex: Enable to get charcode from control symbol by back quote

git-svn-id: svn://tug.org/texlive/trunk@71289 c570f23f-e606-0410-a88d-b1316a301751

11 files changed, 219 insertions, 35 deletions

diff --git a/Build/source/texk/web2c/Makefile.in b/Build/source/texk/web2c/Makefile.in
index fb8579e9938..9f1c3cab36d 100644
--- a/Build/source/texk/web2c/Makefile.in
+++ b/Build/source/texk/web2c/Makefile.in
@@ -3244,22 +3244,24 @@ EXTRA_DIST = PROJECTS cftests cpascal.h help.h w2c/config.h \
 	euptexdir/euptrip/euptrip.diffs euptexdir/euptrip/texmf.cnf \
 	euptexdir/pdfprimitive.test \
 	euptexdir/tests/pdfprimitive-test.tex \
-	euptexdir/tests/pdfprimitive-euptex.log hitexdir/ChangeLog \
-	$(hitex_web) $(hitex_tests) hitexdir/tests/hello.tex \
-	hitexdir/tests/rule.tex pdftexdir/regex/COPYING.LIB \
-	pdftexdir/regex/README $(pdftex_ch_srcs) \
-	pdftexdir/pdftex.defines pdftexdir/ChangeLog pdftexdir/NEWS \
-	pdftexdir/README pdftexdir/change-files.txt $(pdftex_tests) \
-	tests/wprob.tex pdftexdir/tests/pdfimage.tex tests/1-4.jpg \
-	tests/B.pdf tests/basic.tex tests/lily-ledger-broken.png \
-	tests/expanded.tex tests/expanded.txt tests/cnfline.tex \
-	tests/partoken-ok.tex tests/partoken-xfail.tex \
-	$(ttf2afm_tests) pdftexdir/tests/postV3.afm \
-	pdftexdir/tests/postV3.ttf pdftexdir/tests/postV7.afm \
-	pdftexdir/tests/postV7.ttf $(pdftosrc_tests) \
-	pdftexdir/tests/test-13.pdf pdftexdir/tests/test-13.xref \
-	pdftexdir/tests/test-15.pdf pdftexdir/tests/test-15.xref \
-	$(libluasocket_sources) luatexdir/luasocket/src/ftp_lua.c \
+	euptexdir/tests/pdfprimitive-euptex.log \
+	euptexdir/tests/ctrlsym.log euptexdir/tests/ctrlsym.tex \
+	hitexdir/ChangeLog $(hitex_web) $(hitex_tests) \
+	hitexdir/tests/hello.tex hitexdir/tests/rule.tex \
+	pdftexdir/regex/COPYING.LIB pdftexdir/regex/README \
+	$(pdftex_ch_srcs) pdftexdir/pdftex.defines pdftexdir/ChangeLog \
+	pdftexdir/NEWS pdftexdir/README pdftexdir/change-files.txt \
+	$(pdftex_tests) tests/wprob.tex pdftexdir/tests/pdfimage.tex \
+	tests/1-4.jpg tests/B.pdf tests/basic.tex \
+	tests/lily-ledger-broken.png tests/expanded.tex \
+	tests/expanded.txt tests/cnfline.tex tests/partoken-ok.tex \
+	tests/partoken-xfail.tex $(ttf2afm_tests) \
+	pdftexdir/tests/postV3.afm pdftexdir/tests/postV3.ttf \
+	pdftexdir/tests/postV7.afm pdftexdir/tests/postV7.ttf \
+	$(pdftosrc_tests) pdftexdir/tests/test-13.pdf \
+	pdftexdir/tests/test-13.xref pdftexdir/tests/test-15.pdf \
+	pdftexdir/tests/test-15.xref $(libluasocket_sources) \
+	luatexdir/luasocket/src/ftp_lua.c \
 	luatexdir/luasocket/src/headers_lua.c \
 	luatexdir/luasocket/src/http_lua.c \
 	luatexdir/luasocket/src/ltn12_lua.c \
@@ -3569,7 +3571,8 @@ DISTCLEANFILES = CXXLD.sh tangle.c tangle.h tangle.p tangle-web2c \
 	uptests/ygk256*.* uptests/xskipjfmp.* uptrip.diffs \
 	$(nodist_euptex_SOURCES) euptex.web euptex.ch euptex-web2c \
 	euptex.p euptex.pool euptex-tangle euptrip.diffs \
-	pdfprimitive-euptex.* $(nodist_hitex_SOURCES) \
+	pdfprimitive-euptex.* eup-ctrlsym.fmt eup-ctrlsym.log \
+	eup-ctrlsym.out eup-ctrlsym.tex $(nodist_hitex_SOURCES) \
 	$(nodist_hishrink_SOURCES) $(nodist_histretch_SOURCES) \
 	hiformat-tangle hitex-tangle hello.log rule.log \
 	$(nodist_pdftex_SOURCES) pdftex-final.ch pdftex-web2c pdftex.p \
@@ -4514,6 +4517,7 @@ euptex_ch_srcs = \
 euptex_tests = \
 	euptexdir/euptriptest.test euptexdir/eptriptest.test \
 	euptexdir/pdfprimitive.test \
+	euptexdir/euptex-ctrlsym.test \
 	euptexdir/eupver.test \
 	euptexdir/upkcat.test \
 	euptexdir/wcfname.test euptexdir/wcfname0.test
@@ -21984,7 +21988,8 @@ euptex.ch: tie$(EXEEXT) euptex.web $(euptex_ch_srcs)
 euptexdir/euptriptest.log euptexdir/eptriptest.log: \
 	euptex$(EXEEXT) dvitype$(EXEEXT) pltotf$(EXEEXT) tftopl$(EXEEXT)
 euptexdir/pdfprimitive.log: euptex$(EXEEXT)
-euptexdir/eupver.log euptexdir/upkcat.log \
+euptexdir/euptex-ctrlsym.log \
+	euptexdir/eupver.log euptexdir/upkcat.log \
 	euptexdir/wcfname.log euptexdir/wcfname0.log: euptex$(EXEEXT)
 euptrip.diffs: euptex$(EXEEXT) dvitype$(EXEEXT) pltotf$(EXEEXT) tftopl$(EXEEXT)
 	$(triptrap_diffs) $@
diff --git a/Build/source/texk/web2c/euptexdir/ChangeLog b/Build/source/texk/web2c/euptexdir/ChangeLog
index 5d39c9205c8..34397d189d0 100644
--- a/Build/source/texk/web2c/euptexdir/ChangeLog
+++ b/Build/source/texk/web2c/euptexdir/ChangeLog
@@ -1,3 +1,10 @@
+2024-05-18  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
+
+	* euptex-ctrlsym.test, tests/ctrlsym.{tex,log},
+	am/euptex.am: Add tests for getting character codes
+	from control symbols by using backquote "`".
+	https://github.com/texjporg/tex-jp-build/issues/167
+
 2024-05-01  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
 
 	* pdfutils.ch: Revert some patches.
diff --git a/Build/source/texk/web2c/euptexdir/am/euptex.am b/Build/source/texk/web2c/euptexdir/am/euptex.am
index 1103cc70d83..f961b447fcc 100644
--- a/Build/source/texk/web2c/euptexdir/am/euptex.am
+++ b/Build/source/texk/web2c/euptexdir/am/euptex.am
@@ -98,13 +98,15 @@ EXTRA_DIST += \
 euptex_tests = \
 	euptexdir/euptriptest.test euptexdir/eptriptest.test \
 	euptexdir/pdfprimitive.test \
+	euptexdir/euptex-ctrlsym.test \
 	euptexdir/eupver.test \
 	euptexdir/upkcat.test \
 	euptexdir/wcfname.test euptexdir/wcfname0.test
 euptexdir/euptriptest.log euptexdir/eptriptest.log: \
 	euptex$(EXEEXT) dvitype$(EXEEXT) pltotf$(EXEEXT) tftopl$(EXEEXT)
 euptexdir/pdfprimitive.log: euptex$(EXEEXT)
-euptexdir/eupver.log euptexdir/upkcat.log \
+euptexdir/euptex-ctrlsym.log \
+	euptexdir/eupver.log euptexdir/upkcat.log \
 	euptexdir/wcfname.log euptexdir/wcfname0.log: euptex$(EXEEXT)
 
 EXTRA_DIST += $(euptex_tests)
@@ -136,4 +138,8 @@ EXTRA_DIST += \
 	euptexdir/tests/pdfprimitive-euptex.log
 DISTCLEANFILES += pdfprimitive-euptex.*
 
+## euptex-ctrlsym.test
+EXTRA_DIST += euptexdir/tests/ctrlsym.log euptexdir/tests/ctrlsym.tex
+DISTCLEANFILES += eup-ctrlsym.fmt eup-ctrlsym.log eup-ctrlsym.out eup-ctrlsym.tex
+
 # (end of euptex.am)
diff --git a/Build/source/texk/web2c/euptexdir/euptex-ctrlsym.test b/Build/source/texk/web2c/euptexdir/euptex-ctrlsym.test
new file mode 100755
index 00000000000..294e0cf0bf1
--- /dev/null
+++ b/Build/source/texk/web2c/euptexdir/euptex-ctrlsym.test
@@ -0,0 +1,28 @@
+#! /bin/sh -vx
+# Copyright 2024 Japanese TeX Development Community <issue@texjp.org>
+# You may freely use, modify and/or distribute this file.
+
+BinDir=${BinDir:-.}
+ExeExt=${ExeExt:-}
+_euptex=$BinDir/euptex$ExeExt
+
+LC_ALL=C; export LC_ALL;  LANGUAGE=C; export LANGUAGE
+
+TEXMFCNF=$srcdir/../kpathsea;export TEXMFCNF
+TEXINPUTS=".;$srcdir/tests"; export TEXINPUTS
+TEXFORMATS=.; export TEXFORMATS
+
+# get same filename in log
+rm -f eup-ctrlsym.tex
+$LN_S $srcdir/euptexdir/tests/ctrlsym.tex ./eup-ctrlsym.tex
+
+#exit 77
+
+$_euptex -ini -etex eup-ctrlsym || exit 1
+
+$_euptex -etex -fmt=eup-ctrlsym eup-ctrlsym || exit 2
+
+sed 1d eup-ctrlsym.log >eup-ctrlsym.out
+
+diff $srcdir/euptexdir/tests/ctrlsym.log eup-ctrlsym.out || exit 3
+
diff --git a/Build/source/texk/web2c/euptexdir/tests/ctrlsym.log b/Build/source/texk/web2c/euptexdir/tests/ctrlsym.log
new file mode 100644
index 00000000000..1d0eae0c92b
--- /dev/null
+++ b/Build/source/texk/web2c/euptexdir/tests/ctrlsym.log
@@ -0,0 +1,20 @@
+entering extended mode
+ restricted \write18 enabled.
+ %&-line parsing enabled.
+**eup-ctrlsym
+(./eup-ctrlsym.tex 
+A: 38634
+
+\雪 ->snow
+B: 38634. 雪\雪snow
+macro:->snow.
+macro:->snow.
+
+C: 127820
+
+\🍌->banana
+D: 127820. 🍌\🍌 banana
+macro:->banana.
+macro:->banana.
+ )
+No pages of output.
diff --git a/Build/source/texk/web2c/euptexdir/tests/ctrlsym.tex b/Build/source/texk/web2c/euptexdir/tests/ctrlsym.tex
new file mode 100644
index 00000000000..f007708a2bc
--- /dev/null
+++ b/Build/source/texk/web2c/euptexdir/tests/ctrlsym.tex
@@ -0,0 +1,34 @@
+% Copyright 2024 Japanese TeX Development Community <issue@texjp.org>
+% You may freely use, modify and/or distribute this file.
+%
+\ifx\fmtname\undefined
+  \input basic
+  \def\fmtname{basic}
+  \expandafter\dump
+\fi
+%==================
+\begingroup % Just there to minimize tracing output
+  \tracingmacros=2
+  \tracingonline=1
+%--
+  \newlinechar=10
+  \count255=0
+  \count255=`雪 \relax
+  \message{^^JA: \the\count255^^J}
+  \def\雪{snow}
+  \count255=`\雪 \relax
+  \message{B: \the\count255. 雪\string\雪 \雪^^J}
+  \message{\expandafter\meaning\csname 雪\endcsname.^^J}
+  \message{\expandafter\meaning\csname \Uchar"96EA\endcsname.^^J}%
+  \count4095=0
+  \count4095=`🍌 \relax
+  \message{^^JC: \the\count4095^^J}
+  \def\🍌{banana}
+  \count4095=`\🍌 \relax
+  \message{D: \the\count4095. 🍌\string\🍌 \🍌^^J}
+  \message{\expandafter\meaning\csname 🍌\endcsname.^^J}
+  \message{\expandafter\meaning\csname \Uchar"1F34C\endcsname.^^J}%
+%--
+  \setbox0=\vbox{} % no dvi/xdv/pdf output
+\endgroup
+\end
diff --git a/Build/source/texk/web2c/ptexdir/ChangeLog b/Build/source/texk/web2c/ptexdir/ChangeLog
index 73a3fbe1c2b..2fe03aef772 100644
--- a/Build/source/texk/web2c/ptexdir/ChangeLog
+++ b/Build/source/texk/web2c/ptexdir/ChangeLog
@@ -1,3 +1,9 @@
+2024-05-18  Hironori Kitagawa  <h_kitagawa2001@yahoo.co.jp>
+
+	* ptex-base.ch: Enable to get character codes
+	from control symbols by using backquote "`".
+	https://github.com/texjporg/tex-jp-build/issues/167
+
 2024-04-14  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
 
 	* ptriptest.test:
diff --git a/Build/source/texk/web2c/ptexdir/ptex-base.ch b/Build/source/texk/web2c/ptexdir/ptex-base.ch
index 3ed7b821e7b..6b77387d58e 100644
--- a/Build/source/texk/web2c/ptexdir/ptex-base.ch
+++ b/Build/source/texk/web2c/ptexdir/ptex-base.ch
@@ -2582,14 +2582,15 @@ if cur_tok<cs_token_flag then
   end
 else if cur_tok<cs_token_flag+single_base then
   cur_val:=cur_tok-cs_token_flag-active_base
-else
-  { the token is a CS;
-    * if |kanji|<=|cur_cmd|<=|max_char_code|, then CS is let-equal to |wchar_token|
-    * if |max_char_code|<|cur_cmd|, then CS is a multibyte CS
-      => both case should raise "Improper ..." error
-    * otherwise it should be a single-character CS with |cur_val|<=255 }
-  begin if not (cur_cmd<kanji) then cur_cmd:=invalid_char;
-  cur_val:=cur_tok-cs_token_flag-single_base;
+else if cur_tok<cs_token_flag+null_cs then
+  cur_val:=cur_tok-cs_token_flag-single_base
+else { check the cs is a single Japanese character }
+  begin m:=text(cur_tok-cs_token_flag);
+    if str_start[m+1]-str_start[m]
+       = multistrlenshort(str_pool, str_start[m+1], str_start[m])
+       then
+      cur_val:=fromBUFFshort(str_pool, str_start[m+1], str_start[m])
+    else begin cur_cmd:=invalid_char; cur_val:=256; end;
   end;
 if (cur_val>255)and(cur_cmd<kanji) then
   begin print_err("Improper alphabetic or KANJI constant");
diff --git a/Build/source/texk/web2c/uptexdir/ChangeLog b/Build/source/texk/web2c/uptexdir/ChangeLog
index 7637aed37eb..a68dbcce2f4 100644
--- a/Build/source/texk/web2c/uptexdir/ChangeLog
+++ b/Build/source/texk/web2c/uptexdir/ChangeLog
@@ -1,3 +1,10 @@
+2024-05-18  Hironori Kitagawa  <h_kitagawa2001@yahoo.co.jp>
+
+	* ptex-base.ch: Enable to get character codes
+	from control symbols by using backquote "`".
+	* tests/ctlsym_num.tex: Add a test file.
+	https://github.com/texjporg/tex-jp-build/issues/167
+
 2024-05-11  TANAKA Takuji  <ttk@t-lab.opal.ne.jp>
 
 	* uptex-m.ch: Fix unmatched begin-end grouping
diff --git a/Build/source/texk/web2c/uptexdir/ptex-base.ch b/Build/source/texk/web2c/uptexdir/ptex-base.ch
index 3ed7b821e7b..6b77387d58e 100644
--- a/Build/source/texk/web2c/uptexdir/ptex-base.ch
+++ b/Build/source/texk/web2c/uptexdir/ptex-base.ch
@@ -2582,14 +2582,15 @@ if cur_tok<cs_token_flag then
   end
 else if cur_tok<cs_token_flag+single_base then
   cur_val:=cur_tok-cs_token_flag-active_base
-else
-  { the token is a CS;
-    * if |kanji|<=|cur_cmd|<=|max_char_code|, then CS is let-equal to |wchar_token|
-    * if |max_char_code|<|cur_cmd|, then CS is a multibyte CS
-      => both case should raise "Improper ..." error
-    * otherwise it should be a single-character CS with |cur_val|<=255 }
-  begin if not (cur_cmd<kanji) then cur_cmd:=invalid_char;
-  cur_val:=cur_tok-cs_token_flag-single_base;
+else if cur_tok<cs_token_flag+null_cs then
+  cur_val:=cur_tok-cs_token_flag-single_base
+else { check the cs is a single Japanese character }
+  begin m:=text(cur_tok-cs_token_flag);
+    if str_start[m+1]-str_start[m]
+       = multistrlenshort(str_pool, str_start[m+1], str_start[m])
+       then
+      cur_val:=fromBUFFshort(str_pool, str_start[m+1], str_start[m])
+    else begin cur_cmd:=invalid_char; cur_val:=256; end;
   end;
 if (cur_val>255)and(cur_cmd<kanji) then
   begin print_err("Improper alphabetic or KANJI constant");
diff --git a/Build/source/texk/web2c/uptexdir/tests/ctlsym_num.tex b/Build/source/texk/web2c/uptexdir/tests/ctlsym_num.tex
new file mode 100644
index 00000000000..40f695b60b5
--- /dev/null
+++ b/Build/source/texk/web2c/uptexdir/tests/ctlsym_num.tex
@@ -0,0 +1,69 @@
+% Copyright 2024 Japanese TeX Development Community <issue@texjp.org>
+% You may freely use, modify and/or distribute this file.
+%
+\scrollmode
+\newlinechar=10
+
+
+%
+% Following runs on ptex, uptex, eptex and euptex
+%
+\count255=0
+
+\count255=`♪ \relax
+\message{P: \the\count255^^J}
+
+\def\♪{baz}
+\def\雪{qux}
+
+\count255=0
+\count255=`\♪ \relax
+\message{Q: \the\count255. ♪\string\♪ \♪^^J}
+
+\count255=0
+\count255=`雪 \relax
+\message{R: \the\count255^^J}
+\count255=0
+\count255=`\雪 \relax
+\message{S: \the\count255. 雪\string\雪 \雪^^J}
+
+\message{\expandafter\meaning\csname ♪\endcsname.^^J}
+\message{\expandafter\meaning\csname 雪\endcsname.^^J}
+\ifx\Uchar\undefined\else
+\message{\expandafter\meaning\csname \Uchar\jis"2276\endcsname.^^J}
+\message{\expandafter\meaning\csname \Uchar\jis"4063\endcsname.^^J}
+\fi
+
+%
+% Following runs on euptex
+%
+\ifnum\jis"2121="3000
+\ifx\Uchar\undefined
+\else
+
+\count1000=0
+
+\def\🍌{foo}
+\def\𝕯{bar}
+\count1000=`🍌 \relax
+\message{A: \the\count1000^^J}
+\count1000=0
+\count1000=`\🍌 \relax
+\message{B: \the\count1000. 🍌\string\🍌 \🍌^^J}
+
+\count1000=0
+\count1000=`𝕯 \relax
+\message{C: \the\count1000^^J}
+\count1000=0
+\count1000=`\𝕯 \relax
+\message{D: \the\count1000. 𝕯 \string\𝕯  \𝕯^^J}
+
+\message{\expandafter\meaning\csname 🍌\endcsname.^^J}
+\message{\expandafter\meaning\csname 𝕯\endcsname.^^J}
+\message{\expandafter\meaning\csname \Uchar"1F34C\endcsname.^^J}
+\message{\expandafter\meaning\csname \Uchar"1D56F\endcsname.^^J}
+
+\fi
+\fi
+
+\bye