new cross-engine primitives \partokenname and \partokencontext, from Petr Olsak

git-svn-id: svn://tug.org/texlive/trunk@60054 c570f23f-e606-0410-a88d-b1316a301751

19 files changed, 399 insertions, 47 deletions

diff --git a/Build/source/texk/web2c/ChangeLog b/Build/source/texk/web2c/ChangeLog
index 6e8be72092d..e024dfb56c9 100644
--- a/Build/source/texk/web2c/ChangeLog
+++ b/Build/source/texk/web2c/ChangeLog
@@ -1,3 +1,37 @@
+2021-07-25  Petr Olsak  <petr@olsak.net>
+	and Karl Berry  <karl@freefriends.org>
+	and Hironori Kitagawa  <h_kitagawa2001@yahoo.co.jp>
+
+	* partoken.ch: new primitives \partokenname and \partokencontext.
+	* NEWS: mention this.
+
+	* partoken-102.ch (max_command): split off @d since different for pTeX.
+	* enctexdir/enctex-pdftex.ch: rearrange constants.
+	* uptexdir/uptex-m.ch (max_command): now partoken_name is last.
+	* ptexdir/ptex-base.ch (max_command): now partoken_name is last.
+	(insert_group, vcenter_group): remove constant lines from change
+	hunks so that partoken.ch can apply.
+
+	* eptexdir/am/eptex.am (eptex_web_srcs),
+	* euptexdir/am/euptex.am (euptex_web_srcs),
+	* ptexdir/am/ptex.am (ptex_web_srcs),
+	* uptexdir/am/uptex.am (uptex_web_srcs): include partoken.ch.
+	* xetexdir/am/xetex.am (xetex_ch_srcs),
+	* pdftexdir/am/pdftex.am (pdftex_ch_srcs): include partoken.ch
+	and partoken-102.ch.
+
+	* pdftexdir/tests/partoken.test: test it.
+	* tests/partoken-ok.tex,
+	* tests/partoken-xfail.tex: new test files.
+	* tests/cmr10.pfb: needed since the partoken test typesets text.
+	* pdftexdir/am/pdftex.am (pdftex_tests): new test.
+	(pdftexdir/tests/partoken.log): new log.
+	(EXTRA_DIST): new test files.
+	(DISTCLEANFILES): remove new log files.
+
+	* pdftexdir/tests/cnfline.test: use nonstopmode so errors are on
+	terminal, following partoken.test.
+
 2021-07-25  Karl Berry  <karl@freefriends.org>
 
 	* tex.ch: restore "strings are different" change from 2021-07-22.
diff --git a/Build/source/texk/web2c/Makefile.in b/Build/source/texk/web2c/Makefile.in
index 0496527dc54..28b692d962f 100644
--- a/Build/source/texk/web2c/Makefile.in
+++ b/Build/source/texk/web2c/Makefile.in
@@ -3252,7 +3252,8 @@ EXTRA_DIST = PROJECTS cftests cpascal.h help.h w2c/config.h \
 	$(pdftex_tests) tests/wprob.tex pdftexdir/tests/pdfimage.tex \
 	tests/1-4.jpg tests/B.pdf tests/basic.tex \
 	tests/lily-ledger-broken.png tests/expanded.tex \
-	tests/expanded.txt tests/cnfline.tex $(ttf2afm_tests) \
+	tests/expanded.txt tests/cnfline.tex tests/partoken-ok.tex \
+	tests/partoken-xfail.tex $(ttf2afm_tests) \
 	pdftexdir/tests/postV3.afm pdftexdir/tests/postV3.ttf \
 	pdftexdir/tests/postV7.afm pdftexdir/tests/postV7.ttf \
 	$(pdftosrc_tests) pdftexdir/tests/test-13.pdf \
@@ -3570,31 +3571,31 @@ DISTCLEANFILES = CXXLD.sh tangle.c tangle.h tangle.p tangle-web2c \
 	pdfprimitive-euptex.* $(nodist_pdftex_SOURCES) pdftex-final.ch \
 	pdftex-web2c pdftex.p pdftex.pool pdftex-tangle pwprob.log \
 	pwprob.tex pdfimage.fmt pdfimage.log pdfimage.pdf expanded.log \
-	cnfline.log postV3.afm postV7.afm test-13.pdf test-13.xref \
-	test-15.pdf test-15.xref $(nodist_libluatex_sources) \
-	luaimage.* luajitimage.* $(nodist_xetex_SOURCES) xetex.web \
-	xetex-final.ch xetex-web2c xetex.p xetex.pool xetex-tangle \
-	bug73.fmt bug73.log bug73.out bug73.tex filedump.log \
-	filedump.out filedump.tex $(omegaware_programs:=.c) \
-	$(omegaware_programs:=.h) $(omegaware_programs:=.p) \
-	$(omegaware_programs:=-web2c) ofm2opl.web opl2ofm.web \
-	ovf2ovp.web ovp2ovf.web omegaware/bad*.* \
-	omegaware/tests/charwdr.* omegaware/tests/charwdv.* \
-	omegaware/tests/xcheck* omegaware/tests/xlevel1.* \
-	omegaware/tests/xlig*.* ofont*vf omegaware/tests/xpagenum.* \
-	omegaware/tests/xofont* omegaware/tests/Cherokee.tfm \
-	omegaware/tests/OCherokee.ofm omegaware/tests/OCherokee.opl \
-	omegaware/tests/OCherokee.ovf omegaware/tests/xCherokee.* \
-	omegaware/tests/xOCherokee.* ocftest.* omegaware/tests/xinbmp* \
-	omegaware/tests/xoverbmp* omegaware/tests/xrealnum.* \
-	omegaware/tests/xrepeated.* omegaware/tests/sample*.ofm \
-	omegaware/tests/sample*.ovf omegaware/tests/sample1-h.opl \
-	omegaware/tests/xsample*.out omegaware/tests/shortend.* \
-	omegaware/tests/specialhex.ofm omegaware/tests/specialhex.opl \
-	omegaware/tests/specialhex.ovf omegaware/tests/xspecialhex.* \
-	omegaware/tests/yrepeat* omegaware/tests/*yarabic* \
-	$(nodist_aleph_SOURCES) aleph.web aleph.ch aleph-web2c aleph.p \
-	aleph.pool aleph-tangle
+	cnfline.log partoken-ok.log partoken-xfail.log postV3.afm \
+	postV7.afm test-13.pdf test-13.xref test-15.pdf test-15.xref \
+	$(nodist_libluatex_sources) luaimage.* luajitimage.* \
+	$(nodist_xetex_SOURCES) xetex.web xetex-final.ch xetex-web2c \
+	xetex.p xetex.pool xetex-tangle bug73.fmt bug73.log bug73.out \
+	bug73.tex filedump.log filedump.out filedump.tex \
+	$(omegaware_programs:=.c) $(omegaware_programs:=.h) \
+	$(omegaware_programs:=.p) $(omegaware_programs:=-web2c) \
+	ofm2opl.web opl2ofm.web ovf2ovp.web ovp2ovf.web \
+	omegaware/bad*.* omegaware/tests/charwdr.* \
+	omegaware/tests/charwdv.* omegaware/tests/xcheck* \
+	omegaware/tests/xlevel1.* omegaware/tests/xlig*.* ofont*vf \
+	omegaware/tests/xpagenum.* omegaware/tests/xofont* \
+	omegaware/tests/Cherokee.tfm omegaware/tests/OCherokee.ofm \
+	omegaware/tests/OCherokee.opl omegaware/tests/OCherokee.ovf \
+	omegaware/tests/xCherokee.* omegaware/tests/xOCherokee.* \
+	ocftest.* omegaware/tests/xinbmp* omegaware/tests/xoverbmp* \
+	omegaware/tests/xrealnum.* omegaware/tests/xrepeated.* \
+	omegaware/tests/sample*.ofm omegaware/tests/sample*.ovf \
+	omegaware/tests/sample1-h.opl omegaware/tests/xsample*.out \
+	omegaware/tests/shortend.* omegaware/tests/specialhex.ofm \
+	omegaware/tests/specialhex.opl omegaware/tests/specialhex.ovf \
+	omegaware/tests/xspecialhex.* omegaware/tests/yrepeat* \
+	omegaware/tests/*yarabic* $(nodist_aleph_SOURCES) aleph.web \
+	aleph.ch aleph-web2c aleph.p aleph.pool aleph-tangle
 CLEANFILES = $(EXTRA_PROGRAMS) $(EXTRA_LIBRARIES) $(EXTRA_LTLIBRARIES)
 TRIPTRAP_CLEAN = $(am__append_8) $(am__append_18) $(am__append_27) \
 	$(am__append_36) $(am__append_44) $(am__append_60) \
@@ -4308,6 +4309,7 @@ ptex_web_srcs = \
 	tex.web \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch
 
 ptex_ch_srcs = \
@@ -4368,6 +4370,7 @@ eptex_web_srcs = \
 	etexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch \
 	etexdir/tex.ech
 
@@ -4413,6 +4416,7 @@ uptex_web_srcs = \
 	tex.web \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch
 
 uptex_ch_srcs = \
@@ -4478,6 +4482,7 @@ euptex_web_srcs = \
 	etexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch \
 	etexdir/tex.ech
 
@@ -4549,6 +4554,8 @@ pdftex_ch_srcs = \
 	pdftexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken-102.ch \
+	partoken.ch \
 	zlib-fmt.ch \
 	enctexdir/enctex1.ch \
 	enctexdir/enctex-pdftex.ch \
@@ -4563,7 +4570,7 @@ pdftex_ch_srcs = \
 #
 pdftex_tests = pdftexdir/wprob.test pdftexdir/pdftex.test \
   pdftexdir/pdfimage.test pdftexdir/expanded.test \
-  pdftexdir/tests/cnfline.test
+  pdftexdir/tests/cnfline.test pdftexdir/tests/partoken.test
 
 ttf2afm_CPPFLAGS = $(AM_CPPFLAGS) -I$(srcdir)/pdftexdir
 ttf2afm_SOURCES = pdftexdir/ttf2afm.c
@@ -5241,6 +5248,8 @@ xetex_ch_srcs = \
 	xetexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken-102.ch \
+	partoken.ch \
 	$(xetex_ch_synctex) \
 	xetexdir/xetex.ch \
 	$(xetex_post_ch_synctex) \
@@ -21828,7 +21837,7 @@ pdftex.tex: $(srcdir)/pdftexdir/pdftex.web
 
 pdftexdir/wprob.log pdftexdir/pdftex.log \
   pdftexdir/pdfimage.log pdftexdir/expanded.log \
-  pdftexdir/cnfline.log: pdftex$(EXEEXT)
+  pdftexdir/tests/cnfline.log pdftexdir/tests/partoken.log: pdftex$(EXEEXT)
 pdftexdir/ttf2afm.log: ttf2afm$(EXEEXT)
 
 $(pdftosrc_OBJECTS): $(ZLIB_DEPEND) $(LIBPNG_DEPEND) $(XPDF_DEPEND)
diff --git a/Build/source/texk/web2c/NEWS b/Build/source/texk/web2c/NEWS
index 86e75002def..d398bf800f8 100644
--- a/Build/source/texk/web2c/NEWS
+++ b/Build/source/texk/web2c/NEWS
@@ -1,5 +1,10 @@
 This file records noteworthy changes.  (Public domain.)
 
+* for all engines except original TeX:
+- new primitives \partokenname and \partokencontext allow overriding the
+  name of the internally-emitted \par token at blank lines, the end of
+  vboxes, etc.
+
 
 2021 (for TeX Live 2021, 23 March 2021)
 * Knuth 2021 tune-up, with fixes to all of his *.web files. Overview:
diff --git a/Build/source/texk/web2c/enctexdir/enctex-pdftex.ch b/Build/source/texk/web2c/enctexdir/enctex-pdftex.ch
index e1979d892b2..283cd3e84b0 100644
--- a/Build/source/texk/web2c/enctexdir/enctex-pdftex.ch
+++ b/Build/source/texk/web2c/enctexdir/enctex-pdftex.ch
@@ -1,16 +1,16 @@
 % $Id$
 % enctex1.ch is applied first.
 % Then this change is applied for pdfTeX et al.,
-% assuming that tracingstacklevels.ch has been included,
+% assuming that tracingstacklevels.ch plus partoken.ch has been included,
 % as specified in the *.am files.
 
 % encTeX: \mubytein \mubyteout \mubytelog and \specialout
 @x [17.236] l.4954
-@d web2c_int_pars=web2c_int_base+4 {total number of web2c's integer parameters}
+@d web2c_int_pars=web2c_int_base+5 {total number of web2c's integer parameters}
 @y
-@d mubyte_in_code=web2c_int_base+4{if positive then reading mubytes is active}
-@d mubyte_out_code=web2c_int_base+5{if positive then printing mubytes is active}
-@d mubyte_log_code=web2c_int_base+6{if positive then print mubytes to log and terminal}
-@d spec_out_code=web2c_int_base+7 {if positive then print specials by mubytes}
-@d web2c_int_pars=web2c_int_base+8 {total number of web2c's integer parameters}
+@d mubyte_in_code=web2c_int_base+5{if positive then reading mubytes is active}
+@d mubyte_out_code=web2c_int_base+6{if positive then printing mubytes is active}
+@d mubyte_log_code=web2c_int_base+7{if positive then print mubytes to log and terminal}
+@d spec_out_code=web2c_int_base+8 {if positive then print specials by mubytes}
+@d web2c_int_pars=web2c_int_base+9 {total number of web2c's integer parameters}
 @z
diff --git a/Build/source/texk/web2c/eptexdir/am/eptex.am b/Build/source/texk/web2c/eptexdir/am/eptex.am
index 928dd028f60..41e3ccc9a8c 100644
--- a/Build/source/texk/web2c/eptexdir/am/eptex.am
+++ b/Build/source/texk/web2c/eptexdir/am/eptex.am
@@ -55,6 +55,7 @@ eptex_web_srcs = \
 	etexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch \
 	etexdir/tex.ech
 
diff --git a/Build/source/texk/web2c/euptexdir/am/euptex.am b/Build/source/texk/web2c/euptexdir/am/euptex.am
index 597e2e1cd9d..7f163aa6ec1 100644
--- a/Build/source/texk/web2c/euptexdir/am/euptex.am
+++ b/Build/source/texk/web2c/euptexdir/am/euptex.am
@@ -51,6 +51,7 @@ euptex_web_srcs = \
 	etexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch \
 	etexdir/tex.ech
 
diff --git a/Build/source/texk/web2c/partoken-102.ch b/Build/source/texk/web2c/partoken-102.ch
new file mode 100644
index 00000000000..5d50fa2488e
--- /dev/null
+++ b/Build/source/texk/web2c/partoken-102.ch
@@ -0,0 +1,14 @@
+% $Id$
+% Public domain. See partoken.ch (and pdfTeX manual) for main information.
+
+% New internal constant for the main command |\partokenname|.
+% This redefinition of max_command varies, but the rest of the change
+% file works across all engines, so separate it out. This version is
+% used for pdftex and xetex. For the pTeX family, the max_command update
+% is done in the main change files (ptex-base.ch and uptex-m.ch).
+@x
+@d max_command=102 {the largest command code seen at |big_switch|}
+@y
+@d partoken_name=103 {set |par_token| name}
+@d max_command=103 {the largest command code seen at |big_switch|}
+@z
diff --git a/Build/source/texk/web2c/partoken.ch b/Build/source/texk/web2c/partoken.ch
new file mode 100644
index 00000000000..a321c40e9c1
--- /dev/null
+++ b/Build/source/texk/web2c/partoken.ch
@@ -0,0 +1,175 @@
+% $Id$
+% Implementation of |\partokenname| and |\partokencontext| primitives.
+% Public domain. Originally written by Petr Olsak, 2021.
+
+% TeX inserts (let's say) "par-token" at empty lines and in other situations.
+% The primitive meaning of "par-token" is (let's say) "end-paragraph",
+%   but it can be changed by the macro programmer (by |\def\par|, for example).
+% The name of the auto-emitted "par-token" is fixed as |\par| in
+%   classical TeX. 
+% The new |\partokenname| primitive defined here allows changing this cs name.
+% After |\partokenname <control-sequence>| is given, the specified
+%   <control-sequence> will be emitted at empty lines, etc., not |\par|. 
+% Moreover, the given <control-sequence> plays the role of |\par| when
+%   processing not-\long macros ("runaway error").
+% The setting of |\partokenname| is global.
+%
+% The integer register |\partokencontext| allows controlling the places where
+% "par-token" is auto-emitted. The value of |\partokencontext| can be:
+%   0 ... "par-token" is emitted as in classical TeX, i.e. at empty lines,
+%         before \end, \vskip, \hrule, \unvbox, and \halign (if horizontal mode
+%         is current) and in several error recovery situations.
+%         This is the default value.
+%   1 ... "par-token" is emitted as before and at the end of \vbox,
+%         \vtop, and \vcenter if horizontal mode is current here.
+%   2 ... "par-token" is emitted as before and at the end of \noalign,
+%         \vadjust, \output, \insert, and \valign items, if horizontal
+%         mode is current here.
+% Classical TeX does a direct call of the "end-paragraph" routine at the
+% places mentioned in cases 1 and 2.
+% 
+% See the user documentation in the pdftex manual for more,
+% the tests/partoken-test.tex file for basic examples,
+% and the OpTeX format (eventually) for the practical case inspiring
+% this idea.
+
+% New internal constant for integer register |\partokencontext|
+@x
+@d web2c_int_pars=web2c_int_base+4 {total number of web2c's integer parameters}
+@y
+@d partoken_context_code=web2c_int_base+4 {controlling where |partoken| inserted}
+@d web2c_int_pars=web2c_int_base+5 {total number of web2c's integer parameters}
+@z
+
+% Allocation of integer register |\partokencontext|
+@x
+@d tracing_stack_levels==int_par(tracing_stack_levels_code)
+@y
+@d tracing_stack_levels==int_par(tracing_stack_levels_code)
+@d partoken_context==int_par(partoken_context_code)
+@z
+
+% Printing |\show\partokencontext|
+@x
+tracing_stack_levels_code:print_esc("tracingstacklevels");
+@y
+tracing_stack_levels_code:print_esc("tracingstacklevels");
+partoken_context_code:print_esc("partokencontext");
+@z
+
+% Declaration of primitives |\patokenname| and |\partokencontext|
+@x
+  primitive("tracingstacklevels",assign_int,int_base+tracing_stack_levels_code);@/
+@!@:tracing_stack_levels_}{\.{\\tracingstacklevels} primitive@>
+@y
+  primitive("tracingstacklevels",assign_int,int_base+tracing_stack_levels_code);@/
+@!@:tracing_stack_levels_}{\.{\\tracingstacklevels} primitive@>
+  primitive("partokenname",partoken_name,0);@/
+@!@:partoken_name_}{\.{\\partokenname} primitive@>
+  primitive("partokencontext",assign_int,int_base+partoken_context_code);@/
+@!@:partoken_context_}{\.{\\partokencontext} primitive@>
+@z
+
+% Printing |\show\partokenname|
+@x
+vrule: print_esc("vrule");
+@y
+vrule: print_esc("vrule");
+partoken_name: print_esc("partokenname");
+@z
+
+% End of |\vbox| and |\vtop|
+@x
+vbox_group: begin end_graf; package(0);
+  end;
+vtop_group: begin end_graf; package(vtop_code);
+  end;
+@y
+vbox_group: if (partoken_context>0) and (mode=hmode) then 
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end
+  else  begin end_graf; package(0); end;
+vtop_group: if (partoken_context>0) and (mode=hmode) then
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end 
+  else begin end_graf; package(vtop_code); end;
+@z
+
+% End of |\insert| or |\vadjust|
+@x
+insert_group: begin end_graf; q:=split_top_skip; add_glue_ref(q);
+@y
+insert_group: if (partoken_context>1) and (mode=hmode) then
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end 
+  else begin end_graf; q:=split_top_skip; add_glue_ref(q);
+@z
+
+% End of |\output|
+@x
+output_group: @<Resume the page builder...@>;
+@y
+output_group: if (partoken_context>1) and (mode=hmode) then
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end 
+  else @<Resume the page builder...@>;
+@z
+
+% End of item in |\valign|
+@x
+vmode+endv,hmode+endv: do_endv;
+@y
+vmode+endv,hmode+endv: if (partoken_context>1) and (mode=hmode) then
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end
+  else do_endv;
+@z
+
+% End of |\noalign| in |\halign|
+@x
+no_align_group: begin end_graf; unsave; align_peek;
+  end;
+@y
+no_align_group: if (partoken_context>1) and (mode=hmode) then
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end 
+  else begin end_graf; unsave; align_peek;
+  end;
+@z
+
+% End of |\vcenter|
+@x
+vcenter_group: begin end_graf; unsave; save_ptr:=save_ptr-2;
+@y
+vcenter_group: if (partoken_context>0) and (mode=hmode) then 
+  begin
+     back_input; cur_tok := par_token; back_input; token_type := inserted;
+  end else begin end_graf; unsave; save_ptr:=save_ptr-2;
+@z
+
+% Setting new name of the |par-token|
+@x
+any_mode(after_group):begin get_token; save_for_after(cur_tok);
+  end;
+@y
+any_mode(after_group):begin get_token; save_for_after(cur_tok);
+  end;
+any_mode(partoken_name):begin get_token;
+  if cur_cs > 0 then begin
+     par_loc := cur_cs; par_token := cur_tok;
+     end; 
+  end;
+@z
+
+% Undump |par-token| from the format
+@x
+undump(hash_base)(hash_top)(par_loc);
+@y
+undump_int(par_loc);
+@z
diff --git a/Build/source/texk/web2c/pdftexdir/am/pdftex.am b/Build/source/texk/web2c/pdftexdir/am/pdftex.am
index 39dd3179400..07acca7cb7a 100644
--- a/Build/source/texk/web2c/pdftexdir/am/pdftex.am
+++ b/Build/source/texk/web2c/pdftexdir/am/pdftex.am
@@ -71,6 +71,8 @@ pdftex_ch_srcs = \
 	pdftexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken-102.ch \
+	partoken.ch \
 	zlib-fmt.ch \
 	enctexdir/enctex1.ch \
 	enctexdir/enctex-pdftex.ch \
@@ -102,11 +104,11 @@ EXTRA_DIST += \
 #
 pdftex_tests = pdftexdir/wprob.test pdftexdir/pdftex.test \
   pdftexdir/pdfimage.test pdftexdir/expanded.test \
-  pdftexdir/tests/cnfline.test
+  pdftexdir/tests/cnfline.test pdftexdir/tests/partoken.test
 
 pdftexdir/wprob.log pdftexdir/pdftex.log \
   pdftexdir/pdfimage.log pdftexdir/expanded.log \
-  pdftexdir/cnfline.log: pdftex$(EXEEXT)
+  pdftexdir/tests/cnfline.log pdftexdir/tests/partoken.log: pdftex$(EXEEXT)
 
 EXTRA_DIST += $(pdftex_tests)
 
@@ -130,3 +132,7 @@ DISTCLEANFILES += expanded.log
 ## cnfline.test
 EXTRA_DIST += tests/cnfline.tex
 DISTCLEANFILES += cnfline.log
+
+## partoken.test
+EXTRA_DIST += tests/partoken-ok.tex tests/partoken-xfail.tex
+DISTCLEANFILES += partoken-ok.log partoken-xfail.log
diff --git a/Build/source/texk/web2c/pdftexdir/tests/cnfline.test b/Build/source/texk/web2c/pdftexdir/tests/cnfline.test
index b295b81ea90..6bc2b1b8a53 100755
--- a/Build/source/texk/web2c/pdftexdir/tests/cnfline.test
+++ b/Build/source/texk/web2c/pdftexdir/tests/cnfline.test
@@ -8,7 +8,7 @@ LC_ALL=C; export LC_ALL;  LANGUAGE=C; export LANGUAGE
 TEXMFCNF=$srcdir/../kpathsea; export TEXMFCNF
 TEXINPUTS=$srcdir/pdftexdir/tests; export TEXINPUTS
 
-./pdftex -ini --interaction=batchmode \
+./pdftex -ini --interaction=nonstopmode \
   --cnf-line=max_print_line=500 cnfline.tex || exit $?
 
 if grep 'those hyphens are' cnfline.log >/dev/null; then
diff --git a/Build/source/texk/web2c/pdftexdir/tests/partoken.test b/Build/source/texk/web2c/pdftexdir/tests/partoken.test
new file mode 100755
index 00000000000..8c9af4d9f4a
--- /dev/null
+++ b/Build/source/texk/web2c/pdftexdir/tests/partoken.test
@@ -0,0 +1,25 @@
+#! /bin/sh -vx
+# $Id$
+# Public domain. Originally written by Karl Berry, 2021.
+# Check for new \partokenname primitives.
+
+LC_ALL=C; export LC_ALL;  LANGUAGE=C; export LANGUAGE
+
+# running in Build/source/Work/texk/web2c
+# srcdir  is Build/source/texk/web2c
+: ${srcdir=../../../texk/web2c}
+TEXMFCNF=$srcdir/../kpathsea; export TEXMFCNF
+TEXMFDOTDIR=$srcdir/tests; export TEXMFDOTDIR # both fonts and inputs
+
+: ${testengine=pdftex}
+
+# should succeed. nonstopmode instead of batchmode so normal errors are
+# printed on the terminal, and thus show up in test-suite.log.
+if ./$testengine -ini --interaction=nonstopmode partoken-ok.tex; then :; else
+  exit 1; fi
+
+# should fail
+if ./$testengine -ini --interaction=nonstopmode partoken-xfail.tex; then
+  exit 1; fi
+
+exit 0
diff --git a/Build/source/texk/web2c/ptexdir/am/ptex.am b/Build/source/texk/web2c/ptexdir/am/ptex.am
index 800af466b91..2714e21449b 100644
--- a/Build/source/texk/web2c/ptexdir/am/ptex.am
+++ b/Build/source/texk/web2c/ptexdir/am/ptex.am
@@ -1,3 +1,4 @@
+## $Id$
 ## texk/web2c/ptexdir/am/ptex.am: Makefile fragment for pTeX.
 ##
 ## Copyright 2015-2021 Karl Berry <tex-live@tug.org>
@@ -75,6 +76,7 @@ ptex_web_srcs = \
 	tex.web \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch
 
 # Generate ptex.ch
diff --git a/Build/source/texk/web2c/ptexdir/ptex-base.ch b/Build/source/texk/web2c/ptexdir/ptex-base.ch
index 6613cb295dc..4825c40fb6d 100644
--- a/Build/source/texk/web2c/ptexdir/ptex-base.ch
+++ b/Build/source/texk/web2c/ptexdir/ptex-base.ch
@@ -1,3 +1,4 @@
+% $Id$
 % This is a change file for pTeX
 % By Sadayuki Tanaka and ASCII MEDIA WORKS.
 %
@@ -1009,7 +1010,8 @@ kern_node,math_node,penalty_node: begin r:=get_node(small_node_size);
 @d set_interaction=hyph_data+1 {define level of interaction ( \.{\\batchmode}, etc.~)}
 @d set_auto_spacing=set_interaction+1 {set auto spacing mode
   ( \.{\\autospacing}, \.{\\noautospacing}, \.{\\autoxspacing}, \.{\\noautoxspacing} )}
-@d max_command=set_auto_spacing {the largest command code seen at |big_switch|}
+@d partoken_name=set_auto_spacing+1 {set |par_token| name}
+@d max_command=partoken_name {the largest command code seen at |big_switch|}
 @z
 
 @x [16.212] l.4437 - pTeX: last_jchr, direction, adjust direction
@@ -5340,8 +5342,6 @@ push_nest; mode:=-vmode; direction:=adjust_dir; prev_depth:=ignore_depth;
 @z
 
 @x [47.1100] l.21189 - pTeX: free box node, ins_dir
-insert_group: begin end_graf; q:=split_top_skip; add_glue_ref(q);
-  d:=split_max_depth; f:=floating_penalty; unsave; decr(save_ptr);
   {now |saved(0)| is the insertion number, or 255 for |vadjust|}
   p:=vpack(link(head),natural); pop_nest;
   if saved(0)<255 then
@@ -5359,8 +5359,6 @@ insert_group: begin end_graf; q:=split_top_skip; add_glue_ref(q);
   if nest_ptr=0 then build_page;
   end;
 @y
-insert_group: begin end_graf; q:=split_top_skip; add_glue_ref(q);
-  d:=split_max_depth; f:=floating_penalty; unsave; decr(save_ptr);
   {now |saved(0)| is the insertion number, or 255 for |vadjust|}
   p:=vpack(link(head),natural); set_box_dir(p)(direction); pop_nest;
   if saved(0)<255 then
@@ -5938,13 +5936,11 @@ scan_math(nucleus(tail),kcode_noad(tail));
 @z
 
 @x [48.1164] l.22790 - pTeX: vcenter : dir
-vcenter_group: begin end_graf; unsave; save_ptr:=save_ptr-2;
   p:=vpack(link(head),saved(1),saved(0)); pop_nest;
   tail_append(new_noad); type(tail):=vcenter_noad;
   math_type(nucleus(tail)):=sub_box; info(nucleus(tail)):=p;
   end;
 @y
-vcenter_group: begin end_graf; unsave; save_ptr:=save_ptr-2;
   p:=vpack(link(head),saved(1),saved(0));
   set_box_dir(p)(direction); pop_nest;
   if abs(box_dir(p))<>abs(direction) then p:=new_dir_node(p,abs(direction));
diff --git a/Build/source/texk/web2c/tests/cmr10.pfb b/Build/source/texk/web2c/tests/cmr10.pfb
new file mode 100644
index 00000000000..fa8c833d374
--- /dev/null
+++ b/Build/source/texk/web2c/tests/cmr10.pfb
diff --git a/Build/source/texk/web2c/tests/partoken-ok.tex b/Build/source/texk/web2c/tests/partoken-ok.tex
new file mode 100644
index 00000000000..329505db804
--- /dev/null
+++ b/Build/source/texk/web2c/tests/partoken-ok.tex
@@ -0,0 +1,48 @@
+% $Id$
+% Public domain. Originally written by Petr Olsak, 2021.
+% Testing \partokenname and \partokencontext primitives: successful tests.
+% 
+% We want to test in iniTeX state, use "pdftex -ini partoken-test.tex" 
+% Generating dvi output.
+%
+\catcode`\{=1 \catcode`\}=2 \catcode`\#=6
+\hsize=4in \vsize=4in \baselineskip=12pt
+\font\f=cmr10 \f % we want to typeset some text
+
+% Seting different partoken name
+\catcode`\_=11
+\partokenname \_par
+\let \_par = \par
+\def\par{hello}
+% This empty line emits \_par (i.e., no output), no "hello":
+
+% Not-\long macro accepts \par in its argument, but does not accept \_par
+% (emitted here from blank line):
+\def\macro#1{}
+\macro{long test: \par} % without error
+% see partoken-xfail.tex for failing test.
+%
+\let \_endgraf=\_par
+\def \_par{PAR-TOKEN\_endgraf}
+% This empty line prints PAR-TOKEN (plus underfull \hbox
+%   due to \parfillskip=0pt):
+
+% This \vskip prints Test PAR-TOKEN (plus underfull \hbox warning):
+Test \vskip1pt \relax
+%
+% This end-of-\vbox prints 0: Test text. (standard behavior):
+\vbox{\the\partokencontext: Test text.}
+%
+\partokencontext=1
+% This prints 1: Test text.PAR-TOKEN (\_par called at end of \vbox):
+\vbox{\the\partokencontext: Test text.}
+%
+% This prints 1: no-align text (standard behavior):
+\halign{#\cr \noalign{\the\partokencontext: no-align text}}
+%
+\partokencontext=2
+% This prints: 2: no-align textPAR-TOKEN (\_par called at end of \noalign):
+\halign{#\cr \noalign{\the\partokencontext: no-align text}}
+%
+% no blank line to elide final PAR-TOKEN.
+\end
diff --git a/Build/source/texk/web2c/tests/partoken-xfail.tex b/Build/source/texk/web2c/tests/partoken-xfail.tex
new file mode 100644
index 00000000000..6d2208992e5
--- /dev/null
+++ b/Build/source/texk/web2c/tests/partoken-xfail.tex
@@ -0,0 +1,31 @@
+% $Id$
+% Public domain. Originally written by Petr Olsak, 2021.
+% Testing \partokenname and \partokencontext primitives: an expected failure.
+% 
+% We want to test in iniTeX state, use "pdftex -ini partoken-test.tex" 
+%
+\catcode`\{=1 \catcode`\}=2 \catcode`\#=6
+\hsize=4in \vsize=4in \baselineskip=12pt
+\font\f=cmr10 \f
+\nonstopmode
+
+% Seting different partoken name
+\catcode`\_=11
+\let \_par = \par
+\partokenname \_par
+\def\par{hello}
+
+% Not-\long macro does not accept \_par
+% (emitted here from blank line):
+\def\macro#1{}
+% see partoken-ok.tex for successful test.
+\macro{failing long test:
+
+} % generates error: 
+% ! Paragraph ended before \macro was complete.
+% <to be read again> 
+%                    \_par 
+% l.20
+% plus dependent error: Too many }'s.
+
+\end
diff --git a/Build/source/texk/web2c/uptexdir/am/uptex.am b/Build/source/texk/web2c/uptexdir/am/uptex.am
index 8db4c4b8dcf..826dc260f00 100644
--- a/Build/source/texk/web2c/uptexdir/am/uptex.am
+++ b/Build/source/texk/web2c/uptexdir/am/uptex.am
@@ -1,3 +1,4 @@
+## $Id$
 ## texk/web2c/uptexdir/am/uptex.am: Makefile fragment for upTeX.
 ##
 ## Copyright 2016-2021 Karl Berry <tex-live@tug.org>
@@ -68,6 +69,7 @@ uptex_web_srcs = \
 	tex.web \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken.ch \
 	zlib-fmt.ch
 
 # Generate uptex.ch
diff --git a/Build/source/texk/web2c/uptexdir/uptex-m.ch b/Build/source/texk/web2c/uptexdir/uptex-m.ch
index 4e806e6d94b..d3df0324517 100644
--- a/Build/source/texk/web2c/uptexdir/uptex-m.ch
+++ b/Build/source/texk/web2c/uptexdir/uptex-m.ch
@@ -1,3 +1,4 @@
+% $Id$
 % This is a change file for upTeX u1.27
 % By Takuji Tanaka.
 %
@@ -184,9 +185,9 @@ if (kcode_pos=1)or((kcode_pos>=@'11)and(kcode_pos<=@'12))
 @z
 
 @x
-@d max_command=set_auto_spacing {the largest command code seen at |big_switch|}
+@d max_command=partoken_name {the largest command code seen at |big_switch|}
 @y
-@d set_enable_cjk_token=set_auto_spacing+1 {set cjk mode ( \.{\\enablecjktoken}, \.{\\disablecjktoken}, \.{\\forcecjktoken} )}
+@d set_enable_cjk_token=partoken_name+1 {set cjk mode ( \.{\\enablecjktoken}, \.{\\disablecjktoken}, \.{\\forcecjktoken} )}
 @d max_command=set_enable_cjk_token {the largest command code seen at |big_switch|}
 @z
 
diff --git a/Build/source/texk/web2c/xetexdir/am/xetex.am b/Build/source/texk/web2c/xetexdir/am/xetex.am
index ecbdc8ca31f..f12ed3b3e59 100644
--- a/Build/source/texk/web2c/xetexdir/am/xetex.am
+++ b/Build/source/texk/web2c/xetexdir/am/xetex.am
@@ -114,6 +114,8 @@ xetex_ch_srcs = \
 	xetexdir/tex.ch0 \
 	tex.ch \
 	tracingstacklevels.ch \
+	partoken-102.ch \
+	partoken.ch \
 	$(xetex_ch_synctex) \
 	xetexdir/xetex.ch \
 	$(xetex_post_ch_synctex) \