summaryrefslogtreecommitdiff
path: root/Build/source/texk/web2c/ptexdir
diff options
context:
space:
mode:
authorHironobu Yamashita <h.y.acetaminophen@gmail.com>2022-01-22 17:03:22 +0000
committerHironobu Yamashita <h.y.acetaminophen@gmail.com>2022-01-22 17:03:22 +0000
commit76e833603f3eb350549d8b472d5b0c1ebae21713 (patch)
tree0adce490f86927e7a81a2f05330b9590bf80663d /Build/source/texk/web2c/ptexdir
parent87e77a4b4b35459b0bc1aa06f48f2cf0db19ae9e (diff)
[e][u]ptex: Distinguish 8-bit characters and Japanese characters (H. Kitagawa et al.)
For better support of LaTeX3 (expl3). More details in TUGboat 41(2):329--334, 2020. git-svn-id: svn://tug.org/texlive/trunk@61692 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/web2c/ptexdir')
-rw-r--r--Build/source/texk/web2c/ptexdir/ChangeLog11
-rw-r--r--Build/source/texk/web2c/ptexdir/kanji.h4
-rw-r--r--Build/source/texk/web2c/ptexdir/ptex-base.ch424
-rw-r--r--Build/source/texk/web2c/ptexdir/ptex.defines3
-rw-r--r--Build/source/texk/web2c/ptexdir/ptex_version.h2
-rw-r--r--Build/source/texk/web2c/ptexdir/tests/filename_test.sh45
-rw-r--r--Build/source/texk/web2c/ptexdir/tests/printkanji.tex168
7 files changed, 621 insertions, 36 deletions
diff --git a/Build/source/texk/web2c/ptexdir/ChangeLog b/Build/source/texk/web2c/ptexdir/ChangeLog
index c77ab5bbd3f..ce2506f302e 100644
--- a/Build/source/texk/web2c/ptexdir/ChangeLog
+++ b/Build/source/texk/web2c/ptexdir/ChangeLog
@@ -1,3 +1,14 @@
+2022-01-22 Hironori Kitagawa <h_kitagawa2001@yahoo.co.jp>
+
+ * ptex-base.ch: Change type of str_pool to unsigned short.
+ We use only 0--511, and use flag 0x100 for Japanese char.
+ More details in TUGboat 41(2):329--334, 2020.
+ * kanji.h: Adapt to arguments of input_line2() in ptexenc-1.4.0
+ (for pBibTeX).
+ * ptex.defines: Add multistrlen{short,filename}, fromBUFFshort.
+ * ptex_version.h: Version p4.0.0.
+ * tests/filename_test.sh, tests/printkanji.tex: Added.
+
2022-01-10 Karl Berry <karl@freefriends.org>
* am/ptex.am: do not silence tangle-sh.
diff --git a/Build/source/texk/web2c/ptexdir/kanji.h b/Build/source/texk/web2c/ptexdir/kanji.h
index 5be133ecad4..e0edeb2ed70 100644
--- a/Build/source/texk/web2c/ptexdir/kanji.h
+++ b/Build/source/texk/web2c/ptexdir/kanji.h
@@ -65,9 +65,9 @@ extern void init_default_kanji (const_string file_str, const_string internal_str
#endif /* !PRESERVE_FPUTS */
#ifdef PBIBTEX
-#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,pos,size,ptr)
+#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,NULL,pos,size,ptr)
#else
-#define inputline2(fp,buff,pos,size) input_line2(fp,buff,pos,size,NULL)
+#define inputline2(fp,buff,pos,size) input_line2(fp,buff,NULL,pos,size,NULL)
#endif
extern void dump_kanji (gzFile fp);
diff --git a/Build/source/texk/web2c/ptexdir/ptex-base.ch b/Build/source/texk/web2c/ptexdir/ptex-base.ch
index e3b15e30d58..15a76eaeca6 100644
--- a/Build/source/texk/web2c/ptexdir/ptex-base.ch
+++ b/Build/source/texk/web2c/ptexdir/ptex-base.ch
@@ -67,6 +67,10 @@
% based on TeX 3.141592653 (for TL21).
% (2021-06-25) HY pTeX p3.9.1 Various fixes.
% (2021-06-20) HK pTeX p3.10.0 Add \ucs and \toucs.
+% (2022-01-22) HK pTeX p4.0.0 Distinguish 8-bit characters and Japanese characters
+% for better support of LaTeX3 (expl3).
+% Requires ptexenc version 1.4.0.
+% More details in TUGboat 41(2):329--334, 2020.
@x
% Here is TeX material that gets inserted after \input webmac
@@ -79,10 +83,10 @@
@d banner==TeX_banner
@d banner_k==TeX_banner_k
@y
-@d pTeX_version=3
-@d pTeX_minor_version=10
+@d pTeX_version=4
+@d pTeX_minor_version=0
@d pTeX_revision==".0"
-@d pTeX_version_string=='-p3.10.0' {current \pTeX\ version}
+@d pTeX_version_string=='-p4.0.0' {current \pTeX\ version}
@#
@d pTeX_banner=='This is pTeX, Version 3.141592653',pTeX_version_string
@d pTeX_banner_k==pTeX_banner
@@ -97,6 +101,21 @@
@y
@!ASCII_code=0..255; {eight-bit numbers}
@!KANJI_code=0..65535; {sixteen-bit numbers}
+@!ext_ASCII_code=0..32768; { only use 0--511 }
+@z
+
+@x pTeX: xchr
+xchr: array [ASCII_code] of text_char;
+ { specifies conversion of output characters }
+@y
+xchr: array [ext_ASCII_code] of ext_ASCII_code;
+ { specifies conversion of output characters }
+@z
+
+@x pTeX: xchr
+for i:=@'177 to @'377 do xchr[i]:=i;
+@y
+for i:=@'177 to @'777 do xchr[i]:=i;
@z
@x [3.??] l.870 - pTeX:
@@ -114,6 +133,36 @@
@ Kanji code handling.
@z
+@x [3.??] pTeX
+@<Glob...@>=
+@!buffer:^ASCII_code; {lines of characters being read}
+@y
+In \pTeX, we use another array |buffer2[]| to indicate which byte
+is a part of a Japanese character.
+|buffer2[]| is initialized to zero in reading one line from a file
+(|input_ln|). |buffer2[i]| is set to one when |buffer[i]| is known
+to be a part of a Japanese character, in |get_next| routine.
+
+@<Glob...@>=
+@!buffer:^ASCII_code; {lines of characters being read}
+@!buffer2:^ASCII_code;
+@z
+
+@x [4]
+@!packed_ASCII_code = 0..255; {elements of |str_pool| array}
+@y
+@!packed_ASCII_code = 0..32768; {elements of |str_pool| array}
+ { 256..511 are used by Japanese characters }
+@z
+
+@x [4] pTeX: str_eq_buf
+while j<str_start[s+1] do
+ begin if so(str_pool[j])<>buffer[k] then
+@y
+while j<str_start[s+1] do
+ begin if so(str_pool[j])<>buffer2[k]*@"100+buffer[k] then
+@z
+
@x [4.47] l.1325 - pTeX:
@!init function get_strings_started:boolean; {initializes the string pool,
but returns |false| if something goes wrong}
@@ -126,19 +175,11 @@ label done,exit;
var k,@!l:KANJI_code; {small indices or counters}
@z
-@x [4.49] l.1384 - pTeX:
-@<Character |k| cannot be printed@>=
- (k<" ")or(k>"~")
-@y
-@<Character |k| cannot be printed@>=
- not (ismultiprn(k) or xprn[k])
-@z
-
@x [5.54] l.1514 - pTeX: Global variables
@!trick_buf:array[0..ssup_error_line] of ASCII_code; {circular buffer for
pseudoprinting}
@y
-@!trick_buf:array[0..ssup_error_line] of ASCII_code; {circular buffer for
+@!trick_buf:array[0..ssup_error_line] of ext_ASCII_code; {circular buffer for
pseudoprinting}
@!trick_buf2:array[0..ssup_error_line] of 0..2; {pTeX: buffer for KANJI}
@!kcode_pos: 0..2; {pTeX: denotes whether first byte or second byte of KANJI}
@@ -212,14 +253,16 @@ term_only: begin wterm(xchr[s]); incr(term_offset);
no_print: do_nothing;
pseudo: if tally<trick_count then trick_buf[tally mod error_line]:=s;
@y
-procedure print_char(@!s:ASCII_code); {prints a single character}
+procedure print_char(@!s:ext_ASCII_code); {prints a single character}
label exit; {label is not used but nonetheless kept (for other changes?)}
begin if @<Character |s| is the current new-line character@> then
if selector<pseudo then
begin print_ln; return;
end;
-if kcode_pos=1 then kcode_pos:=2
-else if iskanji1(xchr[s]) then
+if s>@"1FF then s:=s mod 256;
+if s<256 then kcode_pos:=0
+else if kcode_pos=1 then kcode_pos:=2
+else if iskanji1(xchr[s-256]) then
begin kcode_pos:=1;
if (selector=term_and_log)or(selector=log_only) then
if file_offset>=max_print_line-1 then
@@ -299,6 +342,7 @@ else if s<256 then
if selector<pseudo then
begin print_ln; return;
end;
+ if xprn[s] then begin print_char(s); return; end;
nl:=new_line_char; new_line_char:=-1;
{temporarily disable new-line character}
j:=str_start[s];
@@ -314,6 +358,64 @@ while j<str_start[s+1] do
exit:end;
@z
+@x
+procedure slow_print(@!s:integer); {prints string |s|}
+var j:pool_pointer; {current character code position}
+begin if (s>=str_ptr) or (s<256) then print(s)
+else begin j:=str_start[s];
+ while j<str_start[s+1] do
+ begin print(so(str_pool[j])); incr(j);
+ end;
+ end;
+end;
+@y
+procedure slow_print(@!s:integer); {prints string |s|}
+var j:pool_pointer; {current character code position}
+c:integer;
+begin if (s>=str_ptr) or (s<256) then print(s)
+else begin j:=str_start[s];
+ while j<str_start[s+1] do
+ begin c:=so(str_pool[j]);
+ if c>=@"100 then print_char(c) else print(c); incr(j);
+ end;
+ end;
+end;
+
+procedure slow_print_filename(@!s:integer);
+ {prints string |s| which represents filename, without code conversion}
+var i,j,l:pool_pointer; p:integer;
+begin if (s>=str_ptr) or (s<256) then print(s)
+else begin i:=str_start[s]; l:=str_start[s+1];
+ while i<l do begin
+ p:=multistrlenshort(str_pool, l, i);
+ if p<>1 then
+ begin for j:=i to i+p-1 do print_char(@"100+(so(str_pool[j]) mod @"100));
+ i:=i+p; end
+ else begin print(so(str_pool[i]) mod @"100); incr(i); end;
+ end;
+ end;
+end;
+
+procedure print_quoted(@!s:integer);
+ {prints string |s| which represents filename,
+ omitting quotes and with code conversion}
+var i,l:pool_pointer; j,p:integer;
+begin if s<>0 then begin
+ i:=str_start[s]; l:=str_start[s+1];
+ while i<l do begin
+ p:=multistrlenshort(str_pool, l, i);
+ if p<>1 then begin
+ for j:=i to i+p-1 do print_char(@"100+(so(str_pool[j]) mod @"100));
+ i:=i+p; end
+ else begin
+ if so(str_pool[i])<>"""" then print(so(str_pool[i]) mod @"100);
+ incr(i); end;
+ end;
+ end;
+end;
+
+@z
+
@x [5.61] l.1656 - pTeX:
@<Initialize the output...@>=
if src_specials_p or file_line_error_style_p or parse_first_line_p then
@@ -341,6 +443,18 @@ else
@ Old versions of \TeX\ needed a procedure called |print_ASCII| whose function
@z
+@x [5.??] - pTeX: term_input
+@p procedure term_input; {gets a line from the terminal}
+@y
+@p procedure@?print_unread_buffer_with_ptenc; forward;@t\2@>@/
+procedure term_input; {gets a line from the terminal}
+@z
+@x [5.??] - pTeX: term_input
+if last<>first then for k:=first to last-1 do print(buffer[k]);
+@y
+if last<>first then print_unread_buffer_with_ptenc(first,last);
+@z
+
@x
@d max_halfword==@"FFFFFFF {largest allowable value in a |halfword|}
@y
@@ -1452,14 +1566,20 @@ end;
tats
@z
-@x [17.???] l.???? - pTeX multibyte control symbol
+@x [18] buffer2
+for k:=j to j+l-1 do append_char(buffer[k]);
+@y
+for k:=j to j+l-1 do append_char(buffer2[k]*@"100+buffer[k]);
+@z
+
+@x [18.???] l.???? - pTeX multibyte control symbol
procedure print_cs(@!p:integer); {prints a purported control sequence}
@y
procedure print_cs(@!p:integer); {prints a purported control sequence}
var j, l:pool_pointer; @!cat:0..max_char_code;
@z
-@x
+@x [18.???]
else begin print_esc(text(p));
print_char(" ");
end;
@@ -1467,8 +1587,8 @@ else begin print_esc(text(p));
else begin l:=text(p);
print_esc(l); j:=str_start[l]; l:=str_start[l+1];
if l>j+1 then begin
- if l-j=multistrlen(ustringcast(str_pool), l, j) then
- begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(str_pool), l, j)));
+ if (str_pool[j]>=@"100)and(l-j=multistrlenshort(str_pool, l, j)) then
+ begin cat:=kcat_code(kcatcodekey(fromBUFFshort(str_pool, l, j)));
if (cat<>other_kchar) then print_char(" ");
end
else print_char(" "); end
@@ -1476,6 +1596,13 @@ else begin l:=text(p);
end;
@z
+@x [18.???] pTeX: ensure buffer2[]=0 in primitive
+ for j:=0 to l-1 do buffer[j]:=so(str_pool[k+j]);
+@y
+ for j:=0 to l-1 do begin
+ buffer[j]:=Lo(so(str_pool[k+j])); buffer2[j]:=Hi(so(str_pool[k+j])); end;
+@z
+
@x [18.265] l.5903 - pTeX: \jfont \tfont
primitive("font",def_font,0);@/
@!@:font_}{\.{\\font} primitive@>
@@ -1634,6 +1761,27 @@ if m+n<=error_line then p:=first_count+m else p:=first_count+(error_line-n-3);
if trick_buf2[(p-1) mod error_line]=1 then p:=p-1;
@z
+@x pTeX: buffer
+if j>0 then for i:=start to j-1 do
+ begin if i=loc then set_trick_count;
+ print(buffer[i]);
+ end
+@y
+if j>0 then begin
+ i:=start;
+ if (loc<=j-1)and(start<=loc) then begin
+ for i:=start to loc-1 do
+ if buffer2[i]>0 then
+ print_char(@"100*buffer2[i]+buffer[i]) else print(buffer[i]);
+ set_trick_count; print_unread_buffer_with_ptenc(loc,j);
+ end
+ else
+ for i:=start to j-1 do
+ if buffer2[i]>0 then
+ print_char(@"100*buffer2[i]+buffer[i]) else print(buffer[i]);
+ end
+@z
+
@x [22.319] l.7157 - pTeX: adjust kanji code token
@ @<Pseudoprint the token list@>=
begin_pseudoprint;
@@ -1660,6 +1808,13 @@ else show_token_list(link(start),loc,100000); {avoid reference count}
done1:
@z
+@x [23.???] pTeX: init the input routines
+first:=buf_size; repeat buffer[first]:=0; decr(first); until first=0;
+@y
+first:=buf_size; repeat buffer[first]:=0; buffer2[first]:=0; decr(first); until first=0;
+@z
+
+
@x [24.341] l.7479 - pTeX: set last_chr
@!cat:0..max_char_code; {|cat_code(cur_chr)|, usually}
@y
@@ -1681,6 +1836,8 @@ begin switch: if loc<=limit then {current line not yet finished}
if multistrlen(ustringcast(buffer), limit+1, loc-1)=2 then
begin cur_chr:=fromBUFF(ustringcast(buffer), limit+1, loc-1);
cur_cmd:=kcat_code(kcatcodekey(cur_chr));
+ for l:=loc-1 to loc-2+multistrlen(ustringcast(buffer), limit+1, loc-1) do
+ buffer2[l]:=1;
incr(loc);
end
else reswitch: cur_cmd:=cat_code(cur_chr);
@@ -1769,7 +1926,10 @@ end
begin if loc>limit then cur_cs:=null_cs {|state| is irrelevant in this case}
else begin k:=loc; cur_chr:=buffer[k]; incr(k);
if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then
- begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k);
+ begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1)));
+ for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do
+ buffer2[l]:=1;
+ incr(k);
end
else cat:=cat_code(cur_chr);
start_cs:
@@ -1832,10 +1992,10 @@ begin if buffer[k]=cur_chr then @+if cat=sup_mark then @+if k<limit then
end
else if c<@'100 then buffer[k-1]:=c+@'100
else buffer[k-1]:=c-@'100;
- limit:=limit-d; first:=first-d;
+ buffer2[k-1]:=0; limit:=limit-d; first:=first-d;
l:=k; cur_chr:=buffer[k-1]; cat:=cat_code(cur_chr);
while l<=limit do
- begin buffer[l]:=buffer[l+d]; incr(l);
+ begin buffer[l]:=buffer[l+d]; buffer2[l]:=buffer2[l+d]; incr(l);
end;
goto start_cs;
end;
@@ -1858,7 +2018,10 @@ end
@ @<Scan ahead in the buffer...@>=
begin repeat cur_chr:=buffer[k]; incr(k);
if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then
- begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k);
+ begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1)));
+ for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do
+ buffer2[l]:=1;
+ incr(k);
end
else cat:=cat_code(cur_chr);
while (buffer[k]=cur_chr)and(cat=sup_mark)and(k<limit) do
@@ -1874,11 +2037,11 @@ begin repeat cur_chr:=buffer[k]; incr(k);
else cur_chr:=c-@'100;
cat:=cat_code(cur_chr);
if (cat=letter)or(cat=sup_mark) then
- begin buffer[k-1]:=cur_chr;
+ begin buffer[k-1]:=cur_chr; buffer2[k-1]:=0;
limit:=limit-d; first:=first-d;
l:=k;
while l<=limit do
- begin buffer[l]:=buffer[l+d]; incr(l);
+ begin buffer[l]:=buffer[l+d]; buffer2[l]:=buffer2[l+d]; incr(l);
end;
end;
end;
@@ -1951,6 +2114,23 @@ else begin {we are done with this token list}
end
@z
+@x [24] pTeX: firm_up_the_line
+ if start<limit then for k:=start to limit-1 do print(buffer[k]);
+ first:=limit; prompt_input("=>"); {wait for user response}
+@.=>@>
+ if last>first then
+ begin for k:=first to last-1 do {move line down in buffer}
+ buffer[k+start-first]:=buffer[k];
+@y
+ if start<limit then for k:=start to limit-1 do
+ if buffer2[k]>=@"100 then print_char(buffer[k]) else print(buffer[k]);
+ first:=limit; prompt_input("=>"); {wait for user response}
+@.=>@>
+ if last>first then
+ begin for k:=first to last-1 do {move line down in buffer}
+ begin buffer[k+start-first]:=buffer[k]; buffer2[k+start-first]:=buffer2[k]; end;
+@z
+
@x [24.365] l.7935 - pTeX: get_token
@p procedure get_token; {sets |cur_cmd|, |cur_chr|, |cur_tok|}
begin no_new_control_sequence:=false; get_next; no_new_control_sequence:=true;
@@ -1982,6 +2162,12 @@ while p<>null do
end;
buffer[j]:=info(p) mod @'400; incr(j); p:=link(p);
end;
+if j>first+1 then
+ begin no_new_control_sequence:=false; cur_cs:=id_lookup(first,j-first);
+ no_new_control_sequence:=true;
+ end
+else if j=first then cur_cs:=null_cs {the list is empty}
+else cur_cs:=single_base+buffer[first] {the list has length one}
@y
@ @<Look up the characters of list |r| in the hash table...@>=
j:=first; p:=link(r);
@@ -1993,10 +2179,17 @@ while p<>null do
@:TeX capacity exceeded buffer size}{\quad buffer size@>
end;
if check_kanji(info(p)) then {|wchar_token|}
- begin buffer[j]:=Hi(info(p)); incr(j);
- end;
+ begin buffer[j]:=Hi(info(p)); buffer2[j]:=1; incr(j); buffer2[j]:=1;
+ end
+ else buffer2[j]:=0;
buffer[j]:=Lo(info(p)); incr(j); p:=link(p);
end;
+if j>first+1 then
+ begin no_new_control_sequence:=false; cur_cs:=id_lookup(first,j-first);
+ no_new_control_sequence:=true;
+ end
+else if j=first then cur_cs:=null_cs {the list is empty}
+else cur_cs:=single_base+buffer[first] {the list has length one}
@z
@x [25.380] l.8221 - pTeX: get_x_token
@@ -2363,8 +2556,8 @@ help6("Dimensions can be in units of em, ex, zw, zh, in, pt, pc,")@/
if t=" " then t:=space_token
else t:=other_token+t;
@y
- if multistrlen(ustringcast(str_pool), pool_ptr, k)=2 then
- begin t:=fromBUFF(ustringcast(str_pool), pool_ptr, k); incr(k);
+ if t>=@"100 then
+ begin t:=fromBUFFshort(str_pool, pool_ptr, k); incr(k);
end
else if t=" " then t:=space_token
else t:=other_token+t;
@@ -2651,6 +2844,28 @@ else begin str_room(1); append_char(c); {contribute |c| to the current string}
end;
@z
+@x [29.518] - print_quoted in pTeX is already defined
+@d print_quoted(#) == {print string |#|, omitting quotes}
+if #<>0 then
+ for j:=str_start[#] to str_start[#+1]-1 do
+ if so(str_pool[j])<>"""" then
+ print(so(str_pool[j]))
+
+@y
+@z
+
+@x
+@d append_to_name(#)==begin c:=#; if not (c="""") then begin incr(k);
+ if k<=file_name_size then name_of_file[k]:=xchr[c];
+ end end
+@y
+@d append_to_name(#)==begin if (#)>=@"100 then c:=(#)-@"100 else c:=#;
+ { Since the type of |c| is |ASCII_code|, above if-statement might not be needed }
+ if not (c="""") then begin incr(k);
+ if k<=file_name_size then name_of_file[k]:=xchr[c];
+ end end
+@z
+
@x [29.526] l.10668 - pTeX: scan file name
loop@+begin if (cur_cmd>other_char)or(cur_chr>255) then {not a character}
begin back_input; goto done;
@@ -2669,8 +2884,8 @@ skip_mode:=false;
loop@+begin
if (cur_cmd=kanji)or(cur_cmd=kana)or(cur_cmd=other_kchar) then {is kanji}
begin str_room(2);
- append_char(Hi(cur_chr)); {kanji upper byte}
- append_char(Lo(cur_chr)); {kanji lower byte}
+ append_char(@"100+Hi(cur_chr)); {kanji upper byte}
+ append_char(@"100+Lo(cur_chr)); {kanji lower byte}
end
else if (cur_cmd>other_char)or(cur_chr>255) then {not an alphabet}
begin back_input; goto done;
@@ -2686,6 +2901,15 @@ done: end_name; name_in_progress:=false;
skip_mode:=true;
@z
+@x [29.???] open_log_file
+if buffer[l]=end_line_char then decr(l);
+for k:=1 to l do print(buffer[k]);
+print_ln; {now the transcript file contains the first line of input}
+@y
+if buffer[l]=end_line_char then decr(l); print_unread_buffer_with_ptenc(1,l+1);
+print_ln; {now the transcript file contains the first line of input}
+@z
+
@x [29.536] l.10834 - pTeX:
begin
if src_specials_p or file_line_error_style_p or parse_first_line_p
@@ -2705,6 +2929,14 @@ else
wlog(')');
@z
+@x [29.???] pTeX - start_input
+print_char("("); incr(open_parens);
+slow_print(full_source_filename_stack[in_open]); update_terminal;
+@y
+print_char("("); incr(open_parens);
+slow_print_filename(full_source_filename_stack[in_open]); update_terminal;
+@z
+
@x [30.560] l.10968 - pTeX:
This is called BigEndian order.
@!@^BigEndian order@>
@@ -6268,6 +6500,24 @@ show_code: @<Show the current meaning of a token, then |goto common_ending|@>;
show_mode: @<Show the current japanese processing mode@>;
@z
+@x dump
+@!format_engine: ^text_char;
+@y
+@!w: four_quarters; {four ASCII codes}
+@!format_engine: ^text_char;
+@z
+
+@x undump
+@!format_engine: ^text_char;
+@!dummy_xord: ASCII_code;
+@!dummy_xchr: text_char;
+@y
+@!w: four_quarters; {four ASCII codes}
+@!format_engine: ^text_char;
+@!dummy_xord: ASCII_code;
+@!dummy_xchr: ext_ASCII_code;
+@z
+
@x
libc_free(format_engine);@/
@y
@@ -6282,6 +6532,28 @@ libc_free(format_engine);
undump_kanji(fmt_file);
@z
+@x
+dump_things(str_pool[0], pool_ptr);
+@y
+for k:=0 to str_ptr do dump_int(str_start[k]);
+k:=0;
+while k+4<pool_ptr do
+ begin dump_four_ASCII; k:=k+4;
+ end;
+k:=pool_ptr-4; dump_four_ASCII;
+@z
+
+@x
+undump_things(str_pool[0], pool_ptr);
+@y
+for k:=0 to str_ptr do undump(0)(pool_ptr)(str_start[k]);
+k:=0;
+while k+4<pool_ptr do
+ begin undump_four_ASCII; k:=k+4;
+ end;
+k:=pool_ptr-4; undump_four_ASCII;
+@z
+
@x l.24982
font_info:=xmalloc_array(fmemory_word, font_mem_size);
@y
@@ -6339,6 +6611,13 @@ undump_things(ctype_base[null_font], font_ptr+1-null_font);
undump_things(char_base[null_font], font_ptr+1-null_font);
@z
+@x
+ buffer:=xmalloc_array (ASCII_code, buf_size);
+@y
+ buffer:=xmalloc_array (ASCII_code, buf_size);
+ buffer2:=xmalloc_array (ASCII_code, buf_size);
+@z
+
@x l.25363 - pTeX
font_info:=xmalloc_array (fmemory_word, font_mem_size);
@y
@@ -6396,6 +6675,40 @@ inhibit_glue_flag:=false;
end
@z
+@x [53.????] \write18{foo} (write_out in tex.ch)
+@!d:integer; {number of characters in incomplete current string}
+@y
+@!k:integer; {loop indices}
+@!d:integer; {number of characters in incomplete current string}
+@z
+
+@x [53.????] \write18{foo} (write_out in tex.ch)
+ for d:=0 to cur_length-1 do
+ begin {|print| gives up if passed |str_ptr|, so do it by hand.}
+ print(so(str_pool[str_start[str_ptr]+d])); {N.B.: not |print_char|}
+ end;
+@y
+ for d:=0 to cur_length-1 do
+ begin {|print| gives up if passed |str_ptr|, so do it by hand.}
+ if so(str_pool[str_start[str_ptr]+d])>=@"100 then
+ print_char(so(str_pool[str_start[str_ptr]+d]))
+ else print(so(str_pool[str_start[str_ptr]+d])); {N.B.: not |print_char|}
+ end;
+@z
+
+@x [53.????] ignore "flag bit" in str_pool for system(3)
+ runsystem_ret := runsystem(conststringcast(addressof(
+ str_pool[str_start[str_ptr]])));
+@y
+ if name_of_file then libc_free(name_of_file);
+ name_of_file := xmalloc(cur_length * 3 + 2);
+ k := 0;
+ for d:=0 to cur_length-1 do
+ append_to_name(str_pool[str_start[str_ptr]+d]);
+ name_of_file[k+1] := 0;
+ runsystem_ret := runsystem(conststringcast(name_of_file+1));
+@z
+
@x [53.????] Implement \immediate, inhibit_glue_flag
begin p:=tail; do_extension; {append a whatsit node}
out_what(tail); {do the action immediately}
@@ -6439,6 +6752,31 @@ else begin inhibit_glue_flag:=false;
@!debug debug_format_file:=true; @+gubed;
@z
+@x pTeX: xchr
+ if eight_bit_p then
+ for k:=0 to 255 do
+ xprn[k]:=1;
+end;
+@y
+ if eight_bit_p then
+ for k:=0 to 255 do
+ xprn[k]:=1;
+end;
+for k:=256 to 511 do xchr[k]:=k;
+@z
+
+@x [54/web2c.???] scan_file_name_braced
+ for i:=str_start[s] to str_start[s+1]-1 do
+ dummy := more_name(str_pool[i]); {add each read character to the current file name}
+@y
+ for i:=str_start[s] to str_start[s+1]-1 do
+ if str_pool[i]>=@"100 then
+ begin str_room(1); append_char(str_pool[i]);
+ end
+ else
+ dummy := more_name(str_pool[i]); {add each read character to the current file name}
+@z
+
@x l.26984 - pTeX
@* \[54] System-dependent changes.
@y
@@ -7501,8 +7839,8 @@ skip_loop: do_nothing;
@ @<Basic printing...@>=
procedure print_kanji(@!s:KANJI_code); {prints a single character}
begin
-if s>255 then
- begin print_char(Hi(s)); print_char(Lo(s));
+if s>@"FF then
+ begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s));
end else print_char(s);
end;
@@ -7536,5 +7874,25 @@ else begin
end;
end;
+@ This procedure is used in printing the second line in showing contexts.
+This part is not read by |get_next| yet, so we don't know which bytes
+are part of Japaense characters when the procedure is called.
+
+@<Basic printing...@>=
+procedure print_unread_buffer_with_ptenc(@!f, @!l: integer);
+{ print |buffer[f..l-1]| with code conversion }
+var @!i,@!j: pool_pointer; @!p: integer;
+begin
+ i:=f;
+ while i<l do begin
+ p:=multistrlen(ustringcast(buffer), l, i);
+ if p<>1 then
+ begin for j:=i to i+p-1 do
+ print_char(@"100+buffer[j]);
+ i:=i+p; end
+ else begin print(buffer[i]); incr(i); end;
+ end;
+end;
+
@* \[56] System-dependent changes.
@z
diff --git a/Build/source/texk/web2c/ptexdir/ptex.defines b/Build/source/texk/web2c/ptexdir/ptex.defines
index e087b545741..462ee6227ea 100644
--- a/Build/source/texk/web2c/ptexdir/ptex.defines
+++ b/Build/source/texk/web2c/ptexdir/ptex.defines
@@ -7,7 +7,10 @@
@define function iskanji1 ();
@define function multistrlen ();
+@define function multistrlenshort ();
+@define function multistrlenfilename ();
@define function fromBUFF ();
+@define function fromBUFFshort ();
@define function toBUFF ();
@define function fromDVI ();
diff --git a/Build/source/texk/web2c/ptexdir/ptex_version.h b/Build/source/texk/web2c/ptexdir/ptex_version.h
index 91d996f48a6..e451d4fcd80 100644
--- a/Build/source/texk/web2c/ptexdir/ptex_version.h
+++ b/Build/source/texk/web2c/ptexdir/ptex_version.h
@@ -1 +1 @@
-#define PTEX_VERSION "p3.10.0"
+#define PTEX_VERSION "p4.0.0"
diff --git a/Build/source/texk/web2c/ptexdir/tests/filename_test.sh b/Build/source/texk/web2c/ptexdir/tests/filename_test.sh
new file mode 100644
index 00000000000..4c04623462e
--- /dev/null
+++ b/Build/source/texk/web2c/ptexdir/tests/filename_test.sh
@@ -0,0 +1,45 @@
+#!/bin/bash
+cat <<'EOF' > filename_日本語.tex
+% UTF-8 encoding
+\def\TEST#1{\ifx#1\relax\else\immediate\write16{[\meaning#1]}\expandafter\TEST\fi}
+\expandafter\TEST\jobname\relax
+
+\immediate\openout0=\jobname.txt
+\immediate\write0{日本語}
+\immediate\closeout0
+
+\openin0=\jobname.txt
+\ifeof0
+ \immediate\write16{File `\jobname.txt' not found.}
+\else
+ \read0 to \TEXT
+ \immediate\write16{TEXT=\TEXT}
+\fi
+\closein0
+
+\input \jobname.txt
+
+\end
+
+EOF
+
+ln -s filename_日本語.tex "filename_日本a.tex"
+eptex -synctex=1 "\input {filename_日本a}"
+ls *.synctex.gz; rm *.synctex.gz
+euptex -synctex=1 "\input {filename_日本a}"
+ls *.synctex.gz; rm *.synctex.gz
+ls "filename_日本a".*
+rm "filename_日本a".*
+ln -s filename_日本語.tex filename_日本ßſa.tex
+euptex -synctex=1 filename_日本ßſa
+ls *.synctex.gz; rm *.synctex.gz
+ls filename_日本ßſa.*
+rm filename_日本ßſa.*
+
+eptex -synctex=1 "-jobname=漢ßſa" filename_日本語
+ls *.synctex.gz; rm *.synctex.gz
+ls 漢*; rm 漢*
+euptex -synctex=1 "-jobname=漢ßſa" filename_日本語
+ls *.synctex.gz; rm *.synctex.gz
+ls 漢*; rm 漢*
+rm filename_日本語.tex
diff --git a/Build/source/texk/web2c/ptexdir/tests/printkanji.tex b/Build/source/texk/web2c/ptexdir/tests/printkanji.tex
new file mode 100644
index 00000000000..a8f1d4cb5a0
--- /dev/null
+++ b/Build/source/texk/web2c/ptexdir/tests/printkanji.tex
@@ -0,0 +1,168 @@
+%\scrollmode
+\tracingstats=1000
+\noautoxspacing
+\newlinechar=`\^^J
+\font\x=ec-lmtt10 \x
+\immediate\openout1=\jobname.out
+\def\MSG#1{%
+ \message{\string{MSG #1\string}}%
+ \immediate\write17{\string{TOT #1\string}}%
+ \immediate\write1{#1}%
+}
+\def\head#1{\message{■#1.}\par\noindent\hbox{■\null}#1.\par}
+\message{^^J}
+
+%================
+\def\A{^^c5^^bf ſ 顛 }
+A: \A
+
+\MSG{\A}
+\show\A
+
+%================
+\par
+\head{\string\meaning}
+
+\edef\B{\meaning\A}
+\meaning\B
+
+\B
+\MSG{\meaning\B}
+\show\B
+
+%================
+\head{\string\jobname}
+\edef\C{:\jobname:}
+\MSG{*あ*\jobname *\C*\meaning\C*}
+\message{^^J}
+
+%================
+\catcode`\^^c5=11
+\catcode`\^^bf=11
+\catcode`\^^e1=11
+\catcode`\^^e3=11
+\catcode`\^^81=11
+\catcode`\^^82=11
+
+\head{oneletter}
+
+\string\^^c4.\string\^^c5.\string\^^ff.
+\count0=`\^^c5%
+\MSG{\string\catcode`\string\^^c5 = \the\count0}%
+\count0=`^^c5%
+\MSG{\string\catcode`^^c5 = \the\count0}%
+\count0=`^^c5^^bf%
+\MSG{\string\catcode`^^c5^^bf = \the\count0}%
+\count0=`顛%
+\MSG{\string\catcode`顛 = \the\count0}%
+
+%================
+\head{csname1}
+
+{\def\顛{hoge}\def\^^c5^^bf{piyo}
+\show\顛
+\show\^^c5^^bf
+\expandafter\show\csname ^^c5^^bf\endcsname
+\MSG{\string\顛=>\meaning\顛}
+\MSG{\string\^^c5^^bf=>\meaning\^^c5^^bf}
+\MSG{\expandafter\string\csname ^^c5^^bf\endcsname
+ =>\expandafter\meaning\csname ^^c5^^bf\endcsname}}
+
+\string\^^c5^^bf
+\expandafter\string\csname ^^c5^^bf\endcsname,
+\expandafter\string\csname ſ\endcsname,
+\expandafter\string\csname 顛\endcsname
+\MSG{\string\^^c5^^bf,
+ \expandafter\string\csname ^^c5^^bf\endcsname,
+ \expandafter\string\csname ſ\endcsname,
+ \expandafter\string\csname 顛\endcsname.}
+
+%================
+
+\def\あ{hoge}
+\message{^^J}
+\def\TEST#1#2{%
+ \expandafter\def\csname#2\endcsname{piyo}
+ \par\toks0={#2}
+ \expandafter\string\csname #1\endcsname => \csname #1\endcsname,\par
+ \expandafter\string\csname #2\endcsname => \csname #2\endcsname,\par
+ \expandafter\string\csname \the\toks0\endcsname => \csname \the\toks0\endcsname.
+ \MSG{%
+ \expandafter\string\csname #1\endcsname => \csname #1\endcsname,
+ \expandafter\string\csname #2\endcsname => \csname #2\endcsname.
+ \expandafter\string\csname \the\toks0\endcsname => \csname \the\toks0\endcsname.
+ }%
+}
+\ifnum\euc"A4A2="A4A2\relax
+ \TEST{あ}{^^a4^^a2}
+\else
+ \TEST{あ}{^^e3^^81^^82}
+\fi
+
+%================
+\head{0xFF}
+\catcode"FF=11
+\message{^^J}
+
+\def\^^ff^^c5^^ff^^bf{あ}
+\edef\E{(\string\^^ff^^c5^^ff^^bf:\meaning\^^ff^^c5^^ff^^bf)}
+\^^ff^^c5^^ff^^bf, \E, \string\^^ff^^c5^^ff^^bf, \meaning\E.
+\MSG{\^^ff^^c5^^ff^^bf, \A, \string\^^ff^^c5^^ff^^bf, \meaning\E.}
+
+\def\^^ff{い}
+\edef\E{(\string\^^ff :\meaning\^^ff)}
+\^^ff, \E, \string\^^ff, \meaning\E.
+\MSG{\^^ff, \E, \string\^^ff, \meaning\E.}
+
+\immediate\closeout1
+
+%========
+\head{contexts}
+
+\errorcontextlines=10000
+\def\@{\A\undefined}
+\edef\a{^^c5^^bf a\A\noexpand\@ ſ 顛 }
+\edef\+#1{\noexpand\^^c5^^bf\noexpand\顛\noexpand\^^ff^^c5^^ff^^bf#1
+ \noexpand\^^ff\meaning\A\noexpand\^^c5^^bf\noexpand\顛}
+\+\a
+
+\^^ff^^c5^^ff^^bf ^^c5^^bf \^^c5^^bf ſ顛\a \^^ff ^^ff\^^c5^^bf
+
+\^^c5\^^ff\^^c4\^^fe
+
+\catcode`\^^c5=11
+\catcode`\^^be=11
+\catcode`\^^bf=11
+\catcode`\^^bd=11
+\catcode`\^^ff=11
+\font\x=ec-lmtt10 \x\scrollmode
+
+\def\^^c5^^bf{a}
+\def\顛{b}
+\def\転{c}
+\message{\string\^^c5^^bf \string\^^ff^^c5}
+
+\def\b{\ž.\ſ.\Ž..\転.\顛.\貼.}
+\b
+
+\ž.\ſ.\Ž..\転.\顛.\貼.
+
+
+%========
+\head{csname2}
+
+SHOW \show\ſ.\show\顛.%
+\expandafter\show\csname ſ\endcsname.%
+\expandafter\show\csname 顛\endcsname
+
+STRING \string\ſ.\string\顛.%
+\expandafter\string\csname ſ\endcsname.%
+\expandafter\string\csname 顛\endcsname
+
+MEANING \meaning\ſ.\meaning\顛.%
+\expandafter\meaning\csname ſ\endcsname.%
+\expandafter\meaning\csname 顛\endcsname.%
+
+\message{\string\^^e3^^81^^82 \string\あ}
+
+\end