diff options
author | Hironobu Yamashita <h.y.acetaminophen@gmail.com> | 2022-01-22 17:03:22 +0000 |
---|---|---|
committer | Hironobu Yamashita <h.y.acetaminophen@gmail.com> | 2022-01-22 17:03:22 +0000 |
commit | 76e833603f3eb350549d8b472d5b0c1ebae21713 (patch) | |
tree | 0adce490f86927e7a81a2f05330b9590bf80663d /Build/source/texk/web2c/ptexdir | |
parent | 87e77a4b4b35459b0bc1aa06f48f2cf0db19ae9e (diff) |
[e][u]ptex: Distinguish 8-bit characters and Japanese characters (H. Kitagawa et al.)
For better support of LaTeX3 (expl3).
More details in TUGboat 41(2):329--334, 2020.
git-svn-id: svn://tug.org/texlive/trunk@61692 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/web2c/ptexdir')
-rw-r--r-- | Build/source/texk/web2c/ptexdir/ChangeLog | 11 | ||||
-rw-r--r-- | Build/source/texk/web2c/ptexdir/kanji.h | 4 | ||||
-rw-r--r-- | Build/source/texk/web2c/ptexdir/ptex-base.ch | 424 | ||||
-rw-r--r-- | Build/source/texk/web2c/ptexdir/ptex.defines | 3 | ||||
-rw-r--r-- | Build/source/texk/web2c/ptexdir/ptex_version.h | 2 | ||||
-rw-r--r-- | Build/source/texk/web2c/ptexdir/tests/filename_test.sh | 45 | ||||
-rw-r--r-- | Build/source/texk/web2c/ptexdir/tests/printkanji.tex | 168 |
7 files changed, 621 insertions, 36 deletions
diff --git a/Build/source/texk/web2c/ptexdir/ChangeLog b/Build/source/texk/web2c/ptexdir/ChangeLog index c77ab5bbd3f..ce2506f302e 100644 --- a/Build/source/texk/web2c/ptexdir/ChangeLog +++ b/Build/source/texk/web2c/ptexdir/ChangeLog @@ -1,3 +1,14 @@ +2022-01-22 Hironori Kitagawa <h_kitagawa2001@yahoo.co.jp> + + * ptex-base.ch: Change type of str_pool to unsigned short. + We use only 0--511, and use flag 0x100 for Japanese char. + More details in TUGboat 41(2):329--334, 2020. + * kanji.h: Adapt to arguments of input_line2() in ptexenc-1.4.0 + (for pBibTeX). + * ptex.defines: Add multistrlen{short,filename}, fromBUFFshort. + * ptex_version.h: Version p4.0.0. + * tests/filename_test.sh, tests/printkanji.tex: Added. + 2022-01-10 Karl Berry <karl@freefriends.org> * am/ptex.am: do not silence tangle-sh. diff --git a/Build/source/texk/web2c/ptexdir/kanji.h b/Build/source/texk/web2c/ptexdir/kanji.h index 5be133ecad4..e0edeb2ed70 100644 --- a/Build/source/texk/web2c/ptexdir/kanji.h +++ b/Build/source/texk/web2c/ptexdir/kanji.h @@ -65,9 +65,9 @@ extern void init_default_kanji (const_string file_str, const_string internal_str #endif /* !PRESERVE_FPUTS */ #ifdef PBIBTEX -#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,pos,size,ptr) +#define inputline2(fp,buff,pos,size,ptr) input_line2(fp,buff,NULL,pos,size,ptr) #else -#define inputline2(fp,buff,pos,size) input_line2(fp,buff,pos,size,NULL) +#define inputline2(fp,buff,pos,size) input_line2(fp,buff,NULL,pos,size,NULL) #endif extern void dump_kanji (gzFile fp); diff --git a/Build/source/texk/web2c/ptexdir/ptex-base.ch b/Build/source/texk/web2c/ptexdir/ptex-base.ch index e3b15e30d58..15a76eaeca6 100644 --- a/Build/source/texk/web2c/ptexdir/ptex-base.ch +++ b/Build/source/texk/web2c/ptexdir/ptex-base.ch @@ -67,6 +67,10 @@ % based on TeX 3.141592653 (for TL21). % (2021-06-25) HY pTeX p3.9.1 Various fixes. % (2021-06-20) HK pTeX p3.10.0 Add \ucs and \toucs. +% (2022-01-22) HK pTeX p4.0.0 Distinguish 8-bit characters and Japanese characters +% for better support of LaTeX3 (expl3). +% Requires ptexenc version 1.4.0. +% More details in TUGboat 41(2):329--334, 2020. @x % Here is TeX material that gets inserted after \input webmac @@ -79,10 +83,10 @@ @d banner==TeX_banner @d banner_k==TeX_banner_k @y -@d pTeX_version=3 -@d pTeX_minor_version=10 +@d pTeX_version=4 +@d pTeX_minor_version=0 @d pTeX_revision==".0" -@d pTeX_version_string=='-p3.10.0' {current \pTeX\ version} +@d pTeX_version_string=='-p4.0.0' {current \pTeX\ version} @# @d pTeX_banner=='This is pTeX, Version 3.141592653',pTeX_version_string @d pTeX_banner_k==pTeX_banner @@ -97,6 +101,21 @@ @y @!ASCII_code=0..255; {eight-bit numbers} @!KANJI_code=0..65535; {sixteen-bit numbers} +@!ext_ASCII_code=0..32768; { only use 0--511 } +@z + +@x pTeX: xchr +xchr: array [ASCII_code] of text_char; + { specifies conversion of output characters } +@y +xchr: array [ext_ASCII_code] of ext_ASCII_code; + { specifies conversion of output characters } +@z + +@x pTeX: xchr +for i:=@'177 to @'377 do xchr[i]:=i; +@y +for i:=@'177 to @'777 do xchr[i]:=i; @z @x [3.??] l.870 - pTeX: @@ -114,6 +133,36 @@ @ Kanji code handling. @z +@x [3.??] pTeX +@<Glob...@>= +@!buffer:^ASCII_code; {lines of characters being read} +@y +In \pTeX, we use another array |buffer2[]| to indicate which byte +is a part of a Japanese character. +|buffer2[]| is initialized to zero in reading one line from a file +(|input_ln|). |buffer2[i]| is set to one when |buffer[i]| is known +to be a part of a Japanese character, in |get_next| routine. + +@<Glob...@>= +@!buffer:^ASCII_code; {lines of characters being read} +@!buffer2:^ASCII_code; +@z + +@x [4] +@!packed_ASCII_code = 0..255; {elements of |str_pool| array} +@y +@!packed_ASCII_code = 0..32768; {elements of |str_pool| array} + { 256..511 are used by Japanese characters } +@z + +@x [4] pTeX: str_eq_buf +while j<str_start[s+1] do + begin if so(str_pool[j])<>buffer[k] then +@y +while j<str_start[s+1] do + begin if so(str_pool[j])<>buffer2[k]*@"100+buffer[k] then +@z + @x [4.47] l.1325 - pTeX: @!init function get_strings_started:boolean; {initializes the string pool, but returns |false| if something goes wrong} @@ -126,19 +175,11 @@ label done,exit; var k,@!l:KANJI_code; {small indices or counters} @z -@x [4.49] l.1384 - pTeX: -@<Character |k| cannot be printed@>= - (k<" ")or(k>"~") -@y -@<Character |k| cannot be printed@>= - not (ismultiprn(k) or xprn[k]) -@z - @x [5.54] l.1514 - pTeX: Global variables @!trick_buf:array[0..ssup_error_line] of ASCII_code; {circular buffer for pseudoprinting} @y -@!trick_buf:array[0..ssup_error_line] of ASCII_code; {circular buffer for +@!trick_buf:array[0..ssup_error_line] of ext_ASCII_code; {circular buffer for pseudoprinting} @!trick_buf2:array[0..ssup_error_line] of 0..2; {pTeX: buffer for KANJI} @!kcode_pos: 0..2; {pTeX: denotes whether first byte or second byte of KANJI} @@ -212,14 +253,16 @@ term_only: begin wterm(xchr[s]); incr(term_offset); no_print: do_nothing; pseudo: if tally<trick_count then trick_buf[tally mod error_line]:=s; @y -procedure print_char(@!s:ASCII_code); {prints a single character} +procedure print_char(@!s:ext_ASCII_code); {prints a single character} label exit; {label is not used but nonetheless kept (for other changes?)} begin if @<Character |s| is the current new-line character@> then if selector<pseudo then begin print_ln; return; end; -if kcode_pos=1 then kcode_pos:=2 -else if iskanji1(xchr[s]) then +if s>@"1FF then s:=s mod 256; +if s<256 then kcode_pos:=0 +else if kcode_pos=1 then kcode_pos:=2 +else if iskanji1(xchr[s-256]) then begin kcode_pos:=1; if (selector=term_and_log)or(selector=log_only) then if file_offset>=max_print_line-1 then @@ -299,6 +342,7 @@ else if s<256 then if selector<pseudo then begin print_ln; return; end; + if xprn[s] then begin print_char(s); return; end; nl:=new_line_char; new_line_char:=-1; {temporarily disable new-line character} j:=str_start[s]; @@ -314,6 +358,64 @@ while j<str_start[s+1] do exit:end; @z +@x +procedure slow_print(@!s:integer); {prints string |s|} +var j:pool_pointer; {current character code position} +begin if (s>=str_ptr) or (s<256) then print(s) +else begin j:=str_start[s]; + while j<str_start[s+1] do + begin print(so(str_pool[j])); incr(j); + end; + end; +end; +@y +procedure slow_print(@!s:integer); {prints string |s|} +var j:pool_pointer; {current character code position} +c:integer; +begin if (s>=str_ptr) or (s<256) then print(s) +else begin j:=str_start[s]; + while j<str_start[s+1] do + begin c:=so(str_pool[j]); + if c>=@"100 then print_char(c) else print(c); incr(j); + end; + end; +end; + +procedure slow_print_filename(@!s:integer); + {prints string |s| which represents filename, without code conversion} +var i,j,l:pool_pointer; p:integer; +begin if (s>=str_ptr) or (s<256) then print(s) +else begin i:=str_start[s]; l:=str_start[s+1]; + while i<l do begin + p:=multistrlenshort(str_pool, l, i); + if p<>1 then + begin for j:=i to i+p-1 do print_char(@"100+(so(str_pool[j]) mod @"100)); + i:=i+p; end + else begin print(so(str_pool[i]) mod @"100); incr(i); end; + end; + end; +end; + +procedure print_quoted(@!s:integer); + {prints string |s| which represents filename, + omitting quotes and with code conversion} +var i,l:pool_pointer; j,p:integer; +begin if s<>0 then begin + i:=str_start[s]; l:=str_start[s+1]; + while i<l do begin + p:=multistrlenshort(str_pool, l, i); + if p<>1 then begin + for j:=i to i+p-1 do print_char(@"100+(so(str_pool[j]) mod @"100)); + i:=i+p; end + else begin + if so(str_pool[i])<>"""" then print(so(str_pool[i]) mod @"100); + incr(i); end; + end; + end; +end; + +@z + @x [5.61] l.1656 - pTeX: @<Initialize the output...@>= if src_specials_p or file_line_error_style_p or parse_first_line_p then @@ -341,6 +443,18 @@ else @ Old versions of \TeX\ needed a procedure called |print_ASCII| whose function @z +@x [5.??] - pTeX: term_input +@p procedure term_input; {gets a line from the terminal} +@y +@p procedure@?print_unread_buffer_with_ptenc; forward;@t\2@>@/ +procedure term_input; {gets a line from the terminal} +@z +@x [5.??] - pTeX: term_input +if last<>first then for k:=first to last-1 do print(buffer[k]); +@y +if last<>first then print_unread_buffer_with_ptenc(first,last); +@z + @x @d max_halfword==@"FFFFFFF {largest allowable value in a |halfword|} @y @@ -1452,14 +1566,20 @@ end; tats @z -@x [17.???] l.???? - pTeX multibyte control symbol +@x [18] buffer2 +for k:=j to j+l-1 do append_char(buffer[k]); +@y +for k:=j to j+l-1 do append_char(buffer2[k]*@"100+buffer[k]); +@z + +@x [18.???] l.???? - pTeX multibyte control symbol procedure print_cs(@!p:integer); {prints a purported control sequence} @y procedure print_cs(@!p:integer); {prints a purported control sequence} var j, l:pool_pointer; @!cat:0..max_char_code; @z -@x +@x [18.???] else begin print_esc(text(p)); print_char(" "); end; @@ -1467,8 +1587,8 @@ else begin print_esc(text(p)); else begin l:=text(p); print_esc(l); j:=str_start[l]; l:=str_start[l+1]; if l>j+1 then begin - if l-j=multistrlen(ustringcast(str_pool), l, j) then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(str_pool), l, j))); + if (str_pool[j]>=@"100)and(l-j=multistrlenshort(str_pool, l, j)) then + begin cat:=kcat_code(kcatcodekey(fromBUFFshort(str_pool, l, j))); if (cat<>other_kchar) then print_char(" "); end else print_char(" "); end @@ -1476,6 +1596,13 @@ else begin l:=text(p); end; @z +@x [18.???] pTeX: ensure buffer2[]=0 in primitive + for j:=0 to l-1 do buffer[j]:=so(str_pool[k+j]); +@y + for j:=0 to l-1 do begin + buffer[j]:=Lo(so(str_pool[k+j])); buffer2[j]:=Hi(so(str_pool[k+j])); end; +@z + @x [18.265] l.5903 - pTeX: \jfont \tfont primitive("font",def_font,0);@/ @!@:font_}{\.{\\font} primitive@> @@ -1634,6 +1761,27 @@ if m+n<=error_line then p:=first_count+m else p:=first_count+(error_line-n-3); if trick_buf2[(p-1) mod error_line]=1 then p:=p-1; @z +@x pTeX: buffer +if j>0 then for i:=start to j-1 do + begin if i=loc then set_trick_count; + print(buffer[i]); + end +@y +if j>0 then begin + i:=start; + if (loc<=j-1)and(start<=loc) then begin + for i:=start to loc-1 do + if buffer2[i]>0 then + print_char(@"100*buffer2[i]+buffer[i]) else print(buffer[i]); + set_trick_count; print_unread_buffer_with_ptenc(loc,j); + end + else + for i:=start to j-1 do + if buffer2[i]>0 then + print_char(@"100*buffer2[i]+buffer[i]) else print(buffer[i]); + end +@z + @x [22.319] l.7157 - pTeX: adjust kanji code token @ @<Pseudoprint the token list@>= begin_pseudoprint; @@ -1660,6 +1808,13 @@ else show_token_list(link(start),loc,100000); {avoid reference count} done1: @z +@x [23.???] pTeX: init the input routines +first:=buf_size; repeat buffer[first]:=0; decr(first); until first=0; +@y +first:=buf_size; repeat buffer[first]:=0; buffer2[first]:=0; decr(first); until first=0; +@z + + @x [24.341] l.7479 - pTeX: set last_chr @!cat:0..max_char_code; {|cat_code(cur_chr)|, usually} @y @@ -1681,6 +1836,8 @@ begin switch: if loc<=limit then {current line not yet finished} if multistrlen(ustringcast(buffer), limit+1, loc-1)=2 then begin cur_chr:=fromBUFF(ustringcast(buffer), limit+1, loc-1); cur_cmd:=kcat_code(kcatcodekey(cur_chr)); + for l:=loc-1 to loc-2+multistrlen(ustringcast(buffer), limit+1, loc-1) do + buffer2[l]:=1; incr(loc); end else reswitch: cur_cmd:=cat_code(cur_chr); @@ -1769,7 +1926,10 @@ end begin if loc>limit then cur_cs:=null_cs {|state| is irrelevant in this case} else begin k:=loc; cur_chr:=buffer[k]; incr(k); if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k); + begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); + for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do + buffer2[l]:=1; + incr(k); end else cat:=cat_code(cur_chr); start_cs: @@ -1832,10 +1992,10 @@ begin if buffer[k]=cur_chr then @+if cat=sup_mark then @+if k<limit then end else if c<@'100 then buffer[k-1]:=c+@'100 else buffer[k-1]:=c-@'100; - limit:=limit-d; first:=first-d; + buffer2[k-1]:=0; limit:=limit-d; first:=first-d; l:=k; cur_chr:=buffer[k-1]; cat:=cat_code(cur_chr); while l<=limit do - begin buffer[l]:=buffer[l+d]; incr(l); + begin buffer[l]:=buffer[l+d]; buffer2[l]:=buffer2[l+d]; incr(l); end; goto start_cs; end; @@ -1858,7 +2018,10 @@ end @ @<Scan ahead in the buffer...@>= begin repeat cur_chr:=buffer[k]; incr(k); if multistrlen(ustringcast(buffer), limit+1, k-1)=2 then - begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); incr(k); + begin cat:=kcat_code(kcatcodekey(fromBUFF(ustringcast(buffer), limit+1, k-1))); + for l:=k-1 to k-2+multistrlen(ustringcast(buffer), limit+1, k-1) do + buffer2[l]:=1; + incr(k); end else cat:=cat_code(cur_chr); while (buffer[k]=cur_chr)and(cat=sup_mark)and(k<limit) do @@ -1874,11 +2037,11 @@ begin repeat cur_chr:=buffer[k]; incr(k); else cur_chr:=c-@'100; cat:=cat_code(cur_chr); if (cat=letter)or(cat=sup_mark) then - begin buffer[k-1]:=cur_chr; + begin buffer[k-1]:=cur_chr; buffer2[k-1]:=0; limit:=limit-d; first:=first-d; l:=k; while l<=limit do - begin buffer[l]:=buffer[l+d]; incr(l); + begin buffer[l]:=buffer[l+d]; buffer2[l]:=buffer2[l+d]; incr(l); end; end; end; @@ -1951,6 +2114,23 @@ else begin {we are done with this token list} end @z +@x [24] pTeX: firm_up_the_line + if start<limit then for k:=start to limit-1 do print(buffer[k]); + first:=limit; prompt_input("=>"); {wait for user response} +@.=>@> + if last>first then + begin for k:=first to last-1 do {move line down in buffer} + buffer[k+start-first]:=buffer[k]; +@y + if start<limit then for k:=start to limit-1 do + if buffer2[k]>=@"100 then print_char(buffer[k]) else print(buffer[k]); + first:=limit; prompt_input("=>"); {wait for user response} +@.=>@> + if last>first then + begin for k:=first to last-1 do {move line down in buffer} + begin buffer[k+start-first]:=buffer[k]; buffer2[k+start-first]:=buffer2[k]; end; +@z + @x [24.365] l.7935 - pTeX: get_token @p procedure get_token; {sets |cur_cmd|, |cur_chr|, |cur_tok|} begin no_new_control_sequence:=false; get_next; no_new_control_sequence:=true; @@ -1982,6 +2162,12 @@ while p<>null do end; buffer[j]:=info(p) mod @'400; incr(j); p:=link(p); end; +if j>first+1 then + begin no_new_control_sequence:=false; cur_cs:=id_lookup(first,j-first); + no_new_control_sequence:=true; + end +else if j=first then cur_cs:=null_cs {the list is empty} +else cur_cs:=single_base+buffer[first] {the list has length one} @y @ @<Look up the characters of list |r| in the hash table...@>= j:=first; p:=link(r); @@ -1993,10 +2179,17 @@ while p<>null do @:TeX capacity exceeded buffer size}{\quad buffer size@> end; if check_kanji(info(p)) then {|wchar_token|} - begin buffer[j]:=Hi(info(p)); incr(j); - end; + begin buffer[j]:=Hi(info(p)); buffer2[j]:=1; incr(j); buffer2[j]:=1; + end + else buffer2[j]:=0; buffer[j]:=Lo(info(p)); incr(j); p:=link(p); end; +if j>first+1 then + begin no_new_control_sequence:=false; cur_cs:=id_lookup(first,j-first); + no_new_control_sequence:=true; + end +else if j=first then cur_cs:=null_cs {the list is empty} +else cur_cs:=single_base+buffer[first] {the list has length one} @z @x [25.380] l.8221 - pTeX: get_x_token @@ -2363,8 +2556,8 @@ help6("Dimensions can be in units of em, ex, zw, zh, in, pt, pc,")@/ if t=" " then t:=space_token else t:=other_token+t; @y - if multistrlen(ustringcast(str_pool), pool_ptr, k)=2 then - begin t:=fromBUFF(ustringcast(str_pool), pool_ptr, k); incr(k); + if t>=@"100 then + begin t:=fromBUFFshort(str_pool, pool_ptr, k); incr(k); end else if t=" " then t:=space_token else t:=other_token+t; @@ -2651,6 +2844,28 @@ else begin str_room(1); append_char(c); {contribute |c| to the current string} end; @z +@x [29.518] - print_quoted in pTeX is already defined +@d print_quoted(#) == {print string |#|, omitting quotes} +if #<>0 then + for j:=str_start[#] to str_start[#+1]-1 do + if so(str_pool[j])<>"""" then + print(so(str_pool[j])) + +@y +@z + +@x +@d append_to_name(#)==begin c:=#; if not (c="""") then begin incr(k); + if k<=file_name_size then name_of_file[k]:=xchr[c]; + end end +@y +@d append_to_name(#)==begin if (#)>=@"100 then c:=(#)-@"100 else c:=#; + { Since the type of |c| is |ASCII_code|, above if-statement might not be needed } + if not (c="""") then begin incr(k); + if k<=file_name_size then name_of_file[k]:=xchr[c]; + end end +@z + @x [29.526] l.10668 - pTeX: scan file name loop@+begin if (cur_cmd>other_char)or(cur_chr>255) then {not a character} begin back_input; goto done; @@ -2669,8 +2884,8 @@ skip_mode:=false; loop@+begin if (cur_cmd=kanji)or(cur_cmd=kana)or(cur_cmd=other_kchar) then {is kanji} begin str_room(2); - append_char(Hi(cur_chr)); {kanji upper byte} - append_char(Lo(cur_chr)); {kanji lower byte} + append_char(@"100+Hi(cur_chr)); {kanji upper byte} + append_char(@"100+Lo(cur_chr)); {kanji lower byte} end else if (cur_cmd>other_char)or(cur_chr>255) then {not an alphabet} begin back_input; goto done; @@ -2686,6 +2901,15 @@ done: end_name; name_in_progress:=false; skip_mode:=true; @z +@x [29.???] open_log_file +if buffer[l]=end_line_char then decr(l); +for k:=1 to l do print(buffer[k]); +print_ln; {now the transcript file contains the first line of input} +@y +if buffer[l]=end_line_char then decr(l); print_unread_buffer_with_ptenc(1,l+1); +print_ln; {now the transcript file contains the first line of input} +@z + @x [29.536] l.10834 - pTeX: begin if src_specials_p or file_line_error_style_p or parse_first_line_p @@ -2705,6 +2929,14 @@ else wlog(')'); @z +@x [29.???] pTeX - start_input +print_char("("); incr(open_parens); +slow_print(full_source_filename_stack[in_open]); update_terminal; +@y +print_char("("); incr(open_parens); +slow_print_filename(full_source_filename_stack[in_open]); update_terminal; +@z + @x [30.560] l.10968 - pTeX: This is called BigEndian order. @!@^BigEndian order@> @@ -6268,6 +6500,24 @@ show_code: @<Show the current meaning of a token, then |goto common_ending|@>; show_mode: @<Show the current japanese processing mode@>; @z +@x dump +@!format_engine: ^text_char; +@y +@!w: four_quarters; {four ASCII codes} +@!format_engine: ^text_char; +@z + +@x undump +@!format_engine: ^text_char; +@!dummy_xord: ASCII_code; +@!dummy_xchr: text_char; +@y +@!w: four_quarters; {four ASCII codes} +@!format_engine: ^text_char; +@!dummy_xord: ASCII_code; +@!dummy_xchr: ext_ASCII_code; +@z + @x libc_free(format_engine);@/ @y @@ -6282,6 +6532,28 @@ libc_free(format_engine); undump_kanji(fmt_file); @z +@x +dump_things(str_pool[0], pool_ptr); +@y +for k:=0 to str_ptr do dump_int(str_start[k]); +k:=0; +while k+4<pool_ptr do + begin dump_four_ASCII; k:=k+4; + end; +k:=pool_ptr-4; dump_four_ASCII; +@z + +@x +undump_things(str_pool[0], pool_ptr); +@y +for k:=0 to str_ptr do undump(0)(pool_ptr)(str_start[k]); +k:=0; +while k+4<pool_ptr do + begin undump_four_ASCII; k:=k+4; + end; +k:=pool_ptr-4; undump_four_ASCII; +@z + @x l.24982 font_info:=xmalloc_array(fmemory_word, font_mem_size); @y @@ -6339,6 +6611,13 @@ undump_things(ctype_base[null_font], font_ptr+1-null_font); undump_things(char_base[null_font], font_ptr+1-null_font); @z +@x + buffer:=xmalloc_array (ASCII_code, buf_size); +@y + buffer:=xmalloc_array (ASCII_code, buf_size); + buffer2:=xmalloc_array (ASCII_code, buf_size); +@z + @x l.25363 - pTeX font_info:=xmalloc_array (fmemory_word, font_mem_size); @y @@ -6396,6 +6675,40 @@ inhibit_glue_flag:=false; end @z +@x [53.????] \write18{foo} (write_out in tex.ch) +@!d:integer; {number of characters in incomplete current string} +@y +@!k:integer; {loop indices} +@!d:integer; {number of characters in incomplete current string} +@z + +@x [53.????] \write18{foo} (write_out in tex.ch) + for d:=0 to cur_length-1 do + begin {|print| gives up if passed |str_ptr|, so do it by hand.} + print(so(str_pool[str_start[str_ptr]+d])); {N.B.: not |print_char|} + end; +@y + for d:=0 to cur_length-1 do + begin {|print| gives up if passed |str_ptr|, so do it by hand.} + if so(str_pool[str_start[str_ptr]+d])>=@"100 then + print_char(so(str_pool[str_start[str_ptr]+d])) + else print(so(str_pool[str_start[str_ptr]+d])); {N.B.: not |print_char|} + end; +@z + +@x [53.????] ignore "flag bit" in str_pool for system(3) + runsystem_ret := runsystem(conststringcast(addressof( + str_pool[str_start[str_ptr]]))); +@y + if name_of_file then libc_free(name_of_file); + name_of_file := xmalloc(cur_length * 3 + 2); + k := 0; + for d:=0 to cur_length-1 do + append_to_name(str_pool[str_start[str_ptr]+d]); + name_of_file[k+1] := 0; + runsystem_ret := runsystem(conststringcast(name_of_file+1)); +@z + @x [53.????] Implement \immediate, inhibit_glue_flag begin p:=tail; do_extension; {append a whatsit node} out_what(tail); {do the action immediately} @@ -6439,6 +6752,31 @@ else begin inhibit_glue_flag:=false; @!debug debug_format_file:=true; @+gubed; @z +@x pTeX: xchr + if eight_bit_p then + for k:=0 to 255 do + xprn[k]:=1; +end; +@y + if eight_bit_p then + for k:=0 to 255 do + xprn[k]:=1; +end; +for k:=256 to 511 do xchr[k]:=k; +@z + +@x [54/web2c.???] scan_file_name_braced + for i:=str_start[s] to str_start[s+1]-1 do + dummy := more_name(str_pool[i]); {add each read character to the current file name} +@y + for i:=str_start[s] to str_start[s+1]-1 do + if str_pool[i]>=@"100 then + begin str_room(1); append_char(str_pool[i]); + end + else + dummy := more_name(str_pool[i]); {add each read character to the current file name} +@z + @x l.26984 - pTeX @* \[54] System-dependent changes. @y @@ -7501,8 +7839,8 @@ skip_loop: do_nothing; @ @<Basic printing...@>= procedure print_kanji(@!s:KANJI_code); {prints a single character} begin -if s>255 then - begin print_char(Hi(s)); print_char(Lo(s)); +if s>@"FF then + begin print_char(@"100+Hi(s)); print_char(@"100+Lo(s)); end else print_char(s); end; @@ -7536,5 +7874,25 @@ else begin end; end; +@ This procedure is used in printing the second line in showing contexts. +This part is not read by |get_next| yet, so we don't know which bytes +are part of Japaense characters when the procedure is called. + +@<Basic printing...@>= +procedure print_unread_buffer_with_ptenc(@!f, @!l: integer); +{ print |buffer[f..l-1]| with code conversion } +var @!i,@!j: pool_pointer; @!p: integer; +begin + i:=f; + while i<l do begin + p:=multistrlen(ustringcast(buffer), l, i); + if p<>1 then + begin for j:=i to i+p-1 do + print_char(@"100+buffer[j]); + i:=i+p; end + else begin print(buffer[i]); incr(i); end; + end; +end; + @* \[56] System-dependent changes. @z diff --git a/Build/source/texk/web2c/ptexdir/ptex.defines b/Build/source/texk/web2c/ptexdir/ptex.defines index e087b545741..462ee6227ea 100644 --- a/Build/source/texk/web2c/ptexdir/ptex.defines +++ b/Build/source/texk/web2c/ptexdir/ptex.defines @@ -7,7 +7,10 @@ @define function iskanji1 (); @define function multistrlen (); +@define function multistrlenshort (); +@define function multistrlenfilename (); @define function fromBUFF (); +@define function fromBUFFshort (); @define function toBUFF (); @define function fromDVI (); diff --git a/Build/source/texk/web2c/ptexdir/ptex_version.h b/Build/source/texk/web2c/ptexdir/ptex_version.h index 91d996f48a6..e451d4fcd80 100644 --- a/Build/source/texk/web2c/ptexdir/ptex_version.h +++ b/Build/source/texk/web2c/ptexdir/ptex_version.h @@ -1 +1 @@ -#define PTEX_VERSION "p3.10.0" +#define PTEX_VERSION "p4.0.0" diff --git a/Build/source/texk/web2c/ptexdir/tests/filename_test.sh b/Build/source/texk/web2c/ptexdir/tests/filename_test.sh new file mode 100644 index 00000000000..4c04623462e --- /dev/null +++ b/Build/source/texk/web2c/ptexdir/tests/filename_test.sh @@ -0,0 +1,45 @@ +#!/bin/bash +cat <<'EOF' > filename_日本語.tex +% UTF-8 encoding +\def\TEST#1{\ifx#1\relax\else\immediate\write16{[\meaning#1]}\expandafter\TEST\fi} +\expandafter\TEST\jobname\relax + +\immediate\openout0=\jobname.txt +\immediate\write0{日本語} +\immediate\closeout0 + +\openin0=\jobname.txt +\ifeof0 + \immediate\write16{File `\jobname.txt' not found.} +\else + \read0 to \TEXT + \immediate\write16{TEXT=\TEXT} +\fi +\closein0 + +\input \jobname.txt + +\end + +EOF + +ln -s filename_日本語.tex "filename_日本a.tex" +eptex -synctex=1 "\input {filename_日本a}" +ls *.synctex.gz; rm *.synctex.gz +euptex -synctex=1 "\input {filename_日本a}" +ls *.synctex.gz; rm *.synctex.gz +ls "filename_日本a".* +rm "filename_日本a".* +ln -s filename_日本語.tex filename_日本ßſa.tex +euptex -synctex=1 filename_日本ßſa +ls *.synctex.gz; rm *.synctex.gz +ls filename_日本ßſa.* +rm filename_日本ßſa.* + +eptex -synctex=1 "-jobname=漢ßſa" filename_日本語 +ls *.synctex.gz; rm *.synctex.gz +ls 漢*; rm 漢* +euptex -synctex=1 "-jobname=漢ßſa" filename_日本語 +ls *.synctex.gz; rm *.synctex.gz +ls 漢*; rm 漢* +rm filename_日本語.tex diff --git a/Build/source/texk/web2c/ptexdir/tests/printkanji.tex b/Build/source/texk/web2c/ptexdir/tests/printkanji.tex new file mode 100644 index 00000000000..a8f1d4cb5a0 --- /dev/null +++ b/Build/source/texk/web2c/ptexdir/tests/printkanji.tex @@ -0,0 +1,168 @@ +%\scrollmode +\tracingstats=1000 +\noautoxspacing +\newlinechar=`\^^J +\font\x=ec-lmtt10 \x +\immediate\openout1=\jobname.out +\def\MSG#1{% + \message{\string{MSG #1\string}}% + \immediate\write17{\string{TOT #1\string}}% + \immediate\write1{#1}% +} +\def\head#1{\message{■#1.}\par\noindent\hbox{■\null}#1.\par} +\message{^^J} + +%================ +\def\A{^^c5^^bf ſ 顛 } +A: \A + +\MSG{\A} +\show\A + +%================ +\par +\head{\string\meaning} + +\edef\B{\meaning\A} +\meaning\B + +\B +\MSG{\meaning\B} +\show\B + +%================ +\head{\string\jobname} +\edef\C{:\jobname:} +\MSG{*あ*\jobname *\C*\meaning\C*} +\message{^^J} + +%================ +\catcode`\^^c5=11 +\catcode`\^^bf=11 +\catcode`\^^e1=11 +\catcode`\^^e3=11 +\catcode`\^^81=11 +\catcode`\^^82=11 + +\head{oneletter} + +\string\^^c4.\string\^^c5.\string\^^ff. +\count0=`\^^c5% +\MSG{\string\catcode`\string\^^c5 = \the\count0}% +\count0=`^^c5% +\MSG{\string\catcode`^^c5 = \the\count0}% +\count0=`^^c5^^bf% +\MSG{\string\catcode`^^c5^^bf = \the\count0}% +\count0=`顛% +\MSG{\string\catcode`顛 = \the\count0}% + +%================ +\head{csname1} + +{\def\顛{hoge}\def\^^c5^^bf{piyo} +\show\顛 +\show\^^c5^^bf +\expandafter\show\csname ^^c5^^bf\endcsname +\MSG{\string\顛=>\meaning\顛} +\MSG{\string\^^c5^^bf=>\meaning\^^c5^^bf} +\MSG{\expandafter\string\csname ^^c5^^bf\endcsname + =>\expandafter\meaning\csname ^^c5^^bf\endcsname}} + +\string\^^c5^^bf +\expandafter\string\csname ^^c5^^bf\endcsname, +\expandafter\string\csname ſ\endcsname, +\expandafter\string\csname 顛\endcsname +\MSG{\string\^^c5^^bf, + \expandafter\string\csname ^^c5^^bf\endcsname, + \expandafter\string\csname ſ\endcsname, + \expandafter\string\csname 顛\endcsname.} + +%================ + +\def\あ{hoge} +\message{^^J} +\def\TEST#1#2{% + \expandafter\def\csname#2\endcsname{piyo} + \par\toks0={#2} + \expandafter\string\csname #1\endcsname => \csname #1\endcsname,\par + \expandafter\string\csname #2\endcsname => \csname #2\endcsname,\par + \expandafter\string\csname \the\toks0\endcsname => \csname \the\toks0\endcsname. + \MSG{% + \expandafter\string\csname #1\endcsname => \csname #1\endcsname, + \expandafter\string\csname #2\endcsname => \csname #2\endcsname. + \expandafter\string\csname \the\toks0\endcsname => \csname \the\toks0\endcsname. + }% +} +\ifnum\euc"A4A2="A4A2\relax + \TEST{あ}{^^a4^^a2} +\else + \TEST{あ}{^^e3^^81^^82} +\fi + +%================ +\head{0xFF} +\catcode"FF=11 +\message{^^J} + +\def\^^ff^^c5^^ff^^bf{あ} +\edef\E{(\string\^^ff^^c5^^ff^^bf:\meaning\^^ff^^c5^^ff^^bf)} +\^^ff^^c5^^ff^^bf, \E, \string\^^ff^^c5^^ff^^bf, \meaning\E. +\MSG{\^^ff^^c5^^ff^^bf, \A, \string\^^ff^^c5^^ff^^bf, \meaning\E.} + +\def\^^ff{い} +\edef\E{(\string\^^ff :\meaning\^^ff)} +\^^ff, \E, \string\^^ff, \meaning\E. +\MSG{\^^ff, \E, \string\^^ff, \meaning\E.} + +\immediate\closeout1 + +%======== +\head{contexts} + +\errorcontextlines=10000 +\def\@{\A\undefined} +\edef\a{^^c5^^bf a\A\noexpand\@ ſ 顛 } +\edef\+#1{\noexpand\^^c5^^bf\noexpand\顛\noexpand\^^ff^^c5^^ff^^bf#1 + \noexpand\^^ff\meaning\A\noexpand\^^c5^^bf\noexpand\顛} +\+\a + +\^^ff^^c5^^ff^^bf ^^c5^^bf \^^c5^^bf ſ顛\a \^^ff ^^ff\^^c5^^bf + +\^^c5\^^ff\^^c4\^^fe + +\catcode`\^^c5=11 +\catcode`\^^be=11 +\catcode`\^^bf=11 +\catcode`\^^bd=11 +\catcode`\^^ff=11 +\font\x=ec-lmtt10 \x\scrollmode + +\def\^^c5^^bf{a} +\def\顛{b} +\def\転{c} +\message{\string\^^c5^^bf \string\^^ff^^c5} + +\def\b{\ž.\ſ.\Ž..\転.\顛.\貼.} +\b + +\ž.\ſ.\Ž..\転.\顛.\貼. + + +%======== +\head{csname2} + +SHOW \show\ſ.\show\顛.% +\expandafter\show\csname ſ\endcsname.% +\expandafter\show\csname 顛\endcsname + +STRING \string\ſ.\string\顛.% +\expandafter\string\csname ſ\endcsname.% +\expandafter\string\csname 顛\endcsname + +MEANING \meaning\ſ.\meaning\顛.% +\expandafter\meaning\csname ſ\endcsname.% +\expandafter\meaning\csname 顛\endcsname.% + +\message{\string\^^e3^^81^^82 \string\あ} + +\end |