diff options
author | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
---|---|---|
committer | Norbert Preining <norbert@preining.info> | 2019-09-02 13:46:59 +0900 |
commit | e0c6872cf40896c7be36b11dcc744620f10adf1d (patch) | |
tree | 60335e10d2f4354b0674ec22d7b53f0f8abee672 /systems/tex-extensions/clasen/hyphenation |
Initial commit
Diffstat (limited to 'systems/tex-extensions/clasen/hyphenation')
-rw-r--r-- | systems/tex-extensions/clasen/hyphenation/classes.ch | 679 | ||||
-rw-r--r-- | systems/tex-extensions/clasen/hyphenation/count.ch | 240 | ||||
l--------- | systems/tex-extensions/clasen/hyphenation/etex.bib | 1 | ||||
-rw-r--r-- | systems/tex-extensions/clasen/hyphenation/hyphenation.ch | 883 | ||||
-rw-r--r-- | systems/tex-extensions/clasen/hyphenation/hyphenation.tex | 249 | ||||
-rw-r--r-- | systems/tex-extensions/clasen/hyphenation/patgen-hyf.ch | 460 |
6 files changed, 2512 insertions, 0 deletions
diff --git a/systems/tex-extensions/clasen/hyphenation/classes.ch b/systems/tex-extensions/clasen/hyphenation/classes.ch new file mode 100644 index 0000000000..27bbda3178 --- /dev/null +++ b/systems/tex-extensions/clasen/hyphenation/classes.ch @@ -0,0 +1,679 @@ +@x +else begin print_err("I can't go on meeting you like this"); +@y +else begin print_err("I can't go on meeting you like this ("); + print(s); print_char(")"); +@z + +@x +@d disc_node=7 {|type| of a discretionary node} +@y +With extended hyphenation, discretionaries have an additional property, +the |hyphen_class| (which is an integer $0\leq\mathit{hyphen_class}\leq 9$). +The penalty charged for a break depends on the |hyphen_class|. + +@d disc_node=7 {|type| of a discretionary node} +@d disc_node_size=3 {number of words to allocate for a discretionary} +@d hyphen_class(#)==mem[#+2].int {the class of this hyphen} +@z + +@x +@p function new_disc:pointer; {creates an empty |disc_node|} +var p:pointer; {the new node} +begin p:=get_node(small_node_size); type(p):=disc_node; +replace_count(p):=0; pre_break(p):=null; post_break(p):=null; +new_disc:=p; +end; +@y +@p function new_disc(c:integer):pointer; {creates an empty |disc_node|} +var p:pointer; {the new node} +begin p:=get_node(disc_node_size); type(p):=disc_node; +replace_count(p):=0; pre_break(p):=null; post_break(p):=null; +hyphen_class(p):=c; new_disc:=p; +end; +@z + +@x +disc_node: begin short_display(pre_break(p)); + short_display(post_break(p));@/ +@y +disc_node: begin short_display(pre_break(p)); + if hyphen_classes_en and(hyphen_class(p)<>1) then print_int(hyphen_class(p)); + short_display(post_break(p));@/ +@z + +@x + disc_node: begin flush_node_list(pre_break(p)); + flush_node_list(post_break(p)); + end; +@y + disc_node: begin flush_node_list(pre_break(p)); + flush_node_list(post_break(p)); + free_node(p,disc_node_size); goto done; + end; +@z + +@x +disc_node: begin r:=get_node(small_node_size); + pre_break(r):=copy_node_list(pre_break(p)); + post_break(r):=copy_node_list(post_break(p)); + end; +@y +disc_node: begin r:=new_disc(hyphen_class(p)); + pre_break(r):=copy_node_list(pre_break(p)); + post_break(r):=copy_node_list(post_break(p)); + end; +@z + +@x +@d hyph_data=99 {hyphenation data ( \.{\\hyphenation}, \.{\\patterns} )} +@y +@d hyph_data=99 {hyphenation data ( \.{\\hyphenation}, \.{\\patterns}, + \.{\\hyphenpenalties} )} +@z + +@x + if m=hmode then if nest[p].pg_field <> @'40600000 then + begin print(" (language"); print_int(nest[p].pg_field mod @'200000); + print(":hyphenmin"); print_int(nest[p].pg_field div @'20000000); + print_char(","); print_int((nest[p].pg_field div @'200000) mod @'100); + print_char(")"); + end; +@y + if m=hmode then if nest[p].pg_field <> @'40601000 then begin + print(" (language"); print_int(nest[p].pg_field mod @'400); + print(":hyphenmin"); print_int(nest[p].pg_field div @'20000000); + print_char(","); print_int((nest[p].pg_field div @'200000) mod @'100); + if hyphen_classes_en then begin + print(":hyphenclasses"); + print_int((nest[p].pg_field div @'400) mod @'20); + end; + print_char(")"); + end; +@z + +@x +@d hyphen_penalty_code=3 {penalty for break after discretionary hyphen} +@d ex_hyphen_penalty_code=4 {penalty for break after explicit hyphen} +@y +@d hyphen_classes_code=3 {number of hyphenation classes} +@d ex_hyphen_class_code=4 {hyphenation class of automatically + inserted empty discretionaries} +@z + +@x +{hyphen classes int_pars go here} +@y +@d hyphen_classes_state_code=58 +@d hyphen_penalty_base=59 {10 penalties for breaks after discretionary hyphens, + numbered from 0 to 9} +@z + +@x +@d hyphen_penalty==int_par(hyphen_penalty_code) +@d ex_hyphen_penalty==int_par(ex_hyphen_penalty_code) +@y +@d hyphen_classes_state==int_par(hyphen_classes_state_code) +@d hyphen_classes_en==(hyphen_classes_state>0) +@d hyphen_penalties(#)==int_par(hyphen_penalty_base+#) +@d default_hyphen_class=1 +@d hyphen_penalty==hyphen_penalties(default_hyphen_class) +@d ex_hyphen_penalty==hyphen_penalties(ex_hyphen_class) +@z + +@x +hyphen_penalty_code:print_esc("hyphenpenalty"); +ex_hyphen_penalty_code:print_esc("exhyphenpenalty"); +@y +hyphen_classes_code:print_esc("hyphenclasses"); +ex_hyphen_class_code:print_esc("exhyphenclass"); +@z + +@x +error_context_lines_code:print_esc("errorcontextlines"); +@y +error_context_lines_code:print_esc("errorcontextlines"); +hyphen_classes_state_code:print_esc("hyphenclassesstate"); +@z + +@x +primitive("hyphenpenalty",assign_int,int_base+hyphen_penalty_code);@/ +@!@:hyphen_penalty_}{\.{\\hyphenpenalty} primitive@> +primitive("exhyphenpenalty",assign_int,int_base+ex_hyphen_penalty_code);@/ +@!@:ex_hyphen_penalty_}{\.{\\exhyphenpenalty} primitive@> +@y +primitive("hyphenclasses",assign_int,int_base+hyphen_classes_code);@/ +@!@:hyphen_classes_}{\.{\\hyphenclasses} primitive@> +primitive("exhyphenclass",assign_int,int_base+ex_hyphen_class_code);@/ +@!@:ex_hyphen_class_}{\.{\\exhyphenclass} primitive@> +@z + +@x +primitive("errorcontextlines",assign_int,int_base+error_context_lines_code);@/ +@!@:error_context_lines_}{\.{\\errorcontextlines} primitive@> +@y +primitive("errorcontextlines",assign_int,int_base+error_context_lines_code);@/ +@!@:error_context_lines_}{\.{\\errorcontextlines} primitive@> +primitive("hyphenpenalty",assign_int,int_base+hyphen_penalty_base+1);@/ +@!@:hyphen_penalty_}{\.{\\hyphenpenalty} primitive@> +primitive("exhyphenpenalty",assign_int,int_base+hyphen_penalty_base + +ex_hyphen_class);@/ +@!@:ex_hyphen_penalty_}{\.{\\exhyphenpenalty} primitive@> +primitive("hyphenclassesstate",assign_int,int_base+hyphen_classes_state_code);@/ +@!@:hyphen_classes_state_}{\.{\\hyphenclassesstate} primitive@> +@z + +@x +escape_char:="\"; end_line_char:=carriage_return; +@y +escape_char:="\"; end_line_char:=carriage_return; +hyphen_classes_state:=0; +@z + +@x +@ Here is a procedure that displays the contents of |eqtb[n]| +symbolically. + +@p@t\4@>@<Declare the procedure called |print_cmd_chr|@>@;@/ +@y +@ Here is a procedure that displays the contents of |eqtb[n]| +symbolically. + +@p@t\4@>@<Declare the procedure called |print_cmd_chr|@>@;@/ +function is_enabled(@!b:boolean;@!j:quarterword;@!k:halfword):boolean; +begin + if not b then begin + print_err("Improper "); print_cmd_chr(j,k); + help1("Sorry, this optional feature has been disabled."); error; + end; + is_enabled:=b; +end; +@z + +@x +init_cur_lang:=prev_graf mod @'200000; +@y +init_cur_lang:=prev_graf mod @'400; +init_hyf_class:=(prev_graf div @'400) mod @'20; +@z + +@x +@ The following code knows that discretionary texts contain +only character nodes, kern nodes, box nodes, rule nodes, and ligature nodes. + +@<Try to break after a discretionary fragment...@>= +begin s:=pre_break(cur_p); disc_width:=0; +if s=null then try_break(ex_hyphen_penalty,hyphenated) +else begin repeat @<Add the width of node |s| to |disc_width|@>; + s:=link(s); + until s=null; + act_width:=act_width+disc_width; + try_break(hyphen_penalty,hyphenated); + act_width:=act_width-disc_width; + end; +@y +@ The following code knows that discretionary texts contain +only character nodes, kern nodes, box nodes, rule nodes, and ligature nodes. +If |ex_hyphen_penalty=0|, we might encounter automatically inserted empty +discretionary nodes of class~0. We never break at these discretionaries, +nevertheless they are necessary to differentiate the output of +\.{\\showhyphens} for automatically inserted hyphens of classes 0 and~1. + +@<Try to break after a discretionary fragment...@>= +begin if hyphen_class(cur_p)>0 then begin + s:=pre_break(cur_p); disc_width:=0; + while s<>null do begin + @<Add the width of node |s| to |disc_width|@>; + s:=link(s); + end; + act_width:=act_width+disc_width; + try_break(hyphen_penalties(hyphen_class(cur_p)),hyphenated); + act_width:=act_width-disc_width; + end; +@z + +@x +begin t:=replace_count(q); +@y +begin t:=replace_count(q); hyphen_class(q):=1; +@z + +@x +cur_lang:=init_cur_lang; l_hyf:=init_l_hyf; r_hyf:=init_r_hyf; +@y +cur_lang:=init_cur_lang; l_hyf:=init_l_hyf; r_hyf:=init_r_hyf; +hyf_class:=init_hyf_class; +@z + +@x +@!hyf_bchar:halfword; {boundary character after $c_n$} +@y +@!hyf_bchar:halfword; {boundary character after $c_n$} +@!hyf_class,@!init_hyf_class:integer; +@z + +@x +@!hyf:array [0..64] of 0..9; {odd values indicate discretionary hyphens} +@y +@!hyf:array [0..64] of small_number; +@z + +@x +@<If no hyphens were found, |return|@>= +for j:=l_hyf to hn-r_hyf do if odd(hyf[j]) then goto found1; +return; +found1: +@y +@<If no hyphens were found, |return|@>= +for j:=l_hyf to hn-r_hyf do if hyf[j]>0 then goto found1; +return; +found1: +@z + +@x +@d set_cur_r==begin if j<n then cur_r:=qi(hu[j+1])@+else cur_r:=bchar; + if odd(hyf[j]) then cur_rh:=hchar@+else cur_rh:=non_char; +@y +@d set_cur_r==begin if j<n then cur_r:=qi(hu[j+1])@+else cur_r:=bchar; + if hyf[j]>0 then cur_rh:=hchar@+else cur_rh:=non_char; +@z + +@x + else begin if hchar<non_char then if odd(hyf[j]) then +@y + else begin if hchar<non_char then if hyf[j]>0 then +@z + +@x + while link(s)>null do s:=link(s); + if odd(hyf[j-1]) then +@y + while link(s)>null do s:=link(s); + if hyf[j-1]>0 then +@z + +@x +repeat r:=get_node(small_node_size); +@y +repeat r:=new_disc(hyf[hyphen_passed]); +@z + +@x +hyphen_passed:=j-1; link(hold_head):=null; +until not odd(hyf[j-1]) +@y +hyphen_passed:=j-1; link(hold_head):=null; +until hyf[j-1]=0 +@z + +@x +found: for j:=0 to l_hyf-1 do hyf[j]:=0; +for j:=0 to r_hyf-1 do hyf[hn-j]:=0 +@y +for j:=l_hyf to hn-r_hyf do hyf[j]:=hyf[j] mod hyf_class; +found: for j:=0 to l_hyf-1 do hyf[j]:=0; +for j:=0 to r_hyf-1 do hyf[hn-j]:=0 +@z + +@x +The words in the table point to lists in |mem| that specify hyphen positions +in their |info| fields. The list for $c_1\ldots c_n$ contains the number |k| if +the word $c_1\ldots c_n$ has a discretionary hyphen between $c_k$ and +$c_{k+1}$. +@y +The words in the table point to lists in |mem| that specify hyphen positions +and classes in their |type| and |subtype| fields. The list for $c_1\ldots c_n$ +contains the pair |k|, |l| if the word $c_1\ldots c_n$ has a discretionary +hyphen of class |l| between $c_k$ and $c_{k+1}$. + +@d hyphen_exception_pos==type +@d hyphen_exception_class==subtype +@z + +@x +while s<>null do + begin hyf[info(s)]:=1; s:=link(s); + end +@y +while s<>null do + begin hyf[hyphen_exception_pos(s)]:=hyphen_exception_class(s); s:=link(s); + end +@z + +@x +var n:0..64; {length of current word; not always a |small_number|} +@y +var n:0..64; {length of current word; not always a |small_number|} + at_hyphen:boolean; +@z + +@x +@ @<Enter as many...@>= +n:=0; p:=null; +@y +@ @<Enter as many...@>= +n:=0; p:=null; at_hyphen:=false; +@z + +@x + if cur_cmd=right_brace then return; + n:=0; p:=null; +@y + if cur_cmd=right_brace then return; + n:=0; p:=null; at_hyphen:=false; +@z + +@x +if cur_chr="-" then @<Append the value |n| to list |p|@> +@y +if cur_chr="-" then begin + at_hyphen:=true; + @<Append the value |n| to list |p|@>; +end else if hyphen_classes_en and at_hyphen and + ("0"<=cur_chr)and(cur_chr<="9") then begin + hyphen_exception_class(p):=cur_chr-"0"; + at_hyphen:=false; +end +@z + +@x +begin if n<63 then + begin q:=get_avail; link(q):=p; info(q):=n; p:=q; + end; +end +@y +begin if n<63 then + begin q:=get_avail; link(q):=p; + hyphen_exception_pos(q):=n; + hyphen_exception_class(q):=1; + p:=q; + end; +end +@z + +@x +@!digit_sensed:boolean; {should the next digit be treated as a letter?} +@y +@!digits_sensed:integer; {should the next digit be treated as a letter?} +@z + +@x +k:=0; hyf[0]:=0; digit_sensed:=false; +loop@+ begin get_x_token; +@y +k:=0; hyf[0]:=0; digits_sensed:=0; +loop@+ begin get_x_token; +@z + +@x + if cur_cmd=right_brace then goto done; + k:=0; hyf[0]:=0; digit_sensed:=false; +@y + if cur_cmd=right_brace then goto done; + k:=0; hyf[0]:=0; digits_sensed:=0; +@z + +@x +if digit_sensed or(cur_chr<"0")or(cur_chr>"9") then +@y +if (cur_chr<"0")or(cur_chr>"9")or + ((not hyphen_classes_en)and(digits_sensed>=1))or(digits_sensed>=2) then +@z + +@x + begin incr(k); hc[k]:=cur_chr; hyf[k]:=0; digit_sensed:=false; +@y + begin incr(k); hc[k]:=cur_chr; hyf[k]:=0; digits_sensed:=0; +@z + +@x +else if k<63 then + begin hyf[k]:=cur_chr-"0"; digit_sensed:=true; + end +@y +else if k<63 then begin + if 10*hyf[k]+cur_chr-"0"<=63 then begin + hyf[k]:=10*hyf[k]+cur_chr-"0"; incr(digits_sensed); + end else begin + print_err("Bad "); print_esc("patterns"); +@.Bad \\patterns@> + help1("(See Appendix H.)"); error; + end; +end +@z + +@x + if mode>0 then tail_append(new_disc); +@y + if mode>0 then tail_append(new_disc(ex_hyphen_class)); +@z + +@x +else norm_min:=h; +end; +@y +else norm_min:=h; +end; +@# +function hyphen_classes:small_number; +var h:integer; +begin + if not hyphen_classes_en then hyphen_classes:=2 + else begin + h:=int_par(hyphen_classes_code); + if h<2 then hyphen_classes:=2@+else if h>10 then hyphen_classes:=10@+ + else hyphen_classes:=h; + end; +end; + +function ex_hyphen_class:small_number; +var h:integer; +begin + if not hyphen_classes_en then ex_hyphen_class:=2 + else begin + h:=int_par(ex_hyphen_class_code); + if h<0 then ex_hyphen_class:=0@+else if h>9 then ex_hyphen_class:=9@+ + else ex_hyphen_class:=h; + end; +end; +@z + +@x +prev_graf:=(norm_min(left_hyphen_min)*@'100+norm_min(right_hyphen_min)) + *@'200000+cur_lang; +@y +prev_graf:=(norm_min(left_hyphen_min)*@'100+norm_min(right_hyphen_min)) + *@'200000+hyphen_classes*@'400+cur_lang; +@z + +@x +primitive("discretionary",discretionary,0); +@!@:discretionary_}{\.{\\discretionary} primitive@> +@y +primitive("discretionary",discretionary,0); +@!@:discretionary_}{\.{\\discretionary} primitive@> +primitive("gendiscretionary",discretionary,2); +@!@:gendiscretionary_}{\.{\\gendiscretionary} primitive@> +@z + +@x +discretionary: if chr_code=1 then + print_esc("-")@+else print_esc("discretionary"); +@y +discretionary: if chr_code=1 then + print_esc("-")@+else if chr_code=0 then print_esc("discretionary") + else print_esc("gendiscretionary"); +@z + +@x +@ The space factor does not change when we append a discretionary node, +but it starts out as 1000 in the subsidiary lists. + +@<Declare act...@>= +procedure append_discretionary; +var c:integer; {hyphen character} +begin tail_append(new_disc); +if cur_chr=1 then + begin c:=hyphen_char[cur_font]; + if c>=0 then if c<256 then pre_break(tail):=new_character(cur_font,c); + end +@y +@ The space factor does not change when we append a discretionary node, +but it starts out as 1000 in the subsidiary lists. We temporarily assign +hyphen class 0 to discretionaries generated by one of the original +primitives. This is necessary to simulate \TeX's behaviour on discretionaries +with empty pre-break text. They charge the \.{\\exhyphenpenalty}, so we +have to assign them to hyphen class |ex_hyphen_class|. Using +\.{\\gendiscretionary}, one can generate discretionaries with empty pre-break +text in other hyphen classes. + +@<Declare act...@>= +procedure append_discretionary; +var c:integer; {hyphen character} + class:integer; {hyphen class} +begin + class:=default_hyphen_class; + if cur_chr<2 then class:=0 + else if is_enabled(hyphen_classes_en,cur_cmd,cur_chr) then begin + scan_int; + if (1<=cur_val)and(cur_val<=9) then class:=cur_val + else begin + print_err("No such "); print_esc("hyphen class"); +@.No such \\hyphenpenalties@> + help2("Only numbers 1..9 are valid here.")@/ + ("Proceed; I'll take 1 instead."); + error; + end; + end; + tail_append(new_disc(class)); + if cur_chr=1 then begin + c:=hyphen_char[cur_font]; + if c>=0 then if c<256 then pre_break(tail):=new_character(cur_font,c); + if pre_break(tail)=null then hyphen_class(tail):=ex_hyphen_class + else hyphen_class(tail):=default_hyphen_class; + end +@z + +@x +@ The three discretionary lists are constructed somewhat as if they were +hboxes. A~subroutine called |build_discretionary| handles the transitions. +(This is sort of fun.) +@y +@ The three discretionary lists are constructed somewhat as if they were +hboxes. A~subroutine called |build_discretionary| handles the transitions. +(This is sort of fun.) + +When we enter this routine, |tail| is a discretionary node whose hyphen class +is zero, if it comes from \.{\\discretionary}. In that case we decide about +the proper hyphen class after reading the pre-break text. +@z + +@x +0:pre_break(tail):=p; +@y +0:begin pre_break(tail):=p; + if hyphen_class(tail)=0 then + if (p=null) then hyphen_class(tail):=ex_hyphen_class + else hyphen_class(tail):=default_hyphen_class; +end; +@z + +@x +primitive("patterns",hyph_data,1); +@!@:patterns_}{\.{\\patterns} primitive@> +@y +primitive("patterns",hyph_data,1); +@!@:patterns_}{\.{\\patterns} primitive@> +primitive("hyphenpenalties",hyph_data,2); +@!@:hyphenpenalties_}{\.{\\hyphenpenalties} primitive@> +@z + +@x +hyph_data: if chr_code=1 then print_esc("patterns") + else print_esc("hyphenation"); +@y +hyph_data: if chr_code=1 then print_esc("patterns") + else if chr_code=0 then print_esc("hyphenation") + else print_esc("hyphenpenalties"); +@z + +@x + else begin new_hyph_exceptions; goto done; + end; +@y + else if cur_chr=0 then begin + new_hyph_exceptions; + goto done; + end else if is_enabled(hyphen_classes_en,cur_cmd,cur_chr) then begin + scan_int; p:=cur_val; + scan_optional_equals; scan_int; + if (1<=p)and(p<=9) then begin + word_define(int_base+hyphen_penalty_base+p,cur_val); + end else begin + print_err("No such "); print_esc("hyphenpenalties"); +@.No such \\hyphenpenalties@> + help2("Only numbers 1..9 are valid here.")@/ + ("Proceed; I'll ignore the assignment I just read."); + error; + end; + end; +@z + +@x +@d what_lang(#)==link(#+1) {language number, in the range |0..255|} +@d what_lhm(#)==type(#+1) {minimum left fragment, in the range |1..63|} +@d what_rhm(#)==subtype(#+1) {minimum right fragment, in the range |1..63|} +@y +@d what_lhm(#)==mem[#+1].qqqq.b0 {minimum left fragment, in the range |1..63|} +@d what_rhm(#)==mem[#+1].qqqq.b1 {minimum right fragment, in the range |1..63|} +@d what_lang(#)==mem[#+1].qqqq.b2 {language number, in the range |0..255|} +@d what_hyf(#)==mem[#+1].qqqq.b3 {number of hyphen classes, in the range |2..10|} +@z + +@x + print_int(what_lhm(p)); print_char(","); + print_int(what_rhm(p)); print_char(")"); +@y + print_int(what_lhm(p)); print_char(","); + print_int(what_rhm(p)); print_char(")"); + if hyphen_classes_en then begin + print("(hyphen classes "); print_int(what_hyf(p)); + print_char(")"); + end; +@z + +@x +@ @d adv_past(#)==@+if subtype(#)=language_node then + begin cur_lang:=what_lang(#); l_hyf:=what_lhm(#); r_hyf:=what_rhm(#);@+end +@y +@ @d adv_past(#)==@+if subtype(#)=language_node then + begin cur_lang:=what_lang(#); l_hyf:=what_lhm(#); r_hyf:=what_rhm(#); + hyf_class:=what_hyf(#);@+end +@z + +@x +if l<>clang then + begin new_whatsit(language_node,small_node_size); + what_lang(tail):=l; clang:=l;@/ + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); +@y +if l<>clang then + begin new_whatsit(language_node,small_node_size); + what_lang(tail):=l; clang:=l;@/ + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); + what_hyf(tail):=hyphen_classes; +@z + +@x + what_lang(tail):=clang; + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); +@y + what_lang(tail):=clang; + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); + what_hyf(tail):=hyphen_classes; +@z + diff --git a/systems/tex-extensions/clasen/hyphenation/count.ch b/systems/tex-extensions/clasen/hyphenation/count.ch new file mode 100644 index 0000000000..7e3a63c1d7 --- /dev/null +++ b/systems/tex-extensions/clasen/hyphenation/count.ch @@ -0,0 +1,240 @@ +@x +{hyphen count int_pars go here} +@y +@d hyphen_count_state_code=69 +@d three_hyphen_demerits_code=70 {demerits for three consecutive hyphens} +@d four_hyphen_demerits_code=71 {demerits for four consecutive hyphens} +@d five_hyphen_demerits_code=72 {demerits for five or more consecutive hyphens} +@z + +@x +@d error_context_lines==int_par(error_context_lines_code) +@y +@d hyphen_count_state==int_par(hyphen_count_state_code) +@d hyphen_count_en==(hyphen_count_state>0) +@d error_context_lines==int_par(error_context_lines_code) +@d three_hyphen_demerits==int_par(three_hyphen_demerits_code) +@d four_hyphen_demerits==int_par(four_hyphen_demerits_code) +@d five_hyphen_demerits==int_par(five_hyphen_demerits_code) +@z + +@x +error_context_lines_code:print_esc("errorcontextlines"); +@y +error_context_lines_code:print_esc("errorcontextlines"); +hyphen_count_state_code:print_esc("hyphencountstate"); +three_hyphen_demerits_code:print_esc("threehyphendemerits"); +four_hyphen_demerits_code:print_esc("fourhyphendemerits"); +five_hyphen_demerits_code:print_esc("fivehyphendemerits"); +@z + +@x +primitive("errorcontextlines",assign_int,int_base+error_context_lines_code);@/ +@!@:error_context_lines_}{\.{\\errorcontextlines} primitive@> +@y +primitive("errorcontextlines",assign_int,int_base+error_context_lines_code);@/ +@!@:error_context_lines_}{\.{\\errorcontextlines} primitive@> +primitive("threehyphendemerits",assign_int,int_base+three_hyphen_demerits_code);@/ +@!@:three_hyphen_demerits_}{\.{\\threehyphendemerits} primitive@> +primitive("fourhyphendemerits",assign_int,int_base+four_hyphen_demerits_code);@/ +@!@:four_hyphen_demerits_}{\.{\\fourhyphendemerits} primitive@> +primitive("fivehyphendemerits",assign_int,int_base+five_hyphen_demerits_code);@/ +@!@:five_hyphen_demerits_}{\.{\\fivehyphendemerits} primitive@> +primitive("hyphencountstate",assign_int,int_base+hyphen_count_state_code);@/ +@!@:hyphen_count_state_}{\.{\\hyphencountstate} primitive@> +@z + +@x +escape_char:="\"; end_line_char:=carriage_return; +@y +escape_char:="\"; end_line_char:=carriage_return; +hyphen_count_state:=0; +@z + +@x +@d tight_fit=3 {fitness classification for lines shrinking 0.5 to 1.0 of their + shrinkability} +@y +@d hyphen_count_limit=6 {treat 6 consecutive hyphens like 5} +@d fitness_classes=4 {number of different fitness classes} +@d line_classes=24 {|hyphen_count_limit*fitness_classes|} +@d tight_fit=3 {fitness classification for lines shrinking 0.5 to 1.0 of their + shrinkability} +@z + +@x +@d active_node_size=3 {number of words in active nodes} +@d fitness==subtype {|very_loose_fit..tight_fit| on final line for this break} +@y +@d active_node_size=3 {number of words in active nodes} +@d line_fitness==subtype {|very_loose_fit..tight_fit| on final line for + this break; also contains information about number of consecutive hyphens} +@d fitness(#)==(line_fitness(#) mod fitness_classes) +@d hyphen_count(#)==(line_fitness(#) div fitness_classes) +@z + +@x +type(last_active):=hyphenated; line_number(last_active):=max_halfword; +subtype(last_active):=0; {the |subtype| is never examined by the algorithm} +@y +type(last_active):=hyphenated; line_number(last_active):=max_halfword; +line_fitness(last_active):=0; {the |fitness| is never examined by the + algorithm} +@z + +@x +@!fit_class:very_loose_fit..tight_fit; {possible fitness class of test line} +@y +@!fit_class:very_loose_fit..tight_fit; {possible fitness class of test line} +@!consec:0..5; {number of consecutive hyphens up to test line} +@!line_class:small_number; {|fit_class+fitness_classes*consec|} +@z + +% We have to insert active nodes not only for the four fitness classes, +% but for each fitness class/consecutive hyphen count combination. Since +% we have |fitness_classes| fitness classes and |hyphen_count_limit| different +% numbers of consecutive hyphens, we end up with up to |line_classes| active +% nodes. The following arrays are no longer indexed by fit_class, but by +% fit_class+fitness_classes*consec, where consec (<hyphen_count_limit) is the +% number of consecutive hyphens. + +@x +@!minimal_demerits:array[very_loose_fit..tight_fit] of integer; {best total + demerits known for current line class and position, given the fitness} +@!minimum_demerits:integer; {best total demerits known for current line class + and position} +@!best_place:array[very_loose_fit..tight_fit] of pointer; {how to achieve + |minimal_demerits|} +@!best_pl_line:array[very_loose_fit..tight_fit] of halfword; {corresponding + line number} +@y +@!minimal_demerits:array[0..line_classes] of integer; {best total + demerits known for current line class and position, given |fit_class+4*consec|} +@!minimum_demerits:integer; {best total demerits known for current line class + and position} +@!best_place:array[0..line_classes] of pointer; {how to achieve |minimal_demerits|} +@!best_pl_line:array[0..line_classes] of halfword; {corresponding line number} +@z + +@x +minimal_demerits[tight_fit]:=awful_bad; +minimal_demerits[decent_fit]:=awful_bad; +minimal_demerits[loose_fit]:=awful_bad; +minimal_demerits[very_loose_fit]:=awful_bad; +@y +for j:=0 to line_classes do minimal_demerits[j]:=awful_bad; +@z + +@x +for fit_class:=very_loose_fit to tight_fit do + begin if minimal_demerits[fit_class]<=minimum_demerits then + @<Insert a new active node + from |best_place[fit_class]| to |cur_p|@>; + minimal_demerits[fit_class]:=awful_bad; + end; +@y +for fit_class:=very_loose_fit to tight_fit do + for consec:=0 to hyphen_count_limit-1 do begin + line_class:=fit_class+fitness_classes*consec; + if minimal_demerits[line_class]<=minimum_demerits then + @<Insert a new active node from |best_place[line_class]| to |cur_p|@>; + minimal_demerits[line_class]:=awful_bad; + end; +@z + +@x +@<Insert a new active node from |best_place[fit_class]| to |cur_p|@>= +begin q:=get_node(passive_node_size); +link(q):=passive; passive:=q; cur_break(q):=cur_p; +@!stat incr(pass_number); serial(q):=pass_number;@+tats@;@/ +prev_break(q):=best_place[fit_class];@/ +q:=get_node(active_node_size); break_node(q):=passive; +line_number(q):=best_pl_line[fit_class]+1; +fitness(q):=fit_class; type(q):=break_type; +total_demerits(q):=minimal_demerits[fit_class]; +link(q):=r; link(prev_r):=q; prev_r:=q; +@!stat if tracing_paragraphs>0 then + @<Print a symbolic description of the new break node@>; +tats@;@/ +end +@y +@<Insert a new active node from |best_place[line_class]| to |cur_p|@>= +begin q:=get_node(passive_node_size); +link(q):=passive; passive:=q; cur_break(q):=cur_p; +@!stat incr(pass_number); serial(q):=pass_number;@+tats@;@/ +prev_break(q):=best_place[line_class];@/ +q:=get_node(active_node_size); break_node(q):=passive; +line_number(q):=best_pl_line[line_class]+1; +line_fitness(q):=line_class; type(q):=break_type; +total_demerits(q):=minimal_demerits[line_class]; +link(q):=r; link(prev_r):=q; prev_r:=q; +@!stat if tracing_paragraphs>0 then + @<Print a symbolic description of the new break node@>; +tats@;@/ +end +@z + +@x +if break_type=hyphenated then print_char("-"); +@y +if break_type=hyphenated then begin print("-("); print_int(consec); print_char(")"); end; +@z + +@x +else @<Set the value of |b| to the badness for shrinking the line, + and compute the corresponding |fit_class|@>; +@y +else @<Set the value of |b| to the badness for shrinking the line, + and compute the corresponding |fit_class|@>; +if break_type=unhyphenated then consec:=0 +else if hyphen_count(r)=hyphen_count_limit-1 then consec:=hyphen_count(r) +else consec:=hyphen_count(r)+1; +line_class:=fit_class+fitness_classes*consec; +@z + +@x +if d<=minimal_demerits[fit_class] then + begin minimal_demerits[fit_class]:=d; + best_place[fit_class]:=break_node(r); best_pl_line[fit_class]:=l; + if d<minimum_demerits then minimum_demerits:=d; + end +@y +if d<=minimal_demerits[line_class] then + begin minimal_demerits[line_class]:=d; + best_place[line_class]:=break_node(r); best_pl_line[line_class]:=l; + if d<minimum_demerits then minimum_demerits:=d; + end +@z + +@x +if (break_type=hyphenated)and(type(r)=hyphenated) then + if cur_p<>null then d:=d+double_hyphen_demerits + else d:=d+final_hyphen_demerits; +@y +if (break_type=hyphenated)and(hyphen_count(r)>0) then + if cur_p=null then d:=d+final_hyphen_demerits + else if not hyphen_count_en then d:=d+double_hyphen_demerits + else case hyphen_count(r) of + 1: d:=d+double_hyphen_demerits; + 2: d:=d+three_hyphen_demerits; + 3: d:=d+four_hyphen_demerits; + othercases d:=d+five_hyphen_demerits; + endcases; +@z + +@x +q:=get_node(active_node_size); +type(q):=unhyphenated; fitness(q):=decent_fit; +@y +q:=get_node(active_node_size); +type(q):=unhyphenated; line_fitness(q):=decent_fit+fitness_classes*0; +@z + + + + + + + + + diff --git a/systems/tex-extensions/clasen/hyphenation/etex.bib b/systems/tex-extensions/clasen/hyphenation/etex.bib new file mode 120000 index 0000000000..59519b0c14 --- /dev/null +++ b/systems/tex-extensions/clasen/hyphenation/etex.bib @@ -0,0 +1 @@ +../etex.bib
\ No newline at end of file diff --git a/systems/tex-extensions/clasen/hyphenation/hyphenation.ch b/systems/tex-extensions/clasen/hyphenation/hyphenation.ch new file mode 100644 index 0000000000..70adb89221 --- /dev/null +++ b/systems/tex-extensions/clasen/hyphenation/hyphenation.ch @@ -0,0 +1,883 @@ +@x +else begin print_err("I can't go on meeting you like this"); +@y +else begin print_err("I can't go on meeting you like this ("); + print(s); print_char(")"); +@z + +@x +@d disc_node=7 {|type| of a discretionary node} +@y +With extended hyphenation, discretionaries have an additional property, +the |hyphen_class| (which is an integer $0\leq\mathit{hyphen_class}\leq 9$). +The penalty charged for a break depends on the |hyphen_class|. + +@d disc_node=7 {|type| of a discretionary node} +@d disc_node_size=3 {number of words to allocate for a discretionary} +@d hyphen_class(#)==mem[#+2].int {the class of this hyphen} +@z + +@x +@p function new_disc:pointer; {creates an empty |disc_node|} +var p:pointer; {the new node} +begin p:=get_node(small_node_size); type(p):=disc_node; +replace_count(p):=0; pre_break(p):=null; post_break(p):=null; +new_disc:=p; +end; +@y +@p function new_disc(c:integer):pointer; {creates an empty |disc_node|} +var p:pointer; {the new node} +begin p:=get_node(disc_node_size); type(p):=disc_node; +replace_count(p):=0; pre_break(p):=null; post_break(p):=null; +hyphen_class(p):=c; new_disc:=p; +end; +@z + +@x +disc_node: begin short_display(pre_break(p)); + short_display(post_break(p));@/ +@y +disc_node: begin short_display(pre_break(p)); + if hyphen_classes_en and(hyphen_class(p)<>1) then print_int(hyphen_class(p)); + short_display(post_break(p));@/ +@z + +@x + disc_node: begin flush_node_list(pre_break(p)); + flush_node_list(post_break(p)); + end; +@y + disc_node: begin flush_node_list(pre_break(p)); + flush_node_list(post_break(p)); + free_node(p,disc_node_size); goto done; + end; +@z + +@x +disc_node: begin r:=get_node(small_node_size); + pre_break(r):=copy_node_list(pre_break(p)); + post_break(r):=copy_node_list(post_break(p)); + end; +@y +disc_node: begin r:=new_disc(hyphen_class(p)); + pre_break(r):=copy_node_list(pre_break(p)); + post_break(r):=copy_node_list(post_break(p)); + end; +@z + +@x +@d hyph_data=99 {hyphenation data ( \.{\\hyphenation}, \.{\\patterns} )} +@y +@d hyph_data=99 {hyphenation data ( \.{\\hyphenation}, \.{\\patterns}, + \.{\\hyphenpenalties} )} +@z + +@x + if m=hmode then if nest[p].pg_field <> @'40600000 then + begin print(" (language"); print_int(nest[p].pg_field mod @'200000); + print(":hyphenmin"); print_int(nest[p].pg_field div @'20000000); + print_char(","); print_int((nest[p].pg_field div @'200000) mod @'100); + print_char(")"); + end; +@y + if m=hmode then if nest[p].pg_field <> @'40601000 then begin + print(" (language"); print_int(nest[p].pg_field mod @'400); + print(":hyphenmin"); print_int(nest[p].pg_field div @'20000000); + print_char(","); print_int((nest[p].pg_field div @'200000) mod @'100); + if hyphen_classes_en then begin + print(":hyphenclasses"); + print_int((nest[p].pg_field div @'400) mod @'20); + end; + print_char(")"); + end; +@z + +@x +@d hyphen_penalty_code=3 {penalty for break after discretionary hyphen} +@d ex_hyphen_penalty_code=4 {penalty for break after explicit hyphen} +@y +@d hyphen_classes_code=3 {number of hyphenation classes} +@d ex_hyphen_class_code=4 {hyphenation class of automatically + inserted empty discretionaries} +@z + +@x +{hyphen classes int_pars go here} +@y +@d hyphen_classes_state_code=58 +@d hyphen_penalty_base=59 {10 penalties for breaks after discretionary hyphens, + numbered from 0 to 9} +@z + +@x +{hyphen count int_pars go here} +@y +@d hyphen_count_state_code=69 +@d three_hyphen_demerits_code=70 {demerits for three consecutive hyphens} +@d four_hyphen_demerits_code=71 {demerits for four consecutive hyphens} +@d five_hyphen_demerits_code=72 {demerits for five or more consecutive hyphens} +@z + +@x +@d hyphen_penalty==int_par(hyphen_penalty_code) +@d ex_hyphen_penalty==int_par(ex_hyphen_penalty_code) +@y +@d hyphen_classes_state==int_par(hyphen_classes_state_code) +@d hyphen_classes_en==(hyphen_classes_state>0) +@d hyphen_penalties(#)==int_par(hyphen_penalty_base+#) +@d default_hyphen_class=1 +@d hyphen_penalty==hyphen_penalties(default_hyphen_class) +@d ex_hyphen_penalty==hyphen_penalties(ex_hyphen_class) +@z + +@x +@d error_context_lines==int_par(error_context_lines_code) +@y +@d hyphen_count_state==int_par(hyphen_count_state_code) +@d hyphen_count_en==(hyphen_count_state>0) +@d error_context_lines==int_par(error_context_lines_code) +@d three_hyphen_demerits==int_par(three_hyphen_demerits_code) +@d four_hyphen_demerits==int_par(four_hyphen_demerits_code) +@d five_hyphen_demerits==int_par(five_hyphen_demerits_code) +@z + +@x +hyphen_penalty_code:print_esc("hyphenpenalty"); +ex_hyphen_penalty_code:print_esc("exhyphenpenalty"); +@y +hyphen_classes_code:print_esc("hyphenclasses"); +ex_hyphen_class_code:print_esc("exhyphenclass"); +@z + +@x +error_context_lines_code:print_esc("errorcontextlines"); +@y +error_context_lines_code:print_esc("errorcontextlines"); +hyphen_count_state_code:print_esc("hyphencountstate"); +three_hyphen_demerits_code:print_esc("threehyphendemerits"); +four_hyphen_demerits_code:print_esc("fourhyphendemerits"); +five_hyphen_demerits_code:print_esc("fivehyphendemerits"); +hyphen_classes_state_code:print_esc("hyphenclassesstate"); +@z + +@x +primitive("hyphenpenalty",assign_int,int_base+hyphen_penalty_code);@/ +@!@:hyphen_penalty_}{\.{\\hyphenpenalty} primitive@> +primitive("exhyphenpenalty",assign_int,int_base+ex_hyphen_penalty_code);@/ +@!@:ex_hyphen_penalty_}{\.{\\exhyphenpenalty} primitive@> +@y +primitive("hyphenclasses",assign_int,int_base+hyphen_classes_code);@/ +@!@:hyphen_classes_}{\.{\\hyphenclasses} primitive@> +primitive("exhyphenclass",assign_int,int_base+ex_hyphen_class_code);@/ +@!@:ex_hyphen_class_}{\.{\\exhyphenclass} primitive@> +@z + +@x +primitive("errorcontextlines",assign_int,int_base+error_context_lines_code);@/ +@!@:error_context_lines_}{\.{\\errorcontextlines} primitive@> +@y +primitive("errorcontextlines",assign_int,int_base+error_context_lines_code);@/ +@!@:error_context_lines_}{\.{\\errorcontextlines} primitive@> +primitive("threehyphendemerits",assign_int,int_base+three_hyphen_demerits_code);@/ +@!@:three_hyphen_demerits_}{\.{\\threehyphendemerits} primitive@> +primitive("fourhyphendemerits",assign_int,int_base+four_hyphen_demerits_code);@/ +@!@:four_hyphen_demerits_}{\.{\\fourhyphendemerits} primitive@> +primitive("fivehyphendemerits",assign_int,int_base+five_hyphen_demerits_code);@/ +@!@:five_hyphen_demerits_}{\.{\\fivehyphendemerits} primitive@> +primitive("hyphencountstate",assign_int,int_base+hyphen_count_state_code);@/ +@!@:hyphen_count_state_}{\.{\\hyphencountstate} primitive@> +primitive("hyphenpenalty",assign_int,int_base+hyphen_penalty_base+1);@/ +@!@:hyphen_penalty_}{\.{\\hyphenpenalty} primitive@> +primitive("exhyphenpenalty",assign_int,int_base+hyphen_penalty_base + +ex_hyphen_class);@/ +@!@:ex_hyphen_penalty_}{\.{\\exhyphenpenalty} primitive@> +primitive("hyphenclassesstate",assign_int,int_base+hyphen_classes_state_code);@/ +@!@:hyphen_classes_state_}{\.{\\hyphenclassesstate} primitive@> +@z + +@x +escape_char:="\"; end_line_char:=carriage_return; +@y +escape_char:="\"; end_line_char:=carriage_return; +hyphen_count_state:=0; +hyphen_classes_state:=0; +@z + +@x +@ Here is a procedure that displays the contents of |eqtb[n]| +symbolically. + +@p@t\4@>@<Declare the procedure called |print_cmd_chr|@>@;@/ +@y +@ Here is a procedure that displays the contents of |eqtb[n]| +symbolically. + +@p@t\4@>@<Declare the procedure called |print_cmd_chr|@>@;@/ +function is_enabled(@!b:boolean;@!j:quarterword;@!k:halfword):boolean; +begin + if not b then begin + print_err("Improper "); print_cmd_chr(j,k); + help1("Sorry, this optional feature has been disabled."); error; + end; + is_enabled:=b; +end; +@z + +@x +init_cur_lang:=prev_graf mod @'200000; +@y +init_cur_lang:=prev_graf mod @'400; +init_hyf_class:=(prev_graf div @'400) mod @'20; +@z + +@x +@d tight_fit=3 {fitness classification for lines shrinking 0.5 to 1.0 of their + shrinkability} +@y +@d hyphen_count_limit=6 {treat 6 consecutive hyphens like 5} +@d fitness_classes=4 {number of different fitness classes} +@d line_classes=24 {|hyphen_count_limit*fitness_classes|} +@d tight_fit=3 {fitness classification for lines shrinking 0.5 to 1.0 of their + shrinkability} +@z + +@x +@d active_node_size=3 {number of words in active nodes} +@d fitness==subtype {|very_loose_fit..tight_fit| on final line for this break} +@y +@d active_node_size=3 {number of words in active nodes} +@d line_fitness==subtype {|very_loose_fit..tight_fit| on final line for + this break; also contains information about number of consecutive hyphens} +@d fitness(#)==(line_fitness(#) mod fitness_classes) +@d hyphen_count(#)==(line_fitness(#) div fitness_classes) +@z + +@x +type(last_active):=hyphenated; line_number(last_active):=max_halfword; +subtype(last_active):=0; {the |subtype| is never examined by the algorithm} +@y +type(last_active):=hyphenated; line_number(last_active):=max_halfword; +line_fitness(last_active):=0; {the |fitness| is never examined by the + algorithm} +@z + +@x +@!fit_class:very_loose_fit..tight_fit; {possible fitness class of test line} +@y +@!fit_class:very_loose_fit..tight_fit; {possible fitness class of test line} +@!consec:0..5; {number of consecutive hyphens up to test line} +@!line_class:small_number; {|fit_class+fitness_classes*consec|} +@z + +@x +@!minimal_demerits:array[very_loose_fit..tight_fit] of integer; {best total + demerits known for current line class and position, given the fitness} +@!minimum_demerits:integer; {best total demerits known for current line class + and position} +@!best_place:array[very_loose_fit..tight_fit] of pointer; {how to achieve + |minimal_demerits|} +@!best_pl_line:array[very_loose_fit..tight_fit] of halfword; {corresponding + line number} +@y +@!minimal_demerits:array[0..line_classes] of integer; {best total + demerits known for current line class and position, given |fit_class+4*consec|} +@!minimum_demerits:integer; {best total demerits known for current line class + and position} +@!best_place:array[0..line_classes] of pointer; {how to achieve |minimal_demerits|} +@!best_pl_line:array[0..line_classes] of halfword; {corresponding line number} +@z + +@x +minimal_demerits[tight_fit]:=awful_bad; +minimal_demerits[decent_fit]:=awful_bad; +minimal_demerits[loose_fit]:=awful_bad; +minimal_demerits[very_loose_fit]:=awful_bad; +@y +for j:=0 to line_classes do minimal_demerits[j]:=awful_bad; +@z + +@x +for fit_class:=very_loose_fit to tight_fit do + begin if minimal_demerits[fit_class]<=minimum_demerits then + @<Insert a new active node + from |best_place[fit_class]| to |cur_p|@>; + minimal_demerits[fit_class]:=awful_bad; + end; +@y +for fit_class:=very_loose_fit to tight_fit do + for consec:=0 to hyphen_count_limit-1 do begin + line_class:=fit_class+fitness_classes*consec; + if minimal_demerits[line_class]<=minimum_demerits then + @<Insert a new active node from |best_place[line_class]| to |cur_p|@>; + minimal_demerits[line_class]:=awful_bad; + end; +@z + +@x +@<Insert a new active node from |best_place[fit_class]| to |cur_p|@>= +begin q:=get_node(passive_node_size); +link(q):=passive; passive:=q; cur_break(q):=cur_p; +@!stat incr(pass_number); serial(q):=pass_number;@+tats@;@/ +prev_break(q):=best_place[fit_class];@/ +q:=get_node(active_node_size); break_node(q):=passive; +line_number(q):=best_pl_line[fit_class]+1; +fitness(q):=fit_class; type(q):=break_type; +total_demerits(q):=minimal_demerits[fit_class]; +link(q):=r; link(prev_r):=q; prev_r:=q; +@!stat if tracing_paragraphs>0 then + @<Print a symbolic description of the new break node@>; +tats@;@/ +end +@y +@<Insert a new active node from |best_place[line_class]| to |cur_p|@>= +begin q:=get_node(passive_node_size); +link(q):=passive; passive:=q; cur_break(q):=cur_p; +@!stat incr(pass_number); serial(q):=pass_number;@+tats@;@/ +prev_break(q):=best_place[line_class];@/ +q:=get_node(active_node_size); break_node(q):=passive; +line_number(q):=best_pl_line[line_class]+1; +line_fitness(q):=line_class; type(q):=break_type; +total_demerits(q):=minimal_demerits[line_class]; +link(q):=r; link(prev_r):=q; prev_r:=q; +@!stat if tracing_paragraphs>0 then + @<Print a symbolic description of the new break node@>; +tats@;@/ +end +@z + +@x +if break_type=hyphenated then print_char("-"); +@y +if break_type=hyphenated then begin print("-("); print_int(consec); print_char(")"); end; +@z + +@x +else @<Set the value of |b| to the badness for shrinking the line, + and compute the corresponding |fit_class|@>; +@y +else @<Set the value of |b| to the badness for shrinking the line, + and compute the corresponding |fit_class|@>; +if break_type=unhyphenated then consec:=0 +else if hyphen_count(r)=hyphen_count_limit-1 then consec:=hyphen_count(r) +else consec:=hyphen_count(r)+1; +line_class:=fit_class+fitness_classes*consec; +@z + +@x +if d<=minimal_demerits[fit_class] then + begin minimal_demerits[fit_class]:=d; + best_place[fit_class]:=break_node(r); best_pl_line[fit_class]:=l; + if d<minimum_demerits then minimum_demerits:=d; + end +@y +if d<=minimal_demerits[line_class] then + begin minimal_demerits[line_class]:=d; + best_place[line_class]:=break_node(r); best_pl_line[line_class]:=l; + if d<minimum_demerits then minimum_demerits:=d; + end +@z + +@x +if (break_type=hyphenated)and(type(r)=hyphenated) then + if cur_p<>null then d:=d+double_hyphen_demerits + else d:=d+final_hyphen_demerits; +@y +if (break_type=hyphenated)and(hyphen_count(r)>0) then + if cur_p=null then d:=d+final_hyphen_demerits + else if not hyphen_count_en then d:=d+double_hyphen_demerits + else case hyphen_count(r) of + 1: d:=d+double_hyphen_demerits; + 2: d:=d+three_hyphen_demerits; + 3: d:=d+four_hyphen_demerits; + othercases d:=d+five_hyphen_demerits; + endcases; +@z + +@x +q:=get_node(active_node_size); +type(q):=unhyphenated; fitness(q):=decent_fit; +@y +q:=get_node(active_node_size); +type(q):=unhyphenated; line_fitness(q):=decent_fit+fitness_classes*0; +@z + +@x +@ The following code knows that discretionary texts contain +only character nodes, kern nodes, box nodes, rule nodes, and ligature nodes. + +@<Try to break after a discretionary fragment...@>= +begin s:=pre_break(cur_p); disc_width:=0; +if s=null then try_break(ex_hyphen_penalty,hyphenated) +else begin repeat @<Add the width of node |s| to |disc_width|@>; + s:=link(s); + until s=null; + act_width:=act_width+disc_width; + try_break(hyphen_penalty,hyphenated); + act_width:=act_width-disc_width; + end; +@y +@ The following code knows that discretionary texts contain +only character nodes, kern nodes, box nodes, rule nodes, and ligature nodes. +If |ex_hyphen_penalty=0|, we might encounter automatically inserted empty +discretionary nodes of class~0. We never break at these discretionaries, +nevertheless they are necessary to differentiate the output of +\.{\\showhyphens} for automatically inserted hyphens of classes 0 and~1. + +@<Try to break after a discretionary fragment...@>= +begin if hyphen_class(cur_p)>0 then begin + s:=pre_break(cur_p); disc_width:=0; + while s<>null do begin + @<Add the width of node |s| to |disc_width|@>; + s:=link(s); + end; + act_width:=act_width+disc_width; + try_break(hyphen_penalties(hyphen_class(cur_p)),hyphenated); + act_width:=act_width-disc_width; + end; +@z + +@x +begin t:=replace_count(q); +@y +begin t:=replace_count(q); hyphen_class(q):=1; +@z + +@x +cur_lang:=init_cur_lang; l_hyf:=init_l_hyf; r_hyf:=init_r_hyf; +@y +cur_lang:=init_cur_lang; l_hyf:=init_l_hyf; r_hyf:=init_r_hyf; +hyf_class:=init_hyf_class; +@z + +@x +@!hyf_bchar:halfword; {boundary character after $c_n$} +@y +@!hyf_bchar:halfword; {boundary character after $c_n$} +@!hyf_class,@!init_hyf_class:integer; +@z + +@x +@!hyf:array [0..64] of 0..9; {odd values indicate discretionary hyphens} +@y +@!hyf:array [0..64] of small_number; +@z + +@x +@<If no hyphens were found, |return|@>= +for j:=l_hyf to hn-r_hyf do if odd(hyf[j]) then goto found1; +return; +found1: +@y +@<If no hyphens were found, |return|@>= +for j:=l_hyf to hn-r_hyf do if hyf[j]>0 then goto found1; +return; +found1: +@z + +@x +@d set_cur_r==begin if j<n then cur_r:=qi(hu[j+1])@+else cur_r:=bchar; + if odd(hyf[j]) then cur_rh:=hchar@+else cur_rh:=non_char; +@y +@d set_cur_r==begin if j<n then cur_r:=qi(hu[j+1])@+else cur_r:=bchar; + if hyf[j]>0 then cur_rh:=hchar@+else cur_rh:=non_char; +@z + +@x + else begin if hchar<non_char then if odd(hyf[j]) then +@y + else begin if hchar<non_char then if hyf[j]>0 then +@z + +@x + while link(s)>null do s:=link(s); + if odd(hyf[j-1]) then +@y + while link(s)>null do s:=link(s); + if hyf[j-1]>0 then +@z + +@x +repeat r:=get_node(small_node_size); +@y +repeat r:=new_disc(hyf[hyphen_passed]); +@z + +@x +hyphen_passed:=j-1; link(hold_head):=null; +until not odd(hyf[j-1]) +@y +hyphen_passed:=j-1; link(hold_head):=null; +until hyf[j-1]=0 +@z + +@x +found: for j:=0 to l_hyf-1 do hyf[j]:=0; +for j:=0 to r_hyf-1 do hyf[hn-j]:=0 +@y +for j:=l_hyf to hn-r_hyf do hyf[j]:=hyf[j] mod hyf_class; +found: for j:=0 to l_hyf-1 do hyf[j]:=0; +for j:=0 to r_hyf-1 do hyf[hn-j]:=0 +@z + +@x +The words in the table point to lists in |mem| that specify hyphen positions +in their |info| fields. The list for $c_1\ldots c_n$ contains the number |k| if +the word $c_1\ldots c_n$ has a discretionary hyphen between $c_k$ and +$c_{k+1}$. +@y +The words in the table point to lists in |mem| that specify hyphen positions +and classes in their |type| and |subtype| fields. The list for $c_1\ldots c_n$ +contains the pair |k|, |l| if the word $c_1\ldots c_n$ has a discretionary +hyphen of class |l| between $c_k$ and $c_{k+1}$. + +@d hyphen_exception_pos==type +@d hyphen_exception_class==subtype +@z + +@x +while s<>null do + begin hyf[info(s)]:=1; s:=link(s); + end +@y +while s<>null do + begin hyf[hyphen_exception_pos(s)]:=hyphen_exception_class(s); s:=link(s); + end +@z + +@x +var n:0..64; {length of current word; not always a |small_number|} +@y +var n:0..64; {length of current word; not always a |small_number|} + at_hyphen:boolean; +@z + +@x +@ @<Enter as many...@>= +n:=0; p:=null; +@y +@ @<Enter as many...@>= +n:=0; p:=null; at_hyphen:=false; +@z + +@x + if cur_cmd=right_brace then return; + n:=0; p:=null; +@y + if cur_cmd=right_brace then return; + n:=0; p:=null; at_hyphen:=false; +@z + +@x +if cur_chr="-" then @<Append the value |n| to list |p|@> +@y +if cur_chr="-" then begin + at_hyphen:=true; + @<Append the value |n| to list |p|@>; +end else if hyphen_classes_en and at_hyphen and + ("0"<=cur_chr)and(cur_chr<="9") then begin + hyphen_exception_class(p):=cur_chr-"0"; + at_hyphen:=false; +end +@z + +@x +begin if n<63 then + begin q:=get_avail; link(q):=p; info(q):=n; p:=q; + end; +end +@y +begin if n<63 then + begin q:=get_avail; link(q):=p; + hyphen_exception_pos(q):=n; + hyphen_exception_class(q):=1; + p:=q; + end; +end +@z + +@x +@!digit_sensed:boolean; {should the next digit be treated as a letter?} +@y +@!digits_sensed:integer; {should the next digit be treated as a letter?} +@z + +@x +k:=0; hyf[0]:=0; digit_sensed:=false; +loop@+ begin get_x_token; +@y +k:=0; hyf[0]:=0; digits_sensed:=0; +loop@+ begin get_x_token; +@z + +@x + if cur_cmd=right_brace then goto done; + k:=0; hyf[0]:=0; digit_sensed:=false; +@y + if cur_cmd=right_brace then goto done; + k:=0; hyf[0]:=0; digits_sensed:=0; +@z + +@x +if digit_sensed or(cur_chr<"0")or(cur_chr>"9") then +@y +if (cur_chr<"0")or(cur_chr>"9")or + ((not hyphen_classes_en)and(digits_sensed>=1))or(digits_sensed>=2) then +@z + +@x + begin incr(k); hc[k]:=cur_chr; hyf[k]:=0; digit_sensed:=false; +@y + begin incr(k); hc[k]:=cur_chr; hyf[k]:=0; digits_sensed:=0; +@z + +@x +else if k<63 then + begin hyf[k]:=cur_chr-"0"; digit_sensed:=true; + end +@y +else if k<63 then begin + if 10*hyf[k]+cur_chr-"0"<=63 then begin + hyf[k]:=10*hyf[k]+cur_chr-"0"; incr(digits_sensed); + end else begin + print_err("Bad "); print_esc("patterns"); +@.Bad \\patterns@> + help1("(See Appendix H.)"); error; + end; +end +@z + +@x + if mode>0 then tail_append(new_disc); +@y + if mode>0 then tail_append(new_disc(ex_hyphen_class)); +@z + +@x +else norm_min:=h; +end; +@y +else norm_min:=h; +end; +@# +function hyphen_classes:small_number; +var h:integer; +begin + if not hyphen_classes_en then hyphen_classes:=2 + else begin + h:=int_par(hyphen_classes_code); + if h<2 then hyphen_classes:=2@+else if h>10 then hyphen_classes:=10@+ + else hyphen_classes:=h; + end; +end; + +function ex_hyphen_class:small_number; +var h:integer; +begin + if not hyphen_classes_en then ex_hyphen_class:=2 + else begin + h:=int_par(ex_hyphen_class_code); + if h<0 then ex_hyphen_class:=0@+else if h>9 then ex_hyphen_class:=9@+ + else ex_hyphen_class:=h; + end; +end; +@z + +@x +prev_graf:=(norm_min(left_hyphen_min)*@'100+norm_min(right_hyphen_min)) + *@'200000+cur_lang; +@y +prev_graf:=(norm_min(left_hyphen_min)*@'100+norm_min(right_hyphen_min)) + *@'200000+hyphen_classes*@'400+cur_lang; +@z + +@x +primitive("discretionary",discretionary,0); +@!@:discretionary_}{\.{\\discretionary} primitive@> +@y +primitive("discretionary",discretionary,0); +@!@:discretionary_}{\.{\\discretionary} primitive@> +primitive("gendiscretionary",discretionary,2); +@!@:gendiscretionary_}{\.{\\gendiscretionary} primitive@> +@z + +@x +discretionary: if chr_code=1 then + print_esc("-")@+else print_esc("discretionary"); +@y +discretionary: if chr_code=1 then + print_esc("-")@+else if chr_code=0 then print_esc("discretionary") + else print_esc("gendiscretionary"); +@z + +@x +@ The space factor does not change when we append a discretionary node, +but it starts out as 1000 in the subsidiary lists. + +@<Declare act...@>= +procedure append_discretionary; +var c:integer; {hyphen character} +begin tail_append(new_disc); +if cur_chr=1 then + begin c:=hyphen_char[cur_font]; + if c>=0 then if c<256 then pre_break(tail):=new_character(cur_font,c); + end +@y +@ The space factor does not change when we append a discretionary node, +but it starts out as 1000 in the subsidiary lists. We temporarily assign +hyphen class 0 to discretionaries generated by one of the original +primitives. This is necessary to simulate \TeX's behaviour on discretionaries +with empty pre-break text. They charge the \.{\\exhyphenpenalty}, so we +have to assign them to hyphen class |ex_hyphen_class|. Using +\.{\\gendiscretionary}, one can generate discretionaries with empty pre-break +text in other hyphen classes. + +@<Declare act...@>= +procedure append_discretionary; +var c:integer; {hyphen character} + class:integer; {hyphen class} +begin + class:=default_hyphen_class; + if cur_chr<2 then class:=0 + else if is_enabled(hyphen_classes_en,cur_cmd,cur_chr) then begin + scan_int; + if (1<=cur_val)and(cur_val<=9) then class:=cur_val + else begin + print_err("No such "); print_esc("hyphen class"); +@.No such \\hyphenpenalties@> + help2("Only numbers 1..9 are valid here.")@/ + ("Proceed; I'll take 1 instead."); + error; + end; + end; + tail_append(new_disc(class)); + if cur_chr=1 then begin + c:=hyphen_char[cur_font]; + if c>=0 then if c<256 then pre_break(tail):=new_character(cur_font,c); + if pre_break(tail)=null then hyphen_class(tail):=ex_hyphen_class + else hyphen_class(tail):=default_hyphen_class; + end +@z + +@x +@ The three discretionary lists are constructed somewhat as if they were +hboxes. A~subroutine called |build_discretionary| handles the transitions. +(This is sort of fun.) +@y +@ The three discretionary lists are constructed somewhat as if they were +hboxes. A~subroutine called |build_discretionary| handles the transitions. +(This is sort of fun.) + +When we enter this routine, |tail| is a discretionary node whose hyphen class +is zero, if it comes from \.{\\discretionary}. In that case we decide about +the proper hyphen class after reading the pre-break text. +@z + +@x +0:pre_break(tail):=p; +@y +0:begin pre_break(tail):=p; + if hyphen_class(tail)=0 then + if (p=null) then hyphen_class(tail):=ex_hyphen_class + else hyphen_class(tail):=default_hyphen_class; +end; +@z + +@x +primitive("patterns",hyph_data,1); +@!@:patterns_}{\.{\\patterns} primitive@> +@y +primitive("patterns",hyph_data,1); +@!@:patterns_}{\.{\\patterns} primitive@> +primitive("hyphenpenalties",hyph_data,2); +@!@:hyphenpenalties_}{\.{\\hyphenpenalties} primitive@> +@z + +@x +hyph_data: if chr_code=1 then print_esc("patterns") + else print_esc("hyphenation"); +@y +hyph_data: if chr_code=1 then print_esc("patterns") + else if chr_code=0 then print_esc("hyphenation") + else print_esc("hyphenpenalties"); +@z + +@x + else begin new_hyph_exceptions; goto done; + end; +@y + else if cur_chr=0 then begin + new_hyph_exceptions; + goto done; + end else if is_enabled(hyphen_classes_en,cur_cmd,cur_chr) then begin + scan_int; p:=cur_val; + scan_optional_equals; scan_int; + if (1<=p)and(p<=9) then begin + word_define(int_base+hyphen_penalty_base+p,cur_val); + end else begin + print_err("No such "); print_esc("hyphenpenalties"); +@.No such \\hyphenpenalties@> + help2("Only numbers 1..9 are valid here.")@/ + ("Proceed; I'll ignore the assignment I just read."); + error; + end; + end; +@z + +@x +@d what_lang(#)==link(#+1) {language number, in the range |0..255|} +@d what_lhm(#)==type(#+1) {minimum left fragment, in the range |1..63|} +@d what_rhm(#)==subtype(#+1) {minimum right fragment, in the range |1..63|} +@y +@d what_lhm(#)==mem[#+1].qqqq.b0 {minimum left fragment, in the range |1..63|} +@d what_rhm(#)==mem[#+1].qqqq.b1 {minimum right fragment, in the range |1..63|} +@d what_lang(#)==mem[#+1].qqqq.b2 {language number, in the range |0..255|} +@d what_hyf(#)==mem[#+1].qqqq.b3 {number of hyphen classes, in the range |2..10|} +@z + +@x + print_int(what_lhm(p)); print_char(","); + print_int(what_rhm(p)); print_char(")"); +@y + print_int(what_lhm(p)); print_char(","); + print_int(what_rhm(p)); print_char(")"); + if hyphen_classes_en then begin + print("(hyphen classes "); print_int(what_hyf(p)); + print_char(")"); + end; +@z + +@x +@ @d adv_past(#)==@+if subtype(#)=language_node then + begin cur_lang:=what_lang(#); l_hyf:=what_lhm(#); r_hyf:=what_rhm(#);@+end +@y +@ @d adv_past(#)==@+if subtype(#)=language_node then + begin cur_lang:=what_lang(#); l_hyf:=what_lhm(#); r_hyf:=what_rhm(#); + hyf_class:=what_hyf(#);@+end +@z + +@x +if l<>clang then + begin new_whatsit(language_node,small_node_size); + what_lang(tail):=l; clang:=l;@/ + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); +@y +if l<>clang then + begin new_whatsit(language_node,small_node_size); + what_lang(tail):=l; clang:=l;@/ + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); + what_hyf(tail):=hyphen_classes; +@z + +@x + what_lang(tail):=clang; + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); +@y + what_lang(tail):=clang; + what_lhm(tail):=norm_min(left_hyphen_min); + what_rhm(tail):=norm_min(right_hyphen_min); + what_hyf(tail):=hyphen_classes; +@z + diff --git a/systems/tex-extensions/clasen/hyphenation/hyphenation.tex b/systems/tex-extensions/clasen/hyphenation/hyphenation.tex new file mode 100644 index 0000000000..5fbbd5a14e --- /dev/null +++ b/systems/tex-extensions/clasen/hyphenation/hyphenation.tex @@ -0,0 +1,249 @@ +\documentclass[preprint]{ltugboat} +\usepackage{shortvrb} +\usepackage{booktabs} + +\providecommand{\meta}[1]{\ensuremath{\langle\textit{#1}\rangle}} +\makeatletter +\providecommand{\eTeX}{$\m@th\varepsilon$-\TeX} +\makeatother +\hyphenation{pa-ra-me-ter} + +\emergencystretch=4mm +\title{An extension of \TeX's hyphenation algorithm} +\author{Matthias Clasen} +%\address{} +\netaddress{maclas@gmx.de} +\setcounter{page}{1} + +\begin{document} +\maketitle +\MakeShortVerb{\|} + +\section{Introduction} + +\TeX{}'s hyphenation algorithm works quite well and has been adapted for +a large number of languages. But there are a few shortcomings, which have +been noticed long ago \cite{Mittelbach:90,MR:92}: +\begin{itemize} +\item + While \TeX{} has provisions to deal with general discretionaries like + \textit{ck$\rightarrow$k-k}, there is no way to generate these automatically + in the same way as ordinary discretionaries. They have to be entered manually + using macros like |"ck| from |german.sty|. +\item + \TeX{} has \cs{doublehyphendemerits} to discourage consecutive hyphenated + lines, but there is no provision to enforce a specification like ``no more + that three consecutive hyphenated lines''. +\item + All breakpoints inserted by the hyphenation algorithm are treated the same, + although their quality might be quite different. Especially for languages + like German it would be nice to prefer breakpoints at word boundaries over + intra-word breakpoints. +\end{itemize} + +I don't tackle the first problem, but I do propose a solution for the other +problems in the sequel. + +\section{Extensions to \TeX{}} + +\subsection{Consecutive hyphens} +\label{sec:linebreaking} + +\TeX's linebreaking algorithm does already keep track of the hyphenation +status of the feasible line breaks. All one has to do to solve the second +problem is to change this boolean parameter into an integer and accumulate +the information for consecutive line breaks. This is not very difficult, since +the data structures for line breaks (the |active_node| node type) has enough +room for this information. + +Well, actually this is not quite true, since it fails to preserve the dynamic +programming principle of optimality and thus might miss optimal solutions. To +do it properly we have to maintain information about optimal solutions not +only for each fitness class, but for each combination of fitness class and +number of consecutive hyphens. This is what I have implemented. + +With this bookkeeping in place, it would in principle be possible to apply +different demerits for arbitrary numbers of consecutive hyphens. I have +implemented this only for three, four, and five lines since higher numbers are +unlikely to be relevant in practice. The new demerits are made available as +integer parameters +\begin{quotation} + \cs{threehyphendemerits},\\ + \cs{fourhyphendemerits}, +\end{quotation} +and +\begin{quotation} + \cs{fivehyphendemerits}. +\end{quotation} + +\subsection{Hyphen classes} +\label{sec:hyphenclasses} + +\TeX's hyphenation algorithm works by assigning numbers to inter-letter + positions in a word and reducing these numbers modulo~2 to obtain a binary + alternative (breakpoint or not). A natural extension of this algorithm would + be to reduce modulo~$n$ to obtain an $n$-ary alternative, e.\,g.\ the class + of the hyphen (where a higher class means a more acceptable breakpoint). + + This of course implies that the patterns have to be generated with this in + mind, i.\,e.\ the number of hyphen classes becomes a parameter of the pattern + generation algorithm like \cs{lefthyphenmin} and \cs{righthyphenmin}. This + has been implemented in a change to \textsf{patgen} which enables + it to deal with up~to~10 hyphen classes and eject inter-letter + values of up~to~63. My preliminary tests seem to indicate that it is indeed + possible to generate useful patterns for more than one hyphen class, but more + extensive experiments are needed. + + The extended \TeX{} knows the number of hyphen classes through the integer + parameter + \begin{quotation} + \cs{hyphenclasses}. + \end{quotation} + It is saved and restored like \cs{lefthyphenmin} and + \cs{righthyphenmin} when the language is changed in the middle of a + paragraph. + + To make use of the hyphen classes, it is necessary to make \TeX{} charge + different penalties for them. To this end, the extended \TeX{} has the + integer parameters + \begin{quotation} + \cs{hyphenpenalties}\meta{hyphen class},\\ + \null\qquad\qquad\qquad\qquad$(1\leq\meta{hyphen class}\leq 9)$. + \end{quotation} + The parameters \cs{hyphen\-penalty} and \cs{exhyphen\-penalty} are now + synonyms for \cs{hyphen\-penalties1} and \cs{hyphen\-penalties}\meta{n}, + where \meta{n} is the value of the new integer parameter + \begin{quotation} + \cs{exhyphenclass}. + \end{quotation} + It determines the class to which the automatically inserted discretionaries + after~|-| and discretionaries with empty pre-break text generated by \cs{-} + or \cs{discretionary} belong (\cs{exhyphen\-class} may also be~0 to disable + breaking after~|-|). + + Discretionaries of any class can be inserted with the help of the + new primitive + \begin{quotation} + \cs{gendiscretionary}\meta{hyphen class}. + \end{quotation} + + If these extensions are enabled, the \TeX{} primitives \cs{patterns} + and \cs{hyphenation} accept slightly more general input: \cs{patterns} + accepts two-digit values up~to~63 between letters and \cs{hyphenation} + accepts one-digit hyphen classes after hyphens. + +\subsection{Enabling the extensions} + + Following the model of \eTeX, the extensions described above are + protected by two integer parameters, + \begin{quotation} + \cs{hyphencountstate} + \end{quotation} + and + \begin{quotation} + \cs{hyphenclassesstate}, + \end{quotation} + which must be set to a positive value to enable them. With both + state variables set to zero the extended \TeX{} still passes the trip test + (with small differences in memory usage caused by the different size of + discretionary nodes). In this mode, the primitives \cs{gendiscretionary} and + \cs{hyphenpenalties} are forbidden and the values of the integer parameters + \cs{exhyphenclass}, \cs{hyphen\-classes}, + \cs{hyphen\-penalties3}, \dots, \cs{hyphen\-penalties9}, + \cs{three\-hy\-phen\-demerits}, + \dots, \cs{five\-hyphen\-demerits} are ignored. + +\section{Generating patterns} + +I have done some experiments to see if it is possible to generate +patterns for more than one hyphen class. My test data was a list +of approximately 25\,000 German words, many of them compound. I used +\TeX's hyphenation to insert hyphens and flagged the inter-word hyphens +manually as hyphen class~2. For the last experiment, I broke all compound +words in their constituents. + +\medskip + +\hspace{-4cm}\hfill +\begin{tabular}{cccccc} +\toprule +break points&\multicolumn{2}{c}{patterns}&good&bad&missed\\ +&kB&\#&\%&\%&\%\\ +\midrule +all&65582&9271&99.02&0.10&0.98\\ +all in one class&32892&4970&99.98&0.01&0.02\\ +only inter-word&37407&5386&99.63&0.01&0.37\\ +only intra-word&14653&2226&99.70&0.04&0.30\\ +\bottomrule +\end{tabular} + +\medskip + +Note that in the first experiment, hyphens of class~1 which are inserted +where a hyphen of class~2 was wanted do count as bad. The percentage +of \emph{really} bad (i.\,e.\ too permissive) hyphens is much smaller. + +It's not too surprising that the patterns for two hyphen classes are bigger +than the patterns for one hyphen class, since they contain more information. + +The results of the third and fourth experiment might encourage a different +approach to the problem of weighting breaks: Replace the current hyphenation +algorithm by two passes of the same algorithm, where the first pass would use +special patterns to find word boundaries and the second would find breakpoints +in the broken words. This would fit nicely with \TeX's three line-breaking +passes. The first pass would be done as always before any hyphenation. The +second try would be done after the insertion of inter-word breaks. Only the +last pass would make use of intra-word breaks. + +But I haven't tried this yet. + +\section{Implementation} + +The changes to \TeX{} the program and \textsf{patgen} sketched above are +realized as a series of change files. + +\begin{itemize} +\item |classes.ch| + extends \TeX's hyphenation routines to deal with hyphens of up to~9 different + classes. It implements the new control sequences described in + section~\ref{sec:hyphenclasses}. +\item |count.ch| + extends \TeX's line breaking routines to deal with sequences of breakpoints + containing more than two consecutive hyphens. It implements the new control + sequences described in section~\ref{sec:linebreaking}. +\item |patgen-classes.ch| + extends \textsf{patgen}s pattern generation algorithm to deal with up~to~10 + different hyphen classes. The new algorithm has one new integer parameter, the + number of hyphen classes (between 2 and~10). This parameter can be specified + in columns 8 and~9 of the first line of the translate file. \textsf{patgen} + will now produce values up~to~63. +\end{itemize} + +\section{Installation} + + The change files are written for \TeX~3.1459 and \textsf{patgen}~2.3. But + it should need only cosmetic changes to make them work with other + versions of \TeX3 or \textsf{patgen}2. + + First, you have to apply the change file + |pre.ch|. The |count.ch| and |classes.ch| change files should be + independent of each other. I have only tried these files with |web2c|, and + for that you need to surround the |web2c| change file with |pre-web2c.ch| and + |post-web2c.ch|. All this boils down to the command line + +\begin{verbatim} +mv tex.ch tex.ch.orig +tie -c tex.ch tex.web pre.ch classes.ch \ + count.ch pre-web2c.ch tex.ch.orig \ + post-web2c.ch +\end{verbatim} + + \makesignature + + \bibliographystyle{plain} + \bibliography{etex} + +\end{document} + + + diff --git a/systems/tex-extensions/clasen/hyphenation/patgen-hyf.ch b/systems/tex-extensions/clasen/hyphenation/patgen-hyf.ch new file mode 100644 index 0000000000..db2fb9c460 --- /dev/null +++ b/systems/tex-extensions/clasen/hyphenation/patgen-hyf.ch @@ -0,0 +1,460 @@ +% This change extends patgens pattern generation algorithm to deal with up to +% 10 different hyphen classes. The new algorithm has one new integer +% parameter, the number of hyphen classes (between 2 and 10). This parameter +% can be specified in columns 8 and 9 of the first line of the translate +% file. patgen will now produce values up to 63. + +@x l.54 +@d banner=='This is PATGEN, Version 2.3' {printed when the program starts} +@y +@d banner=='This is PATGEN, Version 2.3 (with multiple hyphen classes)' + {printed when the program starts} +@z + +@x l.588 +@!max_val=10; {maximum number of levels$+1$, also used to denote bad patterns} +@y +@!max_val=64; {maximum number of levels$+1$, also used to denote bad patterns} +@z + +@x l.603 +if max_val>10 then bad:=5; +@y +if max_val>100 then bad:=5; +@z + +@x l.1107 +print_ln('left_hyphen_min = ',left_hyphen_min:1, + ', right_hyphen_min = ',right_hyphen_min:1, + ', ',imax-edge_of_word:1,' letters'); +@y +print_ln('left_hyphen_min = ',left_hyphen_min:1, + ', right_hyphen_min = ',right_hyphen_min:1, + ', hyphen_class_num = ',hyphen_class_num:1, + ', ',imax-edge_of_word:1,' letters'); +@z + +@x l.1113 +@ @<Globals...@>= +@!imax: internal_code; {largest |internal_code| assigned so far} +@!left_hyphen_min, @!right_hyphen_min: dot_type; + +@ @<Set up default...@>= +begin left_hyphen_min:=2; right_hyphen_min:=3; +@y + +@ @d hyphen_class(#)==(# mod hyphen_class_num) + +@<Globals...@>= +@!imax: internal_code; {largest |internal_code| assigned so far} +@!left_hyphen_min, @!right_hyphen_min: dot_type; +@!hyphen_class_num: 2..10; { allow up to 10 hyphen classes, default is 2 } + +@ @<Set up default...@>= +begin left_hyphen_min:=2; right_hyphen_min:=3; +hyphen_class_num:=2; +@z + +@x +If the values specified for \.{\\lefthyphenmin} and \.{\\righthyphenmin} +are invalid (e.g., blank) new values are read from the terminal. +@y +Another addition is, that columns 8 and~9 may optionally contain a value for +\.{\\hyphenclassnum}. + +If the values specified for \.{\\lefthyphenmin}, \.{\\righthyphenmin} and +\.{\\hyphenclassnum} are invalid (e.g., blank) new values are read from the +terminal. +@z +@x l.1148 +if (n>=1)and(n<max_dot) then right_hyphen_min:=n@+ +else bad:=true; +@y +if (n>=1)and(n<max_dot) then right_hyphen_min:=n@+ +else bad:=true; +if buf[8]=' ' then n:=0 +else if xclass[buf[8]]=digit_class then n:=xint[buf[8]]@+ +else bad:=true; +if xclass[buf[9]]=digit_class then n:=10*n+xint[buf[9]]@+ +else bad:=true; +if (n>=2)and(n<=10) then hyphen_class_num:=n@+ +else bad:=true; +@z + +@x l.1358 +@!more_to_come: boolean; +@y +@!more_to_come: boolean; +@!off_count: array [1..9] of integer; + { off by $<-3$, $-3$, $-2$, $-1$, $0$, $1$, $2$, $3$, $>3$ } +@z + +@x l.1480 +@ @<Output this pattern@>= +begin for d:=0 to pat_len do hval[d]:=0; + repeat d:=hyf_dot(h); + if hval[d]<hyf_val(h) then hval[d]:=hyf_val(h); + h:=hyf_nxt(h); + until h=0; + if hval[0]>0 then write(patout,xdig[hval[0]]); + for d:=1 to pat_len do + begin write_letter(pat[d])(patout); write(patout,xext[pat[d]]); + if hval[d]>0 then write(patout,xdig[hval[d]]); + end; + write_ln(patout); +end +@y +@ Since we have increased |max_val|, we must allow for two-digit values +in patterns. + +@<Output this pattern@>= +begin for d:=0 to pat_len do hval[d]:=0; + repeat d:=hyf_dot(h); + if hval[d]<hyf_val(h) then hval[d]:=hyf_val(h); + h:=hyf_nxt(h); + until h=0; + if hval[0]>9 then write(patout,xdig[hval[0] div 10]); + if hval[0]>0 then write(patout,xdig[hval[0] mod 10]); + for d:=1 to pat_len do + begin write_letter(pat[d])(patout); write(patout,xext[pat[d]]); + if hval[d]>9 then write(patout,xdig[hval[d] div 10]); + if hval[d]>0 then write(patout,xdig[hval[d] mod 10]); + end; + write_ln(patout); +end +@z + +@x l.1502 +@!dots: array[word_index] of hyf_type; {current hyphens} +@y +@!hclass: array[word_index] of digit; {current wanted hyphen classes} +@z + +@x +applied to all following words (until the next global word weight). A +digit at some intercharacter position indicates a weight for that position +only. + +The |read_word| procedure scans a line of input representing a word, +and places the letters into the array |word|, with |word[1]=word[wlen]= +edge_of_word|. The dot appearing between |word[dpos]| and |word[dpos+1]| +is placed in |dots[dpos]|, and the corresponding dot weight in +|dotw[dpos]|. +@y +applied to all following words (until the next global word weight). A +digit at the beginning of some intercharacter position indicates a weight for +that position only. A digit at the end of some intercharacter position +indicates the hyphen class for the hyphen wanted at this position. + +The |read_word| procedure scans a line of input representing a word, +and places the letters into the array |word|, with |word[1]=word[wlen]= +edge_of_word|. The class of the hyphen appearing between |word[dpos]| and +|word[dpos+1]| is placed in |hclass[dpos]|, and the corresponding dot weight in +|dotw[dpos]|. +@z + +@x l.1537 +@p procedure read_word; +label done, found; +var c: text_char; +@!t: trie_pointer; +begin read_buf(dictionary); +word[1]:=edge_of_word; wlen:=1; buf_ptr:=0; +repeat incr(buf_ptr); c:=buf[buf_ptr]; + case xclass[c] of + space_class: goto found; + digit_class: + if wlen=1 then {global word weight} + begin if xint[c]<>word_wt then wt_chg:=true; + word_wt:=xint[c]; + end + else dotw[wlen]:=xint[c]; {dot weight} + hyf_class: dots[wlen]:=xint[c]; {record the dot |c|} + letter_class: {record the letter |c|} + begin incr(wlen); + if wlen=max_len then + begin print_buf; overflow('word length=',max_len:1); + end; + word[wlen]:=xint[c]; dots[wlen]:=no_hyf; dotw[wlen]:=word_wt; + end; + escape_class: {record a multi-character sequence starting with |c|} + begin incr(wlen); + if wlen=max_len then + begin print_buf; overflow('word length=',max_len:1); + end; + get_letter(word[wlen]); dots[wlen]:=no_hyf; dotw[wlen]:=word_wt; + end; + invalid_class: bad_input('Bad character'); +@.Bad character@> + end; +until buf_ptr=max_buf_len; +found: incr(wlen); word[wlen]:=edge_of_word; +end; +@y +@p procedure read_word; +label done, found; +var c: text_char; +i: word_index; +@!t: trie_pointer; +begin read_buf(dictionary); +word[1]:=edge_of_word; wlen:=1; buf_ptr:=0; +for i:=0 to max_len do hclass[i]:=0; +repeat incr(buf_ptr); c:=buf[buf_ptr]; + case xclass[c] of + space_class: goto found; + digit_class: + if wlen=1 then {global word weight} + begin if xint[c]<>word_wt then wt_chg:=true; + word_wt:=xint[c]; + end + else if hclass[wlen]>0 then begin + if hyphen_class_num>xint[c] then hclass[wlen]:=xint[c] + else error('unexpected hyphen class!'); + end else dotw[wlen]:=xint[c]; {dot weight} + hyf_class: + begin + hclass[wlen]:=1; + end; + letter_class: {record the letter |c|} + begin + incr(wlen); + if wlen=max_len then + begin print_buf; overflow('word length=',max_len:1); + end; + word[wlen]:=xint[c]; hclass[wlen]:=0; dotw[wlen]:=word_wt; + end; + escape_class: {record a multi-character sequence starting with |c|} + begin incr(wlen); + if wlen=max_len then + begin print_buf; overflow('word length=',max_len:1); + end; + get_letter(word[wlen]); dotw[wlen]:=word_wt; + end; + invalid_class: bad_input('Bad character'); +@.Bad character@> + end; +until buf_ptr=max_buf_len; +found: incr(wlen); word[wlen]:=edge_of_word; +end; +@z + +@x l.1628 +@ The |change_dots| procedure updates the |dots| array representing the +printing values of the hyphens. Initially, hyphens (and correctly +found hyphens) in the word list are represented by |is_hyf| whereas +non-hyphen positions (and erroneous hyphens) are represented by |no_hyf|. A +Here these values are increased by one for each hyphen found by the +current patterns, thus changing |no_hyf| into |err_hyf| and |is_hyf| +into |found_hyf|. The routine also collects statistics about the number +of good, bad, and missed hyphens. + +@d incr_wt(#)==Incr(#)(dotw[dpos]) + +@p procedure change_dots; +var dpos: word_index; +begin for dpos:=wlen-hyf_max downto hyf_min do + begin if odd(hval[dpos]) then incr(dots[dpos]); + if dots[dpos]=found_hyf then incr_wt(good_count) + else if dots[dpos]=err_hyf then incr_wt(bad_count) + else if dots[dpos]=is_hyf then incr_wt(miss_count); + end; +end; +@y +@ The |change_dots| procedure owes its name to the fact that its job was +once to update an array called |dots| representing the printing values of the +hyphens. This is no longer the case, as |dots| is gone. + +The routine collects statistics about the number of good, bad, and missed hyphens. + +@d incr_wt(#)==Incr(#)(dotw[dpos]) + +@p procedure change_dots; +var dpos: word_index; + have: integer; +begin for dpos:=wlen-hyf_max downto hyf_min do + begin + have:=hyphen_class(hval[dpos]); + {good/bad/miss statistics} + if have>0 then + if have=hclass[dpos] then incr_wt(good_count) + else incr_wt(bad_count) + else if hclass[dpos]>0 then incr_wt(miss_count); + {off statistics} + if have+hclass[dpos]>0 then + if abs(have-hclass[dpos])<=3 then + incr_wt(off_count[have-hclass[dpos]+5]) + else if have<hclass[dpos] then + incr_wt(off_count[1]) + else incr_wt(off_count[9]) + end; +end; +@z + +@x l.1653 +@ The following procedure outputs the word as hyphenated by the current +patterns, including any word weights. Hyphens inhibited by the values of +\.{\\lefthyphenmin} and \.{\\righthyphenmin} are output as well. + +@p procedure output_hyphenated_word; +var dpos: word_index;@/ +@!l: triec_pointer; {for |write_letter|} +begin if wt_chg then {output global word weight} + begin write(pattmp,xdig[word_wt]); wt_chg:=false + end; +for dpos:=2 to wlen-2 do + begin write_letter(word[dpos])(pattmp); write(pattmp,xext[word[dpos]]); + if dots[dpos]<>no_hyf then write(pattmp,xhyf[dots[dpos]]); + if dotw[dpos]<>word_wt then write(pattmp,xdig[dotw[dpos]]); + end; +write_letter(word[wlen-1])(pattmp); write_ln(pattmp,xext[word[wlen-1]]); +end; +@y +@ The following procedure outputs the word as hyphenated by the current +patterns, including the found hyphen classes. A correct hyphen is shown with +|found_hyf|, an incorrect one with |err_hyf|. Hyphens inhibited by the values of +\.{\\lefthyphenmin} and \.{\\righthyphenmin} are {\it not} shown. + +@p procedure output_hyphenated_word; +var dpos: word_index;@/ +@!l: triec_pointer; {for |write_letter|} +begin + for dpos:=2 to hyf_min-1 do begin + write_letter(word[dpos])(pattmp); + write(pattmp,xext[word[dpos]]); + end; + for dpos:=hyf_min to wlen-hyf_max do begin + write_letter(word[dpos])(pattmp); + write(pattmp,xext[word[dpos]]); + if hyphen_class(hval[dpos])>0 then begin + if hyphen_class(hval[dpos])=hclass[dpos] then write(pattmp,xhyf[found_hyf]) + else write(pattmp,xhyf[err_hyf]); + if hyphen_class(hval[dpos])>1 then + write(pattmp,xdig[hyphen_class(hval[dpos])]); + end; + end; + for dpos:=wlen-hyf_max+1 to wlen-1 do begin + write_letter(word[dpos])(pattmp); + write(pattmp,xext[word[dpos]]); + end; + write_ln(pattmp,''); +end; +@z + +@x l.1702 +@ The globals |good_dot| and |bad_dot| will be set to |is_hyf| and +|no_hyf|, or |err_hyf| and |found_hyf|, depending on whether the current +level is odd or even, respectively. The globals |dot_min|, |dot_max|, +and |dot_len| are analogous to |hyf_min|, |hyf_max|, and |hyf_len| +defined earlier. + +@<Globals...@>= +@!good_dot, @!bad_dot: hyf_type; {good and bad hyphens at current level} +@!dot_min, @!dot_max, @!dot_len: word_index; {limits for legal dots} + +@ @<Prepare to read dictionary@>= +if procesp then + begin dot_min:=pat_dot; dot_max:=pat_len-pat_dot; + if dot_min<hyf_min then dot_min:=hyf_min; + if dot_max<hyf_max then dot_max:=hyf_max; + dot_len:=dot_min+dot_max; + if odd(hyph_level) then + begin good_dot:=is_hyf; bad_dot:=no_hyf; + end + else begin good_dot:=err_hyf; bad_dot:=found_hyf; + end; + end; +@y +@ The globals |dot_min|, |dot_max|, and |dot_len| are analogous to |hyf_min|, +|hyf_max|, and |hyf_len| defined earlier. + +@<Globals...@>= +@!dot_min, @!dot_max, @!dot_len: word_index; {limits for legal dots} + +@ @<Prepare to read dictionary@>= +if procesp then + begin dot_min:=pat_dot; dot_max:=pat_len-pat_dot; + if dot_min<hyf_min then dot_min:=hyf_min; + if dot_max<hyf_max then dot_max:=hyf_max; + dot_len:=dot_min+dot_max; + end; +@z + +@x + @!goodp: boolean; +@y + @!goodp: boolean; + have,get: integer; +@z + +@x l.1729 +@<Check this dot position...@>= +if no_more[dpos] then goto continue; +if dots[dpos]=good_dot then goodp:=true else +if dots[dpos]=bad_dot then goodp:=false else goto continue; +@y +@<Check this dot position...@>= +if no_more[dpos] then goto continue; +have:=hyphen_class(hval[dpos]); +get:=hyphen_class(hyph_level); +if abs(get-hclass[dpos])<abs(have-hclass[dpos]) then goodp:=true +else if abs(get-hclass[dpos])>abs(have-hclass[dpos]) then goodp:=false +else goto continue; +@z + +@x l.1750 +@p procedure do_dictionary; +begin good_count:=0; bad_count:=0; miss_count:=0; +@y +@p procedure do_dictionary; +var i: integer; +begin + for i:=1 to 9 do begin + off_count[i]:=0; + end; + good_count:=0; bad_count:=0; miss_count:=0; +@z + +@x l.1771 + if (good_count+miss_count)>0 then + print_ln((100*good_count/(good_count+miss_count)):1:2,' %, ', + (100*bad_count/(good_count+miss_count)):1:2,' %, ', + (100*miss_count/(good_count+miss_count)):1:2,' %'); +@y + if (good_count+miss_count)>0 then + print_ln((100*good_count/(good_count+miss_count)):1:2,' %, ', + (100*bad_count/(good_count+miss_count)):1:2,' %, ', + (100*miss_count/(good_count+miss_count)):1:2,' %'); + print_ln('off by <-3, -3,...: ', + off_count[1]:1, ', ', + off_count[2]:1, ', ', + off_count[3]:1, ', ', + off_count[4]:1, ', ', + off_count[5]:1, ', ', + off_count[6]:1, ', ', + off_count[7]:1, ', ', + off_count[8]:1, ', ', + off_count[9]:1); +@z + +@x l.1835 + digit_class: + begin d:=xint[c]; + if d>=max_val then bad_input('Bad hyphenation value'); +@.Bad hyphenation value@> + if d>max_pat then max_pat:=d; + hval[pat_len]:=d; + end; +@y + digit_class: + begin d:=xint[c]; + if xclass[buf[buf_ptr+1]]=digit_class then begin + incr(buf_ptr); + c:=buf[buf_ptr]; + d:=10*d+xint[c]; + end; + if d>=max_val then bad_input('Bad hyphenation value'); +@.Bad hyphenation value@> + if d>max_pat then max_pat:=d; + hval[pat_len]:=d; + end; +@z |