diff options
author | Luigi Scarso <luigi.scarso@gmail.com> | 2018-09-05 21:57:30 +0000 |
---|---|---|
committer | Luigi Scarso <luigi.scarso@gmail.com> | 2018-09-05 21:57:30 +0000 |
commit | ecd1605a23ef54d2a4a69e1a361172d350e03b3b (patch) | |
tree | 8ea1d9745c24dbc5e755f13d801fa01e88944650 /Build/source/texk/web2c/luatexdir/lang | |
parent | 04563248956e7059aad4fc4f6bf7da18562f7b8c (diff) |
luatex: correct automake file for pplib; replaced cweb files with c files.
git-svn-id: svn://tug.org/texlive/trunk@48594 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/lang')
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hnjalloc.c | 65 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hnjalloc.w | 69 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hyphen.c (renamed from Build/source/texk/web2c/luatexdir/lang/hyphen.w) | 471 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/texlang.c (renamed from Build/source/texk/web2c/luatexdir/lang/texlang.w) | 686 |
4 files changed, 673 insertions, 618 deletions
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c new file mode 100644 index 00000000000..3ebeee9e4d5 --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c @@ -0,0 +1,65 @@ +/* + +hnjalloc.w + +LibHnj is dual licensed under LGPL and MPL. Boilerplate for both licenses +follows. + +LibHnj - a library for high quality hyphenation and justification Copyright (C) +1998 Raph Levien, (C) 2001 ALTLinux, Moscow + +This library is free software; you can redistribute it and/or modify it under the +terms of the GNU Library General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This library is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Library General Public License for more details. + +You should have received a copy of the GNU Library General Public License along +with this library; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307 USA. + +The contents of this file are subject to the Mozilla Public License Version 1.0 +(the "MPL"); you may not use this file except in compliance with the MPL. You may +obtain a copy of the MPL at http://www.mozilla.org/MPL/ + +Software distributed under the MPL is distributed on an "AS IS" basis, WITHOUT +WARRANTY OF ANY KIND, either express or implied. See the MPL for the specific +language governing rights and limitations under the MPL. + +*/ + +/*tex The wrappers for malloc */ + +#include <stdlib.h> +#include <stdio.h> +#include "lang/hnjalloc.h" + +void *hnj_malloc(int size) +{ + void *p; + + p = malloc((size_t) size); + if (p == NULL) { + fprintf(stderr, "can't allocate %d bytes\n", size); + exit(1); + } + return p; +} + +void *hnj_realloc(void *p, int size) +{ + p = realloc(p, (size_t) size); + if (p == NULL) { + fprintf(stderr, "can't allocate %d bytes\n", size); + exit(1); + } + return p; +} + +void hnj_free(void *p) +{ + free(p); +} diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w deleted file mode 100644 index 91fefec9e5d..00000000000 --- a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w +++ /dev/null @@ -1,69 +0,0 @@ -% hnjalloc.w -% -% LibHnj is dual licensed under LGPL and MPL. Boilerplate for both -% licenses follows. -% -% -% LibHnj - a library for high quality hyphenation and justification -% Copyright (C) 1998 Raph Levien, (C) 2001 ALTLinux, Moscow -% -% This library is free software; you can redistribute it and/or -% modify it under the terms of the GNU Library General Public -% License as published by the Free Software Foundation; either -% version 2 of the License, or (at your option) any later version. -% -% This library is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -% Library General Public License for more details. -% -% You should have received a copy of the GNU Library General Public -% License along with this library; if not, write to the -% Free Software Foundation, Inc., 59 Temple Place - Suite 330, -% Boston, MA 02111-1307 USA. -% -% -% -% The contents of this file are subject to the Mozilla Public License -% Version 1.0 (the "MPL"); you may not use this file except in -% compliance with the MPL. You may obtain a copy of the MPL at -% http://www.mozilla.org/MPL/ -% -% Software distributed under the MPL is distributed on an "AS IS" basis, -% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL -% for the specific language governing rights and limitations under the -% MPL. - -@ wrappers for malloc -@c - -#include <stdlib.h> -#include <stdio.h> -#include "lang/hnjalloc.h" - -void *hnj_malloc(int size) -{ - void *p; - - p = malloc((size_t) size); - if (p == NULL) { - fprintf(stderr, "can't allocate %d bytes\n", size); - exit(1); - } - return p; -} - -void *hnj_realloc(void *p, int size) -{ - p = realloc(p, (size_t) size); - if (p == NULL) { - fprintf(stderr, "can't allocate %d bytes\n", size); - exit(1); - } - return p; -} - -void hnj_free(void *p) -{ - free(p); -} diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.w b/Build/source/texk/web2c/luatexdir/lang/hyphen.c index 35befb743de..3a96f195b2f 100644 --- a/Build/source/texk/web2c/luatexdir/lang/hyphen.w +++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.c @@ -1,57 +1,51 @@ -% hyphen.w -% -% Libhnj is dual licensed under LGPL and MPL. Boilerplate for both -% licenses follows. -% -% -% LibHnj - a library for high quality hyphenation and justification -% Copyright (C) 1998 Raph Levien, -% (C) 2001 ALTLinux, Moscow (http://www.alt-linux.org), -% (C) 2001 Peter Novodvorsky (nidd@@cs.msu.su) -% -% This library is free software; you can redistribute it and/or -% modify it under the terms of the GNU Library General Public -% License as published by the Free Software Foundation; either -% version 2 of the License, or (at your option) any later version. -% -% This library is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -% Library General Public License for more details. -% -% You should have received a copy of the GNU Library General Public -% License along with this library; if not, write to the -% Free Software Foundation, Inc., 59 Temple Place - Suite 330, -% Boston, MA 02111-1307 USA. -% -% -% -% The contents of this file are subject to the Mozilla Public License -% Version 1.0 (the "MPL"); you may not use this file except in -% compliance with the MPL. You may obtain a copy of the MPL at -% http://www.mozilla.org/MPL/ -% -% Software distributed under the MPL is distributed on an "AS IS" basis, -% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL -% for the specific language governing rights and limitations under the -% MPL. - - -@ @c +/* +hyphen.c + +This file is derived from libhnj which is is dual licensed under LGPL and MPL. +Boilerplate for both licenses follows. + +LibHnj - a library for high quality hyphenation and justification + +(C) 1998 Raph Levien, +(C) 2001 ALTLinux, Moscow (http://www.alt-linux.org), +(C) 2001 Peter Novodvorsky (nidd@cs.msu.su) + +This library is free software; you can redistribute it and/or modify it under the +terms of the GNU Library General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This library is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Library General Public License for more details. + +You should have received a copy of the GNU Library General Public License along +with this library; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307 USA. + +The contents of this file are subject to the Mozilla Public License Version 1.0 +(the "MPL"); you may not use this file except in compliance with the MPL. You may +obtain a copy of the MPL at http://www.mozilla.org/MPL/ + +Software distributed under the MPL is distributed on an "AS IS" basis, WITHOUT +WARRANTY OF ANY KIND, either express or implied. See the MPL for the specific +language governing rights and limitations under the MPL. + +*/ #include "ptexlib.h" #include "lua/luatex-api.h" -#include <stdlib.h> /* for NULL, malloc */ -#include <stdio.h> /* for fprintf */ -#include <string.h> /* for strdup */ -#include <stdlib.h> /* for malloc used by substring inclusion */ +#include <stdlib.h> /* for NULL, malloc */ +#include <stdio.h> /* for fprintf */ +#include <string.h> /* for strdup */ +#include <stdlib.h> /* for malloc used by substring inclusion */ #define MAXPATHS 40960 #ifdef UNX -# include <unistd.h> /* for exit */ +# include <unistd.h> /* for exit */ #endif #include <kpathsea/c-ctype.h> @@ -60,9 +54,8 @@ #include "lang/hnjalloc.h" -@ TODO: should be moved to separate library +/*tex This could be moved to separate library. */ -@c static unsigned char *hnj_strdup(const unsigned char *s) { unsigned char *new; @@ -75,18 +68,20 @@ static unsigned char *hnj_strdup(const unsigned char *s) return new; } -@* Type definitions. +/*tex -@ a little bit of a hash table implementation. This simply maps strings - to state numbers + First some type definitions and a little bit of a hash table implementation. + This simply maps strings to state numbers -@c -typedef struct _HashTab HashTab; +*/ + +typedef struct _HashTab HashTab; typedef struct _HashEntry HashEntry; -typedef struct _HashIter HashIter; -typedef union _HashVal HashVal; +typedef struct _HashIter HashIter; +typedef union _HashVal HashVal; + +/*tex A cheap, but effective, hack. */ -/* A cheap, but effective, hack. */ #define HASH_SIZE 31627 struct _HashTab { @@ -110,13 +105,13 @@ struct _HashIter { int ndx; }; -@ State machine +/* The state state machine. */ -@c typedef struct _HyphenState HyphenState; typedef struct _HyphenTrans HyphenTrans; + #define MAX_CHARS 256 -#define MAX_NAME 20 +#define MAX_NAME 20 struct _HyphenDict { int num_states; @@ -130,9 +125,14 @@ struct _HyphenDict { struct _HyphenState { char *match; - /*char *repl; */ - /*signed char replindex; */ - /*signed char replcut; */ + /*tex Removed: + + \starttyping + char *repl; + signed char replindex; + signed char replcut; + \stoptyping + */ int fallback_state; int num_trans; HyphenTrans *trans; @@ -143,20 +143,21 @@ struct _HyphenTrans { int new_state; }; +/*tex + + Combine two right-aligned number patterns, 04000 + 020 becomes 04020. This + works also for utf8 sequences because the substring is identical to the last + substring-length bytes of expr except for the (single byte) hyphenation + encoders -@ Combine two right-aligned number patterns, 04000 + 020 becomes 04020 +*/ -@c static char *combine(char *expr, const char *subexpr) { size_t l1 = strlen(expr); size_t l2 = strlen(subexpr); size_t off = l1 - l2; unsigned j; - /* this works also for utf8 sequences because the substring is identical - to the last substring-length bytes of expr except for the (single byte) - hyphenation encoders - */ for (j = 0; j < l2; j++) { if (expr[off + j] < subexpr[j]) expr[off + j] = subexpr[j]; @@ -164,9 +165,8 @@ static char *combine(char *expr, const char *subexpr) return expr; } +/*tex Some original code: */ -@ ORIGINAL CODE -@c static HashIter *new_HashIter(HashTab * h) { HashIter *i = hnj_malloc(sizeof(HashIter)); @@ -176,7 +176,6 @@ static HashIter *new_HashIter(HashTab * h) return i; } - static int nextHashStealPattern(HashIter * i, unsigned char **word, char **pattern) { while (i->cur == NULL) { @@ -204,7 +203,6 @@ static int nextHash(HashIter * i, unsigned char **word) return 1; } - static int eachHash(HashIter * i, unsigned char **word, char **pattern) { while (i->cur == NULL) { @@ -218,16 +216,17 @@ static int eachHash(HashIter * i, unsigned char **word, char **pattern) return 1; } - static void delete_HashIter(HashIter * i) { hnj_free(i); } +/*tex -@ a |char*| hash function from ASU - adapted from Gtk+ +A |char*| hash function from ASU, adapted from |Gtk+|: + +*/ -@c static unsigned int hnj_string_hash(const unsigned char *s) { const unsigned char *p; @@ -240,18 +239,15 @@ static unsigned int hnj_string_hash(const unsigned char *s) h = h ^ g; } } - return h /* \% M */ ; + return h; } +/*tex This assumes that key is not already present! */ -@ assumes that key is not already present! - -@c static void state_insert(HashTab * hashtab, unsigned char *key, int state) { int i; HashEntry *e; - i = (int) (hnj_string_hash(key) % HASH_SIZE); e = hnj_malloc(sizeof(HashEntry)); e->next = hashtab->entries[i]; @@ -260,23 +256,18 @@ static void state_insert(HashTab * hashtab, unsigned char *key, int state) hashtab->entries[i] = e; } +/*tex This also assumes that key is not already present! */ -@ assumes that key is not already present! - -@c static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat) { int i; HashEntry *e; - i = (int) (hnj_string_hash(key) % HASH_SIZE); for (e = hashtab->entries[i]; e; e = e->next) { if (strcmp((char *) e->key, (char *) key) == 0) { if (e->u.hyppat) { - if (hyppat - && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) { - print_err("Conflicting pattern ignored"); - error(); + if (hyppat && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) { + normal_warning("hyphenation","a conflicting pattern has been ignored"); } hnj_free(e->u.hyppat); } @@ -292,15 +283,12 @@ static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat) hashtab->entries[i] = e; } +/*tex We return |state| if found, otherwise |-1|. */ -@ return state if found, otherwise $-1$ - -@c static int state_lookup(HashTab * hashtab, const unsigned char *key) { int i; HashEntry *e; - i = (int) (hnj_string_hash(key) % HASH_SIZE); for (e = hashtab->entries[i]; e; e = e->next) { if (!strcmp((const char *) key, (const char *) e->key)) { @@ -310,15 +298,14 @@ static int state_lookup(HashTab * hashtab, const unsigned char *key) return -1; } +/*tex We return |state| if found, otherwise |-1|. */ -@ return state if found, otherwise $-1$ - -@c static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l) { int i; HashEntry *e; - unsigned char key[256]; /* should be ample */ + /*tex The 256 should be enough. */ + unsigned char key[256]; strncpy((char *) key, (const char *) chars, (size_t) l); key[l] = 0; i = (int) (hnj_string_hash(key) % HASH_SIZE); @@ -330,24 +317,18 @@ static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l) return NULL; } +/*tex Get the state number, allocating a new state if necessary. */ -@ Get the state number, allocating a new state if necessary. - -@c -static int hnj_get_state(HyphenDict * dict, - const unsigned char *str, int *state_num) +static int hnj_get_state(HyphenDict * dict, const unsigned char *str, int *state_num) { *state_num = state_lookup(dict->state_num, str); - if (*state_num >= 0) return *state_num; - state_insert(dict->state_num, hnj_strdup(str), dict->num_states); - /* predicate is true if |dict->num_states| is a power of two */ + /*tex The predicate is true if |dict->num_states| is a power of two: */ if (!(dict->num_states & (dict->num_states - 1))) { dict->states = hnj_realloc(dict->states, - (int) ((dict->num_states << 1) * - (int) sizeof(HyphenState))); + (int) ((dict->num_states << 1) * (int) sizeof(HyphenState))); } dict->states[dict->num_states].match = NULL; dict->states[dict->num_states].fallback_state = -1; @@ -356,70 +337,59 @@ static int hnj_get_state(HyphenDict * dict, return dict->num_states++; } +/*tex -@ Add a transition from state1 to state2 through ch - assumes that the - transition does not already exist + Add a transition from state1 to state2 through ch - assumes that the + transition does not already exist. + +*/ -@c static void hnj_add_trans(HyphenDict * dict, int state1, int state2, int uni_ch) { int num_trans; - /* TH: this test was a bit too strict, it is quite normal for old - patterns to have chars in the range 0-31 or 127-159 (inclusive). - To ease the transition, let's only disallow NUL for now - (this is probably a requirement of the code anyway). - */ + /* + TH: this test was a bit too strict, it is quite normal for old patterns + to have chars in the range 0-31 or 127-159 (inclusive). To ease the + transition, let's only disallow |NUL| for now, which probably is a + requirement of the code anyway. + */ if (uni_ch == 0) { - char errmsg[256]; /* temp hack ... we will have a formatted error */ - snprintf(errmsg, 255, "character out of bounds: u%04x", uni_ch); - errmsg[255] = '\0'; - normal_error("hyphenation",errmsg); /* todo */ + formatted_error("hyphenation","a character is out of bounds: u%04x", uni_ch); } num_trans = dict->states[state1].num_trans; if (num_trans == 0) { dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans)); } else { - /* TH: The old version did - } else if (!(num_trans & (num_trans - 1))) { - ... hnj_realloc(dict->states[state1].trans, - (int) ((num_trans << 1) * - sizeof(HyphenTrans))); - but that is incredibly nasty when adding patters one-at-a-time. - Controlled growth would be nicer than the current +1, but if - noone complains, this is good enough ;) - */ + /* + TH: The old version did: + + \starttyping + } else if (!(num_trans & (num_trans - 1))) { + ... =hnj_realloc(dict->states[state1].trans, + (int) ((num_trans << 1) * sizeof(HyphenTrans))); + \stoptyping + + but that is incredibly nasty when adding patters one-at-a-time. + Controlled growth would be nicer than the current +1, but if no one + complains, and no one did in a decade, this is good enough. + + */ dict->states[state1].trans = hnj_realloc(dict->states[state1].trans, - (int) ((num_trans + 1) * - sizeof(HyphenTrans))); + (int) ((num_trans + 1) * sizeof(HyphenTrans))); } dict->states[state1].trans[num_trans].uni_ch = uni_ch; dict->states[state1].trans[num_trans].new_state = state2; dict->states[state1].num_trans++; } +/*tex -#ifdef VERBOSE - -static unsigned char *get_state_str(int state) -{ - int i; - HashEntry *e; - - for (i = 0; i < HASH_SIZE; i++) - for (e = global->entries[i]; e; e = e->next) - if (e->u.state == state) - return e->key; - return NULL; -} -#endif + We did change the semantics a bit here: |hnj_hyphen_load| used to operate on + a file, but now the argument is a string buffer. +*/ -@ I've changed the semantics a bit here: |hnj_hyphen_load| used to - operate on a file, but now the argument is a string buffer. - -@c -static const unsigned char *next_pattern(size_t * length, - const unsigned char **buf) +static const unsigned char *next_pattern(size_t * length, const unsigned char **buf) { const unsigned char *here, *rover = *buf; while (*rover && isspace(*rover)) @@ -435,25 +405,28 @@ static const unsigned char *next_pattern(size_t * length, } *length = (size_t) (rover - here); *buf = rover; - return *length ? here : NULL; /* zero sensed */ + /*tex Zero sensed: */ + return *length ? here : NULL; } static void init_hash(HashTab ** h) { int i; - if (*h) + if (*h) { return; + } *h = hnj_malloc(sizeof(HashTab)); - for (i = 0; i < HASH_SIZE; i++) + for (i = 0; i < HASH_SIZE; i++) { (*h)->entries[i] = NULL; + } } - static void clear_state_hash(HashTab ** h) { int i; - if (*h == NULL) + if (*h == NULL) { return; + } for (i = 0; i < HASH_SIZE; i++) { HashEntry *e, *next; for (e = (*h)->entries[i]; e; e = next) { @@ -466,19 +439,20 @@ static void clear_state_hash(HashTab ** h) *h = NULL; } - static void clear_hyppat_hash(HashTab ** h) { int i; - if (*h == NULL) + if (*h == NULL) { return; + } for (i = 0; i < HASH_SIZE; i++) { HashEntry *e, *next; for (e = (*h)->entries[i]; e; e = next) { next = e->next; hnj_free(e->key); - if (e->u.hyppat) + if (e->u.hyppat) { hnj_free(e->u.hyppat); + } hnj_free(e); } } @@ -486,7 +460,6 @@ static void clear_hyppat_hash(HashTab ** h) *h = NULL; } - static void init_dict(HyphenDict * dict) { dict->num_states = 1; @@ -502,7 +475,6 @@ static void init_dict(HyphenDict * dict) init_hash(&dict->patterns); } - static void clear_dict(HyphenDict * dict) { int state_num; @@ -519,8 +491,6 @@ static void clear_dict(HyphenDict * dict) clear_state_hash(&dict->state_num); } - - HyphenDict *hnj_hyphen_new(void) { HyphenDict *dict = hnj_malloc(sizeof(HyphenDict)); @@ -528,14 +498,12 @@ HyphenDict *hnj_hyphen_new(void) return dict; } - void hnj_hyphen_clear(HyphenDict * dict) { clear_dict(dict); init_dict(dict); } - void hnj_hyphen_free(HyphenDict * dict) { clear_dict(dict); @@ -568,29 +536,30 @@ unsigned char *hnj_serialize(HyphenDict * dict) return buf; } - void hnj_free_serialize(unsigned char *c) { hnj_free(c); } +/*tex -@ hyphenation pattern: - -signed bytes - -0 indicates end (actually any negative number) + In hyphenation patterns we use signed bytes where |0|, or actually any + negative number, indicates end: -: prio(1+),startpos,length,len1,[replace],len2,[replace] + \starttyping + prio(1+),startpos,length,len1,[replace],len2,[replace] + \starttyping -most basic example is: + A basic example is: -p n 0 0 0 + \starttyping + p n 0 0 0 + \starttyping -for a hyphenation point between characters + for a hyphenation point between characters. +*/ -@c void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) { int state_num, last_state; @@ -601,7 +570,6 @@ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) unsigned char *word; char *pattern; size_t l = 0; - const unsigned char *format; const unsigned char *begin = f; unsigned char *pat; @@ -609,46 +577,23 @@ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) while ((format = next_pattern(&l, &f)) != NULL) { int i, j, e1; if (l>=255) { - help1("Individual patterns should not be longer than 254 bytes total."); - print_err("Pattern of enormous length ignored"); - error(); + normal_warning("hyphenation","a pattern of more than 254 bytes ignored"); continue; } -#if 0 - printf("%s\n",format); - char* repl = strnchr(format, '/',l); - int replindex = 0; - int replcut = 0; - if (repl) { - int clen = l-(repl-format); - l = repl-format; - char * index = strnchr(repl + 1, ',',clen); - if (index) { - char * index2 = strnchr(index + 1, ',',clen-(index-repl)); - if (index2) { - replindex = (signed char) atoi(index + 1) - 1; - replcut = (signed char) atoi(index2 + 1); - } - } else { - hnj_strchomp(repl + 1); - replindex = 0; - replcut = strlen(buf); - } - repl = hnj_strdup(repl + 1); - } -#endif for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) { if (format[i] >= '0' && format[i] <= '9') j++; if (is_utf8_follow(format[i])) e1++; } - /* |l-e1| => number of {\it characters} not {\it bytes} */ - /* |l-j| => number of pattern bytes */ - /* |l-e1-j| => number of pattern characters */ + /*tex + Here |l-e1| is the number of {\em characters} not {\em bytes}, |l-j| + the number of pattern bytes and |l-e1-j| the number of pattern + characters. + */ pat = (unsigned char *) malloc((1 + l - (size_t) j)); org = (char *) malloc((size_t) (2 + l - (size_t) e1 - (size_t) j)); - /* remove hyphenation encoders (digits) from pat */ + /*tex Remove hyphenation encoders (digits) from pat. */ org[0] = '0'; for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) { unsigned char c = format[i]; @@ -665,20 +610,25 @@ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) org[j + 1] = 0; hyppat_insert(dict->patterns, pat, org); } - dict->pat_length += (int) ((f - begin) + 2); /* 2 for spurious spaces */ + /*tex We add 2 bytes for spurious spaces. */ + dict->pat_length += (int) ((f - begin) + 2); init_hash(&dict->merged); v = new_HashIter(dict->patterns); while (nextHash(v, &word)) { int wordsize = (int) strlen((char *) word); int j1, l1; for (l1 = 1; l1 <= wordsize; l1++) { - if (is_utf8_follow(word[l1])) - continue; /* Do not clip an utf8 sequence */ + if (is_utf8_follow(word[l1])) { + /*tex Do not clip an utf8 sequence. */ + continue; + } for (j1 = 1; j1 <= l1; j1++) { char *subpat_pat; int i1 = l1 - j1; - if (is_utf8_follow(word[i1])) - continue; /* Do not start halfway an utf8 sequence */ + if (is_utf8_follow(word[i1])) { + /*tex Do not start halfway an utf8 sequence. */ + continue; + } if ((subpat_pat = hyppat_lookup(dict->patterns, word + i1, j1)) != NULL) { char *newpat_pat; @@ -694,7 +644,8 @@ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) if (is_utf8_follow(newword[i1])) e1++; neworg = malloc((size_t) (l1 + 2 - e1)); - sprintf(neworg, "%0*d", l1 + 1 - e1, 0); /* fill with right amount of '0' */ + /*tex Fill with right amount of zeros: */ + sprintf(neworg, "%0*d", l1 + 1 - e1, 0); hyppat_insert(dict->merged, newword, combine(neworg, subpat_pat)); } else { combine(newpat_pat, subpat_pat); @@ -704,20 +655,15 @@ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) } } delete_HashIter(v); - init_hash(&dict->state_num); state_insert(dict->state_num, hnj_strdup((const unsigned char *) ""), 0); v = new_HashIter(dict->merged); while (nextHashStealPattern(v, &word, &pattern)) { static unsigned char mask[] = { 0x3F, 0x1F, 0xF, 0x7 }; int j1 = (int) strlen((char *) word); -#ifdef VERBOSE - printf("word %s pattern %s, j = %d\n", word, pattern, j1); -#endif state_num = hnj_get_state(dict, word, &found); dict->states[state_num].match = pattern; - - /* now, put in the prefix transitions */ + /*tex Now, put in the prefix transitions. */ while (found < 0) { j1--; last_state = state_num; @@ -741,71 +687,48 @@ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) } delete_HashIter(v); clear_hyppat_hash(&dict->merged); - - /* put in the fallback states */ + /*tex Put in the fallback states. */ { - int i, j = 0; - for (i = 0; i < HASH_SIZE; i++) { - for (e = dict->state_num->entries[i]; e; e = e->next) { - /* do not do state==0 otherwise things get confused */ - if (e->u.state) { - for (j = 1; 1; j++) { - state_num = state_lookup(dict->state_num, e->key + j); - if (state_num >= 0) - break; + int i, j = 0; + for (i = 0; i < HASH_SIZE; i++) { + for (e = dict->state_num->entries[i]; e; e = e->next) { + /*tex Do not do |state==0| otherwise things get confused. */ + if (e->u.state) { + for (j = 1; 1; j++) { + state_num = state_lookup(dict->state_num, e->key + j); + if (state_num >= 0) + break; + } + dict->states[e->u.state].fallback_state = state_num; } - dict->states[e->u.state].fallback_state = state_num; } } } -#ifdef VERBOSE - for (i = 0; i < HASH_SIZE; i++) { - for (e = dict->state_num->entries[i]; e; e = e->next) { - printf("%d string %s state %d, fallback=%d\n", - i, e->key, e->u.state, dict->states[e->u.state].fallback_state); - for (j = 0; j < dict->states[e->u.state].num_trans; j++) { - printf(" u%4x->%d\n", - (int) dict->states[e->u.state].trans[j].uni_ch, - dict->states[e->u.state].trans[j].new_state); - } - } - } -#endif - } clear_state_hash(&dict->state_num); } -@ @c extern halfword insert_syllable_discretionary(halfword t, lang_variables * lan); -void hnj_hyphen_hyphenate(HyphenDict * dict, - halfword first1, - halfword last1, - int length, - halfword left, halfword right, lang_variables * lan) +void hnj_hyphen_hyphenate(HyphenDict * dict, halfword first1, halfword last1, + int length, halfword left, halfword right, lang_variables * lan) { int char_num; halfword here; int state = 0; - /* +2 for dots at each end, +1 for points /outside/ characters */ + /*tex +2 for dots at each end, +1 for points outside characters. */ int ext_word_len = length + 2; int hyphen_len = ext_word_len + 1; char *hyphens = hnj_malloc(hyphen_len + 1); - - /* Add a '.' to beginning and end to facilitate matching */ + /*tex Add a '.' to beginning and end to facilitate matching. */ vlink(begin_point) = first1; vlink(end_point) = vlink(last1); vlink(last1) = end_point; - for (char_num = 0; char_num < hyphen_len; char_num++) { hyphens[char_num] = '0'; } hyphens[hyphen_len] = 0; - - /* now, run the finite state machine */ - for (char_num = 0, here = begin_point; here != vlink(end_point); - here = vlink(here)) { - + /*tex Now, run the finite state machine. */ + for (char_num = 0, here = begin_point; here != vlink(end_point); here = vlink(here)) { int ch; if (here == begin_point || here == end_point) { ch = '.'; @@ -816,28 +739,20 @@ void hnj_hyphen_hyphenate(HyphenDict * dict, } } while (state != -1) { -#if 0 - printf("%*s%s%c",char_num-strlen(get_state_str(state)),"",get_state_str(state),(char)ch); -#endif HyphenState *hstate = &dict->states[state]; int k; for (k = 0; k < hstate->num_trans; k++) { if (hstate->trans[k].uni_ch == ch) { char *match; state = hstate->trans[k].new_state; -#if 0 - printf(" state %d\n",state); -#endif match = dict->states[state].match; if (match) { - /* +2 because: - 1 string length is one bigger than offset - 1 hyphenation starts before first character - */ + /*tex + + We add +2 because 1 string length is one bigger than offset + and 1 hyphenation starts before first character. + */ int offset = (int) (char_num + 2 - (int) strlen(match)); -#if 0 - printf("%*s%s\n", offset,"", match); -#endif int m; for (m = 0; match[m]; m++) { if (hyphens[offset + m] < match[m]) @@ -848,21 +763,19 @@ void hnj_hyphen_hyphenate(HyphenDict * dict, } } state = hstate->fallback_state; -#if 0 - printf(" back to %d\n", state); -#endif } - /* nothing worked, let's go to the next character */ + /*tex Nothing worked, let's go to the next character. */ state = 0; - try_next_letter:; + try_next_letter:; char_num++; } - - /* restore the correct pointers */ + /*tex Restore the correct pointers. */ vlink(last1) = vlink(end_point); + /*tex - /* pattern is \.{\^.\^w\^o\^r\^d\^.\^} |word_len|=4, |ext_word_len|=6, |hyphens|=7 - * check \.{ \^ \^ \^ } so drop first two and stop after |word_len-1| + Pattern is \.{\^.\^w\^o\^r\^d\^.\^} and |word_len|=4, |ext_word_len|=6, + |hyphens|=7; check \.{ \^ \^ \^ } and therefore drop first two and stop + after |word_len-1| */ for (here = first1, char_num = 2; here != left; here = vlink(here)) char_num++; diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.w b/Build/source/texk/web2c/luatexdir/lang/texlang.c index 337d2eca51d..04f96660f5e 100644 --- a/Build/source/texk/web2c/luatexdir/lang/texlang.w +++ b/Build/source/texk/web2c/luatexdir/lang/texlang.c @@ -1,31 +1,32 @@ -% texlang.w -% -% Copyright 2006-2012 Taco Hoekwater <taco@@luatex.org> -% -% This file is part of LuaTeX. -% -% LuaTeX is free software; you can redistribute it and/or modify it under -% the terms of the GNU General Public License as published by the Free -% Software Foundation; either version 2 of the License, or (at your -% option) any later version. -% -% LuaTeX is distributed in the hope that it will be useful, but WITHOUT -% ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -% FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public -% License for more details. -% -% You should have received a copy of the GNU General Public License along -% with LuaTeX; if not, see <http://www.gnu.org/licenses/>. - -@ @c +/* + +texlang.w + +Copyright 2006-2012 Taco Hoekwater <taco@luatex.org> + +This file is part of LuaTeX. + +LuaTeX is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2 of the License, or (at your +option) any later version. + +LuaTeX is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +License for more details. + +You should have received a copy of the GNU General Public License along +with LuaTeX; if not, see <http://www.gnu.org/licenses/>. + +*/ #include "ptexlib.h" #include <string.h> #include "lua/luatex-api.h" -@ Low-level helpers +/*tex Low-level helpers */ -@ @c #define unVERBOSE #define MAX_TEX_LANGUAGES 16384 @@ -60,7 +61,9 @@ struct tex_language *new_language(int n) lang->post_exhyphen_char = 0; lang->hyphenation_min = -1; if (saving_hyph_codes_par) { - /* for now, we might just use specific value for whatever task */ + /*tex + For now, we might just use specific value for whatever task. + */ hj_codes_from_lc_codes(l); } return lang; @@ -82,7 +85,6 @@ struct tex_language *get_language(int n) } } -@ @c void set_pre_hyphen_char(int n, int v) { struct tex_language *l = get_language((int) n); @@ -183,7 +185,6 @@ void load_tex_patterns(int curlang, halfword head) load_patterns(get_language(curlang), (unsigned char *) s); } -@ @c #define STORE_CHAR(l,x) do { \ unsigned xx = get_hj_code(l,x); \ if (!xx || xx <= 32) { \ @@ -192,17 +193,24 @@ void load_tex_patterns(int curlang, halfword head) uindex = uni2string(uindex, xx); \ } while (0) -@ Cleans one word which is returned in |cleaned|, returns the new offset into -|buffer| +/* + + This cleans one word which is returned in |cleaned|, returns the new offset + into |buffer|. + +*/ -@c const char *clean_hyphenation(int id, const char *buff, char **cleaned) { int items = 0; - unsigned char word[MAX_WORD_LEN + 1]; /* work buffer for bytes */ - unsigned uword[MAX_WORD_LEN + 1] = { 0 }; /* work buffer for unicode */ - int u = 0; /* unicode buffer value */ - int i = 0; /* index into buffer */ + /*tex Work buffer for bytes: */ + unsigned char word[MAX_WORD_LEN + 1]; + /*tex Work buffer for \UNICODE: */ + unsigned uword[MAX_WORD_LEN + 1] = { 0 }; + /*tex The \UNICODE\ buffer value: */ + int u = 0; + /*tex The index into buffer: */ + int i = 0; char *uindex = (char *)word; const char *s = buff; @@ -210,21 +218,21 @@ const char *clean_hyphenation(int id, const char *buff, char **cleaned) word[i++] = (unsigned)*s; s++; if ((s-buff)>MAX_WORD_LEN) { - /* todo: this is too strict, should count unicode, not bytes */ + /*tex Todo: this is too strict, should count \UNICODE, not bytes. */ *cleaned = NULL; tex_error("exception too long", NULL); return s; } } - /* now convert the input to unicode */ + /*tex Now convert the input to \UNICODE. */ word[i] = '\0'; utf2uni_strcpy(uword, (const char *)word); - /* build the new word string */ + /*tex Build the new word string. */ i = 0; while (uword[i]>0) { u = uword[i++]; if (u == '-') { - /* skip */ + /*tex Skip. */ } else if (u == '=') { STORE_CHAR(id,'-'); } else if (u == '{') { @@ -259,6 +267,9 @@ const char *clean_hyphenation(int id, const char *buff, char **cleaned) tex_error("exception syntax error", NULL); return s; } + if (uword[i] == '[' && uword[i+1] >= '0' && uword[i+1] <= '9' && uword[i+2] == ']') { + i += 3; + } } else { STORE_CHAR(id,u); } @@ -268,7 +279,6 @@ const char *clean_hyphenation(int id, const char *buff, char **cleaned) return s; } -@ @c void load_hyphenation(struct tex_language *lang, const unsigned char *buff) { const char *s; @@ -322,7 +332,6 @@ void load_tex_hyphenation(int curlang, halfword head) load_hyphenation(get_language(curlang), (unsigned char *) s); } -@ @c static halfword insert_discretionary(halfword t, halfword pre, halfword post, halfword replace, int penalty) { halfword g; @@ -331,21 +340,21 @@ static halfword insert_discretionary(halfword t, halfword pre, halfword post, ha int attr = node_attr(t) ; disc_penalty(d) = penalty; if (t == replace) { - /* prev disc next-next */ + /*tex We have |prev disc next-next|. */ try_couple_nodes(d, vlink(t)); try_couple_nodes(alink(t), d); alink(t) = null; vlink(t) = null; replace = t ; } else { - /* prev disc next */ + /*tex We have |prev disc next|. */ try_couple_nodes(d, vlink(t)); couple_nodes(t, d); } if (replace != null) { f = font(replace); } else { - /* For compound words following explicit hyphens. */ + /*tex For compound words following explicit hyphens. */ f = get_cur_font(); } for (g = pre; g != null; g = vlink(g)) { @@ -406,7 +415,6 @@ halfword insert_syllable_discretionary(halfword t, lang_variables * lan) } set_disc_field(pre_break(n), g); } - if (lan->post_hyphen_char > 0) { t = vlink(n); g = raw_glyph_node(); @@ -423,7 +431,6 @@ halfword insert_syllable_discretionary(halfword t, lang_variables * lan) return n; } -@ @c static halfword insert_character(halfword t, int c) { halfword p; @@ -456,12 +463,12 @@ static halfword compound_word_break(halfword t, int clang) return disc; } -@ @c void set_disc_field(halfword f, halfword t) { if (t != null) { - /* - couple_nodes(f, t); // better not expose f as prev pointer + /*tex + No |couple_nodes(f, t);| as we can better not expose |f| as |prev| + pointer. */ vlink(f) = t ; alink(t) = null ; @@ -472,14 +479,14 @@ void set_disc_field(halfword f, halfword t) } } -@ @c static char *hyphenation_exception(int exceptions, char *w) { char *ret = NULL; lua_checkstack(Luas, 2); lua_rawgeti(Luas, LUA_REGISTRYINDEX, exceptions); - if (lua_istable(Luas, -1)) { /* ?? */ - lua_pushstring(Luas, w); /* word table */ + if (lua_istable(Luas, -1)) { + /*tex Word table: */ + lua_pushstring(Luas, w); lua_rawget(Luas, -2); if (lua_type(Luas, -1) == LUA_TSTRING) { ret = xstrdup(lua_tostring(Luas, -1)); @@ -491,7 +498,6 @@ static char *hyphenation_exception(int exceptions, char *w) return ret; } -@ @c char *exception_strings(struct tex_language *lang) { const char *value; @@ -503,8 +509,8 @@ char *exception_strings(struct tex_language *lang) lua_checkstack(Luas, 2); lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions); if (lua_istable(Luas, -1)) { - /* iterate and join */ - lua_pushnil(Luas); /* first key */ + /*tex Iterate and join. */ + lua_pushnil(Luas); while (lua_next(Luas, -2) != 0) { value = lua_tolstring(Luas, -1, &l); if (current + 2 + l > size) { @@ -520,15 +526,19 @@ char *exception_strings(struct tex_language *lang) return ret; } -@ the sequence from |wordstart| to |r| can contain only normal characters it -could be faster to modify a halfword pointer and return an integer +/*tex + + The sequence from |wordstart| to |r| can contain only normal characters it + could be faster to modify a halfword pointer and return an integer + +*/ -@c static halfword find_exception_part(unsigned int *j, unsigned int *uword, int len) { halfword g = null, gg = null; register unsigned i = *j; - i++; /* this puts uword[i] on the |{| */ + /*tex This puts uword[i] on the |{|. */ + i++; while (i < (unsigned) len && uword[i + 1] != '}') { if (g == null) { gg = new_char(0, (int) uword[i + 1]); @@ -548,7 +558,8 @@ static int count_exception_part(unsigned int *j, unsigned int *uword, int len) { int ret = 0; register unsigned i = *j; - i++; /* this puts uword[i] on the |{| */ + /*tex This puts uword[i] on the |{|. */ + i++; while (i < (unsigned) len && uword[i + 1] != '}') { ret++; i++; @@ -557,22 +568,23 @@ static int count_exception_part(unsigned int *j, unsigned int *uword, int len) return ret; } -@ @c static const char *PAT_ERROR[] = { "Exception discretionaries should contain three pairs of braced items.", "No intervening spaces are allowed.", NULL }; -/* +/*tex + The exceptions are taken as-is: no min values are taken into account. One can add normal patterns on-the-fly if needed. + */ static void do_exception(halfword wordstart, halfword r, char *replacement) { unsigned i; - halfword t; + halfword t, pen; unsigned len; int clang; lang_variables langdata; @@ -584,126 +596,191 @@ static void do_exception(halfword wordstart, halfword r, char *replacement) clang = char_lang(wordstart); langdata.pre_hyphen_char = get_pre_hyphen_char(clang); langdata.post_hyphen_char = get_post_hyphen_char(clang); - for (i = 0; i < len; i++) { - if (uword[i + 1] == '-') { /* a hyphen follows */ - while (vlink(t) != r && (type(t) != glyph_node || !is_simple_character(t))) - t = vlink(t); + if (uword[i + 1] == 0 ) { + /*tex We ran out of the exception pattern. */ + break; + } else if (uword[i + 1] == '-') { + /*tex A hyphen follows. */ if (vlink(t) == r) break; insert_syllable_discretionary(t, &langdata); t = vlink(t); /* skip the new disc */ } else if (uword[i + 1] == '=') { - /* do nothing ? */ + /*tex We skip a disc. */ t = vlink(t); } else if (uword[i + 1] == '{') { + /*tex We ran into an exception |{}{}{}| or |{}{}{}[]|. */ halfword gg, hh, replace = null; int repl; + /*tex |pre| */ gg = find_exception_part(&i, uword, (int) len); if (i == len || uword[i + 1] != '{') { tex_error("broken pattern 1", PAT_ERROR); } + /*tex |post| */ hh = find_exception_part(&i, uword, (int) len); if (i == len || uword[i + 1] != '{') { tex_error("broken pattern 2", PAT_ERROR); } + /*tex |replace| */ repl = count_exception_part(&i, uword, (int) len); if (i == len) { tex_error("broken pattern 3", PAT_ERROR); } - /*i++; *//* jump over the last right brace */ + /*tex Play safe. */ if (vlink(t) == r) break; + /*tex Let's deal with an (optional) replacement. */ if (repl > 0) { + /*tex Assemble the replace stream. */ halfword q = t; replace = vlink(q); while (repl > 0 && q != null) { q = vlink(q); - if (type(q) == glyph_node) { + if (type(q) == glyph_node || type(q) == disc_node) { repl--; + } else { + break ; } } + /*tex Remove it from the main stream */ try_couple_nodes(t, vlink(q)); + /*tex and finish it in the replace. */ vlink(q) = null; + /*tex Sanitize the replace stream (we could use the flattener instead). */ + q = replace ; + while (q != null) { + halfword n = vlink(q); + if (type(q) == disc_node) { + /*tex Beware: the replacement starts after the no_break pointer. */ + halfword r = vlink(no_break(q)); + vlink(no_break(q)) = null; + alink(r) = null ; + /*tex Insert the replacement glyph. */ + if (q == replace) { + replace = r; + } else { + try_couple_nodes(alink(q),r); + } + /*tex Append the glyph (one). */ + try_couple_nodes(r,n); + /*tex Flush the disc. */ + flush_node(q); + } + q = n ; + } + } + /*tex Let's check if we have a penalty spec. */ + if (((i+3) < len) && uword[i+1] == '[' && uword[i+2] >= '0' && uword[i+2] <= '9' && uword[i+3] == ']') { + if (exception_penalty_par > 0) { + if (exception_penalty_par > 100000) { + pen = (uword[i+2] - '0') * exception_penalty_par ; + } else { + pen = exception_penalty_par; + } + } else { + pen = hyphen_penalty_par; + } + i += 3; + } else { + pen = hyphen_penalty_par; + } + /*tex And now we insert a disc node. */ + t = insert_discretionary(t, gg, hh, replace, pen); + /*tex We skip the new disc node. */ + t = vlink(t); + /*tex check if we have two exceptions in a row */ + if (uword[i + 1] == '{') { + i--; } - t = insert_discretionary(t, gg, hh, replace, hyphen_penalty_par); - t = vlink(t); /* skip the new disc */ } else { t = vlink(t); } + /*tex Again we play safe. */ + if (t == null || vlink(t) == r) { + break; + } } } -@ This is a documentation section from the pascal web file. It is not true any -more, but I do not have time right now to rewrite it -- Taco - -When the line-breaking routine is unable to find a feasible sequence of -breakpoints, it makes a second pass over the paragraph, attempting to hyphenate -the hyphenatable words. The goal of hyphenation is to insert discretionary -material into the paragraph so that there are more potential places to break. - -The general rules for hyphenation are somewhat complex and technical, because we -want to be able to hyphenate words that are preceded or followed by punctuation -marks, and because we want the rules to work for languages other than English. We -also must contend with the fact that hyphens might radically alter the ligature -and kerning structure of a word. - -A sequence of characters will be considered for hyphenation only if it belongs to -a ``potentially hyphenatable part'' of the current paragraph. This is a sequence -of nodes $p_0p_1\ldots p_m$ where $p_0$ is a glue node, $p_1\ldots p_{m-1}$ are -either character or ligature or whatsit or implicit kern nodes, and $p_m$ is a -glue or penalty or insertion or adjust or mark or whatsit or explicit kern node. -(Therefore hyphenation is disabled by boxes, math formulas, and discretionary -nodes already inserted by the user.) The ligature nodes among $p_1\ldots p_{m-1}$ -are effectively expanded into the original non-ligature characters; the kern -nodes and whatsits are ignored. Each character |c| is now classified as either a -nonletter (if |lc_code(c)=0|), a lowercase letter (if |lc_code(c)=c|), or an -uppercase letter (otherwise); an uppercase letter is treated as if it were -|lc_code(c)| for purposes of hyphenation. The characters generated by $p_1\ldots -p_{m-1}$ may begin with nonletters; let $c_1$ be the first letter that is not in -the middle of a ligature. Whatsit nodes preceding $c_1$ are ignored; a whatsit -found after $c_1$ will be the terminating node $p_m$. All characters that do not -have the same font as $c_1$ will be treated as nonletters. The |hyphen_char| for -that font must be between 0 and 255, otherwise hyphenation will not be attempted. -\TeX\ looks ahead for as many consecutive letters $c_1\ldots c_n$ as possible; -however, |n| must be less than 64, so a character that would otherwise be -$c_{64}$ is effectively not a letter. Furthermore $c_n$ must not be in the middle -of a ligature. In this way we obtain a string of letters $c_1\ldots c_n$ that are -generated by nodes $p_a\ldots p_b$, where |1<=a<=b+1<=m|. If |n>=l_hyf+r_hyf|, -this string qualifies for hyphenation; however, |uc_hyph| must be positive, if -$c_1$ is uppercase. - -The hyphenation process takes place in three stages. First, the candidate -sequence $c_1\ldots c_n$ is found; then potential positions for hyphens are -determined by referring to hyphenation tables; and finally, the nodes $p_a\ldots -p_b$ are replaced by a new sequence of nodes that includes the discretionary -breaks found. - -Fortunately, we do not have to do all this calculation very often, because of the -way it has been taken out of \TeX's inner loop. For example, when the second -edition of the author's 700-page book {\sl Seminumerical Algorithms} was typeset -by \TeX, only about 1.2 hyphenations needed to be @^Knuth, Donald Ervin@> tried -per paragraph, since the line breaking algorithm needed to use two passes on only -about 5 per cent of the paragraphs. - -When a word been set up to contain a candidate for hyphenation, \TeX\ first looks -to see if it is in the user's exception dictionary. If not, hyphens are inserted -based on patterns that appear within the given word, using an algorithm due to -Frank~M. Liang. @^Liang, Franklin Mark@> - -@ This is incompatible with TEX because the first word of a paragraph can be -hyphenated, but most european users seem to agree that prohibiting hyphenation -there was not the best idea ever. - -@c -/* - More strict: \hyphenationbounds - - 0 = not strict - 1 = strict start - 2 = strict end - 3 = strict start and strict end - +/*tex + + This is a documentation section from the pascal web file. It is not true any + more, but I (Taco) do not have time right now to rewrite it. + + When the line-breaking routine is unable to find a feasible sequence of + breakpoints, it makes a second pass over the paragraph, attempting to + hyphenate the hyphenatable words. The goal of hyphenation is to insert + discretionary material into the paragraph so that there are more potential + places to break. + + The general rules for hyphenation are somewhat complex and technical, because + we want to be able to hyphenate words that are preceded or followed by + punctuation marks, and because we want the rules to work for languages other + than English. We also must contend with the fact that hyphens might radically + alter the ligature and kerning structure of a word. + + A sequence of characters will be considered for hyphenation only if it + belongs to a ``potentially hyphenatable part'' of the current paragraph. This + is a sequence of nodes $p_0p_1\ldots p_m$ where $p_0$ is a glue node, + $p_1\ldots p_{m-1}$ are either character or ligature or whatsit or implicit + kern nodes, and $p_m$ is a glue or penalty or insertion or adjust or mark or + whatsit or explicit kern node. (Therefore hyphenation is disabled by boxes, + math formulas, and discretionary nodes already inserted by the user.) The + ligature nodes among $p_1\ldots p_{m-1}$ are effectively expanded into the + original non-ligature characters; the kern nodes and whatsits are ignored. + Each character |c| is now classified as either a nonletter (if + |lc_code(c)=0|), a lowercase letter (if |lc_code(c)=c|), or an uppercase + letter (otherwise); an uppercase letter is treated as if it were |lc_code(c)| + for purposes of hyphenation. The characters generated by $p_1\ldots p_{m-1}$ + may begin with nonletters; let $c_1$ be the first letter that is not in the + middle of a ligature. Whatsit nodes preceding $c_1$ are ignored; a whatsit + found after $c_1$ will be the terminating node $p_m$. All characters that do + not have the same font as $c_1$ will be treated as nonletters. The + |hyphen_char| for that font must be between 0 and 255, otherwise hyphenation + will not be attempted. \TeX\ looks ahead for as many consecutive letters + $c_1\ldots c_n$ as possible; however, |n| must be less than 64, so a + character that would otherwise be $c_{64}$ is effectively not a letter. + Furthermore $c_n$ must not be in the middle of a ligature. In this way we + obtain a string of letters $c_1\ldots c_n$ that are generated by nodes + $p_a\ldots p_b$, where |1<=a<=b+1<=m|. If |n>=l_hyf+r_hyf|, this string + qualifies for hyphenation; however, |uc_hyph| must be positive, if $c_1$ is + uppercase. + + The hyphenation process takes place in three stages. First, the candidate + sequence $c_1\ldots c_n$ is found; then potential positions for hyphens are + determined by referring to hyphenation tables; and finally, the nodes + $p_a\ldots p_b$ are replaced by a new sequence of nodes that includes the + discretionary breaks found. + + Fortunately, we do not have to do all this calculation very often, because of + the way it has been taken out of \TeX's inner loop. For example, when the + second edition of the author's 700-page book {\sl Seminumerical Algorithms} + was typeset by \TeX, only about 1.2 hyphenations needed to be tried per + paragraph, since the line breaking algorithm needed to use two passes on only + about 5 per cent of the paragraphs. + + When a word been set up to contain a candidate for hyphenation, \TeX\ first + looks to see if it is in the user's exception dictionary. If not, hyphens are + inserted based on patterns that appear within the given word, using an + algorithm due to Frank~M. Liang. + + This is incompatible with \TEX\ because the first word of a paragraph can be + hyphenated, but most European users seem to agree that prohibiting + hyphenation there was not the best idea ever. + + We have some variants available that are controlled by the paremeter + \type {\hyphenationbounds}: + + \starttabulate + \NC \type {0} \NC not strict \NC \NR + \NC \type {1} \NC strict start \NC \NR + \NC \type {2} \NC strict end \NC \NR + \NC \type {3} \NC strict start and strict end \NC \NR + \stoptabulate + + \startbuffer \parindent0pt \hsize=1.1cm 12-34-56 \par 12-34-\hbox{56} \par @@ -716,20 +793,24 @@ there was not the best idea ever. 12-34-\vrule width 1em height 1.5ex \par 12-\hbox{34}-56 \par 12-\vrule width 1em height 1.5ex-56 \par + \stopbuffer -*/ + \typebuffer -/* - We only accept an explicit hyphen when there is a preceding glyph and we skip a sequence of - explicit hyphens as that normally indicates a -- or --- ligature in which case we can in a - worse case usage get bad node lists later on due to messed up ligature building as these - dashes are ligatures in base fonts. This is a side effect of the separating the hyphenation, - ligaturing and kerning steps. A test is cmr with ------. + \startpacked \getbuffer \stopbuffer + + We only accept an explicit hyphen when there is a preceding glyph and we skip + a sequence of explicit hyphens as that normally indicates a \type {--} or + \type {---} ligature in which case we can in a worse case usage get bad node + lists later on due to messed up ligature building as these dashes are + ligatures in base fonts. This is a side effect of the separating the + hyphenation, ligaturing and kerning steps. A test is cmr with \type {------}. - A font handler can collapse successive hyphens but it's not nice to put the burden there. A - somewhat messy border case is ---- but in LuaTeX we don't treat -- and --- special. Also, - traditional TeX will break a line at -foo but this can be disabled by setting the automatic - mode to 1. + A font handler can collapse successive hyphens but it's not nice to put the + burden there. A somewhat messy border case is \type {----} but in \LUATEX\ we + don't treat \type {--} and \type {---} special. Also, traditional \TEX\ will + break a line at \type {-foo} but this can be disabled by setting the + automatic mode to \type {1}. */ @@ -742,79 +823,78 @@ static halfword find_next_wordstart(halfword r, halfword first_language, halfwor halfword t ; while (r != null) { switch (type(r)) { - case boundary_node: - if (subtype(r) == word_boundary) { + case boundary_node: + if (subtype(r) == word_boundary) { + start_ok = 1; + } + break; + case hlist_node: + case vlist_node: + case rule_node: + case dir_node: + case whatsit_node: + if (strict_bound == 1 || strict_bound == 3) { + start_ok = 0; + } + break; + case glue_node: start_ok = 1; - } - break; - case hlist_node: /* new > 0.95 */ - case vlist_node: /* new > 0.95 */ - case rule_node: /* new > 0.95 */ - case dir_node: - case whatsit_node: - if (strict_bound == 1 || strict_bound == 3) { - start_ok = 0; - } - break; - case glue_node: - start_ok = 1; - break; - case math_node: - while (mathlevel > 0) { - r = vlink(r); - if (r == null) - return r; - if (type(r) == math_node) { - if (subtype(r) == before) { - mathlevel++; - } else { - mathlevel--; + break; + case math_node: + while (mathlevel > 0) { + r = vlink(r); + if (r == null) + return r; + if (type(r) == math_node) { + if (subtype(r) == before) { + mathlevel++; + } else { + mathlevel--; + } } } - } - break; - case glyph_node: - if (is_simple_character(r)) { - chr = character(r) ; - if (chr == ex_hyphen_char_par) { - t = vlink(r) ; - if ((automatic_hyphen_mode_par == 0) && (t != null) && (type(t) == glyph_node) && (character(t) != ex_hyphen_char_par)) { - /* we have no word yet and the next character is a non hyphen */ - r = compound_word_break(r, char_lang(r)); - } else { - /* we jump over the sequence of hyphens */ - while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { - r = t ; - t = vlink(r) ; + break; + case glyph_node: + if (is_simple_character(r)) { + chr = character(r) ; + if (chr == ex_hyphen_char_par) { + t = vlink(r) ; + if ((automatic_hyphen_mode_par == 0) && (t != null) && (type(t) == glyph_node) && (character(t) != ex_hyphen_char_par)) { + /*tex We have no word yet and the next character is a non hyphen. */ + r = compound_word_break(r, char_lang(r)); + } else { + /*tex We jump over the sequence of hyphens. */ + while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { + r = t ; + t = vlink(r) ; + } + if (t == null) { + /*tex We reached the end of the list so we have no word start. */ + return null; + } } - if (t == null) { - /* we reached the end of the list so we have no word start */ - return null; + /*tex We need a restart. */ + start_ok = 0; + } else if (start_ok && (char_lang(r)>=first_language) && ((l = get_hj_code(char_lang(r),chr)) > 0)) { + if (char_uchyph(r) || l == chr || l <= 32) { + return r; + } else { + start_ok = 0; } - } - /* we need a restart */ - start_ok = 0; - } else if (start_ok && (char_lang(r)>=first_language) && ((l = get_hj_code(char_lang(r),chr)) > 0)) { - if (char_uchyph(r) || l == chr || l <= 32) { - return r; } else { - start_ok = 0; + /*tex We go on. */ } - } else { - /* go on */ } - } - break; - default: - start_ok = 0; - break; + break; + default: + start_ok = 0; + break; } r = vlink(r); } return r; } -@ @c static int valid_wordend(halfword s, halfword strict_bound) { register halfword r = s; @@ -831,16 +911,16 @@ static int valid_wordend(halfword s, halfword strict_bound) if (r == null || (type(r) == glyph_node && is_simple_character(r) && clang != char_lang(r)) || type(r) == glue_node || type(r) == penalty_node - || (type(r) == kern_node && (subtype(r) == explicit_kern || /* so why not italic correction ? */ + || (type(r) == kern_node && (subtype(r) == explicit_kern || subtype(r) == italic_kern || subtype(r) == accent_kern )) - || ((type(r) == hlist_node || /* new > 0.95 */ - type(r) == vlist_node || /* new > 0.95 */ - type(r) == rule_node || /* new > 0.95 */ - type(r) == dir_node || /* new > 0.97 */ + || ((type(r) == hlist_node || + type(r) == vlist_node || + type(r) == rule_node || + type(r) == dir_node || type(r) == whatsit_node || - type(r) == ins_node || /* yes or no strict test */ - type(r) == adjust_node /* yes or no strict test */ + type(r) == ins_node || + type(r) == adjust_node ) && ! (strict_bound == 2 || strict_bound == 3)) || type(r) == boundary_node ) @@ -848,7 +928,6 @@ static int valid_wordend(halfword s, halfword strict_bound) return 0; } -@ @c void hnj_hyphenation(halfword head, halfword tail) { int lchar, i; @@ -859,37 +938,43 @@ void hnj_hyphenation(halfword head, halfword tail) char *hy = utf8word; char *replacement = NULL; boolean explicit_hyphen = false; + boolean valid_word = false; halfword first_language = first_valid_language_par; halfword strict_bound = hyphenation_bounds_par; halfword s, r = head, wordstart = null, save_tail1 = null, left = null, right = null; - - /* + halfword expstart = null; + boolean compound_hyphen = compound_hyphen_mode_par; + /*tex This first movement assures two things: - \item{a)} that we won't waste lots of time on something that has been handled already - (in that case, none of the glyphs match |simple_character|). - - \item{b)} that the first word can be hyphenated. if the movement was not explicit, - then the indentation at the start of a paragraph list would make |find_next_wordstart()| - look too far ahead. + \startitemize + \startitem + That we won't waste lots of time on something that has been + handled already (in that case, none of the glyphs match + |simple_character|). + \stopitem + \startitem + That the first word can be hyphenated. if the movement was not + explicit, then the indentation at the start of a paragraph list + would make |find_next_wordstart()| look too far ahead. + \stopitem + \stopitemize */ - while (r != null && (type(r) != glyph_node || !is_simple_character(r))) { r = vlink(r); } - /* + /*tex This will make |r| a glyph node with subtype character. */ r = find_next_wordstart(r,first_language,strict_bound); if (r == null) return; - assert(tail != null); save_tail1 = vlink(tail); s = new_penalty(0,word_penalty); couple_nodes(tail, s); - - while (r != null) { /* could be while(1), but let's be paranoid */ + while (r != null) { + /*tex This could be |while(1)|, but let's be paranoid: */ int clang, lhmin, rhmin, hmin; halfword hyf_font; halfword end_word = r; @@ -897,7 +982,7 @@ void hnj_hyphenation(halfword head, halfword tail) assert(is_simple_character(wordstart)); hyf_font = font(wordstart); if (hyphen_char(hyf_font) < 0) { - /* For backward compatibility: */ + /*tex For backward compatibility we set: */ hyf_font = 0; } clang = char_lang(wordstart); @@ -947,19 +1032,28 @@ void hnj_hyphenation(halfword head, halfword tail) r = vlink(r); } if (explicit_hyphen == true) { - /* we are not at the start, so we only need to look ahead */ + /*tex we are not at the start, so we only need to look ahead */ halfword t = vlink(r) ; if ((automatic_hyphen_mode_par == 0 || automatic_hyphen_mode_par == 1) && (t != null) && ((type(t) == glyph_node) && (character(t) != ex_hyphen_char_par))) { - /* we have a word already but the next character may not be a hyphen too */ + /*tex we have a word already but the next character may not be a hyphen too */ r = compound_word_break(r, char_lang(r)); + if (compound_hyphen) { + if (expstart == null) { + expstart = wordstart; + } + explicit_hyphen = false; + hy = uni2string(hy, '-'); + r = t; + continue; + } } else { - /* we jump over the sequence of hyphens */ - while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { + /*tex we jump over the sequence of hyphens */ + while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { r = t ; t = vlink(r) ; } if (t == null) { - /* we reached the end of the list and will quit the loop later */ + /*tex we reached the end of the list and will quit the loop later */ r = null; } } @@ -971,32 +1065,67 @@ void hnj_hyphenation(halfword head, halfword tail) && (lang = tex_languages[clang]) != NULL ) { *hy = 0; + /*tex + this is messy and nasty: we can have a word with a - in it which + is why we have two branches + */ if (lang->exceptions != 0 && (replacement = hyphenation_exception(lang->exceptions, utf8word)) != NULL) { - /* handle the exception and go on to the next word */ - do_exception(wordstart, r, replacement); + /*tex handle the exception and go on to the next word */ + if (expstart == null) { + do_exception(wordstart, r, replacement); + } else { + do_exception(expstart,r,replacement); + } free(replacement); + } else if (expstart != null) { + /*tex We're done already */ } else if (lang->patterns != NULL) { + /*tex A safeguard, not needed but doesn't hurt either: */ + valid_word = true; left = wordstart; for (i = lhmin; i > 1; i--) { left = vlink(left); + if (left == null) { + valid_word = false; + break ; + } while (!is_simple_character(left)) { left = vlink(left); + if (left == null) { + valid_word = false; + break ; + } } - /* if (!left) break; */ - /* what is left overruns right .. a bit messy */ } - right = r; - for (i = rhmin; i > 0; i--) { - right = alink(right); - while (!is_simple_character(right)) { + if (valid_word) { + right = r; + if (right == left) { + valid_word = false; + break ; + } + for (i = rhmin; i > 0; i--) { right = alink(right); + if (right == null || right == left) { + valid_word = false; + break ; + } + while (!is_simple_character(right)) { + right = alink(right); + if (right == null || right == left) { + valid_word = false; + break ; + } + } + } + if (valid_word && expstart == null) { + (void) hnj_hyphen_hyphenate(lang->patterns, wordstart, end_word, wordlen, left, right, &langdata); + } else { + /*tex nothing yet */ } - /* if (!right) break; */ - /* what is right overruns left .. a bit messy */ } - (void) hnj_hyphen_hyphenate(lang->patterns, wordstart, end_word, wordlen, left, right, &langdata); } } + expstart = null ; explicit_hyphen = false; wordlen = 0; hy = utf8word; @@ -1008,36 +1137,40 @@ void hnj_hyphenation(halfword head, halfword tail) vlink(tail) = save_tail1; } -@ @c void new_hyphenation(halfword head, halfword tail) { + int i, top; register int callback_id = 0; if (head == null || vlink(head) == null) return; fix_node_list(head); callback_id = callback_defined(hyphenate_callback); if (callback_id > 0) { + top = lua_gettop(Luas); if (!get_callback(Luas, callback_id)) { - lua_pop(Luas, 2); + lua_settop(Luas, top); return; } nodelist_to_lua(Luas, head); nodelist_to_lua(Luas, tail); - if (lua_pcall(Luas, 2, 0, 0) != 0) { + if ((i=lua_pcall(Luas, 2, 0, 0)) != 0) { formatted_warning("hyphenation","bad specification: %s",lua_tostring(Luas, -1)); - lua_pop(Luas, 2); - lua_error(Luas); + lua_settop(Luas, top); + luatex_error(Luas, (i == LUA_ERRRUN ? 0 : 1)); return; } - lua_pop(Luas, 1); + lua_settop(Luas, top); } else if (callback_id == 0) { hnj_hyphenation(head, tail); } } -@ Dumping and undumping languages. +/*tex + + Dumping and undumping languages: + +*/ -@c #define dump_string(a) \ if (a!=NULL) { \ x = (int)strlen(a)+1; \ @@ -1106,7 +1239,7 @@ static void undump_one_language(int i) lang->post_exhyphen_char = x; undump_int(x); lang->hyphenation_min = x; - /* patterns */ + /*tex patterns */ undump_int(x); if (x > 0) { s = xmalloc((unsigned) x); @@ -1114,7 +1247,7 @@ static void undump_one_language(int i) load_patterns(lang, (unsigned char *) s); free(s); } - /* exceptions */ + /*tex exceptions */ undump_int(x); if (x > 0) { s = xmalloc((unsigned) x); @@ -1137,21 +1270,26 @@ void undump_language_data(void) } } -@ When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named -|new_hyph_exceptions| to do the right thing. +/*tex + + When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named + |new_hyph_exceptions| to do the right thing. + +*/ -@c void new_hyph_exceptions(void) -{ /* enters new exceptions */ +{ (void) scan_toks(false, true); load_tex_hyphenation(language_par, def_ref); flush_list(def_ref); } -@ Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a procedure named -|new_patterns|. +/* + Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a procedure + named |new_patterns|. + +*/ -@c void new_patterns(void) { (void) scan_toks(false, true); @@ -1159,10 +1297,14 @@ void new_patterns(void) flush_list(def_ref); } -@ `\.{\\prehyphenchar}', sets the |pre_break| character, and `\.{\\posthyphenchar}' the -|post_break| character. Their respective defaults are ascii hyphen ("-") and zero (nul). +/*tex + + `\.{\\prehyphenchar}', sets the |pre_break| character, and + `\.{\\posthyphenchar}' the |post_break| character. Their respective defaults + are ascii hyphen ("-") and zero (nul). + +*/ -@c void new_pre_hyphen_char(void) { scan_optional_equals(); @@ -1177,10 +1319,14 @@ void new_post_hyphen_char(void) set_post_hyphen_char(language_par, cur_val); } -@ `\.{\\preexhyphenchar}', sets the |pre_break| character, and `\.{\\postexhyphenchar}' the -|post_break| character. Their defaults are both zero (nul). +/*tex + + `\.{\\preexhyphenchar}', sets the |pre_break| character, and + `\.{\\postexhyphenchar}' the |post_break| character. Their defaults are both + zero (nul). + +*/ -@c void new_pre_exhyphen_char(void) { scan_optional_equals(); |