From ecd1605a23ef54d2a4a69e1a361172d350e03b3b Mon Sep 17 00:00:00 2001 From: Luigi Scarso Date: Wed, 5 Sep 2018 21:57:30 +0000 Subject: luatex: correct automake file for pplib; replaced cweb files with c files. git-svn-id: svn://tug.org/texlive/trunk@48594 c570f23f-e606-0410-a88d-b1316a301751 --- Build/source/texk/web2c/luatexdir/lang/hnjalloc.c | 65 + Build/source/texk/web2c/luatexdir/lang/hnjalloc.w | 69 -- Build/source/texk/web2c/luatexdir/lang/hyphen.c | 788 ++++++++++++ Build/source/texk/web2c/luatexdir/lang/hyphen.w | 875 ------------- Build/source/texk/web2c/luatexdir/lang/texlang.c | 1359 +++++++++++++++++++++ Build/source/texk/web2c/luatexdir/lang/texlang.w | 1213 ------------------ 6 files changed, 2212 insertions(+), 2157 deletions(-) create mode 100644 Build/source/texk/web2c/luatexdir/lang/hnjalloc.c delete mode 100644 Build/source/texk/web2c/luatexdir/lang/hnjalloc.w create mode 100644 Build/source/texk/web2c/luatexdir/lang/hyphen.c delete mode 100644 Build/source/texk/web2c/luatexdir/lang/hyphen.w create mode 100644 Build/source/texk/web2c/luatexdir/lang/texlang.c delete mode 100644 Build/source/texk/web2c/luatexdir/lang/texlang.w (limited to 'Build/source/texk/web2c/luatexdir/lang') diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c new file mode 100644 index 00000000000..3ebeee9e4d5 --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c @@ -0,0 +1,65 @@ +/* + +hnjalloc.w + +LibHnj is dual licensed under LGPL and MPL. Boilerplate for both licenses +follows. + +LibHnj - a library for high quality hyphenation and justification Copyright (C) +1998 Raph Levien, (C) 2001 ALTLinux, Moscow + +This library is free software; you can redistribute it and/or modify it under the +terms of the GNU Library General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This library is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Library General Public License for more details. + +You should have received a copy of the GNU Library General Public License along +with this library; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307 USA. + +The contents of this file are subject to the Mozilla Public License Version 1.0 +(the "MPL"); you may not use this file except in compliance with the MPL. You may +obtain a copy of the MPL at http://www.mozilla.org/MPL/ + +Software distributed under the MPL is distributed on an "AS IS" basis, WITHOUT +WARRANTY OF ANY KIND, either express or implied. See the MPL for the specific +language governing rights and limitations under the MPL. + +*/ + +/*tex The wrappers for malloc */ + +#include +#include +#include "lang/hnjalloc.h" + +void *hnj_malloc(int size) +{ + void *p; + + p = malloc((size_t) size); + if (p == NULL) { + fprintf(stderr, "can't allocate %d bytes\n", size); + exit(1); + } + return p; +} + +void *hnj_realloc(void *p, int size) +{ + p = realloc(p, (size_t) size); + if (p == NULL) { + fprintf(stderr, "can't allocate %d bytes\n", size); + exit(1); + } + return p; +} + +void hnj_free(void *p) +{ + free(p); +} diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w deleted file mode 100644 index 91fefec9e5d..00000000000 --- a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w +++ /dev/null @@ -1,69 +0,0 @@ -% hnjalloc.w -% -% LibHnj is dual licensed under LGPL and MPL. Boilerplate for both -% licenses follows. -% -% -% LibHnj - a library for high quality hyphenation and justification -% Copyright (C) 1998 Raph Levien, (C) 2001 ALTLinux, Moscow -% -% This library is free software; you can redistribute it and/or -% modify it under the terms of the GNU Library General Public -% License as published by the Free Software Foundation; either -% version 2 of the License, or (at your option) any later version. -% -% This library is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -% Library General Public License for more details. -% -% You should have received a copy of the GNU Library General Public -% License along with this library; if not, write to the -% Free Software Foundation, Inc., 59 Temple Place - Suite 330, -% Boston, MA 02111-1307 USA. -% -% -% -% The contents of this file are subject to the Mozilla Public License -% Version 1.0 (the "MPL"); you may not use this file except in -% compliance with the MPL. You may obtain a copy of the MPL at -% http://www.mozilla.org/MPL/ -% -% Software distributed under the MPL is distributed on an "AS IS" basis, -% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL -% for the specific language governing rights and limitations under the -% MPL. - -@ wrappers for malloc -@c - -#include -#include -#include "lang/hnjalloc.h" - -void *hnj_malloc(int size) -{ - void *p; - - p = malloc((size_t) size); - if (p == NULL) { - fprintf(stderr, "can't allocate %d bytes\n", size); - exit(1); - } - return p; -} - -void *hnj_realloc(void *p, int size) -{ - p = realloc(p, (size_t) size); - if (p == NULL) { - fprintf(stderr, "can't allocate %d bytes\n", size); - exit(1); - } - return p; -} - -void hnj_free(void *p) -{ - free(p); -} diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.c b/Build/source/texk/web2c/luatexdir/lang/hyphen.c new file mode 100644 index 00000000000..3a96f195b2f --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.c @@ -0,0 +1,788 @@ +/* + +hyphen.c + +This file is derived from libhnj which is is dual licensed under LGPL and MPL. +Boilerplate for both licenses follows. + +LibHnj - a library for high quality hyphenation and justification + +(C) 1998 Raph Levien, +(C) 2001 ALTLinux, Moscow (http://www.alt-linux.org), +(C) 2001 Peter Novodvorsky (nidd@cs.msu.su) + +This library is free software; you can redistribute it and/or modify it under the +terms of the GNU Library General Public License as published by the Free Software +Foundation; either version 2 of the License, or (at your option) any later +version. + +This library is distributed in the hope that it will be useful, but WITHOUT ANY +WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A +PARTICULAR PURPOSE. See the GNU Library General Public License for more details. + +You should have received a copy of the GNU Library General Public License along +with this library; if not, write to the Free Software Foundation, Inc., 59 Temple +Place - Suite 330, Boston, MA 02111-1307 USA. + +The contents of this file are subject to the Mozilla Public License Version 1.0 +(the "MPL"); you may not use this file except in compliance with the MPL. You may +obtain a copy of the MPL at http://www.mozilla.org/MPL/ + +Software distributed under the MPL is distributed on an "AS IS" basis, WITHOUT +WARRANTY OF ANY KIND, either express or implied. See the MPL for the specific +language governing rights and limitations under the MPL. + +*/ + +#include "ptexlib.h" +#include "lua/luatex-api.h" + +#include /* for NULL, malloc */ +#include /* for fprintf */ +#include /* for strdup */ +#include /* for malloc used by substring inclusion */ + +#define MAXPATHS 40960 + +#ifdef UNX +# include /* for exit */ +#endif + +#include + +#define noVERBOSE + +#include "lang/hnjalloc.h" + +/*tex This could be moved to separate library. */ + +static unsigned char *hnj_strdup(const unsigned char *s) +{ + unsigned char *new; + size_t l; + + l = strlen((const char *) s); + new = hnj_malloc((int) l + 1); + memcpy(new, s, l); + new[l] = 0; + return new; +} + +/*tex + + First some type definitions and a little bit of a hash table implementation. + This simply maps strings to state numbers + +*/ + +typedef struct _HashTab HashTab; +typedef struct _HashEntry HashEntry; +typedef struct _HashIter HashIter; +typedef union _HashVal HashVal; + +/*tex A cheap, but effective, hack. */ + +#define HASH_SIZE 31627 + +struct _HashTab { + HashEntry *entries[HASH_SIZE]; +}; + +union _HashVal { + int state; + char *hyppat; +}; + +struct _HashEntry { + HashEntry *next; + unsigned char *key; + HashVal u; +}; + +struct _HashIter { + HashEntry **e; + HashEntry *cur; + int ndx; +}; + +/* The state state machine. */ + +typedef struct _HyphenState HyphenState; +typedef struct _HyphenTrans HyphenTrans; + +#define MAX_CHARS 256 +#define MAX_NAME 20 + +struct _HyphenDict { + int num_states; + int pat_length; + char cset[MAX_NAME]; + HyphenState *states; + HashTab *patterns; + HashTab *merged; + HashTab *state_num; +}; + +struct _HyphenState { + char *match; + /*tex Removed: + + \starttyping + char *repl; + signed char replindex; + signed char replcut; + \stoptyping + */ + int fallback_state; + int num_trans; + HyphenTrans *trans; +}; + +struct _HyphenTrans { + int uni_ch; + int new_state; +}; + +/*tex + + Combine two right-aligned number patterns, 04000 + 020 becomes 04020. This + works also for utf8 sequences because the substring is identical to the last + substring-length bytes of expr except for the (single byte) hyphenation + encoders + +*/ + +static char *combine(char *expr, const char *subexpr) +{ + size_t l1 = strlen(expr); + size_t l2 = strlen(subexpr); + size_t off = l1 - l2; + unsigned j; + for (j = 0; j < l2; j++) { + if (expr[off + j] < subexpr[j]) + expr[off + j] = subexpr[j]; + } + return expr; +} + +/*tex Some original code: */ + +static HashIter *new_HashIter(HashTab * h) +{ + HashIter *i = hnj_malloc(sizeof(HashIter)); + i->e = h->entries; + i->cur = NULL; + i->ndx = -1; + return i; +} + +static int nextHashStealPattern(HashIter * i, unsigned char **word, char **pattern) +{ + while (i->cur == NULL) { + if (i->ndx >= HASH_SIZE - 1) + return 0; + i->cur = i->e[++i->ndx]; + } + *word = i->cur->key; + *pattern = i->cur->u.hyppat; + i->cur->u.hyppat = NULL; + i->cur = i->cur->next; + return 1; +} + + +static int nextHash(HashIter * i, unsigned char **word) +{ + while (i->cur == NULL) { + if (i->ndx >= HASH_SIZE - 1) + return 0; + i->cur = i->e[++i->ndx]; + } + *word = i->cur->key; + i->cur = i->cur->next; + return 1; +} + +static int eachHash(HashIter * i, unsigned char **word, char **pattern) +{ + while (i->cur == NULL) { + if (i->ndx >= HASH_SIZE - 1) + return 0; + i->cur = i->e[++i->ndx]; + } + *word = i->cur->key; + *pattern = i->cur->u.hyppat; + i->cur = i->cur->next; + return 1; +} + +static void delete_HashIter(HashIter * i) +{ + hnj_free(i); +} + +/*tex + +A |char*| hash function from ASU, adapted from |Gtk+|: + +*/ + +static unsigned int hnj_string_hash(const unsigned char *s) +{ + const unsigned char *p; + unsigned int h = 0, g; + + for (p = s; *p != '\0'; p += 1) { + h = (h << 4) + *p; + if ((g = (h & 0xf0000000))) { + h = h ^ (g >> 24); + h = h ^ g; + } + } + return h; +} + +/*tex This assumes that key is not already present! */ + +static void state_insert(HashTab * hashtab, unsigned char *key, int state) +{ + int i; + HashEntry *e; + i = (int) (hnj_string_hash(key) % HASH_SIZE); + e = hnj_malloc(sizeof(HashEntry)); + e->next = hashtab->entries[i]; + e->key = key; + e->u.state = state; + hashtab->entries[i] = e; +} + +/*tex This also assumes that key is not already present! */ + +static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat) +{ + int i; + HashEntry *e; + i = (int) (hnj_string_hash(key) % HASH_SIZE); + for (e = hashtab->entries[i]; e; e = e->next) { + if (strcmp((char *) e->key, (char *) key) == 0) { + if (e->u.hyppat) { + if (hyppat && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) { + normal_warning("hyphenation","a conflicting pattern has been ignored"); + } + hnj_free(e->u.hyppat); + } + e->u.hyppat = hyppat; + hnj_free(key); + return; + } + } + e = hnj_malloc(sizeof(HashEntry)); + e->next = hashtab->entries[i]; + e->key = key; + e->u.hyppat = hyppat; + hashtab->entries[i] = e; +} + +/*tex We return |state| if found, otherwise |-1|. */ + +static int state_lookup(HashTab * hashtab, const unsigned char *key) +{ + int i; + HashEntry *e; + i = (int) (hnj_string_hash(key) % HASH_SIZE); + for (e = hashtab->entries[i]; e; e = e->next) { + if (!strcmp((const char *) key, (const char *) e->key)) { + return e->u.state; + } + } + return -1; +} + +/*tex We return |state| if found, otherwise |-1|. */ + +static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l) +{ + int i; + HashEntry *e; + /*tex The 256 should be enough. */ + unsigned char key[256]; + strncpy((char *) key, (const char *) chars, (size_t) l); + key[l] = 0; + i = (int) (hnj_string_hash(key) % HASH_SIZE); + for (e = hashtab->entries[i]; e; e = e->next) { + if (!strcmp((char *) key, (char *) e->key)) { + return e->u.hyppat; + } + } + return NULL; +} + +/*tex Get the state number, allocating a new state if necessary. */ + +static int hnj_get_state(HyphenDict * dict, const unsigned char *str, int *state_num) +{ + *state_num = state_lookup(dict->state_num, str); + if (*state_num >= 0) + return *state_num; + state_insert(dict->state_num, hnj_strdup(str), dict->num_states); + /*tex The predicate is true if |dict->num_states| is a power of two: */ + if (!(dict->num_states & (dict->num_states - 1))) { + dict->states = hnj_realloc(dict->states, + (int) ((dict->num_states << 1) * (int) sizeof(HyphenState))); + } + dict->states[dict->num_states].match = NULL; + dict->states[dict->num_states].fallback_state = -1; + dict->states[dict->num_states].num_trans = 0; + dict->states[dict->num_states].trans = NULL; + return dict->num_states++; +} + +/*tex + + Add a transition from state1 to state2 through ch - assumes that the + transition does not already exist. + +*/ + +static void hnj_add_trans(HyphenDict * dict, int state1, int state2, int uni_ch) +{ + int num_trans; + /* + TH: this test was a bit too strict, it is quite normal for old patterns + to have chars in the range 0-31 or 127-159 (inclusive). To ease the + transition, let's only disallow |NUL| for now, which probably is a + requirement of the code anyway. + */ + if (uni_ch == 0) { + formatted_error("hyphenation","a character is out of bounds: u%04x", uni_ch); + } + num_trans = dict->states[state1].num_trans; + if (num_trans == 0) { + dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans)); + } else { + /* + TH: The old version did: + + \starttyping + } else if (!(num_trans & (num_trans - 1))) { + ... =hnj_realloc(dict->states[state1].trans, + (int) ((num_trans << 1) * sizeof(HyphenTrans))); + \stoptyping + + but that is incredibly nasty when adding patters one-at-a-time. + Controlled growth would be nicer than the current +1, but if no one + complains, and no one did in a decade, this is good enough. + + */ + dict->states[state1].trans = hnj_realloc(dict->states[state1].trans, + (int) ((num_trans + 1) * sizeof(HyphenTrans))); + } + dict->states[state1].trans[num_trans].uni_ch = uni_ch; + dict->states[state1].trans[num_trans].new_state = state2; + dict->states[state1].num_trans++; +} + +/*tex + + We did change the semantics a bit here: |hnj_hyphen_load| used to operate on + a file, but now the argument is a string buffer. + +*/ + +static const unsigned char *next_pattern(size_t * length, const unsigned char **buf) +{ + const unsigned char *here, *rover = *buf; + while (*rover && isspace(*rover)) + rover++; + here = rover; + while (*rover) { + if (isspace(*rover)) { + *length = (size_t) (rover - here); + *buf = rover; + return here; + } + rover++; + } + *length = (size_t) (rover - here); + *buf = rover; + /*tex Zero sensed: */ + return *length ? here : NULL; +} + +static void init_hash(HashTab ** h) +{ + int i; + if (*h) { + return; + } + *h = hnj_malloc(sizeof(HashTab)); + for (i = 0; i < HASH_SIZE; i++) { + (*h)->entries[i] = NULL; + } +} + +static void clear_state_hash(HashTab ** h) +{ + int i; + if (*h == NULL) { + return; + } + for (i = 0; i < HASH_SIZE; i++) { + HashEntry *e, *next; + for (e = (*h)->entries[i]; e; e = next) { + next = e->next; + hnj_free(e->key); + hnj_free(e); + } + } + hnj_free(*h); + *h = NULL; +} + +static void clear_hyppat_hash(HashTab ** h) +{ + int i; + if (*h == NULL) { + return; + } + for (i = 0; i < HASH_SIZE; i++) { + HashEntry *e, *next; + for (e = (*h)->entries[i]; e; e = next) { + next = e->next; + hnj_free(e->key); + if (e->u.hyppat) { + hnj_free(e->u.hyppat); + } + hnj_free(e); + } + } + hnj_free(*h); + *h = NULL; +} + +static void init_dict(HyphenDict * dict) +{ + dict->num_states = 1; + dict->pat_length = 0; + dict->states = hnj_malloc(sizeof(HyphenState)); + dict->states[0].match = NULL; + dict->states[0].fallback_state = -1; + dict->states[0].num_trans = 0; + dict->states[0].trans = NULL; + dict->patterns = NULL; + dict->merged = NULL; + dict->state_num = NULL; + init_hash(&dict->patterns); +} + +static void clear_dict(HyphenDict * dict) +{ + int state_num; + for (state_num = 0; state_num < dict->num_states; state_num++) { + HyphenState *hstate = &dict->states[state_num]; + if (hstate->match) + hnj_free(hstate->match); + if (hstate->trans) + hnj_free(hstate->trans); + } + hnj_free(dict->states); + clear_hyppat_hash(&dict->patterns); + clear_hyppat_hash(&dict->merged); + clear_state_hash(&dict->state_num); +} + +HyphenDict *hnj_hyphen_new(void) +{ + HyphenDict *dict = hnj_malloc(sizeof(HyphenDict)); + init_dict(dict); + return dict; +} + +void hnj_hyphen_clear(HyphenDict * dict) +{ + clear_dict(dict); + init_dict(dict); +} + +void hnj_hyphen_free(HyphenDict * dict) +{ + clear_dict(dict); + hnj_free(dict); +} + +unsigned char *hnj_serialize(HyphenDict * dict) +{ + HashIter *v; + unsigned char *word; + char *pattern; + unsigned char *buf = hnj_malloc(dict->pat_length); + unsigned char *cur = buf; + v = new_HashIter(dict->patterns); + while (eachHash(v, &word, &pattern)) { + int i = 0, e = 0; + while (word[e + i]) { + if (pattern[i] != '0') + *cur++ = (unsigned char) pattern[i]; + *cur++ = word[e + i++]; + while (is_utf8_follow(word[e + i])) + *cur++ = word[i + e++]; + } + if (pattern[i] != '0') + *cur++ = (unsigned char) pattern[i]; + *cur++ = ' '; + } + delete_HashIter(v); + *cur = 0; + return buf; +} + +void hnj_free_serialize(unsigned char *c) +{ + hnj_free(c); +} + +/*tex + + In hyphenation patterns we use signed bytes where |0|, or actually any + negative number, indicates end: + + \starttyping + prio(1+),startpos,length,len1,[replace],len2,[replace] + \starttyping + + A basic example is: + + \starttyping + p n 0 0 0 + \starttyping + + for a hyphenation point between characters. + +*/ + +void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) +{ + int state_num, last_state; + int ch; + int found; + HashEntry *e; + HashIter *v; + unsigned char *word; + char *pattern; + size_t l = 0; + const unsigned char *format; + const unsigned char *begin = f; + unsigned char *pat; + char *org; + while ((format = next_pattern(&l, &f)) != NULL) { + int i, j, e1; + if (l>=255) { + normal_warning("hyphenation","a pattern of more than 254 bytes ignored"); + continue; + } + for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) { + if (format[i] >= '0' && format[i] <= '9') + j++; + if (is_utf8_follow(format[i])) + e1++; + } + /*tex + Here |l-e1| is the number of {\em characters} not {\em bytes}, |l-j| + the number of pattern bytes and |l-e1-j| the number of pattern + characters. + */ + pat = (unsigned char *) malloc((1 + l - (size_t) j)); + org = (char *) malloc((size_t) (2 + l - (size_t) e1 - (size_t) j)); + /*tex Remove hyphenation encoders (digits) from pat. */ + org[0] = '0'; + for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) { + unsigned char c = format[i]; + if (is_utf8_follow(c)) { + pat[j + e1++] = c; + } else if (c < '0' || c > '9') { + pat[e1 + j++] = c; + org[j] = '0'; + } else { + org[j] = (char) c; + } + } + pat[e1 + j] = 0; + org[j + 1] = 0; + hyppat_insert(dict->patterns, pat, org); + } + /*tex We add 2 bytes for spurious spaces. */ + dict->pat_length += (int) ((f - begin) + 2); + init_hash(&dict->merged); + v = new_HashIter(dict->patterns); + while (nextHash(v, &word)) { + int wordsize = (int) strlen((char *) word); + int j1, l1; + for (l1 = 1; l1 <= wordsize; l1++) { + if (is_utf8_follow(word[l1])) { + /*tex Do not clip an utf8 sequence. */ + continue; + } + for (j1 = 1; j1 <= l1; j1++) { + char *subpat_pat; + int i1 = l1 - j1; + if (is_utf8_follow(word[i1])) { + /*tex Do not start halfway an utf8 sequence. */ + continue; + } + if ((subpat_pat = + hyppat_lookup(dict->patterns, word + i1, j1)) != NULL) { + char *newpat_pat; + if ((newpat_pat = + hyppat_lookup(dict->merged, word, l1)) == NULL) { + char *neworg; + unsigned char *newword = + (unsigned char *) malloc((size_t) (l1 + 1)); + int e1 = 0; + strncpy((char *) newword, (char *) word, (size_t) l1); + newword[l1] = 0; + for (i1 = 0; i1 < l1; i1++) + if (is_utf8_follow(newword[i1])) + e1++; + neworg = malloc((size_t) (l1 + 2 - e1)); + /*tex Fill with right amount of zeros: */ + sprintf(neworg, "%0*d", l1 + 1 - e1, 0); + hyppat_insert(dict->merged, newword, combine(neworg, subpat_pat)); + } else { + combine(newpat_pat, subpat_pat); + } + } + } + } + } + delete_HashIter(v); + init_hash(&dict->state_num); + state_insert(dict->state_num, hnj_strdup((const unsigned char *) ""), 0); + v = new_HashIter(dict->merged); + while (nextHashStealPattern(v, &word, &pattern)) { + static unsigned char mask[] = { 0x3F, 0x1F, 0xF, 0x7 }; + int j1 = (int) strlen((char *) word); + state_num = hnj_get_state(dict, word, &found); + dict->states[state_num].match = pattern; + /*tex Now, put in the prefix transitions. */ + while (found < 0) { + j1--; + last_state = state_num; + ch = word[j1]; + if (ch >= 0x80) { + int m; + int i1 = 1; + while (is_utf8_follow(word[j1 - i1])) + i1++; + ch = word[j1 - i1] & mask[i1]; + m = j1 - i1; + while (i1--) { + ch = (ch << 6) + (0x3F & word[j1 - i1]); + } + j1 = m; + } + word[j1] = '\0'; + state_num = hnj_get_state(dict, word, &found); + hnj_add_trans(dict, state_num, last_state, ch); + } + } + delete_HashIter(v); + clear_hyppat_hash(&dict->merged); + /*tex Put in the fallback states. */ + { + int i, j = 0; + for (i = 0; i < HASH_SIZE; i++) { + for (e = dict->state_num->entries[i]; e; e = e->next) { + /*tex Do not do |state==0| otherwise things get confused. */ + if (e->u.state) { + for (j = 1; 1; j++) { + state_num = state_lookup(dict->state_num, e->key + j); + if (state_num >= 0) + break; + } + dict->states[e->u.state].fallback_state = state_num; + } + } + } + } + clear_state_hash(&dict->state_num); +} + +extern halfword insert_syllable_discretionary(halfword t, lang_variables * lan); + +void hnj_hyphen_hyphenate(HyphenDict * dict, halfword first1, halfword last1, + int length, halfword left, halfword right, lang_variables * lan) +{ + int char_num; + halfword here; + int state = 0; + /*tex +2 for dots at each end, +1 for points outside characters. */ + int ext_word_len = length + 2; + int hyphen_len = ext_word_len + 1; + char *hyphens = hnj_malloc(hyphen_len + 1); + /*tex Add a '.' to beginning and end to facilitate matching. */ + vlink(begin_point) = first1; + vlink(end_point) = vlink(last1); + vlink(last1) = end_point; + for (char_num = 0; char_num < hyphen_len; char_num++) { + hyphens[char_num] = '0'; + } + hyphens[hyphen_len] = 0; + /*tex Now, run the finite state machine. */ + for (char_num = 0, here = begin_point; here != vlink(end_point); here = vlink(here)) { + int ch; + if (here == begin_point || here == end_point) { + ch = '.'; + } else { + ch = get_hj_code(char_lang(here),character(here)); + if (ch <= 32) { + ch = character(here); + } + } + while (state != -1) { + HyphenState *hstate = &dict->states[state]; + int k; + for (k = 0; k < hstate->num_trans; k++) { + if (hstate->trans[k].uni_ch == ch) { + char *match; + state = hstate->trans[k].new_state; + match = dict->states[state].match; + if (match) { + /*tex + + We add +2 because 1 string length is one bigger than offset + and 1 hyphenation starts before first character. + */ + int offset = (int) (char_num + 2 - (int) strlen(match)); + int m; + for (m = 0; match[m]; m++) { + if (hyphens[offset + m] < match[m]) + hyphens[offset + m] = match[m]; + } + } + goto try_next_letter; + } + } + state = hstate->fallback_state; + } + /*tex Nothing worked, let's go to the next character. */ + state = 0; + try_next_letter:; + char_num++; + } + /*tex Restore the correct pointers. */ + vlink(last1) = vlink(end_point); + /*tex + + Pattern is \.{\^.\^w\^o\^r\^d\^.\^} and |word_len|=4, |ext_word_len|=6, + |hyphens|=7; check \.{ \^ \^ \^ } and therefore drop first two and stop + after |word_len-1| + */ + for (here = first1, char_num = 2; here != left; here = vlink(here)) + char_num++; + for (; here != right; here = vlink(here)) { + if (hyphens[char_num] & 1) + here = insert_syllable_discretionary(here, lan); + char_num++; + } + hnj_free(hyphens); +} diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.w b/Build/source/texk/web2c/luatexdir/lang/hyphen.w deleted file mode 100644 index 35befb743de..00000000000 --- a/Build/source/texk/web2c/luatexdir/lang/hyphen.w +++ /dev/null @@ -1,875 +0,0 @@ -% hyphen.w -% -% Libhnj is dual licensed under LGPL and MPL. Boilerplate for both -% licenses follows. -% -% -% LibHnj - a library for high quality hyphenation and justification -% Copyright (C) 1998 Raph Levien, -% (C) 2001 ALTLinux, Moscow (http://www.alt-linux.org), -% (C) 2001 Peter Novodvorsky (nidd@@cs.msu.su) -% -% This library is free software; you can redistribute it and/or -% modify it under the terms of the GNU Library General Public -% License as published by the Free Software Foundation; either -% version 2 of the License, or (at your option) any later version. -% -% This library is distributed in the hope that it will be useful, -% but WITHOUT ANY WARRANTY; without even the implied warranty of -% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU -% Library General Public License for more details. -% -% You should have received a copy of the GNU Library General Public -% License along with this library; if not, write to the -% Free Software Foundation, Inc., 59 Temple Place - Suite 330, -% Boston, MA 02111-1307 USA. -% -% -% -% The contents of this file are subject to the Mozilla Public License -% Version 1.0 (the "MPL"); you may not use this file except in -% compliance with the MPL. You may obtain a copy of the MPL at -% http://www.mozilla.org/MPL/ -% -% Software distributed under the MPL is distributed on an "AS IS" basis, -% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL -% for the specific language governing rights and limitations under the -% MPL. - - -@ @c - - -#include "ptexlib.h" -#include "lua/luatex-api.h" - -#include /* for NULL, malloc */ -#include /* for fprintf */ -#include /* for strdup */ -#include /* for malloc used by substring inclusion */ - -#define MAXPATHS 40960 - -#ifdef UNX -# include /* for exit */ -#endif - -#include - -#define noVERBOSE - -#include "lang/hnjalloc.h" - -@ TODO: should be moved to separate library - -@c -static unsigned char *hnj_strdup(const unsigned char *s) -{ - unsigned char *new; - size_t l; - - l = strlen((const char *) s); - new = hnj_malloc((int) l + 1); - memcpy(new, s, l); - new[l] = 0; - return new; -} - -@* Type definitions. - -@ a little bit of a hash table implementation. This simply maps strings - to state numbers - -@c -typedef struct _HashTab HashTab; -typedef struct _HashEntry HashEntry; -typedef struct _HashIter HashIter; -typedef union _HashVal HashVal; - -/* A cheap, but effective, hack. */ -#define HASH_SIZE 31627 - -struct _HashTab { - HashEntry *entries[HASH_SIZE]; -}; - -union _HashVal { - int state; - char *hyppat; -}; - -struct _HashEntry { - HashEntry *next; - unsigned char *key; - HashVal u; -}; - -struct _HashIter { - HashEntry **e; - HashEntry *cur; - int ndx; -}; - -@ State machine - -@c -typedef struct _HyphenState HyphenState; -typedef struct _HyphenTrans HyphenTrans; -#define MAX_CHARS 256 -#define MAX_NAME 20 - -struct _HyphenDict { - int num_states; - int pat_length; - char cset[MAX_NAME]; - HyphenState *states; - HashTab *patterns; - HashTab *merged; - HashTab *state_num; -}; - -struct _HyphenState { - char *match; - /*char *repl; */ - /*signed char replindex; */ - /*signed char replcut; */ - int fallback_state; - int num_trans; - HyphenTrans *trans; -}; - -struct _HyphenTrans { - int uni_ch; - int new_state; -}; - - -@ Combine two right-aligned number patterns, 04000 + 020 becomes 04020 - -@c -static char *combine(char *expr, const char *subexpr) -{ - size_t l1 = strlen(expr); - size_t l2 = strlen(subexpr); - size_t off = l1 - l2; - unsigned j; - /* this works also for utf8 sequences because the substring is identical - to the last substring-length bytes of expr except for the (single byte) - hyphenation encoders - */ - for (j = 0; j < l2; j++) { - if (expr[off + j] < subexpr[j]) - expr[off + j] = subexpr[j]; - } - return expr; -} - - -@ ORIGINAL CODE -@c -static HashIter *new_HashIter(HashTab * h) -{ - HashIter *i = hnj_malloc(sizeof(HashIter)); - i->e = h->entries; - i->cur = NULL; - i->ndx = -1; - return i; -} - - -static int nextHashStealPattern(HashIter * i, unsigned char **word, char **pattern) -{ - while (i->cur == NULL) { - if (i->ndx >= HASH_SIZE - 1) - return 0; - i->cur = i->e[++i->ndx]; - } - *word = i->cur->key; - *pattern = i->cur->u.hyppat; - i->cur->u.hyppat = NULL; - i->cur = i->cur->next; - return 1; -} - - -static int nextHash(HashIter * i, unsigned char **word) -{ - while (i->cur == NULL) { - if (i->ndx >= HASH_SIZE - 1) - return 0; - i->cur = i->e[++i->ndx]; - } - *word = i->cur->key; - i->cur = i->cur->next; - return 1; -} - - -static int eachHash(HashIter * i, unsigned char **word, char **pattern) -{ - while (i->cur == NULL) { - if (i->ndx >= HASH_SIZE - 1) - return 0; - i->cur = i->e[++i->ndx]; - } - *word = i->cur->key; - *pattern = i->cur->u.hyppat; - i->cur = i->cur->next; - return 1; -} - - -static void delete_HashIter(HashIter * i) -{ - hnj_free(i); -} - - -@ a |char*| hash function from ASU - adapted from Gtk+ - -@c -static unsigned int hnj_string_hash(const unsigned char *s) -{ - const unsigned char *p; - unsigned int h = 0, g; - - for (p = s; *p != '\0'; p += 1) { - h = (h << 4) + *p; - if ((g = (h & 0xf0000000))) { - h = h ^ (g >> 24); - h = h ^ g; - } - } - return h /* \% M */ ; -} - - -@ assumes that key is not already present! - -@c -static void state_insert(HashTab * hashtab, unsigned char *key, int state) -{ - int i; - HashEntry *e; - - i = (int) (hnj_string_hash(key) % HASH_SIZE); - e = hnj_malloc(sizeof(HashEntry)); - e->next = hashtab->entries[i]; - e->key = key; - e->u.state = state; - hashtab->entries[i] = e; -} - - -@ assumes that key is not already present! - -@c -static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat) -{ - int i; - HashEntry *e; - - i = (int) (hnj_string_hash(key) % HASH_SIZE); - for (e = hashtab->entries[i]; e; e = e->next) { - if (strcmp((char *) e->key, (char *) key) == 0) { - if (e->u.hyppat) { - if (hyppat - && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) { - print_err("Conflicting pattern ignored"); - error(); - } - hnj_free(e->u.hyppat); - } - e->u.hyppat = hyppat; - hnj_free(key); - return; - } - } - e = hnj_malloc(sizeof(HashEntry)); - e->next = hashtab->entries[i]; - e->key = key; - e->u.hyppat = hyppat; - hashtab->entries[i] = e; -} - - -@ return state if found, otherwise $-1$ - -@c -static int state_lookup(HashTab * hashtab, const unsigned char *key) -{ - int i; - HashEntry *e; - - i = (int) (hnj_string_hash(key) % HASH_SIZE); - for (e = hashtab->entries[i]; e; e = e->next) { - if (!strcmp((const char *) key, (const char *) e->key)) { - return e->u.state; - } - } - return -1; -} - - -@ return state if found, otherwise $-1$ - -@c -static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l) -{ - int i; - HashEntry *e; - unsigned char key[256]; /* should be ample */ - strncpy((char *) key, (const char *) chars, (size_t) l); - key[l] = 0; - i = (int) (hnj_string_hash(key) % HASH_SIZE); - for (e = hashtab->entries[i]; e; e = e->next) { - if (!strcmp((char *) key, (char *) e->key)) { - return e->u.hyppat; - } - } - return NULL; -} - - -@ Get the state number, allocating a new state if necessary. - -@c -static int hnj_get_state(HyphenDict * dict, - const unsigned char *str, int *state_num) -{ - *state_num = state_lookup(dict->state_num, str); - - if (*state_num >= 0) - return *state_num; - - state_insert(dict->state_num, hnj_strdup(str), dict->num_states); - /* predicate is true if |dict->num_states| is a power of two */ - if (!(dict->num_states & (dict->num_states - 1))) { - dict->states = hnj_realloc(dict->states, - (int) ((dict->num_states << 1) * - (int) sizeof(HyphenState))); - } - dict->states[dict->num_states].match = NULL; - dict->states[dict->num_states].fallback_state = -1; - dict->states[dict->num_states].num_trans = 0; - dict->states[dict->num_states].trans = NULL; - return dict->num_states++; -} - - -@ Add a transition from state1 to state2 through ch - assumes that the - transition does not already exist - -@c -static void hnj_add_trans(HyphenDict * dict, int state1, int state2, int uni_ch) -{ - int num_trans; - /* TH: this test was a bit too strict, it is quite normal for old - patterns to have chars in the range 0-31 or 127-159 (inclusive). - To ease the transition, let's only disallow NUL for now - (this is probably a requirement of the code anyway). - */ - if (uni_ch == 0) { - char errmsg[256]; /* temp hack ... we will have a formatted error */ - snprintf(errmsg, 255, "character out of bounds: u%04x", uni_ch); - errmsg[255] = '\0'; - normal_error("hyphenation",errmsg); /* todo */ - } - num_trans = dict->states[state1].num_trans; - if (num_trans == 0) { - dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans)); - } else { - /* TH: The old version did - } else if (!(num_trans & (num_trans - 1))) { - ... hnj_realloc(dict->states[state1].trans, - (int) ((num_trans << 1) * - sizeof(HyphenTrans))); - but that is incredibly nasty when adding patters one-at-a-time. - Controlled growth would be nicer than the current +1, but if - noone complains, this is good enough ;) - */ - dict->states[state1].trans = hnj_realloc(dict->states[state1].trans, - (int) ((num_trans + 1) * - sizeof(HyphenTrans))); - } - dict->states[state1].trans[num_trans].uni_ch = uni_ch; - dict->states[state1].trans[num_trans].new_state = state2; - dict->states[state1].num_trans++; -} - - -#ifdef VERBOSE - -static unsigned char *get_state_str(int state) -{ - int i; - HashEntry *e; - - for (i = 0; i < HASH_SIZE; i++) - for (e = global->entries[i]; e; e = e->next) - if (e->u.state == state) - return e->key; - return NULL; -} -#endif - - -@ I've changed the semantics a bit here: |hnj_hyphen_load| used to - operate on a file, but now the argument is a string buffer. - -@c -static const unsigned char *next_pattern(size_t * length, - const unsigned char **buf) -{ - const unsigned char *here, *rover = *buf; - while (*rover && isspace(*rover)) - rover++; - here = rover; - while (*rover) { - if (isspace(*rover)) { - *length = (size_t) (rover - here); - *buf = rover; - return here; - } - rover++; - } - *length = (size_t) (rover - here); - *buf = rover; - return *length ? here : NULL; /* zero sensed */ -} - -static void init_hash(HashTab ** h) -{ - int i; - if (*h) - return; - *h = hnj_malloc(sizeof(HashTab)); - for (i = 0; i < HASH_SIZE; i++) - (*h)->entries[i] = NULL; -} - - -static void clear_state_hash(HashTab ** h) -{ - int i; - if (*h == NULL) - return; - for (i = 0; i < HASH_SIZE; i++) { - HashEntry *e, *next; - for (e = (*h)->entries[i]; e; e = next) { - next = e->next; - hnj_free(e->key); - hnj_free(e); - } - } - hnj_free(*h); - *h = NULL; -} - - -static void clear_hyppat_hash(HashTab ** h) -{ - int i; - if (*h == NULL) - return; - for (i = 0; i < HASH_SIZE; i++) { - HashEntry *e, *next; - for (e = (*h)->entries[i]; e; e = next) { - next = e->next; - hnj_free(e->key); - if (e->u.hyppat) - hnj_free(e->u.hyppat); - hnj_free(e); - } - } - hnj_free(*h); - *h = NULL; -} - - -static void init_dict(HyphenDict * dict) -{ - dict->num_states = 1; - dict->pat_length = 0; - dict->states = hnj_malloc(sizeof(HyphenState)); - dict->states[0].match = NULL; - dict->states[0].fallback_state = -1; - dict->states[0].num_trans = 0; - dict->states[0].trans = NULL; - dict->patterns = NULL; - dict->merged = NULL; - dict->state_num = NULL; - init_hash(&dict->patterns); -} - - -static void clear_dict(HyphenDict * dict) -{ - int state_num; - for (state_num = 0; state_num < dict->num_states; state_num++) { - HyphenState *hstate = &dict->states[state_num]; - if (hstate->match) - hnj_free(hstate->match); - if (hstate->trans) - hnj_free(hstate->trans); - } - hnj_free(dict->states); - clear_hyppat_hash(&dict->patterns); - clear_hyppat_hash(&dict->merged); - clear_state_hash(&dict->state_num); -} - - - -HyphenDict *hnj_hyphen_new(void) -{ - HyphenDict *dict = hnj_malloc(sizeof(HyphenDict)); - init_dict(dict); - return dict; -} - - -void hnj_hyphen_clear(HyphenDict * dict) -{ - clear_dict(dict); - init_dict(dict); -} - - -void hnj_hyphen_free(HyphenDict * dict) -{ - clear_dict(dict); - hnj_free(dict); -} - -unsigned char *hnj_serialize(HyphenDict * dict) -{ - HashIter *v; - unsigned char *word; - char *pattern; - unsigned char *buf = hnj_malloc(dict->pat_length); - unsigned char *cur = buf; - v = new_HashIter(dict->patterns); - while (eachHash(v, &word, &pattern)) { - int i = 0, e = 0; - while (word[e + i]) { - if (pattern[i] != '0') - *cur++ = (unsigned char) pattern[i]; - *cur++ = word[e + i++]; - while (is_utf8_follow(word[e + i])) - *cur++ = word[i + e++]; - } - if (pattern[i] != '0') - *cur++ = (unsigned char) pattern[i]; - *cur++ = ' '; - } - delete_HashIter(v); - *cur = 0; - return buf; -} - - -void hnj_free_serialize(unsigned char *c) -{ - hnj_free(c); -} - - -@ hyphenation pattern: - -signed bytes - -0 indicates end (actually any negative number) - -: prio(1+),startpos,length,len1,[replace],len2,[replace] - -most basic example is: - -p n 0 0 0 - -for a hyphenation point between characters - - -@c -void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) -{ - int state_num, last_state; - int ch; - int found; - HashEntry *e; - HashIter *v; - unsigned char *word; - char *pattern; - size_t l = 0; - - const unsigned char *format; - const unsigned char *begin = f; - unsigned char *pat; - char *org; - while ((format = next_pattern(&l, &f)) != NULL) { - int i, j, e1; - if (l>=255) { - help1("Individual patterns should not be longer than 254 bytes total."); - print_err("Pattern of enormous length ignored"); - error(); - continue; - } -#if 0 - printf("%s\n",format); - char* repl = strnchr(format, '/',l); - int replindex = 0; - int replcut = 0; - if (repl) { - int clen = l-(repl-format); - l = repl-format; - char * index = strnchr(repl + 1, ',',clen); - if (index) { - char * index2 = strnchr(index + 1, ',',clen-(index-repl)); - if (index2) { - replindex = (signed char) atoi(index + 1) - 1; - replcut = (signed char) atoi(index2 + 1); - } - } else { - hnj_strchomp(repl + 1); - replindex = 0; - replcut = strlen(buf); - } - repl = hnj_strdup(repl + 1); - } -#endif - for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) { - if (format[i] >= '0' && format[i] <= '9') - j++; - if (is_utf8_follow(format[i])) - e1++; - } - /* |l-e1| => number of {\it characters} not {\it bytes} */ - /* |l-j| => number of pattern bytes */ - /* |l-e1-j| => number of pattern characters */ - pat = (unsigned char *) malloc((1 + l - (size_t) j)); - org = (char *) malloc((size_t) (2 + l - (size_t) e1 - (size_t) j)); - /* remove hyphenation encoders (digits) from pat */ - org[0] = '0'; - for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) { - unsigned char c = format[i]; - if (is_utf8_follow(c)) { - pat[j + e1++] = c; - } else if (c < '0' || c > '9') { - pat[e1 + j++] = c; - org[j] = '0'; - } else { - org[j] = (char) c; - } - } - pat[e1 + j] = 0; - org[j + 1] = 0; - hyppat_insert(dict->patterns, pat, org); - } - dict->pat_length += (int) ((f - begin) + 2); /* 2 for spurious spaces */ - init_hash(&dict->merged); - v = new_HashIter(dict->patterns); - while (nextHash(v, &word)) { - int wordsize = (int) strlen((char *) word); - int j1, l1; - for (l1 = 1; l1 <= wordsize; l1++) { - if (is_utf8_follow(word[l1])) - continue; /* Do not clip an utf8 sequence */ - for (j1 = 1; j1 <= l1; j1++) { - char *subpat_pat; - int i1 = l1 - j1; - if (is_utf8_follow(word[i1])) - continue; /* Do not start halfway an utf8 sequence */ - if ((subpat_pat = - hyppat_lookup(dict->patterns, word + i1, j1)) != NULL) { - char *newpat_pat; - if ((newpat_pat = - hyppat_lookup(dict->merged, word, l1)) == NULL) { - char *neworg; - unsigned char *newword = - (unsigned char *) malloc((size_t) (l1 + 1)); - int e1 = 0; - strncpy((char *) newword, (char *) word, (size_t) l1); - newword[l1] = 0; - for (i1 = 0; i1 < l1; i1++) - if (is_utf8_follow(newword[i1])) - e1++; - neworg = malloc((size_t) (l1 + 2 - e1)); - sprintf(neworg, "%0*d", l1 + 1 - e1, 0); /* fill with right amount of '0' */ - hyppat_insert(dict->merged, newword, combine(neworg, subpat_pat)); - } else { - combine(newpat_pat, subpat_pat); - } - } - } - } - } - delete_HashIter(v); - - init_hash(&dict->state_num); - state_insert(dict->state_num, hnj_strdup((const unsigned char *) ""), 0); - v = new_HashIter(dict->merged); - while (nextHashStealPattern(v, &word, &pattern)) { - static unsigned char mask[] = { 0x3F, 0x1F, 0xF, 0x7 }; - int j1 = (int) strlen((char *) word); -#ifdef VERBOSE - printf("word %s pattern %s, j = %d\n", word, pattern, j1); -#endif - state_num = hnj_get_state(dict, word, &found); - dict->states[state_num].match = pattern; - - /* now, put in the prefix transitions */ - while (found < 0) { - j1--; - last_state = state_num; - ch = word[j1]; - if (ch >= 0x80) { - int m; - int i1 = 1; - while (is_utf8_follow(word[j1 - i1])) - i1++; - ch = word[j1 - i1] & mask[i1]; - m = j1 - i1; - while (i1--) { - ch = (ch << 6) + (0x3F & word[j1 - i1]); - } - j1 = m; - } - word[j1] = '\0'; - state_num = hnj_get_state(dict, word, &found); - hnj_add_trans(dict, state_num, last_state, ch); - } - } - delete_HashIter(v); - clear_hyppat_hash(&dict->merged); - - /* put in the fallback states */ - { - int i, j = 0; - for (i = 0; i < HASH_SIZE; i++) { - for (e = dict->state_num->entries[i]; e; e = e->next) { - /* do not do state==0 otherwise things get confused */ - if (e->u.state) { - for (j = 1; 1; j++) { - state_num = state_lookup(dict->state_num, e->key + j); - if (state_num >= 0) - break; - } - dict->states[e->u.state].fallback_state = state_num; - } - } - } -#ifdef VERBOSE - for (i = 0; i < HASH_SIZE; i++) { - for (e = dict->state_num->entries[i]; e; e = e->next) { - printf("%d string %s state %d, fallback=%d\n", - i, e->key, e->u.state, dict->states[e->u.state].fallback_state); - for (j = 0; j < dict->states[e->u.state].num_trans; j++) { - printf(" u%4x->%d\n", - (int) dict->states[e->u.state].trans[j].uni_ch, - dict->states[e->u.state].trans[j].new_state); - } - } - } -#endif - } - clear_state_hash(&dict->state_num); -} - -@ @c -extern halfword insert_syllable_discretionary(halfword t, lang_variables * lan); - -void hnj_hyphen_hyphenate(HyphenDict * dict, - halfword first1, - halfword last1, - int length, - halfword left, halfword right, lang_variables * lan) -{ - int char_num; - halfword here; - int state = 0; - /* +2 for dots at each end, +1 for points /outside/ characters */ - int ext_word_len = length + 2; - int hyphen_len = ext_word_len + 1; - char *hyphens = hnj_malloc(hyphen_len + 1); - - /* Add a '.' to beginning and end to facilitate matching */ - vlink(begin_point) = first1; - vlink(end_point) = vlink(last1); - vlink(last1) = end_point; - - for (char_num = 0; char_num < hyphen_len; char_num++) { - hyphens[char_num] = '0'; - } - hyphens[hyphen_len] = 0; - - /* now, run the finite state machine */ - for (char_num = 0, here = begin_point; here != vlink(end_point); - here = vlink(here)) { - - int ch; - if (here == begin_point || here == end_point) { - ch = '.'; - } else { - ch = get_hj_code(char_lang(here),character(here)); - if (ch <= 32) { - ch = character(here); - } - } - while (state != -1) { -#if 0 - printf("%*s%s%c",char_num-strlen(get_state_str(state)),"",get_state_str(state),(char)ch); -#endif - HyphenState *hstate = &dict->states[state]; - int k; - for (k = 0; k < hstate->num_trans; k++) { - if (hstate->trans[k].uni_ch == ch) { - char *match; - state = hstate->trans[k].new_state; -#if 0 - printf(" state %d\n",state); -#endif - match = dict->states[state].match; - if (match) { - /* +2 because: - 1 string length is one bigger than offset - 1 hyphenation starts before first character - */ - int offset = (int) (char_num + 2 - (int) strlen(match)); -#if 0 - printf("%*s%s\n", offset,"", match); -#endif - int m; - for (m = 0; match[m]; m++) { - if (hyphens[offset + m] < match[m]) - hyphens[offset + m] = match[m]; - } - } - goto try_next_letter; - } - } - state = hstate->fallback_state; -#if 0 - printf(" back to %d\n", state); -#endif - } - /* nothing worked, let's go to the next character */ - state = 0; - try_next_letter:; - char_num++; - } - - /* restore the correct pointers */ - vlink(last1) = vlink(end_point); - - /* pattern is \.{\^.\^w\^o\^r\^d\^.\^} |word_len|=4, |ext_word_len|=6, |hyphens|=7 - * check \.{ \^ \^ \^ } so drop first two and stop after |word_len-1| - */ - for (here = first1, char_num = 2; here != left; here = vlink(here)) - char_num++; - for (; here != right; here = vlink(here)) { - if (hyphens[char_num] & 1) - here = insert_syllable_discretionary(here, lan); - char_num++; - } - hnj_free(hyphens); -} diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.c b/Build/source/texk/web2c/luatexdir/lang/texlang.c new file mode 100644 index 00000000000..04f96660f5e --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/texlang.c @@ -0,0 +1,1359 @@ +/* + +texlang.w + +Copyright 2006-2012 Taco Hoekwater + +This file is part of LuaTeX. + +LuaTeX is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free +Software Foundation; either version 2 of the License, or (at your +option) any later version. + +LuaTeX is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +License for more details. + +You should have received a copy of the GNU General Public License along +with LuaTeX; if not, see . + +*/ + +#include "ptexlib.h" +#include +#include "lua/luatex-api.h" + +/*tex Low-level helpers */ + +#define unVERBOSE + +#define MAX_TEX_LANGUAGES 16384 + +static struct tex_language *tex_languages[MAX_TEX_LANGUAGES] = { NULL }; + +static int next_lang_id = 0; + +struct tex_language *new_language(int n) +{ + struct tex_language *lang; + unsigned l; + if (n >= 0) { + l = (unsigned) n; + if (l != (MAX_TEX_LANGUAGES - 1)) + if (next_lang_id <= n) + next_lang_id = n + 1; + } else { + while (tex_languages[next_lang_id] != NULL) + next_lang_id++; + l = (unsigned) next_lang_id++; + } + if (l < (MAX_TEX_LANGUAGES - 1) && tex_languages[l] == NULL) { + lang = xmalloc(sizeof(struct tex_language)); + tex_languages[l] = lang; + lang->id = (int) l; + lang->exceptions = 0; + lang->patterns = NULL; + lang->pre_hyphen_char = '-'; + lang->post_hyphen_char = 0; + lang->pre_exhyphen_char = 0; + lang->post_exhyphen_char = 0; + lang->hyphenation_min = -1; + if (saving_hyph_codes_par) { + /*tex + For now, we might just use specific value for whatever task. + */ + hj_codes_from_lc_codes(l); + } + return lang; + } else { + return NULL; + } +} + +struct tex_language *get_language(int n) +{ + if (n >= 0 && n < MAX_TEX_LANGUAGES) { + if (tex_languages[n] != NULL) { + return tex_languages[n]; + } else { + return new_language(n); + } + } else { + return NULL; + } +} + +void set_pre_hyphen_char(int n, int v) +{ + struct tex_language *l = get_language((int) n); + if (l != NULL) + l->pre_hyphen_char = (int) v; +} + +void set_post_hyphen_char(int n, int v) +{ + struct tex_language *l = get_language((int) n); + if (l != NULL) + l->post_hyphen_char = (int) v; +} + +void set_pre_exhyphen_char(int n, int v) +{ + struct tex_language *l = get_language((int) n); + if (l != NULL) + l->pre_exhyphen_char = (int) v; +} + +void set_post_exhyphen_char(int n, int v) +{ + struct tex_language *l = get_language((int) n); + if (l != NULL) + l->post_exhyphen_char = (int) v; +} + +int get_pre_hyphen_char(int n) +{ + struct tex_language *l = get_language((int) n); + if (l == NULL) + return -1; + return (int) l->pre_hyphen_char; +} + +int get_post_hyphen_char(int n) +{ + struct tex_language *l = get_language((int) n); + if (l == NULL) + return -1; + return (int) l->post_hyphen_char; +} + +int get_pre_exhyphen_char(int n) +{ + struct tex_language *l = get_language((int) n); + if (l == NULL) + return -1; + return (int) l->pre_exhyphen_char; +} + +int get_post_exhyphen_char(int n) +{ + struct tex_language *l = get_language((int) n); + if (l == NULL) + return -1; + return (int) l->post_exhyphen_char; +} + +void set_hyphenation_min(int n, int v) +{ + struct tex_language *l = get_language((int) n); + if (l != NULL) + l->hyphenation_min = (int) v; +} + +int get_hyphenation_min(int n) +{ + struct tex_language *l = get_language((int) n); + if (l == NULL) + return -1; + return (int) l->hyphenation_min; +} + +void load_patterns(struct tex_language *lang, const unsigned char *buff) +{ + if (lang == NULL || buff == NULL || strlen((const char *) buff) == 0) + return; + if (lang->patterns == NULL) { + lang->patterns = hnj_hyphen_new(); + } + hnj_hyphen_load(lang->patterns, buff); +} + +void clear_patterns(struct tex_language *lang) +{ + if (lang == NULL) + return; + if (lang->patterns != NULL) { + hnj_hyphen_clear(lang->patterns); + } +} + +void load_tex_patterns(int curlang, halfword head) +{ + char *s = tokenlist_to_cstring(head, 1, NULL); + load_patterns(get_language(curlang), (unsigned char *) s); +} + +#define STORE_CHAR(l,x) do { \ + unsigned xx = get_hj_code(l,x); \ + if (!xx || xx <= 32) { \ + xx = x; \ + } \ + uindex = uni2string(uindex, xx); \ +} while (0) + +/* + + This cleans one word which is returned in |cleaned|, returns the new offset + into |buffer|. + +*/ + +const char *clean_hyphenation(int id, const char *buff, char **cleaned) +{ + int items = 0; + /*tex Work buffer for bytes: */ + unsigned char word[MAX_WORD_LEN + 1]; + /*tex Work buffer for \UNICODE: */ + unsigned uword[MAX_WORD_LEN + 1] = { 0 }; + /*tex The \UNICODE\ buffer value: */ + int u = 0; + /*tex The index into buffer: */ + int i = 0; + char *uindex = (char *)word; + const char *s = buff; + + while (*s && !isspace((unsigned char)*s)) { + word[i++] = (unsigned)*s; + s++; + if ((s-buff)>MAX_WORD_LEN) { + /*tex Todo: this is too strict, should count \UNICODE, not bytes. */ + *cleaned = NULL; + tex_error("exception too long", NULL); + return s; + } + } + /*tex Now convert the input to \UNICODE. */ + word[i] = '\0'; + utf2uni_strcpy(uword, (const char *)word); + /*tex Build the new word string. */ + i = 0; + while (uword[i]>0) { + u = uword[i++]; + if (u == '-') { + /*tex Skip. */ + } else if (u == '=') { + STORE_CHAR(id,'-'); + } else if (u == '{') { + u = uword[i++]; + items = 0; + while (u && u != '}') { + u = uword[i++]; + } + if (u == '}') { + items++; + u = uword[i++]; + } + while (u && u != '}') { + u = uword[i++]; + } + if (u == '}') { + items++; + u = uword[i++]; + } + if (u == '{') { + u = uword[i++]; + } + while (u && u != '}') { + STORE_CHAR(id,u); + u = uword[i++]; + } + if (u == '}') { + items++; + } + if (items != 3) { + *cleaned = NULL; + tex_error("exception syntax error", NULL); + return s; + } + if (uword[i] == '[' && uword[i+1] >= '0' && uword[i+1] <= '9' && uword[i+2] == ']') { + i += 3; + } + } else { + STORE_CHAR(id,u); + } + } + *uindex = '\0'; + *cleaned = xstrdup((char *) word); + return s; +} + +void load_hyphenation(struct tex_language *lang, const unsigned char *buff) +{ + const char *s; + const char *value; + char *cleaned; + int id ; + if (lang == NULL) + return; + if (lang->exceptions == 0) { + lua_newtable(Luas); + lang->exceptions = luaL_ref(Luas, LUA_REGISTRYINDEX); + } + lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions); + s = (const char *) buff; + id = lang->id; + while (*s) { + while (isspace((unsigned char)*s)) + s++; + if (*s) { + value = s; + s = clean_hyphenation(id, s, &cleaned); + if (cleaned != NULL) { + if ((s - value) > 0) { + lua_pushstring(Luas, cleaned); + lua_pushlstring(Luas, value, (size_t) (s - value)); + lua_rawset(Luas, -3); + } + free(cleaned); + } else { +#ifdef VERBOSE + formatted_warning("hyphenation","skipping invalid hyphenation exception: %s", value); +#endif + } + } + } +} + +void clear_hyphenation(struct tex_language *lang) +{ + if (lang == NULL) + return; + if (lang->exceptions != 0) { + luaL_unref(Luas, LUA_REGISTRYINDEX, lang->exceptions); + lang->exceptions = 0; + } +} + +void load_tex_hyphenation(int curlang, halfword head) +{ + char *s = tokenlist_to_cstring(head, 1, NULL); + load_hyphenation(get_language(curlang), (unsigned char *) s); +} + +static halfword insert_discretionary(halfword t, halfword pre, halfword post, halfword replace, int penalty) +{ + halfword g; + int f; + halfword d = new_node(disc_node, syllable_disc); + int attr = node_attr(t) ; + disc_penalty(d) = penalty; + if (t == replace) { + /*tex We have |prev disc next-next|. */ + try_couple_nodes(d, vlink(t)); + try_couple_nodes(alink(t), d); + alink(t) = null; + vlink(t) = null; + replace = t ; + } else { + /*tex We have |prev disc next|. */ + try_couple_nodes(d, vlink(t)); + couple_nodes(t, d); + } + if (replace != null) { + f = font(replace); + } else { + /*tex For compound words following explicit hyphens. */ + f = get_cur_font(); + } + for (g = pre; g != null; g = vlink(g)) { + font(g) = f; + if (attr != null) { + delete_attribute_ref(node_attr(g)); + node_attr(g) = attr; + attr_list_ref(attr) += 1; + } + } + for (g = post; g != null; g = vlink(g)) { + font(g) = f; + if (attr != null) { + delete_attribute_ref(node_attr(g)); + node_attr(g) = attr; + attr_list_ref(attr) += 1; + } + } + if (attr != null) { + for (g = replace; g != null; g = vlink(g)) { + delete_attribute_ref(node_attr(g)); + node_attr(g) = attr; + attr_list_ref(attr) += 1; + } + delete_attribute_ref(node_attr(d)); + node_attr(d) = attr; + attr_list_ref(attr) += 1; + } + set_disc_field(pre_break(d), pre); + set_disc_field(post_break(d), post); + set_disc_field(no_break(d), replace); + return d; +} + +halfword insert_syllable_discretionary(halfword t, lang_variables * lan) +{ + halfword g, n; + n = new_node(disc_node, syllable_disc); + disc_penalty(n) = hyphen_penalty_par; + couple_nodes(n, vlink(t)); + couple_nodes(t, n); + delete_attribute_ref(node_attr(n)); + if (node_attr(t) != null) { + node_attr(n) = node_attr(t); + attr_list_ref(node_attr(t))++; + } else { + node_attr(n) = null; + } + if (lan->pre_hyphen_char > 0) { + g = raw_glyph_node(); + set_to_character(g); + character(g) = lan->pre_hyphen_char; + font(g) = font(t); + lang_data(g) = lang_data(t); + if (node_attr(t) != null) { + node_attr(g) = node_attr(t); + attr_list_ref(node_attr(t))++; + } + set_disc_field(pre_break(n), g); + } + if (lan->post_hyphen_char > 0) { + t = vlink(n); + g = raw_glyph_node(); + set_to_character(g); + character(g) = lan->post_hyphen_char; + font(g) = font(t); + lang_data(g) = lang_data(t); + if (node_attr(t) != null) { + node_attr(g) = node_attr(t); + attr_list_ref(node_attr(t)) += 1; + } + set_disc_field(post_break(n), g); + } + return n; +} + +static halfword insert_character(halfword t, int c) +{ + halfword p; + p = new_node(glyph_node, 0); + set_to_character(p); + character(p) = c; + if (t != null) { + couple_nodes(t, p); + } + return p; +} + +static halfword compound_word_break(halfword t, int clang) +{ + halfword disc = null; + halfword pre = null; + halfword pos = null; + halfword pre_exhyphen_char = get_pre_exhyphen_char(clang); + halfword post_exhyphen_char = get_post_exhyphen_char(clang); + if (pre_exhyphen_char > 0) { + pre = insert_character(null,pre_exhyphen_char); + } else { + pre = insert_character(null,ex_hyphen_char_par); + } + if (post_exhyphen_char > 0) + pos = insert_character(null,post_exhyphen_char); + disc = insert_discretionary(t,pre,pos,t,ex_hyphen_penalty_par); + subtype(disc) = automatic_disc; + set_automatic_disc_penalty(disc); + return disc; +} + +void set_disc_field(halfword f, halfword t) +{ + if (t != null) { + /*tex + No |couple_nodes(f, t);| as we can better not expose |f| as |prev| + pointer. + */ + vlink(f) = t ; + alink(t) = null ; + tlink(f) = tail_of_list(t); + } else { + vlink(f) = null; + tlink(f) = null; + } +} + +static char *hyphenation_exception(int exceptions, char *w) +{ + char *ret = NULL; + lua_checkstack(Luas, 2); + lua_rawgeti(Luas, LUA_REGISTRYINDEX, exceptions); + if (lua_istable(Luas, -1)) { + /*tex Word table: */ + lua_pushstring(Luas, w); + lua_rawget(Luas, -2); + if (lua_type(Luas, -1) == LUA_TSTRING) { + ret = xstrdup(lua_tostring(Luas, -1)); + } + lua_pop(Luas, 2); + } else { + lua_pop(Luas, 1); + } + return ret; +} + +char *exception_strings(struct tex_language *lang) +{ + const char *value; + size_t size = 0, current = 0; + size_t l = 0; + char *ret = NULL; + if (lang->exceptions == 0) + return NULL; + lua_checkstack(Luas, 2); + lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions); + if (lua_istable(Luas, -1)) { + /*tex Iterate and join. */ + lua_pushnil(Luas); + while (lua_next(Luas, -2) != 0) { + value = lua_tolstring(Luas, -1, &l); + if (current + 2 + l > size) { + ret = xrealloc(ret, (unsigned) ((size + size / 5) + current + l + 1024)); + size = (size + size / 5) + current + l + 1024; + } + *(ret + current) = ' '; + strcpy(ret + current + 1, value); + current += l + 1; + lua_pop(Luas, 1); + } + } + return ret; +} + +/*tex + + The sequence from |wordstart| to |r| can contain only normal characters it + could be faster to modify a halfword pointer and return an integer + +*/ + +static halfword find_exception_part(unsigned int *j, unsigned int *uword, int len) +{ + halfword g = null, gg = null; + register unsigned i = *j; + /*tex This puts uword[i] on the |{|. */ + i++; + while (i < (unsigned) len && uword[i + 1] != '}') { + if (g == null) { + gg = new_char(0, (int) uword[i + 1]); + g = gg; + } else { + halfword s = new_char(0, (int) uword[i + 1]); + couple_nodes(g, s); + g = vlink(g); + } + i++; + } + *j = ++i; + return gg; +} + +static int count_exception_part(unsigned int *j, unsigned int *uword, int len) +{ + int ret = 0; + register unsigned i = *j; + /*tex This puts uword[i] on the |{|. */ + i++; + while (i < (unsigned) len && uword[i + 1] != '}') { + ret++; + i++; + } + *j = ++i; + return ret; +} + +static const char *PAT_ERROR[] = { + "Exception discretionaries should contain three pairs of braced items.", + "No intervening spaces are allowed.", + NULL +}; + +/*tex + + The exceptions are taken as-is: no min values are taken into account. One can + add normal patterns on-the-fly if needed. + +*/ + +static void do_exception(halfword wordstart, halfword r, char *replacement) +{ + unsigned i; + halfword t, pen; + unsigned len; + int clang; + lang_variables langdata; + unsigned uword[MAX_WORD_LEN + 1] = { 0 }; + utf2uni_strcpy(uword, replacement); + len = u_length(uword); + i = 0; + t = wordstart; + clang = char_lang(wordstart); + langdata.pre_hyphen_char = get_pre_hyphen_char(clang); + langdata.post_hyphen_char = get_post_hyphen_char(clang); + for (i = 0; i < len; i++) { + if (uword[i + 1] == 0 ) { + /*tex We ran out of the exception pattern. */ + break; + } else if (uword[i + 1] == '-') { + /*tex A hyphen follows. */ + if (vlink(t) == r) + break; + insert_syllable_discretionary(t, &langdata); + t = vlink(t); /* skip the new disc */ + } else if (uword[i + 1] == '=') { + /*tex We skip a disc. */ + t = vlink(t); + } else if (uword[i + 1] == '{') { + /*tex We ran into an exception |{}{}{}| or |{}{}{}[]|. */ + halfword gg, hh, replace = null; + int repl; + /*tex |pre| */ + gg = find_exception_part(&i, uword, (int) len); + if (i == len || uword[i + 1] != '{') { + tex_error("broken pattern 1", PAT_ERROR); + } + /*tex |post| */ + hh = find_exception_part(&i, uword, (int) len); + if (i == len || uword[i + 1] != '{') { + tex_error("broken pattern 2", PAT_ERROR); + } + /*tex |replace| */ + repl = count_exception_part(&i, uword, (int) len); + if (i == len) { + tex_error("broken pattern 3", PAT_ERROR); + } + /*tex Play safe. */ + if (vlink(t) == r) + break; + /*tex Let's deal with an (optional) replacement. */ + if (repl > 0) { + /*tex Assemble the replace stream. */ + halfword q = t; + replace = vlink(q); + while (repl > 0 && q != null) { + q = vlink(q); + if (type(q) == glyph_node || type(q) == disc_node) { + repl--; + } else { + break ; + } + } + /*tex Remove it from the main stream */ + try_couple_nodes(t, vlink(q)); + /*tex and finish it in the replace. */ + vlink(q) = null; + /*tex Sanitize the replace stream (we could use the flattener instead). */ + q = replace ; + while (q != null) { + halfword n = vlink(q); + if (type(q) == disc_node) { + /*tex Beware: the replacement starts after the no_break pointer. */ + halfword r = vlink(no_break(q)); + vlink(no_break(q)) = null; + alink(r) = null ; + /*tex Insert the replacement glyph. */ + if (q == replace) { + replace = r; + } else { + try_couple_nodes(alink(q),r); + } + /*tex Append the glyph (one). */ + try_couple_nodes(r,n); + /*tex Flush the disc. */ + flush_node(q); + } + q = n ; + } + } + /*tex Let's check if we have a penalty spec. */ + if (((i+3) < len) && uword[i+1] == '[' && uword[i+2] >= '0' && uword[i+2] <= '9' && uword[i+3] == ']') { + if (exception_penalty_par > 0) { + if (exception_penalty_par > 100000) { + pen = (uword[i+2] - '0') * exception_penalty_par ; + } else { + pen = exception_penalty_par; + } + } else { + pen = hyphen_penalty_par; + } + i += 3; + } else { + pen = hyphen_penalty_par; + } + /*tex And now we insert a disc node. */ + t = insert_discretionary(t, gg, hh, replace, pen); + /*tex We skip the new disc node. */ + t = vlink(t); + /*tex check if we have two exceptions in a row */ + if (uword[i + 1] == '{') { + i--; + } + } else { + t = vlink(t); + } + /*tex Again we play safe. */ + if (t == null || vlink(t) == r) { + break; + } + } +} + +/*tex + + This is a documentation section from the pascal web file. It is not true any + more, but I (Taco) do not have time right now to rewrite it. + + When the line-breaking routine is unable to find a feasible sequence of + breakpoints, it makes a second pass over the paragraph, attempting to + hyphenate the hyphenatable words. The goal of hyphenation is to insert + discretionary material into the paragraph so that there are more potential + places to break. + + The general rules for hyphenation are somewhat complex and technical, because + we want to be able to hyphenate words that are preceded or followed by + punctuation marks, and because we want the rules to work for languages other + than English. We also must contend with the fact that hyphens might radically + alter the ligature and kerning structure of a word. + + A sequence of characters will be considered for hyphenation only if it + belongs to a ``potentially hyphenatable part'' of the current paragraph. This + is a sequence of nodes $p_0p_1\ldots p_m$ where $p_0$ is a glue node, + $p_1\ldots p_{m-1}$ are either character or ligature or whatsit or implicit + kern nodes, and $p_m$ is a glue or penalty or insertion or adjust or mark or + whatsit or explicit kern node. (Therefore hyphenation is disabled by boxes, + math formulas, and discretionary nodes already inserted by the user.) The + ligature nodes among $p_1\ldots p_{m-1}$ are effectively expanded into the + original non-ligature characters; the kern nodes and whatsits are ignored. + Each character |c| is now classified as either a nonletter (if + |lc_code(c)=0|), a lowercase letter (if |lc_code(c)=c|), or an uppercase + letter (otherwise); an uppercase letter is treated as if it were |lc_code(c)| + for purposes of hyphenation. The characters generated by $p_1\ldots p_{m-1}$ + may begin with nonletters; let $c_1$ be the first letter that is not in the + middle of a ligature. Whatsit nodes preceding $c_1$ are ignored; a whatsit + found after $c_1$ will be the terminating node $p_m$. All characters that do + not have the same font as $c_1$ will be treated as nonletters. The + |hyphen_char| for that font must be between 0 and 255, otherwise hyphenation + will not be attempted. \TeX\ looks ahead for as many consecutive letters + $c_1\ldots c_n$ as possible; however, |n| must be less than 64, so a + character that would otherwise be $c_{64}$ is effectively not a letter. + Furthermore $c_n$ must not be in the middle of a ligature. In this way we + obtain a string of letters $c_1\ldots c_n$ that are generated by nodes + $p_a\ldots p_b$, where |1<=a<=b+1<=m|. If |n>=l_hyf+r_hyf|, this string + qualifies for hyphenation; however, |uc_hyph| must be positive, if $c_1$ is + uppercase. + + The hyphenation process takes place in three stages. First, the candidate + sequence $c_1\ldots c_n$ is found; then potential positions for hyphens are + determined by referring to hyphenation tables; and finally, the nodes + $p_a\ldots p_b$ are replaced by a new sequence of nodes that includes the + discretionary breaks found. + + Fortunately, we do not have to do all this calculation very often, because of + the way it has been taken out of \TeX's inner loop. For example, when the + second edition of the author's 700-page book {\sl Seminumerical Algorithms} + was typeset by \TeX, only about 1.2 hyphenations needed to be tried per + paragraph, since the line breaking algorithm needed to use two passes on only + about 5 per cent of the paragraphs. + + When a word been set up to contain a candidate for hyphenation, \TeX\ first + looks to see if it is in the user's exception dictionary. If not, hyphens are + inserted based on patterns that appear within the given word, using an + algorithm due to Frank~M. Liang. + + This is incompatible with \TEX\ because the first word of a paragraph can be + hyphenated, but most European users seem to agree that prohibiting + hyphenation there was not the best idea ever. + + We have some variants available that are controlled by the paremeter + \type {\hyphenationbounds}: + + \starttabulate + \NC \type {0} \NC not strict \NC \NR + \NC \type {1} \NC strict start \NC \NR + \NC \type {2} \NC strict end \NC \NR + \NC \type {3} \NC strict start and strict end \NC \NR + \stoptabulate + + \startbuffer + \parindent0pt \hsize=1.1cm + 12-34-56 \par + 12-34-\hbox{56} \par + 12-34-\vrule width 1em height 1.5ex \par + 12-\hbox{34}-56 \par + 12-\vrule width 1em height 1.5ex-56 \par + \hjcode`\1=`\1 \hjcode`\2=`\2 \hjcode`\3=`\3 \hjcode`\4=`\4 \vskip.5cm + 12-34-56 \par + 12-34-\hbox{56} \par + 12-34-\vrule width 1em height 1.5ex \par + 12-\hbox{34}-56 \par + 12-\vrule width 1em height 1.5ex-56 \par + \stopbuffer + + \typebuffer + + \startpacked \getbuffer \stopbuffer + + We only accept an explicit hyphen when there is a preceding glyph and we skip + a sequence of explicit hyphens as that normally indicates a \type {--} or + \type {---} ligature in which case we can in a worse case usage get bad node + lists later on due to messed up ligature building as these dashes are + ligatures in base fonts. This is a side effect of the separating the + hyphenation, ligaturing and kerning steps. A test is cmr with \type {------}. + + A font handler can collapse successive hyphens but it's not nice to put the + burden there. A somewhat messy border case is \type {----} but in \LUATEX\ we + don't treat \type {--} and \type {---} special. Also, traditional \TEX\ will + break a line at \type {-foo} but this can be disabled by setting the + automatic mode to \type {1}. + +*/ + +static halfword find_next_wordstart(halfword r, halfword first_language, halfword strict_bound) +{ + register int l; + register int start_ok = 1; + int mathlevel = 1; + int chr ; + halfword t ; + while (r != null) { + switch (type(r)) { + case boundary_node: + if (subtype(r) == word_boundary) { + start_ok = 1; + } + break; + case hlist_node: + case vlist_node: + case rule_node: + case dir_node: + case whatsit_node: + if (strict_bound == 1 || strict_bound == 3) { + start_ok = 0; + } + break; + case glue_node: + start_ok = 1; + break; + case math_node: + while (mathlevel > 0) { + r = vlink(r); + if (r == null) + return r; + if (type(r) == math_node) { + if (subtype(r) == before) { + mathlevel++; + } else { + mathlevel--; + } + } + } + break; + case glyph_node: + if (is_simple_character(r)) { + chr = character(r) ; + if (chr == ex_hyphen_char_par) { + t = vlink(r) ; + if ((automatic_hyphen_mode_par == 0) && (t != null) && (type(t) == glyph_node) && (character(t) != ex_hyphen_char_par)) { + /*tex We have no word yet and the next character is a non hyphen. */ + r = compound_word_break(r, char_lang(r)); + } else { + /*tex We jump over the sequence of hyphens. */ + while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { + r = t ; + t = vlink(r) ; + } + if (t == null) { + /*tex We reached the end of the list so we have no word start. */ + return null; + } + } + /*tex We need a restart. */ + start_ok = 0; + } else if (start_ok && (char_lang(r)>=first_language) && ((l = get_hj_code(char_lang(r),chr)) > 0)) { + if (char_uchyph(r) || l == chr || l <= 32) { + return r; + } else { + start_ok = 0; + } + } else { + /*tex We go on. */ + } + } + break; + default: + start_ok = 0; + break; + } + r = vlink(r); + } + return r; +} + +static int valid_wordend(halfword s, halfword strict_bound) +{ + register halfword r = s; + register int clang = char_lang(s); + if (r == null) + return 1; + while ( (r != null) && + ( (type(r) == glyph_node && is_simple_character(r) && clang == char_lang(r)) + || (type(r) == kern_node && (subtype(r) == normal)) + ) + ) { + r = vlink(r); + } + if (r == null || (type(r) == glyph_node && is_simple_character(r) && clang != char_lang(r)) + || type(r) == glue_node + || type(r) == penalty_node + || (type(r) == kern_node && (subtype(r) == explicit_kern || + subtype(r) == italic_kern || + subtype(r) == accent_kern )) + || ((type(r) == hlist_node || + type(r) == vlist_node || + type(r) == rule_node || + type(r) == dir_node || + type(r) == whatsit_node || + type(r) == ins_node || + type(r) == adjust_node + ) && ! (strict_bound == 2 || strict_bound == 3)) + || type(r) == boundary_node + ) + return 1; + return 0; +} + +void hnj_hyphenation(halfword head, halfword tail) +{ + int lchar, i; + struct tex_language *lang; + lang_variables langdata; + char utf8word[(4 * MAX_WORD_LEN) + 1] = { 0 }; + int wordlen = 0; + char *hy = utf8word; + char *replacement = NULL; + boolean explicit_hyphen = false; + boolean valid_word = false; + halfword first_language = first_valid_language_par; + halfword strict_bound = hyphenation_bounds_par; + halfword s, r = head, wordstart = null, save_tail1 = null, left = null, right = null; + halfword expstart = null; + boolean compound_hyphen = compound_hyphen_mode_par; + /*tex + This first movement assures two things: + + \startitemize + \startitem + That we won't waste lots of time on something that has been + handled already (in that case, none of the glyphs match + |simple_character|). + \stopitem + \startitem + That the first word can be hyphenated. if the movement was not + explicit, then the indentation at the start of a paragraph list + would make |find_next_wordstart()| look too far ahead. + \stopitem + \stopitemize + */ + while (r != null && (type(r) != glyph_node || !is_simple_character(r))) { + r = vlink(r); + } + /*tex + This will make |r| a glyph node with subtype character. + */ + r = find_next_wordstart(r,first_language,strict_bound); + if (r == null) + return; + assert(tail != null); + save_tail1 = vlink(tail); + s = new_penalty(0,word_penalty); + couple_nodes(tail, s); + while (r != null) { + /*tex This could be |while(1)|, but let's be paranoid: */ + int clang, lhmin, rhmin, hmin; + halfword hyf_font; + halfword end_word = r; + wordstart = r; + assert(is_simple_character(wordstart)); + hyf_font = font(wordstart); + if (hyphen_char(hyf_font) < 0) { + /*tex For backward compatibility we set: */ + hyf_font = 0; + } + clang = char_lang(wordstart); + lhmin = char_lhmin(wordstart); + rhmin = char_rhmin(wordstart); + hmin = get_hyphenation_min(clang); + langdata.pre_hyphen_char = get_pre_hyphen_char(clang); + langdata.post_hyphen_char = get_post_hyphen_char(clang); + while ( r != null + && type(r) == glyph_node + && is_simple_character(r) + && clang == char_lang(r) + && ( ( (clang >= first_language) + && (lchar = get_hj_code(clang,character(r))) > 0 + ) + || ( character(r) == ex_hyphen_char_par + && (lchar = ex_hyphen_char_par) + ) + ) + ) { + if (character(r) == ex_hyphen_char_par) { + explicit_hyphen = true; + break; + } + wordlen++; + if (lchar <= 32) { + if (lchar == 32) { + lchar = 0 ; + } + if (wordlen <= lhmin) { + lhmin = lhmin - lchar + 1 ; + if (lhmin < 0) + lhmin = 1; + } + if (wordlen >= rhmin) { + rhmin = rhmin - lchar + 1 ; + if (rhmin < 0) + rhmin = 1; + } + hmin = hmin - lchar + 1 ; + if (hmin < 0) + rhmin = 1; + lchar = character(r) ; + } + hy = uni2string(hy, (unsigned) lchar); + end_word = r; + r = vlink(r); + } + if (explicit_hyphen == true) { + /*tex we are not at the start, so we only need to look ahead */ + halfword t = vlink(r) ; + if ((automatic_hyphen_mode_par == 0 || automatic_hyphen_mode_par == 1) && (t != null) && ((type(t) == glyph_node) && (character(t) != ex_hyphen_char_par))) { + /*tex we have a word already but the next character may not be a hyphen too */ + r = compound_word_break(r, char_lang(r)); + if (compound_hyphen) { + if (expstart == null) { + expstart = wordstart; + } + explicit_hyphen = false; + hy = uni2string(hy, '-'); + r = t; + continue; + } + } else { + /*tex we jump over the sequence of hyphens */ + while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { + r = t ; + t = vlink(r) ; + } + if (t == null) { + /*tex we reached the end of the list and will quit the loop later */ + r = null; + } + } + } else if ( valid_wordend(r,strict_bound) + && clang >= first_language + && wordlen >= lhmin + rhmin + && (hmin <= 0 || wordlen >= hmin) + && (hyf_font != 0) + && (lang = tex_languages[clang]) != NULL + ) { + *hy = 0; + /*tex + this is messy and nasty: we can have a word with a - in it which + is why we have two branches + */ + if (lang->exceptions != 0 && (replacement = hyphenation_exception(lang->exceptions, utf8word)) != NULL) { + /*tex handle the exception and go on to the next word */ + if (expstart == null) { + do_exception(wordstart, r, replacement); + } else { + do_exception(expstart,r,replacement); + } + free(replacement); + } else if (expstart != null) { + /*tex We're done already */ + } else if (lang->patterns != NULL) { + /*tex A safeguard, not needed but doesn't hurt either: */ + valid_word = true; + left = wordstart; + for (i = lhmin; i > 1; i--) { + left = vlink(left); + if (left == null) { + valid_word = false; + break ; + } + while (!is_simple_character(left)) { + left = vlink(left); + if (left == null) { + valid_word = false; + break ; + } + } + } + if (valid_word) { + right = r; + if (right == left) { + valid_word = false; + break ; + } + for (i = rhmin; i > 0; i--) { + right = alink(right); + if (right == null || right == left) { + valid_word = false; + break ; + } + while (!is_simple_character(right)) { + right = alink(right); + if (right == null || right == left) { + valid_word = false; + break ; + } + } + } + if (valid_word && expstart == null) { + (void) hnj_hyphen_hyphenate(lang->patterns, wordstart, end_word, wordlen, left, right, &langdata); + } else { + /*tex nothing yet */ + } + } + } + } + expstart = null ; + explicit_hyphen = false; + wordlen = 0; + hy = utf8word; + if (r == null) + break; + r = find_next_wordstart(r,first_language,strict_bound); + } + flush_node(vlink(tail)); + vlink(tail) = save_tail1; +} + +void new_hyphenation(halfword head, halfword tail) +{ + int i, top; + register int callback_id = 0; + if (head == null || vlink(head) == null) + return; + fix_node_list(head); + callback_id = callback_defined(hyphenate_callback); + if (callback_id > 0) { + top = lua_gettop(Luas); + if (!get_callback(Luas, callback_id)) { + lua_settop(Luas, top); + return; + } + nodelist_to_lua(Luas, head); + nodelist_to_lua(Luas, tail); + if ((i=lua_pcall(Luas, 2, 0, 0)) != 0) { + formatted_warning("hyphenation","bad specification: %s",lua_tostring(Luas, -1)); + lua_settop(Luas, top); + luatex_error(Luas, (i == LUA_ERRRUN ? 0 : 1)); + return; + } + lua_settop(Luas, top); + } else if (callback_id == 0) { + hnj_hyphenation(head, tail); + } +} + +/*tex + + Dumping and undumping languages: + +*/ + +#define dump_string(a) \ + if (a!=NULL) { \ + x = (int)strlen(a)+1; \ + dump_int(x); dump_things(*a, x); \ + } else { \ + x = 0; dump_int(x); \ + } + +static void dump_one_language(int i) +{ + char *s = NULL; + int x = 0; + struct tex_language *lang; + lang = tex_languages[i]; + dump_int(lang->id); + dump_int(lang->pre_hyphen_char); + dump_int(lang->post_hyphen_char); + dump_int(lang->pre_exhyphen_char); + dump_int(lang->post_exhyphen_char); + dump_int(lang->hyphenation_min); + if (lang->patterns != NULL) { + s = (char *) hnj_serialize(lang->patterns); + } + dump_string(s); + if (s != NULL) { + free(s); + s = NULL; + } + if (lang->exceptions != 0) + s = exception_strings(lang); + dump_string(s); + if (s != NULL) { + free(s); + } + free(lang); +} + +void dump_language_data(void) +{ + int i; + dump_int(next_lang_id); + for (i = 0; i < next_lang_id; i++) { + if (tex_languages[i]) { + dump_int(1); + dump_one_language(i); + } else { + dump_int(0); + } + } +} + +static void undump_one_language(int i) +{ + char *s = NULL; + int x = 0; + struct tex_language *lang = get_language(i); + undump_int(x); + lang->id = x; + undump_int(x); + lang->pre_hyphen_char = x; + undump_int(x); + lang->post_hyphen_char = x; + undump_int(x); + lang->pre_exhyphen_char = x; + undump_int(x); + lang->post_exhyphen_char = x; + undump_int(x); + lang->hyphenation_min = x; + /*tex patterns */ + undump_int(x); + if (x > 0) { + s = xmalloc((unsigned) x); + undump_things(*s, x); + load_patterns(lang, (unsigned char *) s); + free(s); + } + /*tex exceptions */ + undump_int(x); + if (x > 0) { + s = xmalloc((unsigned) x); + undump_things(*s, x); + load_hyphenation(lang, (unsigned char *) s); + free(s); + } +} + +void undump_language_data(void) +{ + int i, x, numlangs; + undump_int(numlangs); + next_lang_id = numlangs; + for (i = 0; i < numlangs; i++) { + undump_int(x); + if (x == 1) { + undump_one_language(i); + } + } +} + +/*tex + + When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named + |new_hyph_exceptions| to do the right thing. + +*/ + +void new_hyph_exceptions(void) +{ + (void) scan_toks(false, true); + load_tex_hyphenation(language_par, def_ref); + flush_list(def_ref); +} + +/* + Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a procedure + named |new_patterns|. + +*/ + +void new_patterns(void) +{ + (void) scan_toks(false, true); + load_tex_patterns(language_par, def_ref); + flush_list(def_ref); +} + +/*tex + + `\.{\\prehyphenchar}', sets the |pre_break| character, and + `\.{\\posthyphenchar}' the |post_break| character. Their respective defaults + are ascii hyphen ("-") and zero (nul). + +*/ + +void new_pre_hyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_pre_hyphen_char(language_par, cur_val); +} + +void new_post_hyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_post_hyphen_char(language_par, cur_val); +} + +/*tex + + `\.{\\preexhyphenchar}', sets the |pre_break| character, and + `\.{\\postexhyphenchar}' the |post_break| character. Their defaults are both + zero (nul). + +*/ + +void new_pre_exhyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_pre_exhyphen_char(language_par, cur_val); +} + +void new_post_exhyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_post_exhyphen_char(language_par, cur_val); +} + +void new_hyphenation_min(void) +{ + scan_optional_equals(); + scan_int(); + set_hyphenation_min(language_par, cur_val); +} + +void new_hj_code(void) +{ + int i ; + scan_int(); + i = cur_val; + scan_optional_equals(); + scan_int(); + set_hj_code(language_par, i, cur_val, -1); +} diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.w b/Build/source/texk/web2c/luatexdir/lang/texlang.w deleted file mode 100644 index 337d2eca51d..00000000000 --- a/Build/source/texk/web2c/luatexdir/lang/texlang.w +++ /dev/null @@ -1,1213 +0,0 @@ -% texlang.w -% -% Copyright 2006-2012 Taco Hoekwater -% -% This file is part of LuaTeX. -% -% LuaTeX is free software; you can redistribute it and/or modify it under -% the terms of the GNU General Public License as published by the Free -% Software Foundation; either version 2 of the License, or (at your -% option) any later version. -% -% LuaTeX is distributed in the hope that it will be useful, but WITHOUT -% ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -% FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public -% License for more details. -% -% You should have received a copy of the GNU General Public License along -% with LuaTeX; if not, see . - -@ @c - -#include "ptexlib.h" -#include -#include "lua/luatex-api.h" - -@ Low-level helpers - -@ @c -#define unVERBOSE - -#define MAX_TEX_LANGUAGES 16384 - -static struct tex_language *tex_languages[MAX_TEX_LANGUAGES] = { NULL }; - -static int next_lang_id = 0; - -struct tex_language *new_language(int n) -{ - struct tex_language *lang; - unsigned l; - if (n >= 0) { - l = (unsigned) n; - if (l != (MAX_TEX_LANGUAGES - 1)) - if (next_lang_id <= n) - next_lang_id = n + 1; - } else { - while (tex_languages[next_lang_id] != NULL) - next_lang_id++; - l = (unsigned) next_lang_id++; - } - if (l < (MAX_TEX_LANGUAGES - 1) && tex_languages[l] == NULL) { - lang = xmalloc(sizeof(struct tex_language)); - tex_languages[l] = lang; - lang->id = (int) l; - lang->exceptions = 0; - lang->patterns = NULL; - lang->pre_hyphen_char = '-'; - lang->post_hyphen_char = 0; - lang->pre_exhyphen_char = 0; - lang->post_exhyphen_char = 0; - lang->hyphenation_min = -1; - if (saving_hyph_codes_par) { - /* for now, we might just use specific value for whatever task */ - hj_codes_from_lc_codes(l); - } - return lang; - } else { - return NULL; - } -} - -struct tex_language *get_language(int n) -{ - if (n >= 0 && n < MAX_TEX_LANGUAGES) { - if (tex_languages[n] != NULL) { - return tex_languages[n]; - } else { - return new_language(n); - } - } else { - return NULL; - } -} - -@ @c -void set_pre_hyphen_char(int n, int v) -{ - struct tex_language *l = get_language((int) n); - if (l != NULL) - l->pre_hyphen_char = (int) v; -} - -void set_post_hyphen_char(int n, int v) -{ - struct tex_language *l = get_language((int) n); - if (l != NULL) - l->post_hyphen_char = (int) v; -} - -void set_pre_exhyphen_char(int n, int v) -{ - struct tex_language *l = get_language((int) n); - if (l != NULL) - l->pre_exhyphen_char = (int) v; -} - -void set_post_exhyphen_char(int n, int v) -{ - struct tex_language *l = get_language((int) n); - if (l != NULL) - l->post_exhyphen_char = (int) v; -} - -int get_pre_hyphen_char(int n) -{ - struct tex_language *l = get_language((int) n); - if (l == NULL) - return -1; - return (int) l->pre_hyphen_char; -} - -int get_post_hyphen_char(int n) -{ - struct tex_language *l = get_language((int) n); - if (l == NULL) - return -1; - return (int) l->post_hyphen_char; -} - -int get_pre_exhyphen_char(int n) -{ - struct tex_language *l = get_language((int) n); - if (l == NULL) - return -1; - return (int) l->pre_exhyphen_char; -} - -int get_post_exhyphen_char(int n) -{ - struct tex_language *l = get_language((int) n); - if (l == NULL) - return -1; - return (int) l->post_exhyphen_char; -} - -void set_hyphenation_min(int n, int v) -{ - struct tex_language *l = get_language((int) n); - if (l != NULL) - l->hyphenation_min = (int) v; -} - -int get_hyphenation_min(int n) -{ - struct tex_language *l = get_language((int) n); - if (l == NULL) - return -1; - return (int) l->hyphenation_min; -} - -void load_patterns(struct tex_language *lang, const unsigned char *buff) -{ - if (lang == NULL || buff == NULL || strlen((const char *) buff) == 0) - return; - if (lang->patterns == NULL) { - lang->patterns = hnj_hyphen_new(); - } - hnj_hyphen_load(lang->patterns, buff); -} - -void clear_patterns(struct tex_language *lang) -{ - if (lang == NULL) - return; - if (lang->patterns != NULL) { - hnj_hyphen_clear(lang->patterns); - } -} - -void load_tex_patterns(int curlang, halfword head) -{ - char *s = tokenlist_to_cstring(head, 1, NULL); - load_patterns(get_language(curlang), (unsigned char *) s); -} - -@ @c -#define STORE_CHAR(l,x) do { \ - unsigned xx = get_hj_code(l,x); \ - if (!xx || xx <= 32) { \ - xx = x; \ - } \ - uindex = uni2string(uindex, xx); \ -} while (0) - -@ Cleans one word which is returned in |cleaned|, returns the new offset into -|buffer| - -@c -const char *clean_hyphenation(int id, const char *buff, char **cleaned) -{ - int items = 0; - unsigned char word[MAX_WORD_LEN + 1]; /* work buffer for bytes */ - unsigned uword[MAX_WORD_LEN + 1] = { 0 }; /* work buffer for unicode */ - int u = 0; /* unicode buffer value */ - int i = 0; /* index into buffer */ - char *uindex = (char *)word; - const char *s = buff; - - while (*s && !isspace((unsigned char)*s)) { - word[i++] = (unsigned)*s; - s++; - if ((s-buff)>MAX_WORD_LEN) { - /* todo: this is too strict, should count unicode, not bytes */ - *cleaned = NULL; - tex_error("exception too long", NULL); - return s; - } - } - /* now convert the input to unicode */ - word[i] = '\0'; - utf2uni_strcpy(uword, (const char *)word); - /* build the new word string */ - i = 0; - while (uword[i]>0) { - u = uword[i++]; - if (u == '-') { - /* skip */ - } else if (u == '=') { - STORE_CHAR(id,'-'); - } else if (u == '{') { - u = uword[i++]; - items = 0; - while (u && u != '}') { - u = uword[i++]; - } - if (u == '}') { - items++; - u = uword[i++]; - } - while (u && u != '}') { - u = uword[i++]; - } - if (u == '}') { - items++; - u = uword[i++]; - } - if (u == '{') { - u = uword[i++]; - } - while (u && u != '}') { - STORE_CHAR(id,u); - u = uword[i++]; - } - if (u == '}') { - items++; - } - if (items != 3) { - *cleaned = NULL; - tex_error("exception syntax error", NULL); - return s; - } - } else { - STORE_CHAR(id,u); - } - } - *uindex = '\0'; - *cleaned = xstrdup((char *) word); - return s; -} - -@ @c -void load_hyphenation(struct tex_language *lang, const unsigned char *buff) -{ - const char *s; - const char *value; - char *cleaned; - int id ; - if (lang == NULL) - return; - if (lang->exceptions == 0) { - lua_newtable(Luas); - lang->exceptions = luaL_ref(Luas, LUA_REGISTRYINDEX); - } - lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions); - s = (const char *) buff; - id = lang->id; - while (*s) { - while (isspace((unsigned char)*s)) - s++; - if (*s) { - value = s; - s = clean_hyphenation(id, s, &cleaned); - if (cleaned != NULL) { - if ((s - value) > 0) { - lua_pushstring(Luas, cleaned); - lua_pushlstring(Luas, value, (size_t) (s - value)); - lua_rawset(Luas, -3); - } - free(cleaned); - } else { -#ifdef VERBOSE - formatted_warning("hyphenation","skipping invalid hyphenation exception: %s", value); -#endif - } - } - } -} - -void clear_hyphenation(struct tex_language *lang) -{ - if (lang == NULL) - return; - if (lang->exceptions != 0) { - luaL_unref(Luas, LUA_REGISTRYINDEX, lang->exceptions); - lang->exceptions = 0; - } -} - -void load_tex_hyphenation(int curlang, halfword head) -{ - char *s = tokenlist_to_cstring(head, 1, NULL); - load_hyphenation(get_language(curlang), (unsigned char *) s); -} - -@ @c -static halfword insert_discretionary(halfword t, halfword pre, halfword post, halfword replace, int penalty) -{ - halfword g; - int f; - halfword d = new_node(disc_node, syllable_disc); - int attr = node_attr(t) ; - disc_penalty(d) = penalty; - if (t == replace) { - /* prev disc next-next */ - try_couple_nodes(d, vlink(t)); - try_couple_nodes(alink(t), d); - alink(t) = null; - vlink(t) = null; - replace = t ; - } else { - /* prev disc next */ - try_couple_nodes(d, vlink(t)); - couple_nodes(t, d); - } - if (replace != null) { - f = font(replace); - } else { - /* For compound words following explicit hyphens. */ - f = get_cur_font(); - } - for (g = pre; g != null; g = vlink(g)) { - font(g) = f; - if (attr != null) { - delete_attribute_ref(node_attr(g)); - node_attr(g) = attr; - attr_list_ref(attr) += 1; - } - } - for (g = post; g != null; g = vlink(g)) { - font(g) = f; - if (attr != null) { - delete_attribute_ref(node_attr(g)); - node_attr(g) = attr; - attr_list_ref(attr) += 1; - } - } - if (attr != null) { - for (g = replace; g != null; g = vlink(g)) { - delete_attribute_ref(node_attr(g)); - node_attr(g) = attr; - attr_list_ref(attr) += 1; - } - delete_attribute_ref(node_attr(d)); - node_attr(d) = attr; - attr_list_ref(attr) += 1; - } - set_disc_field(pre_break(d), pre); - set_disc_field(post_break(d), post); - set_disc_field(no_break(d), replace); - return d; -} - -halfword insert_syllable_discretionary(halfword t, lang_variables * lan) -{ - halfword g, n; - n = new_node(disc_node, syllable_disc); - disc_penalty(n) = hyphen_penalty_par; - couple_nodes(n, vlink(t)); - couple_nodes(t, n); - delete_attribute_ref(node_attr(n)); - if (node_attr(t) != null) { - node_attr(n) = node_attr(t); - attr_list_ref(node_attr(t))++; - } else { - node_attr(n) = null; - } - if (lan->pre_hyphen_char > 0) { - g = raw_glyph_node(); - set_to_character(g); - character(g) = lan->pre_hyphen_char; - font(g) = font(t); - lang_data(g) = lang_data(t); - if (node_attr(t) != null) { - node_attr(g) = node_attr(t); - attr_list_ref(node_attr(t))++; - } - set_disc_field(pre_break(n), g); - } - - if (lan->post_hyphen_char > 0) { - t = vlink(n); - g = raw_glyph_node(); - set_to_character(g); - character(g) = lan->post_hyphen_char; - font(g) = font(t); - lang_data(g) = lang_data(t); - if (node_attr(t) != null) { - node_attr(g) = node_attr(t); - attr_list_ref(node_attr(t)) += 1; - } - set_disc_field(post_break(n), g); - } - return n; -} - -@ @c -static halfword insert_character(halfword t, int c) -{ - halfword p; - p = new_node(glyph_node, 0); - set_to_character(p); - character(p) = c; - if (t != null) { - couple_nodes(t, p); - } - return p; -} - -static halfword compound_word_break(halfword t, int clang) -{ - halfword disc = null; - halfword pre = null; - halfword pos = null; - halfword pre_exhyphen_char = get_pre_exhyphen_char(clang); - halfword post_exhyphen_char = get_post_exhyphen_char(clang); - if (pre_exhyphen_char > 0) { - pre = insert_character(null,pre_exhyphen_char); - } else { - pre = insert_character(null,ex_hyphen_char_par); - } - if (post_exhyphen_char > 0) - pos = insert_character(null,post_exhyphen_char); - disc = insert_discretionary(t,pre,pos,t,ex_hyphen_penalty_par); - subtype(disc) = automatic_disc; - set_automatic_disc_penalty(disc); - return disc; -} - -@ @c -void set_disc_field(halfword f, halfword t) -{ - if (t != null) { - /* - couple_nodes(f, t); // better not expose f as prev pointer - */ - vlink(f) = t ; - alink(t) = null ; - tlink(f) = tail_of_list(t); - } else { - vlink(f) = null; - tlink(f) = null; - } -} - -@ @c -static char *hyphenation_exception(int exceptions, char *w) -{ - char *ret = NULL; - lua_checkstack(Luas, 2); - lua_rawgeti(Luas, LUA_REGISTRYINDEX, exceptions); - if (lua_istable(Luas, -1)) { /* ?? */ - lua_pushstring(Luas, w); /* word table */ - lua_rawget(Luas, -2); - if (lua_type(Luas, -1) == LUA_TSTRING) { - ret = xstrdup(lua_tostring(Luas, -1)); - } - lua_pop(Luas, 2); - } else { - lua_pop(Luas, 1); - } - return ret; -} - -@ @c -char *exception_strings(struct tex_language *lang) -{ - const char *value; - size_t size = 0, current = 0; - size_t l = 0; - char *ret = NULL; - if (lang->exceptions == 0) - return NULL; - lua_checkstack(Luas, 2); - lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions); - if (lua_istable(Luas, -1)) { - /* iterate and join */ - lua_pushnil(Luas); /* first key */ - while (lua_next(Luas, -2) != 0) { - value = lua_tolstring(Luas, -1, &l); - if (current + 2 + l > size) { - ret = xrealloc(ret, (unsigned) ((size + size / 5) + current + l + 1024)); - size = (size + size / 5) + current + l + 1024; - } - *(ret + current) = ' '; - strcpy(ret + current + 1, value); - current += l + 1; - lua_pop(Luas, 1); - } - } - return ret; -} - -@ the sequence from |wordstart| to |r| can contain only normal characters it -could be faster to modify a halfword pointer and return an integer - -@c -static halfword find_exception_part(unsigned int *j, unsigned int *uword, int len) -{ - halfword g = null, gg = null; - register unsigned i = *j; - i++; /* this puts uword[i] on the |{| */ - while (i < (unsigned) len && uword[i + 1] != '}') { - if (g == null) { - gg = new_char(0, (int) uword[i + 1]); - g = gg; - } else { - halfword s = new_char(0, (int) uword[i + 1]); - couple_nodes(g, s); - g = vlink(g); - } - i++; - } - *j = ++i; - return gg; -} - -static int count_exception_part(unsigned int *j, unsigned int *uword, int len) -{ - int ret = 0; - register unsigned i = *j; - i++; /* this puts uword[i] on the |{| */ - while (i < (unsigned) len && uword[i + 1] != '}') { - ret++; - i++; - } - *j = ++i; - return ret; -} - -@ @c -static const char *PAT_ERROR[] = { - "Exception discretionaries should contain three pairs of braced items.", - "No intervening spaces are allowed.", - NULL -}; - -/* - The exceptions are taken as-is: no min values are taken into account. One can - add normal patterns on-the-fly if needed. -*/ - -static void do_exception(halfword wordstart, halfword r, char *replacement) -{ - unsigned i; - halfword t; - unsigned len; - int clang; - lang_variables langdata; - unsigned uword[MAX_WORD_LEN + 1] = { 0 }; - utf2uni_strcpy(uword, replacement); - len = u_length(uword); - i = 0; - t = wordstart; - clang = char_lang(wordstart); - langdata.pre_hyphen_char = get_pre_hyphen_char(clang); - langdata.post_hyphen_char = get_post_hyphen_char(clang); - - for (i = 0; i < len; i++) { - if (uword[i + 1] == '-') { /* a hyphen follows */ - while (vlink(t) != r && (type(t) != glyph_node || !is_simple_character(t))) - t = vlink(t); - if (vlink(t) == r) - break; - insert_syllable_discretionary(t, &langdata); - t = vlink(t); /* skip the new disc */ - } else if (uword[i + 1] == '=') { - /* do nothing ? */ - t = vlink(t); - } else if (uword[i + 1] == '{') { - halfword gg, hh, replace = null; - int repl; - gg = find_exception_part(&i, uword, (int) len); - if (i == len || uword[i + 1] != '{') { - tex_error("broken pattern 1", PAT_ERROR); - } - hh = find_exception_part(&i, uword, (int) len); - if (i == len || uword[i + 1] != '{') { - tex_error("broken pattern 2", PAT_ERROR); - } - repl = count_exception_part(&i, uword, (int) len); - if (i == len) { - tex_error("broken pattern 3", PAT_ERROR); - } - /*i++; *//* jump over the last right brace */ - if (vlink(t) == r) - break; - if (repl > 0) { - halfword q = t; - replace = vlink(q); - while (repl > 0 && q != null) { - q = vlink(q); - if (type(q) == glyph_node) { - repl--; - } - } - try_couple_nodes(t, vlink(q)); - vlink(q) = null; - } - t = insert_discretionary(t, gg, hh, replace, hyphen_penalty_par); - t = vlink(t); /* skip the new disc */ - } else { - t = vlink(t); - } - } -} - -@ This is a documentation section from the pascal web file. It is not true any -more, but I do not have time right now to rewrite it -- Taco - -When the line-breaking routine is unable to find a feasible sequence of -breakpoints, it makes a second pass over the paragraph, attempting to hyphenate -the hyphenatable words. The goal of hyphenation is to insert discretionary -material into the paragraph so that there are more potential places to break. - -The general rules for hyphenation are somewhat complex and technical, because we -want to be able to hyphenate words that are preceded or followed by punctuation -marks, and because we want the rules to work for languages other than English. We -also must contend with the fact that hyphens might radically alter the ligature -and kerning structure of a word. - -A sequence of characters will be considered for hyphenation only if it belongs to -a ``potentially hyphenatable part'' of the current paragraph. This is a sequence -of nodes $p_0p_1\ldots p_m$ where $p_0$ is a glue node, $p_1\ldots p_{m-1}$ are -either character or ligature or whatsit or implicit kern nodes, and $p_m$ is a -glue or penalty or insertion or adjust or mark or whatsit or explicit kern node. -(Therefore hyphenation is disabled by boxes, math formulas, and discretionary -nodes already inserted by the user.) The ligature nodes among $p_1\ldots p_{m-1}$ -are effectively expanded into the original non-ligature characters; the kern -nodes and whatsits are ignored. Each character |c| is now classified as either a -nonletter (if |lc_code(c)=0|), a lowercase letter (if |lc_code(c)=c|), or an -uppercase letter (otherwise); an uppercase letter is treated as if it were -|lc_code(c)| for purposes of hyphenation. The characters generated by $p_1\ldots -p_{m-1}$ may begin with nonletters; let $c_1$ be the first letter that is not in -the middle of a ligature. Whatsit nodes preceding $c_1$ are ignored; a whatsit -found after $c_1$ will be the terminating node $p_m$. All characters that do not -have the same font as $c_1$ will be treated as nonletters. The |hyphen_char| for -that font must be between 0 and 255, otherwise hyphenation will not be attempted. -\TeX\ looks ahead for as many consecutive letters $c_1\ldots c_n$ as possible; -however, |n| must be less than 64, so a character that would otherwise be -$c_{64}$ is effectively not a letter. Furthermore $c_n$ must not be in the middle -of a ligature. In this way we obtain a string of letters $c_1\ldots c_n$ that are -generated by nodes $p_a\ldots p_b$, where |1<=a<=b+1<=m|. If |n>=l_hyf+r_hyf|, -this string qualifies for hyphenation; however, |uc_hyph| must be positive, if -$c_1$ is uppercase. - -The hyphenation process takes place in three stages. First, the candidate -sequence $c_1\ldots c_n$ is found; then potential positions for hyphens are -determined by referring to hyphenation tables; and finally, the nodes $p_a\ldots -p_b$ are replaced by a new sequence of nodes that includes the discretionary -breaks found. - -Fortunately, we do not have to do all this calculation very often, because of the -way it has been taken out of \TeX's inner loop. For example, when the second -edition of the author's 700-page book {\sl Seminumerical Algorithms} was typeset -by \TeX, only about 1.2 hyphenations needed to be @^Knuth, Donald Ervin@> tried -per paragraph, since the line breaking algorithm needed to use two passes on only -about 5 per cent of the paragraphs. - -When a word been set up to contain a candidate for hyphenation, \TeX\ first looks -to see if it is in the user's exception dictionary. If not, hyphens are inserted -based on patterns that appear within the given word, using an algorithm due to -Frank~M. Liang. @^Liang, Franklin Mark@> - -@ This is incompatible with TEX because the first word of a paragraph can be -hyphenated, but most european users seem to agree that prohibiting hyphenation -there was not the best idea ever. - -@c -/* - More strict: \hyphenationbounds - - 0 = not strict - 1 = strict start - 2 = strict end - 3 = strict start and strict end - - \parindent0pt \hsize=1.1cm - 12-34-56 \par - 12-34-\hbox{56} \par - 12-34-\vrule width 1em height 1.5ex \par - 12-\hbox{34}-56 \par - 12-\vrule width 1em height 1.5ex-56 \par - \hjcode`\1=`\1 \hjcode`\2=`\2 \hjcode`\3=`\3 \hjcode`\4=`\4 \vskip.5cm - 12-34-56 \par - 12-34-\hbox{56} \par - 12-34-\vrule width 1em height 1.5ex \par - 12-\hbox{34}-56 \par - 12-\vrule width 1em height 1.5ex-56 \par - -*/ - -/* - We only accept an explicit hyphen when there is a preceding glyph and we skip a sequence of - explicit hyphens as that normally indicates a -- or --- ligature in which case we can in a - worse case usage get bad node lists later on due to messed up ligature building as these - dashes are ligatures in base fonts. This is a side effect of the separating the hyphenation, - ligaturing and kerning steps. A test is cmr with ------. - - A font handler can collapse successive hyphens but it's not nice to put the burden there. A - somewhat messy border case is ---- but in LuaTeX we don't treat -- and --- special. Also, - traditional TeX will break a line at -foo but this can be disabled by setting the automatic - mode to 1. - -*/ - -static halfword find_next_wordstart(halfword r, halfword first_language, halfword strict_bound) -{ - register int l; - register int start_ok = 1; - int mathlevel = 1; - int chr ; - halfword t ; - while (r != null) { - switch (type(r)) { - case boundary_node: - if (subtype(r) == word_boundary) { - start_ok = 1; - } - break; - case hlist_node: /* new > 0.95 */ - case vlist_node: /* new > 0.95 */ - case rule_node: /* new > 0.95 */ - case dir_node: - case whatsit_node: - if (strict_bound == 1 || strict_bound == 3) { - start_ok = 0; - } - break; - case glue_node: - start_ok = 1; - break; - case math_node: - while (mathlevel > 0) { - r = vlink(r); - if (r == null) - return r; - if (type(r) == math_node) { - if (subtype(r) == before) { - mathlevel++; - } else { - mathlevel--; - } - } - } - break; - case glyph_node: - if (is_simple_character(r)) { - chr = character(r) ; - if (chr == ex_hyphen_char_par) { - t = vlink(r) ; - if ((automatic_hyphen_mode_par == 0) && (t != null) && (type(t) == glyph_node) && (character(t) != ex_hyphen_char_par)) { - /* we have no word yet and the next character is a non hyphen */ - r = compound_word_break(r, char_lang(r)); - } else { - /* we jump over the sequence of hyphens */ - while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { - r = t ; - t = vlink(r) ; - } - if (t == null) { - /* we reached the end of the list so we have no word start */ - return null; - } - } - /* we need a restart */ - start_ok = 0; - } else if (start_ok && (char_lang(r)>=first_language) && ((l = get_hj_code(char_lang(r),chr)) > 0)) { - if (char_uchyph(r) || l == chr || l <= 32) { - return r; - } else { - start_ok = 0; - } - } else { - /* go on */ - } - } - break; - default: - start_ok = 0; - break; - } - r = vlink(r); - } - return r; -} - -@ @c -static int valid_wordend(halfword s, halfword strict_bound) -{ - register halfword r = s; - register int clang = char_lang(s); - if (r == null) - return 1; - while ( (r != null) && - ( (type(r) == glyph_node && is_simple_character(r) && clang == char_lang(r)) - || (type(r) == kern_node && (subtype(r) == normal)) - ) - ) { - r = vlink(r); - } - if (r == null || (type(r) == glyph_node && is_simple_character(r) && clang != char_lang(r)) - || type(r) == glue_node - || type(r) == penalty_node - || (type(r) == kern_node && (subtype(r) == explicit_kern || /* so why not italic correction ? */ - subtype(r) == italic_kern || - subtype(r) == accent_kern )) - || ((type(r) == hlist_node || /* new > 0.95 */ - type(r) == vlist_node || /* new > 0.95 */ - type(r) == rule_node || /* new > 0.95 */ - type(r) == dir_node || /* new > 0.97 */ - type(r) == whatsit_node || - type(r) == ins_node || /* yes or no strict test */ - type(r) == adjust_node /* yes or no strict test */ - ) && ! (strict_bound == 2 || strict_bound == 3)) - || type(r) == boundary_node - ) - return 1; - return 0; -} - -@ @c -void hnj_hyphenation(halfword head, halfword tail) -{ - int lchar, i; - struct tex_language *lang; - lang_variables langdata; - char utf8word[(4 * MAX_WORD_LEN) + 1] = { 0 }; - int wordlen = 0; - char *hy = utf8word; - char *replacement = NULL; - boolean explicit_hyphen = false; - halfword first_language = first_valid_language_par; - halfword strict_bound = hyphenation_bounds_par; - halfword s, r = head, wordstart = null, save_tail1 = null, left = null, right = null; - - /* - This first movement assures two things: - - \item{a)} that we won't waste lots of time on something that has been handled already - (in that case, none of the glyphs match |simple_character|). - - \item{b)} that the first word can be hyphenated. if the movement was not explicit, - then the indentation at the start of a paragraph list would make |find_next_wordstart()| - look too far ahead. - */ - - while (r != null && (type(r) != glyph_node || !is_simple_character(r))) { - r = vlink(r); - } - /* - This will make |r| a glyph node with subtype character. - */ - r = find_next_wordstart(r,first_language,strict_bound); - if (r == null) - return; - - assert(tail != null); - save_tail1 = vlink(tail); - s = new_penalty(0,word_penalty); - couple_nodes(tail, s); - - while (r != null) { /* could be while(1), but let's be paranoid */ - int clang, lhmin, rhmin, hmin; - halfword hyf_font; - halfword end_word = r; - wordstart = r; - assert(is_simple_character(wordstart)); - hyf_font = font(wordstart); - if (hyphen_char(hyf_font) < 0) { - /* For backward compatibility: */ - hyf_font = 0; - } - clang = char_lang(wordstart); - lhmin = char_lhmin(wordstart); - rhmin = char_rhmin(wordstart); - hmin = get_hyphenation_min(clang); - langdata.pre_hyphen_char = get_pre_hyphen_char(clang); - langdata.post_hyphen_char = get_post_hyphen_char(clang); - while ( r != null - && type(r) == glyph_node - && is_simple_character(r) - && clang == char_lang(r) - && ( ( (clang >= first_language) - && (lchar = get_hj_code(clang,character(r))) > 0 - ) - || ( character(r) == ex_hyphen_char_par - && (lchar = ex_hyphen_char_par) - ) - ) - ) { - if (character(r) == ex_hyphen_char_par) { - explicit_hyphen = true; - break; - } - wordlen++; - if (lchar <= 32) { - if (lchar == 32) { - lchar = 0 ; - } - if (wordlen <= lhmin) { - lhmin = lhmin - lchar + 1 ; - if (lhmin < 0) - lhmin = 1; - } - if (wordlen >= rhmin) { - rhmin = rhmin - lchar + 1 ; - if (rhmin < 0) - rhmin = 1; - } - hmin = hmin - lchar + 1 ; - if (hmin < 0) - rhmin = 1; - lchar = character(r) ; - } - hy = uni2string(hy, (unsigned) lchar); - end_word = r; - r = vlink(r); - } - if (explicit_hyphen == true) { - /* we are not at the start, so we only need to look ahead */ - halfword t = vlink(r) ; - if ((automatic_hyphen_mode_par == 0 || automatic_hyphen_mode_par == 1) && (t != null) && ((type(t) == glyph_node) && (character(t) != ex_hyphen_char_par))) { - /* we have a word already but the next character may not be a hyphen too */ - r = compound_word_break(r, char_lang(r)); - } else { - /* we jump over the sequence of hyphens */ - while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) { - r = t ; - t = vlink(r) ; - } - if (t == null) { - /* we reached the end of the list and will quit the loop later */ - r = null; - } - } - } else if ( valid_wordend(r,strict_bound) - && clang >= first_language - && wordlen >= lhmin + rhmin - && (hmin <= 0 || wordlen >= hmin) - && (hyf_font != 0) - && (lang = tex_languages[clang]) != NULL - ) { - *hy = 0; - if (lang->exceptions != 0 && (replacement = hyphenation_exception(lang->exceptions, utf8word)) != NULL) { - /* handle the exception and go on to the next word */ - do_exception(wordstart, r, replacement); - free(replacement); - } else if (lang->patterns != NULL) { - left = wordstart; - for (i = lhmin; i > 1; i--) { - left = vlink(left); - while (!is_simple_character(left)) { - left = vlink(left); - } - /* if (!left) break; */ - /* what is left overruns right .. a bit messy */ - } - right = r; - for (i = rhmin; i > 0; i--) { - right = alink(right); - while (!is_simple_character(right)) { - right = alink(right); - } - /* if (!right) break; */ - /* what is right overruns left .. a bit messy */ - } - (void) hnj_hyphen_hyphenate(lang->patterns, wordstart, end_word, wordlen, left, right, &langdata); - } - } - explicit_hyphen = false; - wordlen = 0; - hy = utf8word; - if (r == null) - break; - r = find_next_wordstart(r,first_language,strict_bound); - } - flush_node(vlink(tail)); - vlink(tail) = save_tail1; -} - -@ @c -void new_hyphenation(halfword head, halfword tail) -{ - register int callback_id = 0; - if (head == null || vlink(head) == null) - return; - fix_node_list(head); - callback_id = callback_defined(hyphenate_callback); - if (callback_id > 0) { - if (!get_callback(Luas, callback_id)) { - lua_pop(Luas, 2); - return; - } - nodelist_to_lua(Luas, head); - nodelist_to_lua(Luas, tail); - if (lua_pcall(Luas, 2, 0, 0) != 0) { - formatted_warning("hyphenation","bad specification: %s",lua_tostring(Luas, -1)); - lua_pop(Luas, 2); - lua_error(Luas); - return; - } - lua_pop(Luas, 1); - } else if (callback_id == 0) { - hnj_hyphenation(head, tail); - } -} - -@ Dumping and undumping languages. - -@c -#define dump_string(a) \ - if (a!=NULL) { \ - x = (int)strlen(a)+1; \ - dump_int(x); dump_things(*a, x); \ - } else { \ - x = 0; dump_int(x); \ - } - -static void dump_one_language(int i) -{ - char *s = NULL; - int x = 0; - struct tex_language *lang; - lang = tex_languages[i]; - dump_int(lang->id); - dump_int(lang->pre_hyphen_char); - dump_int(lang->post_hyphen_char); - dump_int(lang->pre_exhyphen_char); - dump_int(lang->post_exhyphen_char); - dump_int(lang->hyphenation_min); - if (lang->patterns != NULL) { - s = (char *) hnj_serialize(lang->patterns); - } - dump_string(s); - if (s != NULL) { - free(s); - s = NULL; - } - if (lang->exceptions != 0) - s = exception_strings(lang); - dump_string(s); - if (s != NULL) { - free(s); - } - free(lang); -} - -void dump_language_data(void) -{ - int i; - dump_int(next_lang_id); - for (i = 0; i < next_lang_id; i++) { - if (tex_languages[i]) { - dump_int(1); - dump_one_language(i); - } else { - dump_int(0); - } - } -} - -static void undump_one_language(int i) -{ - char *s = NULL; - int x = 0; - struct tex_language *lang = get_language(i); - undump_int(x); - lang->id = x; - undump_int(x); - lang->pre_hyphen_char = x; - undump_int(x); - lang->post_hyphen_char = x; - undump_int(x); - lang->pre_exhyphen_char = x; - undump_int(x); - lang->post_exhyphen_char = x; - undump_int(x); - lang->hyphenation_min = x; - /* patterns */ - undump_int(x); - if (x > 0) { - s = xmalloc((unsigned) x); - undump_things(*s, x); - load_patterns(lang, (unsigned char *) s); - free(s); - } - /* exceptions */ - undump_int(x); - if (x > 0) { - s = xmalloc((unsigned) x); - undump_things(*s, x); - load_hyphenation(lang, (unsigned char *) s); - free(s); - } -} - -void undump_language_data(void) -{ - int i, x, numlangs; - undump_int(numlangs); - next_lang_id = numlangs; - for (i = 0; i < numlangs; i++) { - undump_int(x); - if (x == 1) { - undump_one_language(i); - } - } -} - -@ When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named -|new_hyph_exceptions| to do the right thing. - -@c -void new_hyph_exceptions(void) -{ /* enters new exceptions */ - (void) scan_toks(false, true); - load_tex_hyphenation(language_par, def_ref); - flush_list(def_ref); -} - -@ Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a procedure named -|new_patterns|. - -@c -void new_patterns(void) -{ - (void) scan_toks(false, true); - load_tex_patterns(language_par, def_ref); - flush_list(def_ref); -} - -@ `\.{\\prehyphenchar}', sets the |pre_break| character, and `\.{\\posthyphenchar}' the -|post_break| character. Their respective defaults are ascii hyphen ("-") and zero (nul). - -@c -void new_pre_hyphen_char(void) -{ - scan_optional_equals(); - scan_int(); - set_pre_hyphen_char(language_par, cur_val); -} - -void new_post_hyphen_char(void) -{ - scan_optional_equals(); - scan_int(); - set_post_hyphen_char(language_par, cur_val); -} - -@ `\.{\\preexhyphenchar}', sets the |pre_break| character, and `\.{\\postexhyphenchar}' the -|post_break| character. Their defaults are both zero (nul). - -@c -void new_pre_exhyphen_char(void) -{ - scan_optional_equals(); - scan_int(); - set_pre_exhyphen_char(language_par, cur_val); -} - -void new_post_exhyphen_char(void) -{ - scan_optional_equals(); - scan_int(); - set_post_exhyphen_char(language_par, cur_val); -} - -void new_hyphenation_min(void) -{ - scan_optional_equals(); - scan_int(); - set_hyphenation_min(language_par, cur_val); -} - -void new_hj_code(void) -{ - int i ; - scan_int(); - i = cur_val; - scan_optional_equals(); - scan_int(); - set_hj_code(language_par, i, cur_val, -1); -} -- cgit v1.2.3