diff options
author | Taco Hoekwater <taco@elvenkind.com> | 2010-04-04 13:55:46 +0000 |
---|---|---|
committer | Taco Hoekwater <taco@elvenkind.com> | 2010-04-04 13:55:46 +0000 |
commit | f372862c42d234710b6057449b46818e63181c90 (patch) | |
tree | a9ebec66dd31ac37aed442502ebd1b9fe76245ec /Build/source/texk/web2c/luatexdir/lang | |
parent | 2dad291ab056f30816e0aea61970f38033c2f6a5 (diff) |
import luatex 0.60.0 (with small local changes, see luatexdir/ChangeLog)
and do an autoreconf
git-svn-id: svn://tug.org/texlive/trunk@17680 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/lang')
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hnjalloc.h | 10 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hnjalloc.w | 66 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hyphen.c | 850 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hyphen.h (renamed from Build/source/texk/web2c/luatexdir/lang/hnjalloc.c) | 52 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/hyphen.w | 857 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/texlang.h | 82 | ||||
-rw-r--r-- | Build/source/texk/web2c/luatexdir/lang/texlang.w (renamed from Build/source/texk/web2c/luatexdir/lang/texlang.c) | 368 |
7 files changed, 1254 insertions, 1031 deletions
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h index db82fb9c758..1af5a3f0ea3 100644 --- a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h +++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h @@ -35,12 +35,8 @@ */ /* wrappers for malloc */ -void * -hnj_malloc (int size); +void *hnj_malloc(int size); -void * -hnj_realloc (void *p, int size); - -void -hnj_free (void *p); +void *hnj_realloc(void *p, int size); +void hnj_free(void *p); diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w new file mode 100644 index 00000000000..c7d595cdd75 --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w @@ -0,0 +1,66 @@ +% LibHnj is dual licensed under LGPL and MPL. Boilerplate for both +% licenses follows. +% +% +% LibHnj - a library for high quality hyphenation and justification +% Copyright (C) 1998 Raph Levien, (C) 2001 ALTLinux, Moscow +% +% This library is free software; you can redistribute it and/or +% modify it under the terms of the GNU Library General Public +% License as published by the Free Software Foundation; either +% version 2 of the License, or (at your option) any later version. +% +% This library is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +% Library General Public License for more details. +% +% You should have received a copy of the GNU Library General Public +% License along with this library; if not, write to the +% Free Software Foundation, Inc., 59 Temple Place - Suite 330, +% Boston, MA 02111-1307 USA. +% +% +% +% The contents of this file are subject to the Mozilla Public License +% Version 1.0 (the "MPL"); you may not use this file except in +% compliance with the MPL. You may obtain a copy of the MPL at +% http://www.mozilla.org/MPL/ +% +% Software distributed under the MPL is distributed on an "AS IS" basis, +% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL +% for the specific language governing rights and limitations under the +% MPL. + +@ wrappers for malloc +@c + +#include <stdlib.h> +#include <stdio.h> + +void *hnj_malloc(int size) +{ + void *p; + + p = malloc((size_t) size); + if (p == NULL) { + fprintf(stderr, "can't allocate %d bytes\n", size); + exit(1); + } + return p; +} + +void *hnj_realloc(void *p, int size) +{ + p = realloc(p, (size_t) size); + if (p == NULL) { + fprintf(stderr, "can't allocate %d bytes\n", size); + exit(1); + } + return p; +} + +void hnj_free(void *p) +{ + free(p); +} diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.c b/Build/source/texk/web2c/luatexdir/lang/hyphen.c deleted file mode 100644 index edcad69adff..00000000000 --- a/Build/source/texk/web2c/luatexdir/lang/hyphen.c +++ /dev/null @@ -1,850 +0,0 @@ -/* Libhnj is dual licensed under LGPL and MPL. Boilerplate for both - * licenses follows. - */ - -/* LibHnj - a library for high quality hyphenation and justification - * Copyright (C) 1998 Raph Levien, - * (C) 2001 ALTLinux, Moscow (http://www.alt-linux.org), - * (C) 2001 Peter Novodvorsky (nidd@cs.msu.su) - * - * This library is free software; you can redistribute it and/or - * modify it under the terms of the GNU Library General Public - * License as published by the Free Software Foundation; either - * version 2 of the License, or (at your option) any later version. - * - * This library is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU - * Library General Public License for more details. - * - * You should have received a copy of the GNU Library General Public - * License along with this library; if not, write to the - * Free Software Foundation, Inc., 59 Temple Place - Suite 330, - * Boston, MA 02111-1307 USA. -*/ - -/* - * The contents of this file are subject to the Mozilla Public License - * Version 1.0 (the "MPL"); you may not use this file except in - * compliance with the MPL. You may obtain a copy of the MPL at - * http://www.mozilla.org/MPL/ - * - * Software distributed under the MPL is distributed on an "AS IS" basis, - * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL - * for the specific language governing rights and limitations under the - * MPL. - * - */ -#include <stdlib.h> /* for NULL, malloc */ -#include <stdio.h> /* for fprintf */ -#include <string.h> /* for strdup */ -#include <stdlib.h> /* for malloc used by substring inclusion*/ - -#define MAXPATHS 40960 - -#ifdef UNX -#include <unistd.h> /* for exit */ -#endif - -#include <ctype.h> - -/*#define VERBOSE*/ - -#include "hnjalloc.h" -#include "hyphen.h" - -static const char _svn_version[] = - "$Id: hyphen.c 1696 2008-12-28 21:02:34Z oneiros $ $URL: http://scm.foundry.supelec.fr/svn/luatex/trunk/src/texk/web2c/luatexdir/lang/texlang.c $"; - -/* SHOULD BE MOVED TO SEPARATE LIBRARY */ -static unsigned char * hnj_strdup( - const unsigned char *s -) { - unsigned char *new; - int l; - - l = strlen ((const char*)s); - new = hnj_malloc (l + 1); - memcpy (new, s, l); - new[l] = 0; - return new; -} - -static int is_utf8_follow( - unsigned char c -) { - if (c>=0x80 && c<0xC0) return 1; - return 0; -} - -/* -------------------------------------------------------------------- - * - * Type definitions - * - * -------------------------------------------------------------------- - */ - -/* a little bit of a hash table implementation. This simply maps strings - to state numbers */ - -typedef struct _HashTab HashTab; -typedef struct _HashEntry HashEntry; -typedef struct _HashIter HashIter; -typedef union _HashVal HashVal; - -/* A cheap, but effective, hack. */ -#define HASH_SIZE 31627 - -struct _HashTab { - HashEntry *entries[HASH_SIZE]; -}; - -union _HashVal { - int state; - char* hyppat; -}; - -struct _HashEntry { - HashEntry *next; - unsigned char *key; - HashVal u; -}; - -struct _HashIter { - HashEntry** e; - HashEntry* cur; - int ndx; -}; - -/* State machine */ -typedef struct _HyphenState HyphenState; -typedef struct _HyphenTrans HyphenTrans; -#define MAX_CHARS 256 -#define MAX_NAME 20 - -struct _HyphenDict { - int num_states; - int pat_length; - char cset[MAX_NAME]; - HyphenState *states; - HashTab *patterns; - HashTab *merged; - HashTab *state_num; -}; - -struct _HyphenState { - char *match; - /*char *repl;*/ - /*signed char replindex;*/ - /*signed char replcut;*/ - int fallback_state; - int num_trans; - HyphenTrans *trans; -}; - -struct _HyphenTrans { - int uni_ch; - int new_state; -}; - - -/* Combine two right-aligned number patterns, 04000 + 020 becomes 04020*/ -static char *combine( - char *expr, - const char *subexpr -) { - int l1 = strlen(expr); - int l2 = strlen(subexpr); - int off = l1-l2; - int j; - /* this works also for utf8 sequences because the substring is identical - * to the last substring-length bytes of expr except for the (single byte) - * hyphenation encoders - */ - for (j=0; j<l2; j++) { - if (expr[off+j]<subexpr[j]) expr[off+j] = subexpr[j]; - } - return expr; -} - - -/* -------------------------------------------------------------------- - * ORIGINAL CODE - * -------------------------------------------------------------------- - */ - -static HashIter* new_HashIter( - HashTab* h -) { - HashIter* i = hnj_malloc(sizeof(HashIter)); - i->e = h->entries; - i->cur = NULL; - i->ndx = -1; - return i; -} - - -static int nextHashStealPattern( - HashIter*i, - unsigned char**word, - char **pattern -) { - while (i->cur==NULL) { - if (i->ndx >= HASH_SIZE-1) return 0; - i->cur = i->e[++i->ndx]; - } - *word = i->cur->key; - *pattern = i->cur->u.hyppat; - i->cur->u.hyppat = NULL; - i->cur = i->cur->next; - return 1; -} - - -static int nextHash( - HashIter*i, - unsigned char**word -) { - while (i->cur==NULL) { - if (i->ndx >= HASH_SIZE-1) return 0; - i->cur = i->e[++i->ndx]; - } - *word = i->cur->key; - i->cur = i->cur->next; - return 1; -} - - -static int eachHash( - HashIter*i, - unsigned char**word, - char**pattern -) { - while (i->cur==NULL) { - if (i->ndx >= HASH_SIZE-1) return 0; - i->cur = i->e[++i->ndx]; - } - *word = i->cur->key; - *pattern = i->cur->u.hyppat; - i->cur = i->cur->next; - return 1; -} - - -static void delete_HashIter( - HashIter*i -) { - hnj_free(i); -} - - -/* a char* hash function from ASU - adapted from Gtk+ */ -static unsigned int hnj_string_hash ( - const unsigned char *s -) { - const unsigned char *p; - unsigned int h=0, g; - - for (p = s; *p != '\0'; p += 1) { - h = ( h << 4 ) + *p; - if ( ( g = h & 0xf0000000 ) ) { - h = h ^ (g >> 24); - h = h ^ g; - } - } - return h /* % M */; -} - - -/* assumes that key is not already present! */ -static void state_insert( - HashTab *hashtab, - unsigned char *key, - int state -) { - int i; - HashEntry *e; - - i = hnj_string_hash (key) % HASH_SIZE; - e = hnj_malloc (sizeof(HashEntry)); - e->next = hashtab->entries[i]; - e->key = key; - e->u.state = state; - hashtab->entries[i] = e; -} - - -/* assumes that key is not already present! */ -static void hyppat_insert( - HashTab *hashtab, - unsigned char *key, - char* hyppat -) { - int i; - HashEntry *e; - - i = hnj_string_hash (key) % HASH_SIZE; - for (e = hashtab->entries[i]; e; e=e->next) { - if (strcmp((char*)e->key,(char*)key)==0) { - if (e->u.hyppat) { - if (hyppat && strcmp((char*)e->u.hyppat,(char*)hyppat)!=0) { - int s = maketexstring("Conflicting pattern ignored"); - do_print_err(s); - error(); - flush_str(s); - } - hnj_free(e->u.hyppat); - } - e->u.hyppat = hyppat; - hnj_free(key); - return; - } - } - e = hnj_malloc (sizeof(HashEntry)); - e->next = hashtab->entries[i]; - e->key = key; - e->u.hyppat = hyppat; - hashtab->entries[i] = e; -} - - -/* return state if found, otherwise -1 */ -static int state_lookup( - HashTab *hashtab, - const unsigned char *key -) { - int i; - HashEntry *e; - - i = hnj_string_hash (key) % HASH_SIZE; - for (e = hashtab->entries[i]; e; e = e->next) { - if (!strcmp ((const char*)key, (const char*)e->key)) { - return e->u.state; - } - } - return -1; -} - - -/* return state if found, otherwise -1 */ -static char* hyppat_lookup( - HashTab *hashtab, - const unsigned char *chars, - int l -) { - int i; - HashEntry *e; - unsigned char key[128]; /* should be ample*/ - strncpy((char*)key,(const char*)chars,l); key[l]=0; - i = hnj_string_hash (key) % HASH_SIZE; - for (e = hashtab->entries[i]; e; e = e->next) { - if (!strcmp ((char*)key, (char*)e->key)) { - return e->u.hyppat; - } - } - return NULL; -} - - -/* Get the state number, allocating a new state if necessary. */ -static int hnj_get_state( - HyphenDict *dict, - const unsigned char *string, - int *state_num -) { - *state_num = state_lookup(dict->state_num, string); - - if (*state_num >= 0) - return *state_num; - - state_insert(dict->state_num, hnj_strdup(string), dict->num_states); - /* predicate is true if dict->num_states is a power of two */ - if (!(dict->num_states & (dict->num_states - 1))) { - dict->states = hnj_realloc( - dict->states, - (dict->num_states << 1) * sizeof(HyphenState)); - } - dict->states[dict->num_states].match = NULL; - dict->states[dict->num_states].fallback_state = -1; - dict->states[dict->num_states].num_trans = 0; - dict->states[dict->num_states].trans = NULL; - return dict->num_states++; -} - - -/* add a transition from state1 to state2 through ch - assumes that the - transition does not already exist */ -static void hnj_add_trans( - HyphenDict *dict, - int state1, - int state2, - int uni_ch -) { - int num_trans; - /* TH: this test was a bit too strict, it is quite normal for old - patterns to have chars in the range 0-31 or 127-159 (inclusive). - To ease the transition, let's only disallow NUL for now - (this is probably a requirement of the code anyway). - */ - if (uni_ch==0) { - fprintf(stderr,"Character out of bounds: u%04x \n",uni_ch); - exit(1); - } - num_trans = dict->states[state1].num_trans; - if (num_trans == 0) { - dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans)); - } else if (!(num_trans & (num_trans - 1))) { - dict->states[state1].trans = hnj_realloc( - dict->states[state1].trans, - (num_trans << 1) * sizeof(HyphenTrans)); - } - dict->states[state1].trans[num_trans].uni_ch = uni_ch; - dict->states[state1].trans[num_trans].new_state = state2; - dict->states[state1].num_trans++; -} - - -#ifdef VERBOSE - -static unsigned char *get_state_str( - int state -) { - int i; - HashEntry *e; - - for (i = 0; i < HASH_SIZE; i++) - for (e = global->entries[i]; e; e = e->next) - if (e->u.state == state) - return e->key; - return NULL; -} -#endif - - -/* I've changed the semantics a bit here: hnj_hyphen_load used to - operate on a file, but now the argument is a string buffer. - */ - -static const unsigned char* next_pattern( - size_t* length, - const unsigned char **buf -) { - const unsigned char *here, *rover = *buf; - while (*rover && isspace(*rover)) rover++; - here = rover; - while (*rover) { - if (isspace(*rover)) { - *length = rover-here; - *buf = rover; - return here; - } - rover++; - } - *length = rover-here; - *buf = rover; - return *length ? here : NULL; /* zero sensed */ -} - -static void init_hash( - HashTab**h -) { - int i; - if (*h) return; - *h = hnj_malloc(sizeof(HashTab)); - for (i = 0; i < HASH_SIZE; i++) (*h)->entries[i] = NULL; -} - - -static void clear_state_hash( - HashTab**h -) { - int i; - if (*h==NULL) return; - for (i = 0; i < HASH_SIZE; i++) { - HashEntry *e, *next; - for (e = (*h)->entries[i]; e; e = next) { - next = e->next; - hnj_free (e->key); - hnj_free (e); - } - } - hnj_free(*h); - *h=NULL; -} - - -static void clear_hyppat_hash( - HashTab**h -) { - int i; - if (*h==NULL) return; - for (i = 0; i < HASH_SIZE; i++) { - HashEntry *e, *next; - for (e = (*h)->entries[i]; e; e = next) { - next = e->next; - hnj_free(e->key); - if (e->u.hyppat) hnj_free(e->u.hyppat); - hnj_free(e); - } - } - hnj_free(*h); - *h=NULL; -} - - -static void init_dict( - HyphenDict* dict -) { - dict->num_states = 1; - dict->pat_length = 0; - dict->states = hnj_malloc (sizeof(HyphenState)); - dict->states[0].match = NULL; - dict->states[0].fallback_state = -1; - dict->states[0].num_trans = 0; - dict->states[0].trans = NULL; - dict->patterns = NULL; - dict->merged = NULL; - dict->state_num = NULL; - init_hash(&dict->patterns); -} - - -static void clear_dict( - HyphenDict* dict -) { - int state_num; - for (state_num = 0; state_num < dict->num_states; state_num++) { - HyphenState *hstate = &dict->states[state_num]; - if (hstate->match) hnj_free (hstate->match); - if (hstate->trans) hnj_free (hstate->trans); - } - hnj_free (dict->states); - clear_hyppat_hash(&dict->patterns); - clear_hyppat_hash(&dict->merged); - clear_state_hash(&dict->state_num); -} - - - -HyphenDict* hnj_hyphen_new(void) { - HyphenDict* dict = hnj_malloc (sizeof(HyphenDict)); - init_dict(dict); - return dict; -} - - -void hnj_hyphen_clear( - HyphenDict* dict -) { - clear_dict(dict); - init_dict(dict); -} - - -void hnj_hyphen_free( - HyphenDict *dict -) { - clear_dict(dict); - hnj_free(dict); -} - -unsigned char* hnj_serialize( - HyphenDict* dict -) { - HashIter *v; - unsigned char* word; - char* pattern; - unsigned char* buf = hnj_malloc(dict->pat_length); - unsigned char* cur = buf; - v = new_HashIter(dict->patterns); - while (eachHash(v,&word,&pattern)) { - int i=0, e=0; - while(word[e+i]) { - if (pattern[i]!='0') *cur++ = (unsigned char) pattern[i]; - *cur++ = word[e+i++]; - while (is_utf8_follow(word[e+i])) *cur++ = word[i+e++]; - } - if (pattern[i]!='0') *cur++ = (unsigned char) pattern[i]; - *cur++ = ' '; - } - delete_HashIter(v); - *cur = 0; - return buf; -} - - -void hnj_free_serialize( - unsigned char* c -) { - hnj_free(c); -} - - -/* hyphenation pattern: - * signed bytes - * 0 indicates end (actually any negative number) - * : prio(1+),startpos,length,len1,[replace],len2,[replace] - * most basic example is: - * p n 0 0 0 - * for a hyphenation point between characters - */ - - -void hnj_hyphen_load( - HyphenDict* dict, - const unsigned char *f -) { - int state_num, last_state; - int i, j = 0; - int ch; - int found; - HashEntry *e; - HashIter *v; - unsigned char* word; - char* pattern; - size_t l = 0; - - - /***************************************/ - - const unsigned char* format; - const unsigned char* begin = f; - unsigned char *pat; - char *org; - while((format = next_pattern(&l,&f))!=NULL) { - int i,j,e; - /* - printf("%s\n",format); - char* repl = strnchr(format, '/',l); - int replindex = 0; - int replcut = 0; - if (repl) { - int clen = l-(repl-format); - l = repl-format; - char * index = strnchr(repl + 1, ',',clen); - if (index) { - char * index2 = strnchr(index + 1, ',',clen-(index-repl)); - if (index2) { - replindex = (signed char) atoi(index + 1) - 1; - replcut = (signed char) atoi(index2 + 1); - } - } else { - hnj_strchomp(repl + 1); - replindex = 0; - replcut = strlen(buf); - } - repl = hnj_strdup(repl + 1); - } - */ - for (i=0,j=0,e=0; (unsigned)i<l; i++) { - if (format[i]>='0'&&format[i]<='9') j++; - if (is_utf8_follow(format[i])) e++; - } - /* l-e => number of _characters_ not _bytes_*/ - /* l-j => number of pattern bytes */ - /* l-e-j => number of pattern characters*/ - pat = (unsigned char*) malloc(1+l-j); - org = ( char*) malloc(2+l-e-j); - /* remove hyphenation encoders (digits) from pat*/ - org[0] = '0'; - for (i=0,j=0,e=0; (unsigned)i<l; i++) { - unsigned char c = format[i]; - if (is_utf8_follow(c)) { - pat[j+e++] = c; - } else if (c<'0' || c>'9') { - pat[e+j++] = c; - org[j] = '0'; - } else { - org[j] = c; - } - } - pat[e+j] = 0; - org[j+1] = 0; - hyppat_insert(dict->patterns,pat,org); - } - dict->pat_length += (f-begin)+2; /* 2 for spurious spaces*/ - init_hash(&dict->merged); - v = new_HashIter(dict->patterns); - while (nextHash(v,&word)) { - int wordsize = strlen((char*)word); - int j,l; - for (l=1; l<=wordsize; l++) { - if (is_utf8_follow(word[l])) continue; /* Do not clip an utf8 sequence*/ - for (j=1; j<=l; j++) { - char *subpat_pat; - int i = l-j; - if (is_utf8_follow(word[i])) continue; /* Do not start halfway an utf8 sequence*/ - if ((subpat_pat = hyppat_lookup(dict->patterns,word+i,j))!=NULL) { - char* newpat_pat; - if ((newpat_pat = hyppat_lookup(dict->merged,word,l))==NULL) { - char *neworg; - unsigned char *newword=(unsigned char*)malloc(l+1); - int e=0; - strncpy((char*)newword, (char*)word,l); newword[l]=0; - for (i=0; i<l; i++) if (is_utf8_follow(newword[i])) e++; - neworg = malloc(l+2-e); - sprintf(neworg,"%0*d",l+1-e,0); /* fill with right amount of '0'*/ - hyppat_insert(dict->merged,newword,combine(neworg,subpat_pat)); - } else { - combine(newpat_pat,subpat_pat); - } - } - } - } - } - delete_HashIter(v); - - init_hash(&dict->state_num); - state_insert(dict->state_num, hnj_strdup((const unsigned char*)""), 0); - v = new_HashIter(dict->merged); - while (nextHashStealPattern(v,&word,&pattern)) { - static unsigned char mask[] = {0x3F,0x1F,0xF,0x7}; - int j = strlen((char*)word); -#ifdef VERBOSE - printf ("word %s pattern %s, j = %d\n", word, pattern, j); -#endif - state_num = hnj_get_state( dict, word, &found ); - dict->states[state_num].match = pattern; - - /* now, put in the prefix transitions */ - while (found < 0) { - j--; - last_state = state_num; - ch = word[j]; - if (ch>=0x80) { - int m; - int i=1; - while (is_utf8_follow(word[j-i])) i++; - ch = word[j-i] & mask[i]; - m = j-i; - while (i--) { - ch = (ch<<6)+(0x3F & word[j-i]); - } - j = m; - } - word[j] = '\0'; - state_num = hnj_get_state (dict, word, &found); - hnj_add_trans (dict, state_num, last_state, ch); - } - } - delete_HashIter(v); - clear_hyppat_hash(&dict->merged); - - /***************************************/ - - /* put in the fallback states */ - for (i = 0; i < HASH_SIZE; i++) { - for (e = dict->state_num->entries[i]; e; e = e->next) { - /* do not do state==0 otherwise things get confused*/ - if (e->u.state) { - for (j = 1; 1; j++) { - state_num = state_lookup(dict->state_num, e->key + j); - if (state_num >= 0) break; - } - dict->states[e->u.state].fallback_state = state_num; - } - } - } -#ifdef VERBOSE - for (i = 0; i < HASH_SIZE; i++) { - for (e = dict->state_num->entries[i]; e; e = e->next) { - printf ("%d string %s state %d, fallback=%d\n", i, e->key, e->u.state, - dict->states[e->u.state].fallback_state); - for (j = 0; j < dict->states[e->u.state].num_trans; j++) { - printf (" u%4x->%d\n", (int)dict->states[e->u.state].trans[j].uni_ch, - dict->states[e->u.state].trans[j].new_state); - } - } - } -#endif - clear_state_hash(&dict->state_num); -} - - -void hnj_hyphen_hyphenate( - HyphenDict *dict, - halfword first, - halfword last, - int length, - halfword left, - halfword right, - lang_variables *lan -) { - int char_num; - halfword here; - int state = 0; - /* +2 for dots at each end, +1 for points /outside/ characters*/ - int ext_word_len = length+2; - int hyphen_len = ext_word_len+1; - /*char *hyphens = hnj_malloc((hyphen_len*2)+1); */ /* LATER */ - char *hyphens = hnj_malloc(hyphen_len+1); - - /* Add a '.' to beginning and end to facilitate matching*/ - set_vlink(begin_point,first); - set_vlink(end_point,get_vlink(last)); - set_vlink(last,end_point); - - for (char_num = 0; char_num < hyphen_len; char_num++) { - /* hyphens[char_num*2] = '0'; */ /* LATER */ - /* hyphens[char_num*2+1] = '0'; */ /* LATER */ - hyphens[char_num] = '0'; - } - /*hyphens[hyphen_len*2] = 0; */ /* LATER */ - hyphens[hyphen_len] = 0; - - /* now, run the finite state machine */ - for (char_num=0, here=begin_point; here!=get_vlink(end_point); here=get_vlink(here)) { - - int ch; - if (here == begin_point || here == end_point) - ch='.'; - else - ch = get_lc_code(get_character(here)); - while (state!=-1) { - /* printf("%*s%s%c",char_num-strlen(get_state_str(state)),"",get_state_str(state),(char)ch);*/ - HyphenState *hstate = &dict->states[state]; - int k; - for (k = 0; k < hstate->num_trans; k++) { - if (hstate->trans[k].uni_ch == ch) { - char *match; - state = hstate->trans[k].new_state; - /* printf(" state %d\n",state);*/ - match = dict->states[state].match; - if (match) { - /* +2 because: - * 1 string length is one bigger than offset - * 1 hyphenation starts before first character - */ - int offset = char_num + 2 - strlen (match); - /* printf ("%*s%s\n", offset,"", match);*/ - int m; - for (m = 0; match[m]; m++) { - if (hyphens[offset+m] < match[m]) hyphens[offset+m] = match[m]; - } - } - goto try_next_letter; - } - } - state = hstate->fallback_state; - /* printf (" back to %d\n", state);*/ - } - /* nothing worked, let's go to the next character*/ - state = 0; -try_next_letter: ; - char_num++; - } - - /* restore the correct pointers*/ - set_vlink(last,get_vlink(end_point)); - - /* pattern is ^.^w^o^r^d^.^ word_len=4, ext_word_len=6, hyphens=7 - * check ^ ^ ^ so drop first two and stop after word_len-1 - */ - for (here=first,char_num=2; here!=left; here=get_vlink(here)) char_num++; - for (; here!=right; here=get_vlink(here)) { - if (hyphens[char_num] & 1) - here = insert_syllable_discretionary(here, lan); - char_num++; - } - hnj_free(hyphens); -} diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c b/Build/source/texk/web2c/luatexdir/lang/hyphen.h index 6072152f4f5..8e5e5785c93 100644 --- a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c +++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.h @@ -33,42 +33,28 @@ * MPL. * */ -/* wrappers for malloc */ +#ifndef __HYPHEN_H__ +# define __HYPHEN_H__ -#include <stdlib.h> -#include <stdio.h> +# ifdef __cplusplus +extern "C" { +# endif /* __cplusplus */ -#include "hnjalloc.h" + typedef struct _HyphenDict HyphenDict; -void * -hnj_malloc (int size) -{ - void *p; +# include "ptexlib.h" - p = malloc (size); - if (p == NULL) - { - fprintf (stderr, "can't allocate %d bytes\n", size); - exit (1); - } - return p; -} - -void * -hnj_realloc (void *p, int size) -{ - p = realloc (p, size); - if (p == NULL) - { - fprintf (stderr, "can't allocate %d bytes\n", size); - exit (1); - } - return p; -} + HyphenDict *hnj_hyphen_new(void); + void hnj_hyphen_load(HyphenDict * dict, const unsigned char *fn); + void hnj_hyphen_free(HyphenDict * dict); + void hnj_hyphen_clear(HyphenDict * dict); + void hnj_hyphen_hyphenate(HyphenDict * dict, halfword first, halfword last, + int size, halfword left, halfword right, + lang_variables * lan); + unsigned char *hnj_serialize(HyphenDict *); + void hnj_free_serialize(unsigned char *); -void -hnj_free (void *p) -{ - free (p); +# ifdef __cplusplus } - +# endif /* __cplusplus */ +#endif /* __HYPHEN_H__ */ diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.w b/Build/source/texk/web2c/luatexdir/lang/hyphen.w new file mode 100644 index 00000000000..f7bbe6bd843 --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.w @@ -0,0 +1,857 @@ +% Libhnj is dual licensed under LGPL and MPL. Boilerplate for both +% licenses follows. +% +% +% LibHnj - a library for high quality hyphenation and justification +% Copyright (C) 1998 Raph Levien, +% (C) 2001 ALTLinux, Moscow (http://www.alt-linux.org), +% (C) 2001 Peter Novodvorsky (nidd@@cs.msu.su) +% +% This library is free software; you can redistribute it and/or +% modify it under the terms of the GNU Library General Public +% License as published by the Free Software Foundation; either +% version 2 of the License, or (at your option) any later version. +% +% This library is distributed in the hope that it will be useful, +% but WITHOUT ANY WARRANTY; without even the implied warranty of +% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU +% Library General Public License for more details. +% +% You should have received a copy of the GNU Library General Public +% License along with this library; if not, write to the +% Free Software Foundation, Inc., 59 Temple Place - Suite 330, +% Boston, MA 02111-1307 USA. +% +% +% +% The contents of this file are subject to the Mozilla Public License +% Version 1.0 (the "MPL"); you may not use this file except in +% compliance with the MPL. You may obtain a copy of the MPL at +% http://www.mozilla.org/MPL/ +% +% Software distributed under the MPL is distributed on an "AS IS" basis, +% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL +% for the specific language governing rights and limitations under the +% MPL. + + +@ @c +#include <stdlib.h> /* for NULL, malloc */ +#include <stdio.h> /* for fprintf */ +#include <string.h> /* for strdup */ +#include <stdlib.h> /* for malloc used by substring inclusion */ + +#define MAXPATHS 40960 + +#ifdef UNX +# include <unistd.h> /* for exit */ +#endif + +#include <kpathsea/c-ctype.h> + +#define noVERBOSE + +#include "lang/hnjalloc.h" +#include "lang/hyphen.h" + +@ @c +static const char _svn_version[] = + "$Id: hyphen.w 3584 2010-04-02 17:45:55Z hhenkel $ " +"$URL: http://foundry.supelec.fr/svn/luatex/tags/beta-0.60.0/source/texk/web2c/luatexdir/lang/hyphen.w $"; + +@ TODO: should be moved to separate library + +@c +static unsigned char *hnj_strdup(const unsigned char *s) +{ + unsigned char *new; + size_t l; + + l = strlen((const char *) s); + new = hnj_malloc((int) l + 1); + memcpy(new, s, l); + new[l] = 0; + return new; +} + +static int is_utf8_follow(unsigned char c) +{ + if (c >= 0x80 && c < 0xC0) + return 1; + return 0; +} + +@* Type definitions. + +@ a little bit of a hash table implementation. This simply maps strings + to state numbers + +@c +typedef struct _HashTab HashTab; +typedef struct _HashEntry HashEntry; +typedef struct _HashIter HashIter; +typedef union _HashVal HashVal; + +/* A cheap, but effective, hack. */ +#define HASH_SIZE 31627 + +struct _HashTab { + HashEntry *entries[HASH_SIZE]; +}; + +union _HashVal { + int state; + char *hyppat; +}; + +struct _HashEntry { + HashEntry *next; + unsigned char *key; + HashVal u; +}; + +struct _HashIter { + HashEntry **e; + HashEntry *cur; + int ndx; +}; + +@ State machine + +@c +typedef struct _HyphenState HyphenState; +typedef struct _HyphenTrans HyphenTrans; +#define MAX_CHARS 256 +#define MAX_NAME 20 + +struct _HyphenDict { + int num_states; + int pat_length; + char cset[MAX_NAME]; + HyphenState *states; + HashTab *patterns; + HashTab *merged; + HashTab *state_num; +}; + +struct _HyphenState { + char *match; + /*char *repl; */ + /*signed char replindex; */ + /*signed char replcut; */ + int fallback_state; + int num_trans; + HyphenTrans *trans; +}; + +struct _HyphenTrans { + int uni_ch; + int new_state; +}; + + +@ Combine two right-aligned number patterns, 04000 + 020 becomes 04020 + +@c +static char *combine(char *expr, const char *subexpr) +{ + size_t l1 = strlen(expr); + size_t l2 = strlen(subexpr); + size_t off = l1 - l2; + unsigned j; + /* this works also for utf8 sequences because the substring is identical + to the last substring-length bytes of expr except for the (single byte) + hyphenation encoders + */ + for (j = 0; j < l2; j++) { + if (expr[off + j] < subexpr[j]) + expr[off + j] = subexpr[j]; + } + return expr; +} + + +@ ORIGINAL CODE +@c +HashIter *new_HashIter(HashTab * h) +{ + HashIter *i = hnj_malloc(sizeof(HashIter)); + i->e = h->entries; + i->cur = NULL; + i->ndx = -1; + return i; +} + + +int nextHashStealPattern(HashIter * i, unsigned char **word, char **pattern) +{ + while (i->cur == NULL) { + if (i->ndx >= HASH_SIZE - 1) + return 0; + i->cur = i->e[++i->ndx]; + } + *word = i->cur->key; + *pattern = i->cur->u.hyppat; + i->cur->u.hyppat = NULL; + i->cur = i->cur->next; + return 1; +} + + +int nextHash(HashIter * i, unsigned char **word) +{ + while (i->cur == NULL) { + if (i->ndx >= HASH_SIZE - 1) + return 0; + i->cur = i->e[++i->ndx]; + } + *word = i->cur->key; + i->cur = i->cur->next; + return 1; +} + + +int eachHash(HashIter * i, unsigned char **word, char **pattern) +{ + while (i->cur == NULL) { + if (i->ndx >= HASH_SIZE - 1) + return 0; + i->cur = i->e[++i->ndx]; + } + *word = i->cur->key; + *pattern = i->cur->u.hyppat; + i->cur = i->cur->next; + return 1; +} + + +void delete_HashIter(HashIter * i) +{ + hnj_free(i); +} + + +@ a |char*| hash function from ASU - adapted from Gtk+ + +@c +static unsigned int hnj_string_hash(const unsigned char *s) +{ + const unsigned char *p; + unsigned int h = 0, g; + + for (p = s; *p != '\0'; p += 1) { + h = (h << 4) + *p; + if ((g = (h & 0xf0000000))) { + h = h ^ (g >> 24); + h = h ^ g; + } + } + return h /* \% M */ ; +} + + +@ assumes that key is not already present! + +@c +static void state_insert(HashTab * hashtab, unsigned char *key, int state) +{ + int i; + HashEntry *e; + + i = (int) (hnj_string_hash(key) % HASH_SIZE); + e = hnj_malloc(sizeof(HashEntry)); + e->next = hashtab->entries[i]; + e->key = key; + e->u.state = state; + hashtab->entries[i] = e; +} + + +@ assumes that key is not already present! + +@c +static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat) +{ + int i; + HashEntry *e; + + i = (int) (hnj_string_hash(key) % HASH_SIZE); + for (e = hashtab->entries[i]; e; e = e->next) { + if (strcmp((char *) e->key, (char *) key) == 0) { + if (e->u.hyppat) { + if (hyppat + && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) { + print_err("Conflicting pattern ignored"); + error(); + } + hnj_free(e->u.hyppat); + } + e->u.hyppat = hyppat; + hnj_free(key); + return; + } + } + e = hnj_malloc(sizeof(HashEntry)); + e->next = hashtab->entries[i]; + e->key = key; + e->u.hyppat = hyppat; + hashtab->entries[i] = e; +} + + +@ return state if found, otherwise $-1$ + +@c +static int state_lookup(HashTab * hashtab, const unsigned char *key) +{ + int i; + HashEntry *e; + + i = (int) (hnj_string_hash(key) % HASH_SIZE); + for (e = hashtab->entries[i]; e; e = e->next) { + if (!strcmp((const char *) key, (const char *) e->key)) { + return e->u.state; + } + } + return -1; +} + + +@ return state if found, otherwise $-1$ + +@c +static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l) +{ + int i; + HashEntry *e; + unsigned char key[128]; /* should be ample */ + strncpy((char *) key, (const char *) chars, (size_t) l); + key[l] = 0; + i = (int) (hnj_string_hash(key) % HASH_SIZE); + for (e = hashtab->entries[i]; e; e = e->next) { + if (!strcmp((char *) key, (char *) e->key)) { + return e->u.hyppat; + } + } + return NULL; +} + + +@ Get the state number, allocating a new state if necessary. + +@c +static int hnj_get_state(HyphenDict * dict, + const unsigned char *str, int *state_num) +{ + *state_num = state_lookup(dict->state_num, str); + + if (*state_num >= 0) + return *state_num; + + state_insert(dict->state_num, hnj_strdup(str), dict->num_states); + /* predicate is true if |dict->num_states| is a power of two */ + if (!(dict->num_states & (dict->num_states - 1))) { + dict->states = hnj_realloc(dict->states, + (int) ((dict->num_states << 1) * + (int) sizeof(HyphenState))); + } + dict->states[dict->num_states].match = NULL; + dict->states[dict->num_states].fallback_state = -1; + dict->states[dict->num_states].num_trans = 0; + dict->states[dict->num_states].trans = NULL; + return dict->num_states++; +} + + +@ Add a transition from state1 to state2 through ch - assumes that the + transition does not already exist + +@c +static void hnj_add_trans(HyphenDict * dict, int state1, int state2, int uni_ch) +{ + int num_trans; + /* TH: this test was a bit too strict, it is quite normal for old + patterns to have chars in the range 0-31 or 127-159 (inclusive). + To ease the transition, let's only disallow NUL for now + (this is probably a requirement of the code anyway). + */ + if (uni_ch == 0) { + fprintf(stderr, "Character out of bounds: u%04x \n", uni_ch); + exit(1); + } + num_trans = dict->states[state1].num_trans; + if (num_trans == 0) { + dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans)); + } else if (!(num_trans & (num_trans - 1))) { + dict->states[state1].trans = hnj_realloc(dict->states[state1].trans, + (int) (num_trans << 1 * + sizeof(HyphenTrans))); + } + dict->states[state1].trans[num_trans].uni_ch = uni_ch; + dict->states[state1].trans[num_trans].new_state = state2; + dict->states[state1].num_trans++; +} + + +#ifdef VERBOSE + +static unsigned char *get_state_str(int state) +{ + int i; + HashEntry *e; + + for (i = 0; i < HASH_SIZE; i++) + for (e = global->entries[i]; e; e = e->next) + if (e->u.state == state) + return e->key; + return NULL; +} +#endif + + +@ I've changed the semantics a bit here: |hnj_hyphen_load| used to + operate on a file, but now the argument is a string buffer. + +@c +static const unsigned char *next_pattern(size_t * length, + const unsigned char **buf) +{ + const unsigned char *here, *rover = *buf; + while (*rover && isspace(*rover)) + rover++; + here = rover; + while (*rover) { + if (isspace(*rover)) { + *length = (size_t) (rover - here); + *buf = rover; + return here; + } + rover++; + } + *length = (size_t) (rover - here); + *buf = rover; + return *length ? here : NULL; /* zero sensed */ +} + +static void init_hash(HashTab ** h) +{ + int i; + if (*h) + return; + *h = hnj_malloc(sizeof(HashTab)); + for (i = 0; i < HASH_SIZE; i++) + (*h)->entries[i] = NULL; +} + + +static void clear_state_hash(HashTab ** h) +{ + int i; + if (*h == NULL) + return; + for (i = 0; i < HASH_SIZE; i++) { + HashEntry *e, *next; + for (e = (*h)->entries[i]; e; e = next) { + next = e->next; + hnj_free(e->key); + hnj_free(e); + } + } + hnj_free(*h); + *h = NULL; +} + + +static void clear_hyppat_hash(HashTab ** h) +{ + int i; + if (*h == NULL) + return; + for (i = 0; i < HASH_SIZE; i++) { + HashEntry *e, *next; + for (e = (*h)->entries[i]; e; e = next) { + next = e->next; + hnj_free(e->key); + if (e->u.hyppat) + hnj_free(e->u.hyppat); + hnj_free(e); + } + } + hnj_free(*h); + *h = NULL; +} + + +static void init_dict(HyphenDict * dict) +{ + dict->num_states = 1; + dict->pat_length = 0; + dict->states = hnj_malloc(sizeof(HyphenState)); + dict->states[0].match = NULL; + dict->states[0].fallback_state = -1; + dict->states[0].num_trans = 0; + dict->states[0].trans = NULL; + dict->patterns = NULL; + dict->merged = NULL; + dict->state_num = NULL; + init_hash(&dict->patterns); +} + + +static void clear_dict(HyphenDict * dict) +{ + int state_num; + for (state_num = 0; state_num < dict->num_states; state_num++) { + HyphenState *hstate = &dict->states[state_num]; + if (hstate->match) + hnj_free(hstate->match); + if (hstate->trans) + hnj_free(hstate->trans); + } + hnj_free(dict->states); + clear_hyppat_hash(&dict->patterns); + clear_hyppat_hash(&dict->merged); + clear_state_hash(&dict->state_num); +} + + + +HyphenDict *hnj_hyphen_new(void) +{ + HyphenDict *dict = hnj_malloc(sizeof(HyphenDict)); + init_dict(dict); + return dict; +} + + +void hnj_hyphen_clear(HyphenDict * dict) +{ + clear_dict(dict); + init_dict(dict); +} + + +void hnj_hyphen_free(HyphenDict * dict) +{ + clear_dict(dict); + hnj_free(dict); +} + +unsigned char *hnj_serialize(HyphenDict * dict) +{ + HashIter *v; + unsigned char *word; + char *pattern; + unsigned char *buf = hnj_malloc(dict->pat_length); + unsigned char *cur = buf; + v = new_HashIter(dict->patterns); + while (eachHash(v, &word, &pattern)) { + int i = 0, e = 0; + while (word[e + i]) { + if (pattern[i] != '0') + *cur++ = (unsigned char) pattern[i]; + *cur++ = word[e + i++]; + while (is_utf8_follow(word[e + i])) + *cur++ = word[i + e++]; + } + if (pattern[i] != '0') + *cur++ = (unsigned char) pattern[i]; + *cur++ = ' '; + } + delete_HashIter(v); + *cur = 0; + return buf; +} + + +void hnj_free_serialize(unsigned char *c) +{ + hnj_free(c); +} + + +@ hyphenation pattern: + +signed bytes + +0 indicates end (actually any negative number) + +: prio(1+),startpos,length,len1,[replace],len2,[replace] + +most basic example is: + +p n 0 0 0 + +for a hyphenation point between characters + + +@c +void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f) +{ + int state_num, last_state; + int i, j = 0; + int ch; + int found; + HashEntry *e; + HashIter *v; + unsigned char *word; + char *pattern; + size_t l = 0; + + const unsigned char *format; + const unsigned char *begin = f; + unsigned char *pat; + char *org; + while ((format = next_pattern(&l, &f)) != NULL) { + int i, j, e; +#if 0 + printf("%s\n",format); + char* repl = strnchr(format, '/',l); + int replindex = 0; + int replcut = 0; + if (repl) { + int clen = l-(repl-format); + l = repl-format; + char * index = strnchr(repl + 1, ',',clen); + if (index) { + char * index2 = strnchr(index + 1, ',',clen-(index-repl)); + if (index2) { + replindex = (signed char) atoi(index + 1) - 1; + replcut = (signed char) atoi(index2 + 1); + } + } else { + hnj_strchomp(repl + 1); + replindex = 0; + replcut = strlen(buf); + } + repl = hnj_strdup(repl + 1); + } +#endif + for (i = 0, j = 0, e = 0; (unsigned) i < l; i++) { + if (format[i] >= '0' && format[i] <= '9') + j++; + if (is_utf8_follow(format[i])) + e++; + } + /* |l-e| => number of {\it characters} not {\it bytes} */ + /* |l-j| => number of pattern bytes */ + /* |l-e-j| => number of pattern characters */ + pat = (unsigned char *) malloc((1 + l - (size_t) j)); + org = (char *) malloc((size_t) (2 + l - (size_t) e - (size_t) j)); + /* remove hyphenation encoders (digits) from pat */ + org[0] = '0'; + for (i = 0, j = 0, e = 0; (unsigned) i < l; i++) { + unsigned char c = format[i]; + if (is_utf8_follow(c)) { + pat[j + e++] = c; + } else if (c < '0' || c > '9') { + pat[e + j++] = c; + org[j] = '0'; + } else { + org[j] = (char) c; + } + } + pat[e + j] = 0; + org[j + 1] = 0; + hyppat_insert(dict->patterns, pat, org); + } + dict->pat_length += (int) ((f - begin) + 2); /* 2 for spurious spaces */ + init_hash(&dict->merged); + v = new_HashIter(dict->patterns); + while (nextHash(v, &word)) { + int wordsize = (int) strlen((char *) word); + int j, l; + for (l = 1; l <= wordsize; l++) { + if (is_utf8_follow(word[l])) + continue; /* Do not clip an utf8 sequence */ + for (j = 1; j <= l; j++) { + char *subpat_pat; + int i = l - j; + if (is_utf8_follow(word[i])) + continue; /* Do not start halfway an utf8 sequence */ + if ((subpat_pat = + hyppat_lookup(dict->patterns, word + i, j)) != NULL) { + char *newpat_pat; + if ((newpat_pat = + hyppat_lookup(dict->merged, word, l)) == NULL) { + char *neworg; + unsigned char *newword = + (unsigned char *) malloc((size_t) (l + 1)); + int e = 0; + strncpy((char *) newword, (char *) word, (size_t) l); + newword[l] = 0; + for (i = 0; i < l; i++) + if (is_utf8_follow(newword[i])) + e++; + neworg = malloc((size_t) (l + 2 - e)); + sprintf(neworg, "%0*d", l + 1 - e, 0); /* fill with right amount of '0' */ + hyppat_insert(dict->merged, newword, + combine(neworg, subpat_pat)); + } else { + combine(newpat_pat, subpat_pat); + } + } + } + } + } + delete_HashIter(v); + + init_hash(&dict->state_num); + state_insert(dict->state_num, hnj_strdup((const unsigned char *) ""), 0); + v = new_HashIter(dict->merged); + while (nextHashStealPattern(v, &word, &pattern)) { + static unsigned char mask[] = { 0x3F, 0x1F, 0xF, 0x7 }; + int j = (int) strlen((char *) word); +#ifdef VERBOSE + printf("word %s pattern %s, j = %d\n", word, pattern, j); +#endif + state_num = hnj_get_state(dict, word, &found); + dict->states[state_num].match = pattern; + + /* now, put in the prefix transitions */ + while (found < 0) { + j--; + last_state = state_num; + ch = word[j]; + if (ch >= 0x80) { + int m; + int i = 1; + while (is_utf8_follow(word[j - i])) + i++; + ch = word[j - i] & mask[i]; + m = j - i; + while (i--) { + ch = (ch << 6) + (0x3F & word[j - i]); + } + j = m; + } + word[j] = '\0'; + state_num = hnj_get_state(dict, word, &found); + hnj_add_trans(dict, state_num, last_state, ch); + } + } + delete_HashIter(v); + clear_hyppat_hash(&dict->merged); + + /* put in the fallback states */ + for (i = 0; i < HASH_SIZE; i++) { + for (e = dict->state_num->entries[i]; e; e = e->next) { + /* do not do state==0 otherwise things get confused */ + if (e->u.state) { + for (j = 1; 1; j++) { + state_num = state_lookup(dict->state_num, e->key + j); + if (state_num >= 0) + break; + } + dict->states[e->u.state].fallback_state = state_num; + } + } + } +#ifdef VERBOSE + for (i = 0; i < HASH_SIZE; i++) { + for (e = dict->state_num->entries[i]; e; e = e->next) { + printf("%d string %s state %d, fallback=%d\n", i, e->key, + e->u.state, dict->states[e->u.state].fallback_state); + for (j = 0; j < dict->states[e->u.state].num_trans; j++) { + printf(" u%4x->%d\n", + (int) dict->states[e->u.state].trans[j].uni_ch, + dict->states[e->u.state].trans[j].new_state); + } + } + } +#endif + clear_state_hash(&dict->state_num); +} + +@ @c +void hnj_hyphen_hyphenate(HyphenDict * dict, + halfword first, + halfword last, + int length, + halfword left, halfword right, lang_variables * lan) +{ + int char_num; + halfword here; + int state = 0; + /* +2 for dots at each end, +1 for points /outside/ characters */ + int ext_word_len = length + 2; + int hyphen_len = ext_word_len + 1; + char *hyphens = hnj_malloc(hyphen_len + 1); + + /* Add a '.' to beginning and end to facilitate matching */ + set_vlink(begin_point, first); + set_vlink(end_point, get_vlink(last)); + set_vlink(last, end_point); + + for (char_num = 0; char_num < hyphen_len; char_num++) { + hyphens[char_num] = '0'; + } + hyphens[hyphen_len] = 0; + + /* now, run the finite state machine */ + for (char_num = 0, here = begin_point; here != get_vlink(end_point); + here = get_vlink(here)) { + + int ch; + if (here == begin_point || here == end_point) + ch = '.'; + else + ch = get_lc_code(get_character(here)); + while (state != -1) { +#if 0 + printf("%*s%s%c",char_num-strlen(get_state_str(state)),"",get_state_str(state),(char)ch); +#endif + HyphenState *hstate = &dict->states[state]; + int k; + for (k = 0; k < hstate->num_trans; k++) { + if (hstate->trans[k].uni_ch == ch) { + char *match; + state = hstate->trans[k].new_state; +#if 0 + printf(" state %d\n",state); +#endif + match = dict->states[state].match; + if (match) { + /* +2 because: + 1 string length is one bigger than offset + 1 hyphenation starts before first character + */ + int offset = (int) (char_num + 2 - (int) strlen(match)); +#if 0 + printf ("%*s%s\n", offset,"", match); +#endif + int m; + for (m = 0; match[m]; m++) { + if (hyphens[offset + m] < match[m]) + hyphens[offset + m] = match[m]; + } + } + goto try_next_letter; + } + } + state = hstate->fallback_state; +#if 0 + printf (" back to %d\n", state); +#endif + } + /* nothing worked, let's go to the next character */ + state = 0; + try_next_letter:; + char_num++; + } + + /* restore the correct pointers */ + set_vlink(last, get_vlink(end_point)); + + /* pattern is \.{\^.\^w\^o\^r\^d\^.\^} |word_len|=4, |ext_word_len|=6, |hyphens|=7 + * check \.{ \^ \^ \^ } so drop first two and stop after |word_len-1| + */ + for (here = first, char_num = 2; here != left; here = get_vlink(here)) + char_num++; + for (; here != right; here = get_vlink(here)) { + if (hyphens[char_num] & 1) + here = insert_syllable_discretionary(here, lan); + char_num++; + } + hnj_free(hyphens); +} diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.h b/Build/source/texk/web2c/luatexdir/lang/texlang.h new file mode 100644 index 00000000000..b312dab1a3a --- /dev/null +++ b/Build/source/texk/web2c/luatexdir/lang/texlang.h @@ -0,0 +1,82 @@ +/* texlang.h + + Copyright 2009 Taco Hoekwater <taco@luatex.org> + + This file is part of LuaTeX. + + LuaTeX is free software; you can redistribute it and/or modify it under + the terms of the GNU General Public License as published by the Free + Software Foundation; either version 2 of the License, or (at your + option) any later version. + + LuaTeX is distributed in the hope that it will be useful, but WITHOUT + ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public + License for more details. + + You should have received a copy of the GNU General Public License along + with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */ + +/* $Id: texlang.h 3394 2010-01-26 14:20:46Z taco $ */ + +#ifndef TEXLANG_H +# define TEXLANG_H + +typedef struct _lang_variables { + int pre_hyphen_char; + int post_hyphen_char; + int pre_exhyphen_char; + int post_exhyphen_char; +} lang_variables; + +# include "lang/hyphen.h" + +struct tex_language { + HyphenDict *patterns; + int exceptions; /* lua registry pointer, should be replaced */ + int id; + int pre_hyphen_char; + int post_hyphen_char; + int pre_exhyphen_char; + int post_exhyphen_char; +}; + +# define MAX_WORD_LEN 256 /* in chars */ + +extern struct tex_language *new_language(int n); +extern struct tex_language *get_language(int n); +extern void load_patterns(struct tex_language *lang, const unsigned char *buf); +extern void load_hyphenation(struct tex_language *lang, + const unsigned char *buf); +extern int hyphenate_string(struct tex_language *lang, char *w, char **ret); + +extern void new_hyphenation(halfword h, halfword t); +extern void clear_patterns(struct tex_language *lang); +extern void clear_hyphenation(struct tex_language *lang); +extern const char *clean_hyphenation(const char *buffer, char **cleaned); +extern void hnj_hyphenation(halfword head, halfword tail); + +extern void set_pre_hyphen_char(int lan, int val); +extern void set_post_hyphen_char(int lan, int val); +extern int get_pre_hyphen_char(int lan); +extern int get_post_hyphen_char(int lan); + +extern void set_pre_exhyphen_char(int lan, int val); +extern void set_post_exhyphen_char(int lan, int val); +extern int get_pre_exhyphen_char(int lan); +extern int get_post_exhyphen_char(int lan); +extern halfword compound_word_break(halfword t, int clang); + +extern void dump_language_data(void); +extern void undump_language_data(void); +extern char *exception_strings(struct tex_language *lang); + +extern void new_hyph_exceptions(void); +extern void new_patterns(void); +extern void new_pre_hyphen_char(void); +extern void new_post_hyphen_char(void); +extern void new_pre_exhyphen_char(void); +extern void new_post_exhyphen_char(void); + + +#endif diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.c b/Build/source/texk/web2c/luatexdir/lang/texlang.w index 5a17b2c34b6..6fcb7180820 100644 --- a/Build/source/texk/web2c/luatexdir/lang/texlang.c +++ b/Build/source/texk/web2c/luatexdir/lang/texlang.w @@ -1,105 +1,112 @@ -/* texlang.c - - Copyright 2006-2008 Taco Hoekwater <taco@luatex.org> +% texlang.w +% +% Copyright 2006-2010 Taco Hoekwater <taco@@luatex.org> - This file is part of LuaTeX. +% This file is part of LuaTeX. - LuaTeX is free software; you can redistribute it and/or modify it under - the terms of the GNU General Public License as published by the Free - Software Foundation; either version 2 of the License, or (at your - option) any later version. +% LuaTeX is free software; you can redistribute it and/or modify it under +% the terms of the GNU General Public License as published by the Free +% Software Foundation; either version 2 of the License, or (at your +% option) any later version. - LuaTeX is distributed in the hope that it will be useful, but WITHOUT - ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public - License for more details. +% LuaTeX is distributed in the hope that it will be useful, but WITHOUT +% ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +% FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public +% License for more details. - You should have received a copy of the GNU General Public License along - with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */ +% You should have received a copy of the GNU General Public License along +% with LuaTeX; if not, see <http://www.gnu.org/licenses/>. -#include "luatex-api.h" -#include <ptexlib.h> +@ @c +#include "ptexlib.h" #include <string.h> -#include "nodes.h" -#include "font/texfont.h" -#include "hyphen.h" +#include "lua/luatex-api.h" + static const char _svn_version[] = - "$Id: texlang.c 2596 2009-06-25 09:26:59Z taco $ $URL: http://scm.foundry.supelec.fr/svn/luatex/trunk/src/texk/web2c/luatexdir/lang/texlang.c $"; + "$Id: texlang.w 3584 2010-04-02 17:45:55Z hhenkel $ " +"$URL: http://foundry.supelec.fr/svn/luatex/tags/beta-0.60.0/source/texk/web2c/luatexdir/lang/texlang.w $"; + + +@ Low-level helpers -/* functions copied from the fontforge unicode library */ +@c +static char *uni2string(char *utf8_text, unsigned ch) +{ + /* Increment and deposit character */ + if (ch >= 17 * 65536) + return (utf8_text); + + if (ch <= 127) + *utf8_text++ = (char) ch; + else if (ch <= 0x7ff) { + *utf8_text++ = (char) (0xc0 | (ch >> 6)); + *utf8_text++ = (char) (0x80 | (ch & 0x3f)); + } else if (ch <= 0xffff) { + *utf8_text++ = (char) (0xe0 | (ch >> 12)); + *utf8_text++ = (char) (0x80 | ((ch >> 6) & 0x3f)); + *utf8_text++ = (char) (0x80 | (ch & 0x3f)); + } else { + unsigned val = ch - 0x10000; + unsigned u = ((val & 0xf0000) >> 16) + 1, z = (val & 0x0f000) >> 12, y = + (val & 0x00fc0) >> 6, x = val & 0x0003f; + *utf8_text++ = (char) (0xf0 | (u >> 2)); + *utf8_text++ = (char) (0x80 | ((u & 3) << 4) | z); + *utf8_text++ = (char) (0x80 | y); + *utf8_text++ = (char) (0x80 | x); + } + return (utf8_text); +} -static unsigned int *utf82u_strcpy(unsigned int * ubuf, const char *utf8buf) +static unsigned u_length(register unsigned int *str) { - int len = strlen(utf8buf) + 1; + register unsigned len = 0; + while (*str++ != '\0') + ++len; + return (len); +} + + +static void utf82u_strcpy(unsigned int *ubuf, const char *utf8buf) +{ + int len = (int) strlen(utf8buf) + 1; unsigned int *upt = ubuf, *uend = ubuf + len - 1; - const unsigned char *pt = (const unsigned char *) utf8buf, *end = pt + strlen(utf8buf); + const unsigned char *pt = (const unsigned char *) utf8buf, *end = + pt + strlen(utf8buf); int w, w2; while (pt < end && *pt != '\0' && upt < uend) { if (*pt <= 127) *upt = *pt++; else if (*pt <= 0xdf) { - *upt = ((*pt & 0x1f) << 6) | (pt[1] & 0x3f); + *upt = (unsigned int) (((*pt & 0x1f) << 6) | (pt[1] & 0x3f)); pt += 2; } else if (*pt <= 0xef) { - *upt = ((*pt & 0xf) << 12) | ((pt[1] & 0x3f) << 6) | (pt[2] & 0x3f); + *upt = + (unsigned int) (((*pt & 0xf) << 12) | ((pt[1] & 0x3f) << 6) | + (pt[2] & 0x3f)); pt += 3; } else { w = (((*pt & 0x7) << 2) | ((pt[1] & 0x30) >> 4)) - 1; w = (w << 6) | ((pt[1] & 0xf) << 2) | ((pt[2] & 0x30) >> 4); w2 = ((pt[2] & 0xf) << 6) | (pt[3] & 0x3f); - *upt = w * 0x400 + w2 + 0x10000; + *upt = (unsigned int) (w * 0x400 + w2 + 0x10000); pt += 4; } ++upt; } *upt = '\0'; - return (ubuf); -} - -char *utf8_idpb(char *utf8_text, unsigned int ch) -{ - /* Increment and deposit character */ - if (ch >= 17 * 65536) - return (utf8_text); - - if (ch <= 127) - *utf8_text++ = ch; - else if (ch <= 0x7ff) { - *utf8_text++ = 0xc0 | (ch >> 6); - *utf8_text++ = 0x80 | (ch & 0x3f); - } else if (ch <= 0xffff) { - *utf8_text++ = 0xe0 | (ch >> 12); - *utf8_text++ = 0x80 | ((ch >> 6) & 0x3f); - *utf8_text++ = 0x80 | (ch & 0x3f); - } else { - unsigned int val = ch - 0x10000; - int u = ((val & 0xf0000) >> 16) + 1, z = (val & 0x0f000) >> 12, y = - (val & 0x00fc0) >> 6, x = val & 0x0003f; - *utf8_text++ = 0xf0 | (u >> 2); - *utf8_text++ = 0x80 | ((u & 3) << 4) | z; - *utf8_text++ = 0x80 | y; - *utf8_text++ = 0x80 | x; - } - return (utf8_text); -} - -static int u_strlen(register unsigned int * str) -{ - register int len = 0; - while (*str++ != '\0') - ++len; - return (len); } +@ @c #define noVERBOSE #define MAX_TEX_LANGUAGES 32768 static struct tex_language *tex_languages[MAX_TEX_LANGUAGES] = { NULL }; + static int next_lang_id = 0; struct tex_language *new_language(int n) @@ -114,12 +121,12 @@ struct tex_language *new_language(int n) } else { while (tex_languages[next_lang_id] != NULL) next_lang_id++; - l = next_lang_id++; + l = (unsigned) next_lang_id++; } if (l < (MAX_TEX_LANGUAGES - 1) && tex_languages[l] == NULL) { lang = xmalloc(sizeof(struct tex_language)); tex_languages[l] = lang; - lang->id = l; + lang->id = (int) l; lang->exceptions = 0; lang->patterns = NULL; lang->pre_hyphen_char = '-'; @@ -145,14 +152,15 @@ struct tex_language *get_language(int n) } } -void set_pre_hyphen_char(integer n, integer v) +@ @c +void set_pre_hyphen_char(int n, int v) { struct tex_language *l = get_language((int) n); if (l != NULL) l->pre_hyphen_char = (int) v; } -void set_post_hyphen_char(integer n, integer v) +void set_post_hyphen_char(int n, int v) { struct tex_language *l = get_language((int) n); if (l != NULL) @@ -160,14 +168,14 @@ void set_post_hyphen_char(integer n, integer v) } -void set_pre_exhyphen_char(integer n, integer v) +void set_pre_exhyphen_char(int n, int v) { struct tex_language *l = get_language((int) n); if (l != NULL) l->pre_exhyphen_char = (int) v; } -void set_post_exhyphen_char(integer n, integer v) +void set_post_exhyphen_char(int n, int v) { struct tex_language *l = get_language((int) n); if (l != NULL) @@ -175,40 +183,40 @@ void set_post_exhyphen_char(integer n, integer v) } - -integer get_pre_hyphen_char(integer n) +int get_pre_hyphen_char(int n) { struct tex_language *l = get_language((int) n); if (l == NULL) return -1; - return (integer) l->pre_hyphen_char; + return (int) l->pre_hyphen_char; } -integer get_post_hyphen_char(integer n) +int get_post_hyphen_char(int n) { struct tex_language *l = get_language((int) n); if (l == NULL) return -1; - return (integer) l->post_hyphen_char; + return (int) l->post_hyphen_char; } -integer get_pre_exhyphen_char(integer n) +int get_pre_exhyphen_char(int n) { struct tex_language *l = get_language((int) n); if (l == NULL) return -1; - return (integer) l->pre_exhyphen_char; + return (int) l->pre_exhyphen_char; } -integer get_post_exhyphen_char(integer n) +int get_post_exhyphen_char(int n) { struct tex_language *l = get_language((int) n); if (l == NULL) return -1; - return (integer) l->post_exhyphen_char; + return (int) l->post_exhyphen_char; } +@ @c void load_patterns(struct tex_language *lang, const unsigned char *buffer) { if (lang == NULL || buffer == NULL || strlen((const char *) buffer) == 0) @@ -231,12 +239,13 @@ void clear_patterns(struct tex_language *lang) void load_tex_patterns(int curlang, halfword head) { char *s = tokenlist_to_cstring(head, 1, NULL); - load_patterns(get_language(curlang), (const unsigned char *) s); + load_patterns(get_language(curlang), (unsigned char *) s); } +@ @c #define STORE_CHAR(x) do { \ - word[w] = x; \ + word[w] = (unsigned char)x; \ if (w<MAX_WORD_LEN) w++; \ } while (0) @@ -298,9 +307,11 @@ const char *clean_hyphenation(const char *buffer, char **cleaned) return s; } +@ @c void load_hyphenation(struct tex_language *lang, const unsigned char *buffer) { - const char *s, *value; + const char *s; + const char *value; char *cleaned; lua_State *L = Luas; if (lang == NULL) @@ -320,7 +331,7 @@ void load_hyphenation(struct tex_language *lang, const unsigned char *buffer) if (cleaned != NULL) { if ((s - value) > 0) { lua_pushstring(L, cleaned); - lua_pushlstring(L, value, (s - value)); + lua_pushlstring(L, value, (size_t) (s - value)); lua_rawset(L, -3); } free(cleaned); @@ -348,13 +359,14 @@ void clear_hyphenation(struct tex_language *lang) void load_tex_hyphenation(int curlang, halfword head) { char *s = tokenlist_to_cstring(head, 1, NULL); - load_hyphenation(get_language(curlang), (const unsigned char *) s); + load_hyphenation(get_language(curlang), (unsigned char *) s); } -/* TODO: clean this up. The delete_attribute_ref() statements are not very +@ TODO: clean this up. The |delete_attribute_ref()| statements are not very nice, but needed. Also, in the post-break, it would be nicer to get the - attribute list from vlink(n). No rush, as it is currently not used much. */ + attribute list from |vlink(n)|. No rush, as it is currently not used much. +@c halfword insert_discretionary(halfword t, halfword pre, halfword post, halfword replace) { @@ -454,6 +466,7 @@ halfword insert_word_discretionary(halfword t, lang_variables * lan) return insert_discretionary(t, pre, pos, null); } +@ @c halfword compound_word_break(halfword t, int clang) { int disc; @@ -486,7 +499,7 @@ halfword insert_character(halfword t, int c) return p; } - +@ @c void set_disc_field(halfword f, halfword t) { if (t != null) { @@ -500,7 +513,8 @@ void set_disc_field(halfword f, halfword t) -static char *hyphenation_exception(int exceptions, char *w) +@ @c +char *hyphenation_exception(int exceptions, char *w) { char *ret = NULL; lua_State *L = Luas; @@ -520,6 +534,7 @@ static char *hyphenation_exception(int exceptions, char *w) } +@ @c char *exception_strings(struct tex_language *lang) { const char *value; @@ -537,8 +552,11 @@ char *exception_strings(struct tex_language *lang) while (lua_next(L, -2) != 0) { value = lua_tolstring(L, -1, &l); if (current + 2 + l > size) { - ret = xrealloc(ret, (1.2 * size) + current + l + 1024); - size = (1.2 * size) + current + l + 1024; + ret = + xrealloc(ret, + (unsigned) ((size + size / 5) + current + l + + 1024)); + size = (size + size / 5) + current + l + 1024; } *(ret + current) = ' '; strcpy(ret + current + 1, value); @@ -550,20 +568,21 @@ char *exception_strings(struct tex_language *lang) } -/* the sequence from |wordstart| to |r| can contain only normal characters */ -/* it could be faster to modify a halfword pointer and return an integer */ +@ the sequence from |wordstart| to |r| can contain only normal characters +it could be faster to modify a halfword pointer and return an integer -static halfword find_exception_part(unsigned int *j, int *uword, int len) +@c +halfword find_exception_part(unsigned int *j, unsigned int *uword, int len) { halfword g = null, gg = null; - register int i = *j; - i++; /* this puts uword[i] on the '{' */ - while (i < len && uword[i + 1] != '}') { + register unsigned i = *j; + i++; /* this puts uword[i] on the |{| */ + while (i < (unsigned) len && uword[i + 1] != '}') { if (g == null) { - gg = new_char(0, uword[i + 1]); + gg = new_char(0, (int) uword[i + 1]); g = gg; } else { - halfword s = new_char(0, uword[i + 1]); + halfword s = new_char(0, (int) uword[i + 1]); couple_nodes(g, s); g = vlink(g); } @@ -573,12 +592,12 @@ static halfword find_exception_part(unsigned int *j, int *uword, int len) return gg; } -static int count_exception_part(unsigned int *j, int *uword, int len) +int count_exception_part(unsigned int *j, unsigned int *uword, int len) { int ret = 0; - register int i = *j; - i++; /* this puts uword[i] on the '{' */ - while (i < len && uword[i + 1] != '}') { + register unsigned i = *j; + i++; /* this puts uword[i] on the |{| */ + while (i < (unsigned) len && uword[i + 1] != '}') { ret++; i++; } @@ -587,22 +606,23 @@ static int count_exception_part(unsigned int *j, int *uword, int len) } +@ @c static const char *PAT_ERROR[] = { "Exception discretionaries should contain three pairs of braced items.", "No intervening spaces are allowed.", NULL }; -static void do_exception(halfword wordstart, halfword r, char *replacement) +void do_exception(halfword wordstart, halfword r, char *replacement) { unsigned i; halfword t; unsigned len; int clang; lang_variables langdata; - int uword[MAX_WORD_LEN + 1] = { 0 }; - (void) utf82u_strcpy((unsigned int *) uword, replacement); - len = u_strlen((unsigned int *) uword); + unsigned uword[MAX_WORD_LEN + 1] = { 0 }; + utf82u_strcpy(uword, replacement); + len = u_length(uword); i = 0; t = wordstart; clang = char_lang(wordstart); @@ -624,15 +644,15 @@ static void do_exception(halfword wordstart, halfword r, char *replacement) } else if (uword[i + 1] == '{') { halfword gg, hh, replace = null; int repl; - gg = find_exception_part(&i, uword, len); + gg = find_exception_part(&i, uword, (int) len); if (i == len || uword[i + 1] != '{') { tex_error("broken pattern 1", PAT_ERROR); } - hh = find_exception_part(&i, uword, len); + hh = find_exception_part(&i, uword, (int) len); if (i == len || uword[i + 1] != '{') { tex_error("broken pattern 2", PAT_ERROR); } - repl = count_exception_part(&i, uword, len); + repl = count_exception_part(&i, uword, (int) len); if (i == len) { tex_error("broken pattern 3", PAT_ERROR); } @@ -659,7 +679,7 @@ static void do_exception(halfword wordstart, halfword r, char *replacement) } } -/* This is a documentation section from the pascal web file. It is not +@ This is a documentation section from the pascal web file. It is not true any more, but I do not have time right now to rewrite it -- Taco When the line-breaking routine is unable to find a feasible sequence of @@ -722,15 +742,14 @@ When a word been set up to contain a candidate for hyphenation, hyphens are inserted based on patterns that appear within the given word, using an algorithm due to Frank~M. Liang. @^Liang, Franklin Mark@> -*/ -/* - * This is incompatible with TEX because the first word of a paragraph - * can be hyphenated, but most european users seem to agree that - * prohibiting hyphenation there was not the best idea ever. - */ -static halfword find_next_wordstart(halfword r) +@ This is incompatible with TEX because the first word of a paragraph +can be hyphenated, but most european users seem to agree that +prohibiting hyphenation there was not the best idea ever. + +@c +halfword find_next_wordstart(halfword r) { register int l; register int start_ok = 1; @@ -770,27 +789,30 @@ static halfword find_next_wordstart(halfword r) return r; } -static int valid_wordend(halfword s) +@ @c +int valid_wordend(halfword s) { register halfword r = s; register int clang = char_lang(s); if (r == null) return 1; while ((r != null) && ((type(r) == glyph_node && is_simple_character(r) - && clang == char_lang(r)) || (type(r) == kern_node - && subtype(r) == - normal))) { + && clang == char_lang(r)) || + (type(r) == kern_node && (subtype(r) == normal)) + )) { r = vlink(r); } if (r == null || (type(r) == glyph_node && is_simple_character(r) && clang != char_lang(r)) || type(r) == glue_node || type(r) == whatsit_node || type(r) == ins_node || type(r) == adjust_node || type(r) == penalty_node - || (type(r) == kern_node && subtype(r) == explicit)) + || (type(r) == kern_node + && (subtype(r) == explicit || subtype(r) == acc_kern))) return 1; return 0; } +@ @c void hnj_hyphenation(halfword head, halfword tail) { int lchar, i; @@ -804,17 +826,17 @@ void hnj_hyphenation(halfword head, halfword tail) null, right = null; /* this first movement assures two things: - * a) that we won't waste lots of time on something that has been - * handled already (in that case, none of the glyphs match |simple_character|). - * b) that the first word can be hyphenated. if the movement was - * not explicit, then the indentation at the start of a paragraph - * list would make find_next_wordstart() look too far ahead. + \item{a)} that we won't waste lots of time on something that has been + handled already (in that case, none of the glyphs match |simple_character|). + \item{b)} that the first word can be hyphenated. if the movement was + not explicit, then the indentation at the start of a paragraph + list would make |find_next_wordstart()| look too far ahead. */ while (r != null && (type(r) != glyph_node || !is_simple_character(r))) { r = vlink(r); } - /* this will make |r| a glyph node with subtype_character */ + /* this will make |r| a glyph node with subtype character */ r = find_next_wordstart(r); if (r == null) return; @@ -844,7 +866,7 @@ void hnj_hyphenation(halfword head, halfword tail) clang == char_lang(r) && (lchar = get_lc_code(character(r))) > 0) { wordlen++; - hy = utf8_idpb(hy, lchar); + hy = uni2string(hy, (unsigned) lchar); /* this should not be needed any more */ /*if (vlink(r)!=null) alink(vlink(r))=r; */ end_word = r; @@ -897,12 +919,13 @@ void hnj_hyphenation(halfword head, halfword tail) } +@ @c void new_hyphenation(halfword head, halfword tail) { register int callback_id = 0; if (head == null || vlink(head) == null) return; - fix_node_list(head); /* TODO: use couple_nodes() in append_tail()! */ + fix_node_list(head); callback_id = callback_defined(hyphenate_callback); if (callback_id > 0) { lua_State *L = Luas; @@ -924,21 +947,22 @@ void new_hyphenation(halfword head, halfword tail) } } -/* dumping and undumping fonts */ +@ dumping and undumping languages +@c #define dump_string(a) \ if (a!=NULL) { \ - x = strlen(a)+1; \ + x = (int)strlen(a)+1; \ dump_int(x); dump_things(*a, x); \ } else { \ x = 0; dump_int(x); \ } -static void dump_one_language(int i) +void dump_one_language(int i) { char *s = NULL; - integer x = 0; + int x = 0; struct tex_language *lang; lang = tex_languages[i]; dump_int(lang->id); @@ -965,7 +989,7 @@ static void dump_one_language(int i) void dump_language_data(void) { - integer i; + int i; dump_int(next_lang_id); for (i = 0; i < next_lang_id; i++) { if (tex_languages[i]) { @@ -978,10 +1002,10 @@ void dump_language_data(void) } -static void undump_one_language(int i) +void undump_one_language(int i) { char *s = NULL; - integer x = 0; + int x = 0; struct tex_language *lang = get_language(i); undump_int(x); lang->id = x; @@ -996,24 +1020,24 @@ static void undump_one_language(int i) /* patterns */ undump_int(x); if (x > 0) { - s = xmalloc(x); + s = xmalloc((unsigned) x); undump_things(*s, x); - load_patterns(lang, (const unsigned char *) s); + load_patterns(lang, (unsigned char *) s); free(s); } /* exceptions */ undump_int(x); if (x > 0) { - s = xmalloc(x); + s = xmalloc((unsigned) x); undump_things(*s, x); - load_hyphenation(lang, (const unsigned char *) s); + load_hyphenation(lang, (unsigned char *) s); free(s); } } void undump_language_data(void) { - integer i, x, numlangs; + int i, x, numlangs; undump_int(numlangs); next_lang_id = numlangs; for (i = 0; i < numlangs; i++) { @@ -1023,3 +1047,65 @@ void undump_language_data(void) } } } + + +@ When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named +|new_hyph_exceptions| to do the right thing. + +@c +void new_hyph_exceptions(void) +{ /* enters new exceptions */ + (void) scan_toks(false, true); + load_tex_hyphenation(int_par(language_code), def_ref); + flush_list(def_ref); +} + +@ Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a +procedure named |new_patterns|. + +@c +void new_patterns(void) +{ /* initializes the hyphenation pattern data */ + (void) scan_toks(false, true); + load_tex_patterns(int_par(language_code), def_ref); + flush_list(def_ref); +} + +@ `\.{\\prehyphenchar}', sets the |pre_break| character, and +`\.{\\posthyphenchar}' the |post_break| character. Their respective +defaults are ascii hyphen ("-") and zero (nul). + +@c +void new_pre_hyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_pre_hyphen_char(int_par(language_code), cur_val); +} + +void new_post_hyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_post_hyphen_char(int_par(language_code), cur_val); +} + + +@ `\.{\\preexhyphenchar}', sets the |pre_break| character, and +`\.{\\postexhyphenchar}' the |post_break| character. Their +defaults are both zero (nul). + +@c +void new_pre_exhyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_pre_exhyphen_char(int_par(language_code), cur_val); +} + +void new_post_exhyphen_char(void) +{ + scan_optional_equals(); + scan_int(); + set_post_exhyphen_char(int_par(language_code), cur_val); +} |