summaryrefslogtreecommitdiff
path: root/Build/source/texk/web2c/luatexdir/lang
diff options
context:
space:
mode:
authorTaco Hoekwater <taco@elvenkind.com>2010-04-04 13:55:46 +0000
committerTaco Hoekwater <taco@elvenkind.com>2010-04-04 13:55:46 +0000
commitf372862c42d234710b6057449b46818e63181c90 (patch)
treea9ebec66dd31ac37aed442502ebd1b9fe76245ec /Build/source/texk/web2c/luatexdir/lang
parent2dad291ab056f30816e0aea61970f38033c2f6a5 (diff)
import luatex 0.60.0 (with small local changes, see luatexdir/ChangeLog)
and do an autoreconf git-svn-id: svn://tug.org/texlive/trunk@17680 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/lang')
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hnjalloc.h10
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hnjalloc.w66
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hyphen.c850
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hyphen.h (renamed from Build/source/texk/web2c/luatexdir/lang/hnjalloc.c)52
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hyphen.w857
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/texlang.h82
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/texlang.w (renamed from Build/source/texk/web2c/luatexdir/lang/texlang.c)368
7 files changed, 1254 insertions, 1031 deletions
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h
index db82fb9c758..1af5a3f0ea3 100644
--- a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h
+++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h
@@ -35,12 +35,8 @@
*/
/* wrappers for malloc */
-void *
-hnj_malloc (int size);
+void *hnj_malloc(int size);
-void *
-hnj_realloc (void *p, int size);
-
-void
-hnj_free (void *p);
+void *hnj_realloc(void *p, int size);
+void hnj_free(void *p);
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w
new file mode 100644
index 00000000000..c7d595cdd75
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.w
@@ -0,0 +1,66 @@
+% LibHnj is dual licensed under LGPL and MPL. Boilerplate for both
+% licenses follows.
+%
+%
+% LibHnj - a library for high quality hyphenation and justification
+% Copyright (C) 1998 Raph Levien, (C) 2001 ALTLinux, Moscow
+%
+% This library is free software; you can redistribute it and/or
+% modify it under the terms of the GNU Library General Public
+% License as published by the Free Software Foundation; either
+% version 2 of the License, or (at your option) any later version.
+%
+% This library is distributed in the hope that it will be useful,
+% but WITHOUT ANY WARRANTY; without even the implied warranty of
+% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+% Library General Public License for more details.
+%
+% You should have received a copy of the GNU Library General Public
+% License along with this library; if not, write to the
+% Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+% Boston, MA 02111-1307 USA.
+%
+%
+%
+% The contents of this file are subject to the Mozilla Public License
+% Version 1.0 (the "MPL"); you may not use this file except in
+% compliance with the MPL. You may obtain a copy of the MPL at
+% http://www.mozilla.org/MPL/
+%
+% Software distributed under the MPL is distributed on an "AS IS" basis,
+% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL
+% for the specific language governing rights and limitations under the
+% MPL.
+
+@ wrappers for malloc
+@c
+
+#include <stdlib.h>
+#include <stdio.h>
+
+void *hnj_malloc(int size)
+{
+ void *p;
+
+ p = malloc((size_t) size);
+ if (p == NULL) {
+ fprintf(stderr, "can't allocate %d bytes\n", size);
+ exit(1);
+ }
+ return p;
+}
+
+void *hnj_realloc(void *p, int size)
+{
+ p = realloc(p, (size_t) size);
+ if (p == NULL) {
+ fprintf(stderr, "can't allocate %d bytes\n", size);
+ exit(1);
+ }
+ return p;
+}
+
+void hnj_free(void *p)
+{
+ free(p);
+}
diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.c b/Build/source/texk/web2c/luatexdir/lang/hyphen.c
deleted file mode 100644
index edcad69adff..00000000000
--- a/Build/source/texk/web2c/luatexdir/lang/hyphen.c
+++ /dev/null
@@ -1,850 +0,0 @@
-/* Libhnj is dual licensed under LGPL and MPL. Boilerplate for both
- * licenses follows.
- */
-
-/* LibHnj - a library for high quality hyphenation and justification
- * Copyright (C) 1998 Raph Levien,
- * (C) 2001 ALTLinux, Moscow (http://www.alt-linux.org),
- * (C) 2001 Peter Novodvorsky (nidd@cs.msu.su)
- *
- * This library is free software; you can redistribute it and/or
- * modify it under the terms of the GNU Library General Public
- * License as published by the Free Software Foundation; either
- * version 2 of the License, or (at your option) any later version.
- *
- * This library is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
- * Library General Public License for more details.
- *
- * You should have received a copy of the GNU Library General Public
- * License along with this library; if not, write to the
- * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
- * Boston, MA 02111-1307 USA.
-*/
-
-/*
- * The contents of this file are subject to the Mozilla Public License
- * Version 1.0 (the "MPL"); you may not use this file except in
- * compliance with the MPL. You may obtain a copy of the MPL at
- * http://www.mozilla.org/MPL/
- *
- * Software distributed under the MPL is distributed on an "AS IS" basis,
- * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL
- * for the specific language governing rights and limitations under the
- * MPL.
- *
- */
-#include <stdlib.h> /* for NULL, malloc */
-#include <stdio.h> /* for fprintf */
-#include <string.h> /* for strdup */
-#include <stdlib.h> /* for malloc used by substring inclusion*/
-
-#define MAXPATHS 40960
-
-#ifdef UNX
-#include <unistd.h> /* for exit */
-#endif
-
-#include <ctype.h>
-
-/*#define VERBOSE*/
-
-#include "hnjalloc.h"
-#include "hyphen.h"
-
-static const char _svn_version[] =
- "$Id: hyphen.c 1696 2008-12-28 21:02:34Z oneiros $ $URL: http://scm.foundry.supelec.fr/svn/luatex/trunk/src/texk/web2c/luatexdir/lang/texlang.c $";
-
-/* SHOULD BE MOVED TO SEPARATE LIBRARY */
-static unsigned char * hnj_strdup(
- const unsigned char *s
-) {
- unsigned char *new;
- int l;
-
- l = strlen ((const char*)s);
- new = hnj_malloc (l + 1);
- memcpy (new, s, l);
- new[l] = 0;
- return new;
-}
-
-static int is_utf8_follow(
- unsigned char c
-) {
- if (c>=0x80 && c<0xC0) return 1;
- return 0;
-}
-
-/* --------------------------------------------------------------------
- *
- * Type definitions
- *
- * --------------------------------------------------------------------
- */
-
-/* a little bit of a hash table implementation. This simply maps strings
- to state numbers */
-
-typedef struct _HashTab HashTab;
-typedef struct _HashEntry HashEntry;
-typedef struct _HashIter HashIter;
-typedef union _HashVal HashVal;
-
-/* A cheap, but effective, hack. */
-#define HASH_SIZE 31627
-
-struct _HashTab {
- HashEntry *entries[HASH_SIZE];
-};
-
-union _HashVal {
- int state;
- char* hyppat;
-};
-
-struct _HashEntry {
- HashEntry *next;
- unsigned char *key;
- HashVal u;
-};
-
-struct _HashIter {
- HashEntry** e;
- HashEntry* cur;
- int ndx;
-};
-
-/* State machine */
-typedef struct _HyphenState HyphenState;
-typedef struct _HyphenTrans HyphenTrans;
-#define MAX_CHARS 256
-#define MAX_NAME 20
-
-struct _HyphenDict {
- int num_states;
- int pat_length;
- char cset[MAX_NAME];
- HyphenState *states;
- HashTab *patterns;
- HashTab *merged;
- HashTab *state_num;
-};
-
-struct _HyphenState {
- char *match;
- /*char *repl;*/
- /*signed char replindex;*/
- /*signed char replcut;*/
- int fallback_state;
- int num_trans;
- HyphenTrans *trans;
-};
-
-struct _HyphenTrans {
- int uni_ch;
- int new_state;
-};
-
-
-/* Combine two right-aligned number patterns, 04000 + 020 becomes 04020*/
-static char *combine(
- char *expr,
- const char *subexpr
-) {
- int l1 = strlen(expr);
- int l2 = strlen(subexpr);
- int off = l1-l2;
- int j;
- /* this works also for utf8 sequences because the substring is identical
- * to the last substring-length bytes of expr except for the (single byte)
- * hyphenation encoders
- */
- for (j=0; j<l2; j++) {
- if (expr[off+j]<subexpr[j]) expr[off+j] = subexpr[j];
- }
- return expr;
-}
-
-
-/* --------------------------------------------------------------------
- * ORIGINAL CODE
- * --------------------------------------------------------------------
- */
-
-static HashIter* new_HashIter(
- HashTab* h
-) {
- HashIter* i = hnj_malloc(sizeof(HashIter));
- i->e = h->entries;
- i->cur = NULL;
- i->ndx = -1;
- return i;
-}
-
-
-static int nextHashStealPattern(
- HashIter*i,
- unsigned char**word,
- char **pattern
-) {
- while (i->cur==NULL) {
- if (i->ndx >= HASH_SIZE-1) return 0;
- i->cur = i->e[++i->ndx];
- }
- *word = i->cur->key;
- *pattern = i->cur->u.hyppat;
- i->cur->u.hyppat = NULL;
- i->cur = i->cur->next;
- return 1;
-}
-
-
-static int nextHash(
- HashIter*i,
- unsigned char**word
-) {
- while (i->cur==NULL) {
- if (i->ndx >= HASH_SIZE-1) return 0;
- i->cur = i->e[++i->ndx];
- }
- *word = i->cur->key;
- i->cur = i->cur->next;
- return 1;
-}
-
-
-static int eachHash(
- HashIter*i,
- unsigned char**word,
- char**pattern
-) {
- while (i->cur==NULL) {
- if (i->ndx >= HASH_SIZE-1) return 0;
- i->cur = i->e[++i->ndx];
- }
- *word = i->cur->key;
- *pattern = i->cur->u.hyppat;
- i->cur = i->cur->next;
- return 1;
-}
-
-
-static void delete_HashIter(
- HashIter*i
-) {
- hnj_free(i);
-}
-
-
-/* a char* hash function from ASU - adapted from Gtk+ */
-static unsigned int hnj_string_hash (
- const unsigned char *s
-) {
- const unsigned char *p;
- unsigned int h=0, g;
-
- for (p = s; *p != '\0'; p += 1) {
- h = ( h << 4 ) + *p;
- if ( ( g = h & 0xf0000000 ) ) {
- h = h ^ (g >> 24);
- h = h ^ g;
- }
- }
- return h /* % M */;
-}
-
-
-/* assumes that key is not already present! */
-static void state_insert(
- HashTab *hashtab,
- unsigned char *key,
- int state
-) {
- int i;
- HashEntry *e;
-
- i = hnj_string_hash (key) % HASH_SIZE;
- e = hnj_malloc (sizeof(HashEntry));
- e->next = hashtab->entries[i];
- e->key = key;
- e->u.state = state;
- hashtab->entries[i] = e;
-}
-
-
-/* assumes that key is not already present! */
-static void hyppat_insert(
- HashTab *hashtab,
- unsigned char *key,
- char* hyppat
-) {
- int i;
- HashEntry *e;
-
- i = hnj_string_hash (key) % HASH_SIZE;
- for (e = hashtab->entries[i]; e; e=e->next) {
- if (strcmp((char*)e->key,(char*)key)==0) {
- if (e->u.hyppat) {
- if (hyppat && strcmp((char*)e->u.hyppat,(char*)hyppat)!=0) {
- int s = maketexstring("Conflicting pattern ignored");
- do_print_err(s);
- error();
- flush_str(s);
- }
- hnj_free(e->u.hyppat);
- }
- e->u.hyppat = hyppat;
- hnj_free(key);
- return;
- }
- }
- e = hnj_malloc (sizeof(HashEntry));
- e->next = hashtab->entries[i];
- e->key = key;
- e->u.hyppat = hyppat;
- hashtab->entries[i] = e;
-}
-
-
-/* return state if found, otherwise -1 */
-static int state_lookup(
- HashTab *hashtab,
- const unsigned char *key
-) {
- int i;
- HashEntry *e;
-
- i = hnj_string_hash (key) % HASH_SIZE;
- for (e = hashtab->entries[i]; e; e = e->next) {
- if (!strcmp ((const char*)key, (const char*)e->key)) {
- return e->u.state;
- }
- }
- return -1;
-}
-
-
-/* return state if found, otherwise -1 */
-static char* hyppat_lookup(
- HashTab *hashtab,
- const unsigned char *chars,
- int l
-) {
- int i;
- HashEntry *e;
- unsigned char key[128]; /* should be ample*/
- strncpy((char*)key,(const char*)chars,l); key[l]=0;
- i = hnj_string_hash (key) % HASH_SIZE;
- for (e = hashtab->entries[i]; e; e = e->next) {
- if (!strcmp ((char*)key, (char*)e->key)) {
- return e->u.hyppat;
- }
- }
- return NULL;
-}
-
-
-/* Get the state number, allocating a new state if necessary. */
-static int hnj_get_state(
- HyphenDict *dict,
- const unsigned char *string,
- int *state_num
-) {
- *state_num = state_lookup(dict->state_num, string);
-
- if (*state_num >= 0)
- return *state_num;
-
- state_insert(dict->state_num, hnj_strdup(string), dict->num_states);
- /* predicate is true if dict->num_states is a power of two */
- if (!(dict->num_states & (dict->num_states - 1))) {
- dict->states = hnj_realloc(
- dict->states,
- (dict->num_states << 1) * sizeof(HyphenState));
- }
- dict->states[dict->num_states].match = NULL;
- dict->states[dict->num_states].fallback_state = -1;
- dict->states[dict->num_states].num_trans = 0;
- dict->states[dict->num_states].trans = NULL;
- return dict->num_states++;
-}
-
-
-/* add a transition from state1 to state2 through ch - assumes that the
- transition does not already exist */
-static void hnj_add_trans(
- HyphenDict *dict,
- int state1,
- int state2,
- int uni_ch
-) {
- int num_trans;
- /* TH: this test was a bit too strict, it is quite normal for old
- patterns to have chars in the range 0-31 or 127-159 (inclusive).
- To ease the transition, let's only disallow NUL for now
- (this is probably a requirement of the code anyway).
- */
- if (uni_ch==0) {
- fprintf(stderr,"Character out of bounds: u%04x \n",uni_ch);
- exit(1);
- }
- num_trans = dict->states[state1].num_trans;
- if (num_trans == 0) {
- dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans));
- } else if (!(num_trans & (num_trans - 1))) {
- dict->states[state1].trans = hnj_realloc(
- dict->states[state1].trans,
- (num_trans << 1) * sizeof(HyphenTrans));
- }
- dict->states[state1].trans[num_trans].uni_ch = uni_ch;
- dict->states[state1].trans[num_trans].new_state = state2;
- dict->states[state1].num_trans++;
-}
-
-
-#ifdef VERBOSE
-
-static unsigned char *get_state_str(
- int state
-) {
- int i;
- HashEntry *e;
-
- for (i = 0; i < HASH_SIZE; i++)
- for (e = global->entries[i]; e; e = e->next)
- if (e->u.state == state)
- return e->key;
- return NULL;
-}
-#endif
-
-
-/* I've changed the semantics a bit here: hnj_hyphen_load used to
- operate on a file, but now the argument is a string buffer.
- */
-
-static const unsigned char* next_pattern(
- size_t* length,
- const unsigned char **buf
-) {
- const unsigned char *here, *rover = *buf;
- while (*rover && isspace(*rover)) rover++;
- here = rover;
- while (*rover) {
- if (isspace(*rover)) {
- *length = rover-here;
- *buf = rover;
- return here;
- }
- rover++;
- }
- *length = rover-here;
- *buf = rover;
- return *length ? here : NULL; /* zero sensed */
-}
-
-static void init_hash(
- HashTab**h
-) {
- int i;
- if (*h) return;
- *h = hnj_malloc(sizeof(HashTab));
- for (i = 0; i < HASH_SIZE; i++) (*h)->entries[i] = NULL;
-}
-
-
-static void clear_state_hash(
- HashTab**h
-) {
- int i;
- if (*h==NULL) return;
- for (i = 0; i < HASH_SIZE; i++) {
- HashEntry *e, *next;
- for (e = (*h)->entries[i]; e; e = next) {
- next = e->next;
- hnj_free (e->key);
- hnj_free (e);
- }
- }
- hnj_free(*h);
- *h=NULL;
-}
-
-
-static void clear_hyppat_hash(
- HashTab**h
-) {
- int i;
- if (*h==NULL) return;
- for (i = 0; i < HASH_SIZE; i++) {
- HashEntry *e, *next;
- for (e = (*h)->entries[i]; e; e = next) {
- next = e->next;
- hnj_free(e->key);
- if (e->u.hyppat) hnj_free(e->u.hyppat);
- hnj_free(e);
- }
- }
- hnj_free(*h);
- *h=NULL;
-}
-
-
-static void init_dict(
- HyphenDict* dict
-) {
- dict->num_states = 1;
- dict->pat_length = 0;
- dict->states = hnj_malloc (sizeof(HyphenState));
- dict->states[0].match = NULL;
- dict->states[0].fallback_state = -1;
- dict->states[0].num_trans = 0;
- dict->states[0].trans = NULL;
- dict->patterns = NULL;
- dict->merged = NULL;
- dict->state_num = NULL;
- init_hash(&dict->patterns);
-}
-
-
-static void clear_dict(
- HyphenDict* dict
-) {
- int state_num;
- for (state_num = 0; state_num < dict->num_states; state_num++) {
- HyphenState *hstate = &dict->states[state_num];
- if (hstate->match) hnj_free (hstate->match);
- if (hstate->trans) hnj_free (hstate->trans);
- }
- hnj_free (dict->states);
- clear_hyppat_hash(&dict->patterns);
- clear_hyppat_hash(&dict->merged);
- clear_state_hash(&dict->state_num);
-}
-
-
-
-HyphenDict* hnj_hyphen_new(void) {
- HyphenDict* dict = hnj_malloc (sizeof(HyphenDict));
- init_dict(dict);
- return dict;
-}
-
-
-void hnj_hyphen_clear(
- HyphenDict* dict
-) {
- clear_dict(dict);
- init_dict(dict);
-}
-
-
-void hnj_hyphen_free(
- HyphenDict *dict
-) {
- clear_dict(dict);
- hnj_free(dict);
-}
-
-unsigned char* hnj_serialize(
- HyphenDict* dict
-) {
- HashIter *v;
- unsigned char* word;
- char* pattern;
- unsigned char* buf = hnj_malloc(dict->pat_length);
- unsigned char* cur = buf;
- v = new_HashIter(dict->patterns);
- while (eachHash(v,&word,&pattern)) {
- int i=0, e=0;
- while(word[e+i]) {
- if (pattern[i]!='0') *cur++ = (unsigned char) pattern[i];
- *cur++ = word[e+i++];
- while (is_utf8_follow(word[e+i])) *cur++ = word[i+e++];
- }
- if (pattern[i]!='0') *cur++ = (unsigned char) pattern[i];
- *cur++ = ' ';
- }
- delete_HashIter(v);
- *cur = 0;
- return buf;
-}
-
-
-void hnj_free_serialize(
- unsigned char* c
-) {
- hnj_free(c);
-}
-
-
-/* hyphenation pattern:
- * signed bytes
- * 0 indicates end (actually any negative number)
- * : prio(1+),startpos,length,len1,[replace],len2,[replace]
- * most basic example is:
- * p n 0 0 0
- * for a hyphenation point between characters
- */
-
-
-void hnj_hyphen_load(
- HyphenDict* dict,
- const unsigned char *f
-) {
- int state_num, last_state;
- int i, j = 0;
- int ch;
- int found;
- HashEntry *e;
- HashIter *v;
- unsigned char* word;
- char* pattern;
- size_t l = 0;
-
-
- /***************************************/
-
- const unsigned char* format;
- const unsigned char* begin = f;
- unsigned char *pat;
- char *org;
- while((format = next_pattern(&l,&f))!=NULL) {
- int i,j,e;
- /*
- printf("%s\n",format);
- char* repl = strnchr(format, '/',l);
- int replindex = 0;
- int replcut = 0;
- if (repl) {
- int clen = l-(repl-format);
- l = repl-format;
- char * index = strnchr(repl + 1, ',',clen);
- if (index) {
- char * index2 = strnchr(index + 1, ',',clen-(index-repl));
- if (index2) {
- replindex = (signed char) atoi(index + 1) - 1;
- replcut = (signed char) atoi(index2 + 1);
- }
- } else {
- hnj_strchomp(repl + 1);
- replindex = 0;
- replcut = strlen(buf);
- }
- repl = hnj_strdup(repl + 1);
- }
- */
- for (i=0,j=0,e=0; (unsigned)i<l; i++) {
- if (format[i]>='0'&&format[i]<='9') j++;
- if (is_utf8_follow(format[i])) e++;
- }
- /* l-e => number of _characters_ not _bytes_*/
- /* l-j => number of pattern bytes */
- /* l-e-j => number of pattern characters*/
- pat = (unsigned char*) malloc(1+l-j);
- org = ( char*) malloc(2+l-e-j);
- /* remove hyphenation encoders (digits) from pat*/
- org[0] = '0';
- for (i=0,j=0,e=0; (unsigned)i<l; i++) {
- unsigned char c = format[i];
- if (is_utf8_follow(c)) {
- pat[j+e++] = c;
- } else if (c<'0' || c>'9') {
- pat[e+j++] = c;
- org[j] = '0';
- } else {
- org[j] = c;
- }
- }
- pat[e+j] = 0;
- org[j+1] = 0;
- hyppat_insert(dict->patterns,pat,org);
- }
- dict->pat_length += (f-begin)+2; /* 2 for spurious spaces*/
- init_hash(&dict->merged);
- v = new_HashIter(dict->patterns);
- while (nextHash(v,&word)) {
- int wordsize = strlen((char*)word);
- int j,l;
- for (l=1; l<=wordsize; l++) {
- if (is_utf8_follow(word[l])) continue; /* Do not clip an utf8 sequence*/
- for (j=1; j<=l; j++) {
- char *subpat_pat;
- int i = l-j;
- if (is_utf8_follow(word[i])) continue; /* Do not start halfway an utf8 sequence*/
- if ((subpat_pat = hyppat_lookup(dict->patterns,word+i,j))!=NULL) {
- char* newpat_pat;
- if ((newpat_pat = hyppat_lookup(dict->merged,word,l))==NULL) {
- char *neworg;
- unsigned char *newword=(unsigned char*)malloc(l+1);
- int e=0;
- strncpy((char*)newword, (char*)word,l); newword[l]=0;
- for (i=0; i<l; i++) if (is_utf8_follow(newword[i])) e++;
- neworg = malloc(l+2-e);
- sprintf(neworg,"%0*d",l+1-e,0); /* fill with right amount of '0'*/
- hyppat_insert(dict->merged,newword,combine(neworg,subpat_pat));
- } else {
- combine(newpat_pat,subpat_pat);
- }
- }
- }
- }
- }
- delete_HashIter(v);
-
- init_hash(&dict->state_num);
- state_insert(dict->state_num, hnj_strdup((const unsigned char*)""), 0);
- v = new_HashIter(dict->merged);
- while (nextHashStealPattern(v,&word,&pattern)) {
- static unsigned char mask[] = {0x3F,0x1F,0xF,0x7};
- int j = strlen((char*)word);
-#ifdef VERBOSE
- printf ("word %s pattern %s, j = %d\n", word, pattern, j);
-#endif
- state_num = hnj_get_state( dict, word, &found );
- dict->states[state_num].match = pattern;
-
- /* now, put in the prefix transitions */
- while (found < 0) {
- j--;
- last_state = state_num;
- ch = word[j];
- if (ch>=0x80) {
- int m;
- int i=1;
- while (is_utf8_follow(word[j-i])) i++;
- ch = word[j-i] & mask[i];
- m = j-i;
- while (i--) {
- ch = (ch<<6)+(0x3F & word[j-i]);
- }
- j = m;
- }
- word[j] = '\0';
- state_num = hnj_get_state (dict, word, &found);
- hnj_add_trans (dict, state_num, last_state, ch);
- }
- }
- delete_HashIter(v);
- clear_hyppat_hash(&dict->merged);
-
- /***************************************/
-
- /* put in the fallback states */
- for (i = 0; i < HASH_SIZE; i++) {
- for (e = dict->state_num->entries[i]; e; e = e->next) {
- /* do not do state==0 otherwise things get confused*/
- if (e->u.state) {
- for (j = 1; 1; j++) {
- state_num = state_lookup(dict->state_num, e->key + j);
- if (state_num >= 0) break;
- }
- dict->states[e->u.state].fallback_state = state_num;
- }
- }
- }
-#ifdef VERBOSE
- for (i = 0; i < HASH_SIZE; i++) {
- for (e = dict->state_num->entries[i]; e; e = e->next) {
- printf ("%d string %s state %d, fallback=%d\n", i, e->key, e->u.state,
- dict->states[e->u.state].fallback_state);
- for (j = 0; j < dict->states[e->u.state].num_trans; j++) {
- printf (" u%4x->%d\n", (int)dict->states[e->u.state].trans[j].uni_ch,
- dict->states[e->u.state].trans[j].new_state);
- }
- }
- }
-#endif
- clear_state_hash(&dict->state_num);
-}
-
-
-void hnj_hyphen_hyphenate(
- HyphenDict *dict,
- halfword first,
- halfword last,
- int length,
- halfword left,
- halfword right,
- lang_variables *lan
-) {
- int char_num;
- halfword here;
- int state = 0;
- /* +2 for dots at each end, +1 for points /outside/ characters*/
- int ext_word_len = length+2;
- int hyphen_len = ext_word_len+1;
- /*char *hyphens = hnj_malloc((hyphen_len*2)+1); */ /* LATER */
- char *hyphens = hnj_malloc(hyphen_len+1);
-
- /* Add a '.' to beginning and end to facilitate matching*/
- set_vlink(begin_point,first);
- set_vlink(end_point,get_vlink(last));
- set_vlink(last,end_point);
-
- for (char_num = 0; char_num < hyphen_len; char_num++) {
- /* hyphens[char_num*2] = '0'; */ /* LATER */
- /* hyphens[char_num*2+1] = '0'; */ /* LATER */
- hyphens[char_num] = '0';
- }
- /*hyphens[hyphen_len*2] = 0; */ /* LATER */
- hyphens[hyphen_len] = 0;
-
- /* now, run the finite state machine */
- for (char_num=0, here=begin_point; here!=get_vlink(end_point); here=get_vlink(here)) {
-
- int ch;
- if (here == begin_point || here == end_point)
- ch='.';
- else
- ch = get_lc_code(get_character(here));
- while (state!=-1) {
- /* printf("%*s%s%c",char_num-strlen(get_state_str(state)),"",get_state_str(state),(char)ch);*/
- HyphenState *hstate = &dict->states[state];
- int k;
- for (k = 0; k < hstate->num_trans; k++) {
- if (hstate->trans[k].uni_ch == ch) {
- char *match;
- state = hstate->trans[k].new_state;
- /* printf(" state %d\n",state);*/
- match = dict->states[state].match;
- if (match) {
- /* +2 because:
- * 1 string length is one bigger than offset
- * 1 hyphenation starts before first character
- */
- int offset = char_num + 2 - strlen (match);
- /* printf ("%*s%s\n", offset,"", match);*/
- int m;
- for (m = 0; match[m]; m++) {
- if (hyphens[offset+m] < match[m]) hyphens[offset+m] = match[m];
- }
- }
- goto try_next_letter;
- }
- }
- state = hstate->fallback_state;
- /* printf (" back to %d\n", state);*/
- }
- /* nothing worked, let's go to the next character*/
- state = 0;
-try_next_letter: ;
- char_num++;
- }
-
- /* restore the correct pointers*/
- set_vlink(last,get_vlink(end_point));
-
- /* pattern is ^.^w^o^r^d^.^ word_len=4, ext_word_len=6, hyphens=7
- * check ^ ^ ^ so drop first two and stop after word_len-1
- */
- for (here=first,char_num=2; here!=left; here=get_vlink(here)) char_num++;
- for (; here!=right; here=get_vlink(here)) {
- if (hyphens[char_num] & 1)
- here = insert_syllable_discretionary(here, lan);
- char_num++;
- }
- hnj_free(hyphens);
-}
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c b/Build/source/texk/web2c/luatexdir/lang/hyphen.h
index 6072152f4f5..8e5e5785c93 100644
--- a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c
+++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.h
@@ -33,42 +33,28 @@
* MPL.
*
*/
-/* wrappers for malloc */
+#ifndef __HYPHEN_H__
+# define __HYPHEN_H__
-#include <stdlib.h>
-#include <stdio.h>
+# ifdef __cplusplus
+extern "C" {
+# endif /* __cplusplus */
-#include "hnjalloc.h"
+ typedef struct _HyphenDict HyphenDict;
-void *
-hnj_malloc (int size)
-{
- void *p;
+# include "ptexlib.h"
- p = malloc (size);
- if (p == NULL)
- {
- fprintf (stderr, "can't allocate %d bytes\n", size);
- exit (1);
- }
- return p;
-}
-
-void *
-hnj_realloc (void *p, int size)
-{
- p = realloc (p, size);
- if (p == NULL)
- {
- fprintf (stderr, "can't allocate %d bytes\n", size);
- exit (1);
- }
- return p;
-}
+ HyphenDict *hnj_hyphen_new(void);
+ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *fn);
+ void hnj_hyphen_free(HyphenDict * dict);
+ void hnj_hyphen_clear(HyphenDict * dict);
+ void hnj_hyphen_hyphenate(HyphenDict * dict, halfword first, halfword last,
+ int size, halfword left, halfword right,
+ lang_variables * lan);
+ unsigned char *hnj_serialize(HyphenDict *);
+ void hnj_free_serialize(unsigned char *);
-void
-hnj_free (void *p)
-{
- free (p);
+# ifdef __cplusplus
}
-
+# endif /* __cplusplus */
+#endif /* __HYPHEN_H__ */
diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.w b/Build/source/texk/web2c/luatexdir/lang/hyphen.w
new file mode 100644
index 00000000000..f7bbe6bd843
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.w
@@ -0,0 +1,857 @@
+% Libhnj is dual licensed under LGPL and MPL. Boilerplate for both
+% licenses follows.
+%
+%
+% LibHnj - a library for high quality hyphenation and justification
+% Copyright (C) 1998 Raph Levien,
+% (C) 2001 ALTLinux, Moscow (http://www.alt-linux.org),
+% (C) 2001 Peter Novodvorsky (nidd@@cs.msu.su)
+%
+% This library is free software; you can redistribute it and/or
+% modify it under the terms of the GNU Library General Public
+% License as published by the Free Software Foundation; either
+% version 2 of the License, or (at your option) any later version.
+%
+% This library is distributed in the hope that it will be useful,
+% but WITHOUT ANY WARRANTY; without even the implied warranty of
+% MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+% Library General Public License for more details.
+%
+% You should have received a copy of the GNU Library General Public
+% License along with this library; if not, write to the
+% Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+% Boston, MA 02111-1307 USA.
+%
+%
+%
+% The contents of this file are subject to the Mozilla Public License
+% Version 1.0 (the "MPL"); you may not use this file except in
+% compliance with the MPL. You may obtain a copy of the MPL at
+% http://www.mozilla.org/MPL/
+%
+% Software distributed under the MPL is distributed on an "AS IS" basis,
+% WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL
+% for the specific language governing rights and limitations under the
+% MPL.
+
+
+@ @c
+#include <stdlib.h> /* for NULL, malloc */
+#include <stdio.h> /* for fprintf */
+#include <string.h> /* for strdup */
+#include <stdlib.h> /* for malloc used by substring inclusion */
+
+#define MAXPATHS 40960
+
+#ifdef UNX
+# include <unistd.h> /* for exit */
+#endif
+
+#include <kpathsea/c-ctype.h>
+
+#define noVERBOSE
+
+#include "lang/hnjalloc.h"
+#include "lang/hyphen.h"
+
+@ @c
+static const char _svn_version[] =
+ "$Id: hyphen.w 3584 2010-04-02 17:45:55Z hhenkel $ "
+"$URL: http://foundry.supelec.fr/svn/luatex/tags/beta-0.60.0/source/texk/web2c/luatexdir/lang/hyphen.w $";
+
+@ TODO: should be moved to separate library
+
+@c
+static unsigned char *hnj_strdup(const unsigned char *s)
+{
+ unsigned char *new;
+ size_t l;
+
+ l = strlen((const char *) s);
+ new = hnj_malloc((int) l + 1);
+ memcpy(new, s, l);
+ new[l] = 0;
+ return new;
+}
+
+static int is_utf8_follow(unsigned char c)
+{
+ if (c >= 0x80 && c < 0xC0)
+ return 1;
+ return 0;
+}
+
+@* Type definitions.
+
+@ a little bit of a hash table implementation. This simply maps strings
+ to state numbers
+
+@c
+typedef struct _HashTab HashTab;
+typedef struct _HashEntry HashEntry;
+typedef struct _HashIter HashIter;
+typedef union _HashVal HashVal;
+
+/* A cheap, but effective, hack. */
+#define HASH_SIZE 31627
+
+struct _HashTab {
+ HashEntry *entries[HASH_SIZE];
+};
+
+union _HashVal {
+ int state;
+ char *hyppat;
+};
+
+struct _HashEntry {
+ HashEntry *next;
+ unsigned char *key;
+ HashVal u;
+};
+
+struct _HashIter {
+ HashEntry **e;
+ HashEntry *cur;
+ int ndx;
+};
+
+@ State machine
+
+@c
+typedef struct _HyphenState HyphenState;
+typedef struct _HyphenTrans HyphenTrans;
+#define MAX_CHARS 256
+#define MAX_NAME 20
+
+struct _HyphenDict {
+ int num_states;
+ int pat_length;
+ char cset[MAX_NAME];
+ HyphenState *states;
+ HashTab *patterns;
+ HashTab *merged;
+ HashTab *state_num;
+};
+
+struct _HyphenState {
+ char *match;
+ /*char *repl; */
+ /*signed char replindex; */
+ /*signed char replcut; */
+ int fallback_state;
+ int num_trans;
+ HyphenTrans *trans;
+};
+
+struct _HyphenTrans {
+ int uni_ch;
+ int new_state;
+};
+
+
+@ Combine two right-aligned number patterns, 04000 + 020 becomes 04020
+
+@c
+static char *combine(char *expr, const char *subexpr)
+{
+ size_t l1 = strlen(expr);
+ size_t l2 = strlen(subexpr);
+ size_t off = l1 - l2;
+ unsigned j;
+ /* this works also for utf8 sequences because the substring is identical
+ to the last substring-length bytes of expr except for the (single byte)
+ hyphenation encoders
+ */
+ for (j = 0; j < l2; j++) {
+ if (expr[off + j] < subexpr[j])
+ expr[off + j] = subexpr[j];
+ }
+ return expr;
+}
+
+
+@ ORIGINAL CODE
+@c
+HashIter *new_HashIter(HashTab * h)
+{
+ HashIter *i = hnj_malloc(sizeof(HashIter));
+ i->e = h->entries;
+ i->cur = NULL;
+ i->ndx = -1;
+ return i;
+}
+
+
+int nextHashStealPattern(HashIter * i, unsigned char **word, char **pattern)
+{
+ while (i->cur == NULL) {
+ if (i->ndx >= HASH_SIZE - 1)
+ return 0;
+ i->cur = i->e[++i->ndx];
+ }
+ *word = i->cur->key;
+ *pattern = i->cur->u.hyppat;
+ i->cur->u.hyppat = NULL;
+ i->cur = i->cur->next;
+ return 1;
+}
+
+
+int nextHash(HashIter * i, unsigned char **word)
+{
+ while (i->cur == NULL) {
+ if (i->ndx >= HASH_SIZE - 1)
+ return 0;
+ i->cur = i->e[++i->ndx];
+ }
+ *word = i->cur->key;
+ i->cur = i->cur->next;
+ return 1;
+}
+
+
+int eachHash(HashIter * i, unsigned char **word, char **pattern)
+{
+ while (i->cur == NULL) {
+ if (i->ndx >= HASH_SIZE - 1)
+ return 0;
+ i->cur = i->e[++i->ndx];
+ }
+ *word = i->cur->key;
+ *pattern = i->cur->u.hyppat;
+ i->cur = i->cur->next;
+ return 1;
+}
+
+
+void delete_HashIter(HashIter * i)
+{
+ hnj_free(i);
+}
+
+
+@ a |char*| hash function from ASU - adapted from Gtk+
+
+@c
+static unsigned int hnj_string_hash(const unsigned char *s)
+{
+ const unsigned char *p;
+ unsigned int h = 0, g;
+
+ for (p = s; *p != '\0'; p += 1) {
+ h = (h << 4) + *p;
+ if ((g = (h & 0xf0000000))) {
+ h = h ^ (g >> 24);
+ h = h ^ g;
+ }
+ }
+ return h /* \% M */ ;
+}
+
+
+@ assumes that key is not already present!
+
+@c
+static void state_insert(HashTab * hashtab, unsigned char *key, int state)
+{
+ int i;
+ HashEntry *e;
+
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ e = hnj_malloc(sizeof(HashEntry));
+ e->next = hashtab->entries[i];
+ e->key = key;
+ e->u.state = state;
+ hashtab->entries[i] = e;
+}
+
+
+@ assumes that key is not already present!
+
+@c
+static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat)
+{
+ int i;
+ HashEntry *e;
+
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ for (e = hashtab->entries[i]; e; e = e->next) {
+ if (strcmp((char *) e->key, (char *) key) == 0) {
+ if (e->u.hyppat) {
+ if (hyppat
+ && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) {
+ print_err("Conflicting pattern ignored");
+ error();
+ }
+ hnj_free(e->u.hyppat);
+ }
+ e->u.hyppat = hyppat;
+ hnj_free(key);
+ return;
+ }
+ }
+ e = hnj_malloc(sizeof(HashEntry));
+ e->next = hashtab->entries[i];
+ e->key = key;
+ e->u.hyppat = hyppat;
+ hashtab->entries[i] = e;
+}
+
+
+@ return state if found, otherwise $-1$
+
+@c
+static int state_lookup(HashTab * hashtab, const unsigned char *key)
+{
+ int i;
+ HashEntry *e;
+
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ for (e = hashtab->entries[i]; e; e = e->next) {
+ if (!strcmp((const char *) key, (const char *) e->key)) {
+ return e->u.state;
+ }
+ }
+ return -1;
+}
+
+
+@ return state if found, otherwise $-1$
+
+@c
+static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l)
+{
+ int i;
+ HashEntry *e;
+ unsigned char key[128]; /* should be ample */
+ strncpy((char *) key, (const char *) chars, (size_t) l);
+ key[l] = 0;
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ for (e = hashtab->entries[i]; e; e = e->next) {
+ if (!strcmp((char *) key, (char *) e->key)) {
+ return e->u.hyppat;
+ }
+ }
+ return NULL;
+}
+
+
+@ Get the state number, allocating a new state if necessary.
+
+@c
+static int hnj_get_state(HyphenDict * dict,
+ const unsigned char *str, int *state_num)
+{
+ *state_num = state_lookup(dict->state_num, str);
+
+ if (*state_num >= 0)
+ return *state_num;
+
+ state_insert(dict->state_num, hnj_strdup(str), dict->num_states);
+ /* predicate is true if |dict->num_states| is a power of two */
+ if (!(dict->num_states & (dict->num_states - 1))) {
+ dict->states = hnj_realloc(dict->states,
+ (int) ((dict->num_states << 1) *
+ (int) sizeof(HyphenState)));
+ }
+ dict->states[dict->num_states].match = NULL;
+ dict->states[dict->num_states].fallback_state = -1;
+ dict->states[dict->num_states].num_trans = 0;
+ dict->states[dict->num_states].trans = NULL;
+ return dict->num_states++;
+}
+
+
+@ Add a transition from state1 to state2 through ch - assumes that the
+ transition does not already exist
+
+@c
+static void hnj_add_trans(HyphenDict * dict, int state1, int state2, int uni_ch)
+{
+ int num_trans;
+ /* TH: this test was a bit too strict, it is quite normal for old
+ patterns to have chars in the range 0-31 or 127-159 (inclusive).
+ To ease the transition, let's only disallow NUL for now
+ (this is probably a requirement of the code anyway).
+ */
+ if (uni_ch == 0) {
+ fprintf(stderr, "Character out of bounds: u%04x \n", uni_ch);
+ exit(1);
+ }
+ num_trans = dict->states[state1].num_trans;
+ if (num_trans == 0) {
+ dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans));
+ } else if (!(num_trans & (num_trans - 1))) {
+ dict->states[state1].trans = hnj_realloc(dict->states[state1].trans,
+ (int) (num_trans << 1 *
+ sizeof(HyphenTrans)));
+ }
+ dict->states[state1].trans[num_trans].uni_ch = uni_ch;
+ dict->states[state1].trans[num_trans].new_state = state2;
+ dict->states[state1].num_trans++;
+}
+
+
+#ifdef VERBOSE
+
+static unsigned char *get_state_str(int state)
+{
+ int i;
+ HashEntry *e;
+
+ for (i = 0; i < HASH_SIZE; i++)
+ for (e = global->entries[i]; e; e = e->next)
+ if (e->u.state == state)
+ return e->key;
+ return NULL;
+}
+#endif
+
+
+@ I've changed the semantics a bit here: |hnj_hyphen_load| used to
+ operate on a file, but now the argument is a string buffer.
+
+@c
+static const unsigned char *next_pattern(size_t * length,
+ const unsigned char **buf)
+{
+ const unsigned char *here, *rover = *buf;
+ while (*rover && isspace(*rover))
+ rover++;
+ here = rover;
+ while (*rover) {
+ if (isspace(*rover)) {
+ *length = (size_t) (rover - here);
+ *buf = rover;
+ return here;
+ }
+ rover++;
+ }
+ *length = (size_t) (rover - here);
+ *buf = rover;
+ return *length ? here : NULL; /* zero sensed */
+}
+
+static void init_hash(HashTab ** h)
+{
+ int i;
+ if (*h)
+ return;
+ *h = hnj_malloc(sizeof(HashTab));
+ for (i = 0; i < HASH_SIZE; i++)
+ (*h)->entries[i] = NULL;
+}
+
+
+static void clear_state_hash(HashTab ** h)
+{
+ int i;
+ if (*h == NULL)
+ return;
+ for (i = 0; i < HASH_SIZE; i++) {
+ HashEntry *e, *next;
+ for (e = (*h)->entries[i]; e; e = next) {
+ next = e->next;
+ hnj_free(e->key);
+ hnj_free(e);
+ }
+ }
+ hnj_free(*h);
+ *h = NULL;
+}
+
+
+static void clear_hyppat_hash(HashTab ** h)
+{
+ int i;
+ if (*h == NULL)
+ return;
+ for (i = 0; i < HASH_SIZE; i++) {
+ HashEntry *e, *next;
+ for (e = (*h)->entries[i]; e; e = next) {
+ next = e->next;
+ hnj_free(e->key);
+ if (e->u.hyppat)
+ hnj_free(e->u.hyppat);
+ hnj_free(e);
+ }
+ }
+ hnj_free(*h);
+ *h = NULL;
+}
+
+
+static void init_dict(HyphenDict * dict)
+{
+ dict->num_states = 1;
+ dict->pat_length = 0;
+ dict->states = hnj_malloc(sizeof(HyphenState));
+ dict->states[0].match = NULL;
+ dict->states[0].fallback_state = -1;
+ dict->states[0].num_trans = 0;
+ dict->states[0].trans = NULL;
+ dict->patterns = NULL;
+ dict->merged = NULL;
+ dict->state_num = NULL;
+ init_hash(&dict->patterns);
+}
+
+
+static void clear_dict(HyphenDict * dict)
+{
+ int state_num;
+ for (state_num = 0; state_num < dict->num_states; state_num++) {
+ HyphenState *hstate = &dict->states[state_num];
+ if (hstate->match)
+ hnj_free(hstate->match);
+ if (hstate->trans)
+ hnj_free(hstate->trans);
+ }
+ hnj_free(dict->states);
+ clear_hyppat_hash(&dict->patterns);
+ clear_hyppat_hash(&dict->merged);
+ clear_state_hash(&dict->state_num);
+}
+
+
+
+HyphenDict *hnj_hyphen_new(void)
+{
+ HyphenDict *dict = hnj_malloc(sizeof(HyphenDict));
+ init_dict(dict);
+ return dict;
+}
+
+
+void hnj_hyphen_clear(HyphenDict * dict)
+{
+ clear_dict(dict);
+ init_dict(dict);
+}
+
+
+void hnj_hyphen_free(HyphenDict * dict)
+{
+ clear_dict(dict);
+ hnj_free(dict);
+}
+
+unsigned char *hnj_serialize(HyphenDict * dict)
+{
+ HashIter *v;
+ unsigned char *word;
+ char *pattern;
+ unsigned char *buf = hnj_malloc(dict->pat_length);
+ unsigned char *cur = buf;
+ v = new_HashIter(dict->patterns);
+ while (eachHash(v, &word, &pattern)) {
+ int i = 0, e = 0;
+ while (word[e + i]) {
+ if (pattern[i] != '0')
+ *cur++ = (unsigned char) pattern[i];
+ *cur++ = word[e + i++];
+ while (is_utf8_follow(word[e + i]))
+ *cur++ = word[i + e++];
+ }
+ if (pattern[i] != '0')
+ *cur++ = (unsigned char) pattern[i];
+ *cur++ = ' ';
+ }
+ delete_HashIter(v);
+ *cur = 0;
+ return buf;
+}
+
+
+void hnj_free_serialize(unsigned char *c)
+{
+ hnj_free(c);
+}
+
+
+@ hyphenation pattern:
+
+signed bytes
+
+0 indicates end (actually any negative number)
+
+: prio(1+),startpos,length,len1,[replace],len2,[replace]
+
+most basic example is:
+
+p n 0 0 0
+
+for a hyphenation point between characters
+
+
+@c
+void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f)
+{
+ int state_num, last_state;
+ int i, j = 0;
+ int ch;
+ int found;
+ HashEntry *e;
+ HashIter *v;
+ unsigned char *word;
+ char *pattern;
+ size_t l = 0;
+
+ const unsigned char *format;
+ const unsigned char *begin = f;
+ unsigned char *pat;
+ char *org;
+ while ((format = next_pattern(&l, &f)) != NULL) {
+ int i, j, e;
+#if 0
+ printf("%s\n",format);
+ char* repl = strnchr(format, '/',l);
+ int replindex = 0;
+ int replcut = 0;
+ if (repl) {
+ int clen = l-(repl-format);
+ l = repl-format;
+ char * index = strnchr(repl + 1, ',',clen);
+ if (index) {
+ char * index2 = strnchr(index + 1, ',',clen-(index-repl));
+ if (index2) {
+ replindex = (signed char) atoi(index + 1) - 1;
+ replcut = (signed char) atoi(index2 + 1);
+ }
+ } else {
+ hnj_strchomp(repl + 1);
+ replindex = 0;
+ replcut = strlen(buf);
+ }
+ repl = hnj_strdup(repl + 1);
+ }
+#endif
+ for (i = 0, j = 0, e = 0; (unsigned) i < l; i++) {
+ if (format[i] >= '0' && format[i] <= '9')
+ j++;
+ if (is_utf8_follow(format[i]))
+ e++;
+ }
+ /* |l-e| => number of {\it characters} not {\it bytes} */
+ /* |l-j| => number of pattern bytes */
+ /* |l-e-j| => number of pattern characters */
+ pat = (unsigned char *) malloc((1 + l - (size_t) j));
+ org = (char *) malloc((size_t) (2 + l - (size_t) e - (size_t) j));
+ /* remove hyphenation encoders (digits) from pat */
+ org[0] = '0';
+ for (i = 0, j = 0, e = 0; (unsigned) i < l; i++) {
+ unsigned char c = format[i];
+ if (is_utf8_follow(c)) {
+ pat[j + e++] = c;
+ } else if (c < '0' || c > '9') {
+ pat[e + j++] = c;
+ org[j] = '0';
+ } else {
+ org[j] = (char) c;
+ }
+ }
+ pat[e + j] = 0;
+ org[j + 1] = 0;
+ hyppat_insert(dict->patterns, pat, org);
+ }
+ dict->pat_length += (int) ((f - begin) + 2); /* 2 for spurious spaces */
+ init_hash(&dict->merged);
+ v = new_HashIter(dict->patterns);
+ while (nextHash(v, &word)) {
+ int wordsize = (int) strlen((char *) word);
+ int j, l;
+ for (l = 1; l <= wordsize; l++) {
+ if (is_utf8_follow(word[l]))
+ continue; /* Do not clip an utf8 sequence */
+ for (j = 1; j <= l; j++) {
+ char *subpat_pat;
+ int i = l - j;
+ if (is_utf8_follow(word[i]))
+ continue; /* Do not start halfway an utf8 sequence */
+ if ((subpat_pat =
+ hyppat_lookup(dict->patterns, word + i, j)) != NULL) {
+ char *newpat_pat;
+ if ((newpat_pat =
+ hyppat_lookup(dict->merged, word, l)) == NULL) {
+ char *neworg;
+ unsigned char *newword =
+ (unsigned char *) malloc((size_t) (l + 1));
+ int e = 0;
+ strncpy((char *) newword, (char *) word, (size_t) l);
+ newword[l] = 0;
+ for (i = 0; i < l; i++)
+ if (is_utf8_follow(newword[i]))
+ e++;
+ neworg = malloc((size_t) (l + 2 - e));
+ sprintf(neworg, "%0*d", l + 1 - e, 0); /* fill with right amount of '0' */
+ hyppat_insert(dict->merged, newword,
+ combine(neworg, subpat_pat));
+ } else {
+ combine(newpat_pat, subpat_pat);
+ }
+ }
+ }
+ }
+ }
+ delete_HashIter(v);
+
+ init_hash(&dict->state_num);
+ state_insert(dict->state_num, hnj_strdup((const unsigned char *) ""), 0);
+ v = new_HashIter(dict->merged);
+ while (nextHashStealPattern(v, &word, &pattern)) {
+ static unsigned char mask[] = { 0x3F, 0x1F, 0xF, 0x7 };
+ int j = (int) strlen((char *) word);
+#ifdef VERBOSE
+ printf("word %s pattern %s, j = %d\n", word, pattern, j);
+#endif
+ state_num = hnj_get_state(dict, word, &found);
+ dict->states[state_num].match = pattern;
+
+ /* now, put in the prefix transitions */
+ while (found < 0) {
+ j--;
+ last_state = state_num;
+ ch = word[j];
+ if (ch >= 0x80) {
+ int m;
+ int i = 1;
+ while (is_utf8_follow(word[j - i]))
+ i++;
+ ch = word[j - i] & mask[i];
+ m = j - i;
+ while (i--) {
+ ch = (ch << 6) + (0x3F & word[j - i]);
+ }
+ j = m;
+ }
+ word[j] = '\0';
+ state_num = hnj_get_state(dict, word, &found);
+ hnj_add_trans(dict, state_num, last_state, ch);
+ }
+ }
+ delete_HashIter(v);
+ clear_hyppat_hash(&dict->merged);
+
+ /* put in the fallback states */
+ for (i = 0; i < HASH_SIZE; i++) {
+ for (e = dict->state_num->entries[i]; e; e = e->next) {
+ /* do not do state==0 otherwise things get confused */
+ if (e->u.state) {
+ for (j = 1; 1; j++) {
+ state_num = state_lookup(dict->state_num, e->key + j);
+ if (state_num >= 0)
+ break;
+ }
+ dict->states[e->u.state].fallback_state = state_num;
+ }
+ }
+ }
+#ifdef VERBOSE
+ for (i = 0; i < HASH_SIZE; i++) {
+ for (e = dict->state_num->entries[i]; e; e = e->next) {
+ printf("%d string %s state %d, fallback=%d\n", i, e->key,
+ e->u.state, dict->states[e->u.state].fallback_state);
+ for (j = 0; j < dict->states[e->u.state].num_trans; j++) {
+ printf(" u%4x->%d\n",
+ (int) dict->states[e->u.state].trans[j].uni_ch,
+ dict->states[e->u.state].trans[j].new_state);
+ }
+ }
+ }
+#endif
+ clear_state_hash(&dict->state_num);
+}
+
+@ @c
+void hnj_hyphen_hyphenate(HyphenDict * dict,
+ halfword first,
+ halfword last,
+ int length,
+ halfword left, halfword right, lang_variables * lan)
+{
+ int char_num;
+ halfword here;
+ int state = 0;
+ /* +2 for dots at each end, +1 for points /outside/ characters */
+ int ext_word_len = length + 2;
+ int hyphen_len = ext_word_len + 1;
+ char *hyphens = hnj_malloc(hyphen_len + 1);
+
+ /* Add a '.' to beginning and end to facilitate matching */
+ set_vlink(begin_point, first);
+ set_vlink(end_point, get_vlink(last));
+ set_vlink(last, end_point);
+
+ for (char_num = 0; char_num < hyphen_len; char_num++) {
+ hyphens[char_num] = '0';
+ }
+ hyphens[hyphen_len] = 0;
+
+ /* now, run the finite state machine */
+ for (char_num = 0, here = begin_point; here != get_vlink(end_point);
+ here = get_vlink(here)) {
+
+ int ch;
+ if (here == begin_point || here == end_point)
+ ch = '.';
+ else
+ ch = get_lc_code(get_character(here));
+ while (state != -1) {
+#if 0
+ printf("%*s%s%c",char_num-strlen(get_state_str(state)),"",get_state_str(state),(char)ch);
+#endif
+ HyphenState *hstate = &dict->states[state];
+ int k;
+ for (k = 0; k < hstate->num_trans; k++) {
+ if (hstate->trans[k].uni_ch == ch) {
+ char *match;
+ state = hstate->trans[k].new_state;
+#if 0
+ printf(" state %d\n",state);
+#endif
+ match = dict->states[state].match;
+ if (match) {
+ /* +2 because:
+ 1 string length is one bigger than offset
+ 1 hyphenation starts before first character
+ */
+ int offset = (int) (char_num + 2 - (int) strlen(match));
+#if 0
+ printf ("%*s%s\n", offset,"", match);
+#endif
+ int m;
+ for (m = 0; match[m]; m++) {
+ if (hyphens[offset + m] < match[m])
+ hyphens[offset + m] = match[m];
+ }
+ }
+ goto try_next_letter;
+ }
+ }
+ state = hstate->fallback_state;
+#if 0
+ printf (" back to %d\n", state);
+#endif
+ }
+ /* nothing worked, let's go to the next character */
+ state = 0;
+ try_next_letter:;
+ char_num++;
+ }
+
+ /* restore the correct pointers */
+ set_vlink(last, get_vlink(end_point));
+
+ /* pattern is \.{\^.\^w\^o\^r\^d\^.\^} |word_len|=4, |ext_word_len|=6, |hyphens|=7
+ * check \.{ \^ \^ \^ } so drop first two and stop after |word_len-1|
+ */
+ for (here = first, char_num = 2; here != left; here = get_vlink(here))
+ char_num++;
+ for (; here != right; here = get_vlink(here)) {
+ if (hyphens[char_num] & 1)
+ here = insert_syllable_discretionary(here, lan);
+ char_num++;
+ }
+ hnj_free(hyphens);
+}
diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.h b/Build/source/texk/web2c/luatexdir/lang/texlang.h
new file mode 100644
index 00000000000..b312dab1a3a
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/texlang.h
@@ -0,0 +1,82 @@
+/* texlang.h
+
+ Copyright 2009 Taco Hoekwater <taco@luatex.org>
+
+ This file is part of LuaTeX.
+
+ LuaTeX is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2 of the License, or (at your
+ option) any later version.
+
+ LuaTeX is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */
+
+/* $Id: texlang.h 3394 2010-01-26 14:20:46Z taco $ */
+
+#ifndef TEXLANG_H
+# define TEXLANG_H
+
+typedef struct _lang_variables {
+ int pre_hyphen_char;
+ int post_hyphen_char;
+ int pre_exhyphen_char;
+ int post_exhyphen_char;
+} lang_variables;
+
+# include "lang/hyphen.h"
+
+struct tex_language {
+ HyphenDict *patterns;
+ int exceptions; /* lua registry pointer, should be replaced */
+ int id;
+ int pre_hyphen_char;
+ int post_hyphen_char;
+ int pre_exhyphen_char;
+ int post_exhyphen_char;
+};
+
+# define MAX_WORD_LEN 256 /* in chars */
+
+extern struct tex_language *new_language(int n);
+extern struct tex_language *get_language(int n);
+extern void load_patterns(struct tex_language *lang, const unsigned char *buf);
+extern void load_hyphenation(struct tex_language *lang,
+ const unsigned char *buf);
+extern int hyphenate_string(struct tex_language *lang, char *w, char **ret);
+
+extern void new_hyphenation(halfword h, halfword t);
+extern void clear_patterns(struct tex_language *lang);
+extern void clear_hyphenation(struct tex_language *lang);
+extern const char *clean_hyphenation(const char *buffer, char **cleaned);
+extern void hnj_hyphenation(halfword head, halfword tail);
+
+extern void set_pre_hyphen_char(int lan, int val);
+extern void set_post_hyphen_char(int lan, int val);
+extern int get_pre_hyphen_char(int lan);
+extern int get_post_hyphen_char(int lan);
+
+extern void set_pre_exhyphen_char(int lan, int val);
+extern void set_post_exhyphen_char(int lan, int val);
+extern int get_pre_exhyphen_char(int lan);
+extern int get_post_exhyphen_char(int lan);
+extern halfword compound_word_break(halfword t, int clang);
+
+extern void dump_language_data(void);
+extern void undump_language_data(void);
+extern char *exception_strings(struct tex_language *lang);
+
+extern void new_hyph_exceptions(void);
+extern void new_patterns(void);
+extern void new_pre_hyphen_char(void);
+extern void new_post_hyphen_char(void);
+extern void new_pre_exhyphen_char(void);
+extern void new_post_exhyphen_char(void);
+
+
+#endif
diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.c b/Build/source/texk/web2c/luatexdir/lang/texlang.w
index 5a17b2c34b6..6fcb7180820 100644
--- a/Build/source/texk/web2c/luatexdir/lang/texlang.c
+++ b/Build/source/texk/web2c/luatexdir/lang/texlang.w
@@ -1,105 +1,112 @@
-/* texlang.c
-
- Copyright 2006-2008 Taco Hoekwater <taco@luatex.org>
+% texlang.w
+%
+% Copyright 2006-2010 Taco Hoekwater <taco@@luatex.org>
- This file is part of LuaTeX.
+% This file is part of LuaTeX.
- LuaTeX is free software; you can redistribute it and/or modify it under
- the terms of the GNU General Public License as published by the Free
- Software Foundation; either version 2 of the License, or (at your
- option) any later version.
+% LuaTeX is free software; you can redistribute it and/or modify it under
+% the terms of the GNU General Public License as published by the Free
+% Software Foundation; either version 2 of the License, or (at your
+% option) any later version.
- LuaTeX is distributed in the hope that it will be useful, but WITHOUT
- ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
- License for more details.
+% LuaTeX is distributed in the hope that it will be useful, but WITHOUT
+% ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+% FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+% License for more details.
- You should have received a copy of the GNU General Public License along
- with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */
+% You should have received a copy of the GNU General Public License along
+% with LuaTeX; if not, see <http://www.gnu.org/licenses/>.
-#include "luatex-api.h"
-#include <ptexlib.h>
+@ @c
+#include "ptexlib.h"
#include <string.h>
-#include "nodes.h"
-#include "font/texfont.h"
-#include "hyphen.h"
+#include "lua/luatex-api.h"
+
static const char _svn_version[] =
- "$Id: texlang.c 2596 2009-06-25 09:26:59Z taco $ $URL: http://scm.foundry.supelec.fr/svn/luatex/trunk/src/texk/web2c/luatexdir/lang/texlang.c $";
+ "$Id: texlang.w 3584 2010-04-02 17:45:55Z hhenkel $ "
+"$URL: http://foundry.supelec.fr/svn/luatex/tags/beta-0.60.0/source/texk/web2c/luatexdir/lang/texlang.w $";
+
+
+@ Low-level helpers
-/* functions copied from the fontforge unicode library */
+@c
+static char *uni2string(char *utf8_text, unsigned ch)
+{
+ /* Increment and deposit character */
+ if (ch >= 17 * 65536)
+ return (utf8_text);
+
+ if (ch <= 127)
+ *utf8_text++ = (char) ch;
+ else if (ch <= 0x7ff) {
+ *utf8_text++ = (char) (0xc0 | (ch >> 6));
+ *utf8_text++ = (char) (0x80 | (ch & 0x3f));
+ } else if (ch <= 0xffff) {
+ *utf8_text++ = (char) (0xe0 | (ch >> 12));
+ *utf8_text++ = (char) (0x80 | ((ch >> 6) & 0x3f));
+ *utf8_text++ = (char) (0x80 | (ch & 0x3f));
+ } else {
+ unsigned val = ch - 0x10000;
+ unsigned u = ((val & 0xf0000) >> 16) + 1, z = (val & 0x0f000) >> 12, y =
+ (val & 0x00fc0) >> 6, x = val & 0x0003f;
+ *utf8_text++ = (char) (0xf0 | (u >> 2));
+ *utf8_text++ = (char) (0x80 | ((u & 3) << 4) | z);
+ *utf8_text++ = (char) (0x80 | y);
+ *utf8_text++ = (char) (0x80 | x);
+ }
+ return (utf8_text);
+}
-static unsigned int *utf82u_strcpy(unsigned int * ubuf, const char *utf8buf)
+static unsigned u_length(register unsigned int *str)
{
- int len = strlen(utf8buf) + 1;
+ register unsigned len = 0;
+ while (*str++ != '\0')
+ ++len;
+ return (len);
+}
+
+
+static void utf82u_strcpy(unsigned int *ubuf, const char *utf8buf)
+{
+ int len = (int) strlen(utf8buf) + 1;
unsigned int *upt = ubuf, *uend = ubuf + len - 1;
- const unsigned char *pt = (const unsigned char *) utf8buf, *end = pt + strlen(utf8buf);
+ const unsigned char *pt = (const unsigned char *) utf8buf, *end =
+ pt + strlen(utf8buf);
int w, w2;
while (pt < end && *pt != '\0' && upt < uend) {
if (*pt <= 127)
*upt = *pt++;
else if (*pt <= 0xdf) {
- *upt = ((*pt & 0x1f) << 6) | (pt[1] & 0x3f);
+ *upt = (unsigned int) (((*pt & 0x1f) << 6) | (pt[1] & 0x3f));
pt += 2;
} else if (*pt <= 0xef) {
- *upt = ((*pt & 0xf) << 12) | ((pt[1] & 0x3f) << 6) | (pt[2] & 0x3f);
+ *upt =
+ (unsigned int) (((*pt & 0xf) << 12) | ((pt[1] & 0x3f) << 6) |
+ (pt[2] & 0x3f));
pt += 3;
} else {
w = (((*pt & 0x7) << 2) | ((pt[1] & 0x30) >> 4)) - 1;
w = (w << 6) | ((pt[1] & 0xf) << 2) | ((pt[2] & 0x30) >> 4);
w2 = ((pt[2] & 0xf) << 6) | (pt[3] & 0x3f);
- *upt = w * 0x400 + w2 + 0x10000;
+ *upt = (unsigned int) (w * 0x400 + w2 + 0x10000);
pt += 4;
}
++upt;
}
*upt = '\0';
- return (ubuf);
-}
-
-char *utf8_idpb(char *utf8_text, unsigned int ch)
-{
- /* Increment and deposit character */
- if (ch >= 17 * 65536)
- return (utf8_text);
-
- if (ch <= 127)
- *utf8_text++ = ch;
- else if (ch <= 0x7ff) {
- *utf8_text++ = 0xc0 | (ch >> 6);
- *utf8_text++ = 0x80 | (ch & 0x3f);
- } else if (ch <= 0xffff) {
- *utf8_text++ = 0xe0 | (ch >> 12);
- *utf8_text++ = 0x80 | ((ch >> 6) & 0x3f);
- *utf8_text++ = 0x80 | (ch & 0x3f);
- } else {
- unsigned int val = ch - 0x10000;
- int u = ((val & 0xf0000) >> 16) + 1, z = (val & 0x0f000) >> 12, y =
- (val & 0x00fc0) >> 6, x = val & 0x0003f;
- *utf8_text++ = 0xf0 | (u >> 2);
- *utf8_text++ = 0x80 | ((u & 3) << 4) | z;
- *utf8_text++ = 0x80 | y;
- *utf8_text++ = 0x80 | x;
- }
- return (utf8_text);
-}
-
-static int u_strlen(register unsigned int * str)
-{
- register int len = 0;
- while (*str++ != '\0')
- ++len;
- return (len);
}
+@ @c
#define noVERBOSE
#define MAX_TEX_LANGUAGES 32768
static struct tex_language *tex_languages[MAX_TEX_LANGUAGES] = { NULL };
+
static int next_lang_id = 0;
struct tex_language *new_language(int n)
@@ -114,12 +121,12 @@ struct tex_language *new_language(int n)
} else {
while (tex_languages[next_lang_id] != NULL)
next_lang_id++;
- l = next_lang_id++;
+ l = (unsigned) next_lang_id++;
}
if (l < (MAX_TEX_LANGUAGES - 1) && tex_languages[l] == NULL) {
lang = xmalloc(sizeof(struct tex_language));
tex_languages[l] = lang;
- lang->id = l;
+ lang->id = (int) l;
lang->exceptions = 0;
lang->patterns = NULL;
lang->pre_hyphen_char = '-';
@@ -145,14 +152,15 @@ struct tex_language *get_language(int n)
}
}
-void set_pre_hyphen_char(integer n, integer v)
+@ @c
+void set_pre_hyphen_char(int n, int v)
{
struct tex_language *l = get_language((int) n);
if (l != NULL)
l->pre_hyphen_char = (int) v;
}
-void set_post_hyphen_char(integer n, integer v)
+void set_post_hyphen_char(int n, int v)
{
struct tex_language *l = get_language((int) n);
if (l != NULL)
@@ -160,14 +168,14 @@ void set_post_hyphen_char(integer n, integer v)
}
-void set_pre_exhyphen_char(integer n, integer v)
+void set_pre_exhyphen_char(int n, int v)
{
struct tex_language *l = get_language((int) n);
if (l != NULL)
l->pre_exhyphen_char = (int) v;
}
-void set_post_exhyphen_char(integer n, integer v)
+void set_post_exhyphen_char(int n, int v)
{
struct tex_language *l = get_language((int) n);
if (l != NULL)
@@ -175,40 +183,40 @@ void set_post_exhyphen_char(integer n, integer v)
}
-
-integer get_pre_hyphen_char(integer n)
+int get_pre_hyphen_char(int n)
{
struct tex_language *l = get_language((int) n);
if (l == NULL)
return -1;
- return (integer) l->pre_hyphen_char;
+ return (int) l->pre_hyphen_char;
}
-integer get_post_hyphen_char(integer n)
+int get_post_hyphen_char(int n)
{
struct tex_language *l = get_language((int) n);
if (l == NULL)
return -1;
- return (integer) l->post_hyphen_char;
+ return (int) l->post_hyphen_char;
}
-integer get_pre_exhyphen_char(integer n)
+int get_pre_exhyphen_char(int n)
{
struct tex_language *l = get_language((int) n);
if (l == NULL)
return -1;
- return (integer) l->pre_exhyphen_char;
+ return (int) l->pre_exhyphen_char;
}
-integer get_post_exhyphen_char(integer n)
+int get_post_exhyphen_char(int n)
{
struct tex_language *l = get_language((int) n);
if (l == NULL)
return -1;
- return (integer) l->post_exhyphen_char;
+ return (int) l->post_exhyphen_char;
}
+@ @c
void load_patterns(struct tex_language *lang, const unsigned char *buffer)
{
if (lang == NULL || buffer == NULL || strlen((const char *) buffer) == 0)
@@ -231,12 +239,13 @@ void clear_patterns(struct tex_language *lang)
void load_tex_patterns(int curlang, halfword head)
{
char *s = tokenlist_to_cstring(head, 1, NULL);
- load_patterns(get_language(curlang), (const unsigned char *) s);
+ load_patterns(get_language(curlang), (unsigned char *) s);
}
+@ @c
#define STORE_CHAR(x) do { \
- word[w] = x; \
+ word[w] = (unsigned char)x; \
if (w<MAX_WORD_LEN) w++; \
} while (0)
@@ -298,9 +307,11 @@ const char *clean_hyphenation(const char *buffer, char **cleaned)
return s;
}
+@ @c
void load_hyphenation(struct tex_language *lang, const unsigned char *buffer)
{
- const char *s, *value;
+ const char *s;
+ const char *value;
char *cleaned;
lua_State *L = Luas;
if (lang == NULL)
@@ -320,7 +331,7 @@ void load_hyphenation(struct tex_language *lang, const unsigned char *buffer)
if (cleaned != NULL) {
if ((s - value) > 0) {
lua_pushstring(L, cleaned);
- lua_pushlstring(L, value, (s - value));
+ lua_pushlstring(L, value, (size_t) (s - value));
lua_rawset(L, -3);
}
free(cleaned);
@@ -348,13 +359,14 @@ void clear_hyphenation(struct tex_language *lang)
void load_tex_hyphenation(int curlang, halfword head)
{
char *s = tokenlist_to_cstring(head, 1, NULL);
- load_hyphenation(get_language(curlang), (const unsigned char *) s);
+ load_hyphenation(get_language(curlang), (unsigned char *) s);
}
-/* TODO: clean this up. The delete_attribute_ref() statements are not very
+@ TODO: clean this up. The |delete_attribute_ref()| statements are not very
nice, but needed. Also, in the post-break, it would be nicer to get the
- attribute list from vlink(n). No rush, as it is currently not used much. */
+ attribute list from |vlink(n)|. No rush, as it is currently not used much.
+@c
halfword insert_discretionary(halfword t, halfword pre, halfword post,
halfword replace)
{
@@ -454,6 +466,7 @@ halfword insert_word_discretionary(halfword t, lang_variables * lan)
return insert_discretionary(t, pre, pos, null);
}
+@ @c
halfword compound_word_break(halfword t, int clang)
{
int disc;
@@ -486,7 +499,7 @@ halfword insert_character(halfword t, int c)
return p;
}
-
+@ @c
void set_disc_field(halfword f, halfword t)
{
if (t != null) {
@@ -500,7 +513,8 @@ void set_disc_field(halfword f, halfword t)
-static char *hyphenation_exception(int exceptions, char *w)
+@ @c
+char *hyphenation_exception(int exceptions, char *w)
{
char *ret = NULL;
lua_State *L = Luas;
@@ -520,6 +534,7 @@ static char *hyphenation_exception(int exceptions, char *w)
}
+@ @c
char *exception_strings(struct tex_language *lang)
{
const char *value;
@@ -537,8 +552,11 @@ char *exception_strings(struct tex_language *lang)
while (lua_next(L, -2) != 0) {
value = lua_tolstring(L, -1, &l);
if (current + 2 + l > size) {
- ret = xrealloc(ret, (1.2 * size) + current + l + 1024);
- size = (1.2 * size) + current + l + 1024;
+ ret =
+ xrealloc(ret,
+ (unsigned) ((size + size / 5) + current + l +
+ 1024));
+ size = (size + size / 5) + current + l + 1024;
}
*(ret + current) = ' ';
strcpy(ret + current + 1, value);
@@ -550,20 +568,21 @@ char *exception_strings(struct tex_language *lang)
}
-/* the sequence from |wordstart| to |r| can contain only normal characters */
-/* it could be faster to modify a halfword pointer and return an integer */
+@ the sequence from |wordstart| to |r| can contain only normal characters
+it could be faster to modify a halfword pointer and return an integer
-static halfword find_exception_part(unsigned int *j, int *uword, int len)
+@c
+halfword find_exception_part(unsigned int *j, unsigned int *uword, int len)
{
halfword g = null, gg = null;
- register int i = *j;
- i++; /* this puts uword[i] on the '{' */
- while (i < len && uword[i + 1] != '}') {
+ register unsigned i = *j;
+ i++; /* this puts uword[i] on the |{| */
+ while (i < (unsigned) len && uword[i + 1] != '}') {
if (g == null) {
- gg = new_char(0, uword[i + 1]);
+ gg = new_char(0, (int) uword[i + 1]);
g = gg;
} else {
- halfword s = new_char(0, uword[i + 1]);
+ halfword s = new_char(0, (int) uword[i + 1]);
couple_nodes(g, s);
g = vlink(g);
}
@@ -573,12 +592,12 @@ static halfword find_exception_part(unsigned int *j, int *uword, int len)
return gg;
}
-static int count_exception_part(unsigned int *j, int *uword, int len)
+int count_exception_part(unsigned int *j, unsigned int *uword, int len)
{
int ret = 0;
- register int i = *j;
- i++; /* this puts uword[i] on the '{' */
- while (i < len && uword[i + 1] != '}') {
+ register unsigned i = *j;
+ i++; /* this puts uword[i] on the |{| */
+ while (i < (unsigned) len && uword[i + 1] != '}') {
ret++;
i++;
}
@@ -587,22 +606,23 @@ static int count_exception_part(unsigned int *j, int *uword, int len)
}
+@ @c
static const char *PAT_ERROR[] = {
"Exception discretionaries should contain three pairs of braced items.",
"No intervening spaces are allowed.",
NULL
};
-static void do_exception(halfword wordstart, halfword r, char *replacement)
+void do_exception(halfword wordstart, halfword r, char *replacement)
{
unsigned i;
halfword t;
unsigned len;
int clang;
lang_variables langdata;
- int uword[MAX_WORD_LEN + 1] = { 0 };
- (void) utf82u_strcpy((unsigned int *) uword, replacement);
- len = u_strlen((unsigned int *) uword);
+ unsigned uword[MAX_WORD_LEN + 1] = { 0 };
+ utf82u_strcpy(uword, replacement);
+ len = u_length(uword);
i = 0;
t = wordstart;
clang = char_lang(wordstart);
@@ -624,15 +644,15 @@ static void do_exception(halfword wordstart, halfword r, char *replacement)
} else if (uword[i + 1] == '{') {
halfword gg, hh, replace = null;
int repl;
- gg = find_exception_part(&i, uword, len);
+ gg = find_exception_part(&i, uword, (int) len);
if (i == len || uword[i + 1] != '{') {
tex_error("broken pattern 1", PAT_ERROR);
}
- hh = find_exception_part(&i, uword, len);
+ hh = find_exception_part(&i, uword, (int) len);
if (i == len || uword[i + 1] != '{') {
tex_error("broken pattern 2", PAT_ERROR);
}
- repl = count_exception_part(&i, uword, len);
+ repl = count_exception_part(&i, uword, (int) len);
if (i == len) {
tex_error("broken pattern 3", PAT_ERROR);
}
@@ -659,7 +679,7 @@ static void do_exception(halfword wordstart, halfword r, char *replacement)
}
}
-/* This is a documentation section from the pascal web file. It is not
+@ This is a documentation section from the pascal web file. It is not
true any more, but I do not have time right now to rewrite it -- Taco
When the line-breaking routine is unable to find a feasible sequence of
@@ -722,15 +742,14 @@ When a word been set up to contain a candidate for hyphenation,
hyphens are inserted based on patterns that appear within the given word,
using an algorithm due to Frank~M. Liang.
@^Liang, Franklin Mark@>
-*/
-/*
- * This is incompatible with TEX because the first word of a paragraph
- * can be hyphenated, but most european users seem to agree that
- * prohibiting hyphenation there was not the best idea ever.
- */
-static halfword find_next_wordstart(halfword r)
+@ This is incompatible with TEX because the first word of a paragraph
+can be hyphenated, but most european users seem to agree that
+prohibiting hyphenation there was not the best idea ever.
+
+@c
+halfword find_next_wordstart(halfword r)
{
register int l;
register int start_ok = 1;
@@ -770,27 +789,30 @@ static halfword find_next_wordstart(halfword r)
return r;
}
-static int valid_wordend(halfword s)
+@ @c
+int valid_wordend(halfword s)
{
register halfword r = s;
register int clang = char_lang(s);
if (r == null)
return 1;
while ((r != null) && ((type(r) == glyph_node && is_simple_character(r)
- && clang == char_lang(r)) || (type(r) == kern_node
- && subtype(r) ==
- normal))) {
+ && clang == char_lang(r)) ||
+ (type(r) == kern_node && (subtype(r) == normal))
+ )) {
r = vlink(r);
}
if (r == null || (type(r) == glyph_node && is_simple_character(r)
&& clang != char_lang(r)) || type(r) == glue_node
|| type(r) == whatsit_node || type(r) == ins_node
|| type(r) == adjust_node || type(r) == penalty_node
- || (type(r) == kern_node && subtype(r) == explicit))
+ || (type(r) == kern_node
+ && (subtype(r) == explicit || subtype(r) == acc_kern)))
return 1;
return 0;
}
+@ @c
void hnj_hyphenation(halfword head, halfword tail)
{
int lchar, i;
@@ -804,17 +826,17 @@ void hnj_hyphenation(halfword head, halfword tail)
null, right = null;
/* this first movement assures two things:
- * a) that we won't waste lots of time on something that has been
- * handled already (in that case, none of the glyphs match |simple_character|).
- * b) that the first word can be hyphenated. if the movement was
- * not explicit, then the indentation at the start of a paragraph
- * list would make find_next_wordstart() look too far ahead.
+ \item{a)} that we won't waste lots of time on something that has been
+ handled already (in that case, none of the glyphs match |simple_character|).
+ \item{b)} that the first word can be hyphenated. if the movement was
+ not explicit, then the indentation at the start of a paragraph
+ list would make |find_next_wordstart()| look too far ahead.
*/
while (r != null && (type(r) != glyph_node || !is_simple_character(r))) {
r = vlink(r);
}
- /* this will make |r| a glyph node with subtype_character */
+ /* this will make |r| a glyph node with subtype character */
r = find_next_wordstart(r);
if (r == null)
return;
@@ -844,7 +866,7 @@ void hnj_hyphenation(halfword head, halfword tail)
clang == char_lang(r) &&
(lchar = get_lc_code(character(r))) > 0) {
wordlen++;
- hy = utf8_idpb(hy, lchar);
+ hy = uni2string(hy, (unsigned) lchar);
/* this should not be needed any more */
/*if (vlink(r)!=null) alink(vlink(r))=r; */
end_word = r;
@@ -897,12 +919,13 @@ void hnj_hyphenation(halfword head, halfword tail)
}
+@ @c
void new_hyphenation(halfword head, halfword tail)
{
register int callback_id = 0;
if (head == null || vlink(head) == null)
return;
- fix_node_list(head); /* TODO: use couple_nodes() in append_tail()! */
+ fix_node_list(head);
callback_id = callback_defined(hyphenate_callback);
if (callback_id > 0) {
lua_State *L = Luas;
@@ -924,21 +947,22 @@ void new_hyphenation(halfword head, halfword tail)
}
}
-/* dumping and undumping fonts */
+@ dumping and undumping languages
+@c
#define dump_string(a) \
if (a!=NULL) { \
- x = strlen(a)+1; \
+ x = (int)strlen(a)+1; \
dump_int(x); dump_things(*a, x); \
} else { \
x = 0; dump_int(x); \
}
-static void dump_one_language(int i)
+void dump_one_language(int i)
{
char *s = NULL;
- integer x = 0;
+ int x = 0;
struct tex_language *lang;
lang = tex_languages[i];
dump_int(lang->id);
@@ -965,7 +989,7 @@ static void dump_one_language(int i)
void dump_language_data(void)
{
- integer i;
+ int i;
dump_int(next_lang_id);
for (i = 0; i < next_lang_id; i++) {
if (tex_languages[i]) {
@@ -978,10 +1002,10 @@ void dump_language_data(void)
}
-static void undump_one_language(int i)
+void undump_one_language(int i)
{
char *s = NULL;
- integer x = 0;
+ int x = 0;
struct tex_language *lang = get_language(i);
undump_int(x);
lang->id = x;
@@ -996,24 +1020,24 @@ static void undump_one_language(int i)
/* patterns */
undump_int(x);
if (x > 0) {
- s = xmalloc(x);
+ s = xmalloc((unsigned) x);
undump_things(*s, x);
- load_patterns(lang, (const unsigned char *) s);
+ load_patterns(lang, (unsigned char *) s);
free(s);
}
/* exceptions */
undump_int(x);
if (x > 0) {
- s = xmalloc(x);
+ s = xmalloc((unsigned) x);
undump_things(*s, x);
- load_hyphenation(lang, (const unsigned char *) s);
+ load_hyphenation(lang, (unsigned char *) s);
free(s);
}
}
void undump_language_data(void)
{
- integer i, x, numlangs;
+ int i, x, numlangs;
undump_int(numlangs);
next_lang_id = numlangs;
for (i = 0; i < numlangs; i++) {
@@ -1023,3 +1047,65 @@ void undump_language_data(void)
}
}
}
+
+
+@ When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named
+|new_hyph_exceptions| to do the right thing.
+
+@c
+void new_hyph_exceptions(void)
+{ /* enters new exceptions */
+ (void) scan_toks(false, true);
+ load_tex_hyphenation(int_par(language_code), def_ref);
+ flush_list(def_ref);
+}
+
+@ Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a
+procedure named |new_patterns|.
+
+@c
+void new_patterns(void)
+{ /* initializes the hyphenation pattern data */
+ (void) scan_toks(false, true);
+ load_tex_patterns(int_par(language_code), def_ref);
+ flush_list(def_ref);
+}
+
+@ `\.{\\prehyphenchar}', sets the |pre_break| character, and
+`\.{\\posthyphenchar}' the |post_break| character. Their respective
+defaults are ascii hyphen ("-") and zero (nul).
+
+@c
+void new_pre_hyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_pre_hyphen_char(int_par(language_code), cur_val);
+}
+
+void new_post_hyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_post_hyphen_char(int_par(language_code), cur_val);
+}
+
+
+@ `\.{\\preexhyphenchar}', sets the |pre_break| character, and
+`\.{\\postexhyphenchar}' the |post_break| character. Their
+defaults are both zero (nul).
+
+@c
+void new_pre_exhyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_pre_exhyphen_char(int_par(language_code), cur_val);
+}
+
+void new_post_exhyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_post_exhyphen_char(int_par(language_code), cur_val);
+}