summaryrefslogtreecommitdiff
path: root/Build/source/texk/web2c/luatexdir/lang
diff options
context:
space:
mode:
authorKarl Berry <karl@freefriends.org>2021-02-25 19:22:25 +0000
committerKarl Berry <karl@freefriends.org>2021-02-25 19:22:25 +0000
commitad547a6b5986815fda458221149728d9d9ab1d87 (patch)
tree16296910eb3eca724371474ea9aea3994dc69614 /Build/source/texk/web2c/luatexdir/lang
parent947b43de3dd21d58ccc2ffadefc4441ea1c2a813 (diff)
restore Build,TODO from r57911
git-svn-id: svn://tug.org/texlive/trunk@57915 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Build/source/texk/web2c/luatexdir/lang')
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hnjalloc.c65
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hnjalloc.h42
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hyphen.c788
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/hyphen.h58
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/texlang.c1381
-rw-r--r--Build/source/texk/web2c/luatexdir/lang/texlang.h88
6 files changed, 2422 insertions, 0 deletions
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c
new file mode 100644
index 00000000000..3ebeee9e4d5
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.c
@@ -0,0 +1,65 @@
+/*
+
+hnjalloc.w
+
+LibHnj is dual licensed under LGPL and MPL. Boilerplate for both licenses
+follows.
+
+LibHnj - a library for high quality hyphenation and justification Copyright (C)
+1998 Raph Levien, (C) 2001 ALTLinux, Moscow
+
+This library is free software; you can redistribute it and/or modify it under the
+terms of the GNU Library General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This library is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public License along
+with this library; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place - Suite 330, Boston, MA 02111-1307 USA.
+
+The contents of this file are subject to the Mozilla Public License Version 1.0
+(the "MPL"); you may not use this file except in compliance with the MPL. You may
+obtain a copy of the MPL at http://www.mozilla.org/MPL/
+
+Software distributed under the MPL is distributed on an "AS IS" basis, WITHOUT
+WARRANTY OF ANY KIND, either express or implied. See the MPL for the specific
+language governing rights and limitations under the MPL.
+
+*/
+
+/*tex The wrappers for malloc */
+
+#include <stdlib.h>
+#include <stdio.h>
+#include "lang/hnjalloc.h"
+
+void *hnj_malloc(int size)
+{
+ void *p;
+
+ p = malloc((size_t) size);
+ if (p == NULL) {
+ fprintf(stderr, "can't allocate %d bytes\n", size);
+ exit(1);
+ }
+ return p;
+}
+
+void *hnj_realloc(void *p, int size)
+{
+ p = realloc(p, (size_t) size);
+ if (p == NULL) {
+ fprintf(stderr, "can't allocate %d bytes\n", size);
+ exit(1);
+ }
+ return p;
+}
+
+void hnj_free(void *p)
+{
+ free(p);
+}
diff --git a/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h
new file mode 100644
index 00000000000..1af5a3f0ea3
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/hnjalloc.h
@@ -0,0 +1,42 @@
+/* LibHnj is dual licensed under LGPL and MPL. Boilerplate for both
+ * licenses follows.
+ */
+
+/* LibHnj - a library for high quality hyphenation and justification
+ * Copyright (C) 1998 Raph Levien
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307 USA.
+*/
+
+/*
+ * The contents of this file are subject to the Mozilla Public License
+ * Version 1.0 (the "MPL"); you may not use this file except in
+ * compliance with the MPL. You may obtain a copy of the MPL at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the MPL is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL
+ * for the specific language governing rights and limitations under the
+ * MPL.
+ *
+ */
+/* wrappers for malloc */
+
+void *hnj_malloc(int size);
+
+void *hnj_realloc(void *p, int size);
+
+void hnj_free(void *p);
diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.c b/Build/source/texk/web2c/luatexdir/lang/hyphen.c
new file mode 100644
index 00000000000..3a96f195b2f
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.c
@@ -0,0 +1,788 @@
+/*
+
+hyphen.c
+
+This file is derived from libhnj which is is dual licensed under LGPL and MPL.
+Boilerplate for both licenses follows.
+
+LibHnj - a library for high quality hyphenation and justification
+
+(C) 1998 Raph Levien,
+(C) 2001 ALTLinux, Moscow (http://www.alt-linux.org),
+(C) 2001 Peter Novodvorsky (nidd@cs.msu.su)
+
+This library is free software; you can redistribute it and/or modify it under the
+terms of the GNU Library General Public License as published by the Free Software
+Foundation; either version 2 of the License, or (at your option) any later
+version.
+
+This library is distributed in the hope that it will be useful, but WITHOUT ANY
+WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A
+PARTICULAR PURPOSE. See the GNU Library General Public License for more details.
+
+You should have received a copy of the GNU Library General Public License along
+with this library; if not, write to the Free Software Foundation, Inc., 59 Temple
+Place - Suite 330, Boston, MA 02111-1307 USA.
+
+The contents of this file are subject to the Mozilla Public License Version 1.0
+(the "MPL"); you may not use this file except in compliance with the MPL. You may
+obtain a copy of the MPL at http://www.mozilla.org/MPL/
+
+Software distributed under the MPL is distributed on an "AS IS" basis, WITHOUT
+WARRANTY OF ANY KIND, either express or implied. See the MPL for the specific
+language governing rights and limitations under the MPL.
+
+*/
+
+#include "ptexlib.h"
+#include "lua/luatex-api.h"
+
+#include <stdlib.h> /* for NULL, malloc */
+#include <stdio.h> /* for fprintf */
+#include <string.h> /* for strdup */
+#include <stdlib.h> /* for malloc used by substring inclusion */
+
+#define MAXPATHS 40960
+
+#ifdef UNX
+# include <unistd.h> /* for exit */
+#endif
+
+#include <kpathsea/c-ctype.h>
+
+#define noVERBOSE
+
+#include "lang/hnjalloc.h"
+
+/*tex This could be moved to separate library. */
+
+static unsigned char *hnj_strdup(const unsigned char *s)
+{
+ unsigned char *new;
+ size_t l;
+
+ l = strlen((const char *) s);
+ new = hnj_malloc((int) l + 1);
+ memcpy(new, s, l);
+ new[l] = 0;
+ return new;
+}
+
+/*tex
+
+ First some type definitions and a little bit of a hash table implementation.
+ This simply maps strings to state numbers
+
+*/
+
+typedef struct _HashTab HashTab;
+typedef struct _HashEntry HashEntry;
+typedef struct _HashIter HashIter;
+typedef union _HashVal HashVal;
+
+/*tex A cheap, but effective, hack. */
+
+#define HASH_SIZE 31627
+
+struct _HashTab {
+ HashEntry *entries[HASH_SIZE];
+};
+
+union _HashVal {
+ int state;
+ char *hyppat;
+};
+
+struct _HashEntry {
+ HashEntry *next;
+ unsigned char *key;
+ HashVal u;
+};
+
+struct _HashIter {
+ HashEntry **e;
+ HashEntry *cur;
+ int ndx;
+};
+
+/* The state state machine. */
+
+typedef struct _HyphenState HyphenState;
+typedef struct _HyphenTrans HyphenTrans;
+
+#define MAX_CHARS 256
+#define MAX_NAME 20
+
+struct _HyphenDict {
+ int num_states;
+ int pat_length;
+ char cset[MAX_NAME];
+ HyphenState *states;
+ HashTab *patterns;
+ HashTab *merged;
+ HashTab *state_num;
+};
+
+struct _HyphenState {
+ char *match;
+ /*tex Removed:
+
+ \starttyping
+ char *repl;
+ signed char replindex;
+ signed char replcut;
+ \stoptyping
+ */
+ int fallback_state;
+ int num_trans;
+ HyphenTrans *trans;
+};
+
+struct _HyphenTrans {
+ int uni_ch;
+ int new_state;
+};
+
+/*tex
+
+ Combine two right-aligned number patterns, 04000 + 020 becomes 04020. This
+ works also for utf8 sequences because the substring is identical to the last
+ substring-length bytes of expr except for the (single byte) hyphenation
+ encoders
+
+*/
+
+static char *combine(char *expr, const char *subexpr)
+{
+ size_t l1 = strlen(expr);
+ size_t l2 = strlen(subexpr);
+ size_t off = l1 - l2;
+ unsigned j;
+ for (j = 0; j < l2; j++) {
+ if (expr[off + j] < subexpr[j])
+ expr[off + j] = subexpr[j];
+ }
+ return expr;
+}
+
+/*tex Some original code: */
+
+static HashIter *new_HashIter(HashTab * h)
+{
+ HashIter *i = hnj_malloc(sizeof(HashIter));
+ i->e = h->entries;
+ i->cur = NULL;
+ i->ndx = -1;
+ return i;
+}
+
+static int nextHashStealPattern(HashIter * i, unsigned char **word, char **pattern)
+{
+ while (i->cur == NULL) {
+ if (i->ndx >= HASH_SIZE - 1)
+ return 0;
+ i->cur = i->e[++i->ndx];
+ }
+ *word = i->cur->key;
+ *pattern = i->cur->u.hyppat;
+ i->cur->u.hyppat = NULL;
+ i->cur = i->cur->next;
+ return 1;
+}
+
+
+static int nextHash(HashIter * i, unsigned char **word)
+{
+ while (i->cur == NULL) {
+ if (i->ndx >= HASH_SIZE - 1)
+ return 0;
+ i->cur = i->e[++i->ndx];
+ }
+ *word = i->cur->key;
+ i->cur = i->cur->next;
+ return 1;
+}
+
+static int eachHash(HashIter * i, unsigned char **word, char **pattern)
+{
+ while (i->cur == NULL) {
+ if (i->ndx >= HASH_SIZE - 1)
+ return 0;
+ i->cur = i->e[++i->ndx];
+ }
+ *word = i->cur->key;
+ *pattern = i->cur->u.hyppat;
+ i->cur = i->cur->next;
+ return 1;
+}
+
+static void delete_HashIter(HashIter * i)
+{
+ hnj_free(i);
+}
+
+/*tex
+
+A |char*| hash function from ASU, adapted from |Gtk+|:
+
+*/
+
+static unsigned int hnj_string_hash(const unsigned char *s)
+{
+ const unsigned char *p;
+ unsigned int h = 0, g;
+
+ for (p = s; *p != '\0'; p += 1) {
+ h = (h << 4) + *p;
+ if ((g = (h & 0xf0000000))) {
+ h = h ^ (g >> 24);
+ h = h ^ g;
+ }
+ }
+ return h;
+}
+
+/*tex This assumes that key is not already present! */
+
+static void state_insert(HashTab * hashtab, unsigned char *key, int state)
+{
+ int i;
+ HashEntry *e;
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ e = hnj_malloc(sizeof(HashEntry));
+ e->next = hashtab->entries[i];
+ e->key = key;
+ e->u.state = state;
+ hashtab->entries[i] = e;
+}
+
+/*tex This also assumes that key is not already present! */
+
+static void hyppat_insert(HashTab * hashtab, unsigned char *key, char *hyppat)
+{
+ int i;
+ HashEntry *e;
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ for (e = hashtab->entries[i]; e; e = e->next) {
+ if (strcmp((char *) e->key, (char *) key) == 0) {
+ if (e->u.hyppat) {
+ if (hyppat && strcmp((char *) e->u.hyppat, (char *) hyppat) != 0) {
+ normal_warning("hyphenation","a conflicting pattern has been ignored");
+ }
+ hnj_free(e->u.hyppat);
+ }
+ e->u.hyppat = hyppat;
+ hnj_free(key);
+ return;
+ }
+ }
+ e = hnj_malloc(sizeof(HashEntry));
+ e->next = hashtab->entries[i];
+ e->key = key;
+ e->u.hyppat = hyppat;
+ hashtab->entries[i] = e;
+}
+
+/*tex We return |state| if found, otherwise |-1|. */
+
+static int state_lookup(HashTab * hashtab, const unsigned char *key)
+{
+ int i;
+ HashEntry *e;
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ for (e = hashtab->entries[i]; e; e = e->next) {
+ if (!strcmp((const char *) key, (const char *) e->key)) {
+ return e->u.state;
+ }
+ }
+ return -1;
+}
+
+/*tex We return |state| if found, otherwise |-1|. */
+
+static char *hyppat_lookup(HashTab * hashtab, const unsigned char *chars, int l)
+{
+ int i;
+ HashEntry *e;
+ /*tex The 256 should be enough. */
+ unsigned char key[256];
+ strncpy((char *) key, (const char *) chars, (size_t) l);
+ key[l] = 0;
+ i = (int) (hnj_string_hash(key) % HASH_SIZE);
+ for (e = hashtab->entries[i]; e; e = e->next) {
+ if (!strcmp((char *) key, (char *) e->key)) {
+ return e->u.hyppat;
+ }
+ }
+ return NULL;
+}
+
+/*tex Get the state number, allocating a new state if necessary. */
+
+static int hnj_get_state(HyphenDict * dict, const unsigned char *str, int *state_num)
+{
+ *state_num = state_lookup(dict->state_num, str);
+ if (*state_num >= 0)
+ return *state_num;
+ state_insert(dict->state_num, hnj_strdup(str), dict->num_states);
+ /*tex The predicate is true if |dict->num_states| is a power of two: */
+ if (!(dict->num_states & (dict->num_states - 1))) {
+ dict->states = hnj_realloc(dict->states,
+ (int) ((dict->num_states << 1) * (int) sizeof(HyphenState)));
+ }
+ dict->states[dict->num_states].match = NULL;
+ dict->states[dict->num_states].fallback_state = -1;
+ dict->states[dict->num_states].num_trans = 0;
+ dict->states[dict->num_states].trans = NULL;
+ return dict->num_states++;
+}
+
+/*tex
+
+ Add a transition from state1 to state2 through ch - assumes that the
+ transition does not already exist.
+
+*/
+
+static void hnj_add_trans(HyphenDict * dict, int state1, int state2, int uni_ch)
+{
+ int num_trans;
+ /*
+ TH: this test was a bit too strict, it is quite normal for old patterns
+ to have chars in the range 0-31 or 127-159 (inclusive). To ease the
+ transition, let's only disallow |NUL| for now, which probably is a
+ requirement of the code anyway.
+ */
+ if (uni_ch == 0) {
+ formatted_error("hyphenation","a character is out of bounds: u%04x", uni_ch);
+ }
+ num_trans = dict->states[state1].num_trans;
+ if (num_trans == 0) {
+ dict->states[state1].trans = hnj_malloc(sizeof(HyphenTrans));
+ } else {
+ /*
+ TH: The old version did:
+
+ \starttyping
+ } else if (!(num_trans & (num_trans - 1))) {
+ ... =hnj_realloc(dict->states[state1].trans,
+ (int) ((num_trans << 1) * sizeof(HyphenTrans)));
+ \stoptyping
+
+ but that is incredibly nasty when adding patters one-at-a-time.
+ Controlled growth would be nicer than the current +1, but if no one
+ complains, and no one did in a decade, this is good enough.
+
+ */
+ dict->states[state1].trans = hnj_realloc(dict->states[state1].trans,
+ (int) ((num_trans + 1) * sizeof(HyphenTrans)));
+ }
+ dict->states[state1].trans[num_trans].uni_ch = uni_ch;
+ dict->states[state1].trans[num_trans].new_state = state2;
+ dict->states[state1].num_trans++;
+}
+
+/*tex
+
+ We did change the semantics a bit here: |hnj_hyphen_load| used to operate on
+ a file, but now the argument is a string buffer.
+
+*/
+
+static const unsigned char *next_pattern(size_t * length, const unsigned char **buf)
+{
+ const unsigned char *here, *rover = *buf;
+ while (*rover && isspace(*rover))
+ rover++;
+ here = rover;
+ while (*rover) {
+ if (isspace(*rover)) {
+ *length = (size_t) (rover - here);
+ *buf = rover;
+ return here;
+ }
+ rover++;
+ }
+ *length = (size_t) (rover - here);
+ *buf = rover;
+ /*tex Zero sensed: */
+ return *length ? here : NULL;
+}
+
+static void init_hash(HashTab ** h)
+{
+ int i;
+ if (*h) {
+ return;
+ }
+ *h = hnj_malloc(sizeof(HashTab));
+ for (i = 0; i < HASH_SIZE; i++) {
+ (*h)->entries[i] = NULL;
+ }
+}
+
+static void clear_state_hash(HashTab ** h)
+{
+ int i;
+ if (*h == NULL) {
+ return;
+ }
+ for (i = 0; i < HASH_SIZE; i++) {
+ HashEntry *e, *next;
+ for (e = (*h)->entries[i]; e; e = next) {
+ next = e->next;
+ hnj_free(e->key);
+ hnj_free(e);
+ }
+ }
+ hnj_free(*h);
+ *h = NULL;
+}
+
+static void clear_hyppat_hash(HashTab ** h)
+{
+ int i;
+ if (*h == NULL) {
+ return;
+ }
+ for (i = 0; i < HASH_SIZE; i++) {
+ HashEntry *e, *next;
+ for (e = (*h)->entries[i]; e; e = next) {
+ next = e->next;
+ hnj_free(e->key);
+ if (e->u.hyppat) {
+ hnj_free(e->u.hyppat);
+ }
+ hnj_free(e);
+ }
+ }
+ hnj_free(*h);
+ *h = NULL;
+}
+
+static void init_dict(HyphenDict * dict)
+{
+ dict->num_states = 1;
+ dict->pat_length = 0;
+ dict->states = hnj_malloc(sizeof(HyphenState));
+ dict->states[0].match = NULL;
+ dict->states[0].fallback_state = -1;
+ dict->states[0].num_trans = 0;
+ dict->states[0].trans = NULL;
+ dict->patterns = NULL;
+ dict->merged = NULL;
+ dict->state_num = NULL;
+ init_hash(&dict->patterns);
+}
+
+static void clear_dict(HyphenDict * dict)
+{
+ int state_num;
+ for (state_num = 0; state_num < dict->num_states; state_num++) {
+ HyphenState *hstate = &dict->states[state_num];
+ if (hstate->match)
+ hnj_free(hstate->match);
+ if (hstate->trans)
+ hnj_free(hstate->trans);
+ }
+ hnj_free(dict->states);
+ clear_hyppat_hash(&dict->patterns);
+ clear_hyppat_hash(&dict->merged);
+ clear_state_hash(&dict->state_num);
+}
+
+HyphenDict *hnj_hyphen_new(void)
+{
+ HyphenDict *dict = hnj_malloc(sizeof(HyphenDict));
+ init_dict(dict);
+ return dict;
+}
+
+void hnj_hyphen_clear(HyphenDict * dict)
+{
+ clear_dict(dict);
+ init_dict(dict);
+}
+
+void hnj_hyphen_free(HyphenDict * dict)
+{
+ clear_dict(dict);
+ hnj_free(dict);
+}
+
+unsigned char *hnj_serialize(HyphenDict * dict)
+{
+ HashIter *v;
+ unsigned char *word;
+ char *pattern;
+ unsigned char *buf = hnj_malloc(dict->pat_length);
+ unsigned char *cur = buf;
+ v = new_HashIter(dict->patterns);
+ while (eachHash(v, &word, &pattern)) {
+ int i = 0, e = 0;
+ while (word[e + i]) {
+ if (pattern[i] != '0')
+ *cur++ = (unsigned char) pattern[i];
+ *cur++ = word[e + i++];
+ while (is_utf8_follow(word[e + i]))
+ *cur++ = word[i + e++];
+ }
+ if (pattern[i] != '0')
+ *cur++ = (unsigned char) pattern[i];
+ *cur++ = ' ';
+ }
+ delete_HashIter(v);
+ *cur = 0;
+ return buf;
+}
+
+void hnj_free_serialize(unsigned char *c)
+{
+ hnj_free(c);
+}
+
+/*tex
+
+ In hyphenation patterns we use signed bytes where |0|, or actually any
+ negative number, indicates end:
+
+ \starttyping
+ prio(1+),startpos,length,len1,[replace],len2,[replace]
+ \starttyping
+
+ A basic example is:
+
+ \starttyping
+ p n 0 0 0
+ \starttyping
+
+ for a hyphenation point between characters.
+
+*/
+
+void hnj_hyphen_load(HyphenDict * dict, const unsigned char *f)
+{
+ int state_num, last_state;
+ int ch;
+ int found;
+ HashEntry *e;
+ HashIter *v;
+ unsigned char *word;
+ char *pattern;
+ size_t l = 0;
+ const unsigned char *format;
+ const unsigned char *begin = f;
+ unsigned char *pat;
+ char *org;
+ while ((format = next_pattern(&l, &f)) != NULL) {
+ int i, j, e1;
+ if (l>=255) {
+ normal_warning("hyphenation","a pattern of more than 254 bytes ignored");
+ continue;
+ }
+ for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) {
+ if (format[i] >= '0' && format[i] <= '9')
+ j++;
+ if (is_utf8_follow(format[i]))
+ e1++;
+ }
+ /*tex
+ Here |l-e1| is the number of {\em characters} not {\em bytes}, |l-j|
+ the number of pattern bytes and |l-e1-j| the number of pattern
+ characters.
+ */
+ pat = (unsigned char *) malloc((1 + l - (size_t) j));
+ org = (char *) malloc((size_t) (2 + l - (size_t) e1 - (size_t) j));
+ /*tex Remove hyphenation encoders (digits) from pat. */
+ org[0] = '0';
+ for (i = 0, j = 0, e1 = 0; (unsigned) i < l; i++) {
+ unsigned char c = format[i];
+ if (is_utf8_follow(c)) {
+ pat[j + e1++] = c;
+ } else if (c < '0' || c > '9') {
+ pat[e1 + j++] = c;
+ org[j] = '0';
+ } else {
+ org[j] = (char) c;
+ }
+ }
+ pat[e1 + j] = 0;
+ org[j + 1] = 0;
+ hyppat_insert(dict->patterns, pat, org);
+ }
+ /*tex We add 2 bytes for spurious spaces. */
+ dict->pat_length += (int) ((f - begin) + 2);
+ init_hash(&dict->merged);
+ v = new_HashIter(dict->patterns);
+ while (nextHash(v, &word)) {
+ int wordsize = (int) strlen((char *) word);
+ int j1, l1;
+ for (l1 = 1; l1 <= wordsize; l1++) {
+ if (is_utf8_follow(word[l1])) {
+ /*tex Do not clip an utf8 sequence. */
+ continue;
+ }
+ for (j1 = 1; j1 <= l1; j1++) {
+ char *subpat_pat;
+ int i1 = l1 - j1;
+ if (is_utf8_follow(word[i1])) {
+ /*tex Do not start halfway an utf8 sequence. */
+ continue;
+ }
+ if ((subpat_pat =
+ hyppat_lookup(dict->patterns, word + i1, j1)) != NULL) {
+ char *newpat_pat;
+ if ((newpat_pat =
+ hyppat_lookup(dict->merged, word, l1)) == NULL) {
+ char *neworg;
+ unsigned char *newword =
+ (unsigned char *) malloc((size_t) (l1 + 1));
+ int e1 = 0;
+ strncpy((char *) newword, (char *) word, (size_t) l1);
+ newword[l1] = 0;
+ for (i1 = 0; i1 < l1; i1++)
+ if (is_utf8_follow(newword[i1]))
+ e1++;
+ neworg = malloc((size_t) (l1 + 2 - e1));
+ /*tex Fill with right amount of zeros: */
+ sprintf(neworg, "%0*d", l1 + 1 - e1, 0);
+ hyppat_insert(dict->merged, newword, combine(neworg, subpat_pat));
+ } else {
+ combine(newpat_pat, subpat_pat);
+ }
+ }
+ }
+ }
+ }
+ delete_HashIter(v);
+ init_hash(&dict->state_num);
+ state_insert(dict->state_num, hnj_strdup((const unsigned char *) ""), 0);
+ v = new_HashIter(dict->merged);
+ while (nextHashStealPattern(v, &word, &pattern)) {
+ static unsigned char mask[] = { 0x3F, 0x1F, 0xF, 0x7 };
+ int j1 = (int) strlen((char *) word);
+ state_num = hnj_get_state(dict, word, &found);
+ dict->states[state_num].match = pattern;
+ /*tex Now, put in the prefix transitions. */
+ while (found < 0) {
+ j1--;
+ last_state = state_num;
+ ch = word[j1];
+ if (ch >= 0x80) {
+ int m;
+ int i1 = 1;
+ while (is_utf8_follow(word[j1 - i1]))
+ i1++;
+ ch = word[j1 - i1] & mask[i1];
+ m = j1 - i1;
+ while (i1--) {
+ ch = (ch << 6) + (0x3F & word[j1 - i1]);
+ }
+ j1 = m;
+ }
+ word[j1] = '\0';
+ state_num = hnj_get_state(dict, word, &found);
+ hnj_add_trans(dict, state_num, last_state, ch);
+ }
+ }
+ delete_HashIter(v);
+ clear_hyppat_hash(&dict->merged);
+ /*tex Put in the fallback states. */
+ {
+ int i, j = 0;
+ for (i = 0; i < HASH_SIZE; i++) {
+ for (e = dict->state_num->entries[i]; e; e = e->next) {
+ /*tex Do not do |state==0| otherwise things get confused. */
+ if (e->u.state) {
+ for (j = 1; 1; j++) {
+ state_num = state_lookup(dict->state_num, e->key + j);
+ if (state_num >= 0)
+ break;
+ }
+ dict->states[e->u.state].fallback_state = state_num;
+ }
+ }
+ }
+ }
+ clear_state_hash(&dict->state_num);
+}
+
+extern halfword insert_syllable_discretionary(halfword t, lang_variables * lan);
+
+void hnj_hyphen_hyphenate(HyphenDict * dict, halfword first1, halfword last1,
+ int length, halfword left, halfword right, lang_variables * lan)
+{
+ int char_num;
+ halfword here;
+ int state = 0;
+ /*tex +2 for dots at each end, +1 for points outside characters. */
+ int ext_word_len = length + 2;
+ int hyphen_len = ext_word_len + 1;
+ char *hyphens = hnj_malloc(hyphen_len + 1);
+ /*tex Add a '.' to beginning and end to facilitate matching. */
+ vlink(begin_point) = first1;
+ vlink(end_point) = vlink(last1);
+ vlink(last1) = end_point;
+ for (char_num = 0; char_num < hyphen_len; char_num++) {
+ hyphens[char_num] = '0';
+ }
+ hyphens[hyphen_len] = 0;
+ /*tex Now, run the finite state machine. */
+ for (char_num = 0, here = begin_point; here != vlink(end_point); here = vlink(here)) {
+ int ch;
+ if (here == begin_point || here == end_point) {
+ ch = '.';
+ } else {
+ ch = get_hj_code(char_lang(here),character(here));
+ if (ch <= 32) {
+ ch = character(here);
+ }
+ }
+ while (state != -1) {
+ HyphenState *hstate = &dict->states[state];
+ int k;
+ for (k = 0; k < hstate->num_trans; k++) {
+ if (hstate->trans[k].uni_ch == ch) {
+ char *match;
+ state = hstate->trans[k].new_state;
+ match = dict->states[state].match;
+ if (match) {
+ /*tex
+
+ We add +2 because 1 string length is one bigger than offset
+ and 1 hyphenation starts before first character.
+ */
+ int offset = (int) (char_num + 2 - (int) strlen(match));
+ int m;
+ for (m = 0; match[m]; m++) {
+ if (hyphens[offset + m] < match[m])
+ hyphens[offset + m] = match[m];
+ }
+ }
+ goto try_next_letter;
+ }
+ }
+ state = hstate->fallback_state;
+ }
+ /*tex Nothing worked, let's go to the next character. */
+ state = 0;
+ try_next_letter:;
+ char_num++;
+ }
+ /*tex Restore the correct pointers. */
+ vlink(last1) = vlink(end_point);
+ /*tex
+
+ Pattern is \.{\^.\^w\^o\^r\^d\^.\^} and |word_len|=4, |ext_word_len|=6,
+ |hyphens|=7; check \.{ \^ \^ \^ } and therefore drop first two and stop
+ after |word_len-1|
+ */
+ for (here = first1, char_num = 2; here != left; here = vlink(here))
+ char_num++;
+ for (; here != right; here = vlink(here)) {
+ if (hyphens[char_num] & 1)
+ here = insert_syllable_discretionary(here, lan);
+ char_num++;
+ }
+ hnj_free(hyphens);
+}
diff --git a/Build/source/texk/web2c/luatexdir/lang/hyphen.h b/Build/source/texk/web2c/luatexdir/lang/hyphen.h
new file mode 100644
index 00000000000..669f2258f7a
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/hyphen.h
@@ -0,0 +1,58 @@
+/* LibHnj is dual licensed under LGPL and MPL. Boilerplate for both
+ * licenses follows.
+ */
+
+/* LibHnj - a library for high quality hyphenation and justification
+ * Copyright (C) 1998 Raph Levien, (C) 2001 ALTLinux, Moscow
+ *
+ * This library is free software; you can redistribute it and/or
+ * modify it under the terms of the GNU Library General Public
+ * License as published by the Free Software Foundation; either
+ * version 2 of the License, or (at your option) any later version.
+ *
+ * This library is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
+ * Library General Public License for more details.
+ *
+ * You should have received a copy of the GNU Library General Public
+ * License along with this library; if not, write to the
+ * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
+ * Boston, MA 02111-1307 USA.
+*/
+
+/*
+ * The contents of this file are subject to the Mozilla Public License
+ * Version 1.0 (the "MPL"); you may not use this file except in
+ * compliance with the MPL. You may obtain a copy of the MPL at
+ * http://www.mozilla.org/MPL/
+ *
+ * Software distributed under the MPL is distributed on an "AS IS" basis,
+ * WITHOUT WARRANTY OF ANY KIND, either express or implied. See the MPL
+ * for the specific language governing rights and limitations under the
+ * MPL.
+ *
+ */
+#ifndef __HYPHEN_H__
+# define __HYPHEN_H__
+
+# ifdef __cplusplus
+extern "C" {
+# endif /* __cplusplus */
+
+ typedef struct _HyphenDict HyphenDict;
+
+ HyphenDict *hnj_hyphen_new(void);
+ void hnj_hyphen_load(HyphenDict * dict, const unsigned char *fn);
+ void hnj_hyphen_free(HyphenDict * dict);
+ void hnj_hyphen_clear(HyphenDict * dict);
+ void hnj_hyphen_hyphenate(HyphenDict * dict, halfword first, halfword last,
+ int size, halfword left, halfword right,
+ lang_variables * lan);
+ unsigned char *hnj_serialize(HyphenDict *);
+ void hnj_free_serialize(unsigned char *);
+
+# ifdef __cplusplus
+}
+# endif /* __cplusplus */
+#endif /* __HYPHEN_H__ */
diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.c b/Build/source/texk/web2c/luatexdir/lang/texlang.c
new file mode 100644
index 00000000000..ba7614ffafc
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/texlang.c
@@ -0,0 +1,1381 @@
+/*
+
+texlang.w
+
+Copyright 2006-2012 Taco Hoekwater <taco@luatex.org>
+
+This file is part of LuaTeX.
+
+LuaTeX is free software; you can redistribute it and/or modify it under
+the terms of the GNU General Public License as published by the Free
+Software Foundation; either version 2 of the License, or (at your
+option) any later version.
+
+LuaTeX is distributed in the hope that it will be useful, but WITHOUT
+ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+License for more details.
+
+You should have received a copy of the GNU General Public License along
+with LuaTeX; if not, see <http://www.gnu.org/licenses/>.
+
+*/
+
+#include "ptexlib.h"
+#include <string.h>
+#include "lua/luatex-api.h"
+
+/*tex Low-level helpers */
+
+#define unVERBOSE
+
+#define MAX_TEX_LANGUAGES 16384
+
+static struct tex_language *tex_languages[MAX_TEX_LANGUAGES] = { NULL };
+
+static int next_lang_id = 0;
+
+struct tex_language *new_language(int n)
+{
+ struct tex_language *lang;
+ unsigned l;
+ if (n >= 0) {
+ l = (unsigned) n;
+ if (l != (MAX_TEX_LANGUAGES - 1))
+ if (next_lang_id <= n)
+ next_lang_id = n + 1;
+ } else {
+ while (tex_languages[next_lang_id] != NULL)
+ next_lang_id++;
+ l = (unsigned) next_lang_id++;
+ }
+ if (l < (MAX_TEX_LANGUAGES - 1) && tex_languages[l] == NULL) {
+ lang = xmalloc(sizeof(struct tex_language));
+ tex_languages[l] = lang;
+ lang->id = (int) l;
+ lang->exceptions = 0;
+ lang->patterns = NULL;
+ lang->pre_hyphen_char = '-';
+ lang->post_hyphen_char = 0;
+ lang->pre_exhyphen_char = 0;
+ lang->post_exhyphen_char = 0;
+ lang->hyphenation_min = -1;
+ if (saving_hyph_codes_par) {
+ /*tex
+ For now, we might just use specific value for whatever task.
+ */
+ hj_codes_from_lc_codes(l);
+ }
+ return lang;
+ } else {
+ return NULL;
+ }
+}
+
+struct tex_language *get_language(int n)
+{
+ if (n >= 0 && n < MAX_TEX_LANGUAGES) {
+ if (tex_languages[n] != NULL) {
+ return tex_languages[n];
+ } else {
+ return new_language(n);
+ }
+ } else {
+ return NULL;
+ }
+}
+
+void set_pre_hyphen_char(int n, int v)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l != NULL)
+ l->pre_hyphen_char = (int) v;
+}
+
+void set_post_hyphen_char(int n, int v)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l != NULL)
+ l->post_hyphen_char = (int) v;
+}
+
+void set_pre_exhyphen_char(int n, int v)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l != NULL)
+ l->pre_exhyphen_char = (int) v;
+}
+
+void set_post_exhyphen_char(int n, int v)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l != NULL)
+ l->post_exhyphen_char = (int) v;
+}
+
+int get_pre_hyphen_char(int n)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l == NULL)
+ return -1;
+ return (int) l->pre_hyphen_char;
+}
+
+int get_post_hyphen_char(int n)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l == NULL)
+ return -1;
+ return (int) l->post_hyphen_char;
+}
+
+int get_pre_exhyphen_char(int n)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l == NULL)
+ return -1;
+ return (int) l->pre_exhyphen_char;
+}
+
+int get_post_exhyphen_char(int n)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l == NULL)
+ return -1;
+ return (int) l->post_exhyphen_char;
+}
+
+void set_hyphenation_min(int n, int v)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l != NULL)
+ l->hyphenation_min = (int) v;
+}
+
+int get_hyphenation_min(int n)
+{
+ struct tex_language *l = get_language((int) n);
+ if (l == NULL)
+ return -1;
+ return (int) l->hyphenation_min;
+}
+
+void load_patterns(struct tex_language *lang, const unsigned char *buff)
+{
+ if (lang == NULL || buff == NULL || strlen((const char *) buff) == 0)
+ return;
+ if (lang->patterns == NULL) {
+ lang->patterns = hnj_hyphen_new();
+ }
+ hnj_hyphen_load(lang->patterns, buff);
+}
+
+void clear_patterns(struct tex_language *lang)
+{
+ if (lang == NULL)
+ return;
+ if (lang->patterns != NULL) {
+ hnj_hyphen_clear(lang->patterns);
+ }
+}
+
+void load_tex_patterns(int curlang, halfword head)
+{
+ char *s = tokenlist_to_cstring(head, 1, NULL);
+ load_patterns(get_language(curlang), (unsigned char *) s);
+}
+
+#define STORE_CHAR(l,x) do { \
+ unsigned xx = get_hj_code(l,x); \
+ if (!xx || xx <= 32) { \
+ xx = x; \
+ } \
+ uindex = uni2string(uindex, xx); \
+} while (0)
+
+/*
+
+ This cleans one word which is returned in |cleaned|, returns the new offset
+ into |buffer|.
+
+*/
+
+const char *clean_hyphenation(int id, const char *buff, char **cleaned)
+{
+ int items = 0;
+ /*tex Work buffer for bytes: */
+ unsigned char word[MAX_WORD_LEN + 1];
+ /*tex Work buffer for \UNICODE: */
+ unsigned uword[MAX_WORD_LEN + 1] = { 0 };
+ /*tex The \UNICODE\ buffer value: */
+ int u = 0;
+ /*tex The index into buffer: */
+ int i = 0;
+ char *uindex = (char *)word;
+ const char *s = buff;
+
+ while (*s && !isspace((unsigned char)*s)) {
+ word[i++] = (unsigned)*s;
+ s++;
+ if ((s-buff)>MAX_WORD_LEN) {
+ /*tex Todo: this is too strict, should count \UNICODE, not bytes. */
+ *cleaned = NULL;
+ tex_error("exception too long", NULL);
+ return s;
+ }
+ }
+ /*tex Now convert the input to \UNICODE. */
+ word[i] = '\0';
+ utf2uni_strcpy(uword, (const char *)word);
+ /*tex Build the new word string. */
+ i = 0;
+ while (uword[i]>0) {
+ u = uword[i++];
+ if (u == '-') {
+ /*tex Skip. */
+ } else if (u == '=') {
+ STORE_CHAR(id,'-');
+ } else if (u == '{') {
+ u = uword[i++];
+ items = 0;
+ while (u && u != '}') {
+ u = uword[i++];
+ }
+ if (u == '}') {
+ items++;
+ u = uword[i++];
+ }
+ while (u && u != '}') {
+ u = uword[i++];
+ }
+ if (u == '}') {
+ items++;
+ u = uword[i++];
+ }
+ if (u == '{') {
+ u = uword[i++];
+ }
+ while (u && u != '}') {
+ STORE_CHAR(id,u);
+ u = uword[i++];
+ }
+ if (u == '}') {
+ items++;
+ }
+ if (items != 3) {
+ *cleaned = NULL;
+ tex_error("exception syntax error", NULL);
+ return s;
+ }
+ if (uword[i] == '[' && uword[i+1] >= '0' && uword[i+1] <= '9' && uword[i+2] == ']') {
+ i += 3;
+ }
+ } else {
+ STORE_CHAR(id,u);
+ }
+ }
+ *uindex = '\0';
+ *cleaned = xstrdup((char *) word);
+ return s;
+}
+
+void load_hyphenation(struct tex_language *lang, const unsigned char *buff)
+{
+ const char *s;
+ const char *value;
+ char *cleaned;
+ int id ;
+ if (lang == NULL)
+ return;
+ if (lang->exceptions == 0) {
+ lua_newtable(Luas);
+ lang->exceptions = luaL_ref(Luas, LUA_REGISTRYINDEX);
+ }
+ lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions);
+ s = (const char *) buff;
+ id = lang->id;
+ while (*s) {
+ while (isspace((unsigned char)*s))
+ s++;
+ if (*s) {
+ value = s;
+ s = clean_hyphenation(id, s, &cleaned);
+ if (cleaned != NULL) {
+ if ((s - value) > 0) {
+ lua_pushstring(Luas, cleaned);
+ lua_pushlstring(Luas, value, (size_t) (s - value));
+ lua_rawset(Luas, -3);
+ }
+ free(cleaned);
+ } else {
+#ifdef VERBOSE
+ formatted_warning("hyphenation","skipping invalid hyphenation exception: %s", value);
+#endif
+ }
+ }
+ }
+}
+
+void clear_hyphenation(struct tex_language *lang)
+{
+ if (lang == NULL)
+ return;
+ if (lang->exceptions != 0) {
+ luaL_unref(Luas, LUA_REGISTRYINDEX, lang->exceptions);
+ lang->exceptions = 0;
+ }
+}
+
+void load_tex_hyphenation(int curlang, halfword head)
+{
+ char *s = tokenlist_to_cstring(head, 1, NULL);
+ load_hyphenation(get_language(curlang), (unsigned char *) s);
+}
+
+static halfword insert_discretionary(halfword t, halfword pre, halfword post, halfword replace, int penalty)
+{
+ halfword g;
+ int f;
+ halfword d = new_node(disc_node, syllable_disc);
+ int attr = node_attr(t) ;
+ disc_penalty(d) = penalty;
+ if (t == replace) {
+ /*tex We have |prev disc next-next|. */
+ try_couple_nodes(d, vlink(t));
+ try_couple_nodes(alink(t), d);
+ alink(t) = null;
+ vlink(t) = null;
+ replace = t ;
+ } else {
+ /*tex We have |prev disc next|. */
+ try_couple_nodes(d, vlink(t));
+ couple_nodes(t, d);
+ }
+ if (replace != null) {
+ f = font(replace);
+ } else {
+ /*tex For compound words following explicit hyphens. */
+ f = get_cur_font();
+ }
+ for (g = pre; g != null; g = vlink(g)) {
+ font(g) = f;
+ if (attr != null) {
+ delete_attribute_ref(node_attr(g));
+ node_attr(g) = attr;
+ attr_list_ref(attr) += 1;
+ }
+ }
+ for (g = post; g != null; g = vlink(g)) {
+ font(g) = f;
+ if (attr != null) {
+ delete_attribute_ref(node_attr(g));
+ node_attr(g) = attr;
+ attr_list_ref(attr) += 1;
+ }
+ }
+ if (attr != null) {
+ for (g = replace; g != null; g = vlink(g)) {
+ delete_attribute_ref(node_attr(g));
+ node_attr(g) = attr;
+ attr_list_ref(attr) += 1;
+ }
+ delete_attribute_ref(node_attr(d));
+ node_attr(d) = attr;
+ attr_list_ref(attr) += 1;
+ }
+ set_disc_field(pre_break(d), pre);
+ set_disc_field(post_break(d), post);
+ set_disc_field(no_break(d), replace);
+ return d;
+}
+
+halfword insert_syllable_discretionary(halfword t, lang_variables * lan)
+{
+ halfword g, n;
+ n = new_node(disc_node, syllable_disc);
+ disc_penalty(n) = hyphen_penalty_par;
+ couple_nodes(n, vlink(t));
+ couple_nodes(t, n);
+ delete_attribute_ref(node_attr(n));
+ if (node_attr(t) != null) {
+ node_attr(n) = node_attr(t);
+ attr_list_ref(node_attr(t))++;
+ } else {
+ node_attr(n) = null;
+ }
+ if (lan->pre_hyphen_char > 0) {
+ g = raw_glyph_node();
+ set_to_character(g);
+ character(g) = lan->pre_hyphen_char;
+ font(g) = font(t);
+ lang_data(g) = lang_data(t);
+ if (node_attr(t) != null) {
+ node_attr(g) = node_attr(t);
+ attr_list_ref(node_attr(t))++;
+ }
+ set_disc_field(pre_break(n), g);
+ }
+ if (lan->post_hyphen_char > 0) {
+ t = vlink(n);
+ g = raw_glyph_node();
+ set_to_character(g);
+ character(g) = lan->post_hyphen_char;
+ font(g) = font(t);
+ lang_data(g) = lang_data(t);
+ if (node_attr(t) != null) {
+ node_attr(g) = node_attr(t);
+ attr_list_ref(node_attr(t)) += 1;
+ }
+ set_disc_field(post_break(n), g);
+ }
+ return n;
+}
+
+static halfword insert_character(halfword t, int c)
+{
+ halfword p;
+ p = new_node(glyph_node, 0);
+ set_to_character(p);
+ character(p) = c;
+ if (t != null) {
+ couple_nodes(t, p);
+ }
+ return p;
+}
+
+static halfword compound_word_break(halfword t, int clang)
+{
+ halfword disc = null;
+ halfword pre = null;
+ halfword pos = null;
+ halfword pre_exhyphen_char = get_pre_exhyphen_char(clang);
+ halfword post_exhyphen_char = get_post_exhyphen_char(clang);
+ if (pre_exhyphen_char > 0) {
+ pre = insert_character(null,pre_exhyphen_char);
+ } else {
+ pre = insert_character(null,ex_hyphen_char_par);
+ }
+ if (post_exhyphen_char > 0)
+ pos = insert_character(null,post_exhyphen_char);
+ disc = insert_discretionary(t,pre,pos,t,ex_hyphen_penalty_par);
+ subtype(disc) = automatic_disc;
+ set_automatic_disc_penalty(disc);
+ return disc;
+}
+
+void set_disc_field(halfword f, halfword t)
+{
+ if (t != null) {
+ /*tex
+ No |couple_nodes(f, t);| as we can better not expose |f| as |prev|
+ pointer.
+ */
+ vlink(f) = t ;
+ alink(t) = null ;
+ tlink(f) = tail_of_list(t);
+ } else {
+ vlink(f) = null;
+ tlink(f) = null;
+ }
+}
+
+static char *hyphenation_exception(int exceptions, char *w)
+{
+ char *ret = NULL;
+ lua_checkstack(Luas, 2);
+ lua_rawgeti(Luas, LUA_REGISTRYINDEX, exceptions);
+ if (lua_istable(Luas, -1)) {
+ /*tex Word table: */
+ lua_pushstring(Luas, w);
+ lua_rawget(Luas, -2);
+ if (lua_type(Luas, -1) == LUA_TSTRING) {
+ ret = xstrdup(lua_tostring(Luas, -1));
+ }
+ lua_pop(Luas, 2);
+ } else {
+ lua_pop(Luas, 1);
+ }
+ return ret;
+}
+
+char *exception_strings(struct tex_language *lang)
+{
+ const char *value;
+ size_t size = 0, current = 0;
+ size_t l = 0;
+ char *ret = NULL;
+ if (lang->exceptions == 0)
+ return NULL;
+ lua_checkstack(Luas, 2);
+ lua_rawgeti(Luas, LUA_REGISTRYINDEX, lang->exceptions);
+ if (lua_istable(Luas, -1)) {
+ /*tex Iterate and join. */
+ lua_pushnil(Luas);
+ while (lua_next(Luas, -2) != 0) {
+ value = lua_tolstring(Luas, -1, &l);
+ if (current + 2 + l > size) {
+ ret = xrealloc(ret, (unsigned) ((size + size / 5) + current + l + 1024));
+ size = (size + size / 5) + current + l + 1024;
+ }
+ *(ret + current) = ' ';
+ strcpy(ret + current + 1, value);
+ current += l + 1;
+ lua_pop(Luas, 1);
+ }
+ }
+ return ret;
+}
+
+/*tex
+
+ The sequence from |wordstart| to |r| can contain only normal characters it
+ could be faster to modify a halfword pointer and return an integer
+
+*/
+
+static halfword find_exception_part(unsigned int *j, unsigned int *uword, int len)
+{
+ halfword g = null, gg = null;
+ register unsigned i = *j;
+ /*tex This puts uword[i] on the |{|. */
+ i++;
+ while (i < (unsigned) len && uword[i + 1] != '}') {
+ if (g == null) {
+ gg = new_char(0, (int) uword[i + 1]);
+ g = gg;
+ } else {
+ halfword s = new_char(0, (int) uword[i + 1]);
+ couple_nodes(g, s);
+ g = vlink(g);
+ }
+ i++;
+ }
+ *j = ++i;
+ return gg;
+}
+
+static int count_exception_part(unsigned int *j, unsigned int *uword, int len)
+{
+ int ret = 0;
+ register unsigned i = *j;
+ /*tex This puts uword[i] on the |{|. */
+ i++;
+ while (i < (unsigned) len && uword[i + 1] != '}') {
+ ret++;
+ i++;
+ }
+ *j = ++i;
+ return ret;
+}
+
+static const char *PAT_ERROR[] = {
+ "Exception discretionaries should contain three pairs of braced items.",
+ "No intervening spaces are allowed.",
+ NULL
+};
+
+/*tex
+
+ The exceptions are taken as-is: no min values are taken into account. One can
+ add normal patterns on-the-fly if needed.
+
+*/
+
+static void do_exception(halfword wordstart, halfword r, char *replacement)
+{
+ unsigned i;
+ halfword t, pen;
+ unsigned len;
+ int clang;
+ lang_variables langdata;
+ unsigned uword[MAX_WORD_LEN + 1] = { 0 };
+ utf2uni_strcpy(uword, replacement);
+ len = u_length(uword);
+ i = 0;
+ t = wordstart;
+ clang = char_lang(wordstart);
+ langdata.pre_hyphen_char = get_pre_hyphen_char(clang);
+ langdata.post_hyphen_char = get_post_hyphen_char(clang);
+ for (i = 0; i < len; i++) {
+ if (uword[i + 1] == 0 ) {
+ /*tex We ran out of the exception pattern. */
+ break;
+ } else if (uword[i + 1] == '-') {
+ /*tex A hyphen follows. */
+ if (vlink(t) == r)
+ break;
+ insert_syllable_discretionary(t, &langdata);
+ t = vlink(t); /* skip the new disc */
+ } else if (uword[i + 1] == '=') {
+ /*tex We skip a disc. */
+ t = vlink(t);
+ } else if (uword[i + 1] == '{') {
+ /*tex We ran into an exception |{}{}{}| or |{}{}{}[]|. */
+ halfword gg, hh, replace = null;
+ int repl;
+ /*tex |pre| */
+ gg = find_exception_part(&i, uword, (int) len);
+ if (i == len || uword[i + 1] != '{') {
+ tex_error("broken pattern 1", PAT_ERROR);
+ }
+ /*tex |post| */
+ hh = find_exception_part(&i, uword, (int) len);
+ if (i == len || uword[i + 1] != '{') {
+ tex_error("broken pattern 2", PAT_ERROR);
+ }
+ /*tex |replace| */
+ repl = count_exception_part(&i, uword, (int) len);
+ if (i == len) {
+ tex_error("broken pattern 3", PAT_ERROR);
+ }
+ /*tex Play safe. */
+ if (vlink(t) == r)
+ break;
+ /*tex Let's deal with an (optional) replacement. */
+ if (repl > 0) {
+ /*tex Assemble the replace stream. */
+ halfword q = t;
+ replace = vlink(q);
+ while (repl > 0 && q != null) {
+ q = vlink(q);
+ if (type(q) == glyph_node || type(q) == disc_node) {
+ repl--;
+ } else {
+ break ;
+ }
+ }
+ /*tex Remove it from the main stream */
+ try_couple_nodes(t, vlink(q));
+ /*tex and finish it in the replace. */
+ vlink(q) = null;
+ /*tex Sanitize the replace stream (we could use the flattener instead). */
+ q = replace ;
+ while (q != null) {
+ halfword n = vlink(q);
+ if (type(q) == disc_node) {
+ /*tex Beware: the replacement starts after the no_break pointer. */
+ halfword r = vlink(no_break(q));
+ vlink(no_break(q)) = null;
+ alink(r) = null ;
+ /*tex Insert the replacement glyph. */
+ if (q == replace) {
+ replace = r;
+ } else {
+ try_couple_nodes(alink(q),r);
+ }
+ /*tex Append the glyph (one). */
+ try_couple_nodes(r,n);
+ /*tex Flush the disc. */
+ flush_node(q);
+ }
+ q = n ;
+ }
+ }
+ /*tex Let's check if we have a penalty spec. */
+ if (((i+3) < len) && uword[i+1] == '[' && uword[i+2] >= '0' && uword[i+2] <= '9' && uword[i+3] == ']') {
+ if (exception_penalty_par > 0) {
+ if (exception_penalty_par > 100000) {
+ pen = (uword[i+2] - '0') * exception_penalty_par ;
+ } else {
+ pen = exception_penalty_par;
+ }
+ } else {
+ pen = hyphen_penalty_par;
+ }
+ i += 3;
+ } else {
+ pen = hyphen_penalty_par;
+ }
+ /*tex And now we insert a disc node. */
+ t = insert_discretionary(t, gg, hh, replace, pen);
+ /*tex We skip the new disc node. */
+ t = vlink(t);
+ /*tex check if we have two exceptions in a row */
+ if (uword[i + 1] == '{') {
+ i--;
+ }
+ } else {
+ t = vlink(t);
+ }
+ /*tex Again we play safe. */
+ if (t == null || vlink(t) == r) {
+ break;
+ }
+ }
+}
+
+/*tex
+
+ This is a documentation section from the pascal web file. It is not true any
+ more, but I (Taco) do not have time right now to rewrite it.
+
+ When the line-breaking routine is unable to find a feasible sequence of
+ breakpoints, it makes a second pass over the paragraph, attempting to
+ hyphenate the hyphenatable words. The goal of hyphenation is to insert
+ discretionary material into the paragraph so that there are more potential
+ places to break.
+
+ The general rules for hyphenation are somewhat complex and technical, because
+ we want to be able to hyphenate words that are preceded or followed by
+ punctuation marks, and because we want the rules to work for languages other
+ than English. We also must contend with the fact that hyphens might radically
+ alter the ligature and kerning structure of a word.
+
+ A sequence of characters will be considered for hyphenation only if it
+ belongs to a ``potentially hyphenatable part'' of the current paragraph. This
+ is a sequence of nodes $p_0p_1\ldots p_m$ where $p_0$ is a glue node,
+ $p_1\ldots p_{m-1}$ are either character or ligature or whatsit or implicit
+ kern nodes, and $p_m$ is a glue or penalty or insertion or adjust or mark or
+ whatsit or explicit kern node. (Therefore hyphenation is disabled by boxes,
+ math formulas, and discretionary nodes already inserted by the user.) The
+ ligature nodes among $p_1\ldots p_{m-1}$ are effectively expanded into the
+ original non-ligature characters; the kern nodes and whatsits are ignored.
+ Each character |c| is now classified as either a nonletter (if
+ |lc_code(c)=0|), a lowercase letter (if |lc_code(c)=c|), or an uppercase
+ letter (otherwise); an uppercase letter is treated as if it were |lc_code(c)|
+ for purposes of hyphenation. The characters generated by $p_1\ldots p_{m-1}$
+ may begin with nonletters; let $c_1$ be the first letter that is not in the
+ middle of a ligature. Whatsit nodes preceding $c_1$ are ignored; a whatsit
+ found after $c_1$ will be the terminating node $p_m$. All characters that do
+ not have the same font as $c_1$ will be treated as nonletters. The
+ |hyphen_char| for that font must be between 0 and 255, otherwise hyphenation
+ will not be attempted. \TeX\ looks ahead for as many consecutive letters
+ $c_1\ldots c_n$ as possible; however, |n| must be less than 64, so a
+ character that would otherwise be $c_{64}$ is effectively not a letter.
+ Furthermore $c_n$ must not be in the middle of a ligature. In this way we
+ obtain a string of letters $c_1\ldots c_n$ that are generated by nodes
+ $p_a\ldots p_b$, where |1<=a<=b+1<=m|. If |n>=l_hyf+r_hyf|, this string
+ qualifies for hyphenation; however, |uc_hyph| must be positive, if $c_1$ is
+ uppercase.
+
+ The hyphenation process takes place in three stages. First, the candidate
+ sequence $c_1\ldots c_n$ is found; then potential positions for hyphens are
+ determined by referring to hyphenation tables; and finally, the nodes
+ $p_a\ldots p_b$ are replaced by a new sequence of nodes that includes the
+ discretionary breaks found.
+
+ Fortunately, we do not have to do all this calculation very often, because of
+ the way it has been taken out of \TeX's inner loop. For example, when the
+ second edition of the author's 700-page book {\sl Seminumerical Algorithms}
+ was typeset by \TeX, only about 1.2 hyphenations needed to be tried per
+ paragraph, since the line breaking algorithm needed to use two passes on only
+ about 5 per cent of the paragraphs.
+
+ When a word been set up to contain a candidate for hyphenation, \TeX\ first
+ looks to see if it is in the user's exception dictionary. If not, hyphens are
+ inserted based on patterns that appear within the given word, using an
+ algorithm due to Frank~M. Liang.
+
+ This is incompatible with \TEX\ because the first word of a paragraph can be
+ hyphenated, but most European users seem to agree that prohibiting
+ hyphenation there was not the best idea ever.
+
+ We have some variants available that are controlled by the paremeter
+ \type {\hyphenationbounds}:
+
+ \starttabulate
+ \NC \type {0} \NC not strict \NC \NR
+ \NC \type {1} \NC strict start \NC \NR
+ \NC \type {2} \NC strict end \NC \NR
+ \NC \type {3} \NC strict start and strict end \NC \NR
+ \stoptabulate
+
+ \startbuffer
+ \parindent0pt \hsize=1.1cm
+ 12-34-56 \par
+ 12-34-\hbox{56} \par
+ 12-34-\vrule width 1em height 1.5ex \par
+ 12-\hbox{34}-56 \par
+ 12-\vrule width 1em height 1.5ex-56 \par
+ \hjcode`\1=`\1 \hjcode`\2=`\2 \hjcode`\3=`\3 \hjcode`\4=`\4 \vskip.5cm
+ 12-34-56 \par
+ 12-34-\hbox{56} \par
+ 12-34-\vrule width 1em height 1.5ex \par
+ 12-\hbox{34}-56 \par
+ 12-\vrule width 1em height 1.5ex-56 \par
+ \stopbuffer
+
+ \typebuffer
+
+ \startpacked \getbuffer \stopbuffer
+
+ We only accept an explicit hyphen when there is a preceding glyph and we skip
+ a sequence of explicit hyphens as that normally indicates a \type {--} or
+ \type {---} ligature in which case we can in a worse case usage get bad node
+ lists later on due to messed up ligature building as these dashes are
+ ligatures in base fonts. This is a side effect of the separating the
+ hyphenation, ligaturing and kerning steps. A test is cmr with \type {------}.
+
+ A font handler can collapse successive hyphens but it's not nice to put the
+ burden there. A somewhat messy border case is \type {----} but in \LUATEX\ we
+ don't treat \type {--} and \type {---} special. Also, traditional \TEX\ will
+ break a line at \type {-foo} but this can be disabled by setting the
+ automatic mode to \type {1}.
+
+*/
+
+static halfword find_next_wordstart(halfword r, halfword first_language, halfword strict_bound)
+{
+ register int l;
+ register int start_ok = 1;
+ int mathlevel = 1;
+ int chr ;
+ halfword t ;
+ while (r != null) {
+ switch (type(r)) {
+ case boundary_node:
+ if (subtype(r) == word_boundary) {
+ start_ok = 1;
+ }
+ break;
+ case hlist_node:
+ case vlist_node:
+ case rule_node:
+ case dir_node:
+ case whatsit_node:
+ if (strict_bound == 1 || strict_bound == 3) {
+ start_ok = 0;
+ }
+ break;
+ case glue_node:
+ start_ok = 1;
+ break;
+ case math_node:
+ while (mathlevel > 0) {
+ r = vlink(r);
+ if (r == null)
+ return r;
+ if (type(r) == math_node) {
+ if (subtype(r) == before) {
+ mathlevel++;
+ } else {
+ mathlevel--;
+ }
+ }
+ }
+ break;
+ case glyph_node:
+ if (is_simple_character(r)) {
+ chr = character(r) ;
+ if (chr == ex_hyphen_char_par) {
+ t = vlink(r) ;
+ if ((automatic_hyphen_mode_par == 0) && (t != null) && (type(t) == glyph_node) && (character(t) != ex_hyphen_char_par)) {
+ /*tex We have no word yet and the next character is a non hyphen. */
+ r = compound_word_break(r, char_lang(r));
+ } else {
+ /*tex We jump over the sequence of hyphens. */
+ while ((t != null) && (type(t) == glyph_node) && (character(t) == ex_hyphen_char_par)) {
+ r = t ;
+ t = vlink(r) ;
+ }
+ if (t == null) {
+ /*tex We reached the end of the list so we have no word start. */
+ return null;
+ }
+ }
+ /*tex We need a restart. */
+ start_ok = 0;
+ } else if (start_ok && (char_lang(r)>=first_language) && ((l = get_hj_code(char_lang(r),chr)) > 0)) {
+ if (char_uchyph(r) || l == chr || l <= 32) {
+ return r;
+ } else {
+ start_ok = 0;
+ }
+ } else {
+ /*tex We go on. */
+ }
+ }
+ break;
+ default:
+ start_ok = 0;
+ break;
+ }
+ r = vlink(r);
+ }
+ return r;
+}
+
+static int valid_wordend(halfword s, halfword strict_bound)
+{
+ register halfword r = s;
+ register int clang = char_lang(s);
+ if (r == null)
+ return 1;
+ while ( (r != null) &&
+ ( (type(r) == glyph_node && is_simple_character(r) && clang == char_lang(r))
+ || (type(r) == kern_node && (subtype(r) == normal))
+ )
+ ) {
+ r = vlink(r);
+ }
+ if (r == null || (type(r) == glyph_node && is_simple_character(r) && clang != char_lang(r))
+ || type(r) == glue_node
+ || type(r) == penalty_node
+ || (type(r) == kern_node && (subtype(r) == explicit_kern ||
+ subtype(r) == italic_kern ||
+ subtype(r) == accent_kern ))
+ || ((type(r) == hlist_node ||
+ type(r) == vlist_node ||
+ type(r) == rule_node ||
+ type(r) == dir_node ||
+ type(r) == whatsit_node ||
+ type(r) == ins_node ||
+ type(r) == adjust_node
+ ) && ! (strict_bound == 2 || strict_bound == 3))
+ || type(r) == boundary_node
+ )
+ return 1;
+ return 0;
+}
+
+void hnj_hyphenation(halfword head, halfword tail)
+{
+ int lchar, i;
+ struct tex_language *lang;
+ lang_variables langdata;
+ char utf8word[(4 * MAX_WORD_LEN) + 1] = { 0 };
+ int wordlen = 0;
+ char *hy = utf8word;
+ char *replacement = NULL;
+ boolean explicit_hyphen = false;
+ boolean valid_word = false;
+ halfword first_language = first_valid_language_par;
+ halfword strict_bound = hyphenation_bounds_par;
+ halfword s, r = head, wordstart = null, save_tail1 = null, left = null, right = null;
+ halfword expstart = null;
+ boolean compound_hyphen = compound_hyphen_mode_par;
+ /*tex Let's play safe: */
+ if (tail == null)
+ return;
+ /*tex
+ This first movement assures two things:
+
+ \startitemize
+ \startitem
+ That we won't waste lots of time on something that has been
+ handled already (in that case, none of the glyphs match
+ |simple_character|).
+ \stopitem
+ \startitem
+ That the first word can be hyphenated. if the movement was not
+ explicit, then the indentation at the start of a paragraph list
+ would make |find_next_wordstart()| look too far ahead.
+ \stopitem
+ \stopitemize
+ */
+ while (r != null && (type(r) != glyph_node || !is_simple_character(r))) {
+ r = vlink(r);
+ }
+ /*tex
+ This will make |r| a glyph node with subtype character.
+ */
+ r = find_next_wordstart(r,first_language,strict_bound);
+ if (r == null)
+ return;
+ if (tail == null)
+ return;
+ save_tail1 = vlink(tail);
+ s = new_penalty(0,word_penalty);
+ couple_nodes(tail, s);
+ while (r != null) {
+ /*tex This could be |while(1)|, but let's be paranoid: */
+ int clang, lhmin, rhmin, hmin;
+ halfword hyf_font;
+ halfword end_word = r;
+ wordstart = r;
+ /*assert(is_simple_character(wordstart));*/
+ if (!(is_simple_character(wordstart))){
+ if (!(is_character(wordstart))) {
+ tex_error("the word doesn't start with a character", NULL);
+ } else if (is_ligature(wordstart)) {
+ tex_error("the word starts with a ligature", NULL);
+ } else if (is_ghost(wordstart)) {
+ tex_error("the word starts with a ghost glyph", NULL);
+ } else {
+ tex_error("the word doesn't start with a simple character", NULL);
+ }
+ }
+ hyf_font = font(wordstart);
+ if (hyphen_char(hyf_font) < 0) {
+ /*tex For backward compatibility we set: */
+ hyf_font = 0;
+ }
+ clang = char_lang(wordstart);
+ lhmin = char_lhmin(wordstart);
+ rhmin = char_rhmin(wordstart);
+ hmin = get_hyphenation_min(clang);
+ langdata.pre_hyphen_char = get_pre_hyphen_char(clang);
+ langdata.post_hyphen_char = get_post_hyphen_char(clang);
+ while ( r != null
+ && type(r) == glyph_node
+ && is_simple_character(r)
+ && clang == char_lang(r)
+ && ( ( (clang >= first_language)
+ && (lchar = get_hj_code(clang,character(r))) > 0
+ )
+ || ( character(r) == ex_hyphen_char_par
+ && (lchar = ex_hyphen_char_par)
+ )
+ )
+ ) {
+ if (character(r) == ex_hyphen_char_par) {
+ explicit_hyphen = true;
+ break;
+ }
+ wordlen++;
+ if (wordlen >= MAX_WORD_LEN) {
+ while (r && type(r) == glyph_node) {
+ r = vlink(r);
+ }
+ goto PICKUP;
+ }
+ if (lchar <= 32) {
+ if (lchar == 32) {
+ lchar = 0 ;
+ }
+ if (wordlen <= lhmin) {
+ lhmin = lhmin - lchar + 1 ;
+ if (lhmin < 0)
+ lhmin = 1;
+ }
+ if (wordlen >= rhmin) {
+ rhmin = rhmin - lchar + 1 ;
+ if (rhmin < 0)
+ rhmin = 1;
+ }
+ hmin = hmin - lchar + 1 ;
+ if (hmin < 0)
+ rhmin = 1;
+ lchar = character(r) ;
+ }
+ hy = uni2string(hy, (unsigned) lchar);
+ end_word = r;
+ r = vlink(r);
+ }
+ if (explicit_hyphen == true) {
+ /*tex we are not at the start, so we only need to look ahead */
+ halfword t = vlink(r) ;
+ if ((automatic_hyphen_mode_par == 0 || automatic_hyphen_mode_par == 1) && (t != null) && ((type(t) == glyph_node) && is_simple_character(t) && (character(t) != ex_hyphen_char_par))) {
+ /*tex we have a word already but the next character may not be a hyphen too */
+ r = compound_word_break(r, char_lang(r));
+ if (compound_hyphen) {
+ if (expstart == null) {
+ expstart = wordstart;
+ }
+ explicit_hyphen = false;
+ hy = uni2string(hy, '-');
+ r = t;
+ continue;
+ }
+ } else {
+ /*tex we jump over the sequence of hyphens */
+ while ((t != null) && (type(t) == glyph_node) && is_simple_character(t) && (character(t) == ex_hyphen_char_par)) {
+ r = t ;
+ t = vlink(r) ;
+ }
+ if (t == null) {
+ /*tex we reached the end of the list and will quit the loop later */
+ r = null;
+ }
+ }
+ } else if ( valid_wordend(r,strict_bound)
+ && clang >= first_language
+ && wordlen >= lhmin + rhmin
+ && (hmin <= 0 || wordlen >= hmin)
+ && (hyf_font != 0)
+ && (lang = tex_languages[clang]) != NULL
+ ) {
+ *hy = 0;
+ /*tex
+ this is messy and nasty: we can have a word with a - in it which
+ is why we have two branches
+ */
+ if (lang->exceptions != 0 && (replacement = hyphenation_exception(lang->exceptions, utf8word)) != NULL) {
+ /*tex handle the exception and go on to the next word */
+ if (expstart == null) {
+ do_exception(wordstart, r, replacement);
+ } else {
+ do_exception(expstart,r,replacement);
+ }
+ free(replacement);
+ } else if (expstart != null) {
+ /*tex We're done already */
+ } else if (lang->patterns != NULL) {
+ /*tex A safeguard, not needed but doesn't hurt either: */
+ valid_word = true;
+ left = wordstart;
+ for (i = lhmin; i > 1; i--) {
+ left = vlink(left);
+ if (left == null) {
+ valid_word = false;
+ break ;
+ }
+ while (!is_simple_character(left)) {
+ left = vlink(left);
+ if (left == null) {
+ valid_word = false;
+ break ;
+ }
+ }
+ }
+ if (valid_word) {
+ right = r;
+ if (right == left) {
+ valid_word = false;
+ break ;
+ }
+ for (i = rhmin; i > 0; i--) {
+ right = alink(right);
+ if (right == null || right == left) {
+ valid_word = false;
+ break ;
+ }
+ while (!is_simple_character(right)) {
+ right = alink(right);
+ if (right == null || right == left) {
+ valid_word = false;
+ break ;
+ }
+ }
+ }
+ if (valid_word && expstart == null) {
+ (void) hnj_hyphen_hyphenate(lang->patterns, wordstart, end_word, wordlen, left, right, &langdata);
+ } else {
+ /*tex nothing yet */
+ }
+ }
+ }
+ }
+PICKUP:
+ expstart = null ;
+ explicit_hyphen = false;
+ wordlen = 0;
+ hy = utf8word;
+ if (r == null)
+ break;
+ r = find_next_wordstart(r,first_language,strict_bound);
+ }
+ flush_node(vlink(tail));
+ vlink(tail) = save_tail1;
+}
+
+void new_hyphenation(halfword head, halfword tail)
+{
+ int i, top;
+ register int callback_id = 0;
+ if (head == null || vlink(head) == null)
+ return;
+ fix_node_list(head);
+ callback_id = callback_defined(hyphenate_callback);
+ if (callback_id > 0) {
+ top = lua_gettop(Luas);
+ if (!get_callback(Luas, callback_id)) {
+ lua_settop(Luas, top);
+ return;
+ }
+ nodelist_to_lua(Luas, head);
+ nodelist_to_lua(Luas, tail);
+ if ((i=lua_pcall(Luas, 2, 0, 0)) != 0) {
+ formatted_warning("hyphenation","bad specification: %s",lua_tostring(Luas, -1));
+ lua_settop(Luas, top);
+ luatex_error(Luas, (i == LUA_ERRRUN ? 0 : 1));
+ return;
+ }
+ lua_settop(Luas, top);
+ } else if (callback_id == 0) {
+ hnj_hyphenation(head, tail);
+ }
+}
+
+/*tex
+
+ Dumping and undumping languages:
+
+*/
+
+#define dump_string(a) \
+ if (a!=NULL) { \
+ x = (int)strlen(a)+1; \
+ dump_int(x); dump_things(*a, x); \
+ } else { \
+ x = 0; dump_int(x); \
+ }
+
+static void dump_one_language(int i)
+{
+ char *s = NULL;
+ int x = 0;
+ struct tex_language *lang;
+ lang = tex_languages[i];
+ dump_int(lang->id);
+ dump_int(lang->pre_hyphen_char);
+ dump_int(lang->post_hyphen_char);
+ dump_int(lang->pre_exhyphen_char);
+ dump_int(lang->post_exhyphen_char);
+ dump_int(lang->hyphenation_min);
+ if (lang->patterns != NULL) {
+ s = (char *) hnj_serialize(lang->patterns);
+ }
+ dump_string(s);
+ if (s != NULL) {
+ free(s);
+ s = NULL;
+ }
+ if (lang->exceptions != 0)
+ s = exception_strings(lang);
+ dump_string(s);
+ if (s != NULL) {
+ free(s);
+ }
+ free(lang);
+}
+
+void dump_language_data(void)
+{
+ int i;
+ dump_int(next_lang_id);
+ for (i = 0; i < next_lang_id; i++) {
+ if (tex_languages[i]) {
+ dump_int(1);
+ dump_one_language(i);
+ } else {
+ dump_int(0);
+ }
+ }
+}
+
+static void undump_one_language(int i)
+{
+ char *s = NULL;
+ int x = 0;
+ struct tex_language *lang = get_language(i);
+ undump_int(x);
+ lang->id = x;
+ undump_int(x);
+ lang->pre_hyphen_char = x;
+ undump_int(x);
+ lang->post_hyphen_char = x;
+ undump_int(x);
+ lang->pre_exhyphen_char = x;
+ undump_int(x);
+ lang->post_exhyphen_char = x;
+ undump_int(x);
+ lang->hyphenation_min = x;
+ /*tex patterns */
+ undump_int(x);
+ if (x > 0) {
+ s = xmalloc((unsigned) x);
+ undump_things(*s, x);
+ load_patterns(lang, (unsigned char *) s);
+ free(s);
+ }
+ /*tex exceptions */
+ undump_int(x);
+ if (x > 0) {
+ s = xmalloc((unsigned) x);
+ undump_things(*s, x);
+ load_hyphenation(lang, (unsigned char *) s);
+ free(s);
+ }
+}
+
+void undump_language_data(void)
+{
+ int i, x, numlangs;
+ undump_int(numlangs);
+ next_lang_id = numlangs;
+ for (i = 0; i < numlangs; i++) {
+ undump_int(x);
+ if (x == 1) {
+ undump_one_language(i);
+ }
+ }
+}
+
+/*tex
+
+ When \TeX\ has scanned `\.{\\hyphenation}', it calls on a procedure named
+ |new_hyph_exceptions| to do the right thing.
+
+*/
+
+void new_hyph_exceptions(void)
+{
+ (void) scan_toks(false, true);
+ load_tex_hyphenation(language_par, def_ref);
+ flush_list(def_ref);
+}
+
+/*
+ Similarly, when \TeX\ has scanned `\.{\\patterns}', it calls on a procedure
+ named |new_patterns|.
+
+*/
+
+void new_patterns(void)
+{
+ (void) scan_toks(false, true);
+ load_tex_patterns(language_par, def_ref);
+ flush_list(def_ref);
+}
+
+/*tex
+
+ `\.{\\prehyphenchar}', sets the |pre_break| character, and
+ `\.{\\posthyphenchar}' the |post_break| character. Their respective defaults
+ are ascii hyphen ("-") and zero (nul).
+
+*/
+
+void new_pre_hyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_pre_hyphen_char(language_par, cur_val);
+}
+
+void new_post_hyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_post_hyphen_char(language_par, cur_val);
+}
+
+/*tex
+
+ `\.{\\preexhyphenchar}', sets the |pre_break| character, and
+ `\.{\\postexhyphenchar}' the |post_break| character. Their defaults are both
+ zero (nul).
+
+*/
+
+void new_pre_exhyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_pre_exhyphen_char(language_par, cur_val);
+}
+
+void new_post_exhyphen_char(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_post_exhyphen_char(language_par, cur_val);
+}
+
+void new_hyphenation_min(void)
+{
+ scan_optional_equals();
+ scan_int();
+ set_hyphenation_min(language_par, cur_val);
+}
+
+void new_hj_code(void)
+{
+ int i ;
+ scan_int();
+ i = cur_val;
+ scan_optional_equals();
+ scan_int();
+ set_hj_code(language_par, i, cur_val, -1);
+}
diff --git a/Build/source/texk/web2c/luatexdir/lang/texlang.h b/Build/source/texk/web2c/luatexdir/lang/texlang.h
new file mode 100644
index 00000000000..0fa11d4c82f
--- /dev/null
+++ b/Build/source/texk/web2c/luatexdir/lang/texlang.h
@@ -0,0 +1,88 @@
+/* texlang.h
+
+ Copyright 2009 Taco Hoekwater <taco@luatex.org>
+
+ This file is part of LuaTeX.
+
+ LuaTeX is free software; you can redistribute it and/or modify it under
+ the terms of the GNU General Public License as published by the Free
+ Software Foundation; either version 2 of the License, or (at your
+ option) any later version.
+
+ LuaTeX is distributed in the hope that it will be useful, but WITHOUT
+ ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public
+ License for more details.
+
+ You should have received a copy of the GNU General Public License along
+ with LuaTeX; if not, see <http://www.gnu.org/licenses/>. */
+
+#ifndef TEXLANG_H
+# define TEXLANG_H
+
+typedef struct _lang_variables {
+ int pre_hyphen_char;
+ int post_hyphen_char;
+ int pre_exhyphen_char;
+ int post_exhyphen_char;
+} lang_variables;
+
+# include "lang/hyphen.h"
+
+struct tex_language {
+ HyphenDict *patterns;
+ int exceptions; /* lua registry pointer, should be replaced */
+ int id;
+ int pre_hyphen_char;
+ int post_hyphen_char;
+ int pre_exhyphen_char;
+ int post_exhyphen_char;
+ int hyphenation_min;
+};
+
+# define MAX_WORD_LEN 65536 /* in chars */
+
+extern struct tex_language *new_language(int n);
+extern struct tex_language *get_language(int n);
+extern void load_patterns(struct tex_language *lang, const unsigned char *buf);
+extern void load_hyphenation(struct tex_language *lang, const unsigned char *buf);
+extern int hyphenate_string(struct tex_language *lang, char *w, char **ret);
+
+extern void new_hyphenation(halfword h, halfword t);
+extern void clear_patterns(struct tex_language *lang);
+extern void clear_hyphenation(struct tex_language *lang);
+extern const char *clean_hyphenation(int id, const char *buffer, char **cleaned);
+extern void hnj_hyphenation(halfword head, halfword tail);
+
+extern void set_pre_hyphen_char(int lan, int val);
+extern void set_post_hyphen_char(int lan, int val);
+extern int get_pre_hyphen_char(int lan);
+extern int get_post_hyphen_char(int lan);
+
+extern void set_pre_exhyphen_char(int lan, int val);
+extern void set_post_exhyphen_char(int lan, int val);
+extern int get_pre_exhyphen_char(int lan);
+extern int get_post_exhyphen_char(int lan);
+
+extern void set_hyphenation_min(int lan, int val);
+extern int get_hyphenation_min(int lan);
+
+/* extern halfword compound_word_break(halfword t, int clang); */
+
+extern void dump_language_data(void);
+extern void undump_language_data(void);
+extern char *exception_strings(struct tex_language *lang);
+
+extern void new_hyph_exceptions(void);
+extern void new_patterns(void);
+extern void new_pre_hyphen_char(void);
+extern void new_post_hyphen_char(void);
+extern void new_pre_exhyphen_char(void);
+extern void new_post_exhyphen_char(void);
+extern void new_hyphenation_min(void);
+extern void new_hj_code(void);
+
+extern void set_disc_field(halfword f, halfword t);
+extern halfword insert_syllable_discretionary(halfword t, lang_variables * lan);
+
+#endif