summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/mkxl/lang-hyp.lmt
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/mkxl/lang-hyp.lmt')
-rw-r--r--Master/texmf-dist/tex/context/base/mkxl/lang-hyp.lmt1806
1 files changed, 1806 insertions, 0 deletions
diff --git a/Master/texmf-dist/tex/context/base/mkxl/lang-hyp.lmt b/Master/texmf-dist/tex/context/base/mkxl/lang-hyp.lmt
new file mode 100644
index 00000000000..c356dbf3c47
--- /dev/null
+++ b/Master/texmf-dist/tex/context/base/mkxl/lang-hyp.lmt
@@ -0,0 +1,1806 @@
+if not modules then modules = { } end modules ['lang-hyp'] = {
+ version = 1.001,
+ comment = "companion to lang-ini.mkiv",
+ author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
+ copyright = "PRAGMA ADE / ConTeXt Development Team",
+ license = "see context related readme files"
+}
+
+-- In an automated workflow hypenation of long titles can be somewhat problematic
+-- especially when demands conflict. For that reason I played a bit with a Lua based
+-- variant of the traditional hyphenation machinery. This mechanism has been extended
+-- several times in projects, of which a good description can be found in TUGboat,
+-- Volume 27 (2006), No. 2 — Proceedings of EuroTEX2006: Automatic non-standard
+-- hyphenation in OpenOffice.org by László Németh.
+--
+-- Being the result of two days experimenting the following implementation is probably
+-- not completely okay yet. If there is demand I might add some more features and plugs.
+-- The performance is quite okay but can probably improved a bit, although this is not
+-- the most critital code. For instance, on a metafun manual run the overhead is about
+-- 0.3 seconds on 19 seconds which is not that bad.
+--
+-- In the procecess of wrapping up (for the ctx conference proceedings) I cleaned up
+-- and extended the code a bit. It can be used in production.
+--
+-- . a l g o r i t h m .
+-- 4l1g4
+-- l g o3
+-- 1g o
+-- 2i t h
+-- 4h1m
+-- ---------------------
+-- 4 1 4 3 2 0 4 1
+-- a l-g o-r i t h-m
+
+-- . a s s z o n n y a l .
+-- s1s z/sz=sz,1,3
+-- n1n y/ny=ny,1,3
+-- -----------------------
+-- 0 1 0 0 0 1 0 0 0/sz=sz,2,3,ny=ny,6,3
+-- a s-s z o n-n y a l/sz=sz,2,3,ny=ny,6,3
+--
+-- ab1cd/ef=gh,2,2 : acd - efd (pattern/replacement,start,length
+--
+-- todo : support hjcodes (<32 == length) like luatex does now (no need/demand so far)
+-- maybe : support hyphenation over range (can alsready be done using attributes/language)
+-- maybe : reset dictionary.hyphenated when a pattern is added and/or forced reset option
+-- todo : check subtypes (because they have subtle meanings in the line breaking)
+--
+-- word start (in tex engine):
+--
+-- boundary : yes when wordboundary
+-- hlist : when hyphenationbounds 1 or 3
+-- vlist : when hyphenationbounds 1 or 3
+-- rule : when hyphenationbounds 1 or 3
+-- dir : when hyphenationbounds 1 or 3
+-- whatsit : when hyphenationbounds 1 or 3
+-- glue : yes
+-- math : skipped
+-- glyph : exhyphenchar (one only) : yes (so no -- ---)
+-- otherwise : yes
+--
+-- word end (in tex engine):
+--
+-- boundary : yes
+-- glyph : yes when different language
+-- glue : yes
+-- penalty : yes
+-- kern : yes when not italic (for some historic reason)
+-- hlist : when hyphenationbounds 2 or 3
+-- vlist : when hyphenationbounds 2 or 3
+-- rule : when hyphenationbounds 2 or 3
+-- dir : when hyphenationbounds 2 or 3
+-- whatsit : when hyphenationbounds 2 or 3
+-- ins : when hyphenationbounds 2 or 3
+-- adjust : when hyphenationbounds 2 or 3
+
+local type, rawget, rawset, tonumber, next = type, rawget, rawset, tonumber, next
+
+local P, R, S, Cg, Cf, Ct, Cc, C, Carg, Cs = lpeg.P, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cf, lpeg.Ct, lpeg.Cc, lpeg.C, lpeg.Carg, lpeg.Cs
+local lpegmatch = lpeg.match
+
+local context = context
+
+local concat = table.concat
+local insert = table.insert
+local remove = table.remove
+local formatters = string.formatters
+local utfchar = utf.char
+local utfbyte = utf.byte
+
+if not characters then
+ require("char-ini")
+end
+
+local setmetatableindex = table.setmetatableindex
+
+-- \enabletrackers[hyphenator.steps=silent] will not write to the terminal
+
+local trace_steps = false trackers.register("hyphenator.steps", function(v) trace_steps = v end)
+local trace_visualize = false trackers.register("hyphenator.visualize",function(v) trace_visualize = v end)
+
+local report = logs.reporter("hyphenator")
+
+local implement = interfaces and interfaces.implement or function() end
+
+languages = languages or { }
+local hyphenators = languages.hyphenators or { }
+languages.hyphenators = hyphenators
+local traditional = hyphenators.traditional or { }
+hyphenators.traditional = traditional
+
+local dictionaries = setmetatableindex(function(t,k)
+ local v = {
+ patterns = { },
+ hyphenated = { },
+ specials = { },
+ exceptions = { },
+ loaded = false,
+ }
+ t[k] = v
+ return v
+end)
+
+hyphenators.dictionaries = dictionaries
+
+local character = lpeg.patterns.utf8character
+local digit = R("09")
+local weight = digit/tonumber + Cc(0)
+local fence = P(".")
+local hyphen = P("-")
+local space = P(" ")
+local char = character - space
+local validcharacter = (character - S("./"))
+local keycharacter = character - S("/")
+----- basepart = Ct( (Cc(0) * fence)^-1 * (weight * validcharacter)^1 * weight * (fence * Cc(0))^-1)
+local specpart = (P("/") * Cf ( Ct("") *
+ Cg ( Cc("before") * C((1-P("="))^1) * P("=") ) *
+ Cg ( Cc("after") * C((1-P(","))^1) ) *
+ ( P(",") *
+ Cg ( Cc("start") * ((1-P(","))^1/tonumber) * P(",") ) *
+ Cg ( Cc("length") * ((1-P(-1) )^1/tonumber) )
+ )^-1
+ , rawset))^-1
+
+local make_hashkey_p = Cs((digit/"" + keycharacter)^1)
+----- make_pattern_p = basepart * specpart
+local make_hashkey_e = Cs((hyphen/"" + keycharacter)^1)
+local make_pattern_e = Ct(P(char) * (hyphen * Cc(true) * P(char) + P(char) * Cc(false))^1) -- catch . and char after -
+
+-- local make_hashkey_c = Cs((digit + keycharacter/"")^1)
+-- local make_pattern_c = Ct((P(1)/tonumber)^1)
+
+-- local cache = setmetatableindex(function(t,k)
+-- local n = lpegmatch(make_hashkey_c,k)
+-- local v = lpegmatch(make_pattern_c,n)
+-- t[k] = v
+-- return v
+-- end)
+--
+-- local weight_n = digit + Cc("0")
+-- local basepart_n = Cs( (Cc("0") * fence)^-1 * (weight * validcharacter)^1 * weight * (fence * Cc("0"))^-1) / cache
+-- local make_pattern_n = basepart_n * specpart
+
+local make_pattern_c = Ct((P(1)/tonumber)^1)
+
+-- us + nl: 17664 entries -> 827 unique (saves some 3M)
+
+local cache = setmetatableindex(function(t,k)
+ local v = lpegmatch(make_pattern_c,k)
+ t[k] = v
+ return v
+end)
+
+local weight_n = digit + Cc("0")
+local fence_n = fence / "0"
+local char_n = validcharacter / ""
+local basepart_n = Cs(fence_n^-1 * (weight_n * char_n)^1 * weight_n * fence_n^-1) / cache
+local make_pattern_n = basepart_n * specpart
+
+local function register_pattern(patterns,specials,str,specification)
+ local k = lpegmatch(make_hashkey_p,str)
+ -- local v1, v2 = lpegmatch(make_pattern_p,str)
+ local v1, v2 = lpegmatch(make_pattern_n,str)
+ patterns[k] = v1 -- is this key still ok for complex patterns
+ if specification then
+ specials[k] = specification
+ elseif v2 then
+ specials[k] = v2
+ end
+end
+
+local function unregister_pattern(patterns,specials,str)
+ local k = lpegmatch(make_hashkey_p,str)
+ patterns[k] = nil
+ specials[k] = nil
+end
+
+local p_lower = lpeg.patterns.utf8lower
+
+local function register_exception(exceptions,str,specification)
+ local l = lpegmatch(p_lower,str)
+ local k = lpegmatch(make_hashkey_e,l)
+ local v = lpegmatch(make_pattern_e,l)
+ exceptions[k] = v
+end
+
+local p_pattern = ((Carg(1) * Carg(2) * C(char^1)) / register_pattern + 1)^1
+local p_exception = ((Carg(1) * C(char^1)) / register_exception + 1)^1
+local p_split = Ct(C(character)^1)
+
+function traditional.loadpatterns(language,filename)
+ local dictionary = dictionaries[language]
+ if not dictionary.loaded then
+ if not filename or filename == "" then
+ filename = "lang-" .. language
+ end
+ filename = file.addsuffix(filename,"lua")
+ local fullname = resolvers.findfile(filename)
+ if fullname and fullname ~= "" then
+ local specification = dofile(fullname)
+ if specification then
+ local patterns = specification.patterns
+ if patterns then
+ local data = patterns.data
+ if data and data ~= "" then
+ lpegmatch(p_pattern,data,1,dictionary.patterns,dictionary.specials)
+ end
+ end
+ local exceptions = specification.exceptions
+ if exceptions then
+ local data = exceptions.data
+ if data and data ~= "" then
+ lpegmatch(p_exception,data,1,dictionary.exceptions)
+ end
+ end
+ end
+ end
+ dictionary.loaded = true
+ end
+ return dictionary
+end
+
+local lcchars = characters.lcchars
+local uccodes = characters.uccodes
+local categories = characters.categories
+local nofwords = 0
+local nofhashed = 0
+
+local steps = nil
+local f_show = formatters["%w%s"]
+
+local function show_log()
+ if trace_steps == true then
+ report()
+ local w = #steps[1][1]
+ for i=1,#steps do
+ local s = steps[i]
+ report("%s%w%S %S",s[1],w - #s[1] + 3,s[2],s[3] or "")
+ end
+ report()
+ end
+end
+
+local function show_1(wsplit)
+ local u = concat(wsplit," ")
+ steps = { { f_show(0,u), f_show(0,u) } }
+end
+
+local function show_2(c,m,wsplit,done,i,spec)
+ local s = lpegmatch(p_split,c)
+ local t = { }
+ local n = #m
+ local w = #wsplit
+ for j=1,n do
+ t[#t+1] = m[j]
+ t[#t+1] = s[j]
+ end
+ local m = 2*i-2
+ local l = #t
+ local s = spec and table.sequenced(spec) or ""
+ if m == 0 then
+ steps[#steps+1] = { f_show(m, concat(t,"",2)), f_show(1,concat(done," ",2,#done),s) }
+ elseif i+1 == w then
+ steps[#steps+1] = { f_show(m-1,concat(t,"",1,#t-1)), f_show(1,concat(done," ",2,#done),s) }
+ else
+ steps[#steps+1] = { f_show(m-1,concat(t)), f_show(1,concat(done," ",2,#done),s) }
+ end
+end
+
+local function show_3(wsplit,done)
+ local t = { }
+ local h = { }
+ local n = #wsplit
+ for i=1,n do
+ local w = wsplit[i]
+ if i > 1 then
+ local d = done[i]
+ t[#t+1] = i > 2 and d % 2 == 1 and "-" or " "
+ h[#h+1] = d
+ end
+ t[#t+1] = w
+ h[#h+1] = w
+ end
+ steps[#steps+1] = { f_show(0,concat(h)), f_show(0,concat(t)) }
+ show_log()
+end
+
+local function show_4(wsplit,done)
+ steps = { { concat(wsplit," ") } }
+ show_log()
+end
+
+function traditional.lasttrace()
+ return steps
+end
+
+-- We could reuse the w table but as we cache the resolved words there is not much gain in
+-- that complication.
+--
+-- Beware: word can be a table and when n is passed to we can assume reuse so we need to
+-- honor that n then.
+--
+-- todo: a fast variant for tex ... less lookups (we could check is dictionary has changed)
+-- ... although due to caching the already done words, we don't do much here
+
+local function hyphenate(dictionary,word,n) -- odd is okay
+ nofwords = nofwords + 1
+ local hyphenated = dictionary.hyphenated
+ local isstring = type(word) == "string"
+ if isstring then
+ local done = hyphenated[word]
+ if done ~= nil then
+ return done
+ end
+ elseif n then
+ local done = hyphenated[concat(word,"",1,n)]
+ if done ~= nil then
+ return done
+ end
+ else
+ local done = hyphenated[concat(word)]
+ if done ~= nil then
+ return done
+ end
+ end
+ local key
+ if isstring then
+ key = word
+ word = lpegmatch(p_split,word)
+ if not n then
+ n = #word
+ end
+ else
+ if not n then
+ n = #word
+ end
+ key = concat(word,"",1,n)
+ end
+ local l = 1
+ local w = { "." }
+ -- local d = dictionary.codehash
+ for i=1,n do
+ local c = word[i]
+ -- l = l + (d[c] or 1)
+ l = l + 1
+ w[l] = lcchars[c] or c
+ end
+ l = l + 1
+ w[l] = "."
+ local c = concat(w,"",2,l-1)
+ --
+ local done = hyphenated[c]
+ if done ~= nil then
+ hyphenated[key] = done
+ nofhashed = nofhashed + 1
+ return done
+ end
+ --
+ local exceptions = dictionary.exceptions
+ local exception = exceptions[c]
+ if exception then
+ if trace_steps then
+ show_4(w,exception)
+ end
+ hyphenated[key] = exception
+ nofhashed = nofhashed + 1
+ return exception
+ end
+ --
+ if trace_steps then
+ show_1(w)
+ end
+ --
+ local specials = dictionary.specials
+ local patterns = dictionary.patterns
+ --
+ local spec
+ for i=1,l do
+ for j=i,l do
+ local c = concat(w,"",i,j)
+ local m = patterns[c]
+ if m then
+ local s = specials[c]
+ if not done then
+ done = { }
+ spec = nil
+ -- the string that we resolve has explicit fences (.) so done starts at
+ -- the first fence and runs upto the last one so we need one slot less
+ for i=1,l do
+ done[i] = 0
+ end
+ end
+ -- we run over the pattern that always has a (zero) value for each character
+ -- plus one more as we look at both sides
+ for k=1,#m do
+ local new = m[k]
+ if not new then
+ break
+ elseif new == true then
+ report("fatal error")
+ break
+ elseif new > 0 then
+ local pos = i + k - 1
+ local old = done[pos]
+ if not old then
+ -- break ?
+ elseif new > old then
+ done[pos] = new
+ if s then
+ local b = i + (s.start or 1) - 1
+ if b > 0 then
+ local e = b + (s.length or 2) - 1
+ if e > 0 then
+ if pos >= b and pos <= e then
+ if spec then
+ spec[pos] = { s, k - 1 }
+ else
+ spec = { [pos] = { s, k - 1 } }
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ end
+ if trace_steps and done then
+ show_2(c,m,w,done,i,s)
+ end
+ end
+ end
+ end
+ if trace_steps and done then
+ show_3(w,done)
+ end
+ if done then
+ local okay = false
+ for i=3,#done do
+ if done[i] % 2 == 1 then
+ done[i-2] = spec and spec[i] or true
+ okay = true
+ else
+ done[i-2] = false
+ end
+ end
+ if okay then
+ done[#done] = nil
+ done[#done] = nil
+ else
+ done = false
+ end
+ else
+ done = false
+ end
+ hyphenated[key] = done
+ nofhashed = nofhashed + 1
+ return done
+end
+
+function traditional.gettrace(language,word)
+ if not word or word == "" then
+ return
+ end
+ local dictionary = dictionaries[language]
+ if dictionary then
+ local hyphenated = dictionary.hyphenated
+ hyphenated[word] = nil
+ hyphenate(dictionary,word)
+ return steps
+ end
+end
+
+local methods = setmetatableindex(function(t,k) local v = hyphenate t[k] = v return v end)
+
+function traditional.installmethod(name,f)
+ if rawget(methods,name) then
+ report("overloading %a is not permitted",name)
+ else
+ methods[name] = f
+ end
+end
+
+local s_detail_1 = "-"
+local f_detail_2 = formatters["%s-%s"]
+local f_detail_3 = formatters["{%s}{%s}{}"]
+local f_detail_4 = formatters["{%s%s}{%s%s}{%s}"]
+
+function traditional.injecthyphens(dictionary,word,specification)
+ if not word then
+ return false
+ end
+ if not specification then
+ return word
+ end
+ local hyphens = hyphenate(dictionary,word)
+ if not hyphens then
+ return word
+ end
+
+ -- the following code is similar to code later on but here we have strings while there
+ -- we have hyphen specs
+
+ local word = lpegmatch(p_split,word)
+ local size = #word
+
+ local leftmin = specification.leftcharmin or 2
+ local rightmin = size - (specification.rightcharmin or leftmin)
+ local leftchar = specification.leftchar
+ local rightchar = specification.rightchar
+
+ local result = { }
+ local rsize = 0
+ local position = 1
+
+ while position <= size do
+ if position >= leftmin and position <= rightmin then
+ local hyphen = hyphens[position]
+ if not hyphen then
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ elseif hyphen == true then
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ rsize = rsize + 1
+ if leftchar and rightchar then
+ result[rsize] = f_detail_3(rightchar,leftchar)
+ else
+ result[rsize] = s_detail_1
+ end
+ position = position + 1
+ else
+ local o, h = hyphen[2]
+ if o then
+ h = hyphen[1]
+ else
+ h = hyphen
+ o = 1
+ end
+ local b = position - o + (h.start or 1)
+ local e = b + (h.length or 2) - 1
+ if b > 0 and e >= b then
+ for i=1,b-position do
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ end
+ rsize = rsize + 1
+ if leftchar and rightchar then
+ result[rsize] = f_detail_4(h.before,rightchar,leftchar,h.after,concat(word,"",b,e))
+ else
+ result[rsize] = f_detail_2(h.before,h.after)
+ end
+ position = e + 1
+ else
+ -- error
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ end
+ end
+ else
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ end
+ end
+ return concat(result)
+end
+
+do
+
+ local word = C((1-space)^1)
+ local spaces = space^1
+
+ local u_pattern = (Carg(1) * Carg(2) * word / unregister_pattern + spaces)^1
+ local r_pattern = (Carg(1) * Carg(2) * word * Carg(3) / register_pattern + spaces)^1
+ local e_pattern = (Carg(1) * word / register_exception + spaces)^1
+
+ function traditional.registerpattern(language,str,specification)
+ local dictionary = dictionaries[language]
+ if specification == false then
+ lpegmatch(u_pattern,str,1,dictionary.patterns,dictionary.specials)
+ -- unregister_pattern(dictionary.patterns,dictionary.specials,str)
+ else
+ lpegmatch(r_pattern,str,1,dictionary.patterns,dictionary.specials,type(specification) == "table" and specification or false)
+ -- register_pattern(dictionary.patterns,dictionary.specials,str,specification)
+ end
+ end
+
+ function traditional.registerexception(language,str)
+ lpegmatch(e_pattern,str,1,dictionaries[language].exceptions)
+ end
+
+end
+
+-- todo: unicodes or utfhash ?
+
+if context then
+
+ local nodecodes = nodes.nodecodes
+ local disccodes = nodes.disccodes
+
+ local glyph_code = nodecodes.glyph
+ local disc_code = nodecodes.disc
+ local math_code = nodecodes.math
+ local hlist_code = nodecodes.hlist
+
+ local automaticdisc_code = disccodes.automatic
+ local regulardisc_code = disccodes.regular
+
+ local nuts = nodes.nuts
+ local tonode = nodes.tonode
+ local nodepool = nuts.pool
+
+ local new_disc = nodepool.disc
+ local new_penalty = nodepool.penalty
+
+ local getfield = nuts.getfield
+ local getfont = nuts.getfont
+ local getid = nuts.getid
+ local getattr = nuts.getattr
+ local getnext = nuts.getnext
+ local getprev = nuts.getprev
+ local getsubtype = nuts.getsubtype
+ local getlist = nuts.getlist
+ local getlanguage = nuts.getlanguage
+ local getattrlist = nuts.getattrlist
+ local setattrlist = nuts.setattrlist
+ local isglyph = nuts.isglyph
+ local ischar = nuts.ischar
+
+ local setchar = nuts.setchar
+ local setdisc = nuts.setdisc
+ local setlink = nuts.setlink
+ local setprev = nuts.setprev
+ local setnext = nuts.setnext
+
+ local insert_before = nuts.insert_before
+ local insert_after = nuts.insert_after
+ local copy_node = nuts.copy
+ local copy_list = nuts.copy_list
+ local remove_node = nuts.remove
+ local end_of_math = nuts.end_of_math
+ local node_tail = nuts.tail
+
+ local nexthlist = nuts.traversers.hlist
+ local nextdisc = nuts.traversers.disc
+
+ local setcolor = nodes.tracers.colors.set
+
+ local variables = interfaces.variables
+ local v_reset = variables.reset
+ local v_yes = variables.yes
+ local v_word = variables.word
+ local v_all = variables.all
+
+ local settings_to_array = utilities.parsers.settings_to_array
+
+ local unsetvalue = attributes.unsetvalue
+ local texsetattribute = tex.setattribute
+
+ local prehyphenchar = language.prehyphenchar
+ local posthyphenchar = language.posthyphenchar
+ local preexhyphenchar = language.preexhyphenchar
+ local postexhyphenchar = language.postexhyphenchar
+
+ local a_hyphenation = attributes.private("hyphenation")
+
+ local interwordpenalty = 5000
+
+ function traditional.loadpatterns(language)
+ return dictionaries[language]
+ end
+
+ -- for the moment we use an independent data structure
+
+ setmetatableindex(dictionaries,function(t,k)
+ if type(k) == "string" then
+ -- this will force a load if not yet loaded (we need a nicer way) for the moment
+ -- that will do (nneeded for examples that register a pattern specification
+ languages.getnumber(k)
+ end
+ local specification = languages.getdata(k)
+ local dictionary = {
+ patterns = { },
+ exceptions = { },
+ hyphenated = { },
+ specials = { },
+ instance = false,
+ characters = { },
+ unicodes = { },
+ }
+ if specification then
+ local resources = specification.resources
+ if resources then
+ local characters = dictionary.characters or { }
+ local unicodes = dictionary.unicodes or { }
+ for i=1,#resources do
+ local r = resources[i]
+ if not r.in_dictionary then
+ r.in_dictionary = true
+ local patterns = r.patterns
+ if patterns then
+ local data = patterns.data
+ if data then
+ -- regular patterns
+ lpegmatch(p_pattern,data,1,dictionary.patterns,dictionary.specials)
+ end
+ local extra = patterns.extra
+ if extra then
+ -- special patterns
+ lpegmatch(p_pattern,extra,1,dictionary.patterns,dictionary.specials)
+ end
+ end
+ local exceptions = r.exceptions
+ if exceptions then
+ local data = exceptions.data
+ if data and data ~= "" then
+ lpegmatch(p_exception,data,1,dictionary.exceptions)
+ end
+ end
+ local usedchars = lpegmatch(p_split,patterns.characters)
+ for i=1,#usedchars do
+ local char = usedchars[i]
+ local code = utfbyte(char)
+ local upper = uccodes[code]
+ characters[char] = code
+ unicodes [code] = char
+ if type(upper) == "table" then
+ for i=1,#upper do
+ local u = upper[i]
+ unicodes[u] = utfchar(u)
+ end
+ else
+ unicodes[upper] = utfchar(upper)
+ end
+ end
+ end
+ end
+ dictionary.characters = characters
+ dictionary.unicodes = unicodes
+ setmetatableindex(characters,function(t,k) local v = k and utfbyte(k) t[k] = v return v end)
+ end
+ t[specification.number] = dictionary
+ dictionary.instance = specification.instance -- needed for hyphenchars
+ end
+ t[k] = dictionary
+ return dictionary
+ end)
+
+ -- Beware: left and right min doesn't mean that in a 1 mmm hsize there can be snippets
+ -- with less characters than either of them! This could be an option but such a narrow
+ -- hsize doesn't make sense anyway.
+
+ -- We assume that featuresets are defined global ... local definitions (also mid paragraph)
+ -- make not much sense anyway. For the moment we assume no predefined sets so we don't need
+ -- to store them. Nor do we need to hash them in order to save space ... no sane user will
+ -- define many of them.
+
+ local featuresets = hyphenators.featuresets or { }
+ hyphenators.featuresets = featuresets
+
+ storage.shared.noflanguagesfeaturesets = storage.shared.noflanguagesfeaturesets or 0
+
+ local noffeaturesets = storage.shared.noflanguagesfeaturesets
+
+ storage.register("languages/hyphenators/featuresets",featuresets,"languages.hyphenators.featuresets")
+
+ ----- hash = table.sequenced(featureset,",") -- no need now
+
+ local function register(name,featureset)
+ noffeaturesets = noffeaturesets + 1
+ featureset.attribute = noffeaturesets
+ featuresets[noffeaturesets] = featureset -- access by attribute
+ featuresets[name] = featureset -- access by name
+ storage.shared.noflanguagesfeaturesets = noffeaturesets
+ return noffeaturesets
+ end
+
+ local function makeset(...)
+ -- a bit overkill, supporting variants but who cares
+ local set = { }
+ for i=1,select("#",...) do
+ local list = select(i,...)
+ local kind = type(list)
+ local used = nil
+ if kind == "string" then
+ if list == v_all then
+ -- not ok ... now all get ignored
+ return setmetatableindex(function(t,k) local v = utfchar(k) t[k] = v return v end)
+ elseif list ~= "" then
+ used = lpegmatch(p_split,list)
+ set = set or { }
+ for i=1,#used do
+ local char = used[i]
+ set[utfbyte(char)] = char
+ end
+ end
+ elseif kind == "table" then
+ if next(list) then
+ set = set or { }
+ for byte, char in next, list do
+ set[byte] = char == true and utfchar(byte) or char
+ end
+ elseif #list > 0 then
+ set = set or { }
+ for i=1,#list do
+ local l = list[i]
+ if type(l) == "number" then
+ set[l] = utfchar(l)
+ else
+ set[utfbyte(l)] = l
+ end
+ end
+ end
+ end
+ end
+ return set
+ end
+
+ -- category pd (tex also sees --- and -- as hyphens but do we really want that
+
+ local defaulthyphens = {
+ [0x002D] = true, -- HYPHEN-MINUS
+ [0x00AD] = 0x002D, -- SOFT HYPHEN (active in ConTeXt)
+ -- [0x058A] = true, -- ARMENIAN HYPHEN
+ -- [0x1400] = true, -- CANADIAN SYLLABICS HYPHEN
+ -- [0x1806] = true, -- MONGOLIAN TODO SOFT HYPHEN
+ [0x2010] = true, -- HYPHEN
+ -- [0x2011] = true, -- NON-BREAKING HYPHEN
+ -- [0x2012] = true, -- FIGURE DASH
+ [0x2013] = true, -- EN DASH
+ [0x2014] = true, -- EM DASH
+ -- [0x2015] = true, -- HORIZONTAL BAR
+ -- [0x2027] = true, -- HYPHENATION POINT
+ -- [0x2E17] = true, -- DOUBLE OBLIQUE HYPHEN
+ -- [0x2E1A] = true, -- HYPHEN WITH DIAERESIS
+ -- [0x2E3A] = true, -- TWO-EM DASH
+ -- [0x2E3B] = true, -- THREE-EM DASH
+ -- [0x2E40] = true, -- DOUBLE HYPHEN
+ -- [0x301C] = true, -- WAVE DASH
+ -- [0x3030] = true, -- WAVY DASH
+ -- [0x30A0] = true, -- KATAKANA-HIRAGANA DOUBLE HYPHEN
+ -- [0xFE31] = true, -- PRESENTATION FORM FOR VERTICAL EM DASH
+ -- [0xFE32] = true, -- PRESENTATION FORM FOR VERTICAL EN DASH
+ -- [0xFE58] = true, -- SMALL EM DASH
+ -- [0xFE63] = true, -- SMALL HYPHEN-MINUS
+ -- [0xFF0D] = true, -- FULLWIDTH HYPHEN-MINUS
+ }
+
+ local defaultjoiners = {
+ [0x200C] = true, -- nzwj
+ [0x200D] = true, -- zwj
+ }
+
+ local function somehyphenchar(c)
+ c = tonumber(c)
+ return c ~= 0 and c or nil
+ end
+
+ local function definefeatures(name,featureset)
+ local extrachars = featureset.characters -- "[]()"
+ local hyphenchars = featureset.hyphens
+ local joinerchars = featureset.joiners
+ local alternative = featureset.alternative
+ local rightwordmin = tonumber(featureset.rightwordmin)
+ local charmin = tonumber(featureset.charmin) -- luatex now also has hyphenationmin
+ local leftcharmin = tonumber(featureset.leftcharmin)
+ local rightcharmin = tonumber(featureset.rightcharmin)
+ local leftchar = somehyphenchar(featureset.leftchar)
+ local rightchar = somehyphenchar(featureset.rightchar)
+ local rightchars = featureset.rightchars
+local rightedge = featureset.rightedge
+local autohyphen = v_yes -- featureset.autohyphen -- insert disc
+local hyphenonly = v_yes -- featureset.hyphenonly -- don't hyphenate around
+ rightchars = rightchars == v_word and true or tonumber(rightchars)
+ joinerchars = joinerchars == v_yes and defaultjoiners or joinerchars -- table
+ hyphenchars = hyphenchars == v_yes and defaulthyphens or hyphenchars -- table
+ -- not yet ok: extrachars have to be ignored so it cannot be all)
+ featureset.extrachars = makeset(joinerchars or "",extrachars or "")
+ featureset.hyphenchars = makeset(hyphenchars or "")
+ featureset.alternative = alternative or "hyphenate"
+ featureset.rightwordmin = rightwordmin and rightwordmin > 0 and rightwordmin or nil
+ featureset.charmin = charmin and charmin > 0 and charmin or nil
+ featureset.leftcharmin = leftcharmin and leftcharmin > 0 and leftcharmin or nil
+ featureset.rightcharmin = rightcharmin and rightcharmin > 0 and rightcharmin or nil
+ featureset.rightchars = rightchars
+ featureset.leftchar = leftchar
+ featureset.rightchar = rightchar
+ -- featureset.strict = rightedge == "tex"
+featureset.autohyphen = autohyphen == v_yes
+featureset.hyphenonly = hyphenonly == v_yes
+ return register(name,featureset)
+ end
+
+ local function setfeatures(n)
+ if not n or n == v_reset then
+ n = false
+ else
+ local f = featuresets[n]
+ if not f and type(n) == "string" then
+ local t = settings_to_array(n)
+ local s = { }
+ for i=1,#t do
+ local ti = t[i]
+ local fs = featuresets[ti]
+ if fs then
+ for k, v in next, fs do
+ s[k] = v
+ end
+ end
+ end
+ n = register(n,s)
+ else
+ n = f and f.attribute
+ end
+ end
+ texsetattribute(a_hyphenation,n or unsetvalue)
+ end
+
+ traditional.definefeatures = definefeatures
+ traditional.setfeatures = setfeatures
+
+ implement {
+ name = "definehyphenationfeatures",
+ actions = definefeatures,
+ arguments = {
+ "string",
+ {
+ { "characters" },
+ { "hyphens" },
+ { "joiners" },
+ { "rightchars" },
+ { "rightwordmin", "integer" },
+ { "charmin", "integer" },
+ { "leftcharmin", "integer" },
+ { "rightcharmin", "integer" },
+ { "leftchar", "integer" },
+ { "rightchar", "integer" },
+ { "alternative" },
+ { "rightedge" },
+ }
+ }
+ }
+
+ implement {
+ name = "sethyphenationfeatures",
+ actions = setfeatures,
+ arguments = "string"
+ }
+
+ implement {
+ name = "registerhyphenationpattern",
+ actions = traditional.registerpattern,
+ arguments = { "string", "string", "boolean" }
+ }
+
+ implement {
+ name = "registerhyphenationexception",
+ actions = traditional.registerexception,
+ arguments = "2 strings",
+ }
+
+ -- This is a relative large function with local variables and local functions. A previous
+ -- implementation had the functions outside but this is cleaner and as efficient. The test
+ -- runs 100 times over tufte.tex, knuth.tex, zapf.tex, ward.tex and darwin.tex in lower
+ -- and uppercase with a 1mm hsize.
+ --
+ -- language=0 language>0 4 | 3 * slower
+ --
+ -- tex 2.34 | 1.30 2.55 | 1.45 0.21 | 0.15
+ -- lua 2.42 | 1.38 3.30 | 1.84 0.88 | 0.46
+ --
+ -- Of course we have extra overhead (virtual Lua machine) but also we check attributes and
+ -- support specific local options). The test puts the typeset text in boxes and discards
+ -- it. If we also flush the runtime is 4.31|2.56 and 4.99|2.94 seconds so the relative
+ -- difference is (somehow) smaller. The test has 536 pages. There is a little bit of extra
+ -- overhead because we store the patterns in a different way.
+ --
+ -- As usual I will look for speedups. Some 0.01 seconds could be gained by sharing patterns
+ -- which is not impressive but it does save some 3M memory on this test. (Some optimizations
+ -- already brought the 3.30 seconds down to 3.14 but it all depends on aggressive caching.)
+
+ -- As we kick in the hyphenator before fonts get handled, we don't look at implicit (font)
+ -- kerns or ligatures.
+
+ local starttiming = statistics.starttiming
+ local stoptiming = statistics.stoptiming
+
+ -- local strictids = {
+ -- [nodecodes.hlist] = true,
+ -- [nodecodes.vlist] = true,
+ -- [nodecodes.rule] = true,
+ -- [nodecodes.dir] = true,
+ -- [nodecodes.whatsit] = true,
+ -- [nodecodes.ins] = true,
+ -- [nodecodes.adjust] = true,
+ --
+ -- [nodecodes.math] = true,
+ -- [nodecodes.disc] = true,
+ --
+ -- [nodecodes.accent] = true, -- never used in context
+ -- }
+
+ -- a lot of overhead when only one char
+
+ function traditional.hyphenate(head)
+
+ local first = head
+ local tail = nil
+ local last = nil
+ local current = first
+ local dictionary = nil
+ local instance = nil
+ local characters = nil
+ local unicodes = nil
+ local exhyphenchar = tex.exhyphenchar
+ local extrachars = nil
+ local hyphenchars = nil
+ local language = nil
+ local lastfont = nil
+ local start = nil
+ local stop = nil
+ local word = { } -- we reuse this table
+ local size = 0
+ local leftchar = false
+ local rightchar = false -- utfbyte("-")
+ local leftexchar = false
+ local rightexchar = false -- utfbyte("-")
+ local leftmin = 0
+ local rightmin = 0
+ local charmin = 1
+ local leftcharmin = nil
+ local rightcharmin = nil
+ ----- leftwordmin = nil
+ local rightwordmin = nil
+ local rightchars = nil
+ local leftchar = nil
+ local rightchar = nil
+ local attr = nil
+ local lastwordlast = nil
+ local hyphenated = hyphenate
+ ----- strict = nil
+ local exhyphenpenalty = tex.exhyphenpenalty
+ local hyphenpenalty = tex.hyphenpenalty
+ local autohyphen = false
+ local hyphenonly = false
+
+ -- We cannot use an 'enabled' boolean (false when no characters or extras) because we
+ -- can have plugins that set a characters metatable and so) ... it doesn't save much
+ -- anyway. Using (unicodes and unicodes[code]) and a nil table when no characters also
+ -- doesn't save much. So there not that much to gain for languages that don't hyphenate.
+ --
+ -- enabled = (unicodes and (next(unicodes) or getmetatable(unicodes)))
+ -- or (extrachars and next(extrachars))
+ --
+ -- This can be used to not add characters i.e. keep size 0 but then we need to check for
+ -- attributes that change it, which costs time too. Not much to gain there.
+
+ starttiming(traditional)
+
+ local function insertpenalty()
+ local p = new_penalty(interwordpenalty)
+ setattrlist(p,last)
+ if trace_visualize then
+ nuts.setvisual(p,"penalty")
+ end
+ last = getprev(last)
+ first, last = insert_after(first,last,p)
+ end
+
+ local function synchronizefeatureset(a)
+ local f = a and featuresets[a]
+ if f then
+ hyphenated = methods[f.alternative or "hyphenate"]
+ extrachars = f.extrachars
+ hyphenchars = f.hyphenchars
+ rightwordmin = f.rightwordmin
+ charmin = f.charmin
+ leftcharmin = f.leftcharmin
+ rightcharmin = f.rightcharmin
+ leftchar = f.leftchar
+ rightchar = f.rightchar
+ -- strict = f.strict and strictids
+ rightchars = f.rightchars
+ autohyphen = f.autohyphen
+ hyphenonly = f.hyphenonly
+ if rightwordmin and rightwordmin > 0 and lastwordlast ~= rightwordmin then
+ -- so we can change mid paragraph but it's kind of unpredictable then
+ if not tail then
+ tail = node_tail(first)
+ end
+ last = tail
+ local inword = false
+ local count = 0
+ while last and rightwordmin > 0 do
+ local id = getid(last)
+ if id == glyph_code then
+ count = count + 1
+ inword = true
+ if trace_visualize then
+ setcolor(last,"darkgreen")
+ end
+ elseif inword then
+ inword = false
+ rightwordmin = rightwordmin - 1
+ if rightchars == true then
+ if rightwordmin > 0 then
+ insertpenalty()
+ end
+ elseif rightchars and count <= rightchars then
+ insertpenalty()
+ end
+ end
+ last = getprev(last)
+ end
+ lastwordlast = rightwordmin
+ end
+ if not charmin or charmin == 0 then
+ charmin = 1
+ end
+ else
+ hyphenated = methods.hyphenate
+ extrachars = false
+ hyphenchars = false
+ rightwordmin = false
+ charmin = 1
+ leftcharmin = false
+ rightcharmin = false
+ leftchar = false
+ rightchar = false
+ -- strict = false
+ autohyphen = false
+ hyphenonly = false
+ end
+
+ return a
+ end
+
+ local function flush(hyphens) -- todo: no need for result
+
+ local rightmin = size - rightmin
+ local result = { }
+ local rsize = 0
+ local position = 1
+
+ -- todo: remember last dics and don't go back to before that (plus message) ...
+ -- for simplicity we also assume that we don't start with a dics node
+ --
+ -- there can be a conflict: if we backtrack then we can end up in another disc
+ -- and get out of sync (dup chars and so)
+
+ while position <= size do
+ if position >= leftmin and position <= rightmin then
+ local hyphen = hyphens[position]
+ if not hyphen then
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ elseif hyphen == true then
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ rsize = rsize + 1
+ result[rsize] = true
+ position = position + 1
+ else
+ local o, h = hyphen[2]
+ if o then
+ -- { hyphen, offset)
+ h = hyphen[1]
+ else
+ -- hyphen
+ h = hyphen
+ o = 1
+ end
+ local b = position - o + (h.start or 1)
+ local e = b + (h.length or 2) - 1
+ if b > 0 and e >= b then
+ for i=1,b-position do
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ end
+ rsize = rsize + 1
+ result[rsize] = {
+ h.before or "", -- pre
+ h.after or "", -- post
+ concat(word,"",b,e), -- replace
+ h.right, -- optional after pre
+ h.left, -- optional before post
+ }
+ position = e + 1
+ else
+ -- error
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ end
+ end
+ else
+ rsize = rsize + 1
+ result[rsize] = word[position]
+ position = position + 1
+ end
+ end
+
+ local function serialize(replacement,leftchar,rightchar)
+ if not replacement then
+ return
+ elseif replacement == true then
+ local glyph = copy_node(stop)
+ setchar(glyph,leftchar or rightchar)
+ return glyph
+ end
+ local head = nil
+ local current = nil
+ if leftchar then
+ head = copy_node(stop)
+ current = head
+ setchar(head,leftchar)
+ end
+ local rsize = #replacement
+ if rsize == 1 then
+ local glyph = copy_node(stop)
+ setchar(glyph,characters[replacement])
+ if head then
+ insert_after(current,current,glyph)
+ else
+ head = glyph
+ end
+ current = glyph
+ elseif rsize > 0 then
+ local list = lpegmatch(p_split,replacement) -- this is an utf split (could be cached)
+ for i=1,#list do
+ local glyph = copy_node(stop)
+ setchar(glyph,characters[list[i]])
+ if head then
+ insert_after(current,current,glyph)
+ else
+ head = glyph
+ end
+ current = glyph
+ end
+ end
+ if rightchar then
+ local glyph = copy_node(stop)
+ insert_after(current,current,glyph)
+ setchar(glyph,rightchar)
+ end
+ return head
+ end
+
+ local current = start
+ local attrnode = start -- will be different, just the first char
+
+ for i=1,rsize do
+ local r = result[i]
+ if r == true then
+ local disc = new_disc()
+ local pre = nil
+ local post = nil
+ if rightchar then
+ pre = serialize(true,rightchar)
+ end
+ if leftchar then
+ post = serialize(true,leftchar)
+ end
+ setdisc(disc,pre,post,nil,regulardisc_code,hyphenpenalty)
+ if attrnode then
+ setattrlist(disc,attrnode)
+ end
+ -- could be a replace as well
+ insert_before(first,current,disc)
+ elseif type(r) == "table" then
+ local disc = new_disc()
+ local pre = r[1]
+ local post = r[2]
+ local replace = r[3]
+ local right = r[4] ~= false and rightchar
+ local left = r[5] ~= false and leftchar
+ if pre then
+ if pre ~= "" then
+ pre = serialize(pre,false,right)
+ else
+ pre = nil
+ end
+ end
+ if post then
+ if post ~= "" then
+ post = serialize(post,left,false)
+ else
+ post = nil
+ end
+ end
+ if replace then
+ if replace ~= "" then
+ replace = serialize(replace)
+ else
+ replace = nil
+ end
+ end
+ -- maybe regular code
+ setdisc(disc,pre,post,replace,regulardisc_code,hyphenpenalty)
+ if attrnode then
+ setattrlist(disc,attrnode)
+ end
+ insert_before(first,current,disc)
+ else
+ setchar(current,characters[r])
+ if i < rsize then
+ current = getnext(current)
+ end
+ end
+ end
+ if current and current ~= stop then
+ local current = getnext(current)
+ local last = getnext(stop)
+ while current ~= last do
+ first, current = remove_node(first,current,true)
+ end
+ end
+
+ end
+
+ local function inject(leftchar,rightchar,code,attrnode)
+ if first ~= current then
+ local disc = new_disc()
+ first, current, glyph = remove_node(first,current)
+ first, current = insert_before(first,current,disc)
+ if trace_visualize then
+ setcolor(glyph,"darkred") -- these get checked
+ setcolor(disc,"darkgreen") -- in the colorizer
+ end
+ local pre = nil
+ local post = nil
+ local replace = glyph
+ if leftchar and leftchar > 0 then
+ post = copy_node(glyph)
+ setchar(post,leftchar)
+ end
+ pre = copy_node(glyph)
+ setchar(pre,rightchar and rightchar > 0 and rightchar or code)
+ setdisc(disc,pre,post,replace,automaticdisc_code,hyphenpenalty) -- ex ?
+ if attrnode then
+ setattrlist(disc,attrnode)
+ end
+ end
+ return current
+ end
+
+ local function injectseries(current,last,next,attrnode)
+ local disc = new_disc()
+ local start = current
+ first, current = insert_before(first,current,disc)
+ setprev(start)
+ setnext(last)
+ if next then
+ setlink(current,next)
+ else
+ setnext(current)
+ end
+ local pre = copy_list(start)
+ local post = nil
+ local replace = start
+ setdisc(disc,pre,post,replace,automaticdisc_code,hyphenpenalty) -- ex ?
+ if attrnode then
+ setattrlist(disc,attrnode)
+ end
+ return current
+ end
+
+ local a = getattr(first,a_hyphenation)
+ if a ~= attr then
+ attr = synchronizefeatureset(a)
+ end
+
+ -- The first attribute in a word determines the way a word gets hyphenated and if
+ -- relevant, other properties are also set then. We could optimize for silly one-char
+ -- cases but it has no priority as the code is still not that much slower than the
+ -- native hyphenator and this variant also provides room for extensions.
+
+ local skipping = false
+
+ -- In "word word word." the sequences "word" and "." can be a different font!
+
+ while current and current ~= last do -- and current
+ local code, id = isglyph(current)
+ if code then
+ if skipping then
+ current = getnext(current)
+ else
+ local lang = getlanguage(current)
+ local font = getfont(current)
+ if lang ~= language or font ~= lastfont then
+ if dictionary and size > charmin and leftmin + rightmin <= size then
+ -- only german has many words starting with an uppercase character
+ if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
+ -- skip
+ else
+ local hyphens = hyphenated(dictionary,word,size)
+ if hyphens then
+ flush(hyphens)
+ end
+ end
+ end
+ lastfont = font
+ if language ~= lang and lang > 0 then
+ --
+ dictionary = dictionaries[lang]
+ instance = dictionary.instance
+ characters = dictionary.characters
+ unicodes = dictionary.unicodes
+ --
+ local a = getattr(current,a_hyphenation)
+ attr = synchronizefeatureset(a)
+ leftchar = leftchar or (instance and posthyphenchar (instance)) -- we can make this more
+ rightchar = rightchar or (instance and prehyphenchar (instance)) -- efficient if needed
+ leftexchar = (instance and preexhyphenchar (instance))
+ rightexchar = (instance and postexhyphenchar(instance))
+ leftmin = leftcharmin or getfield(current,"lhmin")
+ rightmin = rightcharmin or getfield(current,"rhmin")
+ if not leftchar or leftchar < 0 then
+ leftchar = false
+ end
+ if not rightchar or rightchar < 0 then
+ rightchar = false
+ end
+ --
+ local char = unicodes[code] or (extrachars and extrachars[code])
+ if char then
+ word[1] = char
+ size = 1
+ start = current
+ else
+ size = 0
+ end
+ else
+ size = 0
+ end
+ language = lang
+ elseif language <= 0 then
+ --
+ elseif size > 0 then
+ local char = unicodes[code] or (extrachars and extrachars[code])
+ if char then
+ size = size + 1
+ word[size] = char
+ elseif dictionary then
+ if not hyphenonly or code ~= exhyphenchar then
+ if size > charmin and leftmin + rightmin <= size then
+ if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
+ -- skip
+ else
+ local hyphens = hyphenated(dictionary,word,size)
+ if hyphens then
+ flush(hyphens)
+ end
+ end
+ end
+ end
+ size = 0
+ if code == exhyphenchar then -- normally the -
+ local next = getnext(current)
+ local last = current
+ local font = getfont(current)
+ while next and ischar(next,font) == code do
+ last = next
+ next = getnext(next)
+ end
+ if not autohyphen then
+ current = last
+ elseif current == last then
+ current = inject(leftexchar,rightexchar,code,current)
+ else
+ current = injectseries(current,last,next,current)
+ end
+ if hyphenonly then
+ skipping = true
+ end
+ elseif hyphenchars then
+ local char = hyphenchars[code]
+ if char == true then
+ char = code
+ end
+ if char then
+ current = inject(leftchar and char or nil,rightchar and char or nil,char,current)
+ end
+ end
+ end
+ else
+ local a = getattr(current,a_hyphenation)
+ if a ~= attr then
+ attr = synchronizefeatureset(a) -- influences extrachars
+ leftchar = leftchar or (instance and posthyphenchar (instance)) -- we can make this more
+ rightchar = rightchar or (instance and prehyphenchar (instance)) -- efficient if needed
+ leftexchar = (instance and preexhyphenchar (instance))
+ rightexchar = (instance and postexhyphenchar(instance))
+ leftmin = leftcharmin or getfield(current,"lhmin")
+ rightmin = rightcharmin or getfield(current,"rhmin")
+ if not leftchar or leftchar < 0 then
+ leftchar = false
+ end
+ if not rightchar or rightchar < 0 then
+ rightchar = false
+ end
+ end
+ --
+ local char = unicodes[code] or (extrachars and extrachars[code])
+ if char then
+ word[1] = char
+ size = 1
+ start = current
+ end
+ end
+ stop = current
+ current = getnext(current)
+ end
+ else
+ if skipping then
+ skipping = false
+ end
+ if id == disc_code then
+ size = 0
+ current = getnext(current)
+ if hyphenonly then
+ skipping = true
+ end
+ -- elseif strict and strict[id] then
+ -- current = id == math_code and getnext(end_of_math(current)) or getnext(current)
+ -- size = 0
+ else
+ current = id == math_code and getnext(end_of_math(current)) or getnext(current)
+ end
+ if size > 0 then
+ if dictionary and size > charmin and leftmin + rightmin <= size then
+ if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
+ -- skip
+ else
+ local hyphens = hyphenated(dictionary,word,size)
+ if hyphens then
+ flush(hyphens)
+ end
+ end
+ end
+ size = 0
+ end
+ end
+ end
+ -- we can have quit due to last so we need to flush the last seen word, we could move
+ -- this in the loop and test for current but ... messy
+ if dictionary and size > charmin and leftmin + rightmin <= size then
+ if categories[word[1]] == "lu" and getfield(start,"uchyph") < 0 then
+ -- skip
+ else
+ local hyphens = hyphenated(dictionary,word,size)
+ if hyphens then
+ flush(hyphens)
+ end
+ end
+ end
+
+ stoptiming(traditional)
+
+ return head
+ end
+
+ statistics.register("hyphenation",function()
+ if nofwords > 0 or statistics.elapsed(traditional) > 0 then
+ return string.format("%s words hyphenated, %s unique, used time %s",
+ nofwords,nofhashed,statistics.elapsedseconds(traditional) or 0)
+ end
+ end)
+
+ local texmethod = "builders.kernel.hyphenation"
+ local oldmethod = texmethod
+ local newmethod = texmethod
+
+ -- local newmethod = "languages.hyphenators.traditional.hyphenate"
+ --
+ -- nodes.tasks.prependaction("processors","words",newmethod)
+ -- nodes.tasks.disableaction("processors",oldmethod)
+ --
+ -- nodes.tasks.replaceaction("processors","words",oldmethod,newmethod)
+
+ -- \enabledirectives[hyphenators.method=traditional]
+ -- \enabledirectives[hyphenators.method=builtin]
+
+ -- push / pop ? check first attribute
+
+ -- local replaceaction = nodes.tasks.replaceaction -- no longer overload this way (too many local switches)
+
+ local hyphenate = language.hyphenate
+ local hyphenating = nuts.hyphenating
+ local methods = { }
+ local usedmethod = false
+ local stack = { }
+
+ local original = hyphenating and
+ function(head)
+ return (hyphenating(head))
+ end
+ or
+ function(head)
+ hyphenate(tonode(head))
+ return head -- a nut
+ end
+
+ -- local has_language = language.has_language
+ --
+ -- local function original(head) -- kernel.hyphenation(head)
+ -- local h = tonode(head)
+ -- if has_language(h) then
+ -- hyphenate(h)
+ -- end
+ -- return head
+ -- end
+
+ local getcount = tex.getcount
+
+ hyphenators.methods = methods
+ local optimize = false
+
+ directives.register("hyphenator.optimize", function(v) optimize = v end)
+
+ function hyphenators.handler(head,groupcode)
+ if usedmethod then
+ if optimize and (groupcode == "hbox" or groupcode == "adjusted_hbox") then
+ if getcount("hyphenstate") > 0 then
+ forced = false
+ return usedmethod(head)
+ else
+ return head
+ end
+ else
+ return usedmethod(head)
+ end
+ else
+ return head
+ end
+ end
+
+ methods.tex = original
+ methods.original = original
+ methods.expanded = original -- was expanded before 1.005
+ methods.traditional = languages.hyphenators.traditional.hyphenate
+ methods.none = false -- function(head) return head, false end
+
+ usedmethod = original
+
+ local function setmethod(method)
+ usedmethod = type(method) == "string" and methods[method]
+ if usedmethod == nil then
+ usedmethod = methods.tex
+ end
+ end
+ local function pushmethod(method)
+ insert(stack,usedmethod)
+ usedmethod = type(method) == "string" and methods[method]
+ if usedmethod == nil then
+ usedmethod = methods.tex
+ end
+ end
+ local function popmethod()
+ usedmethod = remove(stack) or methods.tex
+ end
+
+ hyphenators.setmethod = setmethod
+ hyphenators.pushmethod = pushmethod
+ hyphenators.popmethod = popmethod
+
+ directives.register("hyphenators.method",setmethod)
+
+ function hyphenators.setup(specification)
+ local method = specification.method
+ if method then
+ setmethod(method)
+ end
+ end
+
+ implement { name = "sethyphenationmethod", actions = setmethod, arguments = "string" }
+ implement { name = "pushhyphenation", actions = pushmethod, arguments = "string" }
+ implement { name = "pophyphenation", actions = popmethod }
+
+ -- can become a runtime loaded one:
+
+ local context = context
+ local ctx_NC = context.NC
+ local ctx_NR = context.NR
+ local ctx_verbatim = context.verbatim
+
+ function hyphenators.showhyphenationtrace(language,word)
+ if not word or word == "" then
+ return
+ end
+ local saved = trace_steps
+ trace_steps = "silent"
+ local steps = traditional.gettrace(language,word)
+ trace_steps = saved
+ if steps then
+ local n = #steps
+ if n > 0 then
+ context.starttabulate { "|r|l|l|l|" }
+ for i=1,n do
+ local s = steps[i]
+ ctx_NC() if i > 1 and i < n then context(i-1) end
+ ctx_NC() ctx_verbatim(s[1])
+ ctx_NC() ctx_verbatim(s[2])
+ ctx_NC() ctx_verbatim(s[3])
+ ctx_NC()
+ ctx_NR()
+ end
+ context.stoptabulate()
+ end
+ end
+ end
+
+ implement {
+ name = "showhyphenationtrace",
+ actions = hyphenators.showhyphenationtrace,
+ arguments = "2 strings",
+ }
+
+ function nodes.stripdiscretionaries(head)
+ for l in nexthlist, head do
+ for d in nextdisc, getlist(l) do
+ remove_node(h,false,true)
+ end
+ end
+ return head
+ end
+
+
+else
+
+ -- traditional.loadpatterns("nl","lang-nl")
+ -- traditional.loadpatterns("de","lang-de")
+ -- traditional.loadpatterns("us","lang-us")
+
+ -- traditional.registerpattern("nl","e1ë", { start = 1, length = 2, before = "e", after = "e" } )
+ -- traditional.registerpattern("nl","oo7ë", { start = 2, length = 3, before = "o", after = "e" } )
+ -- traditional.registerpattern("de","qqxc9xkqq",{ start = 3, length = 4, before = "ab", after = "cd" } )
+
+ -- local specification = {
+ -- leftcharmin = 2,
+ -- rightcharmin = 2,
+ -- leftchar = "<",
+ -- rightchar = ">",
+ -- }
+
+ -- print("reëel", traditional.injecthyphens(dictionaries.nl,"reëel", specification),"r{e>}{<e}{eë}el")
+ -- print("reeëel", traditional.injecthyphens(dictionaries.nl,"reeëel", specification),"re{e>}{<e}{eë}el")
+ -- print("rooëel", traditional.injecthyphens(dictionaries.nl,"rooëel", specification),"r{o>}{<e}{ooë}el")
+
+ -- print( "qxcxkq", traditional.injecthyphens(dictionaries.de, "qxcxkq", specification),"")
+ -- print( "qqxcxkqq", traditional.injecthyphens(dictionaries.de, "qqxcxkqq", specification),"")
+ -- print( "qqqxcxkqqq", traditional.injecthyphens(dictionaries.de, "qqqxcxkqqq", specification),"")
+ -- print("qqqqxcxkqqqq",traditional.injecthyphens(dictionaries.de,"qqqqxcxkqqqq",specification),"")
+
+ -- print("kunstmatig", traditional.injecthyphens(dictionaries.nl,"kunstmatig", specification),"")
+ -- print("kunststofmatig", traditional.injecthyphens(dictionaries.nl,"kunststofmatig", specification),"")
+ -- print("kunst[stof]matig", traditional.injecthyphens(dictionaries.nl,"kunst[stof]matig", specification),"")
+
+ -- traditional.loadpatterns("us","lang-us")
+
+ -- local specification = {
+ -- leftcharmin = 2,
+ -- rightcharmin = 2,
+ -- leftchar = false,
+ -- rightchar = false,
+ -- }
+
+ -- trace_steps = true
+
+ -- print("components", traditional.injecthyphens(dictionaries.us,"components", specification),"")
+ -- print("single", traditional.injecthyphens(dictionaries.us,"single", specification),"sin-gle")
+ -- print("everyday", traditional.injecthyphens(dictionaries.us,"everyday", specification),"every-day")
+ -- print("associate", traditional.injecthyphens(dictionaries.us,"associate", specification),"as-so-ciate")
+ -- print("philanthropic", traditional.injecthyphens(dictionaries.us,"philanthropic", specification),"phil-an-thropic")
+ -- print("projects", traditional.injecthyphens(dictionaries.us,"projects", specification),"projects")
+ -- print("Associate", traditional.injecthyphens(dictionaries.us,"Associate", specification),"As-so-ciate")
+ -- print("Philanthropic", traditional.injecthyphens(dictionaries.us,"Philanthropic", specification),"Phil-an-thropic")
+ -- print("Projects", traditional.injecthyphens(dictionaries.us,"Projects", specification),"Projects")
+
+end
+