summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/lang-wrd.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lang-wrd.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/lang-wrd.lua381
1 files changed, 285 insertions, 96 deletions
diff --git a/Master/texmf-dist/tex/context/base/lang-wrd.lua b/Master/texmf-dist/tex/context/base/lang-wrd.lua
index 095e44443d9..558d5744faf 100644
--- a/Master/texmf-dist/tex/context/base/lang-wrd.lua
+++ b/Master/texmf-dist/tex/context/base/lang-wrd.lua
@@ -8,44 +8,59 @@ if not modules then modules = { } end modules ['lang-ini'] = {
local utf = unicode.utf8
local lower, utfchar = string.lower, utf.char
+local concat = table.concat
local lpegmatch = lpeg.match
+local P, S, Cs = lpeg.P, lpeg.S, lpeg.Cs
-languages.words = languages.words or { }
+local report_words = logs.reporter("languages","words")
-local words = languages.words
+local nodes, node, languages = nodes, node, languages
-words.data = words.data or { }
-words.enables = false
-words.threshold = 4
+languages.words = languages.words or { }
+local words = languages.words
+
+words.data = words.data or { }
+words.enables = false
+words.threshold = 4
+
+local numbers = languages.numbers
+local registered = languages.registered
local set_attribute = node.set_attribute
local unset_attribute = node.unset_attribute
local traverse_nodes = node.traverse
-local node_id = node.id
local wordsdata = words.data
local chardata = characters.data
+local tasks = nodes.tasks
-local glyph_node = node_id('glyph')
-local disc_node = node_id('disc')
-local kern_node = node_id('kern')
+local nodecodes = nodes.nodecodes
+local kerncodes = nodes.kerncodes
-words.colors = {
- ["known"] = "green",
- ["unknown"] = "red",
-}
+local glyph_code = nodecodes.glyph
+local disc_code = nodecodes.disc
+local kern_code = nodecodes.kern
+
+local kerning_code = kerncodes.kerning
+local lowerchar = characters.lower
-local spacing = lpeg.S(" \n\r\t")
-local markup = lpeg.S("-=")
-local lbrace = lpeg.P("{")
-local rbrace = lpeg.P("}")
+local a_color = attributes.private('color')
+local colist = attributes.list[a_color]
+
+local is_letter = characters.is_letter -- maybe is_character as variant
+
+local spacing = S(" \n\r\t")
+local markup = S("-=")
+local lbrace = P("{")
+local rbrace = P("}")
local disc = (lbrace * (1-rbrace)^0 * rbrace)^1 -- or just 3 times, time this
-local word = lpeg.Cs((markup/"" + disc/"" + (1-spacing))^1)
+local word = Cs((markup/"" + disc/"" + (1-spacing))^1)
local loaded = { } -- we share lists
function words.load(tag,filename)
- local fullname = resolvers.find_file(filename,'other text file') or ""
+ local fullname = resolvers.findfile(filename,'other text file') or ""
if fullname ~= "" then
+ report_words("loading word file '%s'",fullname)
statistics.starttiming(languages)
local list = loaded[fullname]
if not list then
@@ -57,7 +72,7 @@ function words.load(tag,filename)
wordsdata[tag] = list
statistics.stoptiming(languages)
else
- logs.report("languages","missing words file '%s'",filename)
+ report_words("missing word file '%s'",filename)
end
end
@@ -65,149 +80,323 @@ function words.found(id, str)
local tag = languages.numbers[id]
if tag then
local data = wordsdata[tag]
- return data and (data[str] or data[lower(str)])
- else
- return false
+ if data then
+ if data[str] then
+ return 1
+ elseif data[lower(str)] then
+ return 2
+ end
+ end
end
end
-- The following code is an adaption of experimental code for
-- hyphenating and spell checking.
-local function mark_words(head,whenfound) -- can be optimized
- local current, start, str, language, n = head, nil, "", nil, 0
+-- there is an n=1 problem somewhere in nested boxes
+
+--~ local function mark_words(head,whenfound) -- can be optimized and shared
+--~ local current, start, str, language, n, done = head, nil, "", nil, 0, false
+--~ local function action()
+--~ if #str > 0 then
+--~ local f = whenfound(language,str)
+--~ if f then
+--~ done = true
+--~ for i=1,n do
+--~ f(start)
+--~ start = start.next
+--~ end
+--~ end
+--~ end
+--~ str, start, n = "", nil, 0
+--~ end
+--~ while current do
+--~ local id = current.id
+--~ if id == glyph_code then
+--~ local a = current.lang
+--~ if a then
+--~ if a ~= language then
+--~ if start then
+--~ action()
+--~ end
+--~ language = a
+--~ end
+--~ elseif start then
+--~ action()
+--~ language = a
+--~ end
+--~ local components = current.components
+--~ if components then
+--~ start = start or current
+--~ n = n + 1
+--~ for g in traverse_nodes(components) do
+--~ str = str .. utfchar(g.char)
+--~ end
+--~ else
+--~ local code = current.char
+--~ local data = chardata[code]
+--~ if is_letter[data.category] then
+--~ start = start or current
+--~ n = n + 1
+--~ str = str .. utfchar(code) -- slow, maybe str should be a table (and given max)
+--~ elseif start then
+--~ action()
+--~ end
+--~ end
+--~ elseif id == disc_code then
+--~ if n > 0 then
+--~ n = n + 1
+--~ end
+--~ elseif id == kern_code and current.subtype == kerning_code and start then
+--~ -- ok
+--~ elseif start then
+--~ action()
+--~ end
+--~ current = current.next
+--~ end
+--~ if start then
+--~ action()
+--~ end
+--~ return head, done
+--~ end
+
+local function mark_words(head,whenfound) -- can be optimized and shared
+ local current, language, done = head, nil, nil, 0, false
+ local str, s, nds, n = { }, 0, { }, 0 -- n could also be a table, saves calls
local function action()
- if #str > 0 then
- local f = whenfound(language,str)
- if f then
+ if s > 0 then
+ local word = concat(str,"",1,s)
+ local mark = whenfound(language,word)
+ if mark then
+ done = true
for i=1,n do
- f(start)
- start = start.next
+ mark(nds[i])
end
end
end
- str, start, n = "", nil, 0
+ n, s = 0, 0
end
while current do
local id = current.id
- if id == glyph_node then
+ if id == glyph_code then
local a = current.lang
if a then
if a ~= language then
- if start then
+ if s > 0 then
action()
end
language = a
end
- elseif start then
+ elseif s > 0 then
action()
language = a
end
local components = current.components
if components then
- start = start or current
n = n + 1
+ nds[n] = current
for g in traverse_nodes(components) do
- str = str .. utfchar(g.char)
+ s = s + 1
+ str[s] = utfchar(g.char)
end
else
local code = current.char
- if chardata[code].uccode or chardata[code].lccode then
- start = start or current
+ local data = chardata[code]
+ if is_letter[data.category] then
n = n + 1
- str = str .. utfchar(code)
- elseif start then
+ nds[n] = current
+ s = s + 1
+ str[s] = utfchar(code)
+ elseif s > 0 then
action()
end
end
- elseif id == disc_node then
+ elseif id == disc_code then -- take the replace
if n > 0 then
n = n + 1
+ nds[n] = current
end
- elseif id == kern_node and current.subtype == 0 and start then
+ elseif id == kern_code and current.subtype == kerning_code and s > 0 then
-- ok
- elseif start then
+ elseif s > 0 then
action()
end
current = current.next
end
- if start then
+ if s > 0 then
action()
end
- return head
+ return head, done
end
-words.methods = { }
-words.method = 1
+local methods = { }
+words.methods = methods
-local methods = words.methods
+local enablers = { }
+words.enablers = enablers
-methods[1] = function(head, attribute, yes, nop)
- local right, wrong = false, false
- if yes then right = function(n) set_attribute(n,attribute,yes) end end
- if nop then wrong = function(n) set_attribute(n,attribute,nop) end end
- for n in traverse_nodes(head) do
- unset_attribute(n,attribute) -- hm, not that selective (reset color)
+local wordmethod = 1
+local enabled = false
+
+function words.check(head)
+ if enabled then
+ return methods[wordmethod](head)
+ else
+ return head, false
end
- local found, done = words.found, false
- mark_words(head, function(language,str)
- if #str < words.threshold then
- return false
- elseif found(language,str) then
- done = true
- return right
- else
- done = true
- return wrong
- end
- end)
- return head, done
end
-local list, dump = { }, false -- todo: per language
+function words.enable(settings)
+ local method = settings.method
+ wordmethod = method and tonumber(method) or wordmethod or 1
+ local e = enablers[wordmethod]
+ if e then e(settings) end
+ tasks.enableaction("processors","languages.words.check")
+ enabled = true
+end
-local lower = characters.lower
+function words.disable()
+ enabled = false
+end
+
+-- colors
+
+local cache = { } -- can also be done with method 1 -- frozen colors once used
-methods[2] = function(head, attribute)
- dump = true
- mark_words(head, function(language,str)
- if #str >= words.threshold then
- str = lower(str)
- list[str] = (list[str] or 0) + 1
+setmetatable(cache, {
+ __index = function(t,k) -- k == language, numbers[k] == tag
+ local c
+ if type(k) == "string" then
+ c = colist[k]
+ elseif k < 0 then
+ c = colist["word:unset"]
+ else
+ c = colist["word:" .. (numbers[k] or "unset")] or colist["word:unknown"]
end
- end)
- return head, true
+ local v = c and function(n) set_attribute(n,a_color,c) end or false
+ t[k] = v
+ return v
+ end
+} )
+
+-- method 1
+
+local function sweep(language,str)
+ if #str < words.threshold then
+ return false
+ elseif words.found(language,str) then -- can become a local wordsfound
+ return cache["word:yes"] -- maybe variables.yes
+ else
+ return cache["word:no"]
+ end
+end
+
+methods[1] = function(head)
+ for n in traverse_nodes(head) do
+ unset_attribute(n,a_color) -- hm, not that selective (reset color)
+ end
+ return mark_words(head,sweep)
end
-words.used = list
+-- method 2
+
+local dumpname = nil
+local dumpthem = false
+local listname = "document"
+
+local category = { }
+local categories = { }
+
+setmetatable(categories, {
+ __index = function(t,k)
+ local languages = { }
+ setmetatable(languages, {
+ __index = function(t,k)
+ local r = registered[k]
+ local v = {
+ number = language,
+ parent = r and r.parent or nil,
+ patterns = r and r.patterns or nil,
+ tag = r and r.tag or nil,
+ list = { },
+ total = 0,
+ unique = 0,
+ }
+ t[k] = v
+ return v
+ end
+ } )
+ local v = {
+ languages = languages,
+ total = 0,
+ }
+ t[k] = v
+ return v
+ end
+} )
+
+local collected = {
+ total = 0,
+ version = 1.000,
+ categories = categories,
+}
+
+enablers[2] = function(settings)
+ local name = settings.list
+ listname = name and name ~= "" and name or "document"
+ category = collected.categories[listname]
+end
-function words.dump_used_words(name)
- if dump then
- logs.report("languages","saving list of used words in '%s'",name)
- io.savedata(name,table.serialize(list))
+local function sweep(language,str)
+ if #str >= words.threshold then
+ str = lowerchar(str)
+ local words = category.languages[numbers[language] or "unset"]
+ local list = words.list
+ local ls = list[str]
+ if ls then
+ list[str] = ls + 1
+ else
+ list[str] = 1
+ words.unique = words.unique + 1
+ end
+ collected.total = collected.total + 1
+ category.total = category.total + 1
+ words.total = words.total + 1
end
end
-local color = attributes.private('color')
+methods[2] = function(head)
+ dumpthem = true
+ return mark_words(head,sweep)
+end
-function words.check(head)
- if words.enabled and head.next then
- local colors = words.colors
- local alc = attributes.list[color]
- return methods[words.method](head, color, alc[colors.known], alc[colors.unknown])
- else
- return head, false
+local function dumpusedwords()
+ if dumpthem then
+ collected.threshold = words.threshold
+ dumpname = dumpname or file.addsuffix(tex.jobname,"words")
+ report_words("saving list of used words in '%s'",dumpname)
+ io.savedata(dumpname,table.serialize(collected,true))
+ -- table.tofile(dumpname,list,true)
end
end
-function words.enable(method)
- tasks.enableaction("processors","languages.words.check")
- words.method = method or words.method or 1
- words.enabled = true
+directives.register("languages.words.dump", function(v)
+ dumpname = type(v) == "string" and v ~= "" and v
+end)
+
+luatex.registerstopactions(dumpusedwords)
+
+-- method 3
+
+local function sweep(language,str)
+ return cache[language]
end
-function words.disable()
- words.enabled = false
+methods[3] = function(head)
+ for n in traverse_nodes(head) do
+ unset_attribute(n,a_color)
+ end
+ return mark_words(head,sweep)
end
-- for the moment we hook it into the attribute handler