diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lang-wrd.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/lang-wrd.lua | 381 |
1 files changed, 285 insertions, 96 deletions
diff --git a/Master/texmf-dist/tex/context/base/lang-wrd.lua b/Master/texmf-dist/tex/context/base/lang-wrd.lua index 095e44443d9..558d5744faf 100644 --- a/Master/texmf-dist/tex/context/base/lang-wrd.lua +++ b/Master/texmf-dist/tex/context/base/lang-wrd.lua @@ -8,44 +8,59 @@ if not modules then modules = { } end modules ['lang-ini'] = { local utf = unicode.utf8 local lower, utfchar = string.lower, utf.char +local concat = table.concat local lpegmatch = lpeg.match +local P, S, Cs = lpeg.P, lpeg.S, lpeg.Cs -languages.words = languages.words or { } +local report_words = logs.reporter("languages","words") -local words = languages.words +local nodes, node, languages = nodes, node, languages -words.data = words.data or { } -words.enables = false -words.threshold = 4 +languages.words = languages.words or { } +local words = languages.words + +words.data = words.data or { } +words.enables = false +words.threshold = 4 + +local numbers = languages.numbers +local registered = languages.registered local set_attribute = node.set_attribute local unset_attribute = node.unset_attribute local traverse_nodes = node.traverse -local node_id = node.id local wordsdata = words.data local chardata = characters.data +local tasks = nodes.tasks -local glyph_node = node_id('glyph') -local disc_node = node_id('disc') -local kern_node = node_id('kern') +local nodecodes = nodes.nodecodes +local kerncodes = nodes.kerncodes -words.colors = { - ["known"] = "green", - ["unknown"] = "red", -} +local glyph_code = nodecodes.glyph +local disc_code = nodecodes.disc +local kern_code = nodecodes.kern + +local kerning_code = kerncodes.kerning +local lowerchar = characters.lower -local spacing = lpeg.S(" \n\r\t") -local markup = lpeg.S("-=") -local lbrace = lpeg.P("{") -local rbrace = lpeg.P("}") +local a_color = attributes.private('color') +local colist = attributes.list[a_color] + +local is_letter = characters.is_letter -- maybe is_character as variant + +local spacing = S(" \n\r\t") +local markup = S("-=") +local lbrace = P("{") +local rbrace = P("}") local disc = (lbrace * (1-rbrace)^0 * rbrace)^1 -- or just 3 times, time this -local word = lpeg.Cs((markup/"" + disc/"" + (1-spacing))^1) +local word = Cs((markup/"" + disc/"" + (1-spacing))^1) local loaded = { } -- we share lists function words.load(tag,filename) - local fullname = resolvers.find_file(filename,'other text file') or "" + local fullname = resolvers.findfile(filename,'other text file') or "" if fullname ~= "" then + report_words("loading word file '%s'",fullname) statistics.starttiming(languages) local list = loaded[fullname] if not list then @@ -57,7 +72,7 @@ function words.load(tag,filename) wordsdata[tag] = list statistics.stoptiming(languages) else - logs.report("languages","missing words file '%s'",filename) + report_words("missing word file '%s'",filename) end end @@ -65,149 +80,323 @@ function words.found(id, str) local tag = languages.numbers[id] if tag then local data = wordsdata[tag] - return data and (data[str] or data[lower(str)]) - else - return false + if data then + if data[str] then + return 1 + elseif data[lower(str)] then + return 2 + end + end end end -- The following code is an adaption of experimental code for -- hyphenating and spell checking. -local function mark_words(head,whenfound) -- can be optimized - local current, start, str, language, n = head, nil, "", nil, 0 +-- there is an n=1 problem somewhere in nested boxes + +--~ local function mark_words(head,whenfound) -- can be optimized and shared +--~ local current, start, str, language, n, done = head, nil, "", nil, 0, false +--~ local function action() +--~ if #str > 0 then +--~ local f = whenfound(language,str) +--~ if f then +--~ done = true +--~ for i=1,n do +--~ f(start) +--~ start = start.next +--~ end +--~ end +--~ end +--~ str, start, n = "", nil, 0 +--~ end +--~ while current do +--~ local id = current.id +--~ if id == glyph_code then +--~ local a = current.lang +--~ if a then +--~ if a ~= language then +--~ if start then +--~ action() +--~ end +--~ language = a +--~ end +--~ elseif start then +--~ action() +--~ language = a +--~ end +--~ local components = current.components +--~ if components then +--~ start = start or current +--~ n = n + 1 +--~ for g in traverse_nodes(components) do +--~ str = str .. utfchar(g.char) +--~ end +--~ else +--~ local code = current.char +--~ local data = chardata[code] +--~ if is_letter[data.category] then +--~ start = start or current +--~ n = n + 1 +--~ str = str .. utfchar(code) -- slow, maybe str should be a table (and given max) +--~ elseif start then +--~ action() +--~ end +--~ end +--~ elseif id == disc_code then +--~ if n > 0 then +--~ n = n + 1 +--~ end +--~ elseif id == kern_code and current.subtype == kerning_code and start then +--~ -- ok +--~ elseif start then +--~ action() +--~ end +--~ current = current.next +--~ end +--~ if start then +--~ action() +--~ end +--~ return head, done +--~ end + +local function mark_words(head,whenfound) -- can be optimized and shared + local current, language, done = head, nil, nil, 0, false + local str, s, nds, n = { }, 0, { }, 0 -- n could also be a table, saves calls local function action() - if #str > 0 then - local f = whenfound(language,str) - if f then + if s > 0 then + local word = concat(str,"",1,s) + local mark = whenfound(language,word) + if mark then + done = true for i=1,n do - f(start) - start = start.next + mark(nds[i]) end end end - str, start, n = "", nil, 0 + n, s = 0, 0 end while current do local id = current.id - if id == glyph_node then + if id == glyph_code then local a = current.lang if a then if a ~= language then - if start then + if s > 0 then action() end language = a end - elseif start then + elseif s > 0 then action() language = a end local components = current.components if components then - start = start or current n = n + 1 + nds[n] = current for g in traverse_nodes(components) do - str = str .. utfchar(g.char) + s = s + 1 + str[s] = utfchar(g.char) end else local code = current.char - if chardata[code].uccode or chardata[code].lccode then - start = start or current + local data = chardata[code] + if is_letter[data.category] then n = n + 1 - str = str .. utfchar(code) - elseif start then + nds[n] = current + s = s + 1 + str[s] = utfchar(code) + elseif s > 0 then action() end end - elseif id == disc_node then + elseif id == disc_code then -- take the replace if n > 0 then n = n + 1 + nds[n] = current end - elseif id == kern_node and current.subtype == 0 and start then + elseif id == kern_code and current.subtype == kerning_code and s > 0 then -- ok - elseif start then + elseif s > 0 then action() end current = current.next end - if start then + if s > 0 then action() end - return head + return head, done end -words.methods = { } -words.method = 1 +local methods = { } +words.methods = methods -local methods = words.methods +local enablers = { } +words.enablers = enablers -methods[1] = function(head, attribute, yes, nop) - local right, wrong = false, false - if yes then right = function(n) set_attribute(n,attribute,yes) end end - if nop then wrong = function(n) set_attribute(n,attribute,nop) end end - for n in traverse_nodes(head) do - unset_attribute(n,attribute) -- hm, not that selective (reset color) +local wordmethod = 1 +local enabled = false + +function words.check(head) + if enabled then + return methods[wordmethod](head) + else + return head, false end - local found, done = words.found, false - mark_words(head, function(language,str) - if #str < words.threshold then - return false - elseif found(language,str) then - done = true - return right - else - done = true - return wrong - end - end) - return head, done end -local list, dump = { }, false -- todo: per language +function words.enable(settings) + local method = settings.method + wordmethod = method and tonumber(method) or wordmethod or 1 + local e = enablers[wordmethod] + if e then e(settings) end + tasks.enableaction("processors","languages.words.check") + enabled = true +end -local lower = characters.lower +function words.disable() + enabled = false +end + +-- colors + +local cache = { } -- can also be done with method 1 -- frozen colors once used -methods[2] = function(head, attribute) - dump = true - mark_words(head, function(language,str) - if #str >= words.threshold then - str = lower(str) - list[str] = (list[str] or 0) + 1 +setmetatable(cache, { + __index = function(t,k) -- k == language, numbers[k] == tag + local c + if type(k) == "string" then + c = colist[k] + elseif k < 0 then + c = colist["word:unset"] + else + c = colist["word:" .. (numbers[k] or "unset")] or colist["word:unknown"] end - end) - return head, true + local v = c and function(n) set_attribute(n,a_color,c) end or false + t[k] = v + return v + end +} ) + +-- method 1 + +local function sweep(language,str) + if #str < words.threshold then + return false + elseif words.found(language,str) then -- can become a local wordsfound + return cache["word:yes"] -- maybe variables.yes + else + return cache["word:no"] + end +end + +methods[1] = function(head) + for n in traverse_nodes(head) do + unset_attribute(n,a_color) -- hm, not that selective (reset color) + end + return mark_words(head,sweep) end -words.used = list +-- method 2 + +local dumpname = nil +local dumpthem = false +local listname = "document" + +local category = { } +local categories = { } + +setmetatable(categories, { + __index = function(t,k) + local languages = { } + setmetatable(languages, { + __index = function(t,k) + local r = registered[k] + local v = { + number = language, + parent = r and r.parent or nil, + patterns = r and r.patterns or nil, + tag = r and r.tag or nil, + list = { }, + total = 0, + unique = 0, + } + t[k] = v + return v + end + } ) + local v = { + languages = languages, + total = 0, + } + t[k] = v + return v + end +} ) + +local collected = { + total = 0, + version = 1.000, + categories = categories, +} + +enablers[2] = function(settings) + local name = settings.list + listname = name and name ~= "" and name or "document" + category = collected.categories[listname] +end -function words.dump_used_words(name) - if dump then - logs.report("languages","saving list of used words in '%s'",name) - io.savedata(name,table.serialize(list)) +local function sweep(language,str) + if #str >= words.threshold then + str = lowerchar(str) + local words = category.languages[numbers[language] or "unset"] + local list = words.list + local ls = list[str] + if ls then + list[str] = ls + 1 + else + list[str] = 1 + words.unique = words.unique + 1 + end + collected.total = collected.total + 1 + category.total = category.total + 1 + words.total = words.total + 1 end end -local color = attributes.private('color') +methods[2] = function(head) + dumpthem = true + return mark_words(head,sweep) +end -function words.check(head) - if words.enabled and head.next then - local colors = words.colors - local alc = attributes.list[color] - return methods[words.method](head, color, alc[colors.known], alc[colors.unknown]) - else - return head, false +local function dumpusedwords() + if dumpthem then + collected.threshold = words.threshold + dumpname = dumpname or file.addsuffix(tex.jobname,"words") + report_words("saving list of used words in '%s'",dumpname) + io.savedata(dumpname,table.serialize(collected,true)) + -- table.tofile(dumpname,list,true) end end -function words.enable(method) - tasks.enableaction("processors","languages.words.check") - words.method = method or words.method or 1 - words.enabled = true +directives.register("languages.words.dump", function(v) + dumpname = type(v) == "string" and v ~= "" and v +end) + +luatex.registerstopactions(dumpusedwords) + +-- method 3 + +local function sweep(language,str) + return cache[language] end -function words.disable() - words.enabled = false +methods[3] = function(head) + for n in traverse_nodes(head) do + unset_attribute(n,a_color) + end + return mark_words(head,sweep) end -- for the moment we hook it into the attribute handler |