diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lang-ini.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/lang-ini.lua | 307 |
1 files changed, 161 insertions, 146 deletions
diff --git a/Master/texmf-dist/tex/context/base/lang-ini.lua b/Master/texmf-dist/tex/context/base/lang-ini.lua index 83e9b899f87..e188ad36cc9 100644 --- a/Master/texmf-dist/tex/context/base/lang-ini.lua +++ b/Master/texmf-dist/tex/context/base/lang-ini.lua @@ -6,6 +6,13 @@ if not modules then modules = { } end modules ['lang-ini'] = { license = "see context related readme files" } +-- needs a cleanup (share locals) + +local utf = unicode.utf8 + +local find, lower, format, utfchar = string.find, string.lower, string.format, utf.char +local concat = table.concat + if lang.use_new then lang.use_new(true) end languages = languages or {} @@ -13,6 +20,8 @@ languages.version = 1.009 languages.hyphenation = languages.hyphenation or { } languages.hyphenation.data = languages.hyphenation.data or { } +local langdata = languages.hyphenation.data + -- 002D : hyphen-minus (ascii) -- 2010 : hyphen -- 2011 : nonbreakable hyphen @@ -23,157 +32,151 @@ languages.hyphenation.data = languages.hyphenation.data or { } --~ lang:clear_hyphenation() -do - -- we can consider hiding data (faster access too) - - --~ local function filter(filename,what) - --~ local data = io.loaddata(input.find_file(texmf.instance,filename)) - --~ local data = data:match(string.format("\\%s%%s*(%%b{})",what or "patterns")) - --~ return data:match("{%s*(.-)%s*}") or "" - --~ end - - -- loading the 26 languages that we normally load in mkiv, the string based variant - -- takes .84 seconds (probably due to the sub's) while the lpeg variant takes .78 - -- seconds - - local leftbrace = lpeg.P("{") - local rightbrace = lpeg.P("}") - local spaces = lpeg.S(" \r\n\t\f") - local spacing = spaces^0 - local validchar = 1-(spaces+rightbrace+leftbrace) - local validword = validchar^1 - local content = spacing * leftbrace * spacing * lpeg.C((spacing * validword)^0) * spacing * rightbrace * lpeg.P(true) - - local command = lpeg.P("\\patterns") - local parser = (1-command)^0 * command * content - - local function filterpatterns(filename) - if filename:find("%.rpl") then - return io.loaddata(input.find_file(texmf.instance,filename)) or "" - else - return parser:match(io.loaddata(input.find_file(texmf.instance,filename)) or "") - end - end +-- we can consider hiding data (faster access too) - local command = lpeg.P("\\hyphenation") - local parser = (1-command)^0 * command * content +--~ local function filter(filename,what) +--~ local data = io.loaddata(resolvers.find_file(filename)) +--~ local data = data:match(string.format("\\%s%%s*(%%b{})",what or "patterns")) +--~ return data:match("{%s*(.-)%s*}") or "" +--~ end - local function filterexceptions(filename) - if filename:find("%.rhl") then - return io.loaddata(input.find_file(texmf.instance,filename)) or "" - else - return parser:match(io.loaddata(input.find_file(texmf.instance,filename)) or {}) -- "" ? - end - end +-- loading the 26 languages that we normally load in mkiv, the string based variant +-- takes .84 seconds (probably due to the sub's) while the lpeg variant takes .78 +-- seconds - local function record(tag) - local data = languages.hyphenation.data[tag] - if not data then - data = lang.new() - languages.hyphenation.data[tag] = data - end - return data - end +local leftbrace = lpeg.P("{") +local rightbrace = lpeg.P("}") +local spaces = lpeg.S(" \r\n\t\f") +local spacing = spaces^0 +local validchar = 1-(spaces+rightbrace+leftbrace) +local validword = validchar^1 +local content = spacing * leftbrace * spacing * lpeg.C((spacing * validword)^0) * spacing * rightbrace * lpeg.P(true) - languages.hyphenation.record = record +local command = lpeg.P("\\patterns") +local parser = (1-command)^0 * command * content - function languages.hyphenation.define(tag) - local data = record(tag) - return data:id() +local function filterpatterns(filename) + if find(filename,"%.rpl") then + return io.loaddata(resolvers.find_file(filename)) or "" + else + return parser:match(io.loaddata(resolvers.find_file(filename)) or "") end +end - function languages.hyphenation.number(tag) - local d = languages.hyphenation.data[tag] - return (d and d:id()) or 0 - end +local command = lpeg.P("\\hyphenation") +local parser = (1-command)^0 * command * content - function languages.hyphenation.load(tag, filename, filter, target) - input.starttiming(languages) - local data = record(tag) - filename = (filename and filename ~= "" and input.find_file(texmf.instance,filename)) or "" - local ok = filename ~= "" - if ok then - lang[target](data,filterpatterns(filename)) - else - lang[target](data,"") - end - languages.hyphenation.data[tag] = data - input.stoptiming(languages) - return ok +local function filterexceptions(filename) + if find(filename,"%.rhl") then + return io.loaddata(resolvers.find_file(filename)) or "" + else + return parser:match(io.loaddata(resolvers.find_file(filename)) or {}) -- "" ? end +end - function languages.hyphenation.loadpatterns(tag, patterns) - return languages.hyphenation.load(tag, patterns, filterpatterns, "patterns") +local function record(tag) + local data = langdata[tag] + if not data then + data = lang.new() + langdata[tag] = data or 0 end + return data +end - function languages.hyphenation.loadexceptions(tag, exceptions) - return languages.hyphenation.load(tag, patterns, filterexceptions, "hyphenation") - end +languages.hyphenation.record = record - function languages.hyphenation.exceptions(tag, ...) - local data = record(tag) - data:hyphenation(...) - end +function languages.hyphenation.define(tag) + local data = record(tag) + return data:id() +end - function languages.hyphenation.hyphenate(tag, str) - return lang.hyphenate(record(tag), str) - end +function languages.hyphenation.number(tag) + local d = langdata[tag] + return (d and d:id()) or 0 +end - function languages.hyphenation.lefthyphenmin(tag, value) - local data = record(tag) - if value then data:lefthyphenmin(value) end - return data:lefthyphenmin() - end - function languages.hyphenation.righthyphenmin(tag, value) - local data = record(tag) - if value then data:righthyphenmin(value) end - return data:righthyphenmin() +local function loadthem(tag, filename, filter, target) + statistics.starttiming(languages) + local data = record(tag) + filename = (filename and filename ~= "" and resolvers.find_file(filename)) or "" + local ok = filename ~= "" + if ok then + lang[target](data,filterpatterns(filename)) + else + lang[target](data,"") end + langdata[tag] = data + statistics.stoptiming(languages) + return ok +end - function languages.hyphenation.n() - return table.count(languages.hyphenation.data) - end +function languages.hyphenation.loadpatterns(tag, patterns) + return loadthem(tag, patterns, filterpatterns, "patterns") +end +function languages.hyphenation.loadexceptions(tag, exceptions) + return loadthem(tag, patterns, filterexceptions, "hyphenation") end -do +function languages.hyphenation.exceptions(tag, ...) + local data = record(tag) + data:hyphenation(...) +end - -- we can speed this one up with locals if needed +function languages.hyphenation.hyphenate(tag, str) + return lang.hyphenate(record(tag), str) +end - local function tolang(what) - if type(what) == "number" then - return languages.hyphenation.data[languages.numbers[what]] - elseif type(what) == "string" then - return languages.hyphenation.data[what] - else - return what - end - end +function languages.hyphenation.lefthyphenmin(tag, value) + local data = record(tag) + if value then data:lefthyphenmin(value) end + return data:lefthyphenmin() +end +function languages.hyphenation.righthyphenmin(tag, value) + local data = record(tag) + if value then data:righthyphenmin(value) end + return data:righthyphenmin() +end - function languages.prehyphenchar(what) - return lang.prehyphenchar(tolang(what)) - end - function languages.posthyphenchar(what) - return lang.posthyphenchar(tolang(what)) - end +function languages.hyphenation.n() + return table.count(langdata) +end - languages.tolang = tolang +-- we can speed this one up with locals if needed +local function tolang(what) + local kind = type(what) + if kind == "number" then + return langdata[languages.numbers[what]] + elseif kind == "string" then + return langdata[what] + else + return what + end end +function languages.prehyphenchar(what) + return lang.prehyphenchar(tolang(what)) +end +function languages.posthyphenchar(what) + return lang.posthyphenchar(tolang(what)) +end + +languages.tolang = tolang + languages.registered = languages.registered or { } languages.associated = languages.associated or { } languages.numbers = languages.numbers or { } -input.storage.register(false,"languages/registered",languages.registered,"languages.registered") -input.storage.register(false,"languages/associated",languages.associated,"languages.associated") +storage.register("languages/registered",languages.registered,"languages.registered") +storage.register("languages/associated",languages.associated,"languages.associated") function languages.register(tag,parent,patterns,exceptions) parent = parent or tag languages.registered[tag] = { parent = parent, - patterns = patterns or string.format("lang-%s.pat",parent), - exceptions = exceptions or string.format("lang-%s.hyp",parent), + patterns = patterns or format("lang-%s.pat",parent), + exceptions = exceptions or format("lang-%s.hyp",parent), loaded = false, number = 0, } @@ -197,7 +200,7 @@ end function languages.loadable(tag) local l = languages.registered[tag] - if l and l.patterns and input.find_file(texmf.instance,patterns) then + if l and l.patterns and resolvers.find_file(patterns) then return true else return false @@ -243,11 +246,11 @@ end function languages.hyphenation.loadwords(tag, filename) local id = languages.hyphenation.number(tag) if id > 0 then - local l = lang.new(id) - input.starttiming(languages) + local l = lang.new(id) or 0 + statistics.starttiming(languages) local data = io.loaddata(filename) or "" l:hyphenation(data) - input.stoptiming(languages) + statistics.stoptiming(languages) end end @@ -264,13 +267,12 @@ function languages.logger.report() if l.loaded then local p = (l.patterns and "pat") or '-' local e = (l.exceptions and "exc") or '-' - result[#result+1] = string.format("%s:%s:%s:%s:%s", tag, l.parent, p, e, l.number) + result[#result+1] = format("%s:%s:%s:%s:%s", tag, l.parent, p, e, l.number) end end - return (#result > 0 and table.concat(result," ")) or "none" + return (#result > 0 and concat(result," ")) or "none" end - languages.words = languages.words or {} languages.words.data = languages.words.data or {} languages.words.enable = false @@ -291,15 +293,15 @@ do local word = lpeg.Cs((markup/"" + disc/"" + (1-spacing))^1) function languages.words.load(tag, filename) - local filename = input.find_file(texmf.instance,filename,'other text file') or "" + local filename = resolvers.find_file(filename,'other text file') or "" if filename ~= "" then - input.starttiming(languages) + statistics.starttiming(languages) local data = io.loaddata(filename) or "" local words = languages.words.data[tag] or {} parser = (spacing + word/function(s) words[s] = true end)^0 parser:match(data) languages.words.data[tag] = words - input.stoptiming(languages) + statistics.stoptiming(languages) end end @@ -309,7 +311,7 @@ function languages.words.found(id, str) local tag = languages.numbers[id] if tag then local data = languages.words.data[tag] - return data and (data[str] or data[str:lower()]) + return data and (data[str] or data[lower(str)]) else return false end @@ -322,12 +324,10 @@ do local glyph, disc, kern = node.id('glyph'), node.id('disc'), node.id('kern') - local bynode = node.traverse - local bychar = string.utfcharacters + local bynode = node.traverse + local chardata = characters.data local function mark_words(head,found) -- can be optimized - local cd = characters.data - local uc = utf.char local current, start, str, language, n = head, nil, "", nil, 0 local function action() if #str > 0 then @@ -356,25 +356,25 @@ do action() language = a end - if current.subtype > 0 then + local components = current.components + if components then start = start or current n = n + 1 - for g in bynode(current.components) do - str = str .. uc(g.char) + for g in bynode(components) do + str = str .. utfchar(g.char) end else local code = current.char - if cd[code].uccode or cd[code].lccode then + if chardata[code].uccode or chardata[code].lccode then start = start or current n = n + 1 - str = str .. uc(code) - else - if start then - action() - end + str = str .. utfchar(code) + elseif start then + action() end end elseif id == disc then + if n > 0 then n = n + 1 end -- ok elseif id == kern and current.subtype == 0 and start then -- ok @@ -392,18 +392,20 @@ do languages.words.methods = { } languages.words.method = 1 + local lw = languages.words + languages.words.methods[1] = function(head, attribute, yes, nop) local set = node.set_attribute local unset = node.unset_attribute - local wrong, right = false, false - if nop then wrong = function(n) set(n,attribute,nop) end end + local right, wrong = false, false if yes then right = function(n) set(n,attribute,yes) end end + if nop then wrong = function(n) set(n,attribute,nop) end end for n in node.traverse(head) do unset(n,attribute) -- hm end local found, done = languages.words.found, false mark_words(head, function(language,str) - if #str < languages.words.threshold then + if #str < lw.threshold then return false elseif found(language,str) then done = true @@ -416,11 +418,10 @@ do return head, done end - local lw = languages.words + local color = attributes.private('color') function languages.words.check(head) - if head.next and lw.enable then - local color = attributes.numbers['color'] + if lw.enable and head.next then local colors = lw.colors local alc = attributes.list[color] return lw.methods[lw.method](head, color, alc[colors.known], alc[colors.unknown]) @@ -439,7 +440,8 @@ end --~ return languages.check(head) --~ end ---~ chars.plugins.language = { +--~ chars.plugins[chars.plugins+1] = { +--~ name = "language", --~ namespace = languagehacks, --~ processor = languagehacks.process --~ } @@ -451,3 +453,16 @@ languages.associate('uk','latn','eng') languages.associate('nl','latn','nld') languages.associate('de','latn','deu') languages.associate('fr','latn','fra') + +statistics.register("loaded patterns", function() + local result = languages.logger.report() + if result ~= "none" then + return result + end +end) + +statistics.register("language load time", function() + if statistics.elapsedindeed(languages) then + return format("%s seconds, n=%s", statistics.elapsedtime(languages), languages.hyphenation.n()) + end +end) |