diff options
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lang-ini.lua')
-rw-r--r-- | Master/texmf-dist/tex/context/base/lang-ini.lua | 281 |
1 files changed, 67 insertions, 214 deletions
diff --git a/Master/texmf-dist/tex/context/base/lang-ini.lua b/Master/texmf-dist/tex/context/base/lang-ini.lua index e188ad36cc9..239e5390c55 100644 --- a/Master/texmf-dist/tex/context/base/lang-ini.lua +++ b/Master/texmf-dist/tex/context/base/lang-ini.lua @@ -1,6 +1,6 @@ if not modules then modules = { } end modules ['lang-ini'] = { version = 1.001, - comment = "companion to lang-ini.tex", + comment = "companion to lang-ini.mkiv", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" @@ -9,11 +9,12 @@ if not modules then modules = { } end modules ['lang-ini'] = { -- needs a cleanup (share locals) local utf = unicode.utf8 - -local find, lower, format, utfchar = string.find, string.lower, string.format, utf.char +local utfbyte = utf.byte +local format = string.format local concat = table.concat +local lpegmatch = lpeg.match -if lang.use_new then lang.use_new(true) end +local trace_patterns = false trackers.register("languages.patterns", function(v) trace_patterns = v end) languages = languages or {} languages.version = 1.009 @@ -31,18 +32,13 @@ local langdata = languages.hyphenation.data --~ string =lang:hyphenation() --~ lang:clear_hyphenation() - -- we can consider hiding data (faster access too) ---~ local function filter(filename,what) ---~ local data = io.loaddata(resolvers.find_file(filename)) ---~ local data = data:match(string.format("\\%s%%s*(%%b{})",what or "patterns")) ---~ return data:match("{%s*(.-)%s*}") or "" ---~ end - -- loading the 26 languages that we normally load in mkiv, the string based variant -- takes .84 seconds (probably due to the sub's) while the lpeg variant takes .78 -- seconds +-- +-- the following lpeg can probably be improved (it was one of the first I made) local leftbrace = lpeg.P("{") local rightbrace = lpeg.P("}") @@ -56,10 +52,10 @@ local command = lpeg.P("\\patterns") local parser = (1-command)^0 * command * content local function filterpatterns(filename) - if find(filename,"%.rpl") then + if file.extname(filename) == "rpl" then return io.loaddata(resolvers.find_file(filename)) or "" else - return parser:match(io.loaddata(resolvers.find_file(filename)) or "") + return lpegmatch(parser,io.loaddata(resolvers.find_file(filename)) or "") end end @@ -67,10 +63,10 @@ local command = lpeg.P("\\hyphenation") local parser = (1-command)^0 * command * content local function filterexceptions(filename) - if find(filename,"%.rhl") then + if file.extname(filename) == "rhl" then return io.loaddata(resolvers.find_file(filename)) or "" else - return parser:match(io.loaddata(resolvers.find_file(filename)) or {}) -- "" ? + return lpegmatch(parser,io.loaddata(resolvers.find_file(filename)) or {}) -- "" ? end end @@ -95,14 +91,22 @@ function languages.hyphenation.number(tag) return (d and d:id()) or 0 end +lang.exceptions = lang.hyphenation + local function loadthem(tag, filename, filter, target) statistics.starttiming(languages) local data = record(tag) - filename = (filename and filename ~= "" and resolvers.find_file(filename)) or "" - local ok = filename ~= "" + local fullname = (filename and filename ~= "" and resolvers.find_file(filename)) or "" + local ok = fullname ~= "" if ok then - lang[target](data,filterpatterns(filename)) + if trace_patterns then + logs.report("languages","filtering %s for language '%s' from '%s'",target,tag,fullname) + end + lang[target](data,filterpatterns(fullname)) else + if trace_patterns then + logs.report("languages","no %s for language '%s' in '%s'",target,tag,filename or "?") + end lang[target](data,"") end langdata[tag] = data @@ -115,7 +119,7 @@ function languages.hyphenation.loadpatterns(tag, patterns) end function languages.hyphenation.loadexceptions(tag, exceptions) - return loadthem(tag, patterns, filterexceptions, "hyphenation") + return loadthem(tag, patterns, filterexceptions, "exceptions") end function languages.hyphenation.exceptions(tag, ...) @@ -142,12 +146,24 @@ function languages.hyphenation.n() return table.count(langdata) end +languages.registered = languages.registered or { } +languages.associated = languages.associated or { } +languages.numbers = languages.numbers or { } + +storage.register("languages/registered",languages.registered,"languages.registered") +storage.register("languages/associated",languages.associated,"languages.associated") + +local numbers = languages.numbers +local registered = languages.registered +local associated = languages.associated + -- we can speed this one up with locals if needed local function tolang(what) local kind = type(what) if kind == "number" then - return langdata[languages.numbers[what]] + local w = what >= 0 and what <= 0x7FFF and numbers[what] + return (w and langdata[w]) or 0 elseif kind == "string" then return langdata[what] else @@ -155,6 +171,20 @@ local function tolang(what) end end +function languages.setup(what,settings) + what = languages.tolang(what or tex.language) + local lefthyphen = settings.lefthyphen + local righthyphen = settings.righthyphen + lefthyphen = lefthyphen ~= "" and lefthyphen or nil + righthyphen = righthyphen ~= "" and righthyphen or nil + lefthyphen = lefthyphen and utfbyte(lefthyphen) or 0 + righthyphen = righthyphen and utfbyte(righthyphen) or 0 + lang.posthyphenchar(what,lefthyphen) + lang.prehyphenchar (what,righthyphen) + lang.postexhyphenchar(what,lefthyphen) + lang.preexhyphenchar (what,righthyphen) +end + function languages.prehyphenchar(what) return lang.prehyphenchar(tolang(what)) end @@ -164,16 +194,9 @@ end languages.tolang = tolang -languages.registered = languages.registered or { } -languages.associated = languages.associated or { } -languages.numbers = languages.numbers or { } - -storage.register("languages/registered",languages.registered,"languages.registered") -storage.register("languages/associated",languages.associated,"languages.associated") - function languages.register(tag,parent,patterns,exceptions) parent = parent or tag - languages.registered[tag] = { + registered[tag] = { parent = parent, patterns = patterns or format("lang-%s.pat",parent), exceptions = exceptions or format("lang-%s.hyp",parent), @@ -183,14 +206,14 @@ function languages.register(tag,parent,patterns,exceptions) end function languages.associate(tag,script,language) - languages.associated[tag] = { script, language } + associated[tag] = { script, language } end function languages.association(tag) if type(tag) == "number" then - tag = languages.numbers[tag] + tag = numbers[tag] end - local lat = tag and languages.associated[tag] + local lat = tag and associated[tag] if lat then return lat[1], lat[2] else @@ -199,7 +222,7 @@ function languages.association(tag) end function languages.loadable(tag) - local l = languages.registered[tag] + local l = registered[tag] if l and l.patterns and resolvers.find_file(patterns) then return true else @@ -213,8 +236,8 @@ function languages.enable(tags) -- beware: we cannot set tex.language, but need tex.normallanguage for i=1,#tags do local tag = tags[i] - local l = languages.registered[tag] - if l then + local l = registered[tag] + if l and l ~= "" then if not l.loaded then local tag = l.parent local number = languages.hyphenation.number(tag) @@ -225,9 +248,12 @@ function languages.enable(tags) l.number = languages.hyphenation.define(tag) languages.hyphenation.loadpatterns(tag,l.patterns) languages.hyphenation.loadexceptions(tag,l.exceptions) - languages.numbers[l.number] = tag + numbers[l.number] = tag end l.loaded = true + if trace_patterns then + logs.report("languages","assigning number %s",l.number) + end end if l.number > 0 then return l.number @@ -261,9 +287,11 @@ languages.hyphenation.loadexceptions("zerolanguage") -- else bug languages.logger = languages.logger or { } function languages.logger.report() - local result = {} - for _, tag in ipairs(table.sortedkeys(languages.registered)) do - local l = languages.registered[tag] + local result = { } + local sorted = table.sortedkeys(registered) + for i=1,#sorted do + local tag = sorted[i] + local l = registered[tag] if l.loaded then local p = (l.patterns and "pat") or '-' local e = (l.exceptions and "exc") or '-' @@ -273,179 +301,6 @@ function languages.logger.report() return (#result > 0 and concat(result," ")) or "none" end -languages.words = languages.words or {} -languages.words.data = languages.words.data or {} -languages.words.enable = false -languages.words.threshold = 4 - -languages.words.colors = { - ["known"] = "green", - ["unknown"] = "red", -} - -do - - local spacing = lpeg.S(" \n\r\t") - local markup = lpeg.S("-=") - local lbrace = lpeg.P("{") - local rbrace = lpeg.P("}") - local disc = (lbrace * (1-rbrace)^0 * rbrace)^1 -- or just 3 times, time this - local word = lpeg.Cs((markup/"" + disc/"" + (1-spacing))^1) - - function languages.words.load(tag, filename) - local filename = resolvers.find_file(filename,'other text file') or "" - if filename ~= "" then - statistics.starttiming(languages) - local data = io.loaddata(filename) or "" - local words = languages.words.data[tag] or {} - parser = (spacing + word/function(s) words[s] = true end)^0 - parser:match(data) - languages.words.data[tag] = words - statistics.stoptiming(languages) - end - end - -end - -function languages.words.found(id, str) - local tag = languages.numbers[id] - if tag then - local data = languages.words.data[tag] - return data and (data[str] or data[lower(str)]) - else - return false - end -end - --- The following code is an adaption of experimental code for --- hyphenating and spell checking. - -do - - local glyph, disc, kern = node.id('glyph'), node.id('disc'), node.id('kern') - - local bynode = node.traverse - local chardata = characters.data - - local function mark_words(head,found) -- can be optimized - local current, start, str, language, n = head, nil, "", nil, 0 - local function action() - if #str > 0 then - local f = found(language,str) - if f then - for i=1,n do - f(start) - start = start.next - end - end - end - str, start, n = "", nil, 0 - end - while current do - local id = current.id - if id == glyph then - local a = current.lang - if a then - if a ~= language then - if start then - action() - end - language = a - end - elseif start then - action() - language = a - end - local components = current.components - if components then - start = start or current - n = n + 1 - for g in bynode(components) do - str = str .. utfchar(g.char) - end - else - local code = current.char - if chardata[code].uccode or chardata[code].lccode then - start = start or current - n = n + 1 - str = str .. utfchar(code) - elseif start then - action() - end - end - elseif id == disc then - if n > 0 then n = n + 1 end - -- ok - elseif id == kern and current.subtype == 0 and start then - -- ok - elseif start then - action() - end - current = current.next - end - if start then - action() - end - return head - end - - languages.words.methods = { } - languages.words.method = 1 - - local lw = languages.words - - languages.words.methods[1] = function(head, attribute, yes, nop) - local set = node.set_attribute - local unset = node.unset_attribute - local right, wrong = false, false - if yes then right = function(n) set(n,attribute,yes) end end - if nop then wrong = function(n) set(n,attribute,nop) end end - for n in node.traverse(head) do - unset(n,attribute) -- hm - end - local found, done = languages.words.found, false - mark_words(head, function(language,str) - if #str < lw.threshold then - return false - elseif found(language,str) then - done = true - return right - else - done = true - return wrong - end - end) - return head, done - end - - local color = attributes.private('color') - - function languages.words.check(head) - if lw.enable and head.next then - local colors = lw.colors - local alc = attributes.list[color] - return lw.methods[lw.method](head, color, alc[colors.known], alc[colors.unknown]) - else - return head, false - end - end - -end - --- for the moment we hook it into the attribute handler - ---~ languagehacks = { } - ---~ function languagehacks.process(namespace,attribute,head) ---~ return languages.check(head) ---~ end - ---~ chars.plugins[chars.plugins+1] = { ---~ name = "language", ---~ namespace = languagehacks, ---~ processor = languagehacks.process ---~ } - -- must happen at the tex end languages.associate('en','latn','eng') @@ -462,7 +317,5 @@ statistics.register("loaded patterns", function() end) statistics.register("language load time", function() - if statistics.elapsedindeed(languages) then - return format("%s seconds, n=%s", statistics.elapsedtime(languages), languages.hyphenation.n()) - end + return statistics.elapsedseconds(languages, format(", n=%s",languages.hyphenation.n())) end) |