summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/lang-ini.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lang-ini.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/lang-ini.lua307
1 files changed, 161 insertions, 146 deletions
diff --git a/Master/texmf-dist/tex/context/base/lang-ini.lua b/Master/texmf-dist/tex/context/base/lang-ini.lua
index 83e9b899f87..e188ad36cc9 100644
--- a/Master/texmf-dist/tex/context/base/lang-ini.lua
+++ b/Master/texmf-dist/tex/context/base/lang-ini.lua
@@ -6,6 +6,13 @@ if not modules then modules = { } end modules ['lang-ini'] = {
license = "see context related readme files"
}
+-- needs a cleanup (share locals)
+
+local utf = unicode.utf8
+
+local find, lower, format, utfchar = string.find, string.lower, string.format, utf.char
+local concat = table.concat
+
if lang.use_new then lang.use_new(true) end
languages = languages or {}
@@ -13,6 +20,8 @@ languages.version = 1.009
languages.hyphenation = languages.hyphenation or { }
languages.hyphenation.data = languages.hyphenation.data or { }
+local langdata = languages.hyphenation.data
+
-- 002D : hyphen-minus (ascii)
-- 2010 : hyphen
-- 2011 : nonbreakable hyphen
@@ -23,157 +32,151 @@ languages.hyphenation.data = languages.hyphenation.data or { }
--~ lang:clear_hyphenation()
-do
- -- we can consider hiding data (faster access too)
-
- --~ local function filter(filename,what)
- --~ local data = io.loaddata(input.find_file(texmf.instance,filename))
- --~ local data = data:match(string.format("\\%s%%s*(%%b{})",what or "patterns"))
- --~ return data:match("{%s*(.-)%s*}") or ""
- --~ end
-
- -- loading the 26 languages that we normally load in mkiv, the string based variant
- -- takes .84 seconds (probably due to the sub's) while the lpeg variant takes .78
- -- seconds
-
- local leftbrace = lpeg.P("{")
- local rightbrace = lpeg.P("}")
- local spaces = lpeg.S(" \r\n\t\f")
- local spacing = spaces^0
- local validchar = 1-(spaces+rightbrace+leftbrace)
- local validword = validchar^1
- local content = spacing * leftbrace * spacing * lpeg.C((spacing * validword)^0) * spacing * rightbrace * lpeg.P(true)
-
- local command = lpeg.P("\\patterns")
- local parser = (1-command)^0 * command * content
-
- local function filterpatterns(filename)
- if filename:find("%.rpl") then
- return io.loaddata(input.find_file(texmf.instance,filename)) or ""
- else
- return parser:match(io.loaddata(input.find_file(texmf.instance,filename)) or "")
- end
- end
+-- we can consider hiding data (faster access too)
- local command = lpeg.P("\\hyphenation")
- local parser = (1-command)^0 * command * content
+--~ local function filter(filename,what)
+--~ local data = io.loaddata(resolvers.find_file(filename))
+--~ local data = data:match(string.format("\\%s%%s*(%%b{})",what or "patterns"))
+--~ return data:match("{%s*(.-)%s*}") or ""
+--~ end
- local function filterexceptions(filename)
- if filename:find("%.rhl") then
- return io.loaddata(input.find_file(texmf.instance,filename)) or ""
- else
- return parser:match(io.loaddata(input.find_file(texmf.instance,filename)) or {}) -- "" ?
- end
- end
+-- loading the 26 languages that we normally load in mkiv, the string based variant
+-- takes .84 seconds (probably due to the sub's) while the lpeg variant takes .78
+-- seconds
- local function record(tag)
- local data = languages.hyphenation.data[tag]
- if not data then
- data = lang.new()
- languages.hyphenation.data[tag] = data
- end
- return data
- end
+local leftbrace = lpeg.P("{")
+local rightbrace = lpeg.P("}")
+local spaces = lpeg.S(" \r\n\t\f")
+local spacing = spaces^0
+local validchar = 1-(spaces+rightbrace+leftbrace)
+local validword = validchar^1
+local content = spacing * leftbrace * spacing * lpeg.C((spacing * validword)^0) * spacing * rightbrace * lpeg.P(true)
- languages.hyphenation.record = record
+local command = lpeg.P("\\patterns")
+local parser = (1-command)^0 * command * content
- function languages.hyphenation.define(tag)
- local data = record(tag)
- return data:id()
+local function filterpatterns(filename)
+ if find(filename,"%.rpl") then
+ return io.loaddata(resolvers.find_file(filename)) or ""
+ else
+ return parser:match(io.loaddata(resolvers.find_file(filename)) or "")
end
+end
- function languages.hyphenation.number(tag)
- local d = languages.hyphenation.data[tag]
- return (d and d:id()) or 0
- end
+local command = lpeg.P("\\hyphenation")
+local parser = (1-command)^0 * command * content
- function languages.hyphenation.load(tag, filename, filter, target)
- input.starttiming(languages)
- local data = record(tag)
- filename = (filename and filename ~= "" and input.find_file(texmf.instance,filename)) or ""
- local ok = filename ~= ""
- if ok then
- lang[target](data,filterpatterns(filename))
- else
- lang[target](data,"")
- end
- languages.hyphenation.data[tag] = data
- input.stoptiming(languages)
- return ok
+local function filterexceptions(filename)
+ if find(filename,"%.rhl") then
+ return io.loaddata(resolvers.find_file(filename)) or ""
+ else
+ return parser:match(io.loaddata(resolvers.find_file(filename)) or {}) -- "" ?
end
+end
- function languages.hyphenation.loadpatterns(tag, patterns)
- return languages.hyphenation.load(tag, patterns, filterpatterns, "patterns")
+local function record(tag)
+ local data = langdata[tag]
+ if not data then
+ data = lang.new()
+ langdata[tag] = data or 0
end
+ return data
+end
- function languages.hyphenation.loadexceptions(tag, exceptions)
- return languages.hyphenation.load(tag, patterns, filterexceptions, "hyphenation")
- end
+languages.hyphenation.record = record
- function languages.hyphenation.exceptions(tag, ...)
- local data = record(tag)
- data:hyphenation(...)
- end
+function languages.hyphenation.define(tag)
+ local data = record(tag)
+ return data:id()
+end
- function languages.hyphenation.hyphenate(tag, str)
- return lang.hyphenate(record(tag), str)
- end
+function languages.hyphenation.number(tag)
+ local d = langdata[tag]
+ return (d and d:id()) or 0
+end
- function languages.hyphenation.lefthyphenmin(tag, value)
- local data = record(tag)
- if value then data:lefthyphenmin(value) end
- return data:lefthyphenmin()
- end
- function languages.hyphenation.righthyphenmin(tag, value)
- local data = record(tag)
- if value then data:righthyphenmin(value) end
- return data:righthyphenmin()
+local function loadthem(tag, filename, filter, target)
+ statistics.starttiming(languages)
+ local data = record(tag)
+ filename = (filename and filename ~= "" and resolvers.find_file(filename)) or ""
+ local ok = filename ~= ""
+ if ok then
+ lang[target](data,filterpatterns(filename))
+ else
+ lang[target](data,"")
end
+ langdata[tag] = data
+ statistics.stoptiming(languages)
+ return ok
+end
- function languages.hyphenation.n()
- return table.count(languages.hyphenation.data)
- end
+function languages.hyphenation.loadpatterns(tag, patterns)
+ return loadthem(tag, patterns, filterpatterns, "patterns")
+end
+function languages.hyphenation.loadexceptions(tag, exceptions)
+ return loadthem(tag, patterns, filterexceptions, "hyphenation")
end
-do
+function languages.hyphenation.exceptions(tag, ...)
+ local data = record(tag)
+ data:hyphenation(...)
+end
- -- we can speed this one up with locals if needed
+function languages.hyphenation.hyphenate(tag, str)
+ return lang.hyphenate(record(tag), str)
+end
- local function tolang(what)
- if type(what) == "number" then
- return languages.hyphenation.data[languages.numbers[what]]
- elseif type(what) == "string" then
- return languages.hyphenation.data[what]
- else
- return what
- end
- end
+function languages.hyphenation.lefthyphenmin(tag, value)
+ local data = record(tag)
+ if value then data:lefthyphenmin(value) end
+ return data:lefthyphenmin()
+end
+function languages.hyphenation.righthyphenmin(tag, value)
+ local data = record(tag)
+ if value then data:righthyphenmin(value) end
+ return data:righthyphenmin()
+end
- function languages.prehyphenchar(what)
- return lang.prehyphenchar(tolang(what))
- end
- function languages.posthyphenchar(what)
- return lang.posthyphenchar(tolang(what))
- end
+function languages.hyphenation.n()
+ return table.count(langdata)
+end
- languages.tolang = tolang
+-- we can speed this one up with locals if needed
+local function tolang(what)
+ local kind = type(what)
+ if kind == "number" then
+ return langdata[languages.numbers[what]]
+ elseif kind == "string" then
+ return langdata[what]
+ else
+ return what
+ end
end
+function languages.prehyphenchar(what)
+ return lang.prehyphenchar(tolang(what))
+end
+function languages.posthyphenchar(what)
+ return lang.posthyphenchar(tolang(what))
+end
+
+languages.tolang = tolang
+
languages.registered = languages.registered or { }
languages.associated = languages.associated or { }
languages.numbers = languages.numbers or { }
-input.storage.register(false,"languages/registered",languages.registered,"languages.registered")
-input.storage.register(false,"languages/associated",languages.associated,"languages.associated")
+storage.register("languages/registered",languages.registered,"languages.registered")
+storage.register("languages/associated",languages.associated,"languages.associated")
function languages.register(tag,parent,patterns,exceptions)
parent = parent or tag
languages.registered[tag] = {
parent = parent,
- patterns = patterns or string.format("lang-%s.pat",parent),
- exceptions = exceptions or string.format("lang-%s.hyp",parent),
+ patterns = patterns or format("lang-%s.pat",parent),
+ exceptions = exceptions or format("lang-%s.hyp",parent),
loaded = false,
number = 0,
}
@@ -197,7 +200,7 @@ end
function languages.loadable(tag)
local l = languages.registered[tag]
- if l and l.patterns and input.find_file(texmf.instance,patterns) then
+ if l and l.patterns and resolvers.find_file(patterns) then
return true
else
return false
@@ -243,11 +246,11 @@ end
function languages.hyphenation.loadwords(tag, filename)
local id = languages.hyphenation.number(tag)
if id > 0 then
- local l = lang.new(id)
- input.starttiming(languages)
+ local l = lang.new(id) or 0
+ statistics.starttiming(languages)
local data = io.loaddata(filename) or ""
l:hyphenation(data)
- input.stoptiming(languages)
+ statistics.stoptiming(languages)
end
end
@@ -264,13 +267,12 @@ function languages.logger.report()
if l.loaded then
local p = (l.patterns and "pat") or '-'
local e = (l.exceptions and "exc") or '-'
- result[#result+1] = string.format("%s:%s:%s:%s:%s", tag, l.parent, p, e, l.number)
+ result[#result+1] = format("%s:%s:%s:%s:%s", tag, l.parent, p, e, l.number)
end
end
- return (#result > 0 and table.concat(result," ")) or "none"
+ return (#result > 0 and concat(result," ")) or "none"
end
-
languages.words = languages.words or {}
languages.words.data = languages.words.data or {}
languages.words.enable = false
@@ -291,15 +293,15 @@ do
local word = lpeg.Cs((markup/"" + disc/"" + (1-spacing))^1)
function languages.words.load(tag, filename)
- local filename = input.find_file(texmf.instance,filename,'other text file') or ""
+ local filename = resolvers.find_file(filename,'other text file') or ""
if filename ~= "" then
- input.starttiming(languages)
+ statistics.starttiming(languages)
local data = io.loaddata(filename) or ""
local words = languages.words.data[tag] or {}
parser = (spacing + word/function(s) words[s] = true end)^0
parser:match(data)
languages.words.data[tag] = words
- input.stoptiming(languages)
+ statistics.stoptiming(languages)
end
end
@@ -309,7 +311,7 @@ function languages.words.found(id, str)
local tag = languages.numbers[id]
if tag then
local data = languages.words.data[tag]
- return data and (data[str] or data[str:lower()])
+ return data and (data[str] or data[lower(str)])
else
return false
end
@@ -322,12 +324,10 @@ do
local glyph, disc, kern = node.id('glyph'), node.id('disc'), node.id('kern')
- local bynode = node.traverse
- local bychar = string.utfcharacters
+ local bynode = node.traverse
+ local chardata = characters.data
local function mark_words(head,found) -- can be optimized
- local cd = characters.data
- local uc = utf.char
local current, start, str, language, n = head, nil, "", nil, 0
local function action()
if #str > 0 then
@@ -356,25 +356,25 @@ do
action()
language = a
end
- if current.subtype > 0 then
+ local components = current.components
+ if components then
start = start or current
n = n + 1
- for g in bynode(current.components) do
- str = str .. uc(g.char)
+ for g in bynode(components) do
+ str = str .. utfchar(g.char)
end
else
local code = current.char
- if cd[code].uccode or cd[code].lccode then
+ if chardata[code].uccode or chardata[code].lccode then
start = start or current
n = n + 1
- str = str .. uc(code)
- else
- if start then
- action()
- end
+ str = str .. utfchar(code)
+ elseif start then
+ action()
end
end
elseif id == disc then
+ if n > 0 then n = n + 1 end
-- ok
elseif id == kern and current.subtype == 0 and start then
-- ok
@@ -392,18 +392,20 @@ do
languages.words.methods = { }
languages.words.method = 1
+ local lw = languages.words
+
languages.words.methods[1] = function(head, attribute, yes, nop)
local set = node.set_attribute
local unset = node.unset_attribute
- local wrong, right = false, false
- if nop then wrong = function(n) set(n,attribute,nop) end end
+ local right, wrong = false, false
if yes then right = function(n) set(n,attribute,yes) end end
+ if nop then wrong = function(n) set(n,attribute,nop) end end
for n in node.traverse(head) do
unset(n,attribute) -- hm
end
local found, done = languages.words.found, false
mark_words(head, function(language,str)
- if #str < languages.words.threshold then
+ if #str < lw.threshold then
return false
elseif found(language,str) then
done = true
@@ -416,11 +418,10 @@ do
return head, done
end
- local lw = languages.words
+ local color = attributes.private('color')
function languages.words.check(head)
- if head.next and lw.enable then
- local color = attributes.numbers['color']
+ if lw.enable and head.next then
local colors = lw.colors
local alc = attributes.list[color]
return lw.methods[lw.method](head, color, alc[colors.known], alc[colors.unknown])
@@ -439,7 +440,8 @@ end
--~ return languages.check(head)
--~ end
---~ chars.plugins.language = {
+--~ chars.plugins[chars.plugins+1] = {
+--~ name = "language",
--~ namespace = languagehacks,
--~ processor = languagehacks.process
--~ }
@@ -451,3 +453,16 @@ languages.associate('uk','latn','eng')
languages.associate('nl','latn','nld')
languages.associate('de','latn','deu')
languages.associate('fr','latn','fra')
+
+statistics.register("loaded patterns", function()
+ local result = languages.logger.report()
+ if result ~= "none" then
+ return result
+ end
+end)
+
+statistics.register("language load time", function()
+ if statistics.elapsedindeed(languages) then
+ return format("%s seconds, n=%s", statistics.elapsedtime(languages), languages.hyphenation.n())
+ end
+end)