summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/tex/context/base/lang-ini.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/tex/context/base/lang-ini.lua')
-rw-r--r--Master/texmf-dist/tex/context/base/lang-ini.lua281
1 files changed, 67 insertions, 214 deletions
diff --git a/Master/texmf-dist/tex/context/base/lang-ini.lua b/Master/texmf-dist/tex/context/base/lang-ini.lua
index e188ad36cc9..239e5390c55 100644
--- a/Master/texmf-dist/tex/context/base/lang-ini.lua
+++ b/Master/texmf-dist/tex/context/base/lang-ini.lua
@@ -1,6 +1,6 @@
if not modules then modules = { } end modules ['lang-ini'] = {
version = 1.001,
- comment = "companion to lang-ini.tex",
+ comment = "companion to lang-ini.mkiv",
author = "Hans Hagen, PRAGMA-ADE, Hasselt NL",
copyright = "PRAGMA ADE / ConTeXt Development Team",
license = "see context related readme files"
@@ -9,11 +9,12 @@ if not modules then modules = { } end modules ['lang-ini'] = {
-- needs a cleanup (share locals)
local utf = unicode.utf8
-
-local find, lower, format, utfchar = string.find, string.lower, string.format, utf.char
+local utfbyte = utf.byte
+local format = string.format
local concat = table.concat
+local lpegmatch = lpeg.match
-if lang.use_new then lang.use_new(true) end
+local trace_patterns = false trackers.register("languages.patterns", function(v) trace_patterns = v end)
languages = languages or {}
languages.version = 1.009
@@ -31,18 +32,13 @@ local langdata = languages.hyphenation.data
--~ string =lang:hyphenation()
--~ lang:clear_hyphenation()
-
-- we can consider hiding data (faster access too)
---~ local function filter(filename,what)
---~ local data = io.loaddata(resolvers.find_file(filename))
---~ local data = data:match(string.format("\\%s%%s*(%%b{})",what or "patterns"))
---~ return data:match("{%s*(.-)%s*}") or ""
---~ end
-
-- loading the 26 languages that we normally load in mkiv, the string based variant
-- takes .84 seconds (probably due to the sub's) while the lpeg variant takes .78
-- seconds
+--
+-- the following lpeg can probably be improved (it was one of the first I made)
local leftbrace = lpeg.P("{")
local rightbrace = lpeg.P("}")
@@ -56,10 +52,10 @@ local command = lpeg.P("\\patterns")
local parser = (1-command)^0 * command * content
local function filterpatterns(filename)
- if find(filename,"%.rpl") then
+ if file.extname(filename) == "rpl" then
return io.loaddata(resolvers.find_file(filename)) or ""
else
- return parser:match(io.loaddata(resolvers.find_file(filename)) or "")
+ return lpegmatch(parser,io.loaddata(resolvers.find_file(filename)) or "")
end
end
@@ -67,10 +63,10 @@ local command = lpeg.P("\\hyphenation")
local parser = (1-command)^0 * command * content
local function filterexceptions(filename)
- if find(filename,"%.rhl") then
+ if file.extname(filename) == "rhl" then
return io.loaddata(resolvers.find_file(filename)) or ""
else
- return parser:match(io.loaddata(resolvers.find_file(filename)) or {}) -- "" ?
+ return lpegmatch(parser,io.loaddata(resolvers.find_file(filename)) or {}) -- "" ?
end
end
@@ -95,14 +91,22 @@ function languages.hyphenation.number(tag)
return (d and d:id()) or 0
end
+lang.exceptions = lang.hyphenation
+
local function loadthem(tag, filename, filter, target)
statistics.starttiming(languages)
local data = record(tag)
- filename = (filename and filename ~= "" and resolvers.find_file(filename)) or ""
- local ok = filename ~= ""
+ local fullname = (filename and filename ~= "" and resolvers.find_file(filename)) or ""
+ local ok = fullname ~= ""
if ok then
- lang[target](data,filterpatterns(filename))
+ if trace_patterns then
+ logs.report("languages","filtering %s for language '%s' from '%s'",target,tag,fullname)
+ end
+ lang[target](data,filterpatterns(fullname))
else
+ if trace_patterns then
+ logs.report("languages","no %s for language '%s' in '%s'",target,tag,filename or "?")
+ end
lang[target](data,"")
end
langdata[tag] = data
@@ -115,7 +119,7 @@ function languages.hyphenation.loadpatterns(tag, patterns)
end
function languages.hyphenation.loadexceptions(tag, exceptions)
- return loadthem(tag, patterns, filterexceptions, "hyphenation")
+ return loadthem(tag, patterns, filterexceptions, "exceptions")
end
function languages.hyphenation.exceptions(tag, ...)
@@ -142,12 +146,24 @@ function languages.hyphenation.n()
return table.count(langdata)
end
+languages.registered = languages.registered or { }
+languages.associated = languages.associated or { }
+languages.numbers = languages.numbers or { }
+
+storage.register("languages/registered",languages.registered,"languages.registered")
+storage.register("languages/associated",languages.associated,"languages.associated")
+
+local numbers = languages.numbers
+local registered = languages.registered
+local associated = languages.associated
+
-- we can speed this one up with locals if needed
local function tolang(what)
local kind = type(what)
if kind == "number" then
- return langdata[languages.numbers[what]]
+ local w = what >= 0 and what <= 0x7FFF and numbers[what]
+ return (w and langdata[w]) or 0
elseif kind == "string" then
return langdata[what]
else
@@ -155,6 +171,20 @@ local function tolang(what)
end
end
+function languages.setup(what,settings)
+ what = languages.tolang(what or tex.language)
+ local lefthyphen = settings.lefthyphen
+ local righthyphen = settings.righthyphen
+ lefthyphen = lefthyphen ~= "" and lefthyphen or nil
+ righthyphen = righthyphen ~= "" and righthyphen or nil
+ lefthyphen = lefthyphen and utfbyte(lefthyphen) or 0
+ righthyphen = righthyphen and utfbyte(righthyphen) or 0
+ lang.posthyphenchar(what,lefthyphen)
+ lang.prehyphenchar (what,righthyphen)
+ lang.postexhyphenchar(what,lefthyphen)
+ lang.preexhyphenchar (what,righthyphen)
+end
+
function languages.prehyphenchar(what)
return lang.prehyphenchar(tolang(what))
end
@@ -164,16 +194,9 @@ end
languages.tolang = tolang
-languages.registered = languages.registered or { }
-languages.associated = languages.associated or { }
-languages.numbers = languages.numbers or { }
-
-storage.register("languages/registered",languages.registered,"languages.registered")
-storage.register("languages/associated",languages.associated,"languages.associated")
-
function languages.register(tag,parent,patterns,exceptions)
parent = parent or tag
- languages.registered[tag] = {
+ registered[tag] = {
parent = parent,
patterns = patterns or format("lang-%s.pat",parent),
exceptions = exceptions or format("lang-%s.hyp",parent),
@@ -183,14 +206,14 @@ function languages.register(tag,parent,patterns,exceptions)
end
function languages.associate(tag,script,language)
- languages.associated[tag] = { script, language }
+ associated[tag] = { script, language }
end
function languages.association(tag)
if type(tag) == "number" then
- tag = languages.numbers[tag]
+ tag = numbers[tag]
end
- local lat = tag and languages.associated[tag]
+ local lat = tag and associated[tag]
if lat then
return lat[1], lat[2]
else
@@ -199,7 +222,7 @@ function languages.association(tag)
end
function languages.loadable(tag)
- local l = languages.registered[tag]
+ local l = registered[tag]
if l and l.patterns and resolvers.find_file(patterns) then
return true
else
@@ -213,8 +236,8 @@ function languages.enable(tags)
-- beware: we cannot set tex.language, but need tex.normallanguage
for i=1,#tags do
local tag = tags[i]
- local l = languages.registered[tag]
- if l then
+ local l = registered[tag]
+ if l and l ~= "" then
if not l.loaded then
local tag = l.parent
local number = languages.hyphenation.number(tag)
@@ -225,9 +248,12 @@ function languages.enable(tags)
l.number = languages.hyphenation.define(tag)
languages.hyphenation.loadpatterns(tag,l.patterns)
languages.hyphenation.loadexceptions(tag,l.exceptions)
- languages.numbers[l.number] = tag
+ numbers[l.number] = tag
end
l.loaded = true
+ if trace_patterns then
+ logs.report("languages","assigning number %s",l.number)
+ end
end
if l.number > 0 then
return l.number
@@ -261,9 +287,11 @@ languages.hyphenation.loadexceptions("zerolanguage") -- else bug
languages.logger = languages.logger or { }
function languages.logger.report()
- local result = {}
- for _, tag in ipairs(table.sortedkeys(languages.registered)) do
- local l = languages.registered[tag]
+ local result = { }
+ local sorted = table.sortedkeys(registered)
+ for i=1,#sorted do
+ local tag = sorted[i]
+ local l = registered[tag]
if l.loaded then
local p = (l.patterns and "pat") or '-'
local e = (l.exceptions and "exc") or '-'
@@ -273,179 +301,6 @@ function languages.logger.report()
return (#result > 0 and concat(result," ")) or "none"
end
-languages.words = languages.words or {}
-languages.words.data = languages.words.data or {}
-languages.words.enable = false
-languages.words.threshold = 4
-
-languages.words.colors = {
- ["known"] = "green",
- ["unknown"] = "red",
-}
-
-do
-
- local spacing = lpeg.S(" \n\r\t")
- local markup = lpeg.S("-=")
- local lbrace = lpeg.P("{")
- local rbrace = lpeg.P("}")
- local disc = (lbrace * (1-rbrace)^0 * rbrace)^1 -- or just 3 times, time this
- local word = lpeg.Cs((markup/"" + disc/"" + (1-spacing))^1)
-
- function languages.words.load(tag, filename)
- local filename = resolvers.find_file(filename,'other text file') or ""
- if filename ~= "" then
- statistics.starttiming(languages)
- local data = io.loaddata(filename) or ""
- local words = languages.words.data[tag] or {}
- parser = (spacing + word/function(s) words[s] = true end)^0
- parser:match(data)
- languages.words.data[tag] = words
- statistics.stoptiming(languages)
- end
- end
-
-end
-
-function languages.words.found(id, str)
- local tag = languages.numbers[id]
- if tag then
- local data = languages.words.data[tag]
- return data and (data[str] or data[lower(str)])
- else
- return false
- end
-end
-
--- The following code is an adaption of experimental code for
--- hyphenating and spell checking.
-
-do
-
- local glyph, disc, kern = node.id('glyph'), node.id('disc'), node.id('kern')
-
- local bynode = node.traverse
- local chardata = characters.data
-
- local function mark_words(head,found) -- can be optimized
- local current, start, str, language, n = head, nil, "", nil, 0
- local function action()
- if #str > 0 then
- local f = found(language,str)
- if f then
- for i=1,n do
- f(start)
- start = start.next
- end
- end
- end
- str, start, n = "", nil, 0
- end
- while current do
- local id = current.id
- if id == glyph then
- local a = current.lang
- if a then
- if a ~= language then
- if start then
- action()
- end
- language = a
- end
- elseif start then
- action()
- language = a
- end
- local components = current.components
- if components then
- start = start or current
- n = n + 1
- for g in bynode(components) do
- str = str .. utfchar(g.char)
- end
- else
- local code = current.char
- if chardata[code].uccode or chardata[code].lccode then
- start = start or current
- n = n + 1
- str = str .. utfchar(code)
- elseif start then
- action()
- end
- end
- elseif id == disc then
- if n > 0 then n = n + 1 end
- -- ok
- elseif id == kern and current.subtype == 0 and start then
- -- ok
- elseif start then
- action()
- end
- current = current.next
- end
- if start then
- action()
- end
- return head
- end
-
- languages.words.methods = { }
- languages.words.method = 1
-
- local lw = languages.words
-
- languages.words.methods[1] = function(head, attribute, yes, nop)
- local set = node.set_attribute
- local unset = node.unset_attribute
- local right, wrong = false, false
- if yes then right = function(n) set(n,attribute,yes) end end
- if nop then wrong = function(n) set(n,attribute,nop) end end
- for n in node.traverse(head) do
- unset(n,attribute) -- hm
- end
- local found, done = languages.words.found, false
- mark_words(head, function(language,str)
- if #str < lw.threshold then
- return false
- elseif found(language,str) then
- done = true
- return right
- else
- done = true
- return wrong
- end
- end)
- return head, done
- end
-
- local color = attributes.private('color')
-
- function languages.words.check(head)
- if lw.enable and head.next then
- local colors = lw.colors
- local alc = attributes.list[color]
- return lw.methods[lw.method](head, color, alc[colors.known], alc[colors.unknown])
- else
- return head, false
- end
- end
-
-end
-
--- for the moment we hook it into the attribute handler
-
---~ languagehacks = { }
-
---~ function languagehacks.process(namespace,attribute,head)
---~ return languages.check(head)
---~ end
-
---~ chars.plugins[chars.plugins+1] = {
---~ name = "language",
---~ namespace = languagehacks,
---~ processor = languagehacks.process
---~ }
-
-- must happen at the tex end
languages.associate('en','latn','eng')
@@ -462,7 +317,5 @@ statistics.register("loaded patterns", function()
end)
statistics.register("language load time", function()
- if statistics.elapsedindeed(languages) then
- return format("%s seconds, n=%s", statistics.elapsedtime(languages), languages.hyphenation.n())
- end
+ return statistics.elapsedseconds(languages, format(", n=%s",languages.hyphenation.n()))
end)