summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/scripts/context/lua/mtx-patterns.lua')
-rw-r--r--Master/texmf-dist/scripts/context/lua/mtx-patterns.lua660
1 files changed, 399 insertions, 261 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
index 293016991ff..bbf47513354 100644
--- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
+++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
@@ -6,312 +6,453 @@ if not modules then modules = { } end modules ['mtx-patterns'] = {
license = "see context related readme files"
}
-local format = string.format
+local format, find, concat, gsub, match, gmatch = string.format, string.find, table.concat, string.gsub, string.match, string.gmatch
+local byte, char = utf.byte, utf.char
+local addsuffix = file.addsuffix
+local lpegmatch, validutf8 = lpeg.match, lpeg.patterns.validutf8
+
+local helpinfo = [[
+--convert generate context language files (mnemonic driven, if not given then all)
+--check check pattern file (or those used by context when no file given)
+--path source path where hyph-foo.tex files are stored
+--destination destination path
+
+examples of usage:
+
+mtxrun --script pattern --check hyph-*.tex
+mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns
+mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns
+mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns
+]]
+
+local application = logs.application {
+ name = "mtx-patterns",
+ banner = "ConTeXt Pattern File Management 0.20",
+ helpinfo = helpinfo,
+}
+
+local report = application.report
scripts = scripts or { }
scripts.patterns = scripts.patterns or { }
-scripts.patterns.list = {
- { "??", "hyph-ar.tex", "arabic" },
- { "bg", "hyph-bg.tex", "bulgarian" },
- { "ca", "hyph-ca.tex", "catalan" },
- { "??", "hyph-cop.tex", "coptic" },
- { "cs", "hyph-cs.tex", "czech" },
- { "cy", "hyph-cy.tex", "welsh" },
- { "da", "hyph-da.tex", "danish" },
- { "deo", "hyph-de-1901.tex", "german, old spelling" },
- { "de", "hyph-de-1996.tex", "german, new spelling" },
---~ { "??", "hyph-el-monoton.tex", "" },
---~ { "??", "hyph-el-polyton.tex", "" },
- { "agr", "hyph-grc", "ancient greek" },
---~ { "???", "hyph-x-ibycus", "ancient greek in ibycus encoding" },
---~ { "gr", "", "" },
- { "eo", "hyph-eo.tex", "esperanto" },
- { "gb", "hyph-en-gb.tex", "british english" },
- { "us", "hyph-en-us.tex", "american english" },
- { "es", "hyph-es.tex", "spanish" },
- { "et", "hyph-et.tex", "estonian" },
- { "eu", "hyph-eu.tex", "basque" }, -- ba is Bashkir!
- { "fa", "hyph-fa.tex", "farsi" },
- { "fi", "hyph-fi.tex", "finnish" },
- { "fr", "hyph-fr.tex", "french" },
--- { "??", "hyph-ga.tex", "" },
--- { "??", "hyph-gl.tex", "" },
--- { "??", "hyph-grc.tex", "" },
- { "hr", "hyph-hr.tex", "croatian" },
- { "??", "hyph-hsb.tex", "upper sorbian" },
- { "hu", "hyph-hu.tex", "hungarian" },
- { "??", "hyph-ia.tex", "interlingua" },
- { "??", "hyph-id.tex", "indonesian" },
- { "is", "hyph-is.tex", "icelandic" },
- { "it", "hyph-it.tex", "italian" },
- { "la", "hyph-la.tex", "latin" },
- { "lt", "hyph-lt.tex", "lithuanian" },
- { "mn", "hyph-mn-cyrl.tex", "mongolian, cyrillic script" },
- { "nb", "hyph-nb.tex", "norwegian bokmål" },
- { "nl", "hyph-nl.tex", "dutch" },
- { "nn", "hyph-nn.tex", "norwegian nynorsk" },
- { "pl", "hyph-pl.tex", "polish" },
- { "pt", "hyph-pt.tex", "portuguese" },
- { "ro", "hyph-ro.tex", "romanian" },
- { "ru", "hyph-ru.tex", "russian" },
- { "sk", "hyph-sk.tex", "slovak" },
- { "sl", "hyph-sl.tex", "slovenian" },
- { "sr", "hyph-sr-cyrl.tex", "serbian" },
- { "sv", "hyph-sv.tex", "swedish" },
- { "tr", "hyph-tr.tex", "turkish" },
- { "tk", "hyph-tk.tex", "turkman" },
- { "uk", "hyph-uk.tex", "ukrainian" },
- { "zh", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" },
+local permitted_characters = table.tohash {
+ 0x0009, -- tab
+ 0x0027, -- apostrofe
+ 0x02BC, -- modifier apostrofe (used in greek)
+ 0x002D, -- hyphen
+ 0x200C, -- zwnj
+ 0x2019, -- quote right
+ 0x1FBD, -- greek, but no letter: symbol modifier
+ 0x1FBF, -- greek, but no letter: symbol modifier
}
+local ignored_ancient_greek = table.tohash {
+ 0x1FD3, -- greekiotadialytikatonos (also 0x0390)
+ 0x1FE3, -- greekupsilondialytikatonos (also 0x03B0)
+ 0x1FBD, -- greek, but no letter: symbol modifier
+ 0x1FBF, -- greek, but no letter: symbol modifier
+ 0x03F2, -- greeksigmalunate
+ 0x02BC, -- modifier apostrofe)
+}
--- stripped down from lpeg example:
+local ignored_french = table.tohash {
+ 0x02BC, -- modifier apostrofe
+}
-local utf = unicode.utf8
+local replaced_whatever = {
+ [char(0x2019)] = char(0x0027)
+}
-local cont = lpeg.R("\128\191") -- continuation byte
+scripts.patterns.list = {
+ { "af", "hyph-af", "afrikaans" },
+ -- { "ar", "hyph-ar", "arabic" },
+ -- { "as", "hyph-as", "assamese" },
+ { "bg", "hyph-bg", "bulgarian" },
+ -- { "bn", "hyph-bn", "bengali" },
+ { "ca", "hyph-ca", "catalan" },
+ -- { "??", "hyph-cop", "coptic" },
+ { "cs", "hyph-cs", "czech" },
+ { "cy", "hyph-cy", "welsh" },
+ { "da", "hyph-da", "danish" },
+ { "deo", "hyph-de-1901", "german, old spelling" },
+ { "de", "hyph-de-1996", "german, new spelling" },
+ -- { "??", "hyph-de-ch-1901", "swiss german" },
+ -- { "??", "hyph-el-monoton", "greek" },
+ -- { "gr", "hyph-el-polyton", "greek" },
+ { "agr", "hyph-grc", "ancient greek", ignored_ancient_greek },
+ { "gb", "hyph-en-gb", "british english" },
+ { "us", "hyph-en-us", "american english" },
+ -- { "eo", "hyph-eo", "esperanto" },
+ { "es", "hyph-es", "spanish" },
+ { "et", "hyph-et", "estonian" },
+ { "eu", "hyph-eu", "basque" },
+ -- { "fa", "hyph-fa", "farsi" },
+ { "fi", "hyph-fi", "finnish" },
+ { "fr", "hyph-fr", "french", ignored_french },
+ -- { "??", "hyph-ga", "irish" },
+ -- { "??", "hyph-gl", "galician" },
+ -- { "gu", "hyph-gu", "gujarati" },
+ -- { "hi", "hyph-hi", "hindi" },
+ { "hr", "hyph-hr", "croatian" },
+ -- { "??", "hyph-hsb", "upper sorbian" },
+ { "hu", "hyph-hu", "hungarian" },
+ -- { "hy", "hyph-hy", "armenian" },
+ -- { "??", "hyph-ia", "interlingua" },
+ -- { "??", "hyph-id", "indonesian" },
+ { "is", "hyph-is", "icelandic" },
+ { "it", "hyph-it", "italian" },
+ -- { "??", "hyph-kmr", "kurmanji" },
+ -- { "kn", "hyph-kn", "kannada" },
+ { "la", "hyph-la", "latin" },
+ -- { "lo", "hyph-lo", "lao" },
+ { "lt", "hyph-lt", "lithuanian" },
+ { "lv", "hyph-lv", "latvian" },
+ -- { "ml", "hyph-ml", "..." },
+ { "mn", "hyph-mn-cyrl", "mongolian, cyrillic script" },
+ -- { "mr", "hyph-mr", "..." },
+ { "nb", "hyph-nb", "norwegian bokmål" },
+ { "nl", "hyph-nl", "dutch" },
+ { "nn", "hyph-nn", "norwegian nynorsk" },
+ -- { "or", "hyph-or", "oriya" },
+ -- { "pa", "hyph-pa", "panjabi" },
+ -- { "", "hyph-", "" },
+ { "pl", "hyph-pl", "polish" },
+ { "pt", "hyph-pt", "portuguese" },
+ { "ro", "hyph-ro", "romanian" },
+ { "ru", "hyph-ru", "russian" },
+ -- { "sa", "hyph-sa", "sanskrit" },
+ { "sk", "hyph-sk", "slovak" },
+ { "sl", "hyph-sl", "slovenian" },
+ { "sr", "hyph-sr-cyrl", "serbian" },
+ -- { "sr", "hyph-sr-latn", "serbian" },
+ { "sv", "hyph-sv", "swedish" },
+ -- { "ta", "hyph-ta", "tamil" },
+ -- { "te", "hyph-te", "telugu" },
+ { "tk", "hyph-tk", "turkmen" },
+ { "tr", "hyph-tr", "turkish" },
+ { "uk", "hyph-uk", "ukrainian" },
+ { "zh", "hyph-zh-latn", "zh-latn, chinese pinyin" },
+}
-local utf8 = lpeg.R("\0\127")
- + lpeg.R("\194\223") * cont
- + lpeg.R("\224\239") * cont * cont
- + lpeg.R("\240\244") * cont * cont * cont
+-- stripped down from lpeg example:
-local validutf = (utf8^0/function() return true end) * (lpeg.P(-1)/function() return false end)
+local utf = unicode.utf8
function utf.check(str)
- return lpeg.match(validutf,str)
+ return lpeg.match(lpeg.patterns.validutf8,str)
end
-local permitted_commands = table.tohash {
- "message",
- "endinput"
-}
+-- *.tex
+-- *.hyp.txt *.pat.txt *.lic.txt *.chr.txt
-local permitted_characters = table.tohash {
- 0x0009, -- tab
- 0x0027, -- apostrofe
- 0x002D, -- hyphen
- 0x200C, --
-}
-
-function scripts.patterns.load(path,name,mnemonic,fullcheck)
+function scripts.patterns.load(path,name,mnemonic,ignored)
local fullname = file.join(path,name)
- local data = io.loaddata(fullname) or ""
- local byte, char = utf.byte, utf.char
- if data ~= "" then
- data = data:gsub("([\n\r])\\input ([^ \n\r]+)", function(previous,subname)
- local subname = file.addsuffix(subname,"tex")
- local subfull = file.join(file.dirname(fullname),subname)
- local subdata = io.loaddata(subfull) or ""
- if subdata == "" then
- if mnemonic then
- logs.simple("no subfile %s for language %s",subname,mnemonic)
- else
- logs.simple("no subfile %s",name)
+ local texfile = addsuffix(fullname,"tex")
+ local hypfile = addsuffix(fullname,"hyp.txt")
+ local patfile = addsuffix(fullname,"pat.txt")
+ local licfile = addsuffix(fullname,"lic.txt")
+ -- local chrfile = addsuffix(fullname,"chr.txt")
+ local okay = true
+ local hyphenations, patterns, comment, stripset = "", "", "", ""
+ local splitpatternsnew, splithyphenationsnew = { }, { }
+ local splitpatternsold, splithyphenationsold = { }, { }
+ local usedpatterncharacters, usedhyphenationcharacters = { }, { }
+ if lfs.isfile(patfile) then
+ report("using txt files %s.[hyp|pat|lic].txt",name)
+ comment, patterns, hyphenations = io.loaddata(licfile) or "", io.loaddata(patfile) or "", io.loaddata(hypfile) or ""
+ hypfile, patfile, licfile = hypfile, patfile, licfile
+ elseif lfs.isfile(texfile) then
+ report("using tex file %s.txt",name)
+ local data = io.loaddata(texfile) or ""
+ if data ~= "" then
+ data = gsub(data,"([\n\r])\\input ([^ \n\r]+)", function(previous,subname)
+ local subname = addsuffix(subname,"tex")
+ local subfull = file.join(file.dirname(texfile),subname)
+ local subdata = io.loaddata(subfull) or ""
+ if subdata == "" then
+ report("no subfile %s",subname)
end
- end
- return previous .. subdata
- end)
- local comment = data:match("^(.-)[\n\r]\\patterns") or ""
- local n, okay = 0, true
- local cd = characters.data
- for line in data:gmatch("[^ \n\r]+") do
- local ok = utf.check(line)
- n = n + 1
- if not ok then
- okay = false
- line = line:gsub("%%","%%%%")
- if fullcheck then
- if mnemonic then
- logs.simple("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line)
- else
- logs.simple("invalid utf in file %s, line %s: %s",name,n,line)
- end
- else
- if mnemonic then
- logs.simple("file %s for %s contains invalid utf",name,mnemonic)
- else
- logs.simple("file %s contains invalid utf",name)
+ return previous .. subdata
+ end)
+ data = gsub(data,"%%.-[\n\r]","")
+ data = gsub(data," *[\n\r]+","\n")
+ patterns = match(data,"\\patterns[%s]*{[%s]*(.-)[%s]*}") or ""
+ hyphenations = match(data,"\\hyphenation[%s]*{[%s]*(.-)[%s]*}") or ""
+ comment = match(data,"^(.-)[\n\r]\\patterns") or ""
+ else
+ okay = false
+ end
+ else
+ okay = false
+ end
+ if okay then
+ -- split into lines
+ local how = lpeg.patterns.whitespace^1
+ splitpatternsnew = lpeg.split(how,patterns)
+ splithyphenationsnew = lpeg.split(how,hyphenations)
+ end
+ if okay then
+ -- remove comments
+ local function check(data,splitdata,name)
+ if find(data,"%%") then
+ for i=1,#splitdata do
+ local line = splitdata[i]
+ if find(line,"%%") then
+ splitdata[i] = gsub(line,"%%.*$","")
+ report("removing comment: %s",line)
end
- break
end
end
end
- local c, h = { }, { }
- for line in data:gmatch("[^\n\r]+") do
- local txt, cmt = line:match("^(.-)%%(.*)$")
- if not txt then
- txt, cmt = line, ""
- end
- for s in txt:gmatch("\\([a-zA-Z]+)") do
- h[s] = (h[s] or 0) + 1
- end
- for s in cmt:gmatch("\\([a-zA-Z]+)") do
- c[s] = (c[s] or 0) + 1
- end
- end
- h.patterns = nil
- h.hyphenation = nil
- for k, v in next, h do
- if not permitted_commands[k] then okay = false end
- if mnemonic then
- logs.simple("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v)
- else
- logs.simple("command \\%s found in file %s, n=%s",k,name,v)
+ check(patterns,splitpatternsnew,patfile)
+ check(hyphenations,splithyphenationsnew,hypfile)
+ end
+ if okay then
+ -- remove lines with commands
+ local function check(data,splitdata,name)
+ if find(data,"\\") then
+ for i=1,#splitdata do
+ local line = splitdata[i]
+ if find(line,"\\") then
+ splitdata[i] = ""
+ report("removing line with command: %s",line)
+ end
+ end
end
end
- if not environment.argument("fast") then
- for k, v in next, c do
- if mnemonic then
- logs.simple("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v)
- else
- logs.simple("command \\%s found in comment of file %s, n=%s",k,name,v)
+ check(patterns,splitpatternsnew,patfile)
+ check(hyphenations,splithyphenationsnew,hypfile)
+ end
+ if okay then
+ -- check for valid utf
+ local function check(data,splitdata,name)
+ for i=1,#splitdata do
+ local line = splitdata[i]
+ local ok = lpegmatch(validutf8,line)
+ if not ok then
+ splitdata[i] = ""
+ report("removing line with invalid utf: %s",line)
end
end
+ -- check for commands being used in comments
end
- data = data:gsub("%%.-[\n\r]","")
- data = data:gsub(" *[\n\r]+","\n")
- local patterns = data:match("\\patterns[%s]*{[%s]*(.-)[%s]*}") or ""
- local hyphenations = data:match("\\hyphenation[%s]*{[%s]*(.-)[%s]*}") or ""
- patterns = patterns:gsub(" +","\n")
- hyphenations = hyphenations:gsub(" +","\n")
- local p, h = { }, { }
- local pats, hyps = { } , { }
- local pused, hused = { } , { }
- local period = byte(".")
- for line in patterns:gmatch("[^ \n\r]+") do
- local ok = true
- for b in line:utfvalues() do
- if b == period then
- -- ok
- else
- local ct = cd[b].category
- if ct == "lu" or ct == "ll" then
- pused[char(b)] = true
- elseif ct == "nd" then
- -- ok
+ check(patterns,splitpatternsnew,patfile)
+ check(hyphenations,splithyphenationsnew,hypfile)
+ end
+ if okay then
+ -- remove funny lines
+ local cd = characters.data
+ local stripped = { }
+ local function check(splitdata,special,name)
+ local used = { }
+ for i=1,#splitdata do
+ local line = splitdata[i]
+ for b in line:utfvalues() do -- could be an lpeg
+ if b == special then
+ -- not registered
+ elseif permitted_characters[b] then
+ used[char(b)] = true
else
- p[b] = (p[b] or 0) + 1
- ok = false
+ local cdb = cd[b]
+ if not cdb then
+ report("no entry in chardata for character %s (0x%04X)",char(b),b)
+ else
+ local ct = cd[b].category
+ if ct == "lu" or ct == "ll" then
+ used[char(b)] = true
+ elseif ct == "nd" then
+ -- number
+ else
+ report("removing line with suspected utf character %s (0x%04X), category %s: %s",char(b),b,ct,line)
+ splitdata[i] = ""
+ break
+ end
+ end
end
end
end
- if ok then
- pats[#pats+1] = line
- end
+ return used
end
- local hyphen = byte("-")
- for line in hyphenations:gmatch("[^ \n\r]+") do
- local ok = true
- for b in line:utfvalues() do
- if b == hyphen then
- -- ok
- else
- local ct = cd[b].category
- if ct == "lu" or ct == "ll" then
- hused[char(b)] = true
+ usedpatterncharacters = check(splitpatternsnew,byte("."))
+ usedhyphenationcharacters = check(splithyphenationsnew,byte("-"))
+ for k, v in next, stripped do
+ report("entries that contain character %s (0x%04X) have been omitted",char(k),k)
+ end
+ end
+ if okay then
+ local function stripped(what,ignored)
+ -- ignored (per language)
+ local p = nil
+ if ignored then
+ for k, v in next, ignored do
+ if p then
+ p = p + lpeg.P(char(k))
else
- h[b] = (h[b] or 0) + 1
- ok = false
+ p = lpeg.P(char(k))
end
end
+ p = lpeg.P{ p + 1 * lpeg.V(1) } -- anywhere
end
- if ok then
- hyps[#hyps+1] = line
- end
- end
- local stripped = { }
- for k, v in next, p do
- if mnemonic then
- logs.simple("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v)
- else
- logs.simple("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v)
+ -- replaced (all languages)
+ local r = nil
+ for k, v in next, replaced_whatever do
+ if r then
+ r = r + lpeg.P(k)/v
+ else
+ r = lpeg.P(k)/v
+ end
end
- if not permitted_characters[k] then
- okay = false
- else
- stripped[k] = true
+ r = lpeg.Cs((r + 1)^0)
+ local result = { }
+ for i=1,#what do
+ local line = what[i]
+ if p and lpegmatch(p,line) then
+ report("discarding conflicting pattern: %s",line)
+ else -- we can speed this up by testing for replacements in the string
+ local l = lpegmatch(r,line)
+ if l ~= line then
+ report("sanitizing pattern: %s -> %s (for old patterns)",line,l)
+ end
+ result[#result+1] = l
+ end
end
+ return result
end
- for k, v in next, h do
- if mnemonic then
- logs.simple("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v)
- else
- logs.simple("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v)
- end
- if not permitted_characters[k] then
- okay = false
- else
- stripped[k] = true
+
+ splitpatternsold = stripped(splitpatternsnew,ignored)
+ splithyphenationsold = stripped(splithyphenationsnew,ignored)
+
+ end
+ if okay then
+ -- discarding duplicates
+ local function check(data,splitdata,name)
+ local used, collected = { }, { }
+ for i=1,#splitdata do
+ local line = splitdata[i]
+ if line == "" then
+ -- discard
+ elseif used[line] then
+ -- discard
+ repo("discarding duplicate pattern: %s",line)
+ else
+ used[line] = true
+ collected[#collected+1] = line
+ end
end
+ return collected
end
- local stripset = ""
- for k, v in next, stripped do
- logs.simple("entries that contain character %s will be omitted",char(k))
- stripset = stripset .. "%" .. char(k)
- end
- return okay, pats, hyps, comment, stripset, pused, hused
- else
- if mnemonic then
- logs.simple("no file %s for language %s",fullname,mnemonic)
- else
- logs.simple("no file %s",fullname)
- end
- return false, { }, { }, "", "", { }, { }
+ splitpatternsnew = check(patterns,splitpatternsnew,patfile)
+ splithyphenationsnew = check(hyphenations,splithyphenationsnew,hypfile)
+ splitpatternsold = check(patterns,splitpatternsold,patfile)
+ splithyphenationsold = check(hyphenations,splithyphenationsold,hypfile)
+ end
+ if not okay then
+ report("no valid file %s.*",name)
end
+ return okay, splitpatternsnew, splithyphenationsnew, splitpatternsold, splithyphenationsold, comment, stripset, usedpatterncharacters, usedhyphenationcharacters
end
-function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused)
- local nofpatterns = #patterns
- local nofhyphenations = #hyphenations
- local pu = table.concat(table.sortedkeys(pused), " ")
- local hu = table.concat(table.sortedkeys(hused), " ")
- logs.simple("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations)
+function scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenationsnew,patternsold,hyphenationsold,comment,stripped,pused,hused,ignored)
+ local nofpatternsnew, nofhyphenationsnew = #patternsnew, #hyphenationsnew
+ local nofpatternsold, nofhyphenationsold = #patternsold, #hyphenationsold
+ report("language %s has %s old and %s new patterns and %s old and %s new exceptions",mnemonic,nofpatternsold,nofpatternsnew,nofhyphenationsold,nofhyphenationsnew)
if mnemonic ~= "??" then
+ local pu = concat(table.sortedkeys(pused), " ")
+ local hu = concat(table.sortedkeys(hused), " ")
+
local rmefile = file.join(destination,"lang-"..mnemonic..".rme")
local patfile = file.join(destination,"lang-"..mnemonic..".pat")
local hypfile = file.join(destination,"lang-"..mnemonic..".hyp")
+ local luafile = file.join(destination,"lang-"..mnemonic..".lua") -- suffix might change to llg
+
local topline = "% generated by mtxrun --script pattern --convert"
local banner = "% for comment and copyright, see " .. rmefile
- logs.simple("saving language data for %s",mnemonic)
+ report("saving language data for %s",mnemonic)
if not comment or comment == "" then comment = "% no comment" end
if not type(destination) == "string" then destination = "." end
+
+ local lines = string.splitlines(comment)
+ for i=1,#lines do
+ if not find(lines[i],"^%%") then
+ lines[i] = "% " .. lines[i]
+ end
+ end
+
+ local metadata = {
+ -- texcomment = comment,
+ texcomment = concat(lines,"\n"),
+ source = name,
+ mnemonic = mnemonic,
+ }
+
+ local patterndata, hyphenationdata
+ if nofpatternsnew > 0 then
+ patterndata = {
+ n = nofpatternsnew,
+ data = concat(patternsnew," ") or nil,
+ characters = concat(table.sortedkeys(pused),""),
+ minhyphenmin = 1, -- determined by pattern author
+ minhyphenmax = 1, -- determined by pattern author
+ }
+ else
+ patterndata = {
+ n = 0,
+ }
+ end
+ if nofhyphenationsnew > 0 then
+ hyphenationdata = {
+ n = nofhyphenationsnew,
+ data = concat(hyphenationsnew," "),
+ characters = concat(table.sortedkeys(hused),""),
+ }
+ else
+ hyphenationdata = {
+ n = 0,
+ }
+ end
+ local data = {
+ -- a prelude to language goodies, like we have font goodies and in
+ -- mkiv we can use this file directly
+ version = "1.001",
+ comment = topline,
+ metadata = metadata,
+ patterns = patterndata,
+ exceptions = hyphenationdata,
+ }
+
os.remove(rmefile)
os.remove(patfile)
os.remove(hypfile)
+ os.remove(luafile)
+
io.savedata(rmefile,format("%s\n\n%s",topline,comment))
- io.savedata(patfile,format("%s\n\n%s\n\n%% used: %s\n\n\\patterns{\n%s}",topline,banner,pu,table.concat(patterns,"\n")))
- io.savedata(hypfile,format("%s\n\n%s\n\n%% used: %s\n\n\\hyphenation{\n%s}",topline,banner,hu,table.concat(hyphenations,"\n")))
+ io.savedata(patfile,format("%s\n\n%s\n\n%% used: %s\n\n\\patterns{\n%s}",topline,banner,pu,concat(patternsold,"\n")))
+ io.savedata(hypfile,format("%s\n\n%s\n\n%% used: %s\n\n\\hyphenation{\n%s}",topline,banner,hu,concat(hyphenationsold,"\n")))
+ io.savedata(luafile,table.serialize(data,true))
end
end
function scripts.patterns.prepare()
- dofile(resolvers.find_file("char-def.lua"))
+ dofile(resolvers.findfile("char-def.lua"))
end
function scripts.patterns.check()
local path = environment.argument("path") or "."
- local found = false
local files = environment.files
+ local only = false
if #files > 0 then
- for i=1,#files do
- local name = files[i]
- logs.simple("checking language file %s", name)
- local okay = scripts.patterns.load(path,name,nil,not environment.argument("fast"))
- if #environment.files > 1 then
- logs.simple("")
- end
- end
- else
- for k, v in next, scripts.patterns.list do
- local mnemonic, name = v[1], v[2]
- logs.simple("checking language %s, file %s", mnemonic, name)
- local okay = scripts.patterns.load(path,name,mnemonic,not environment.argument("fast"))
+ only = table.tohash(files)
+ end
+ for k, v in next, scripts.patterns.list do
+ local mnemonic, name, ignored = v[1], v[2], v[4]
+ if not only or only[mnemonic] then
+ report("checking language %s, file %s", mnemonic, name)
+ local okay = scripts.patterns.load(path,name,mnemonic,ignored)
if not okay then
- logs.simple("there are errors that need to be fixed")
+ report("there are errors that need to be fixed")
end
- logs.simple("")
+ report()
end
end
end
@@ -319,36 +460,34 @@ end
function scripts.patterns.convert()
local path = environment.argument("path") or "."
if path == "" then
- logs.simple("provide sourcepath using --path ")
+ report("provide sourcepath using --path ")
else
local destination = environment.argument("destination") or "."
if path == destination then
- logs.simple("source path and destination path should differ (use --path and/or --destination)")
+ resport("source path and destination path should differ (use --path and/or --destination)")
else
+ local files = environment.files
+ local only = false
+ if #files > 0 then
+ only = table.tohash(files)
+ end
for k, v in next, scripts.patterns.list do
- local mnemonic, name = v[1], v[2]
- logs.simple("converting language %s, file %s", mnemonic, name)
- local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false)
- if okay then
- scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused)
- else
- logs.simple("convertion aborted due to error(s)")
+ local mnemonic, name, ignored = v[1], v[2], v[4]
+ if not only or only[mnemonic] then
+ report("converting language %s, file %s", mnemonic, name)
+ local okay, patternsnew, hyphenationsnew, patternsold, hyphenationsold, comment, stripped, pused, hused = scripts.patterns.load(path,name,mnemonic,ignored)
+ if okay then
+ scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenationsnew,patternsold,hyphenationsold,comment,stripped,pused,hused,ignored)
+ else
+ report("convertion aborted due to error(s)")
+ end
+ report()
end
- logs.simple("")
end
end
end
end
-logs.extendbanner("ConTeXt Pattern File Management 0.20",true)
-
-messages.help = [[
---convert generate context language files (mnemonic driven, if not given then all)
---check check pattern file (or those used by context when no file given)
-
---fast only report filenames, no lines
-]]
-
if environment.argument("check") then
scripts.patterns.prepare()
scripts.patterns.check()
@@ -356,11 +495,10 @@ elseif environment.argument("convert") then
scripts.patterns.prepare()
scripts.patterns.convert()
else
- logs.help(messages.help)
+ application.help()
end
-- mtxrun --script pattern --check hyph-*.tex
-- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns
--- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns
--- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination=e:/tmp/patterns
--- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/branches/luatex/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns
+-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns
+-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns