if not modules then modules = { } end modules ['mtx-patterns'] = { version = 1.001, comment = "companion to mtxrun.lua", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright = "PRAGMA ADE / ConTeXt Development Team", license = "see context related readme files" } local format = string.format scripts = scripts or { } scripts.patterns = scripts.patterns or { } scripts.patterns.list = { { "??", "hyph-ar.tex", "arabic" }, { "bg", "hyph-bg.tex", "bulgarian" }, { "ca", "hyph-ca.tex", "catalan" }, { "??", "hyph-cop.tex", "coptic" }, { "cs", "hyph-cs.tex", "czech" }, { "cy", "hyph-cy.tex", "welsh" }, { "da", "hyph-da.tex", "danish" }, { "deo", "hyph-de-1901.tex", "german, old spelling" }, { "de", "hyph-de-1996.tex", "german, new spelling" }, --~ { "??", "hyph-el-monoton.tex", "" }, --~ { "??", "hyph-el-polyton.tex", "" }, { "agr", "hyph-grc", "ancient greek" }, --~ { "???", "hyph-x-ibycus", "ancient greek in ibycus encoding" }, --~ { "gr", "", "" }, { "eo", "hyph-eo.tex", "esperanto" }, { "gb", "hyph-en-gb.tex", "british english" }, { "us", "hyph-en-us.tex", "american english" }, { "es", "hyph-es.tex", "spanish" }, { "et", "hyph-et.tex", "estonian" }, { "eu", "hyph-eu.tex", "basque" }, -- ba is Bashkir! { "fa", "hyph-fa.tex", "farsi" }, { "fi", "hyph-fi.tex", "finnish" }, { "fr", "hyph-fr.tex", "french" }, -- { "??", "hyph-ga.tex", "" }, -- { "??", "hyph-gl.tex", "" }, -- { "??", "hyph-grc.tex", "" }, { "hr", "hyph-hr.tex", "croatian" }, { "??", "hyph-hsb.tex", "upper sorbian" }, { "hu", "hyph-hu.tex", "hungarian" }, { "??", "hyph-ia.tex", "interlingua" }, { "??", "hyph-id.tex", "indonesian" }, { "is", "hyph-is.tex", "icelandic" }, { "it", "hyph-it.tex", "italian" }, { "la", "hyph-la.tex", "latin" }, { "lt", "hyph-lt.tex", "lithuanian" }, { "mn", "hyph-mn-cyrl.tex", "mongolian, cyrillic script" }, { "nb", "hyph-nb.tex", "norwegian bokmål" }, { "nl", "hyph-nl.tex", "dutch" }, { "nn", "hyph-nn.tex", "norwegian nynorsk" }, { "pl", "hyph-pl.tex", "polish" }, { "pt", "hyph-pt.tex", "portuguese" }, { "ro", "hyph-ro.tex", "romanian" }, { "ru", "hyph-ru.tex", "russian" }, { "sk", "hyph-sk.tex", "slovak" }, { "sl", "hyph-sl.tex", "slovenian" }, { "sr", "hyph-sr-cyrl.tex", "serbian" }, { "sv", "hyph-sv.tex", "swedish" }, { "tr", "hyph-tr.tex", "turkish" }, { "tk", "hyph-tk.tex", "turkman" }, { "uk", "hyph-uk.tex", "ukrainian" }, { "zh", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" }, } -- stripped down from lpeg example: local utf = unicode.utf8 local cont = lpeg.R("\128\191") -- continuation byte local utf8 = lpeg.R("\0\127") + lpeg.R("\194\223") * cont + lpeg.R("\224\239") * cont * cont + lpeg.R("\240\244") * cont * cont * cont local validutf = (utf8^0/function() return true end) * (lpeg.P(-1)/function() return false end) function utf.check(str) return lpeg.match(validutf,str) end local permitted_commands = table.tohash { "message", "endinput" } local permitted_characters = table.tohash { 0x0009, -- tab 0x0027, -- apostrofe 0x002D, -- hyphen 0x200C, -- } function scripts.patterns.load(path,name,mnemonic,fullcheck) local fullname = file.join(path,name) local data = io.loaddata(fullname) or "" local byte, char = utf.byte, utf.char if data ~= "" then data = data:gsub("([\n\r])\\input ([^ \n\r]+)", function(previous,subname) local subname = file.addsuffix(subname,"tex") local subfull = file.join(file.dirname(fullname),subname) local subdata = io.loaddata(subfull) or "" if subdata == "" then if mnemonic then logs.simple("no subfile %s for language %s",subname,mnemonic) else logs.simple("no subfile %s",name) end end return previous .. subdata end) local comment = data:match("^(.-)[\n\r]\\patterns") or "" local n, okay = 0, true local cd = characters.data for line in data:gmatch("[^ \n\r]+") do local ok = utf.check(line) n = n + 1 if not ok then okay = false line = line:gsub("%%","%%%%") if fullcheck then if mnemonic then logs.simple("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line) else logs.simple("invalid utf in file %s, line %s: %s",name,n,line) end else if mnemonic then logs.simple("file %s for %s contains invalid utf",name,mnemonic) else logs.simple("file %s contains invalid utf",name) end break end end end local c, h = { }, { } for line in data:gmatch("[^\n\r]+") do local txt, cmt = line:match("^(.-)%%(.*)$") if not txt then txt, cmt = line, "" end for s in txt:gmatch("\\([a-zA-Z]+)") do h[s] = (h[s] or 0) + 1 end for s in cmt:gmatch("\\([a-zA-Z]+)") do c[s] = (c[s] or 0) + 1 end end h.patterns = nil h.hyphenation = nil for k, v in next, h do if not permitted_commands[k] then okay = false end if mnemonic then logs.simple("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) else logs.simple("command \\%s found in file %s, n=%s",k,name,v) end end if not environment.argument("fast") then for k, v in next, c do if mnemonic then logs.simple("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) else logs.simple("command \\%s found in comment of file %s, n=%s",k,name,v) end end end data = data:gsub("%%.-[\n\r]","") data = data:gsub(" *[\n\r]+","\n") local patterns = data:match("\\patterns[%s]*{[%s]*(.-)[%s]*}") or "" local hyphenations = data:match("\\hyphenation[%s]*{[%s]*(.-)[%s]*}") or "" patterns = patterns:gsub(" +","\n") hyphenations = hyphenations:gsub(" +","\n") local p, h = { }, { } local pats, hyps = { } , { } local pused, hused = { } , { } local period = byte(".") for line in patterns:gmatch("[^ \n\r]+") do local ok = true for b in line:utfvalues() do if b == period then -- ok else local ct = cd[b].category if ct == "lu" or ct == "ll" then pused[char(b)] = true elseif ct == "nd" then -- ok else p[b] = (p[b] or 0) + 1 ok = false end end end if ok then pats[#pats+1] = line end end local hyphen = byte("-") for line in hyphenations:gmatch("[^ \n\r]+") do local ok = true for b in line:utfvalues() do if b == hyphen then -- ok else local ct = cd[b].category if ct == "lu" or ct == "ll" then hused[char(b)] = true else h[b] = (h[b] or 0) + 1 ok = false end end end if ok then hyps[#hyps+1] = line end end local stripped = { } for k, v in next, p do if mnemonic then logs.simple("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else logs.simple("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v) end if not permitted_characters[k] then okay = false else stripped[k] = true end end for k, v in next, h do if mnemonic then logs.simple("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) else logs.simple("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v) end if not permitted_characters[k] then okay = false else stripped[k] = true end end local stripset = "" for k, v in next, stripped do logs.simple("entries that contain character %s will be omitted",char(k)) stripset = stripset .. "%" .. char(k) end return okay, pats, hyps, comment, stripset, pused, hused else if mnemonic then logs.simple("no file %s for language %s",fullname,mnemonic) else logs.simple("no file %s",fullname) end return false, { }, { }, "", "", { }, { } end end function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) local nofpatterns = #patterns local nofhyphenations = #hyphenations local pu = table.concat(table.sortedkeys(pused), " ") local hu = table.concat(table.sortedkeys(hused), " ") logs.simple("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations) if mnemonic ~= "??" then local rmefile = file.join(destination,"lang-"..mnemonic..".rme") local patfile = file.join(destination,"lang-"..mnemonic..".pat") local hypfile = file.join(destination,"lang-"..mnemonic..".hyp") local topline = "% generated by mtxrun --script pattern --convert" local banner = "% for comment and copyright, see " .. rmefile logs.simple("saving language data for %s",mnemonic) if not comment or comment == "" then comment = "% no comment" end if not type(destination) == "string" then destination = "." end os.remove(rmefile) os.remove(patfile) os.remove(hypfile) io.savedata(rmefile,format("%s\n\n%s",topline,comment)) io.savedata(patfile,format("%s\n\n%s\n\n%% used: %s\n\n\\patterns{\n%s}",topline,banner,pu,table.concat(patterns,"\n"))) io.savedata(hypfile,format("%s\n\n%s\n\n%% used: %s\n\n\\hyphenation{\n%s}",topline,banner,hu,table.concat(hyphenations,"\n"))) end end function scripts.patterns.prepare() dofile(resolvers.find_file("char-def.lua")) end function scripts.patterns.check() local path = environment.argument("path") or "." local found = false local files = environment.files if #files > 0 then for i=1,#files do local name = files[i] logs.simple("checking language file %s", name) local okay = scripts.patterns.load(path,name,nil,not environment.argument("fast")) if #environment.files > 1 then logs.simple("") end end else for k, v in next, scripts.patterns.list do local mnemonic, name = v[1], v[2] logs.simple("checking language %s, file %s", mnemonic, name) local okay = scripts.patterns.load(path,name,mnemonic,not environment.argument("fast")) if not okay then logs.simple("there are errors that need to be fixed") end logs.simple("") end end end function scripts.patterns.convert() local path = environment.argument("path") or "." if path == "" then logs.simple("provide sourcepath using --path ") else local destination = environment.argument("destination") or "." if path == destination then logs.simple("source path and destination path should differ (use --path and/or --destination)") else for k, v in next, scripts.patterns.list do local mnemonic, name = v[1], v[2] logs.simple("converting language %s, file %s", mnemonic, name) local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) if okay then scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) else logs.simple("convertion aborted due to error(s)") end logs.simple("") end end end end logs.extendbanner("ConTeXt Pattern File Management 0.20",true) messages.help = [[ --convert generate context language files (mnemonic driven, if not given then all) --check check pattern file (or those used by context when no file given) --fast only report filenames, no lines ]] if environment.argument("check") then scripts.patterns.prepare() scripts.patterns.check() elseif environment.argument("convert") then scripts.patterns.prepare() scripts.patterns.convert() else logs.help(messages.help) end -- mtxrun --script pattern --check hyph-*.tex -- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns -- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns -- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination=e:/tmp/patterns -- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/branches/luatex/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns