diff options
author | Taco Hoekwater <taco@elvenkind.com> | 2011-06-01 08:54:21 +0000 |
---|---|---|
committer | Taco Hoekwater <taco@elvenkind.com> | 2011-06-01 08:54:21 +0000 |
commit | d7ccb42582f85acf30568913610ccf4d602023fb (patch) | |
tree | 7292e3545a420676878e7451b68892d360c62cb6 /Master/texmf-dist/scripts/context/lua/mtx-patterns.lua | |
parent | 2d62a6fe9b80def59c392268022f1f9a2d6e358f (diff) |
commit context 2011.05.18
git-svn-id: svn://tug.org/texlive/trunk@22719 c570f23f-e606-0410-a88d-b1316a301751
Diffstat (limited to 'Master/texmf-dist/scripts/context/lua/mtx-patterns.lua')
-rw-r--r-- | Master/texmf-dist/scripts/context/lua/mtx-patterns.lua | 660 |
1 files changed, 399 insertions, 261 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua index 293016991ff..bbf47513354 100644 --- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua +++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua @@ -6,312 +6,453 @@ if not modules then modules = { } end modules ['mtx-patterns'] = { license = "see context related readme files" } -local format = string.format +local format, find, concat, gsub, match, gmatch = string.format, string.find, table.concat, string.gsub, string.match, string.gmatch +local byte, char = utf.byte, utf.char +local addsuffix = file.addsuffix +local lpegmatch, validutf8 = lpeg.match, lpeg.patterns.validutf8 + +local helpinfo = [[ +--convert generate context language files (mnemonic driven, if not given then all) +--check check pattern file (or those used by context when no file given) +--path source path where hyph-foo.tex files are stored +--destination destination path + +examples of usage: + +mtxrun --script pattern --check hyph-*.tex +mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns +mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns +mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns +]] + +local application = logs.application { + name = "mtx-patterns", + banner = "ConTeXt Pattern File Management 0.20", + helpinfo = helpinfo, +} + +local report = application.report scripts = scripts or { } scripts.patterns = scripts.patterns or { } -scripts.patterns.list = { - { "??", "hyph-ar.tex", "arabic" }, - { "bg", "hyph-bg.tex", "bulgarian" }, - { "ca", "hyph-ca.tex", "catalan" }, - { "??", "hyph-cop.tex", "coptic" }, - { "cs", "hyph-cs.tex", "czech" }, - { "cy", "hyph-cy.tex", "welsh" }, - { "da", "hyph-da.tex", "danish" }, - { "deo", "hyph-de-1901.tex", "german, old spelling" }, - { "de", "hyph-de-1996.tex", "german, new spelling" }, ---~ { "??", "hyph-el-monoton.tex", "" }, ---~ { "??", "hyph-el-polyton.tex", "" }, - { "agr", "hyph-grc", "ancient greek" }, ---~ { "???", "hyph-x-ibycus", "ancient greek in ibycus encoding" }, ---~ { "gr", "", "" }, - { "eo", "hyph-eo.tex", "esperanto" }, - { "gb", "hyph-en-gb.tex", "british english" }, - { "us", "hyph-en-us.tex", "american english" }, - { "es", "hyph-es.tex", "spanish" }, - { "et", "hyph-et.tex", "estonian" }, - { "eu", "hyph-eu.tex", "basque" }, -- ba is Bashkir! - { "fa", "hyph-fa.tex", "farsi" }, - { "fi", "hyph-fi.tex", "finnish" }, - { "fr", "hyph-fr.tex", "french" }, --- { "??", "hyph-ga.tex", "" }, --- { "??", "hyph-gl.tex", "" }, --- { "??", "hyph-grc.tex", "" }, - { "hr", "hyph-hr.tex", "croatian" }, - { "??", "hyph-hsb.tex", "upper sorbian" }, - { "hu", "hyph-hu.tex", "hungarian" }, - { "??", "hyph-ia.tex", "interlingua" }, - { "??", "hyph-id.tex", "indonesian" }, - { "is", "hyph-is.tex", "icelandic" }, - { "it", "hyph-it.tex", "italian" }, - { "la", "hyph-la.tex", "latin" }, - { "lt", "hyph-lt.tex", "lithuanian" }, - { "mn", "hyph-mn-cyrl.tex", "mongolian, cyrillic script" }, - { "nb", "hyph-nb.tex", "norwegian bokmål" }, - { "nl", "hyph-nl.tex", "dutch" }, - { "nn", "hyph-nn.tex", "norwegian nynorsk" }, - { "pl", "hyph-pl.tex", "polish" }, - { "pt", "hyph-pt.tex", "portuguese" }, - { "ro", "hyph-ro.tex", "romanian" }, - { "ru", "hyph-ru.tex", "russian" }, - { "sk", "hyph-sk.tex", "slovak" }, - { "sl", "hyph-sl.tex", "slovenian" }, - { "sr", "hyph-sr-cyrl.tex", "serbian" }, - { "sv", "hyph-sv.tex", "swedish" }, - { "tr", "hyph-tr.tex", "turkish" }, - { "tk", "hyph-tk.tex", "turkman" }, - { "uk", "hyph-uk.tex", "ukrainian" }, - { "zh", "hyph-zh-latn.tex", "zh-latn, chinese Pinyin" }, +local permitted_characters = table.tohash { + 0x0009, -- tab + 0x0027, -- apostrofe + 0x02BC, -- modifier apostrofe (used in greek) + 0x002D, -- hyphen + 0x200C, -- zwnj + 0x2019, -- quote right + 0x1FBD, -- greek, but no letter: symbol modifier + 0x1FBF, -- greek, but no letter: symbol modifier } +local ignored_ancient_greek = table.tohash { + 0x1FD3, -- greekiotadialytikatonos (also 0x0390) + 0x1FE3, -- greekupsilondialytikatonos (also 0x03B0) + 0x1FBD, -- greek, but no letter: symbol modifier + 0x1FBF, -- greek, but no letter: symbol modifier + 0x03F2, -- greeksigmalunate + 0x02BC, -- modifier apostrofe) +} --- stripped down from lpeg example: +local ignored_french = table.tohash { + 0x02BC, -- modifier apostrofe +} -local utf = unicode.utf8 +local replaced_whatever = { + [char(0x2019)] = char(0x0027) +} -local cont = lpeg.R("\128\191") -- continuation byte +scripts.patterns.list = { + { "af", "hyph-af", "afrikaans" }, + -- { "ar", "hyph-ar", "arabic" }, + -- { "as", "hyph-as", "assamese" }, + { "bg", "hyph-bg", "bulgarian" }, + -- { "bn", "hyph-bn", "bengali" }, + { "ca", "hyph-ca", "catalan" }, + -- { "??", "hyph-cop", "coptic" }, + { "cs", "hyph-cs", "czech" }, + { "cy", "hyph-cy", "welsh" }, + { "da", "hyph-da", "danish" }, + { "deo", "hyph-de-1901", "german, old spelling" }, + { "de", "hyph-de-1996", "german, new spelling" }, + -- { "??", "hyph-de-ch-1901", "swiss german" }, + -- { "??", "hyph-el-monoton", "greek" }, + -- { "gr", "hyph-el-polyton", "greek" }, + { "agr", "hyph-grc", "ancient greek", ignored_ancient_greek }, + { "gb", "hyph-en-gb", "british english" }, + { "us", "hyph-en-us", "american english" }, + -- { "eo", "hyph-eo", "esperanto" }, + { "es", "hyph-es", "spanish" }, + { "et", "hyph-et", "estonian" }, + { "eu", "hyph-eu", "basque" }, + -- { "fa", "hyph-fa", "farsi" }, + { "fi", "hyph-fi", "finnish" }, + { "fr", "hyph-fr", "french", ignored_french }, + -- { "??", "hyph-ga", "irish" }, + -- { "??", "hyph-gl", "galician" }, + -- { "gu", "hyph-gu", "gujarati" }, + -- { "hi", "hyph-hi", "hindi" }, + { "hr", "hyph-hr", "croatian" }, + -- { "??", "hyph-hsb", "upper sorbian" }, + { "hu", "hyph-hu", "hungarian" }, + -- { "hy", "hyph-hy", "armenian" }, + -- { "??", "hyph-ia", "interlingua" }, + -- { "??", "hyph-id", "indonesian" }, + { "is", "hyph-is", "icelandic" }, + { "it", "hyph-it", "italian" }, + -- { "??", "hyph-kmr", "kurmanji" }, + -- { "kn", "hyph-kn", "kannada" }, + { "la", "hyph-la", "latin" }, + -- { "lo", "hyph-lo", "lao" }, + { "lt", "hyph-lt", "lithuanian" }, + { "lv", "hyph-lv", "latvian" }, + -- { "ml", "hyph-ml", "..." }, + { "mn", "hyph-mn-cyrl", "mongolian, cyrillic script" }, + -- { "mr", "hyph-mr", "..." }, + { "nb", "hyph-nb", "norwegian bokmål" }, + { "nl", "hyph-nl", "dutch" }, + { "nn", "hyph-nn", "norwegian nynorsk" }, + -- { "or", "hyph-or", "oriya" }, + -- { "pa", "hyph-pa", "panjabi" }, + -- { "", "hyph-", "" }, + { "pl", "hyph-pl", "polish" }, + { "pt", "hyph-pt", "portuguese" }, + { "ro", "hyph-ro", "romanian" }, + { "ru", "hyph-ru", "russian" }, + -- { "sa", "hyph-sa", "sanskrit" }, + { "sk", "hyph-sk", "slovak" }, + { "sl", "hyph-sl", "slovenian" }, + { "sr", "hyph-sr-cyrl", "serbian" }, + -- { "sr", "hyph-sr-latn", "serbian" }, + { "sv", "hyph-sv", "swedish" }, + -- { "ta", "hyph-ta", "tamil" }, + -- { "te", "hyph-te", "telugu" }, + { "tk", "hyph-tk", "turkmen" }, + { "tr", "hyph-tr", "turkish" }, + { "uk", "hyph-uk", "ukrainian" }, + { "zh", "hyph-zh-latn", "zh-latn, chinese pinyin" }, +} -local utf8 = lpeg.R("\0\127") - + lpeg.R("\194\223") * cont - + lpeg.R("\224\239") * cont * cont - + lpeg.R("\240\244") * cont * cont * cont +-- stripped down from lpeg example: -local validutf = (utf8^0/function() return true end) * (lpeg.P(-1)/function() return false end) +local utf = unicode.utf8 function utf.check(str) - return lpeg.match(validutf,str) + return lpeg.match(lpeg.patterns.validutf8,str) end -local permitted_commands = table.tohash { - "message", - "endinput" -} +-- *.tex +-- *.hyp.txt *.pat.txt *.lic.txt *.chr.txt -local permitted_characters = table.tohash { - 0x0009, -- tab - 0x0027, -- apostrofe - 0x002D, -- hyphen - 0x200C, -- -} - -function scripts.patterns.load(path,name,mnemonic,fullcheck) +function scripts.patterns.load(path,name,mnemonic,ignored) local fullname = file.join(path,name) - local data = io.loaddata(fullname) or "" - local byte, char = utf.byte, utf.char - if data ~= "" then - data = data:gsub("([\n\r])\\input ([^ \n\r]+)", function(previous,subname) - local subname = file.addsuffix(subname,"tex") - local subfull = file.join(file.dirname(fullname),subname) - local subdata = io.loaddata(subfull) or "" - if subdata == "" then - if mnemonic then - logs.simple("no subfile %s for language %s",subname,mnemonic) - else - logs.simple("no subfile %s",name) + local texfile = addsuffix(fullname,"tex") + local hypfile = addsuffix(fullname,"hyp.txt") + local patfile = addsuffix(fullname,"pat.txt") + local licfile = addsuffix(fullname,"lic.txt") + -- local chrfile = addsuffix(fullname,"chr.txt") + local okay = true + local hyphenations, patterns, comment, stripset = "", "", "", "" + local splitpatternsnew, splithyphenationsnew = { }, { } + local splitpatternsold, splithyphenationsold = { }, { } + local usedpatterncharacters, usedhyphenationcharacters = { }, { } + if lfs.isfile(patfile) then + report("using txt files %s.[hyp|pat|lic].txt",name) + comment, patterns, hyphenations = io.loaddata(licfile) or "", io.loaddata(patfile) or "", io.loaddata(hypfile) or "" + hypfile, patfile, licfile = hypfile, patfile, licfile + elseif lfs.isfile(texfile) then + report("using tex file %s.txt",name) + local data = io.loaddata(texfile) or "" + if data ~= "" then + data = gsub(data,"([\n\r])\\input ([^ \n\r]+)", function(previous,subname) + local subname = addsuffix(subname,"tex") + local subfull = file.join(file.dirname(texfile),subname) + local subdata = io.loaddata(subfull) or "" + if subdata == "" then + report("no subfile %s",subname) end - end - return previous .. subdata - end) - local comment = data:match("^(.-)[\n\r]\\patterns") or "" - local n, okay = 0, true - local cd = characters.data - for line in data:gmatch("[^ \n\r]+") do - local ok = utf.check(line) - n = n + 1 - if not ok then - okay = false - line = line:gsub("%%","%%%%") - if fullcheck then - if mnemonic then - logs.simple("invalid utf in language %s, file %s, line %s: %s",mnemonic,name,n,line) - else - logs.simple("invalid utf in file %s, line %s: %s",name,n,line) - end - else - if mnemonic then - logs.simple("file %s for %s contains invalid utf",name,mnemonic) - else - logs.simple("file %s contains invalid utf",name) + return previous .. subdata + end) + data = gsub(data,"%%.-[\n\r]","") + data = gsub(data," *[\n\r]+","\n") + patterns = match(data,"\\patterns[%s]*{[%s]*(.-)[%s]*}") or "" + hyphenations = match(data,"\\hyphenation[%s]*{[%s]*(.-)[%s]*}") or "" + comment = match(data,"^(.-)[\n\r]\\patterns") or "" + else + okay = false + end + else + okay = false + end + if okay then + -- split into lines + local how = lpeg.patterns.whitespace^1 + splitpatternsnew = lpeg.split(how,patterns) + splithyphenationsnew = lpeg.split(how,hyphenations) + end + if okay then + -- remove comments + local function check(data,splitdata,name) + if find(data,"%%") then + for i=1,#splitdata do + local line = splitdata[i] + if find(line,"%%") then + splitdata[i] = gsub(line,"%%.*$","") + report("removing comment: %s",line) end - break end end end - local c, h = { }, { } - for line in data:gmatch("[^\n\r]+") do - local txt, cmt = line:match("^(.-)%%(.*)$") - if not txt then - txt, cmt = line, "" - end - for s in txt:gmatch("\\([a-zA-Z]+)") do - h[s] = (h[s] or 0) + 1 - end - for s in cmt:gmatch("\\([a-zA-Z]+)") do - c[s] = (c[s] or 0) + 1 - end - end - h.patterns = nil - h.hyphenation = nil - for k, v in next, h do - if not permitted_commands[k] then okay = false end - if mnemonic then - logs.simple("command \\%s found in language %s, file %s, n=%s",k,mnemonic,name,v) - else - logs.simple("command \\%s found in file %s, n=%s",k,name,v) + check(patterns,splitpatternsnew,patfile) + check(hyphenations,splithyphenationsnew,hypfile) + end + if okay then + -- remove lines with commands + local function check(data,splitdata,name) + if find(data,"\\") then + for i=1,#splitdata do + local line = splitdata[i] + if find(line,"\\") then + splitdata[i] = "" + report("removing line with command: %s",line) + end + end end end - if not environment.argument("fast") then - for k, v in next, c do - if mnemonic then - logs.simple("command \\%s found in comment of language %s, file %s, n=%s",k,mnemonic,name,v) - else - logs.simple("command \\%s found in comment of file %s, n=%s",k,name,v) + check(patterns,splitpatternsnew,patfile) + check(hyphenations,splithyphenationsnew,hypfile) + end + if okay then + -- check for valid utf + local function check(data,splitdata,name) + for i=1,#splitdata do + local line = splitdata[i] + local ok = lpegmatch(validutf8,line) + if not ok then + splitdata[i] = "" + report("removing line with invalid utf: %s",line) end end + -- check for commands being used in comments end - data = data:gsub("%%.-[\n\r]","") - data = data:gsub(" *[\n\r]+","\n") - local patterns = data:match("\\patterns[%s]*{[%s]*(.-)[%s]*}") or "" - local hyphenations = data:match("\\hyphenation[%s]*{[%s]*(.-)[%s]*}") or "" - patterns = patterns:gsub(" +","\n") - hyphenations = hyphenations:gsub(" +","\n") - local p, h = { }, { } - local pats, hyps = { } , { } - local pused, hused = { } , { } - local period = byte(".") - for line in patterns:gmatch("[^ \n\r]+") do - local ok = true - for b in line:utfvalues() do - if b == period then - -- ok - else - local ct = cd[b].category - if ct == "lu" or ct == "ll" then - pused[char(b)] = true - elseif ct == "nd" then - -- ok + check(patterns,splitpatternsnew,patfile) + check(hyphenations,splithyphenationsnew,hypfile) + end + if okay then + -- remove funny lines + local cd = characters.data + local stripped = { } + local function check(splitdata,special,name) + local used = { } + for i=1,#splitdata do + local line = splitdata[i] + for b in line:utfvalues() do -- could be an lpeg + if b == special then + -- not registered + elseif permitted_characters[b] then + used[char(b)] = true else - p[b] = (p[b] or 0) + 1 - ok = false + local cdb = cd[b] + if not cdb then + report("no entry in chardata for character %s (0x%04X)",char(b),b) + else + local ct = cd[b].category + if ct == "lu" or ct == "ll" then + used[char(b)] = true + elseif ct == "nd" then + -- number + else + report("removing line with suspected utf character %s (0x%04X), category %s: %s",char(b),b,ct,line) + splitdata[i] = "" + break + end + end end end end - if ok then - pats[#pats+1] = line - end + return used end - local hyphen = byte("-") - for line in hyphenations:gmatch("[^ \n\r]+") do - local ok = true - for b in line:utfvalues() do - if b == hyphen then - -- ok - else - local ct = cd[b].category - if ct == "lu" or ct == "ll" then - hused[char(b)] = true + usedpatterncharacters = check(splitpatternsnew,byte(".")) + usedhyphenationcharacters = check(splithyphenationsnew,byte("-")) + for k, v in next, stripped do + report("entries that contain character %s (0x%04X) have been omitted",char(k),k) + end + end + if okay then + local function stripped(what,ignored) + -- ignored (per language) + local p = nil + if ignored then + for k, v in next, ignored do + if p then + p = p + lpeg.P(char(k)) else - h[b] = (h[b] or 0) + 1 - ok = false + p = lpeg.P(char(k)) end end + p = lpeg.P{ p + 1 * lpeg.V(1) } -- anywhere end - if ok then - hyps[#hyps+1] = line - end - end - local stripped = { } - for k, v in next, p do - if mnemonic then - logs.simple("invalid character %s (0x%04X) in patterns of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) - else - logs.simple("invalid character %s (0x%04X) in patterns of file %s, n=%s",char(k),k,name,v) + -- replaced (all languages) + local r = nil + for k, v in next, replaced_whatever do + if r then + r = r + lpeg.P(k)/v + else + r = lpeg.P(k)/v + end end - if not permitted_characters[k] then - okay = false - else - stripped[k] = true + r = lpeg.Cs((r + 1)^0) + local result = { } + for i=1,#what do + local line = what[i] + if p and lpegmatch(p,line) then + report("discarding conflicting pattern: %s",line) + else -- we can speed this up by testing for replacements in the string + local l = lpegmatch(r,line) + if l ~= line then + report("sanitizing pattern: %s -> %s (for old patterns)",line,l) + end + result[#result+1] = l + end end + return result end - for k, v in next, h do - if mnemonic then - logs.simple("invalid character %s (0x%04X) in exceptions of language %s, file %s, n=%s",char(k),k,mnemonic,name,v) - else - logs.simple("invalid character %s (0x%04X) in exceptions of file %s, n=%s",char(k),k,name,v) - end - if not permitted_characters[k] then - okay = false - else - stripped[k] = true + + splitpatternsold = stripped(splitpatternsnew,ignored) + splithyphenationsold = stripped(splithyphenationsnew,ignored) + + end + if okay then + -- discarding duplicates + local function check(data,splitdata,name) + local used, collected = { }, { } + for i=1,#splitdata do + local line = splitdata[i] + if line == "" then + -- discard + elseif used[line] then + -- discard + repo("discarding duplicate pattern: %s",line) + else + used[line] = true + collected[#collected+1] = line + end end + return collected end - local stripset = "" - for k, v in next, stripped do - logs.simple("entries that contain character %s will be omitted",char(k)) - stripset = stripset .. "%" .. char(k) - end - return okay, pats, hyps, comment, stripset, pused, hused - else - if mnemonic then - logs.simple("no file %s for language %s",fullname,mnemonic) - else - logs.simple("no file %s",fullname) - end - return false, { }, { }, "", "", { }, { } + splitpatternsnew = check(patterns,splitpatternsnew,patfile) + splithyphenationsnew = check(hyphenations,splithyphenationsnew,hypfile) + splitpatternsold = check(patterns,splitpatternsold,patfile) + splithyphenationsold = check(hyphenations,splithyphenationsold,hypfile) + end + if not okay then + report("no valid file %s.*",name) end + return okay, splitpatternsnew, splithyphenationsnew, splitpatternsold, splithyphenationsold, comment, stripset, usedpatterncharacters, usedhyphenationcharacters end -function scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) - local nofpatterns = #patterns - local nofhyphenations = #hyphenations - local pu = table.concat(table.sortedkeys(pused), " ") - local hu = table.concat(table.sortedkeys(hused), " ") - logs.simple("language %s has %s patterns and %s exceptions",mnemonic,nofpatterns,nofhyphenations) +function scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenationsnew,patternsold,hyphenationsold,comment,stripped,pused,hused,ignored) + local nofpatternsnew, nofhyphenationsnew = #patternsnew, #hyphenationsnew + local nofpatternsold, nofhyphenationsold = #patternsold, #hyphenationsold + report("language %s has %s old and %s new patterns and %s old and %s new exceptions",mnemonic,nofpatternsold,nofpatternsnew,nofhyphenationsold,nofhyphenationsnew) if mnemonic ~= "??" then + local pu = concat(table.sortedkeys(pused), " ") + local hu = concat(table.sortedkeys(hused), " ") + local rmefile = file.join(destination,"lang-"..mnemonic..".rme") local patfile = file.join(destination,"lang-"..mnemonic..".pat") local hypfile = file.join(destination,"lang-"..mnemonic..".hyp") + local luafile = file.join(destination,"lang-"..mnemonic..".lua") -- suffix might change to llg + local topline = "% generated by mtxrun --script pattern --convert" local banner = "% for comment and copyright, see " .. rmefile - logs.simple("saving language data for %s",mnemonic) + report("saving language data for %s",mnemonic) if not comment or comment == "" then comment = "% no comment" end if not type(destination) == "string" then destination = "." end + + local lines = string.splitlines(comment) + for i=1,#lines do + if not find(lines[i],"^%%") then + lines[i] = "% " .. lines[i] + end + end + + local metadata = { + -- texcomment = comment, + texcomment = concat(lines,"\n"), + source = name, + mnemonic = mnemonic, + } + + local patterndata, hyphenationdata + if nofpatternsnew > 0 then + patterndata = { + n = nofpatternsnew, + data = concat(patternsnew," ") or nil, + characters = concat(table.sortedkeys(pused),""), + minhyphenmin = 1, -- determined by pattern author + minhyphenmax = 1, -- determined by pattern author + } + else + patterndata = { + n = 0, + } + end + if nofhyphenationsnew > 0 then + hyphenationdata = { + n = nofhyphenationsnew, + data = concat(hyphenationsnew," "), + characters = concat(table.sortedkeys(hused),""), + } + else + hyphenationdata = { + n = 0, + } + end + local data = { + -- a prelude to language goodies, like we have font goodies and in + -- mkiv we can use this file directly + version = "1.001", + comment = topline, + metadata = metadata, + patterns = patterndata, + exceptions = hyphenationdata, + } + os.remove(rmefile) os.remove(patfile) os.remove(hypfile) + os.remove(luafile) + io.savedata(rmefile,format("%s\n\n%s",topline,comment)) - io.savedata(patfile,format("%s\n\n%s\n\n%% used: %s\n\n\\patterns{\n%s}",topline,banner,pu,table.concat(patterns,"\n"))) - io.savedata(hypfile,format("%s\n\n%s\n\n%% used: %s\n\n\\hyphenation{\n%s}",topline,banner,hu,table.concat(hyphenations,"\n"))) + io.savedata(patfile,format("%s\n\n%s\n\n%% used: %s\n\n\\patterns{\n%s}",topline,banner,pu,concat(patternsold,"\n"))) + io.savedata(hypfile,format("%s\n\n%s\n\n%% used: %s\n\n\\hyphenation{\n%s}",topline,banner,hu,concat(hyphenationsold,"\n"))) + io.savedata(luafile,table.serialize(data,true)) end end function scripts.patterns.prepare() - dofile(resolvers.find_file("char-def.lua")) + dofile(resolvers.findfile("char-def.lua")) end function scripts.patterns.check() local path = environment.argument("path") or "." - local found = false local files = environment.files + local only = false if #files > 0 then - for i=1,#files do - local name = files[i] - logs.simple("checking language file %s", name) - local okay = scripts.patterns.load(path,name,nil,not environment.argument("fast")) - if #environment.files > 1 then - logs.simple("") - end - end - else - for k, v in next, scripts.patterns.list do - local mnemonic, name = v[1], v[2] - logs.simple("checking language %s, file %s", mnemonic, name) - local okay = scripts.patterns.load(path,name,mnemonic,not environment.argument("fast")) + only = table.tohash(files) + end + for k, v in next, scripts.patterns.list do + local mnemonic, name, ignored = v[1], v[2], v[4] + if not only or only[mnemonic] then + report("checking language %s, file %s", mnemonic, name) + local okay = scripts.patterns.load(path,name,mnemonic,ignored) if not okay then - logs.simple("there are errors that need to be fixed") + report("there are errors that need to be fixed") end - logs.simple("") + report() end end end @@ -319,36 +460,34 @@ end function scripts.patterns.convert() local path = environment.argument("path") or "." if path == "" then - logs.simple("provide sourcepath using --path ") + report("provide sourcepath using --path ") else local destination = environment.argument("destination") or "." if path == destination then - logs.simple("source path and destination path should differ (use --path and/or --destination)") + resport("source path and destination path should differ (use --path and/or --destination)") else + local files = environment.files + local only = false + if #files > 0 then + only = table.tohash(files) + end for k, v in next, scripts.patterns.list do - local mnemonic, name = v[1], v[2] - logs.simple("converting language %s, file %s", mnemonic, name) - local okay, patterns, hyphenations, comment, stripped, pused, hused = scripts.patterns.load(path,name,false) - if okay then - scripts.patterns.save(destination,mnemonic,patterns,hyphenations,comment,stripped,pused,hused) - else - logs.simple("convertion aborted due to error(s)") + local mnemonic, name, ignored = v[1], v[2], v[4] + if not only or only[mnemonic] then + report("converting language %s, file %s", mnemonic, name) + local okay, patternsnew, hyphenationsnew, patternsold, hyphenationsold, comment, stripped, pused, hused = scripts.patterns.load(path,name,mnemonic,ignored) + if okay then + scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenationsnew,patternsold,hyphenationsold,comment,stripped,pused,hused,ignored) + else + report("convertion aborted due to error(s)") + end + report() end - logs.simple("") end end end end -logs.extendbanner("ConTeXt Pattern File Management 0.20",true) - -messages.help = [[ ---convert generate context language files (mnemonic driven, if not given then all) ---check check pattern file (or those used by context when no file given) - ---fast only report filenames, no lines -]] - if environment.argument("check") then scripts.patterns.prepare() scripts.patterns.check() @@ -356,11 +495,10 @@ elseif environment.argument("convert") then scripts.patterns.prepare() scripts.patterns.convert() else - logs.help(messages.help) + application.help() end -- mtxrun --script pattern --check hyph-*.tex -- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --check --fast --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --destination=e:/tmp/patterns --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/branches/luatex/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns |