summaryrefslogtreecommitdiff
path: root/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
diff options
context:
space:
mode:
Diffstat (limited to 'Master/texmf-dist/scripts/context/lua/mtx-patterns.lua')
-rw-r--r--Master/texmf-dist/scripts/context/lua/mtx-patterns.lua127
1 files changed, 111 insertions, 16 deletions
diff --git a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
index 7144da9b090..dca81568e41 100644
--- a/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
+++ b/Master/texmf-dist/scripts/context/lua/mtx-patterns.lua
@@ -27,6 +27,8 @@ local helpinfo = [[
<flag name="path"><short>source path where hyph-foo.tex files are stored</short></flag>
<flag name="destination"><short>destination path</short></flag>
<flag name="specification"><short>additional patterns: e.g.: =cy,hyph-cy,welsh</short></flag>
+ <flag name="compress"><short>compress data</short></flag>
+ <flag name="words"><short>update words in given file</short></flag>
</subcategory>
</category>
</flags>
@@ -148,10 +150,11 @@ scripts.patterns.list = {
{ "sv", "hyph-sv", "swedish" },
-- { "ta", "hyph-ta", "tamil" },
-- { "te", "hyph-te", "telugu" },
+ { "th", "hyph-th", "thai" },
{ "tk", "hyph-tk", "turkmen" },
{ "tr", "hyph-tr", "turkish" },
{ "uk", "hyph-uk", "ukrainian" },
- { "zh", "hyph-zh-latn", "zh-latn, chinese pinyin" },
+ { "zh", "hyph-zh-latn-pinyin","zh-latn, chinese pinyin" },
}
-- stripped down from lpeg example:
@@ -166,6 +169,7 @@ end
-- *.hyp.txt *.pat.txt *.lic.txt *.chr.txt
function scripts.patterns.load(path,name,mnemonic,ignored)
+ local basename = name
local fullname = file.join(path,name)
local texfile = addsuffix(fullname,"tex")
local hypfile = addsuffix(fullname,"hyp.txt")
@@ -190,7 +194,7 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
local subfull = file.join(file.dirname(texfile),subname)
local subdata = io.loaddata(subfull) or ""
if subdata == "" then
- report("no subfile %s",subname)
+ report("%s: no subfile %s",basename,subname)
end
return previous .. subdata
end)
@@ -219,7 +223,7 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
local line = splitdata[i]
if find(line,"%%") then
splitdata[i] = gsub(line,"%%.*$","")
- report("removing comment: %s",line)
+ report("%s: removing comment: %s",basename,line)
end
end
end
@@ -235,7 +239,7 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
local line = splitdata[i]
if find(line,"\\") then
splitdata[i] = ""
- report("removing line with command: %s",line)
+ report("%s: removing line with command: %s",basename,line)
end
end
end
@@ -251,7 +255,7 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
local ok = lpegmatch(validutf8,line)
if not ok then
splitdata[i] = ""
- report("removing line with invalid utf: %s",line)
+ report("%s: removing line with invalid utf: %s",basename,line)
end
end
-- check for commands being used in comments
@@ -275,15 +279,15 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
else
local cdb = cd[b]
if not cdb then
- report("no entry in chardata for character %s (0x%04X)",char(b),b)
+ report("%s: no entry in chardata for character %C",basename,b)
else
local ct = cdb.category
- if ct == "lu" or ct == "ll" then
+ if ct == "lu" or ct == "ll" or ct == "lo" or ct == "mn" then -- hm, really mn ?
used[char(b)] = true
elseif ct == "nd" then
-- number
else
- report("removing line with suspected utf character %s (0x%04X), category %s: %s",char(b),b,ct,line)
+ report("%s: removing line with suspected utf character %C, category %s: %s",basename,b,ct,line)
splitdata[i] = ""
break
end
@@ -296,7 +300,7 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
usedpatterncharactersnew = check(splitpatternsnew,byte("."))
usedhyphenationcharactersnew = check(splithyphenationsnew,byte("-"))
for k, v in next, stripped do
- report("entries that contain character %s (0x%04X) have been omitted",char(k),k)
+ report("%s: entries that contain character %C have been omitted",basename,k)
end
end
if okay then
@@ -327,11 +331,11 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
for i=1,#what do
local line = what[i]
if p and lpegmatch(p,line) then
- report("discarding conflicting pattern: %s",line)
+ report("%s: discarding conflicting pattern: %s",basename,line)
else -- we can speed this up by testing for replacements in the string
local l = lpegmatch(r,line)
if l ~= line then
- report("sanitizing pattern: %s -> %s (for old patterns)",line,l)
+ report("%s: sanitizing pattern: %s -> %s (for old patterns)",basename,line,l)
end
result[#result+1] = l
end
@@ -353,7 +357,7 @@ function scripts.patterns.load(path,name,mnemonic,ignored)
-- discard
elseif used[line] then
-- discard
- report("discarding duplicate pattern: %s",line)
+ report("%s: discarding duplicate pattern: %s",basename,line)
else
used[line] = true
collected[#collected+1] = line
@@ -411,6 +415,8 @@ function scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenation
if not comment or comment == "" then comment = "% no comment" end
if not type(destination) == "string" then destination = "." end
+ local compression = environment.arguments.compress and "zlib" or nil
+
local lines = string.splitlines(comment)
for i=1,#lines do
if not find(lines[i],"^%%") then
@@ -427,9 +433,12 @@ function scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenation
local patterndata, hyphenationdata
if nofpatternsnew > 0 then
+ local data = concat(patternsnew," ")
patterndata = {
n = nofpatternsnew,
- data = concat(patternsnew," ") or nil,
+ compression = compression,
+ length = #data,
+ data = compression and zlib.compress(data,9) or data,
characters = concat(table.sortedkeys(pusednew),""),
minhyphenmin = 1, -- determined by pattern author
minhyphenmax = 1, -- determined by pattern author
@@ -440,10 +449,13 @@ function scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenation
}
end
if nofhyphenationsnew > 0 then
+ local data = concat(hyphenationsnew," ")
hyphenationdata = {
- n = nofhyphenationsnew,
- data = concat(hyphenationsnew," "),
- characters = concat(table.sortedkeys(husednew),""),
+ n = nofhyphenationsnew,
+ compression = compression,
+ length = #data,
+ data = compression and zlib.compress(data,9) or data,
+ characters = concat(table.sortedkeys(husednew),""),
}
else
hyphenationdata = {
@@ -541,12 +553,83 @@ function scripts.patterns.convert()
end
end
+local function valid(filename)
+ local specification = table.load(filename)
+ if not specification then
+ return false
+ end
+ local lists = specification.lists
+ if not lists then
+ return false
+ end
+ return specification, lists
+end
+
+function scripts.patterns.words()
+ if environment.arguments.update then
+ local compress = environment.arguments.compress
+ for i=1,#environment.files do
+ local filename = environment.files[i]
+ local fullname = resolvers.findfile(filename)
+ if fullname and fullname ~= "" then
+ report("checking file %a",fullname)
+ local specification, lists = valid(fullname)
+ if specification and #lists> 0 then
+ report("updating %a of language %a",filename,specification.language)
+ for i=1,#lists do
+ local entry = lists[i]
+ local filename = entry.filename
+ if filename then
+ local fullname = resolvers.findfile(filename)
+ if fullname then
+ report("adding words from %a",fullname)
+ local data = io.loaddata(fullname) or ""
+ data = string.strip(data)
+ data = string.gsub(data,"%s+"," ")
+ if compress then
+ entry.data = zlib.compress(data,9)
+ entry.compression = "zlib"
+ entry.length = #data
+ else
+ entry.data = data
+ entry.compression = nil
+ entry.length = #data
+ end
+ else
+ entry.data = ""
+ entry.compression = nil
+ entry.length = 0
+ end
+ else
+ entry.data = ""
+ entry.compression = nil
+ entry.length = 0
+ end
+ end
+ specification.version = "1.00"
+ specification.timestamp = os.localtime()
+ report("updated file %a is saved",filename)
+ table.save(filename,specification)
+ else
+ report("no file %a",filename)
+ end
+ else
+ report("nothing done")
+ end
+ end
+ else
+ report("provide --update")
+ end
+end
+
if environment.argument("check") then
scripts.patterns.prepare()
scripts.patterns.check()
elseif environment.argument("convert") then
scripts.patterns.prepare()
scripts.patterns.convert()
+elseif environment.argument("words") then
+ scripts.patterns.words() -- for the moment here
elseif environment.argument("exporthelp") then
application.export(environment.argument("exporthelp"),environment.files[1])
else
@@ -557,3 +640,15 @@ end
-- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns
-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns
-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns
+
+-- copy /Y *.hyp e:\tex-context\tex\texmf-context\tex\context\patterns
+-- copy /Y *.pat e:\tex-context\tex\texmf-context\tex\context\patterns
+-- copy /Y *.rme e:\tex-context\tex\texmf-context\tex\context\patterns
+-- copy /Y *.lua e:\tex-context\tex\texmf-context\tex\context\patterns
+
+-- move /Y *.hyp e:\tex-context\tex\texmf-mine\tex\context\patterns
+-- move /Y *.pat e:\tex-context\tex\texmf-mine\tex\context\patterns
+-- move /Y *.rme e:\tex-context\tex\texmf-mine\tex\context\patterns
+-- move /Y *.lua e:\tex-context\tex\texmf-mine\tex\context\patterns
+
+-- mtxrun --script pattern --words --update word-th.lua --compress