diff options
Diffstat (limited to 'macros/luatex/generic/lualibs/lualibs-util-zip.lua')
-rw-r--r-- | macros/luatex/generic/lualibs/lualibs-util-zip.lua | 684 |
1 files changed, 684 insertions, 0 deletions
diff --git a/macros/luatex/generic/lualibs/lualibs-util-zip.lua b/macros/luatex/generic/lualibs/lualibs-util-zip.lua new file mode 100644 index 0000000000..bd8fdf287f --- /dev/null +++ b/macros/luatex/generic/lualibs/lualibs-util-zip.lua @@ -0,0 +1,684 @@ +if not modules then modules = { } end modules ['util-zip'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This module is mostly meant for relative simple zip and unzip tasks. We can read +-- and write zip files but with limitations. Performance is quite good and it makes +-- us independent of zip tools, which (for some reason) are not always installed. +-- +-- This is an lmtx module and at some point will be lmtx only but for a while we +-- keep some hybrid functionality. + +local type, tostring, tonumber = type, tostring, tonumber +local sort, concat = table.sort, table.concat + +local find, format, sub, gsub = string.find, string.format, string.sub, string.gsub +local osdate, ostime, osclock = os.date, os.time, os.clock +local ioopen = io.open +local loaddata, savedata = io.loaddata, io.savedata +local filejoin, isdir, dirname, mkdirs = file.join, lfs.isdir, file.dirname, dir.mkdirs +local suffix, suffixes = file.suffix, file.suffixes +local openfile = io.open + +gzip = gzip or { } -- so in luatex we keep the old ones too + +if not zlib then + zlib = xzip -- in luametatex we shadow the old one +elseif not xzip then + xzip = zlib +end + +local files = utilities.files +local openfile = files.open +local closefile = files.close +local readstring = files.readstring +local readcardinal2 = files.readcardinal2le +local readcardinal4 = files.readcardinal4le +local setposition = files.setposition +local getposition = files.getposition + +local band = bit32.band +local rshift = bit32.rshift +local lshift = bit32.lshift + +local zlibdecompress = zlib.decompress +local zlibdecompresssize = zlib.decompresssize +local zlibchecksum = zlib.crc32 + +if not CONTEXTLMTXMODE or CONTEXTLMTXMODE == 0 then + local cs = zlibchecksum + zlibchecksum = function(str,n) return cs(n or 0, str) end +end + +local decompress = function(source) return zlibdecompress (source,-15) end -- auto +local decompresssize = function(source,targetsize) return zlibdecompresssize(source,targetsize,-15) end -- auto +local calculatecrc = function(buffer,initial) return zlibchecksum (initial or 0,buffer) end + +local zipfiles = { } +utilities.zipfiles = zipfiles + +local openzipfile, closezipfile, unzipfile, foundzipfile, getziphash, getziplist do + + function openzipfile(name) + return { + name = name, + handle = openfile(name,0), + } + end + + local function collect(z) + if not z.list then + local list = { } + local hash = { } + local position = 0 + local index = 0 + local handle = z.handle + while true do + setposition(handle,position) + local signature = readstring(handle,4) + if signature == "PK\3\4" then + -- [local file header 1] + -- [encryption header 1] + -- [file data 1] + -- [data descriptor 1] + local version = readcardinal2(handle) + local flag = readcardinal2(handle) + local method = readcardinal2(handle) + local filetime = readcardinal2(handle) + local filedate = readcardinal2(handle) + local crc32 = readcardinal4(handle) + local compressed = readcardinal4(handle) + local uncompressed = readcardinal4(handle) + local namelength = readcardinal2(handle) + local extralength = readcardinal2(handle) + local filename = readstring(handle,namelength) + local descriptor = band(flag,8) ~= 0 + local encrypted = band(flag,1) ~= 0 + local acceptable = method == 0 or method == 8 + -- 30 bytes of header including the signature + local skipped = 0 + local size = 0 + if encrypted then + size = readcardinal2(handle) + skipbytes(size) + skipped = skipped + size + 2 + skipbytes(8) + skipped = skipped + 8 + size = readcardinal2(handle) + skipbytes(size) + skipped = skipped + size + 2 + size = readcardinal4(handle) + skipbytes(size) + skipped = skipped + size + 4 + size = readcardinal2(handle) + skipbytes(size) + skipped = skipped + size + 2 + end + position = position + 30 + namelength + extralength + skipped + if descriptor then + setposition(handle,position + compressed) + crc32 = readcardinal4(handle) + compressed = readcardinal4(handle) + uncompressed = readcardinal4(handle) + end + if acceptable then + index = index + 1 + local data = { + filename = filename, + index = index, + position = position, + method = method, + compressed = compressed, + uncompressed = uncompressed, + crc32 = crc32, + encrypted = encrypted, + } + hash[filename] = data + list[index] = data + else + -- maybe a warning when encrypted + end + position = position + compressed + else + break + end + z.list = list + z.hash = hash + end + end + end + + function getziplist(z) + local list = z.list + if not list then + collect(z) + end + return z.list + end + + function getziphash(z) + local hash = z.hash + if not hash then + collect(z) + end + return z.hash + end + + function foundzipfile(z,name) + return getziphash(z)[name] + end + + function closezipfile(z) + local f = z.handle + if f then + closefile(f) + z.handle = nil + end + end + + function unzipfile(z,filename,check) + local hash = z.hash + if not hash then + hash = zipfiles.hash(z) + end + local data = hash[filename] -- normalize + if not data then + -- lower and cleanup + -- only name + end + if data then + local handle = z.handle + local position = data.position + local compressed = data.compressed + if compressed > 0 then + setposition(handle,position) + local result = readstring(handle,compressed) + if data.method == 8 then + if decompresssize then + result = decompresssize(result,data.uncompressed) + else + result = decompress(result) + end + end + if check and data.crc32 ~= calculatecrc(result) then + print("checksum mismatch") + return "" + end + return result + else + return "" + end + end + end + + zipfiles.open = openzipfile + zipfiles.close = closezipfile + zipfiles.unzip = unzipfile + zipfiles.hash = getziphash + zipfiles.list = getziplist + zipfiles.found = foundzipfile + +end + +if xzip then -- flate then do + + local writecardinal1 = files.writebyte + local writecardinal2 = files.writecardinal2le + local writecardinal4 = files.writecardinal4le + + local logwriter = logs.writer + + local globpattern = dir.globpattern +-- local compress = flate.flate_compress +-- local checksum = flate.update_crc32 + local compress = xzip.compress + local checksum = xzip.crc32 + + -- local function fromdostime(dostime,dosdate) + -- return ostime { + -- year = (dosdate >> 9) + 1980, -- 25 .. 31 + -- month = (dosdate >> 5) & 0x0F, -- 21 .. 24 + -- day = (dosdate ) & 0x1F, -- 16 .. 20 + -- hour = (dostime >> 11) , -- 11 .. 15 + -- min = (dostime >> 5) & 0x3F, -- 5 .. 10 + -- sec = (dostime ) & 0x1F, -- 0 .. 4 + -- } + -- end + -- + -- local function todostime(time) + -- local t = osdate("*t",time) + -- return + -- ((t.year - 1980) << 9) + (t.month << 5) + t.day, + -- (t.hour << 11) + (t.min << 5) + (t.sec >> 1) + -- end + + local function fromdostime(dostime,dosdate) + return ostime { + year = rshift(dosdate, 9) + 1980, -- 25 .. 31 + month = band(rshift(dosdate, 5), 0x0F), -- 21 .. 24 + day = band( (dosdate ), 0x1F), -- 16 .. 20 + hour = band(rshift(dostime,11) ), -- 11 .. 15 + min = band(rshift(dostime, 5), 0x3F), -- 5 .. 10 + sec = band( (dostime ), 0x1F), -- 0 .. 4 + } + end + + local function todostime(time) + local t = osdate("*t",time) + return + lshift(t.year - 1980, 9) + lshift(t.month,5) + t.day, + lshift(t.hour ,11) + lshift(t.min ,5) + rshift(t.sec,1) + end + + local function openzip(filename,level,comment,verbose) + local f = ioopen(filename,"wb") + if f then + return { + filename = filename, + handle = f, + list = { }, + level = tonumber(level) or 3, + comment = tostring(comment), + verbose = verbose, + uncompressed = 0, + compressed = 0, + } + end + end + + local function writezip(z,name,data,level,time) + local f = z.handle + local list = z.list + local level = tonumber(level) or z.level or 3 + local method = 8 + local zipped = compress(data,level) + local checksum = checksum(data) + local verbose = z.verbose + -- + if not zipped then + method = 0 + zipped = data + end + -- + local start = f:seek() + local compressed = #zipped + local uncompressed = #data + -- + z.compressed = z.compressed + compressed + z.uncompressed = z.uncompressed + uncompressed + -- + if verbose then + local pct = 100 * compressed/uncompressed + if pct >= 100 then + logwriter(format("%10i %s",uncompressed,name)) + else + logwriter(format("%10i %02.1f %s",uncompressed,pct,name)) + end + end + -- + f:write("\x50\x4b\x03\x04") -- PK.. 0x04034b50 + -- + writecardinal2(f,0) -- minimum version + writecardinal2(f,0) -- flag + writecardinal2(f,method) -- method + writecardinal2(f,0) -- time + writecardinal2(f,0) -- date + writecardinal4(f,checksum) -- crc32 + writecardinal4(f,compressed) -- compressed + writecardinal4(f,uncompressed) -- uncompressed + writecardinal2(f,#name) -- namelength + writecardinal2(f,0) -- extralength + -- + f:write(name) -- name + f:write(zipped) + -- + list[#list+1] = { #zipped, #data, name, checksum, start, time or 0 } + end + + local function closezip(z) + local f = z.handle + local list = z.list + local comment = z.comment + local verbose = z.verbose + local count = #list + local start = f:seek() + -- + for i=1,count do + local l = list[i] + local compressed = l[1] + local uncompressed = l[2] + local name = l[3] + local checksum = l[4] + local start = l[5] + local time = l[6] + local date, time = todostime(time) + f:write('\x50\x4b\x01\x02') + writecardinal2(f,0) -- version made by + writecardinal2(f,0) -- version needed to extract + writecardinal2(f,0) -- flags + writecardinal2(f,8) -- method + writecardinal2(f,time) -- time + writecardinal2(f,date) -- date + writecardinal4(f,checksum) -- crc32 + writecardinal4(f,compressed) -- compressed + writecardinal4(f,uncompressed) -- uncompressed + writecardinal2(f,#name) -- namelength + writecardinal2(f,0) -- extralength + writecardinal2(f,0) -- commentlength + writecardinal2(f,0) -- nofdisks -- ? + writecardinal2(f,0) -- internal attr (type) + writecardinal4(f,0) -- external attr (mode) + writecardinal4(f,start) -- local offset + f:write(name) -- name + end + -- + local stop = f:seek() + local size = stop - start + -- + f:write('\x50\x4b\x05\x06') + writecardinal2(f,0) -- disk + writecardinal2(f,0) -- disks + writecardinal2(f,count) -- entries + writecardinal2(f,count) -- entries + writecardinal4(f,size) -- dir size + writecardinal4(f,start) -- dir offset + if type(comment) == "string" and comment ~= "" then + writecardinal2(f,#comment) -- comment length + f:write(comment) -- comemnt + else + writecardinal2(f,0) + end + -- + if verbose then + local compressed = z.compressed + local uncompressed = z.uncompressed + local filename = z.filename + -- + local pct = 100 * compressed/uncompressed + logwriter("") + if pct >= 100 then + logwriter(format("%10i %s",uncompressed,filename)) + else + logwriter(format("%10i %02.1f %s",uncompressed,pct,filename)) + end + end + -- + f:close() + end + + local function zipdir(zipname,path,level,verbose) + if type(zipname) == "table" then + verbose = zipname.verbose + level = zipname.level + path = zipname.path + zipname = zipname.zipname + end + if not zipname or zipname == "" then + return + end + if not path or path == "" then + path = "." + end + if not isdir(path) then + return + end + path = gsub(path,"\\+","/") + path = gsub(path,"/+","/") + local list = { } + local count = 0 + globpattern(path,"",true,function(name,size,time) + count = count + 1 + list[count] = { name, time } + end) + sort(list,function(a,b) + return a[1] < b[1] + end) + local zipf = openzip(zipname,level,comment,verbose) + if zipf then + local p = #path + 2 + for i=1,count do + local li = list[i] + local name = li[1] + local time = li[2] + local data = loaddata(name) + local name = sub(name,p,#name) + writezip(zipf,name,data,level,time,verbose) + end + closezip(zipf) + end + end + + local function unzipdir(zipname,path,verbose) + if type(zipname) == "table" then + verbose = zipname.verbose + path = zipname.path + zipname = zipname.zipname + end + if not zipname or zipname == "" then + return + end + if not path or path == "" then + path = "." + end + local z = openzipfile(zipname) + if z then + local list = getziplist(z) + if list then + local total = 0 + local count = #list + local step = number.idiv(count,10) + local done = 0 + local steps = verbose == "steps" + local time = steps and osclock() + for i=1,count do + local l = list[i] + local n = l.filename + local d = unzipfile(z,n) -- true for check + if d then + local p = filejoin(path,n) + if mkdirs(dirname(p)) then + if steps then + total = total + #d + done = done + 1 + if done >= step then + done = 0 + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",i,count,total,osclock()-time)) + end + elseif verbose then + logwriter(n) + end + savedata(p,d) + end + else + logwriter(format("problem with file %s",n)) + end + end + if steps then + logwriter(format("%4i files of %4i done, %10i bytes, %0.3f seconds",count,count,total,osclock()-time)) + end + closezipfile(z) + return true + else + closezipfile(z) + end + end + end + + zipfiles.zipdir = zipdir + zipfiles.unzipdir = unzipdir + +end + +-- todo: compress/decompress that work with offset in string + +-- We only have a few official methods here: +-- +-- local decompressed = gzip.load (filename) +-- local resultsize = gzip.save (filename,compresslevel) +-- local compressed = gzip.compress (str,compresslevel) +-- local decompressed = gzip.decompress (str) +-- local iscompressed = gzip.compressed (str) +-- local suffix, okay = gzip.suffix (filename) +-- +-- In LuaMetaTeX we have only xzip which implements a very few methods: +-- +-- compress (str,level,method,window,memory,strategy) +-- decompress (str,window) +-- adler32 (str,checksum) +-- crc32 (str,checksum) + +local pattern = "^\x1F\x8B\x08" +local gziplevel = 3 + +function gzip.suffix(filename) + local suffix, extra = suffixes(filename) + local gzipped = extra == "gz" + return suffix, gzipped +end + +function gzip.compressed(s) + return s and find(s,pattern) +end + +local getdecompressed +local putcompressed + +if gzip.compress then + + local gzipwindow = 15 + 16 -- +16: gzip, +32: gzip|zlib + + local compress = zlib.compress + local decompress = zlib.decompress + + getdecompressed = function(str) + return decompress(str,gzipwindow) -- pass offset + end + + putcompressed = function(str,level) + return compress(str,level or gziplevel,nil,gzipwindow) + end + +else + + -- Special window values are: flate: -15, zlib: 15, gzip : -15 + + local gzipwindow = -15 -- miniz needs this + local identifier = "\x1F\x8B" + + local compress = zlib.compress + local decompress = zlib.decompress + local zlibchecksum = zlib.crc32 + + if not CONTEXTLMTXMODE or CONTEXTLMTXMODE == 0 then + local cs = zlibchecksum + zlibchecksum = function(str,n) return cs(n or 0, str) end + end + + local streams = utilities.streams + local openstream = streams.openstring + local closestream = streams.close + local getposition = streams.getposition + local readbyte = streams.readbyte + local readcardinal4 = streams.readcardinal4le + local readcardinal2 = streams.readcardinal2le + local readstring = streams.readstring + local readcstring = streams.readcstring + local skipbytes = streams.skip + + local tocardinal1 = streams.tocardinal1 + local tocardinal4 = streams.tocardinal4le + + getdecompressed = function(str) + local s = openstream(str) + local identifier = readstring(s,2) + local method = readbyte(s,1) + local flags = readbyte(s,1) + local timestamp = readcardinal4(s) + local compression = readbyte(s,1) + local operating = readbyte(s,1) + -- local isjusttext = (flags & 0x01 ~= 0) and true or false + -- local extrasize = (flags & 0x04 ~= 0) and readcardinal2(s) or 0 + -- local filename = (flags & 0x08 ~= 0) and readcstring(s) or "" + -- local comment = (flags & 0x10 ~= 0) and readcstring(s) or "" + -- local checksum = (flags & 0x02 ~= 0) and readcardinal2(s) or 0 + local isjusttext = band(flags,0x01) ~= 0 and true or false + local extrasize = band(flags,0x04) ~= 0 and readcardinal2(s) or 0 + local filename = band(flags,0x08) ~= 0 and readcstring(s) or "" + local comment = band(flags,0x10) ~= 0 and readcstring(s) or "" + local checksum = band(flags,0x02) ~= 0 and readcardinal2(s) or 0 + local compressed = readstring(s,#str) + local data = decompress(compressed,gzipwindow) -- pass offset + return data + end + + putcompressed = function(str,level,originalname) + return concat { + identifier, -- 2 identifier + tocardinal1(0x08), -- 1 method + tocardinal1(0x08), -- 1 flags + tocardinal4(os.time()), -- 4 mtime + tocardinal1(0x02), -- 1 compression (2 or 4) + tocardinal1(0xFF), -- 1 operating + (originalname or "unknownname") .. "\0", + compress(str,level,nil,gzipwindow), + tocardinal4(zlibchecksum(str)), -- 4 + tocardinal4(#str), -- 4 + } + end + +end + +function gzip.load(filename) + local f = openfile(filename,"rb") + if not f then + -- invalid file + else + local data = f:read("*all") + f:close() + if data and data ~= "" then + if suffix(filename) == "gz" then + data = getdecompressed(data) + end + return data + end + end +end + +function gzip.save(filename,data,level,originalname) + if suffix(filename) ~= "gz" then + filename = filename .. ".gz" + end + local f = openfile(filename,"wb") + if f then + data = putcompressed(data or "",level or gziplevel,originalname) + f:write(data) + f:close() + return #data + end +end + +function gzip.compress(s,level) + if s and not find(s,pattern) then + if not level then + level = gziplevel + elseif level <= 0 then + return s + elseif level > 9 then + level = 9 + end + return putcompressed(s,level or gziplevel) or s + end +end + +function gzip.decompress(s) + if s and find(s,pattern) then + return getdecompressed(s) + else + return s + end +end + +-- return zipfiles |