diff options
Diffstat (limited to 'tex/context/base/mkiv/util-zip.lua')
| -rw-r--r-- | tex/context/base/mkiv/util-zip.lua | 549 |
1 files changed, 549 insertions, 0 deletions
diff --git a/tex/context/base/mkiv/util-zip.lua b/tex/context/base/mkiv/util-zip.lua new file mode 100644 index 000000000..7d252a74f --- /dev/null +++ b/tex/context/base/mkiv/util-zip.lua @@ -0,0 +1,549 @@ +if not modules then modules = { } end modules ['util-zip'] = { + version = 1.001, + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This module is mostly meant for relative simple zip and unzip tasks. We can read +-- and write zip files but with limitations. Performance is quite good and it makes +-- us independent of zip tools, which (for some reason) are not always installed. +-- +-- This is an lmtx module and at some point will be lmtx only but for a while we +-- keep some hybrid functionality. + +local type, tostring, tonumber = type, tostring, tonumber +local sort = table.sort + +local find, format, sub, gsub = string.find, string.format, string.sub, string.gsub +local osdate, ostime = os.date, os.time +local ioopen = io.open +local loaddata, savedata = io.loaddata, io.savedata +local filejoin, isdir, dirname, mkdirs = file.join, lfs.isdir, file.dirname, dir.mkdirs + +local files = utilities.files +local openfile = files.open +local closefile = files.close +local readstring = files.readstring +local readcardinal2 = files.readcardinal2le +local readcardinal4 = files.readcardinal4le +local setposition = files.setposition +local getposition = files.getposition + +local band = bit32.band +local rshift = bit32.rshift +local lshift = bit32.lshift + +local decompress, calculatecrc + +if flate then + + decompress = flate.flate_decompress + calculatecrc = flate.update_crc32 + +else + + local zlibdecompress = zlib.decompress + local zlibchecksum = zlib.crc32 + + decompress = function(source,targetsize) + local target = zlibdecompress(source,-15) + if target then + return target + else + return false, 1 + end + end + + calculatecrc = function(buffer,initial) + return zlibchecksum(initial or 0,buffer) + end + +end + +local zipfiles = { } +utilities.zipfiles = zipfiles + +local openzipfile, closezipfile, unzipfile, foundzipfile, getziphash, getziplist do + + function openzipfile(name) + return { + name = name, + handle = openfile(name,0), + } + end + + local function collect(z) + if not z.list then + local list = { } + local hash = { } + local position = 0 + local index = 0 + local handle = z.handle + while true do + setposition(handle,position) + local signature = readstring(handle,4) + if signature == "PK\3\4" then + -- [local file header 1] + -- [encryption header 1] + -- [file data 1] + -- [data descriptor 1] + local version = readcardinal2(handle) + local flag = readcardinal2(handle) + local method = readcardinal2(handle) + local filetime = readcardinal2(handle) + local filedate = readcardinal2(handle) + local crc32 = readcardinal4(handle) + local compressed = readcardinal4(handle) + local uncompressed = readcardinal4(handle) + local namelength = readcardinal2(handle) + local extralength = readcardinal2(handle) + local filename = readstring(handle,namelength) + local descriptor = band(flag,8) ~= 0 + local encrypted = band(flag,1) ~= 0 + local acceptable = method == 0 or method == 8 + -- 30 bytes of header including the signature + local skipped = 0 + local size = 0 + if encrypted then + size = readcardinal2(handle) + skipbytes(size) + skipped = skipped + size + 2 + skipbytes(8) + skipped = skipped + 8 + size = readcardinal2(handle) + skipbytes(size) + skipped = skipped + size + 2 + size = readcardinal4(handle) + skipbytes(size) + skipped = skipped + size + 4 + size = readcardinal2(handle) + skipbytes(size) + skipped = skipped + size + 2 + end + position = position + 30 + namelength + extralength + skipped + if descriptor then + setposition(handle,position + compressed) + crc32 = readcardinal4(handle) + compressed = readcardinal4(handle) + uncompressed = readcardinal4(handle) + end + if acceptable then + index = index + 1 + local data = { + filename = filename, + index = index, + position = position, + method = method, + compressed = compressed, + uncompressed = uncompressed, + crc32 = crc32, + encrypted = encrypted, + } + hash[filename] = data + list[index] = data + else + -- maybe a warning when encrypted + end + position = position + compressed + else + break + end + z.list = list + z.hash = hash + end + end + end + + function getziplist(z) + local list = z.list + if not list then + collect(z) + end + return z.list + end + + function getziphash(z) + local hash = z.hash + if not hash then + collect(z) + end + return z.hash + end + + function foundzipfile(z,name) + return getziphash(z)[name] + end + + function closezipfile(z) + local f = z.handle + if f then + closefile(f) + z.handle = nil + end + end + + function unzipfile(z,filename,check) + local hash = z.hash + if not hash then + hash = zipfiles.hash(z) + end + local data = hash[filename] -- normalize + if not data then + -- lower and cleanup + -- only name + end + if data then + local handle = z.handle + local position = data.position + local compressed = data.compressed + if compressed > 0 then + setposition(handle,position) + local result = readstring(handle,compressed) + if data.method == 8 then + result = decompress(result,data.uncompressed) + end + if check and data.crc32 ~= calculatecrc(result) then + print("checksum mismatch") + return "" + end + return result + else + return "" + end + end + end + + zipfiles.open = openzipfile + zipfiles.close = closezipfile + zipfiles.unzip = unzipfile + zipfiles.hash = getziphash + zipfiles.list = getziplist + zipfiles.found = foundzipfile + +end + +if flate then do + + local writecardinal1 = files.writebyte + local writecardinal2 = files.writecardinal2le + local writecardinal4 = files.writecardinal4le + + local logwriter = logs.writer + + local globpattern = dir.globpattern + local compress = flate.flate_compress + local checksum = flate.update_crc32 + + -- local function fromdostime(dostime,dosdate) + -- return ostime { + -- year = (dosdate >> 9) + 1980, -- 25 .. 31 + -- month = (dosdate >> 5) & 0x0F, -- 21 .. 24 + -- day = (dosdate ) & 0x1F, -- 16 .. 20 + -- hour = (dostime >> 11) , -- 11 .. 15 + -- min = (dostime >> 5) & 0x3F, -- 5 .. 10 + -- sec = (dostime ) & 0x1F, -- 0 .. 4 + -- } + -- end + -- + -- local function todostime(time) + -- local t = osdate("*t",time) + -- return + -- ((t.year - 1980) << 9) + (t.month << 5) + t.day, + -- (t.hour << 11) + (t.min << 5) + (t.sec >> 1) + -- end + + local function fromdostime(dostime,dosdate) + return ostime { + year = rshift(dosdate, 9) + 1980, -- 25 .. 31 + month = band(rshift(dosdate, 5), 0x0F), -- 21 .. 24 + day = band( (dosdate ), 0x1F), -- 16 .. 20 + hour = band(rshift(dostime,11) ), -- 11 .. 15 + min = band(rshift(dostime, 5), 0x3F), -- 5 .. 10 + sec = band( (dostime ), 0x1F), -- 0 .. 4 + } + end + + local function todostime(time) + local t = osdate("*t",time) + return + lshift(t.year - 1980, 9) + lshift(t.month,5) + t.day, + lshift(t.hour ,11) + lshift(t.min ,5) + rshift(t.sec,1) + end + + local function openzip(filename,level,comment,verbose) + local f = ioopen(filename,"wb") + if f then + return { + filename = filename, + handle = f, + list = { }, + level = tonumber(level) or 3, + comment = tostring(comment), + verbose = verbose, + uncompressed = 0, + compressed = 0, + } + end + end + + local function writezip(z,name,data,level,time) + local f = z.handle + local list = z.list + local level = tonumber(level) or z.level or 3 + local method = 8 + local zipped = compress(data,level) + local checksum = checksum(data) + local verbose = z.verbose + -- + if not zipped then + method = 0 + zipped = data + end + -- + local start = f:seek() + local compressed = #zipped + local uncompressed = #data + -- + z.compressed = z.compressed + compressed + z.uncompressed = z.uncompressed + uncompressed + -- + if verbose then + local pct = 100 * compressed/uncompressed + if pct >= 100 then + logwriter(format("%10i %s",uncompressed,name)) + else + logwriter(format("%10i %02.1f %s",uncompressed,pct,name)) + end + end + -- + f:write("\x50\x4b\x03\x04") -- PK.. 0x04034b50 + -- + writecardinal2(f,0) -- minimum version + writecardinal2(f,0) -- flag + writecardinal2(f,method) -- method + writecardinal2(f,0) -- time + writecardinal2(f,0) -- date + writecardinal4(f,checksum) -- crc32 + writecardinal4(f,compressed) -- compressed + writecardinal4(f,uncompressed) -- uncompressed + writecardinal2(f,#name) -- namelength + writecardinal2(f,0) -- extralength + -- + f:write(name) -- name + f:write(zipped) + -- + list[#list+1] = { #zipped, #data, name, checksum, start, time or 0 } + end + + local function closezip(z) + local f = z.handle + local list = z.list + local comment = z.comment + local verbose = z.verbose + local count = #list + local start = f:seek() + -- + for i=1,count do + local l = list[i] + local compressed = l[1] + local uncompressed = l[2] + local name = l[3] + local checksum = l[4] + local start = l[5] + local time = l[6] + local date, time = todostime(time) + f:write('\x50\x4b\x01\x02') + writecardinal2(f,0) -- version made by + writecardinal2(f,0) -- version needed to extract + writecardinal2(f,0) -- flags + writecardinal2(f,8) -- method + writecardinal2(f,time) -- time + writecardinal2(f,date) -- date + writecardinal4(f,checksum) -- crc32 + writecardinal4(f,compressed) -- compressed + writecardinal4(f,uncompressed) -- uncompressed + writecardinal2(f,#name) -- namelength + writecardinal2(f,0) -- extralength + writecardinal2(f,0) -- commentlength + writecardinal2(f,0) -- nofdisks -- ? + writecardinal2(f,0) -- internal attr (type) + writecardinal4(f,0) -- external attr (mode) + writecardinal4(f,start) -- local offset + f:write(name) -- name + end + -- + local stop = f:seek() + local size = stop - start + -- + f:write('\x50\x4b\x05\x06') + writecardinal2(f,0) -- disk + writecardinal2(f,0) -- disks + writecardinal2(f,count) -- entries + writecardinal2(f,count) -- entries + writecardinal4(f,size) -- dir size + writecardinal4(f,start) -- dir offset + if type(comment) == "string" and comment ~= "" then + writecardinal2(f,#comment) -- comment length + f:write(comment) -- comemnt + else + writecardinal2(f,0) + end + -- + if verbose then + local compressed = z.compressed + local uncompressed = z.uncompressed + local filename = z.filename + -- + local pct = 100 * compressed/uncompressed + logwriter("") + if pct >= 100 then + logwriter(format("%10i %s",uncompressed,filename)) + else + logwriter(format("%10i %02.1f %s",uncompressed,pct,filename)) + end + end + -- + f:close() + end + + local function zipdir(zipname,path,level,verbose) + if type(zipname) == "table" then + verbose = zipname.verbose + level = zipname.level + path = zipname.path + zipname = zipname.zipname + end + if not zipname or zipname == "" then + return + end + if not path or path == "" then + path = "." + end + if not isdir(path) then + return + end + path = gsub(path,"\\+","/") + path = gsub(path,"/+","/") + local list = { } + local count = 0 + globpattern(path,"",true,function(name,size,time) + count = count + 1 + list[count] = { name, time } + end) + sort(list,function(a,b) + return a[1] < b[1] + end) + local zipf = openzip(zipname,level,comment,verbose) + if zipf then + local p = #path + 2 + for i=1,count do + local li = list[i] + local name = li[1] + local time = li[2] + local data = loaddata(name) + local name = sub(name,p,#name) + writezip(zipf,name,data,level,time,verbose) + end + closezip(zipf) + end + end + + local function unzipdir(zipname,path,verbose) + if type(zipname) == "table" then + verbose = zipname.verbose + path = zipname.path + zipname = zipname.zipname + end + if not zipname or zipname == "" then + return + end + if not path or path == "" then + path = "." + end + local z = openzipfile(zipname) + if z then + local list = getziplist(z) + if list then + local total = 0 + local count = #list + local step = number.idiv(count,10) + local done = 0 + for i=1,count do + local l = list[i] + local n = l.filename + local d = unzipfile(z,n) -- true for check + local p = filejoin(path,n) + if mkdirs(dirname(p)) then + if verbose == "steps" then + total = total + #d + done = done + 1 + if done >= step then + done = 0 + logwriter(format("%4i files of %4i done, %10i bytes",i,count,total)) + end + elseif verbose then + logwriter(n) + end + savedata(p,d) + end + end + if verbose == "steps" then + logwriter(format("%4i files of %4i done, %10i bytes",count,count,total)) + end + closezipfile(z) + return true + else + closezipfile(z) + end + end + end + + zipfiles.zipdir = zipdir + zipfiles.unzipdir = unzipdir + +end end + +if flate then + + local streams = utilities.streams + local openfile = streams.open + local closestream = streams.close + local setposition = streams.setposition + local getsize = streams.size + local readcardinal4 = streams.readcardinal4le + local getstring = streams.getstring + local decompress = flate.gz_decompress + + -- id1=1 id2=1 method=1 flags=1 mtime=4(le) extra=1 os=1 + -- flags:8 comment=...<nul> flags:4 name=...<nul> flags:2 extra=...<nul> flags:1 crc=2 + -- data:? + -- crc=4 size=4 + + function zipfiles.gunzipfile(filename) + local strm = openfile(filename) + if strm then + setposition(strm,getsize(strm) - 4 + 1) + local size = readcardinal4(strm) + local data = decompress(getstring(strm),size) + closestream(strm) + return data + end + end + +elseif gzip then + + local openfile = gzip.open + + function zipfiles.gunzipfile(filename) + local g = openfile(filename,"rb") + if g then + local d = g:read("*a") + d:close() + return d + end + end + +end + +return zipfiles |