diff options
Diffstat (limited to 'scripts/context/lua/mtxrun.lua')
| -rw-r--r-- | scripts/context/lua/mtxrun.lua | 3122 |
1 files changed, 2399 insertions, 723 deletions
diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 7b711a88d..873770cac 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 4734, stripped down to: 2626 +-- original size: 5125, stripped down to: 2881 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -162,6 +162,283 @@ if flush then local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end end +FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +if not FFISUPPORTED then + local okay;okay,ffi=pcall(require,"ffi") + FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +end +if not FFISUPPORTED then + ffi=nil +elseif not ffi.number then + ffi.number=tonumber +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 9667, stripped down to: 6678 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b==false then + return nil + elseif b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] or false +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end end -- of closure @@ -170,7 +447,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10949, stripped down to: 8037 +-- original size: 10587, stripped down to: 7815 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -460,7 +737,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38185, stripped down to: 20990 +-- original size: 37748, stripped down to: 20111 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -552,6 +829,7 @@ patterns.nonwhitespace=nonwhitespace local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local nospacer=Cs((whitespace^1/""+nonwhitespace^1)^0) local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) @@ -561,6 +839,7 @@ local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.nospacer=nospacer patterns.b_collapser=b_collapser patterns.m_collapser=m_collapser patterns.e_collapser=e_collapser @@ -1027,27 +1306,7 @@ function lpeg.append(list,pp,delayed,checked) end local p_false=P(false) local p_true=P(true) -local function make(t) - local function making(t) - local p=p_false - local keys=sortedkeys(t) - for i=1,#keys do - local k=keys[i] - if k~="" then - local v=t[k] - if v==true then - p=p+P(k)*p_true - elseif v==false then - else - p=p+P(k)*making(v) - end - end - end - if t[""] then - p=p+p_true - end - return p - end +local function make(t,rest) local p=p_false local keys=sortedkeys(t) for i=1,#keys do @@ -1058,10 +1317,13 @@ local function make(t) p=p+P(k)*p_true elseif v==false then else - p=p+P(k)*making(v) + p=p+P(k)*make(v,v[""]) end end end + if rest then + p=p+p_true + end return p end local function collapse(t,x) @@ -1264,7 +1526,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 372, stripped down to: 329 +-- original size: 361, stripped down to: 322 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1283,7 +1545,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5983, stripped down to: 2959 +-- original size: 6419, stripped down to: 3339 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1321,22 +1583,26 @@ end local stripper=patterns.stripper local fullstripper=patterns.fullstripper local collapser=patterns.collapser +local nospacer=patterns.nospacer local longtostring=patterns.longtostring function string.strip(str) - return lpegmatch(stripper,str) or "" + return str and lpegmatch(stripper,str) or "" end function string.fullstrip(str) - return lpegmatch(fullstripper,str) or "" + return str and lpegmatch(fullstripper,str) or "" end function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" + return str and lpegmatch(collapser,str) or "" +end +function string.nospaces(str) + return str and lpegmatch(nospacer,str) or "" end function string.longtostring(str) - return lpegmatch(longtostring,str) or "" + return str and lpegmatch(longtostring,str) or "" end local pattern=P(" ")^0*P(-1) function string.is_empty(str) - if str=="" then + if not str or str=="" then return true else return lpegmatch(pattern,str) and true or false @@ -1381,6 +1647,21 @@ function string.tformat(fmt,...) end string.quote=string.quoted string.unquote=string.unquoted +if not string.bytetable then + local limit=5000 + function string.bytetable(str) + local n=#str + if n>limit then + local t={ byte(str,1,limit) } + for i=limit+1,n do + t[i]=byte(str,i) + end + return t + else + return { byte(str,1,n) } + end + end +end end -- of closure @@ -1389,7 +1670,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 36997, stripped down to: 22376 +-- original size: 39608, stripped down to: 23165 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1716,19 +1997,23 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions,metacheck local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } -local function simple_table(t) +local function is_simple_table(t,hexify) local nt=#t if nt>0 then local n=0 for _,v in next,t do n=n+1 + if type(v)=="table" then + return nil + end end + local haszero=rawget(t,0) if n==nt then local tt={} for i=1,nt do @@ -1738,10 +2023,10 @@ local function simple_table(t) if hexify then tt[i]=format("0x%X",v) else - tt[i]=tostring(v) + tt[i]=v end elseif tv=="string" then - tt[i]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then tt[i]=v and "true" or "false" else @@ -1749,10 +2034,32 @@ local function simple_table(t) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + if hexify then + tt[i+1]=format("0x%X",v) + else + tt[i+1]=v + end + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil end +table.is_simple_table=is_simple_table local propername=patterns.propername local function dummy() end local function do_serialize(root,name,depth,level,indexed) @@ -1786,7 +2093,7 @@ local function do_serialize(root,name,depth,level,indexed) if compact then last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -1814,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -1851,6 +2158,7 @@ local function do_serialize(root,name,depth,level,indexed) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) @@ -1873,6 +2181,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,v)) else @@ -1888,13 +2197,14 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={},",depth,k and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then if tk=="number" then if hexify then @@ -1904,6 +2214,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={ %s },",depth,k and "true" or "false",concat(st,", "))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else @@ -1924,6 +2235,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%s,",depth,tostring(k),v and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%s,",depth,k,v and "true" or "false")) else @@ -1940,6 +2252,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=load(%q),",depth,k and "true" or "false",f)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=load(%q),",depth,k,f)) else @@ -1955,6 +2268,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",tostring(v))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,tostring(v))) else @@ -1976,6 +2290,7 @@ local function serialize(_handle,root,name,specification) functions=specification.functions compact=specification.compact inline=specification.inline and compact + metacheck=specification.metacheck if functions==nil then functions=true end @@ -1985,6 +2300,9 @@ local function serialize(_handle,root,name,specification) if inline==nil then inline=compact end + if metacheck==nil then + metacheck=true + end else noquotes=false hexify=false @@ -1992,6 +2310,7 @@ local function serialize(_handle,root,name,specification) compact=true inline=true functions=true + metacheck=true end if tname=="string" then if name=="return" then @@ -2015,7 +2334,7 @@ local function serialize(_handle,root,name,specification) handle("t={") end if root then - if getmetatable(root) then + if metacheck and getmetatable(root) then local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end @@ -2091,6 +2410,38 @@ local function flattened(t,f,depth) return f end table.flattened=flattened +local function collapsed(t,f,h) + if f==nil then + f={} + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsed(v,f,h) + elseif not h[v] then + f[#f+1]=v + h[v]=true + end + end + return f +end +local function collapsedhash(t,h) + if h==nil then + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsedhash(v,h) + else + h[v]=true + end + end + return h +end +table.collapsed=collapsed +table.collapsedhash=collapsedhash local function unnest(t,f) if not f then f={} @@ -2197,6 +2548,12 @@ function table.swapped(t,s) end return n end +function table.hashed(t) + for i=1,#t do + t[t[i]]=i + end + return t +end function table.mirrored(t) local n={} for k,v in next,t do @@ -2365,7 +2722,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 9001, stripped down to: 6512 +-- original size: 11790, stripped down to: 6961 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2375,6 +2732,7 @@ if not modules then modules={} end modules ['l-io']={ license="see context related readme files" } local io=io +local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor @@ -2384,50 +2742,56 @@ if string.find(os.getenv("PATH"),";",1,true) then else io.fileseparator,io.pathseparator="/",":" end -local function readall(f) - return f:read("*all") -end +local large=2^24 +local medium=large/16 +local small=medium/8 local function readall(f) local size=f:seek("end") - if size==0 then - return "" - elseif size<1024*1024 then + if size>0 then f:seek("set",0) - return f:read('*all') - else - local done=f:seek("set",0) - local step - if size<1024*1024 then - step=1024*1024 - elseif size>16*1024*1024 then - step=16*1024*1024 - else - step=floor(size/(1024*1024))*1024*1024/8 - end - local data={} - while true do - local r=f:read(step) - if not r then - return concat(data) - else - data[#data+1]=r - end - end + return f:read(size) + else + return "" end end io.readall=readall function io.loaddata(filename,textmode) - local f=io.open(filename,(textmode and 'r') or 'rb') + local f=open(filename,(textmode and 'r') or 'rb') if f then - local data=readall(f) + local size=f:seek("end") + local data=nil + if size>0 then + f:seek("set",0) + data=f:read(size) + end f:close() - if #data>0 then - return data + return data + end +end +function io.copydata(source,target,action) + local f=open(source,"rb") + if f then + local g=open(target,"wb") + if g then + local size=f:seek("end") + if size>0 then + f:seek("set",0) + local data=f:read(size) + if action then + data=action(data) + end + if data then + g:write(data) + end + end + g:close() end + f:close() + flush() end end function io.savedata(filename,data,joiner) - local f=io.open(filename,"wb") + local f=open(filename,"wb") if f then if type(data)=="table" then f:write(concat(data,joiner or "")) @@ -2437,40 +2801,70 @@ function io.savedata(filename,data,joiner) f:write(data or "") end f:close() - io.flush() + flush() return true else return false end end -function io.loadlines(filename,n) - local f=io.open(filename,'r') - if not f then - elseif n then - local lines={} - for i=1,n do - local line=f:read("*lines") - if line then - lines[#lines+1]=line - else - break +if fio and fio.readline then + local readline=fio.readline + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=readline(f) + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=readline(f) + f:close() + if line and #line>0 then + return line end end - f:close() - lines=concat(lines,"\n") - if #lines>0 then - return lines - end - else - local line=f:read("*line") or "" - f:close() - if #line>0 then - return line + end +else + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end end end end function io.loadchunk(filename,n) - local f=io.open(filename,'rb') + local f=open(filename,'rb') if f then local data=f:read(n or 1024) f:close() @@ -2480,7 +2874,7 @@ function io.loadchunk(filename,n) end end function io.exists(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return false else @@ -2489,7 +2883,7 @@ function io.exists(filename) end end function io.size(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return 0 else @@ -2498,11 +2892,11 @@ function io.size(filename) return s end end -function io.noflines(f) +local function noflines(f) if type(f)=="string" then - local f=io.open(filename) + local f=open(filename) if f then - local n=f and io.noflines(f) or 0 + local n=f and noflines(f) or 0 f:close() return n else @@ -2517,6 +2911,7 @@ function io.noflines(f) return n end end +io.noflines=noflines local nextchar={ [ 4]=function(f) return f:read(1,1,1,1) @@ -2594,16 +2989,16 @@ function io.bytes(f,n) end function io.ask(question,default,options) while true do - io.write(question) + write(question) if options then - io.write(format(" [%s]",concat(options,"|"))) + write(format(" [%s]",concat(options,"|"))) end if default then - io.write(format(" [%s]",default)) + write(format(" [%s]",default)) end - io.write(format(" ")) - io.flush() - local answer=io.read() + write(format(" ")) + flush() + local answer=read() answer=gsub(answer,"^%s*(.*)%s*$","%1") if answer=="" and default then return default @@ -2625,7 +3020,7 @@ function io.ask(question,default,options) end end end -local function readnumber(f,n,m) +local function readnumber(f,n,m) if m then f:seek("set",n) n=m @@ -2634,31 +3029,31 @@ local function readnumber(f,n,m) return byte(f:read(1)) elseif n==2 then local a,b=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==3 then local a,b,c=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==4 then local a,b,c,d=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==8 then local a,b=readnumber(f,4),readnumber(f,4) - return 256*a+b + return 0x100*a+b elseif n==12 then local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-2 then local b,a=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==-3 then local c,b,a=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-4 then local d,c,b,a=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==-8 then local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) - return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + return 0x100000000000000*a+0x1000000000000*b+0x10000000000*c+0x100000000*d+0x1000000*e+0x10000*f+0x100*g+h else return 0 end @@ -2680,7 +3075,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 5146, stripped down to: 2933 +-- original size: 5358, stripped down to: 3177 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2693,6 +3088,7 @@ local tostring,tonumber=tostring,tonumber local format,floor,match,rep=string.format,math.floor,string.match,string.rep local concat,insert=table.concat,table.insert local lpegmatch=lpeg.match +local floor=math.floor number=number or {} local number=number if bit32 then @@ -2817,6 +3213,26 @@ end function number.bits(n) return { bits(n,1) } end +function number.bytetodecimal(b) + local d=floor(b*100/255+0.5) + if d>100 then + return 100 + elseif d<-100 then + return -100 + else + return d + end +end +function number.decimaltobyte(d) + local b=floor(d*255/100+0.5) + if b>255 then + return 255 + elseif b<-255 then + return -255 + else + return b + end +end end -- of closure @@ -2825,7 +3241,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 2010, stripped down to: 1186 +-- original size: 1923, stripped down to: 1133 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2898,7 +3314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16390, stripped down to: 9734 +-- original size: 16268, stripped down to: 9246 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2974,7 +3390,7 @@ if not os.__getenv__ then end local execute=os.execute local iopopen=io.popen -function os.resultof(command) +local function resultof(command) local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" @@ -2984,9 +3400,13 @@ function os.resultof(command) return "" end end +os.resultof=resultof +function os.pipeto(command) + return iopopen(command,"w") +end if not io.fileseparator then if find(os.getenv("PATH"),";",1,true) then - io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "windows" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" end @@ -3029,17 +3449,6 @@ setmetatable(os,{ __index=function(t,k) return r and r(t,k) or nil end }) local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" -local function guess() - local architecture=os.resultof("uname -m") or "" - if architecture~="" then - return architecture - end - architecture=os.getenv("HOSTTYPE") or "" - if architecture~="" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end if platform~="" then os.platform=platform elseif os.type=="windows" then @@ -3056,7 +3465,7 @@ elseif os.type=="windows" then end elseif name=="linux" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="linux-64" elseif find(architecture,"ppc",1,true) then @@ -3070,7 +3479,7 @@ elseif name=="linux" then end elseif name=="macosx" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + local platform,architecture="",resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" elseif find(architecture,"i386",1,true) then @@ -3086,7 +3495,7 @@ elseif name=="macosx" then end elseif name=="sunos" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"sparc",1,true) then platform="solaris-sparc" else @@ -3098,7 +3507,7 @@ elseif name=="sunos" then end elseif name=="freebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else @@ -3110,7 +3519,7 @@ elseif name=="freebsd" then end elseif name=="kfreebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else @@ -3241,7 +3650,7 @@ if not os.sleep then end end local function isleapyear(year) - return (year%400==0) or ((year%100~=0) and (year%4==0)) + return (year%4==0) and (year%100~=0 or year%400==0) end os.isleapyear=isleapyear local days={ 31,28,31,30,31,30,31,31,30,31,30,31 } @@ -3280,7 +3689,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21648, stripped down to: 10238 +-- original size: 20997, stripped down to: 9986 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3617,13 +4026,15 @@ function file.robustname(str,strict) end end end -file.readdata=io.loaddata -file.savedata=io.savedata +local loaddata=io.loaddata +local savedata=io.savedata +file.readdata=loaddata +file.savedata=savedata function file.copy(oldname,newname) if oldname and newname then - local data=io.loaddata(oldname) + local data=loaddata(oldname) if data and data~="" then - file.savedata(newname,data) + savedata(newname,data) end end end @@ -3660,7 +4071,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1265, stripped down to: 1038 +-- original size: 1211, stripped down to: 1002 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3714,7 +4125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3355, stripped down to: 2321 +-- original size: 3309, stripped down to: 2314 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3744,6 +4155,8 @@ do if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + md5.sumhexa=md5.hex + md5.sumHEXA=md5.HEX end end function file.needsupdating(oldname,newname,threshold) @@ -3802,7 +4215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12897, stripped down to: 5882 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4019,7 +4432,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 17358, stripped down to: 11378 +-- original size: 17703, stripped down to: 11691 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4283,6 +4696,31 @@ local function globfiles(path,recurse,func,files) return files end dir.globfiles=globfiles +local function globdirs(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + if recurse then + globdirs(path.."/"..name,recurse,func,files) + end + end + end + end + end + return files +end +dir.globdirs=globdirs function dir.ls(pattern) return concat(glob(pattern),"\n") end @@ -4447,9 +4885,13 @@ end file.expandname=dir.expandname local stack={} function dir.push(newdir) - insert(stack,currentdir()) + local curdir=currentdir() + insert(stack,curdir) if newdir and newdir~="" then chdir(newdir) + return newdir + else + return curdir end end function dir.pop() @@ -4484,7 +4926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1919, stripped down to: 1621 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4556,7 +4998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 38699, stripped down to: 16321 +-- original size: 38263, stripped down to: 16330 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -5167,6 +5609,23 @@ function utf.chrlen(u) (u<0xFC and 5) or (u<0xFE and 6) or 0 end +local extract=bit32.extract +local char=string.char +function unicode.toutf32string(n) + if n<=0xFF then + return + char(n).."\000\000\000" + elseif n<=0xFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8)).."\000\000" + elseif n<=0xFFFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8)).."\000" + else + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8))..char(extract(n,24,8)) + end +end end -- of closure @@ -5175,7 +5634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 1012, stripped down to: 912 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5215,7 +5674,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 36053, stripped down to: 19685 +-- original size: 36148, stripped down to: 20179 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5227,7 +5686,7 @@ if not modules then modules={} end modules ['util-str']={ utilities=utilities or {} utilities.strings=utilities.strings or {} local strings=utilities.strings -local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local format,gsub,rep,sub,find=string.format,string.gsub,string.rep,string.sub,string.find local load,dump=load,string.dump local tonumber,type,tostring=tonumber,type,tostring local unpack,concat=table.unpack,table.concat @@ -5410,6 +5869,25 @@ function number.signed(i) return "-",-i end end +local digit=patterns.digit +local period=patterns.period +local three=digit*digit*digit +local splitter=Cs ( + (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) +) +patterns.formattednumber=splitter +function number.formatted(n,sep1,sep2) + local s=type(s)=="string" and n or format("%0.2f",n) + if sep1==true then + return lpegmatch(splitter,s,1,".",",") + elseif sep1=="." then + return lpegmatch(splitter,s,1,sep1,sep2 or ",") + elseif sep1=="," then + return lpegmatch(splitter,s,1,sep1,sep2 or ".") + else + return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") + end +end local zero=P("0")^1/"" local plus=P("+")/"" local minus=P("-") @@ -5435,6 +5913,27 @@ function number.sparseexponent(f,n) end return tostring(n) end +local hf={} +local hs={} +setmetatable(hf,{ __index=function(t,k) + local v="%."..k.."f" + t[k]=v + return v +end } ) +setmetatable(hs,{ __index=function(t,k) + local v="%"..k.."s" + t[k]=v + return v +end } ) +function number.formattedfloat(n,b,a) + local s=format(hf[a],n) + local l=(b or 0)+(a or 0)+1 + if #s<l then + return format(hs[l],s) + else + return s + end +end local template=[[ %s %s @@ -5462,6 +5961,7 @@ local autodouble=string.autodouble local sequenced=table.sequenced local formattednumber=number.formatted local sparseexponent=number.sparseexponent +local formattedfloat=number.formattedfloat ]] else environment={ @@ -5485,6 +5985,7 @@ else sequenced=table.sequenced, formattednumber=number.formatted, sparseexponent=number.sparseexponent, + formattedfloat=number.formattedfloat } end local arguments={ "a1" } @@ -5495,6 +5996,7 @@ setmetatable(arguments,{ __index=function(t,k) end }) local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_sub=(C((S("+-")+R("09"))^0)+Cc(0))*P(".")*(C((S("+-")+R("09"))^0)+Cc(0)) local prefix_tab=P("{")*C((1-P("}"))^0)*P("}")+C((1-R("az","AZ","09","%%"))^0) local format_s=function(f) n=n+1 @@ -5545,6 +6047,10 @@ local format_F=function(f) return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) end end +local format_k=function(b,a) + n=n+1 + return format("formattedfloat(a%s,%i,%i)",n,b or 0,a or 0) +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5693,25 +6199,6 @@ end local format_W=function(f) return format("nspaces[%s]",tonumber(f) or 0) end -local digit=patterns.digit -local period=patterns.period -local three=digit*digit*digit -local splitter=Cs ( - (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) -) -patterns.formattednumber=splitter -function number.formatted(n,sep1,sep2) - local s=type(s)=="string" and n or format("%0.2f",n) - if sep1==true then - return lpegmatch(splitter,s,1,".",",") - elseif sep1=="." then - return lpegmatch(splitter,s,1,sep1,sep2 or ",") - elseif sep1=="," then - return lpegmatch(splitter,s,1,sep1,sep2 or ".") - else - return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") - end -end local format_m=function(f) n=n+1 if not f or f=="" then @@ -5736,9 +6223,16 @@ end local format_extension=function(extensions,f,name) local extension=extensions[name] or "tostring(%s)" local f=tonumber(f) or 1 + local w=find(extension,"%.%.%.") if f==0 then + if w then + extension=gsub(extension,"%.%.%.","") + end return extension elseif f==1 then + if w then + extension=gsub(extension,"%.%.%.","%%s") + end n=n+1 local a="a"..n return format(extension,a,a) @@ -5746,10 +6240,13 @@ local format_extension=function(extensions,f,name) local a="a"..(n+f+1) return format(extension,a,a) else + if w then + extension=gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s") + end local t={} for i=1,f do n=n+1 - t[#t+1]="a"..n + t[i]="a"..n end return format(extension,unpack(t)) end @@ -5762,7 +6259,8 @@ local builder=Cs { "start", +V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") -+V("N") ++V("N") ++V("k") +V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") +V("W") +V("a") @@ -5789,6 +6287,7 @@ local builder=Cs { "start", ["S"]=(prefix_any*P("S"))/format_S, ["Q"]=(prefix_any*P("Q"))/format_S, ["N"]=(prefix_any*P("N"))/format_N, + ["k"]=(prefix_sub*P("k"))/format_k, ["c"]=(prefix_any*P("c"))/format_c, ["C"]=(prefix_any*P("C"))/format_C, ["r"]=(prefix_any*P("r"))/format_r, @@ -5909,7 +6408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28680, stripped down to: 18636 +-- original size: 27407, stripped down to: 17116 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5923,7 +6422,7 @@ utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort -local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local setmetatable,getmetatable,tonumber,tostring,rawget=setmetatable,getmetatable,tonumber,tostring,rawget local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs @@ -6063,7 +6562,7 @@ function table.tocsv(t,specification) r[f]=tostring(field) end end - result[#result+1]=concat(r,separator) + result[i+1]=concat(r,separator) end return concat(result,"\n") else @@ -6295,11 +6794,12 @@ function table.autokey(t,k) return v end local selfmapper={ __index=function(t,k) t[k]=k return k end } -function table.twowaymapper(t) - if not t then - t={} - else - for i=0,#t do +function table.twowaymapper(t) + if not t then + t={} + else + local zero=rawget(t,0) + for i=zero and 0 or 1,#t do local ti=t[i] if ti then local i=tostring(i) @@ -6307,7 +6807,6 @@ function table.twowaymapper(t) t[ti]=i end end - t[""]=t[0] or "" end setmetatable(t,selfmapper) return t @@ -6346,6 +6845,7 @@ local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize +local is_simple_table=table.is_simple_table local function serialize(root,name,specification) if type(specification)=="table" then return original_serialize(root,name,specification) @@ -6353,54 +6853,6 @@ local function serialize(root,name,specification) local t local n=1 local unknown=false - local function simple_table(t) - local nt=#t - if nt>0 then - local n=0 - for _,v in next,t do - n=n+1 - if type(v)=="table" then - return nil - end - end - local haszero=t[0] - if n==nt then - local tt={} - for i=1,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i]=v - elseif tv=="string" then - tt[i]=format("%q",v) - elseif tv=="boolean" then - tt[i]=v and "true" or "false" - else - return nil - end - end - return tt - elseif haszero and (n==nt+1) then - local tt={} - for i=0,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i+1]=v - elseif tv=="string" then - tt[i+1]=format("%q",v) - elseif tv=="boolean" then - tt[i+1]=v and "true" or "false" - else - return nil - end - end - tt[1]="[0] = "..tt[1] - return tt - end - end - return nil - end local function do_serialize(root,name,depth,level,indexed) if level>0 then n=n+1 @@ -6425,7 +6877,7 @@ local function serialize(root,name,specification) local last=0 last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -6448,7 +6900,7 @@ local function serialize(root,name,specification) if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else - local st=simple_table(v) + local st=is_simple_table(v) if st then n=n+1 t[n]=f_val_seq(depth,st) else @@ -6492,7 +6944,7 @@ local function serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else - local st=simple_table(v) + local st=is_simple_table(v) if not st then do_serialize(v,k,depth,level+1) elseif tk=="number" then @@ -6552,11 +7004,11 @@ local function serialize(root,name,specification) end if root then if getmetatable(root) then - local dummy=root._w_h_a_t_e_v_e_r_ + local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - local st=simple_table(root) + local st=is_simple_table(root) if st then return t[1]..f_fin_seq(st) else @@ -6570,7 +7022,12 @@ local function serialize(root,name,specification) end table.serialize=serialize if setinspector then - setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) + setinspector("table",function(v) + if type(v)=="table" then + print(serialize(v,"table",{ metacheck=false })) + return true + end + end) end @@ -6580,7 +7037,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fil"] = package.loaded["util-fil"] or true --- original size: 3577, stripped down to: 2870 +-- original size: 7567, stripped down to: 5575 if not modules then modules={} end modules ['util-fil']={ version=1.001, @@ -6590,7 +7047,9 @@ if not modules then modules={} end modules ['util-fil']={ license="see context related readme files" } local byte=string.byte -local extract=bit32.extract +local char=string.char +local extract=bit32 and bit32.extract +local floor=math.floor utilities=utilities or {} local files={} utilities.files=files @@ -6609,6 +7068,7 @@ end function files.size(f) return f:seek("end") end +files.getsize=files.size function files.setposition(f,n) if zerobased[f] then f:seek("set",n) @@ -6646,6 +7106,10 @@ end function files.readbytes(f,n) return byte(f:read(n),1,n) end +function files.readbytetable(f,n) + local s=f:read(n or 1) + return { byte(s,1,#s) } +end function files.readchar(f) return f:read(1) end @@ -6655,7 +7119,7 @@ end function files.readinteger1(f) local n=byte(f:read(1)) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6663,55 +7127,107 @@ end files.readcardinal1=files.readbyte files.readcardinal=files.readcardinal1 files.readinteger=files.readinteger1 +files.readsignedbyte=files.readinteger1 function files.readcardinal2(f) local a,b=byte(f:read(2),1,2) return 0x100*a+b end +function files.readcardinal2le(f) + local b,a=byte(f:read(2),1,2) + return 0x100*a+b +end function files.readinteger2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function files.readinteger2le(f) + local b,a=byte(f:read(2),1,2) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function files.readcardinal3(f) local a,b,c=byte(f:read(3),1,3) return 0x10000*a+0x100*b+c end +function files.readcardinal3le(f) + local c,b,a=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readinteger3(f) + local a,b,c=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function files.readinteger3le(f) + local c,b,a=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function files.readcardinal4(f) local a,b,c,d=byte(f:read(4),1,4) return 0x1000000*a+0x10000*b+0x100*c+d end +function files.readcardinal4le(f) + local d,c,b,a=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end function files.readinteger4(f) local a,b,c,d=byte(f:read(4),1,4) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.readfixed4(f) - local a,b,c,d=byte(f:read(4),1,4) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF +function files.readinteger4le(f) + local d,c,b,a=byte(f:read(4),1,4) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n+(0x100*c+d)/0xFFFF + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.read2dot14(f) +function files.readfixed2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - n=extract(n,30,2) - return n+m/0x4000 + return (0x100*a+b )+(0x100*c+d)/0x10000 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function files.skipshort(f,n) @@ -6720,6 +7236,55 @@ end function files.skiplong(f,n) f:read(4*(n or 1)) end +function files.writecardinal2(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + f:write(b,a) +end +function files.writecardinal4(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + n=floor(n/256) + local c=char(n%256) + n=floor(n/256) + local d=char(n%256) + f:write(d,c,b,a) +end +function files.writestring(f,s) + f:write(char(byte(s,1,#s))) +end +function files.writebyte(f,b) + f:write(char(b)) +end +if fio and fio.readcardinal1 then + files.readcardinal1=fio.readcardinal1 + files.readcardinal2=fio.readcardinal2 + files.readcardinal3=fio.readcardinal3 + files.readcardinal4=fio.readcardinal4 + files.readinteger1=fio.readinteger1 + files.readinteger2=fio.readinteger2 + files.readinteger3=fio.readinteger3 + files.readinteger4=fio.readinteger4 + files.read2dot14=fio.read2dot14 + files.setposition=fio.setposition + files.getposition=fio.getposition + files.readbyte=files.readcardinal1 + files.readsignedbyte=files.readinteger1 + files.readcardinal=files.readcardinal1 + files.readinteger=files.readinteger1 + local skipposition=fio.skipposition + files.skipposition=skipposition + files.readbytes=fio.readbytes + files.readbytetable=fio.readbytetable + function files.skipshort(f,n) + skipposition(f,2*(n or 1)) + end + function files.skiplong(f,n) + skipposition(f,4*(n or 1)) + end +end end -- of closure @@ -6728,7 +7293,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 4264, stripped down to: 3349 +-- original size: 8716, stripped down to: 6754 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -6738,7 +7303,7 @@ if not modules then modules={} end modules ['util-sac']={ license="see context related readme files" } local byte,sub=string.byte,string.sub -local extract=bit32.extract +local extract=bit32 and bit32.extract utilities=utilities or {} local streams={} utilities.streams=streams @@ -6796,6 +7361,12 @@ function streams.readbytes(f,n) f[2]=j return byte(f[1],i,j-1) end +function streams.readbytetable(f,n) + local i=f[2] + local j=i+n + f[2]=j + return { byte(f[1],i,j-1) } +end function streams.skipbytes(f,n) f[2]=f[2]+n end @@ -6815,7 +7386,7 @@ function streams.readinteger1(f) f[2]=i+1 local n=byte(f[1],i) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6830,16 +7401,33 @@ function streams.readcardinal2(f) local a,b=byte(f[1],i,j) return 0x100*a+b end +function streams.readcardinal2LE(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + return 0x100*a+b +end function streams.readinteger2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function streams.readinteger2le(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function streams.readcardinal3(f) @@ -6849,6 +7437,35 @@ function streams.readcardinal3(f) local a,b,c=byte(f[1],i,j) return 0x10000*a+0x100*b+c end +function streams.readcardinal3le(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readinteger3(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function streams.readinteger3le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function streams.readcardinal4(f) local i=f[2] local j=i+3 @@ -6861,11 +7478,21 @@ function streams.readinteger4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d + end +end +function streams.readinteger4le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local d,c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 + else + return 0x1000000*a+0x10000*b+0x100*c+d end end function streams.readfixed4(f) @@ -6873,26 +7500,38 @@ function streams.readfixed4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - return n+(0x100*c+d)/0xFFFF + return (0x100*a+b )+(0x100*c+d)/0x10000 end end -function streams.read2dot14(f) +function streams.readfixed2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 - else - n=extract(n,30,2) - return n+m/0x4000 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function streams.skipshort(f,n) @@ -6901,6 +7540,92 @@ end function streams.skiplong(f,n) f[2]=f[2]+4*(n or 1) end +if sio and sio.readcardinal2 then + local readcardinal1=sio.readcardinal1 + local readcardinal2=sio.readcardinal2 + local readcardinal3=sio.readcardinal3 + local readcardinal4=sio.readcardinal4 + local readinteger1=sio.readinteger1 + local readinteger2=sio.readinteger2 + local readinteger3=sio.readinteger3 + local readinteger4=sio.readinteger4 + local readfixed2=sio.readfixed2 + local readfixed4=sio.readfixed4 + local read2dot14=sio.read2dot14 + local readbytes=sio.readbytes + local readbytetable=sio.readbytetable + function streams.readcardinal1(f) + local i=f[2] + f[2]=i+1 + return readcardinal1(f[1],i) + end + function streams.readcardinal2(f) + local i=f[2] + f[2]=i+2 + return readcardinal2(f[1],i) + end + function streams.readcardinal3(f) + local i=f[2] + f[2]=i+3 + return readcardinal3(f[1],i) + end + function streams.readcardinal4(f) + local i=f[2] + f[2]=i+4 + return readcardinal4(f[1],i) + end + function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + return readinteger1(f[1],i) + end + function streams.readinteger2(f) + local i=f[2] + f[2]=i+2 + return readinteger2(f[1],i) + end + function streams.readinteger3(f) + local i=f[2] + f[2]=i+3 + return readinteger3(f[1],i) + end + function streams.readinteger4(f) + local i=f[2] + f[2]=i+4 + return readinteger4(f[1],i) + end + function streams.read2dot4(f) + local i=f[2] + f[2]=i+2 + return read2dot4(f[1],i) + end + function streams.readbytes(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytes(f[1],i,n) + end + function streams.readbytetable(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytetable(f[1],i,n) + end + streams.readbyte=streams.readcardinal1 + streams.readsignedbyte=streams.readinteger1 + streams.readcardinal=streams.readcardinal1 + streams.readinteger=streams.readinteger1 +end end -- of closure @@ -6909,7 +7634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4100, stripped down to: 2852 +-- original size: 3926, stripped down to: 2742 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -7049,7 +7774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 23411, stripped down to: 16177 +-- original size: 22883, stripped down to: 16045 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -7211,6 +7936,21 @@ function parsers.settings_to_array(str,strict) return { str } end end +function parsers.settings_to_numbers(str) + if not str or str=="" then + return {} + end + if type(str)=="table" then + elseif find(str,",",1,true) then + str=lpegmatch(pattern,str) + else + return { tonumber(str) } + end + for i=1,#str do + str[i]=tonumber(str[i]) + end + return str +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) local pattern=spaces*Ct(value*(separator*value)^0) function parsers.settings_to_array_obey_fences(str) @@ -7587,7 +8327,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2350, stripped down to: 1847 +-- original size: 2274, stripped down to: 1781 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7668,7 +8408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12862, stripped down to: 9104 +-- original size: 12454, stripped down to: 8840 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7854,7 +8594,6 @@ function setters.list(t) return user,system end function setters.show(t) - local category=t.name local list=setters.list(t) t.report() for k=1,#list do @@ -7981,7 +8720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 30767, stripped down to: 21355 +-- original size: 30007, stripped down to: 20818 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -8027,6 +8766,14 @@ if tex and (tex.jobname or tex.formatname) then if texio.setescape then texio.setescape(0) end + if arg then + for k,v in next,arg do + if v=="--ansi" or v=="--c:ansi" then + variant="ansi" + break + end + end + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -8612,7 +9359,6 @@ function logs.stop_page_number() end logs.flush() end -local report_files=logs.reporter("files") local nesting=0 local verbose=false local hasscheme=url.hasscheme @@ -8774,7 +9520,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6917, stripped down to: 5484 +-- original size: 8036, stripped down to: 5567 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8806,11 +9552,13 @@ end local function resettiming(instance) timers[instance or "notimer"]={ timing=0,loadtime=0 } end +local ticks=clock +local seconds=function(n) return n or 0 end local function starttiming(instance) local timer=timers[instance or "notimer"] local it=timer.timing or 0 if it==0 then - timer.starttime=clock() + timer.starttime=ticks() if not timer.loadtime then timer.loadtime=0 end @@ -8824,12 +9572,13 @@ local function stoptiming(instance) timer.timing=it-1 else local starttime=timer.starttime - if starttime then - local stoptime=clock() + if starttime and starttime>0 then + local stoptime=ticks() local loadtime=stoptime-starttime timer.stoptime=stoptime timer.loadtime=timer.loadtime+loadtime timer.timing=0 + timer.starttime=0 return loadtime end end @@ -8840,7 +9589,7 @@ local function elapsed(instance) return instance or 0 else local timer=timers[instance or "notimer"] - return timer and timer.loadtime or 0 + return timer and seconds(timer.loadtime) or 0 end end local function elapsedtime(instance) @@ -8888,10 +9637,13 @@ function statistics.show() local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) - if jit then - local jitstatus={ jit.status() } - if jitstatus[1] then - register("luajit options",concat(jitstatus," ",2)) + if TEXENGINE=="luajittex" and JITSUPPORTED then + local jitstatus=jit.status + if jitstatus then + local jitstatus={ jitstatus() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end end end register("lua properties",function() @@ -8955,7 +9707,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 6039, stripped down to: 3616 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -9102,7 +9854,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 5142, stripped down to: 3611 +-- original size: 5396, stripped down to: 3708 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -9224,6 +9976,17 @@ function luautilities.loadstripped(...) return load(dump(l,true)) end end +local finalizers={} +setmetatable(finalizers,{ + __gc=function(t) + for i=1,#t do + pcall(t[i]) + end + end +} ) +function luautilities.registerfinalizer(f) + finalizers[#finalizers+1]=f +end end -- of closure @@ -9232,7 +9995,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 4030, stripped down to: 2718 +-- original size: 8911, stripped down to: 6504 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -9242,75 +10005,230 @@ if not modules then modules={} end modules ['util-deb']={ license="see context related readme files" } local debug=require "debug" -local getinfo=debug.getinfo -local type,next,tostring=type,next,tostring -local format,find=string.format,string.find -local is_boolean=string.is_boolean +local getinfo,sethook=debug.getinfo,debug.sethook +local type,next,tostring,tonumber=type,next,tostring,tonumber +local format,find,sub,gsub=string.format,string.find,string.sub,string.gsub +local insert,remove,sort=table.insert,table.remove,table.sort +local setmetatableindex=table.setmetatableindex utilities=utilities or {} local debugger=utilities.debugger or {} utilities.debugger=debugger -local counters={} -local names={} local report=logs.reporter("debugger") -local function hook() - local f=getinfo(2) - if f then - local n="unknown" - if f.what=="C" then - n=f.name or '<anonymous>' - if not names[n] then - names[n]=format("%42s",n) +local ticks=os.gettimeofday or os.clock +local seconds=function(n) return n or 0 end +local overhead=0 +local dummycalls=10*1000 +local nesting=0 +local names={} +local initialize=false +if not (FFISUPPORTED and ffi) then +elseif os.type=="windows" then + initialize=function() + local kernel=ffilib("kernel32","system") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end end - else - n=f.name or f.namewhat or f.what - if not n or n=="" then - n="?" + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end + initialize=false + end +elseif os.type=="unix" then + initialize=function() + local C=ffi.C + local tonumber=ffi.number or tonumber + ffi.cdef [[ + /* what a mess */ + typedef int clk_id_t; + typedef enum { CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_PROCESS_CPUTIME_ID } clk_id; + typedef struct timespec { long sec; long nsec; } ctx_timespec; + int clock_gettime(clk_id_t timerid, struct timespec *t); + ]] + local target=ffi.new("ctx_timespec[?]",1) + local clock=C.CLOCK_PROCESS_CPUTIME_ID + ticks=function () + C.clock_gettime(clock,target) + return tonumber(target[0].sec*1000000000+target[0].nsec) + end + seconds=function(ticks) + return ticks/1000000000 + end + initialize=false + end +end +setmetatableindex(names,function(t,name) + local v=setmetatableindex(function(t,source) + local v=setmetatableindex(function(t,line) + local v={ total=0,count=0 } + t[line]=v + return v + end) + t[source]=v + return v + end) + t[name]=v + return v +end) +local function hook(where) + local f=getinfo(2,"nSl") + if f then + local source=f.short_src + if not source then + return + end + local line=f.linedefined or 0 + local name=f.name + if not name then + local what=f.what + if what=="C" then + name="<anonymous>" + else + name=f.namewhat or what or "<unknown>" end - if not names[n] then - names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + local data=names[name][source][line] + if where=="call" then + data.count=data.count+1 + insert(data,ticks()) + elseif where=="return" then + local t=remove(data) + if t then + data.total=data.total+ticks()-t end end - counters[n]=(counters[n] or 0)+1 end end -function debugger.showstats(printer,threshold) - printer=printer or report - threshold=threshold or 0 - local total,grandtotal,functions=0,0,0 +function debugger.showstats(printer,threshold) + local printer=printer or report + local calls=0 + local functions=0 local dataset={} - for name,count in next,counters do - dataset[#dataset+1]={ name,count } + local length=0 + local wholetime=0 + local threshold=threshold or 0 + for name,sources in next,names do + for source,lines in next,sources do + for line,data in next,lines do + local count=data.count + if count>threshold then + if #name>length then + length=#name + end + local total=data.total + local real=total + if real>0 then + real=total-(count*overhead/dummycalls) + if real<0 then + real=0 + end + wholetime=wholetime+real + end + if line<0 then + line=0 + end + dataset[#dataset+1]={ real,total,count,name,source,line } + end + end + end end - table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + sort(dataset,function(a,b) + if a[1]==b[1] then + if a[2]==b[2] then + if a[3]==b[3] then + if a[4]==b[4] then + if a[5]==b[5] then + return a[6]<b[6] + else + return a[5]<b[5] + end + else + return a[4]<b[4] + end + else + return b[3]<a[3] + end + else + return b[2]<a[2] + end + else + return b[1]<a[1] + end + end) + if length>50 then + length=50 + end + local fmt=string.formatters["%4.9k %4.9k %3.3k %8i %-"..length.."s %4i %s"] for i=1,#dataset do - local d=dataset[i] - local name=d[1] - local count=d[2] - if count>threshold and not find(name,"for generator") then - printer(format("%8i %s\n",count,names[name])) - total=total+count - end - grandtotal=grandtotal+count + local data=dataset[i] + local real=data[1] + local total=data[2] + local count=data[3] + local name=data[4] + local source=data[5] + local line=data[6] + local percent=real/wholetime + calls=calls+count functions=functions+1 + name=gsub(name,"%s+"," ") + if #name>length then + name=sub(name,1,length) + end + printer(fmt(seconds(total),seconds(real),percent,count,name,line,source)) end - printer("\n") - printer(format("functions : % 10i\n",functions)) - printer(format("total : % 10i\n",total)) - printer(format("grand total: % 10i\n",grandtotal)) - printer(format("threshold : % 10i\n",threshold)) + printer("") + printer(format("functions : %i",functions)) + printer(format("calls : %i",calls)) + printer(format("overhead : %f",seconds(overhead/1000))) end function debugger.savestats(filename,threshold) local f=io.open(filename,'w') if f then - debugger.showstats(function(str) f:write(str) end,threshold) + debugger.showstats(function(str) f:write(str,"\n") end,threshold) f:close() end end function debugger.enable() - debug.sethook(hook,"c") + if nesting==0 then + running=true + if initialize then + initialize() + end + sethook(hook,"cr") + local function dummy() end + local t=ticks() + for i=1,dummycalls do + dummy() + end + overhead=ticks()-t + end + if nesting>0 then + nesting=nesting+1 + end end function debugger.disable() - debug.sethook() + if nesting>0 then + nesting=nesting-1 + end + if nesting==0 then + sethook() + end end local function showtraceback(rep) local level=2 @@ -9334,9 +10252,661 @@ end -- of closure do -- create closure to overcome 200 locals limit +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true + +-- original size: 7100, stripped down to: 3978 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20309, stripped down to: 13848 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + if type(root)=="table" then + root,what=root[1],root[2] + end + if type(root)=="string" and root~="" then + root=collapsepath(expandname(root)) + if what=="r" or what=="ro" or what=="readable" then + what="read" + elseif what=="w" or what=="wo" or what=="writable" then + what="write" + end + validroots[root]=what=="write" or false + end + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + elseif filenamelogger then + filenamelogger(name,"*",name,false) + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + if trace then + report("resultof: %s",command) + end + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + if trace then + report("execute: %s",command) + end + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + if trace then + report("pipeto: %s",command) + end + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +function sandbox.getrunner(name) + return name and validrunners[name] +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if FFISUPPORTED and ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7757, stripped down to: 6015 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9511,154 +11081,9 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8284, stripped down to: 5176 +-- original size: 9246, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9845,7 +11270,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6358, stripped down to: 4257 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9998,7 +11423,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 56973, stripped down to: 35872 +-- original size: 57003, stripped down to: 35696 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -10013,7 +11438,7 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset +local type,next,setmetatable,getmetatable,tonumber,rawset,select=type,next,setmetatable,getmetatable,tonumber,rawset,select local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local sort=table.sort local utfchar=utf.char @@ -10140,6 +11565,7 @@ local function add_empty(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nt, __p__=top } dt[nt]=t @@ -10161,6 +11587,7 @@ local function add_begin(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nil, __p__=stack[level] } setmetatable(top,mt) @@ -10188,6 +11615,7 @@ local function add_end(spacing,namespace,tag) dt=top.dt nt=#dt+1 dt[nt]=toclose + toclose.ni=nt if toclose.at.xmlns then remove(xmlns) end @@ -10232,7 +11660,13 @@ local function add_special(what,spacing,text) if strip and (what=="@cm@" or what=="@dt@") then else nt=nt+1 - dt[nt]={ special=true,ns="",tg=what,dt={ text } } + dt[nt]={ + special=true, + ns="", + tg=what, + ni=nil, + dt={ text }, + } end end local function set_message(txt) @@ -10285,7 +11719,6 @@ do end local p_rest=(1-P(";"))^0 local p_many=P(1)^0 - local p_char=lpegpatterns.utf8character local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) xml.parsedentitylpeg=parsedentity local predefined_unified={ @@ -10327,13 +11760,27 @@ do [ [[}]] ]="&U+7D;", [ [[~]] ]="&U+7E;", } + local privates_x={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[']] ]="&U+27;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } local privates_n={ } local escaped=utf.remapper(privates_u,"dynamic") local unprivatized=utf.remapper(privates_p,"dynamic") local unspecialized=utf.remapper(privates_s,"dynamic") + local despecialized=utf.remapper(privates_x,"dynamic") xml.unprivatized=unprivatized xml.unspecialized=unspecialized + xml.despecialized=despecialized xml.escaped=escaped local function unescaped(s) local p=privates_n[s] @@ -10778,6 +12225,10 @@ local grammar_unparsed_text=P { "preamble", local function _xmlconvert_(data,settings) settings=settings or {} preparexmlstate(settings) + local preprocessor=settings.preprocessor + if data and data~="" and type(preprocessor)=="function" then + data=preprocessor(data,settings) or data + end if settings.parent_root then mt=getmetatable(settings.parent_root) else @@ -10919,14 +12370,24 @@ function xml.toxml(data) return data end end -local function copy(old) +local function copy(old,p) if old then local new={} for k,v in next,old do - if type(v)=="table" then - new[k]=table.copy(v) - else + local t=type(v)=="table" + if k=="at" then + local t={} + for k,v in next,v do + t[k]=v + end + new[k]=t + elseif k=="dt" then + v.__p__=nil + v=copy(v,new) new[k]=v + v.__p__=p + else + new[k]=v end end local mt=getmetatable(old) @@ -11157,18 +12618,26 @@ local xmlfilehandler=newhandlers { function xml.save(root,name) serialize(root,xmlfilehandler,name) end -local result +local result,r,threshold={},0,512 local xmlstringhandler=newhandlers { name="string", initialize=function() - result={} + r=0 return result end, finalize=function() - return concat(result) + local done=concat(result,"",1,r) + r=0 + if r>threshold then + result={} + end + return done end, handle=function(...) - result[#result+1]=concat {... } + for i=1,select("#",...) do + r=r+1 + result[r]=select(i,...) + end end, } local function xmltostring(root) @@ -11320,7 +12789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 53892, stripped down to: 32508 +-- original size: 53301, stripped down to: 32477 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -11702,6 +13171,14 @@ local function apply_expression(list,expression,order) end return collected end +local function apply_selector(list,specification) + if xml.applyselector then + apply_selector=xml.applyselector + return apply_selector(list,specification) + else + return list + end +end local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb local spaces=S(" \n\r\t\f")^0 local lp_space=S(" \n\r\t\f") @@ -11825,6 +13302,9 @@ end local function register_nodes(nodetest,nodes) return { kind="nodes",nodetest=nodetest,nodes=nodes } end +local function register_selector(specification) + return { kind="selector",specification=specification } +end local function register_expression(expression) local converted=lpegmatch(converter,expression) local runner=load(format(template_e,converted)) @@ -11865,34 +13345,36 @@ local pathparser=Ct { "patterns", (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), - step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + step=((V("shortcuts")+V("selector")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, - shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor")+V("s_lastmatch"), shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), s_descendant=P("**")*Cc(register_descendant), - s_child=P("*")*no_nextcolon*Cc(register_child ), - s_parent=P("..")*Cc(register_parent ), - s_self=P("." )*Cc(register_self ), - s_root=P("^^")*Cc(register_root ), - s_ancestor=P("^")*Cc(register_ancestor ), - descendant=P("descendant::")*Cc(register_descendant ), - child=P("child::")*Cc(register_child ), - parent=P("parent::")*Cc(register_parent ), - self=P("self::")*Cc(register_self ), - root=P('root::')*Cc(register_root ), - ancestor=P('ancestor::')*Cc(register_ancestor ), - descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), - ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), - following=P('following::')*Cc(register_following ), - following_sibling=P('following-sibling::')*Cc(register_following_sibling ), - preceding=P('preceding::')*Cc(register_preceding ), - preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), - reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), - last_match=P('last-match::')*Cc(register_last_match ), + s_child=P("*")*no_nextcolon*Cc(register_child), + s_parent=P("..")*Cc(register_parent), + s_self=P("." )*Cc(register_self), + s_root=P("^^")*Cc(register_root), + s_ancestor=P("^")*Cc(register_ancestor), + s_lastmatch=P("=")*Cc(register_last_match), + descendant=P("descendant::")*Cc(register_descendant), + child=P("child::")*Cc(register_child), + parent=P("parent::")*Cc(register_parent), + self=P("self::")*Cc(register_self), + root=P('root::')*Cc(register_root), + ancestor=P('ancestor::')*Cc(register_ancestor), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self), + following=P('following::')*Cc(register_following), + following_sibling=P('following-sibling::')*Cc(register_following_sibling), + preceding=P('preceding::')*Cc(register_preceding), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling), + last_match=P('last-match::')*Cc(register_last_match), + selector=P("{")*C((1-P("}"))^1)*P("}")/register_selector, nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -12042,6 +13524,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) p.matched=p.matched+1 @@ -12083,6 +13567,9 @@ do elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) + report_lpath("% 10i : se : %s ",(collected and #collected) or 0,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") @@ -12114,6 +13601,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then return pi.finalizer(collected) end @@ -12170,6 +13659,13 @@ do function xml.lastmatch() return lastmatch end + local stack={} + function xml.pushmatch() + insert(stack,lastmatch) + end + function xml.popmatch() + lastmatch=remove(stack) + end end local applylpath=xml.applylpath function xml.filter(root,pattern) @@ -12449,7 +13945,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3787, stripped down to: 2003 +-- original size: 3684, stripped down to: 1957 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -12518,7 +14014,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 30566, stripped down to: 21741 +-- original size: 29835, stripped down to: 21174 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12534,7 +14030,6 @@ local xml=xml local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath -local xmlfilter=xml.filter local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip @@ -12738,7 +14233,17 @@ function xml.replace(root,pattern,whatever) report('replacing',pattern,c,e) end local d=p.dt - d[e.ni]=copiedelement(element,p) + local n=e.ni + local t=copiedelement(element,p) + if type(t)=="table" then + d[n]=t[1] + for i=2,#t do + n=n+1 + insert(d,n,t[i]) + end + else + d[n]=t + end redo_ni(d) end end @@ -13161,7 +14666,7 @@ local obsolete=xml.obsolete xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect xml.delete_element=xml.delete obsolete.delete_element=xml.delete -xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.replace_element=xml.replace obsolete.replace_element=xml.replace xml.each_element=xml.each obsolete.each_element=xml.each xml.process_elements=xml.process obsolete.process_elements=xml.process xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter @@ -13379,7 +14884,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10719, stripped down to: 7841 +-- original size: 10274, stripped down to: 7538 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -13757,7 +15262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6534, stripped down to: 5072 +-- original size: 6407, stripped down to: 4965 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13907,6 +15412,7 @@ function reporters.export(t,methods,filename) if filename then local fullname=file.replacesuffix(filename,method) t.report("saving export in %a",fullname) + dir.mkdirs(file.pathpart(fullname)) io.savedata(fullname,result) else reporters.lines(t,result) @@ -13927,7 +15433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11444, stripped down to: 7830 +-- original size: 11099, stripped down to: 7516 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -14072,11 +15578,6 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if type(profiler)=="table" and not jit then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) -end local prefixes=utilities.storage.allocate() resolvers.prefixes=prefixes local resolved={} @@ -14183,7 +15684,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18619, stripped down to: 11042 +-- original size: 17958, stripped down to: 10705 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -14199,7 +15700,6 @@ local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next local isdir=lfs.isdir -local ostype=os.type local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -14568,7 +16068,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9649, stripped down to: 7131 +-- original size: 9342, stripped down to: 6887 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -14852,7 +16352,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16066, stripped down to: 11938 +-- original size: 16088, stripped down to: 11435 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -15056,18 +16556,6 @@ end caches.getreadablepaths=getreadablepaths caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd=getreadablepaths(...) - for i=1,#rd do - local path=rd[i] - local fullname=file.join(path,filename) - if is_readable(fullname) then - usedreadables[i]=true - return fullname,path - end - end - return caches.setfirstwritablefile(filename,...) -end -function caches.getfirstreadablefile_TEST_ME_FIRST(filename,...) local fullname,path=caches.setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path @@ -15096,18 +16584,22 @@ end function caches.setluanames(path,name) return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end -function caches.loaddata(readables,name) +function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } end for i=1,#readables do local path=readables[i] - local tmaname,tmcname=caches.setluanames(path,name) local loader=false + local tmaname,tmcname=caches.setluanames(path,name) if isfile(tmcname) then loader=loadfile(tmcname) end if not loader and isfile(tmaname) then + local tmacrap,tmcname=caches.setluanames(writable,name) + if isfile(tmcname) then + loader=loadfile(tmcname) + end utilities.lua.compile(tmaname,tmcname) if isfile(tmcname) then loader=loadfile(tmcname) @@ -15228,7 +16720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5488, stripped down to: 4101 +-- original size: 5310, stripped down to: 3980 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -15240,7 +16732,6 @@ if not modules then modules={} end modules ['data-met']={ local find,format=string.find,string.format local sequenced=table.sequenced local addurlscheme,urlhashed=url.addscheme,url.hashed -local getcurrentdir=lfs.currentdir local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) @@ -15347,7 +16838,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67241, stripped down to: 46427 +-- original size: 67524, stripped down to: 46632 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15401,6 +16892,7 @@ resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf @@ -15444,7 +16936,6 @@ local function resolvevariable(k) end local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) @@ -15590,23 +17081,29 @@ local function identify_configuration_files() end reportcriticalvariables(cnfspec) local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname=resolvers.luacnfname - for i=1,#cnfpaths do - local filepath=cnfpaths[i] - local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolveprefix(filename) - if trace_locating then - local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) - report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) - end - if isfile(realname) then - specification[#specification+1]=filename + local function locatecnf(luacnfname,kind) + for i=1,#cnfpaths do + local filepath=cnfpaths[i] + local filename=collapsepath(filejoin(filepath,luacnfname)) + local realname=resolveprefix(filename) if trace_locating then - report_resolving("found configuration file %a",realname) + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) + report_resolving("looking for %s %a on %s path %a from specification %a", + kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath) + end + if isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found %s configuration file %a",kind,realname) + end end end end + locatecnf(resolvers.luacnfname,"regular") + if #specification==0 then + locatecnf(resolvers.luacnffallback,"fallback") + end if trace_locating then report_resolving() end @@ -16903,7 +18400,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4236, stripped down to: 3144 +-- original size: 4090, stripped down to: 3059 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -17025,7 +18522,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 935, stripped down to: 838 +-- original size: 910, stripped down to: 823 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -17055,7 +18552,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 548, stripped down to: 483 +-- original size: 530, stripped down to: 475 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -17078,7 +18575,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3976, stripped down to: 3391 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -17186,7 +18683,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5148, stripped down to: 3680 +-- original size: 5029, stripped down to: 3607 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -17256,7 +18753,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name) + stored=caches.loaddata(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -17305,7 +18802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4000, stripped down to: 3052 +-- original size: 4045, stripped down to: 3110 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -17350,7 +18847,7 @@ function resolvers.automount(usecache) end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) -function statistics.savefmtstatus(texname,formatbanner,sourcefile) +function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") @@ -17361,6 +18858,10 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) sourcefile=sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) + lua.registerfinalizer(function() + logs.report("format banner","%s",banner) + logs.newline() + end) end end function statistics.checkfmtstatus(texname) @@ -17396,7 +18897,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 9036, stripped down to: 7041 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -17633,7 +19134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8712, stripped down to: 5726 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17822,7 +19323,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6779, stripped down to: 5444 +-- original size: 6653, stripped down to: 5467 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17868,11 +19369,21 @@ function resolvers.schemes.cleanname(specification) end return hash end -local cached,loaded,reused,thresholds,handlers={},{},{},{},{} -local function runcurl(name,cachename) - local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name - os.execute(command) -end +local cached={} +local loaded={} +local reused={} +local thresholds={} +local handlers={} +local runner=sandbox.registerrunner { + name="curl resolver", + method="execute", + program="curl", + template="--silent -- insecure --create-dirs --output %cachename% %original%", + checkers={ + cachename="cache", + original="url", + } +} local function fetch(specification) local original=specification.original local scheme=specification.scheme @@ -17894,7 +19405,10 @@ local function fetch(specification) report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") end logs.flush() - runcurl(original,cachename) + runner { + original=original, + cachename=cachename, + } end end if io.exists(cachename) then @@ -18003,7 +19517,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4447, stripped down to: 3302 +-- original size: 4207, stripped down to: 3137 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -18045,8 +19559,6 @@ function helpers.cleanpath(path) return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib -local getextraluapaths=package.extraluapaths -local getextralibpaths=package.extralibpaths local registerpath=helpers.registerpath local lualibfile=helpers.lualibfile local luaformatpaths @@ -18112,7 +19624,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2494, stripped down to: 2047 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -18179,7 +19691,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2674, stripped down to: 1658 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -18235,7 +19747,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2815, stripped down to: 2415 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -18315,7 +19827,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11846, stripped down to: 6059 +-- original size: 13595, stripped down to: 7500 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -18324,35 +19836,51 @@ if not modules then modules={} end modules ['util-lib']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } -local gsub,find=string.gsub,string.find -local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join -local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles -local loaded=package.loaded -local report_swiglib=logs.reporter("swiglib") -local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local type=type +local next=next +local pcall=pcall +local gsub=string.gsub +local find=string.find +local sort=table.sort +local pathpart=file.pathpart +local nameonly=file.nameonly +local joinfile=file.join +local removesuffix=file.removesuffix +local findfile=resolvers.findfile +local findfiles=resolvers.findfiles +local expandpaths=resolvers.expandedpathlistfromvariable +local qualifiedpath=file.is_qualified_path +local isfile=lfs.isfile local done=false -local function requireswiglib(required,version) - local trace_swiglib=trace_swiglib or package.helpers.trace - local library=loaded[required] - if library==nil then - if trace_swiglib then - report_swiglib("requiring library %a with version %a",required,version or "any") - end - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) +local function locate(required,version,trace,report,action) + if type(required)~="string" then + report("provide a proper library name") + return + end + if trace then + report("requiring library %a with version %a",required,version or "any") + end + local found_library=nil + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + if qualifiedpath(required) then + if isfile(required) then + found_library=required + end + else local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false local engine=environment.ownmain or false - if trace_swiglib and not done then - local list=resolvers.expandedpathlistfromvariable("lib") + if trace and not done then + local list=expandpaths("lib") for i=1,#list do - report_swiglib("tds path %i: %s",i,list[i]) + report("tds path %i: %s",i,list[i]) end end local function found(locate,asked_library,how,...) - if trace_swiglib then - report_swiglib("checking %s: %a",how,asked_library) + if trace then + report("checking %s: %a",how,asked_library) end return locate(asked_library,...) end @@ -18360,45 +19888,45 @@ local function requireswiglib(required,version) local found=nil if version then local asked_library=joinfile(required_path,version,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end if not found or found=="" then local asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end return found and found~="" and found or false end local function attempt(checkpattern) - if trace_swiglib then - report_swiglib("checking tds lib paths strictly") + if trace then + report("checking tds lib paths strictly") end local found=findfile and check(findfile,"lib") if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace_swiglib then - report_swiglib("checking tds lib paths with wildcard") + if trace then + report("checking tds lib paths with wildcard") end local asked_library=joinfile(required_path,".*",required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","latest version",asked_library) + if trace then + report("checking %s: %a","latest version",asked_library) end local list=findfiles(asked_library,"lib",true) if list and #list>0 then - table.sort(list) + sort(list) local found=list[#list] if found and (not checkpattern or find(found,checkpattern)) then return found end end - if trace_swiglib then - report_swiglib("checking lib paths") + if trace then + report("checking lib paths") end package.extralibpath(environment.ownpath) local paths=package.libpaths() @@ -18410,89 +19938,143 @@ local function requireswiglib(required,version) end return false end - local found_library=nil if engine then - if trace_swiglib then - report_swiglib("attemp 1, engine %a",engine) + if trace then + report("attemp 1, engine %a",engine) end found_library=attempt("/"..engine.."/") if not found_library then - if trace_swiglib then - report_swiglib("attemp 2, no engine",asked_library) + if trace then + report("attemp 2, no engine",asked_library) end found_library=attempt() end else found_library=attempt() end - if not found_library then - if trace_swiglib then - report_swiglib("not found: %a",required) - end + end + if not found_library then + if trace then + report("not found: %a",required) + end + library=false + else + if trace then + report("found: %a",found_library) + end + local message,result=action(found_library,required_base) + if result then + library=result + else library=false + report("load error: message %a, library %a",tostring(message),found_library or "no library") + end + end + if not library then + report("unknown: %a",required) + elseif trace then + report("stored: %a",required) + end + return library +end +do + local report_swiglib=logs.reporter("swiglib") + local trace_swiglib=false + local savedrequire=require + local loadedlibs={} + local loadlib=package.loadlib + local pushdir=dir.push + local popdir=dir.pop + trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) + function requireswiglib(required,version) + local library=loadedlibs[library] + if library==nil then + local trace_swiglib=trace_swiglib or package.helpers.trace + library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) + pushdir(pathpart(name)) + local opener="luaopen_"..base + if trace_swiglib then + report_swiglib("opening: %a with %a",name,opener) + end + local library,message=loadlib(name,opener) + local libtype=type(library) + if libtype=="function" then + library=library() + message=true + else + report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") + library=false + end + popdir() + return message,library + end) + loadedlibs[required]=library or false + end + return library + end + function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) else - local path=pathpart(found_library) - local base=nameonly(found_library) - dir.push(path) + return savedrequire(name) + end + end + local swiglibs={} + local initializer="core" + function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) if trace_swiglib then - report_swiglib("found: %a",found_library) - end - local message=nil - local opener="luaopen_"..required_base - library,message=package.loadlib(found_library,opener) - local libtype=type(library) - if libtype=="function" then - library=library() + report_swiglib("loading %a",name) + end + if not find(name,"%."..initializer.."$") then + fullname="swiglib."..name.."."..initializer else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + fullname="swiglib."..name end - dir.pop() + library=requireswiglib(fullname,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) end - if not library then - report_swiglib("unknown: %a",required) - elseif trace_swiglib then - report_swiglib("stored: %a",required) - end - loaded[required]=library - else - report_swiglib("reused: %a",required) + return library end - return library + statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end + end) end -local savedrequire=require -function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) +if FFISUPPORTED and ffi and ffi.load then + local report_ffilib=logs.reporter("ffilib") + local trace_ffilib=false + local savedffiload=ffi.load + trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) + local function locateindeed(name) + local message,library=pcall(savedffiload,removesuffix(name)) + if type(library)=="userdata" then + return library + else + return false + end end -end -local swiglibs={} -local initializer="core" -function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) + function ffilib(required,version) + if version=="system" then + return locateindeed(name) + else + return locate(required,version,trace_ffilib,report_ffilib,locateindeed) end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer + end + function ffi.load(name) + local library=ffilib(name) + if type(library)=="userdata" then + return library else - fullname="swiglib."..name + report_ffilib("trying to load %a using normal loader",name) + return savedffiload(name) end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end -statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end -end) end -- of closure @@ -18501,7 +20083,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5914, stripped down to: 2584 +-- original size: 5703, stripped down to: 2507 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -18604,7 +20186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 6967, stripped down to: 5631 +-- original size: 9144, stripped down to: 7291 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -18618,23 +20200,67 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local trackers=environment.argument("trackers") - local directives=environment.argument("directives") +local function primaryflags() + local arguments=environment.arguments + local flags={} + if arguments.silent then + flags[#flags+1]="--interaction=batchmode" + end + if arguments.jit then + flags[#flags+1]="--jiton" + end + return concat(flags," ") +end +local function secondaryflags() + local arguments=environment.arguments + local trackers=arguments.trackers + local directives=arguments.directives local flags={} if trackers and trackers~="" then - flags={ "--trackers="..quoted(trackers) } + flags[#flags+1]="--c:trackers="..quoted(trackers) end if directives and directives~="" then - flags={ "--directives="..quoted(directives) } + flags[#flags+1]="--c:directives="..quoted(directives) + end + if arguments.silent then + flags[#flags+1]="--c:silent" end - if environment.argument("jit") then - flags={ "--jiton" } + if arguments.jit then + flags[#flags+1]="--c:jiton" + end + if arguments.ansi then + flags[#flags+1]="--c:ansi" end return concat(flags," ") end -function environment.make_format(name,silent) +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] +local checkers={ + primaryflags="string", + secondaryflags="string", + luafile="readable", + texfile="readable", + redirect="string", + dump="string", +} +local runners={ + luatex=sandbox.registerrunner { + name="make luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="make luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} +function environment.make_format(name,arguments) local engine=environment.ownmain or "luatex" + local silent=environment.arguments.silent local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" if path~="" then @@ -18690,11 +20316,20 @@ function environment.make_format(name,silent) lfs.chdir(olddir) return end - local dump=os.platform=="unix" and "\\\\dump" or "\\dump" - if silent then + local specification={ + primaryflags=primaryflags(), + secondaryflags=secondaryflags(), + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), + dump=os.platform=="unix" and "\\\\dump" or "\\dump", + } + local runner=runners[engine] + if not runner then + report_format("format %a cannot be generated, no runner available for engine %a",name,engine) + elseif silent then statistics.starttiming() - local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - local result=os.execute(command) + specification.redirect="> temp.log" + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -18703,9 +20338,7 @@ function environment.make_format(name,silent) end os.remove("temp.log") else - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - report_format("running command: %s\n",command) - os.execute(command) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -18718,6 +20351,30 @@ function environment.make_format(name,silent) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -18739,9 +20396,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -18750,10 +20416,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 797557 --- stripped bytes : 289197 +-- original bytes : 842443 +-- stripped bytes : 306317 -- end library merge @@ -18777,6 +20443,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -18810,8 +20477,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) @@ -18983,7 +20651,7 @@ local helpinfo = [[ <metadata> <entry name="name">mtxrun</entry> <entry name="detail">ConTeXt TDS Runner Tool</entry> - <entry name="version">1.31</entry> + <entry name="version">1.32</entry> </metadata> <flags> <category name="basic"> @@ -19046,6 +20714,7 @@ local helpinfo = [[ </subcategory> <subcategory> <flag name="expand-braces"><short>expand complex variable</short></flag> + <flag name="resolve-path"><short>expand variable (completely resolve paths)</short></flag> <flag name="expand-path"><short>expand variable (resolve paths)</short></flag> <flag name="expand-var"><short>expand variable (resolve references)</short></flag> <flag name="show-path"><short>show path expansion of ...</short></flag> @@ -19063,7 +20732,7 @@ local helpinfo = [[ local application = logs.application { name = "mtxrun", - banner = "ConTeXt TDS Runner Tool 1.31", + banner = "ConTeXt TDS Runner Tool 1.32", helpinfo = helpinfo, } @@ -20029,6 +21698,13 @@ elseif e_argument("expand-path") then environment.initializearguments(environment.arguments_after) resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) +elseif e_argument("resolve-path") then + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.cleanedpathlist, environment.files) + elseif e_argument("expand-var") or e_argument("expand-variable") then -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) |