From fcd7cdd22ff42dab791f9f825b642caa3cc63300 Mon Sep 17 00:00:00 2001 From: Hans Hagen Date: Fri, 17 Feb 2017 14:32:57 +0100 Subject: 2017-02-17 13:47:00 --- doc/context/documents/general/qrcs/setup-cs.pdf | Bin 799581 -> 799320 bytes doc/context/documents/general/qrcs/setup-de.pdf | Bin 801364 -> 801321 bytes doc/context/documents/general/qrcs/setup-en.pdf | Bin 804340 -> 804334 bytes doc/context/documents/general/qrcs/setup-fr.pdf | Bin 799393 -> 799309 bytes doc/context/documents/general/qrcs/setup-it.pdf | Bin 800062 -> 799949 bytes doc/context/documents/general/qrcs/setup-nl.pdf | Bin 796873 -> 796734 bytes doc/context/documents/general/qrcs/setup-ro.pdf | Bin 796511 -> 796677 bytes scripts/context/lua/mtxlibs.lua | 2 + scripts/context/lua/mtxrun.lua | 1315 ++++++++++++++++---- scripts/context/stubs/mswin/mtxrun.lua | 1315 ++++++++++++++++---- scripts/context/stubs/unix/mtxrun | 1315 ++++++++++++++++---- scripts/context/stubs/win64/mtxrun.lua | 1315 ++++++++++++++++---- tex/context/base/mkii/cont-new.mkii | 2 +- tex/context/base/mkii/context.mkii | 2 +- tex/context/base/mkiv/cont-new.mkiv | 2 +- tex/context/base/mkiv/context.mkiv | 2 +- tex/context/base/mkiv/font-ocl.lua | 2 +- tex/context/base/mkiv/luat-fmt.lua | 53 +- tex/context/base/mkiv/node-rul.lua | 1 - tex/context/base/mkiv/status-files.pdf | Bin 25624 -> 25656 bytes tex/context/base/mkiv/status-lua.pdf | Bin 372482 -> 372480 bytes tex/context/base/mkiv/util-sbx.lua | 36 +- tex/context/interface/mkiv/i-context.pdf | Bin 804340 -> 804334 bytes tex/context/interface/mkiv/i-readme.pdf | Bin 60766 -> 60764 bytes tex/generic/context/luatex/luatex-fonts-merged.lua | 4 +- 25 files changed, 4304 insertions(+), 1062 deletions(-) diff --git a/doc/context/documents/general/qrcs/setup-cs.pdf b/doc/context/documents/general/qrcs/setup-cs.pdf index 86ad7dd61..4f0705c25 100644 Binary files a/doc/context/documents/general/qrcs/setup-cs.pdf and b/doc/context/documents/general/qrcs/setup-cs.pdf differ diff --git a/doc/context/documents/general/qrcs/setup-de.pdf b/doc/context/documents/general/qrcs/setup-de.pdf index ff512abba..6357aa4b7 100644 Binary files a/doc/context/documents/general/qrcs/setup-de.pdf and b/doc/context/documents/general/qrcs/setup-de.pdf differ diff --git a/doc/context/documents/general/qrcs/setup-en.pdf b/doc/context/documents/general/qrcs/setup-en.pdf index a3be00bdf..52980cfbd 100644 Binary files a/doc/context/documents/general/qrcs/setup-en.pdf and b/doc/context/documents/general/qrcs/setup-en.pdf differ diff --git a/doc/context/documents/general/qrcs/setup-fr.pdf b/doc/context/documents/general/qrcs/setup-fr.pdf index cfed2bafc..3d4e175b2 100644 Binary files a/doc/context/documents/general/qrcs/setup-fr.pdf and b/doc/context/documents/general/qrcs/setup-fr.pdf differ diff --git a/doc/context/documents/general/qrcs/setup-it.pdf b/doc/context/documents/general/qrcs/setup-it.pdf index 9b5e83cbe..70a6427d1 100644 Binary files a/doc/context/documents/general/qrcs/setup-it.pdf and b/doc/context/documents/general/qrcs/setup-it.pdf differ diff --git a/doc/context/documents/general/qrcs/setup-nl.pdf b/doc/context/documents/general/qrcs/setup-nl.pdf index 85e91d228..fb3ece5f3 100644 Binary files a/doc/context/documents/general/qrcs/setup-nl.pdf and b/doc/context/documents/general/qrcs/setup-nl.pdf differ diff --git a/doc/context/documents/general/qrcs/setup-ro.pdf b/doc/context/documents/general/qrcs/setup-ro.pdf index 21a3d7cee..9378cf85d 100644 Binary files a/doc/context/documents/general/qrcs/setup-ro.pdf and b/doc/context/documents/general/qrcs/setup-ro.pdf differ diff --git a/scripts/context/lua/mtxlibs.lua b/scripts/context/lua/mtxlibs.lua index 6eee507ae..5e547cdee 100644 --- a/scripts/context/lua/mtxlibs.lua +++ b/scripts/context/lua/mtxlibs.lua @@ -68,6 +68,7 @@ local owntree = ownpath local ownlibs = { "l-lua.lua", + "l-sandbox.lua", "l-package.lua", "l-lpeg.lua", "l-function.lua", @@ -103,6 +104,7 @@ local ownlibs = { "util-mrg.lua", "util-tpl.lua", + "util-sbx.lua", "util-env.lua", -- "luat-env.lua", -- not relevant outside context diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 6d9f4afba..f85c8eba0 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -176,6 +176,275 @@ else end +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 10855, stripped down to: 6942 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] +end +if TEXENGINE=="luajittex" or not ffi then + local ok + ok,ffi=pcall(require,"ffi") +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end + + end -- of closure do -- create closure to overcome 200 locals limit @@ -9632,123 +9901,761 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-mrg"] = package.loaded["util-mrg"] or true +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7313, stripped down to: 4076 -if not modules then modules={} end modules ['util-mrg']={ +if not modules then modules={} end modules ['util-tpl']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local gsub,format=string.gsub,string.format -local concat=table.concat -local type,next=type,next -local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg -local lpegmatch,patterns=lpeg.match,lpeg.patterns -utilities=utilities or {} -local merger=utilities.merger or {} -utilities.merger=merger -merger.strip_comment=true -local report=logs.reporter("system","merge") -utilities.report=report -local m_begin_merge="begin library merge" -local m_end_merge="end library merge" -local m_begin_closure="do -- create closure to overcome 200 locals limit" -local m_end_closure="end -- of closure" -local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" -local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" -local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" -local m_report=[[ --- used libraries : %s --- skipped libraries : %s --- original bytes : %s --- stripped bytes : %s -]] -local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] -local function self_fake() - return m_faked -end -local function self_nothing() - return "" -end -local function self_load(name) - local data=io.loaddata(name) or "" - if data=="" then - report("unknown file %a",name) +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" else - report("inserting file %a",name) + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end end - return data or "" -end -local space=patterns.space -local eol=patterns.newline -local equals=P("=")^0 -local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 -local close=P("]")*C(equals)*P("]") -local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) -local longstring=open*(1-closeeq)^0*close -local quoted=patterns.quoted -local digit=patterns.digit -local emptyline=space^0*eol -local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") -local operator2=S("*+/") -local operator3=S("-") -local operator4=P("..") -local separator=S(",;") -local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" -local strings=quoted -local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" -local longstr=longstring -local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" -local optionalspaces=space^0/"" -local mandatespaces=space^1/"" -local optionalspacing=(eol+space)^0/"" -local mandatespacing=(eol+space)^1/"" -local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces -local lines=emptyline^2/"\n" -local spaces=(space*space)/" " -local compact=Cs (( - ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 -)^1 ) -local strip=Cs((emptyline^2/"\n"+1)^0) -local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) -function merger.compact(data) - return lpegmatch(strip,lpegmatch(compact,data)) end -local function self_compact(data) - local delta=0 - if merger.strip_comment then - local before=#data - data=lpegmatch(compact,data) - data=lpegmatch(strip,data) - local after=#data - delta=before-after - report("original size %s, compacted to %s, stripped %s",before,after,delta) - data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) end - return lpegmatch(stripreturn,data) or data,delta end -local function self_save(name,data) - if data~="" then - io.savedata(name,data) - report("saving %s with size %s",name,#data) +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) end end -local function self_swap(data,code) - return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end -local function self_libs(libs,list) - local result,f,frozen,foundpath={},nil,false,nil - result[#result+1]="\n" - if type(libs)=='string' then libs={ libs } end - if type(list)=='string' then list={ list } end - for i=1,#libs do - local lib=libs[i] - for j=1,#list do - local pth=gsub(list[j],"\\","/") +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20222, stripped down to: 13792 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + root=collapsepath(expandname(root)) + if platform=="windows" then + root=lower(root) + end + validroots[root]=what=="write" or false + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + if platform=="windows" then + asked=lower(asked) + end + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + else + if filenamelogger then + filenamelogger(name,"*",name,false) + end + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-mrg"] = package.loaded["util-mrg"] or true + +-- original size: 7985, stripped down to: 6153 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +merger.strip_comment=true +local report=logs.reporter("system","merge") +utilities.report=report +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + report("unknown file %a",name) + else + report("inserting file %a",name) + end + return data or "" +end +local space=patterns.space +local eol=patterns.newline +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local quoted=patterns.quoted +local digit=patterns.digit +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local operator4=P("..") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local optionalspaces=space^0/"" +local mandatespaces=space^1/"" +local optionalspacing=(eol+space)^0/"" +local mandatespacing=(eol+space)^1/"" +local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end +local function self_compact(data) + local delta=0 + if merger.strip_comment then + local before=#data + data=lpegmatch(compact,data) + data=lpegmatch(strip,data) + local after=#data + delta=before-after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data,delta +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + report("saving %s with size %s",name,#data) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") report("checking library path %a",pth) local name=pth.."/"..lib if lfs.isfile(name) then @@ -9805,151 +10712,6 @@ function merger.selfclean(name) end -end -- of closure - -do -- create closure to overcome 200 locals limit - -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - end -- of closure do -- create closure to overcome 200 locals limit @@ -18976,7 +19738,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 8391, stripped down to: 6761 +-- original size: 9392, stripped down to: 7485 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -19023,7 +19785,7 @@ local function secondaryflags() end return concat(flags," ") end -local template=[[--ini %primaryflags% --lua="%luafile%" "%texfile%" %secondaryflags% %dump% %redirect%]] +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ primaryflags="string", secondaryflags="string", @@ -19109,8 +19871,8 @@ function environment.make_format(name,arguments) local specification={ primaryflags=primaryflags(), secondaryflags=secondaryflags(), - luafile=usedluastub, - texfile=fulltexsourcename, + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } local runner=runners[engine] @@ -19119,7 +19881,7 @@ function environment.make_format(name,arguments) elseif silent then statistics.starttiming() specification.redirect="> temp.log" - local result=makeformat(specification) + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -19128,7 +19890,7 @@ function environment.make_format(name,arguments) end os.remove("temp.log") else - makeformat(specification) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -19141,6 +19903,30 @@ function environment.make_format(name,arguments) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -19162,9 +19948,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -19173,10 +19968,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 816868 --- stripped bytes : 299010 +-- original bytes : 848946 +-- stripped bytes : 309630 -- end library merge @@ -19200,6 +19995,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -19233,8 +20029,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 6d9f4afba..f85c8eba0 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -176,6 +176,275 @@ else end +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 10855, stripped down to: 6942 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] +end +if TEXENGINE=="luajittex" or not ffi then + local ok + ok,ffi=pcall(require,"ffi") +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end + + end -- of closure do -- create closure to overcome 200 locals limit @@ -9632,123 +9901,761 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-mrg"] = package.loaded["util-mrg"] or true +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7313, stripped down to: 4076 -if not modules then modules={} end modules ['util-mrg']={ +if not modules then modules={} end modules ['util-tpl']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local gsub,format=string.gsub,string.format -local concat=table.concat -local type,next=type,next -local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg -local lpegmatch,patterns=lpeg.match,lpeg.patterns -utilities=utilities or {} -local merger=utilities.merger or {} -utilities.merger=merger -merger.strip_comment=true -local report=logs.reporter("system","merge") -utilities.report=report -local m_begin_merge="begin library merge" -local m_end_merge="end library merge" -local m_begin_closure="do -- create closure to overcome 200 locals limit" -local m_end_closure="end -- of closure" -local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" -local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" -local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" -local m_report=[[ --- used libraries : %s --- skipped libraries : %s --- original bytes : %s --- stripped bytes : %s -]] -local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] -local function self_fake() - return m_faked -end -local function self_nothing() - return "" -end -local function self_load(name) - local data=io.loaddata(name) or "" - if data=="" then - report("unknown file %a",name) +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" else - report("inserting file %a",name) + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end end - return data or "" -end -local space=patterns.space -local eol=patterns.newline -local equals=P("=")^0 -local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 -local close=P("]")*C(equals)*P("]") -local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) -local longstring=open*(1-closeeq)^0*close -local quoted=patterns.quoted -local digit=patterns.digit -local emptyline=space^0*eol -local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") -local operator2=S("*+/") -local operator3=S("-") -local operator4=P("..") -local separator=S(",;") -local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" -local strings=quoted -local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" -local longstr=longstring -local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" -local optionalspaces=space^0/"" -local mandatespaces=space^1/"" -local optionalspacing=(eol+space)^0/"" -local mandatespacing=(eol+space)^1/"" -local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces -local lines=emptyline^2/"\n" -local spaces=(space*space)/" " -local compact=Cs (( - ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 -)^1 ) -local strip=Cs((emptyline^2/"\n"+1)^0) -local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) -function merger.compact(data) - return lpegmatch(strip,lpegmatch(compact,data)) end -local function self_compact(data) - local delta=0 - if merger.strip_comment then - local before=#data - data=lpegmatch(compact,data) - data=lpegmatch(strip,data) - local after=#data - delta=before-after - report("original size %s, compacted to %s, stripped %s",before,after,delta) - data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) end - return lpegmatch(stripreturn,data) or data,delta end -local function self_save(name,data) - if data~="" then - io.savedata(name,data) - report("saving %s with size %s",name,#data) +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) end end -local function self_swap(data,code) - return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end -local function self_libs(libs,list) - local result,f,frozen,foundpath={},nil,false,nil - result[#result+1]="\n" - if type(libs)=='string' then libs={ libs } end - if type(list)=='string' then list={ list } end - for i=1,#libs do - local lib=libs[i] - for j=1,#list do - local pth=gsub(list[j],"\\","/") +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20222, stripped down to: 13792 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + root=collapsepath(expandname(root)) + if platform=="windows" then + root=lower(root) + end + validroots[root]=what=="write" or false + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + if platform=="windows" then + asked=lower(asked) + end + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + else + if filenamelogger then + filenamelogger(name,"*",name,false) + end + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-mrg"] = package.loaded["util-mrg"] or true + +-- original size: 7985, stripped down to: 6153 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +merger.strip_comment=true +local report=logs.reporter("system","merge") +utilities.report=report +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + report("unknown file %a",name) + else + report("inserting file %a",name) + end + return data or "" +end +local space=patterns.space +local eol=patterns.newline +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local quoted=patterns.quoted +local digit=patterns.digit +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local operator4=P("..") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local optionalspaces=space^0/"" +local mandatespaces=space^1/"" +local optionalspacing=(eol+space)^0/"" +local mandatespacing=(eol+space)^1/"" +local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end +local function self_compact(data) + local delta=0 + if merger.strip_comment then + local before=#data + data=lpegmatch(compact,data) + data=lpegmatch(strip,data) + local after=#data + delta=before-after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data,delta +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + report("saving %s with size %s",name,#data) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") report("checking library path %a",pth) local name=pth.."/"..lib if lfs.isfile(name) then @@ -9805,151 +10712,6 @@ function merger.selfclean(name) end -end -- of closure - -do -- create closure to overcome 200 locals limit - -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - end -- of closure do -- create closure to overcome 200 locals limit @@ -18976,7 +19738,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 8391, stripped down to: 6761 +-- original size: 9392, stripped down to: 7485 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -19023,7 +19785,7 @@ local function secondaryflags() end return concat(flags," ") end -local template=[[--ini %primaryflags% --lua="%luafile%" "%texfile%" %secondaryflags% %dump% %redirect%]] +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ primaryflags="string", secondaryflags="string", @@ -19109,8 +19871,8 @@ function environment.make_format(name,arguments) local specification={ primaryflags=primaryflags(), secondaryflags=secondaryflags(), - luafile=usedluastub, - texfile=fulltexsourcename, + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } local runner=runners[engine] @@ -19119,7 +19881,7 @@ function environment.make_format(name,arguments) elseif silent then statistics.starttiming() specification.redirect="> temp.log" - local result=makeformat(specification) + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -19128,7 +19890,7 @@ function environment.make_format(name,arguments) end os.remove("temp.log") else - makeformat(specification) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -19141,6 +19903,30 @@ function environment.make_format(name,arguments) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -19162,9 +19948,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -19173,10 +19968,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 816868 --- stripped bytes : 299010 +-- original bytes : 848946 +-- stripped bytes : 309630 -- end library merge @@ -19200,6 +19995,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -19233,8 +20029,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 6d9f4afba..f85c8eba0 100644 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -176,6 +176,275 @@ else end +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 10855, stripped down to: 6942 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] +end +if TEXENGINE=="luajittex" or not ffi then + local ok + ok,ffi=pcall(require,"ffi") +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end + + end -- of closure do -- create closure to overcome 200 locals limit @@ -9632,123 +9901,761 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-mrg"] = package.loaded["util-mrg"] or true +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7313, stripped down to: 4076 -if not modules then modules={} end modules ['util-mrg']={ +if not modules then modules={} end modules ['util-tpl']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local gsub,format=string.gsub,string.format -local concat=table.concat -local type,next=type,next -local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg -local lpegmatch,patterns=lpeg.match,lpeg.patterns -utilities=utilities or {} -local merger=utilities.merger or {} -utilities.merger=merger -merger.strip_comment=true -local report=logs.reporter("system","merge") -utilities.report=report -local m_begin_merge="begin library merge" -local m_end_merge="end library merge" -local m_begin_closure="do -- create closure to overcome 200 locals limit" -local m_end_closure="end -- of closure" -local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" -local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" -local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" -local m_report=[[ --- used libraries : %s --- skipped libraries : %s --- original bytes : %s --- stripped bytes : %s -]] -local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] -local function self_fake() - return m_faked -end -local function self_nothing() - return "" -end -local function self_load(name) - local data=io.loaddata(name) or "" - if data=="" then - report("unknown file %a",name) +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" else - report("inserting file %a",name) + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end end - return data or "" -end -local space=patterns.space -local eol=patterns.newline -local equals=P("=")^0 -local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 -local close=P("]")*C(equals)*P("]") -local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) -local longstring=open*(1-closeeq)^0*close -local quoted=patterns.quoted -local digit=patterns.digit -local emptyline=space^0*eol -local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") -local operator2=S("*+/") -local operator3=S("-") -local operator4=P("..") -local separator=S(",;") -local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" -local strings=quoted -local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" -local longstr=longstring -local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" -local optionalspaces=space^0/"" -local mandatespaces=space^1/"" -local optionalspacing=(eol+space)^0/"" -local mandatespacing=(eol+space)^1/"" -local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces -local lines=emptyline^2/"\n" -local spaces=(space*space)/" " -local compact=Cs (( - ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 -)^1 ) -local strip=Cs((emptyline^2/"\n"+1)^0) -local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) -function merger.compact(data) - return lpegmatch(strip,lpegmatch(compact,data)) end -local function self_compact(data) - local delta=0 - if merger.strip_comment then - local before=#data - data=lpegmatch(compact,data) - data=lpegmatch(strip,data) - local after=#data - delta=before-after - report("original size %s, compacted to %s, stripped %s",before,after,delta) - data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) end - return lpegmatch(stripreturn,data) or data,delta end -local function self_save(name,data) - if data~="" then - io.savedata(name,data) - report("saving %s with size %s",name,#data) +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) end end -local function self_swap(data,code) - return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end -local function self_libs(libs,list) - local result,f,frozen,foundpath={},nil,false,nil - result[#result+1]="\n" - if type(libs)=='string' then libs={ libs } end - if type(list)=='string' then list={ list } end - for i=1,#libs do - local lib=libs[i] - for j=1,#list do - local pth=gsub(list[j],"\\","/") +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20222, stripped down to: 13792 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + root=collapsepath(expandname(root)) + if platform=="windows" then + root=lower(root) + end + validroots[root]=what=="write" or false + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + if platform=="windows" then + asked=lower(asked) + end + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + else + if filenamelogger then + filenamelogger(name,"*",name,false) + end + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-mrg"] = package.loaded["util-mrg"] or true + +-- original size: 7985, stripped down to: 6153 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +merger.strip_comment=true +local report=logs.reporter("system","merge") +utilities.report=report +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + report("unknown file %a",name) + else + report("inserting file %a",name) + end + return data or "" +end +local space=patterns.space +local eol=patterns.newline +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local quoted=patterns.quoted +local digit=patterns.digit +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local operator4=P("..") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local optionalspaces=space^0/"" +local mandatespaces=space^1/"" +local optionalspacing=(eol+space)^0/"" +local mandatespacing=(eol+space)^1/"" +local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end +local function self_compact(data) + local delta=0 + if merger.strip_comment then + local before=#data + data=lpegmatch(compact,data) + data=lpegmatch(strip,data) + local after=#data + delta=before-after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data,delta +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + report("saving %s with size %s",name,#data) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") report("checking library path %a",pth) local name=pth.."/"..lib if lfs.isfile(name) then @@ -9805,151 +10712,6 @@ function merger.selfclean(name) end -end -- of closure - -do -- create closure to overcome 200 locals limit - -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - end -- of closure do -- create closure to overcome 200 locals limit @@ -18976,7 +19738,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 8391, stripped down to: 6761 +-- original size: 9392, stripped down to: 7485 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -19023,7 +19785,7 @@ local function secondaryflags() end return concat(flags," ") end -local template=[[--ini %primaryflags% --lua="%luafile%" "%texfile%" %secondaryflags% %dump% %redirect%]] +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ primaryflags="string", secondaryflags="string", @@ -19109,8 +19871,8 @@ function environment.make_format(name,arguments) local specification={ primaryflags=primaryflags(), secondaryflags=secondaryflags(), - luafile=usedluastub, - texfile=fulltexsourcename, + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } local runner=runners[engine] @@ -19119,7 +19881,7 @@ function environment.make_format(name,arguments) elseif silent then statistics.starttiming() specification.redirect="> temp.log" - local result=makeformat(specification) + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -19128,7 +19890,7 @@ function environment.make_format(name,arguments) end os.remove("temp.log") else - makeformat(specification) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -19141,6 +19903,30 @@ function environment.make_format(name,arguments) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -19162,9 +19948,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -19173,10 +19968,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 816868 --- stripped bytes : 299010 +-- original bytes : 848946 +-- stripped bytes : 309630 -- end library merge @@ -19200,6 +19995,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -19233,8 +20029,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua index 6d9f4afba..f85c8eba0 100644 --- a/scripts/context/stubs/win64/mtxrun.lua +++ b/scripts/context/stubs/win64/mtxrun.lua @@ -176,6 +176,275 @@ else end +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 10855, stripped down to: 6942 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] +end +if TEXENGINE=="luajittex" or not ffi then + local ok + ok,ffi=pcall(require,"ffi") +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end + + end -- of closure do -- create closure to overcome 200 locals limit @@ -9632,123 +9901,761 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-mrg"] = package.loaded["util-mrg"] or true +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7313, stripped down to: 4076 -if not modules then modules={} end modules ['util-mrg']={ +if not modules then modules={} end modules ['util-tpl']={ version=1.001, comment="companion to luat-lib.mkiv", author="Hans Hagen, PRAGMA-ADE, Hasselt NL", copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local gsub,format=string.gsub,string.format -local concat=table.concat -local type,next=type,next -local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg -local lpegmatch,patterns=lpeg.match,lpeg.patterns -utilities=utilities or {} -local merger=utilities.merger or {} -utilities.merger=merger -merger.strip_comment=true -local report=logs.reporter("system","merge") -utilities.report=report -local m_begin_merge="begin library merge" -local m_end_merge="end library merge" -local m_begin_closure="do -- create closure to overcome 200 locals limit" -local m_end_closure="end -- of closure" -local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" -local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" -local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" -local m_report=[[ --- used libraries : %s --- skipped libraries : %s --- original bytes : %s --- stripped bytes : %s -]] -local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] -local function self_fake() - return m_faked -end -local function self_nothing() - return "" -end -local function self_load(name) - local data=io.loaddata(name) or "" - if data=="" then - report("unknown file %a",name) +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" else - report("inserting file %a",name) + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end end - return data or "" -end -local space=patterns.space -local eol=patterns.newline -local equals=P("=")^0 -local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 -local close=P("]")*C(equals)*P("]") -local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) -local longstring=open*(1-closeeq)^0*close -local quoted=patterns.quoted -local digit=patterns.digit -local emptyline=space^0*eol -local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") -local operator2=S("*+/") -local operator3=S("-") -local operator4=P("..") -local separator=S(",;") -local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" -local strings=quoted -local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" -local longstr=longstring -local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" -local optionalspaces=space^0/"" -local mandatespaces=space^1/"" -local optionalspacing=(eol+space)^0/"" -local mandatespacing=(eol+space)^1/"" -local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces -local lines=emptyline^2/"\n" -local spaces=(space*space)/" " -local compact=Cs (( - ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 -)^1 ) -local strip=Cs((emptyline^2/"\n"+1)^0) -local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) -function merger.compact(data) - return lpegmatch(strip,lpegmatch(compact,data)) end -local function self_compact(data) - local delta=0 - if merger.strip_comment then - local before=#data - data=lpegmatch(compact,data) - data=lpegmatch(strip,data) - local after=#data - delta=before-after - report("original size %s, compacted to %s, stripped %s",before,after,delta) - data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) end - return lpegmatch(stripreturn,data) or data,delta end -local function self_save(name,data) - if data~="" then - io.savedata(name,data) - report("saving %s with size %s",name,#data) +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) end end -local function self_swap(data,code) - return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end -local function self_libs(libs,list) - local result,f,frozen,foundpath={},nil,false,nil - result[#result+1]="\n" - if type(libs)=='string' then libs={ libs } end - if type(list)=='string' then list={ list } end - for i=1,#libs do - local lib=libs[i] - for j=1,#list do - local pth=gsub(list[j],"\\","/") +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20222, stripped down to: 13792 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + root=collapsepath(expandname(root)) + if platform=="windows" then + root=lower(root) + end + validroots[root]=what=="write" or false + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + if platform=="windows" then + asked=lower(asked) + end + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + else + if filenamelogger then + filenamelogger(name,"*",name,false) + end + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-mrg"] = package.loaded["util-mrg"] or true + +-- original size: 7985, stripped down to: 6153 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +merger.strip_comment=true +local report=logs.reporter("system","merge") +utilities.report=report +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + report("unknown file %a",name) + else + report("inserting file %a",name) + end + return data or "" +end +local space=patterns.space +local eol=patterns.newline +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local quoted=patterns.quoted +local digit=patterns.digit +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local operator4=P("..") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local optionalspaces=space^0/"" +local mandatespaces=space^1/"" +local optionalspacing=(eol+space)^0/"" +local mandatespacing=(eol+space)^1/"" +local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end +local function self_compact(data) + local delta=0 + if merger.strip_comment then + local before=#data + data=lpegmatch(compact,data) + data=lpegmatch(strip,data) + local after=#data + delta=before-after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data,delta +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + report("saving %s with size %s",name,#data) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") report("checking library path %a",pth) local name=pth.."/"..lib if lfs.isfile(name) then @@ -9805,151 +10712,6 @@ function merger.selfclean(name) end -end -- of closure - -do -- create closure to overcome 200 locals limit - -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - end -- of closure do -- create closure to overcome 200 locals limit @@ -18976,7 +19738,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 8391, stripped down to: 6761 +-- original size: 9392, stripped down to: 7485 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -19023,7 +19785,7 @@ local function secondaryflags() end return concat(flags," ") end -local template=[[--ini %primaryflags% --lua="%luafile%" "%texfile%" %secondaryflags% %dump% %redirect%]] +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] local checkers={ primaryflags="string", secondaryflags="string", @@ -19109,8 +19871,8 @@ function environment.make_format(name,arguments) local specification={ primaryflags=primaryflags(), secondaryflags=secondaryflags(), - luafile=usedluastub, - texfile=fulltexsourcename, + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), dump=os.platform=="unix" and "\\\\dump" or "\\dump", } local runner=runners[engine] @@ -19119,7 +19881,7 @@ function environment.make_format(name,arguments) elseif silent then statistics.starttiming() specification.redirect="> temp.log" - local result=makeformat(specification) + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -19128,7 +19890,7 @@ function environment.make_format(name,arguments) end os.remove("temp.log") else - makeformat(specification) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -19141,6 +19903,30 @@ function environment.make_format(name,arguments) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -19162,9 +19948,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -19173,10 +19968,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 816868 --- stripped bytes : 299010 +-- original bytes : 848946 +-- stripped bytes : 309630 -- end library merge @@ -19200,6 +19995,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -19233,8 +20029,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) diff --git a/tex/context/base/mkii/cont-new.mkii b/tex/context/base/mkii/cont-new.mkii index f08a0cb1c..360538e07 100644 --- a/tex/context/base/mkii/cont-new.mkii +++ b/tex/context/base/mkii/cont-new.mkii @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2017.02.17 10:17} +\newcontextversion{2017.02.17 13:41} %D This file is loaded at runtime, thereby providing an %D excellent place for hacks, patches, extensions and new diff --git a/tex/context/base/mkii/context.mkii b/tex/context/base/mkii/context.mkii index 70ca44d90..48aec809e 100644 --- a/tex/context/base/mkii/context.mkii +++ b/tex/context/base/mkii/context.mkii @@ -20,7 +20,7 @@ %D your styles an modules. \edef\contextformat {\jobname} -\edef\contextversion{2017.02.17 10:17} +\edef\contextversion{2017.02.17 13:41} %D For those who want to use this: diff --git a/tex/context/base/mkiv/cont-new.mkiv b/tex/context/base/mkiv/cont-new.mkiv index e83b97c3a..dbbb8154a 100644 --- a/tex/context/base/mkiv/cont-new.mkiv +++ b/tex/context/base/mkiv/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2017.02.17 10:17} +\newcontextversion{2017.02.17 13:41} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/mkiv/context.mkiv b/tex/context/base/mkiv/context.mkiv index e8aae9575..e4bf79f0b 100644 --- a/tex/context/base/mkiv/context.mkiv +++ b/tex/context/base/mkiv/context.mkiv @@ -39,7 +39,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2017.02.17 10:17} +\edef\contextversion{2017.02.17 13:41} \edef\contextkind {beta} %D For those who want to use this: diff --git a/tex/context/base/mkiv/font-ocl.lua b/tex/context/base/mkiv/font-ocl.lua index 824f09125..3583e15d0 100644 --- a/tex/context/base/mkiv/font-ocl.lua +++ b/tex/context/base/mkiv/font-ocl.lua @@ -269,7 +269,7 @@ do reporter = report_svg, } - if notrunner then + if not runner then -- -- poor mans variant for generic: -- diff --git a/tex/context/base/mkiv/luat-fmt.lua b/tex/context/base/mkiv/luat-fmt.lua index fe3c1042c..f61c659fa 100644 --- a/tex/context/base/mkiv/luat-fmt.lua +++ b/tex/context/base/mkiv/luat-fmt.lua @@ -51,7 +51,8 @@ end -- The silent option is Taco. It's a bit of a hack because we cannot yet mess -- with directives. In fact, I could probably clean up the maker a bit by now. -local template = [[--ini %primaryflags% --lua="%luafile%" "%texfile%" %secondaryflags% %dump% %redirect%]] +local template = [[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] + local checkers = { primaryflags = "string", secondaryflags = "string", @@ -146,8 +147,8 @@ function environment.make_format(name,arguments) local specification = { primaryflags = primaryflags(), secondaryflags = secondaryflags(), - luafile = usedluastub, - texfile = fulltexsourcename, + luafile = quoted(usedluastub), + texfile = quoted(fulltexsourcename), dump = os.platform == "unix" and "\\\\dump" or "\\dump", } local runner = runners[engine] @@ -156,7 +157,7 @@ function environment.make_format(name,arguments) elseif silent then statistics.starttiming() specification.redirect = "> temp.log" - local result = makeformat(specification) + local result = runner(specification) local runtime = statistics.stoptiming() if result ~= 0 then print(format("%s silent make > fatal error when making format %q",engine,name)) -- we use a basic print @@ -165,7 +166,7 @@ function environment.make_format(name,arguments) end os.remove("temp.log") else - makeformat(specification) + runner(specification) end -- remove related mem files local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" @@ -181,6 +182,33 @@ function environment.make_format(name,arguments) lfs.chdir(olddir) end +local template = [[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] + +local checkers = { + flags = "string", + more = "string", + fmtfile = "readable", -- "cache" + luafile = "readable", -- "cache" + texfile = "readable", -- "cache" +} + +local runners = { + luatex = sandbox.registerrunner { + name = "run luatex format", + program = "luatex", + template = template, + checkers = checkers, + reporter = report_format, + }, + luajittex = sandbox.registerrunner { + name = "run luajittex format", + program = "luajittex", + template = template, + checkers = checkers, + reporter = report_format, + }, +} + function environment.run_format(name,data,more) if name and name ~= "" then local engine = environment.ownmain or "luatex" @@ -202,9 +230,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command = format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner = runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags = primaryflags(), + fmtfile = quoted(barename), + luafile = quoted(luaname), + texfile = quoted(data), + more = more, + } + end end end end diff --git a/tex/context/base/mkiv/node-rul.lua b/tex/context/base/mkiv/node-rul.lua index 8646fe447..4ec651d3b 100644 --- a/tex/context/base/mkiv/node-rul.lua +++ b/tex/context/base/mkiv/node-rul.lua @@ -118,7 +118,6 @@ local setmetatableindex = table.setmetatableindex local striprange = nodes.striprange local processwords = nodes.processwords -local processranges = nodes.processranges -- diff --git a/tex/context/base/mkiv/status-files.pdf b/tex/context/base/mkiv/status-files.pdf index c01ed7449..b70fcf6f0 100644 Binary files a/tex/context/base/mkiv/status-files.pdf and b/tex/context/base/mkiv/status-files.pdf differ diff --git a/tex/context/base/mkiv/status-lua.pdf b/tex/context/base/mkiv/status-lua.pdf index d9528aa62..c47682fe6 100644 Binary files a/tex/context/base/mkiv/status-lua.pdf and b/tex/context/base/mkiv/status-lua.pdf differ diff --git a/tex/context/base/mkiv/util-sbx.lua b/tex/context/base/mkiv/util-sbx.lua index 03593cb9b..9cedcc1a6 100644 --- a/tex/context/base/mkiv/util-sbx.lua +++ b/tex/context/base/mkiv/util-sbx.lua @@ -186,9 +186,16 @@ local function validfilename(name,what) end local function readable(name,finalized) - if platform == "windows" then - name = lower(name) -- we assume ascii names - end +-- if platform == "windows" then -- yes or no +-- name = lower(name) -- we assume ascii names +-- end + return validfilename(name,"r") +end + +local function normalizedreadable(name,finalized) +-- if platform == "windows" then -- yes or no +-- name = lower(name) -- we assume ascii names +-- end local valid = validfilename(name,"r") if valid then return normalized(valid) @@ -196,18 +203,27 @@ local function readable(name,finalized) end local function writeable(name,finalized) - if platform == "windows" then - name = lower(name) -- we assume ascii names - end +-- if platform == "windows" then +-- name = lower(name) -- we assume ascii names +-- end + return validfilename(name,"w") +end + +local function normalizedwriteable(name,finalized) +-- if platform == "windows" then +-- name = lower(name) -- we assume ascii names +-- end local valid = validfilename(name,"w") if valid then return normalized(valid) end end -validators.writeable = writeable -validators.readable = readable -validators.filename = readable +validators.readable = readable +validators.writeable = normalizedwriteable +validators.normalizedreadable = normalizedreadable +validators.normalizedwriteable = writeable +validators.filename = readable table.setmetatableindex(validators,function(t,k) if k then @@ -384,7 +400,7 @@ function sandbox.registerrunner(specification) end local name = specification.name if type(name) ~= "string" then - report("invalid name, string expected") + report("invalid name, string expected",name) return end if validrunners[name] then diff --git a/tex/context/interface/mkiv/i-context.pdf b/tex/context/interface/mkiv/i-context.pdf index a3be00bdf..52980cfbd 100644 Binary files a/tex/context/interface/mkiv/i-context.pdf and b/tex/context/interface/mkiv/i-context.pdf differ diff --git a/tex/context/interface/mkiv/i-readme.pdf b/tex/context/interface/mkiv/i-readme.pdf index bcb7a4fb6..5c51ce0b6 100644 Binary files a/tex/context/interface/mkiv/i-readme.pdf and b/tex/context/interface/mkiv/i-readme.pdf differ diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 8ececb5a9..952ed19ac 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : c:/data/develop/context/sources/luatex-fonts-merged.lua -- parent file : c:/data/develop/context/sources/luatex-fonts.lua --- merge date : 02/17/17 10:17:41 +-- merge date : 02/17/17 13:41:40 do -- begin closure to overcome local limits and interference @@ -23888,7 +23888,7 @@ do template="--shell > temp-otf-svg-shape.log", reporter=report_svg, } - if notrunner then + if not runner then runner=function() return io.open("inkscape --shell > temp-otf-svg-shape.log","w") end -- cgit v1.2.3