diff options
Diffstat (limited to 'tex')
52 files changed, 2178 insertions, 1066 deletions
diff --git a/tex/context/base/cont-new.mkiv b/tex/context/base/cont-new.mkiv index 6beccd49a..74025600b 100644 --- a/tex/context/base/cont-new.mkiv +++ b/tex/context/base/cont-new.mkiv @@ -11,7 +11,7 @@ %C therefore copyrighted by \PRAGMA. See mreadme.pdf for %C details. -\newcontextversion{2014.12.21 22:25} +\newcontextversion{2014.12.28 19:50} %D This file is loaded at runtime, thereby providing an excellent place for %D hacks, patches, extensions and new features. diff --git a/tex/context/base/cont-run.lua b/tex/context/base/cont-run.lua new file mode 100644 index 000000000..01090119d --- /dev/null +++ b/tex/context/base/cont-run.lua @@ -0,0 +1,241 @@ +if not modules then modules = { } end modules ['cont-run'] = { + version = 1.001, + comment = "companion to cont-yes.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- When a style is loaded there is a good change that we never enter +-- this code. + +local report = logs.reporter("system") + +local type, tostring = type, tostring + +local report = logs.reporter("sandbox","call") +local fastserialize = table.fastserialize +local quoted = string.quoted +local possiblepath = sandbox.possiblepath + +local qualified = { } +local writeable = { } +local readable = { } +local blocked = { } +local trace_files = false +local trace_calls = false +local nofcalls = 0 +local nofrejected = 0 +local logfilename = "sandbox.log" + +local function registerstats() + statistics.register("sandboxing", function() + if trace_files then + return string.format("%s calls, %s rejected, logdata in '%s'",nofcalls,nofrejected,logfilename) + else + return string.format("%s calls, %s rejected",nofcalls,nofrejected) + end + end) + registerstats = false +end + +local function logsandbox(details) + local comment = details.comment + local result = details.result + local arguments = details.arguments + for i=1,#arguments do + local argument = arguments[i] + local t = type(argument) + if t == "string" then + arguments[i] = quoted(argument) + if trace_files and possiblepath(argument) then + local q = qualified[argument] + if q then + local c = q[comment] + if c then + local r = c[result] + if r then + c[result] = r + 1 + else + c[result] = r + end + else + q[comment] = { + [result] = 1 + } + end + else + qualified[argument] = { + [comment] = { + [result] = 1 + } + } + end + end + elseif t == "table" then + arguments[i] = fastserialize(argument) + else + arguments[i] = tostring(argument) + end + end + if trace_calls then + report("%s(%,t) => %l",details.comment,arguments,result) + end + nofcalls = nofcalls + 1 + if not result then + nofrejected = nofrejected + 1 + end +end + +local ioopen = sandbox.original(io.open) + +local function logsandboxfiles(name,what,asked,okay) + -- we're only interested in permitted access + if not okay then + blocked [asked] = blocked [asked] or 0 + 1 + elseif what == "*" or what == "w" then + writeable[asked] = writeable[asked] or 0 + 1 + else + readable [asked] = readable [asked] or 0 + 1 + end +end + +function sandbox.logcalls() + if not trace_calls then + trace_calls = true + sandbox.setlogger(logsandbox) + if registerstats then + registerstats() + end + end +end + +function sandbox.logfiles() + if not trace_files then + trace_files = true + sandbox.setlogger(logsandbox) + sandbox.setfilenamelogger(logsandboxfiles) + luatex.registerstopactions(function() + table.save(logfilename,{ + calls = { + nofcalls = nofcalls, + nofrejected = nofrejected, + filenames = qualified, + }, + checkednames = { + readable = readable, + writeable = writeable, + blocked = blocked, + }, + }) + end) + if registerstats then + registerstats() + end + end +end + +trackers.register("sandbox.tracecalls",sandbox.logcalls) +trackers.register("sandbox.tracefiles",sandbox.logfiles) + +local sandboxing = environment.arguments.sandbox + +if sandboxing then + + report("enabling sandbox") + + sandbox.enable() + + if type(sandboxing) == "string" then + sandboxing = utilities.parsers.settings_to_hash(sandboxing) + if sandboxing.calls then + sandbox.logcalls() + end + if sandboxing.files then + sandbox.logfiles() + end + end + + -- Nicer would be if we could just disable write 18 and keep os.execute + -- which in fact we can do by defining write18 as macro instead of + -- primitive ... todo. + + -- We block some potential escapes from protection. + + context [[ + \let\primitive \relax + \let\normalprimitive\relax + \let\normalwrite \relax + ]] + +end + +function commands.processjob() + + environment.initializefilenames() -- todo: check if we really need to pre-prep the filename + + local arguments = environment.arguments + local suffix = environment.suffix + local filename = environment.filename -- hm, not inputfilename ! + + if suffix == "xml" or arguments.forcexml then + + -- Maybe we should move the preamble parsing here as it + -- can be part of (any) loaded (sub) file. The \starttext + -- wrapping might go away. + + report("processing as xml: %s",filename) + + context.starttext() + context.xmlprocess("main",filename,"") + context.stoptext() + + elseif suffix == "cld" or arguments.forcecld then + + report("processing as cld: %s",filename) + + context.runfile(filename) + + elseif suffix == "lua" or arguments.forcelua then + + -- The wrapping might go away. Why is is it there in the + -- first place. + + report("processing as lua: %s",filename) + + context.starttext() + context.ctxlua(string.format('dofile("%s")',filename)) + context.stoptext() + + elseif suffix == "mp" or arguments.forcemp then + + report("processing as metapost: %s",filename) + + context.starttext() + context.processMPfigurefile(filename) + context.stoptext() + + -- elseif suffix == "prep" then + -- + -- -- Why do we wrap here. Because it can be xml? Let's get rid + -- -- of prepping in general. + -- + -- context.starttext() + -- context.input(filename) + -- context.stoptext() + + else + + -- \writestatus{system}{processing as tex} + -- We have a regular tex file so no \starttext yet as we can + -- load fonts. + + -- context.enabletrackers { "resolvers.*" } + context.input(filename) + -- context.disabletrackers { "resolvers.*" } + + end + + context.finishjob() + +end diff --git a/tex/context/base/cont-run.mkiv b/tex/context/base/cont-run.mkiv new file mode 100644 index 000000000..fcca7b581 --- /dev/null +++ b/tex/context/base/cont-run.mkiv @@ -0,0 +1,20 @@ +%D \module +%D [ file=cont-run, +%D version=2014.12.26, +%D title=\CONTEXT\ Core Macros, +%D subtitle=Runner, +%D author=Hans Hagen, +%D date=\currentdate, +%D copyright={PRAGMA ADE \& \CONTEXT\ Development Team}] +%C +%C This module is part of the \CONTEXT\ macro||package and is +%C therefore copyrighted by \PRAGMA. See mreadme.pdf for +%C details. + +\writestatus{loading}{ConTeXt Core Macros / Runner} + +\unprotect + +\registerctxluafile{cont-run}{1.001} + +\protect \endinput diff --git a/tex/context/base/cont-yes.mkiv b/tex/context/base/cont-yes.mkiv index 1a10fc30e..bc8a0c64b 100644 --- a/tex/context/base/cont-yes.mkiv +++ b/tex/context/base/cont-yes.mkiv @@ -15,79 +15,83 @@ % wraping as we can assume proper styling. It's a left-over from % mkii that we need to get rid of. -\startluacode - - -- When a style is loaded there is a good change that we never enter - -- this code. - - local report = logs.reporter("system") - - environment.initializefilenames() -- todo: check if we really need to pre-prep the filename - - local arguments = environment.arguments - local suffix = environment.suffix - local filename = environment.filename -- hm, not inputfilename ! - - if suffix == "xml" or arguments.forcexml then - - -- Maybe we should move the preamble parsing here as it - -- can be part of (any) loaded (sub) file. The \starttext - -- wrapping might go away. - - report("processing as xml: %s",filename) - - context.starttext() - context.xmlprocess("main",filename,"") - context.stoptext() - - elseif suffix == "cld" or arguments.forcecld then - - report("processing as cld: %s",filename) - - context.runfile(filename) - - elseif suffix == "lua" or arguments.forcelua then - - -- The wrapping might go away. Why is is it there in the - -- first place. - - report("processing as lua: %s",filename) - - context.starttext() - context.ctxlua(string.format('dofile("%s")',filename)) - context.stoptext() - - elseif suffix == "mp" or arguments.forcemp then - - report("processing as metapost: %s",filename) - - context.starttext() - context.processMPfigurefile(filename) - context.stoptext() - - -- elseif suffix == "prep" then - -- - -- -- Why do we wrap here. Because it can be xml? Let's get rid - -- -- of prepping in general. - -- - -- context.starttext() - -- context.input(filename) - -- context.stoptext() - - else - - -- \writestatus{system}{processing as tex} - -- We have a regular tex file so no \starttext yet as we can - -- load fonts. - - -- context.enabletrackers { "resolvers.*" } - context.input(filename) - -- context.disabletrackers { "resolvers.*" } - - end - - context.finishjob() - -\stopluacode +% now moved to cont-run.lua +% +% \startluacode +% +% -- When a style is loaded there is a good change that we never enter +% -- this code. +% +% local report = logs.reporter("system") +% +% environment.initializefilenames() -- todo: check if we really need to pre-prep the filename +% +% local arguments = environment.arguments +% local suffix = environment.suffix +% local filename = environment.filename -- hm, not inputfilename ! +% +% if suffix == "xml" or arguments.forcexml then +% +% -- Maybe we should move the preamble parsing here as it +% -- can be part of (any) loaded (sub) file. The \starttext +% -- wrapping might go away. +% +% report("processing as xml: %s",filename) +% +% context.starttext() +% context.xmlprocess("main",filename,"") +% context.stoptext() +% +% elseif suffix == "cld" or arguments.forcecld then +% +% report("processing as cld: %s",filename) +% +% context.runfile(filename) +% +% elseif suffix == "lua" or arguments.forcelua then +% +% -- The wrapping might go away. Why is is it there in the +% -- first place. +% +% report("processing as lua: %s",filename) +% +% context.starttext() +% context.ctxlua(string.format('dofile("%s")',filename)) +% context.stoptext() +% +% elseif suffix == "mp" or arguments.forcemp then +% +% report("processing as metapost: %s",filename) +% +% context.starttext() +% context.processMPfigurefile(filename) +% context.stoptext() +% +% -- elseif suffix == "prep" then +% -- +% -- -- Why do we wrap here. Because it can be xml? Let's get rid +% -- -- of prepping in general. +% -- +% -- context.starttext() +% -- context.input(filename) +% -- context.stoptext() +% +% else +% +% -- \writestatus{system}{processing as tex} +% -- We have a regular tex file so no \starttext yet as we can +% -- load fonts. +% +% -- context.enabletrackers { "resolvers.*" } +% context.input(filename) +% -- context.disabletrackers { "resolvers.*" } +% +% end +% +% context.finishjob() +% +% \stopluacode + +\ctxcommand{processjob()} % from cont-run.lua \endinput diff --git a/tex/context/base/context-version.pdf b/tex/context/base/context-version.pdf Binary files differindex f86b51291..c7de956c7 100644 --- a/tex/context/base/context-version.pdf +++ b/tex/context/base/context-version.pdf diff --git a/tex/context/base/context.mkiv b/tex/context/base/context.mkiv index 51b6e654a..f726b4521 100644 --- a/tex/context/base/context.mkiv +++ b/tex/context/base/context.mkiv @@ -28,7 +28,7 @@ %D up and the dependencies are more consistent. \edef\contextformat {\jobname} -\edef\contextversion{2014.12.21 22:25} +\edef\contextversion{2014.12.28 19:50} \edef\contextkind {beta} %D For those who want to use this: @@ -528,6 +528,8 @@ \loadmarkfile{back-exp} +\loadmarkfile{cont-run} % the main runner (used in cont-yes.mkiv) + \setupcurrentlanguage[\defaultlanguagetag] \prependtoks diff --git a/tex/context/base/core-con.lua b/tex/context/base/core-con.lua index 6aaed7954..9a48255bd 100644 --- a/tex/context/base/core-con.lua +++ b/tex/context/base/core-con.lua @@ -562,14 +562,11 @@ local function convert(method,n,language) end local sequence = sequences[method] if sequence then - local set = sequences.set - if set then - local max = #set - if n > max then - return set[(n-1) % max + 1] - else - return set[n] - end + local max = #sequence + if n > max then + return sequence[(n-1) % max + 1] + else + return sequence[n] end end return n diff --git a/tex/context/base/core-ctx.lua b/tex/context/base/core-ctx.lua index 18978a530..e9d3eddce 100644 --- a/tex/context/base/core-ctx.lua +++ b/tex/context/base/core-ctx.lua @@ -254,7 +254,7 @@ function ctxrunner.load(ctxname) for i=1,#runners do local command = runners[i] report_prepfiles("command: %s",command) - local result = os.spawn(command) or 0 + local result = os.execute(command) or 0 -- if result > 0 then -- report_prepfiles("error, return code: %s",result) -- end diff --git a/tex/context/base/core-env.mkiv b/tex/context/base/core-env.mkiv index 5447288d7..1e47ac517 100644 --- a/tex/context/base/core-env.mkiv +++ b/tex/context/base/core-env.mkiv @@ -348,7 +348,7 @@ \unexpanded\def\startmodeset {\pushmacro\c_syst_modes_set_done - \setfalse\conditionalfalse + \setfalse\c_syst_modes_set_done \doifnextoptionalcselse\syst_modes_set_start\syst_modes_set_quit} \def\syst_modes_set_start[#1]% diff --git a/tex/context/base/data-crl.lua b/tex/context/base/data-crl.lua index 445bd5b0a..fba5a6230 100644 --- a/tex/context/base/data-crl.lua +++ b/tex/context/base/data-crl.lua @@ -28,7 +28,7 @@ local function runcurl(specification) if not io.exists(cachename) then cached[original] = cachename local command = "curl --silent --create-dirs --output " .. cachename .. " " .. original - os.spawn(command) + os.execute(command) end if io.exists(cachename) then cached[original] = cachename diff --git a/tex/context/base/data-res.lua b/tex/context/base/data-res.lua index 13d7627d2..8ae0500cd 100644 --- a/tex/context/base/data-res.lua +++ b/tex/context/base/data-res.lua @@ -858,7 +858,7 @@ end function resolvers.expandedpathlist(str,extra_too) if not str then return { } - elseif instance.savelists then + elseif instance.savelists then -- hm, what if two cases, with and without extra_too str = lpegmatch(dollarstripper,str) local lists = instance.lists local lst = lists[str] @@ -884,6 +884,14 @@ function resolvers.expandpathfromvariable(str) return joinpath(resolvers.expandedpathlistfromvariable(str)) end +function resolvers.cleanedpathlist(v) + local t = resolvers.expandedpathlist(v) + for i=1,#t do + t[i] = resolvers.resolve(resolvers.cleanpath(t[i])) + end + return t +end + function resolvers.expandbraces(str) -- output variable and brace expansion of STRING -- local ori = resolvers.variable(str) -- if ori == "" then @@ -1175,14 +1183,220 @@ local function check_subpath(fname) end end -local function find_intree(filename,filetype,wantedfiles,allresults) +-- old one / keep as reference + +-- local function find_intree(filename,filetype,wantedfiles,allresults) +-- local typespec = resolvers.variableofformat(filetype) +-- local pathspec = pathspecs[filetype] +-- local pathlist = resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files +-- local method = "intree" +-- if pathlist and #pathlist > 0 then +-- -- list search +-- local filelist = collect_files(wantedfiles) -- okay, a bit over the top when we just look relative to the current path +-- local dirlist = { } +-- if filelist then +-- for i=1,#filelist do +-- dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble +-- end +-- end +-- if trace_detail then +-- report_resolving("checking filename %a in tree",filename) +-- end +-- local result = { } +-- -- pathlist : resolved +-- -- dirlist : unresolved or resolved +-- -- filelist : unresolved +-- for k=1,#pathlist do +-- local path = pathlist[k] +-- local pathname = lpegmatch(inhibitstripper,path) +-- local doscan = path == pathname -- no ^!! +-- if not find (pathname,'//$') then +-- doscan = false -- we check directly on the path +-- end +-- local done = false +-- -- using file list +-- if filelist then -- database +-- -- compare list entries with permitted pattern -- /xx /xx// +-- local expression = makepathexpression(pathname) +-- if trace_detail then +-- report_resolving("using pattern %a for path %a",expression,pathname) +-- end +-- for k=1,#filelist do +-- local fl = filelist[k] +-- local f = fl[2] +-- local d = dirlist[k] +-- -- resolve is new: +-- if find(d,expression) or find(resolveprefix(d),expression) then +-- -- todo, test for readable +-- result[#result+1] = resolveprefix(fl[3]) -- no shortcut +-- done = true +-- if allresults then +-- if trace_detail then +-- report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) +-- end +-- else +-- if trace_detail then +-- report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) +-- end +-- break +-- end +-- elseif trace_detail then +-- report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) +-- end +-- end +-- end +-- if done then +-- method = "database" +-- else +-- method = "filesystem" -- bonus, even when !! is specified +-- pathname = gsub(pathname,"/+$","") +-- pathname = resolveprefix(pathname) +-- local scheme = url.hasscheme(pathname) +-- if not scheme or scheme == "file" then +-- local pname = gsub(pathname,"%.%*$",'') +-- if not find(pname,"*",1,true) then +-- if can_be_dir(pname) then +-- -- hm, rather useless as we don't go deeper and if we would we could also +-- -- auto generate the file database .. however, we need this for extra paths +-- -- that are not hashed (like sources on my machine) .. so, this is slightly +-- -- out of order but at least fast (and we seldom end up here, only when a file +-- -- is not already found +-- if trace_detail then +-- report_resolving("quick root scan for %a",pname) +-- end +-- for k=1,#wantedfiles do +-- local w = wantedfiles[k] +-- local fname = check_subpath(filejoin(pname,w)) +-- if fname then +-- result[#result+1] = fname +-- done = true +-- if not allresults then +-- break +-- end +-- end +-- end +-- if not done and doscan then +-- -- collect files in path (and cache the result) +-- if trace_detail then +-- report_resolving("scanning filesystem for %a",pname) +-- end +-- local files = resolvers.simplescanfiles(pname,false,true) +-- for k=1,#wantedfiles do +-- local w = wantedfiles[k] +-- local subpath = files[w] +-- if not subpath or subpath == "" then +-- -- rootscan already done +-- elseif type(subpath) == "string" then +-- local fname = check_subpath(filejoin(pname,subpath,w)) +-- if fname then +-- result[#result+1] = fname +-- done = true +-- if not allresults then +-- break +-- end +-- end +-- else +-- for i=1,#subpath do +-- local sp = subpath[i] +-- if sp == "" then +-- -- roottest already done +-- else +-- local fname = check_subpath(filejoin(pname,sp,w)) +-- if fname then +-- result[#result+1] = fname +-- done = true +-- if not allresults then +-- break +-- end +-- end +-- end +-- end +-- if done and not allresults then +-- break +-- end +-- end +-- end +-- end +-- end +-- else +-- -- no access needed for non existing path, speedup (esp in large tree with lots of fake) +-- end +-- else +-- -- we can have extra_paths that are urls +-- for k=1,#wantedfiles do +-- -- independent url scanner +-- local fname = methodhandler('finders',pathname .. "/" .. wantedfiles[k]) +-- if fname then +-- result[#result+1] = fname +-- done = true +-- if not allresults then +-- break +-- end +-- end +-- end +-- end +-- end +-- -- todo recursive scanning +-- if done and not allresults then +-- break +-- end +-- end +-- if #result > 0 then +-- return method, result +-- end +-- end +-- end + +-- this caching is not really needed (seldom accessed) but more readable +-- we could probably move some to a higher level but then we need to adapt +-- more code ... maybe some day + +local pathlists = setmetatableindex(function(list,filetype) local typespec = resolvers.variableofformat(filetype) local pathlist = resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) -- only extra path with user files - local method = "intree" + local entry = { } if pathlist and #pathlist > 0 then + for k=1,#pathlist do + local path = pathlist[k] + local pathname = lpegmatch(inhibitstripper,path) + local expression = makepathexpression(pathname) + local barename = gsub(pathname,"/+$","") + barename = resolveprefix(barename) + local scheme = url.hasscheme(barename) + local schemename = gsub(barename,"%.%*$",'') -- after scheme + local prescanned = path ~= pathname -- ^!! + local resursive = find(pathname,'//$') + entry[k] = { + path = path, + pathname = pathname, + prescanned = prescanned, + recursive = recursive, + expression = expression, + barename = barename, + scheme = scheme, + schemename = schemename, + } + end + entry.typespec = typespec + list[filetype] = entry + else + list[filetype] = false + end + return entry +end) + +-- pathlist : resolved +-- dirlist : unresolved or resolved +-- filelist : unresolved + +local function find_intree(filename,filetype,wantedfiles,allresults) + local pathlist = pathlists[filetype] + if pathlist then -- list search + local method = "intree" local filelist = collect_files(wantedfiles) -- okay, a bit over the top when we just look relative to the current path - local dirlist = { } + local dirlist = { } + local result = { } if filelist then for i=1,#filelist do dirlist[i] = filedirname(filelist[i][3]) .. "/" -- was [2] .. gamble @@ -1191,29 +1405,22 @@ local function find_intree(filename,filetype,wantedfiles,allresults) if trace_detail then report_resolving("checking filename %a in tree",filename) end - local result = { } - -- pathlist : resolved - -- dirlist : unresolved or resolved - -- filelist : unresolved for k=1,#pathlist do - local path = pathlist[k] - local pathname = lpegmatch(inhibitstripper,path) - local doscan = path == pathname -- no ^!! - if not find (pathname,'//$') then - doscan = false -- we check directly on the path - end - local done = false + local entry = pathlist[k] + local path = entry.path + local pathname = entry.pathname + local done = false -- using file list if filelist then -- database -- compare list entries with permitted pattern -- /xx /xx// - local expression = makepathexpression(pathname) + local expression = entry.expression if trace_detail then report_resolving("using pattern %a for path %a",expression,pathname) end for k=1,#filelist do local fl = filelist[k] - local f = fl[2] - local d = dirlist[k] + local f = fl[2] + local d = dirlist[k] -- resolve is new: if find(d,expression) or find(resolveprefix(d),expression) then -- todo, test for readable @@ -1237,71 +1444,74 @@ local function find_intree(filename,filetype,wantedfiles,allresults) if done then method = "database" else - method = "filesystem" -- bonus, even when !! is specified - pathname = gsub(pathname,"/+$","") - pathname = resolveprefix(pathname) - local scheme = url.hasscheme(pathname) + -- beware: we don't honor allresults here in a next attempt (done false) + -- but that is kind of special anyway + method = "filesystem" -- bonus, even when !! is specified + local scheme = entry.scheme if not scheme or scheme == "file" then - local pname = gsub(pathname,"%.%*$",'') + local pname = entry.schemename if not find(pname,"*",1,true) then if can_be_dir(pname) then -- hm, rather useless as we don't go deeper and if we would we could also -- auto generate the file database .. however, we need this for extra paths -- that are not hashed (like sources on my machine) .. so, this is slightly - -- out of order but at least fast (anbd we seldom end up here, only when a file + -- out of order but at least fast (and we seldom end up here, only when a file -- is not already found - if trace_detail then - report_resolving("quick root scan for %a",pname) - end - for k=1,#wantedfiles do - local w = wantedfiles[k] - local fname = check_subpath(filejoin(pname,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end - end - end - if not done and doscan then - -- collect files in path (and cache the result) +-- inspect(entry) + if not done and not entry.prescanned then if trace_detail then - report_resolving("scanning filesystem for %a",pname) + report_resolving("quick root scan for %a",pname) end - local files = resolvers.simplescanfiles(pname,false,true) for k=1,#wantedfiles do local w = wantedfiles[k] - local subpath = files[w] - if not subpath or subpath == "" then - -- rootscan already done - elseif type(subpath) == "string" then - local fname = check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break - end + local fname = check_subpath(filejoin(pname,w)) + if fname then + result[#result+1] = fname + done = true + if not allresults then + break end - else - for i=1,#subpath do - local sp = subpath[i] - if sp == "" then - -- roottest already done - else - local fname = check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1] = fname - done = true - if not allresults then - break + end + end + if not done and entry.recursive then -- maybe also when allresults + -- collect files in path (and cache the result) + if trace_detail then + report_resolving("scanning filesystem for %a",pname) + end + local files = resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w = wantedfiles[k] + local subpath = files[w] + if not subpath or subpath == "" then + -- rootscan already done + elseif type(subpath) == "string" then + local fname = check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1] = fname + done = true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp = subpath[i] + if sp == "" then + -- roottest already done + else + local fname = check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1] = fname + done = true + if not allresults then + break + end end end end - end - if done and not allresults then - break + if done and not allresults then + break + end end end end @@ -1314,10 +1524,11 @@ local function find_intree(filename,filetype,wantedfiles,allresults) -- we can have extra_paths that are urls for k=1,#wantedfiles do -- independent url scanner - local fname = methodhandler('finders',pathname .. "/" .. wantedfiles[k]) + local pname = entry.barename + local fname = methodhandler('finders',pname .. "/" .. wantedfiles[k]) if fname then result[#result+1] = fname - doen = true + done = true if not allresults then break end diff --git a/tex/context/base/data-sch.lua b/tex/context/base/data-sch.lua index 1e1077b03..d79e0c7ef 100644 --- a/tex/context/base/data-sch.lua +++ b/tex/context/base/data-sch.lua @@ -65,7 +65,7 @@ local cached, loaded, reused, thresholds, handlers = { }, { }, { }, { }, { } local function runcurl(name,cachename) -- we use sockets instead or the curl library when possible local command = "curl --silent --insecure --create-dirs --output " .. cachename .. " " .. name - os.spawn(command) + os.execute(command) end local function fetch(specification) diff --git a/tex/context/base/data-zip.lua b/tex/context/base/data-zip.lua index a9d4d7a95..2be88e0fc 100644 --- a/tex/context/base/data-zip.lua +++ b/tex/context/base/data-zip.lua @@ -37,18 +37,6 @@ local archives = zip.archives zip.registeredfiles = zip.registeredfiles or { } local registeredfiles = zip.registeredfiles -local limited = false - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - zip.open = i_limiter.protect(zip.open) - limited = true - end - end -end) - local function validzip(str) -- todo: use url splitter if not find(str,"^zip://") then return "zip:///" .. str diff --git a/tex/context/base/font-afm.lua b/tex/context/base/font-afm.lua index ca5616a1e..a96c6686e 100644 --- a/tex/context/base/font-afm.lua +++ b/tex/context/base/font-afm.lua @@ -48,6 +48,11 @@ local definers = fonts.definers local readers = fonts.readers local constructors = fonts.constructors +local fontloader = fontloader +local font_to_table = fontloader.to_table +local open_font = fontloader.open +local close_font = fontloader.close + local afm = constructors.newhandler("afm") local pfb = constructors.newhandler("pfb") @@ -222,10 +227,10 @@ end local function get_indexes(data,pfbname) data.resources.filename = resolvers.unresolve(pfbname) -- no shortcut - local pfbblob = fontloader.open(pfbname) + local pfbblob = open_font(pfbname) if pfbblob then local characters = data.characters - local pfbdata = fontloader.to_table(pfbblob) + local pfbdata = font_to_table(pfbblob) if pfbdata then local glyphs = pfbdata.glyphs if glyphs then @@ -251,7 +256,7 @@ local function get_indexes(data,pfbname) elseif trace_loading then report_afm("no data in pfb file %a",pfbname) end - fontloader.close(pfbblob) + close_font(pfbblob) elseif trace_loading then report_afm("invalid pfb file %a",pfbname) end diff --git a/tex/context/base/font-ctx.lua b/tex/context/base/font-ctx.lua index f764edb6d..ee2a71ac5 100644 --- a/tex/context/base/font-ctx.lua +++ b/tex/context/base/font-ctx.lua @@ -220,7 +220,6 @@ function constructors.trytosharefont(target,tfmdata) end end - directives.register("fonts.checksharing",function(v) if not v then report_defining("font sharing in backend is disabled") @@ -228,19 +227,6 @@ directives.register("fonts.checksharing",function(v) constructors.sharefonts = v end) -local limited = false - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - fontloader.open = i_limiter.protect(fontloader.open) - fontloader.info = i_limiter.protect(fontloader.info) - limited = true - end - end -end) - function definers.resetnullfont() -- resetting is needed because tikz misuses nullfont local parameters = fonts.nulldata.parameters diff --git a/tex/context/base/font-ini.lua b/tex/context/base/font-ini.lua index 884b22474..c547f89ac 100644 --- a/tex/context/base/font-ini.lua +++ b/tex/context/base/font-ini.lua @@ -29,4 +29,4 @@ fonts.readers = { } fonts.definers = { methods = { } } fonts.loggers = { register = function() end } -fontloader.totable = fontloader.to_table +fontloader.totable = fontloader.to_table -- not used diff --git a/tex/context/base/font-inj.lua b/tex/context/base/font-inj.lua index 3b933829d..d0b073db4 100644 --- a/tex/context/base/font-inj.lua +++ b/tex/context/base/font-inj.lua @@ -19,8 +19,6 @@ local trace_injections = false trackers.register("fonts.injections", function(v local report_injections = logs.reporter("fonts","injections") -report_injections("using experimental injector") - local attributes, nodes, node = attributes, nodes, node fonts = fonts diff --git a/tex/context/base/font-mis.lua b/tex/context/base/font-mis.lua index 22f4ccc58..2f7d12e9a 100644 --- a/tex/context/base/font-mis.lua +++ b/tex/context/base/font-mis.lua @@ -25,6 +25,12 @@ local otf = handlers.otf otf.version = otf.version or 2.802 otf.cache = otf.cache or containers.define("fonts", "otf", otf.version, true) +local fontloader = fontloader +local font_to_table = fontloader.to_table +local open_font = fontloader.open +local get_font_info = fontloader.info +local close_font = fontloader.close + function otf.loadcached(filename,format,sub) -- no recache when version mismatch local name = file.basename(file.removesuffix(filename)) @@ -54,10 +60,10 @@ function fonts.helpers.getfeatures(name,t,script,language) -- maybe per font typ if data and data.resources and data.resources.features then return data.resources.features else - local ff = fontloader.open(filename) + local ff = open_font(filename) if ff then - local data = fontloader.to_table(ff) - fontloader.close(ff) + local data = font_to_table(ff) + close_font(ff) local features = { } for k=1,#featuregroups do local what = featuregroups[k] diff --git a/tex/context/base/font-otf.lua b/tex/context/base/font-otf.lua index 1bb608fd5..44ad89325 100644 --- a/tex/context/base/font-otf.lua +++ b/tex/context/base/font-otf.lua @@ -56,13 +56,14 @@ otf.glists = { "gsub", "gpos" } otf.version = 2.802 -- beware: also sync font-mis.lua otf.cache = containers.define("fonts", "otf", otf.version, true) -local fontdata = fonts.hashes.identifiers -local chardata = characters and characters.data -- not used - +local hashes = fonts.hashes local definers = fonts.definers local readers = fonts.readers local constructors = fonts.constructors +local fontdata = hashes and hashes.identifiers +local chardata = characters and characters.data -- not used + local otffeatures = constructors.newfeatures("otf") local registerotffeature = otffeatures.register @@ -84,7 +85,12 @@ local applyruntimefixes = fonts.treatments and fonts.treatments.applyfixes local wildcard = "*" local default = "dflt" -local fontloaderfields = fontloader.fields +local fontloader = fontloader +local open_font = fontloader.open +local close_font = fontloader.close +local font_fields = fontloader.fields +local apply_featurefile = fontloader.apply_featurefile + local mainfields = nil local glyphfields = nil -- not used yet @@ -137,7 +143,7 @@ local function load_featurefile(raw,featurefile) if trace_loading then report_otf("using featurefile %a", featurefile) end - fontloader.apply_featurefile(raw, featurefile) + apply_featurefile(raw, featurefile) end end @@ -437,12 +443,12 @@ function otf.load(filename,sub,featurefile) -- second argument (format) is gone report_otf("loading %a, hash %a",filename,hash) local fontdata, messages if sub then - fontdata, messages = fontloader.open(filename,sub) + fontdata, messages = open_font(filename,sub) else - fontdata, messages = fontloader.open(filename) + fontdata, messages = open_font(filename) end if fontdata then - mainfields = mainfields or (fontloaderfields and fontloaderfields(fontdata)) + mainfields = mainfields or (font_fields and font_fields(fontdata)) end if trace_loading and messages and #messages > 0 then if type(messages) == "string" then @@ -526,7 +532,7 @@ function otf.load(filename,sub,featurefile) -- second argument (format) is gone report_otf("preprocessing and caching time %s, packtime %s", elapsedtime(data),packdata and elapsedtime(packtime) or 0) end - fontloader.close(fontdata) -- free memory + close_font(fontdata) -- free memory if cleanup > 3 then collectgarbage("collect") end diff --git a/tex/context/base/font-syn.lua b/tex/context/base/font-syn.lua index 450bdad75..bf46c8573 100644 --- a/tex/context/base/font-syn.lua +++ b/tex/context/base/font-syn.lua @@ -35,6 +35,13 @@ local findfile = resolvers.findfile local cleanpath = resolvers.cleanpath local resolveprefix = resolvers.resolve +local fontloader = fontloader +local font_to_table = fontloader.to_table +local open_font = fontloader.open +local get_font_info = fontloader.info +local close_font = fontloader.close +local font_fields = fontloader.fields + local settings_to_hash = utilities.parsers.settings_to_hash_tolerant local trace_names = false trackers.register("fonts.names", function(v) trace_names = v end) @@ -50,7 +57,7 @@ using a table that has keys filtered from the font related files.</p> fonts = fonts or { } -- also used elsewhere -local names = font.names or allocate { } +local names = fonts.names or allocate { } fonts.names = names local filters = names.filters or { } @@ -298,10 +305,10 @@ end but to keep the overview, we define them here.</p> --ldx]]-- -filters.otf = fontloader.info -filters.ttf = fontloader.info -filters.ttc = fontloader.info -filters.dfont = fontloader.info +filters.otf = get_font_info +filters.ttf = get_font_info +filters.ttc = get_font_info +filters.dfont = get_font_info -- We had this as temporary solution because we needed a bit more info but in the -- meantime it got an interesting side effect: currently luatex delays loading of e.g. @@ -311,12 +318,12 @@ filters.dfont = fontloader.info -- missing: names, units_per_em, design_range_bottom, design_range_top, design_size, -- pfminfo, top_side_bearing --- function fontloader.fullinfo(...) -- check with taco what we get / could get --- local ff = fontloader.open(...) +-- local function get_full_info(...) -- check with taco what we get / could get +-- local ff = open_font(...) -- if ff then --- local d = ff -- and fontloader.to_table(ff) +-- local d = ff -- and font_to_table(ff) -- d.glyphs, d.subfonts, d.gpos, d.gsub, d.lookups = nil, nil, nil, nil, nil --- fontloader.close(ff) +-- close_font(ff) -- return d -- else -- return nil, "error in loading font" @@ -327,10 +334,10 @@ filters.dfont = fontloader.info -- return these keys/values (and maybe some more) but at least we close the loader which -- might save some memory in the end. --- function fontloader.fullinfo(name) --- local ff = fontloader.open(name) +-- local function get_full_info(name) +-- local ff = open_font(name) -- if ff then --- local fields = table.tohash(fontloader.fields(ff),true) -- isn't that one stable +-- local fields = table.tohash(font_fields(ff),true) -- isn't that one stable -- local d = { -- names = fields.names and ff.names, -- familyname = fields.familyname and ff.familyname, @@ -349,7 +356,7 @@ filters.dfont = fontloader.info -- setmetatableindex(d,function(t,k) -- report_names("warning, trying to access field %a in font table of %a",k,name) -- end) --- fontloader.close(ff) +-- close_font(ff) -- return d -- else -- return nil, "error in loading font" @@ -360,11 +367,11 @@ filters.dfont = fontloader.info local fields = nil -function fontloader.fullinfo(name) - local ff = fontloader.open(name) +local function get_full_info(name) + local ff = open_font(name) if ff then if not fields then - fields = table.tohash(fontloader.fields(ff),true) + fields = table.tohash(font_fields(ff),true) end -- unfortunately luatex aborts when a field is not available local d = { @@ -385,7 +392,7 @@ function fontloader.fullinfo(name) setmetatableindex(d,function(t,k) report_names("warning, trying to access field %a in font table of %a",k,name) end) - fontloader.close(ff) + close_font(ff) return d else return nil, "error in loading font" @@ -397,26 +404,20 @@ end -- current version that somehow happens not that often (on my machine I end up with -- soem 3 GB extra before that happens). --- function fontloader.fullinfo(...) --- local ff = fontloader.open(...) +-- local function get_full_info(...) +-- local ff = open_font(...) -- if ff then -- local d = { } -- ff is userdata so [1] or # fails on it -- setmetatableindex(d,ff) --- return d -- garbage collection will do the fontloader.close(ff) +-- return d -- garbage collection will do the close_font(ff) -- else -- return nil, "error in loading font" -- end -- end --- We don't get the design_* values here as for that the fontloader has to load feature --- info and therefore we're not much better off than using 'open'. --- --- if tonumber(status.luatex_version) > 78 or (tonumber(status.luatex_version) == 78 and tonumber(status.luatex_revision) > 0) then --- fontloader.fullinfo = fontloader.info --- end - -filters.otf = fontloader.fullinfo -filters.ttf = fontloader.fullinfo +fontloader.fullinfo = get_full_info +filters .otf = get_full_info +filters .ttf = get_full_info function filters.afm(name) -- we could parse the afm file as well, and then report an error but @@ -446,7 +447,7 @@ function filters.afm(name) end function filters.pfb(name) - return fontloader.info(name) + return get_font_info(name) end --[[ldx-- diff --git a/tex/context/base/grph-inc.lua b/tex/context/base/grph-inc.lua index 80d878019..ca17a181c 100644 --- a/tex/context/base/grph-inc.lua +++ b/tex/context/base/grph-inc.lua @@ -77,11 +77,13 @@ local variables = interfaces.variables local codeinjections = backends.codeinjections local nodeinjections = backends.nodeinjections -local trace_figures = false trackers.register("graphics.locating", function(v) trace_figures = v end) -local trace_bases = false trackers.register("graphics.bases", function(v) trace_bases = v end) -local trace_programs = false trackers.register("graphics.programs", function(v) trace_programs = v end) -local trace_conversion = false trackers.register("graphics.conversion", function(v) trace_conversion = v end) -local trace_inclusion = false trackers.register("graphics.inclusion", function(v) trace_inclusion = v end) +local trace_figures = false trackers.register ("graphics.locating", function(v) trace_figures = v end) +local trace_bases = false trackers.register ("graphics.bases", function(v) trace_bases = v end) +local trace_programs = false trackers.register ("graphics.programs", function(v) trace_programs = v end) +local trace_conversion = false trackers.register ("graphics.conversion", function(v) trace_conversion = v end) +local trace_inclusion = false trackers.register ("graphics.inclusion", function(v) trace_inclusion = v end) + +local extra_check = false directives.register("graphics.extracheck", function(v) extra_check = v end) local report_inclusion = logs.reporter("graphics","inclusion") local report_figures = logs.reporter("system","graphics") @@ -241,6 +243,13 @@ local figures_magics = allocate { { format = "pdf", pattern = (1 - P("%PDF"))^0 * P("%PDF") }, } +local figures_native = allocate { + pdf = true, + jpg = true, + jp2 = true, + png = true, +} + figures.formats = figures_formats -- frozen figures.magics = figures_magics -- frozen figures.order = figures_order -- frozen @@ -580,6 +589,18 @@ local function forbiddenname(filename) end end +local function rejected(specification) + if extra_check then + local fullname = specification.fullname + if fullname and figures_native[file.suffix(fullname)] and not figures.guess(fullname) then + specification.comment = "probably a bade file" + specification.found = false + report_inclusion("file %a looks bad",fullname) + return true + end + end +end + local function register(askedname,specification) if not specification then specification = { askedname = askedname, comment = "invalid specification" } @@ -591,7 +612,7 @@ local function register(askedname,specification) if trace_figures then report_inclusion("format %a internally supported by engine",specification.format) end - else + elseif not rejected(specification) then local format = specification.format if format then local conversion = specification.conversion @@ -749,7 +770,11 @@ local function register(askedname,specification) specification.found = false end end - specification.foundname = specification.foundname or specification.fullname + if specification.found then + specification.foundname = specification.foundname or specification.fullname + else + specification.foundname = nil + end specification.badname = figures.badname(askedname) local askedhash = f_hash_part(askedname,specification.conversion or "default",specification.resolution or "default") figures_found[askedhash] = specification @@ -1044,7 +1069,7 @@ function identifiers.default(data) du.fullname = fullname -- can be cached ds.fullname = foundname -- original ds.format = l.format - ds.status = (l.found and 10) or 0 + ds.status = (l.bugged and 0) or (l.found and 10) or 0 end return data end @@ -1159,7 +1184,9 @@ function checkers.generic(data) } codeinjections.setfigurecolorspace(data,figure) codeinjections.setfiguremask(data,figure) - figure = figure and images.check(images.scan(figure)) or false + if figure then + figure = images.check(images.scan(figure)) or false + end local f, d = codeinjections.setfigurealternative(data,figure) figure, data = f or figure, d or data figures_loaded[hash] = figure @@ -1226,7 +1253,6 @@ function includers.generic(data) end) image.next = pager pager.prev = image - local box = hpack(image) -- images.node(figure) not longer valid indexed[figure.index] = figure @@ -1431,7 +1457,7 @@ local function runprogram(binary,argument,variables) if trace_conversion or trace_programs then report_inclusion("running command: %s",command) end - os.spawn(command) + os.execute(command) end end diff --git a/tex/context/base/l-dir.lua b/tex/context/base/l-dir.lua index c56af1b73..81ac65e50 100644 --- a/tex/context/base/l-dir.lua +++ b/tex/context/base/l-dir.lua @@ -6,7 +6,8 @@ if not modules then modules = { } end modules ['l-dir'] = { license = "see context related readme files" } --- dir.expandname will be merged with cleanpath and collapsepath +-- todo: dir.expandname will be sped up and merged with cleanpath and collapsepath +-- todo: keep track of currentdir (chdir, pushdir, popdir) local type, select = type, select local find, gmatch, match, gsub, sub = string.find, string.gmatch, string.match, string.gsub, string.sub @@ -490,52 +491,63 @@ end dir.makedirs = dir.mkdirs --- we can only define it here as it uses dir.current -if onwindows then +do - function dir.expandname(str) -- will be merged with cleanpath and collapsepath\ - local first, nothing, last = match(str,"^(//)(//*)(.*)$") - if first then - first = dir.current() .. "/" -- dir.current sanitizes - end - if not first then - first, last = match(str,"^(//)/*(.*)$") - end - if not first then - first, last = match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d = currentdir() - if chdir(first) then - first = dir.current() + -- we can only define it here as it uses dir.chdir and we also need to + -- make sure we use the non sandboxed variant because otherwise we get + -- into a recursive loop due to usage of expandname in the file resolver + + local chdir = sandbox and sandbox.original(chdir) or chdir + + if onwindows then + + local xcurrentdir = dir.current + + function dir.expandname(str) -- will be merged with cleanpath and collapsepath\ + local first, nothing, last = match(str,"^(//)(//*)(.*)$") + if first then + first = xcurrentdir() .. "/" -- xcurrentdir sanitizes + end + if not first then + first, last = match(str,"^(//)/*(.*)$") + end + if not first then + first, last = match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d = currentdir() -- push / pop + if chdir(first) then + first = xcurrentdir() -- xcurrentdir sanitizes + end + chdir(d) end - chdir(d) + end + if not first then + first, last = xcurrentdir(), str + end + last = gsub(last,"//","/") + last = gsub(last,"/%./","/") + last = gsub(last,"^/*","") + first = gsub(first,"/*$","") + if last == "" or last == "." then + return first + else + return first .. "/" .. last end end - if not first then - first, last = dir.current(), str - end - last = gsub(last,"//","/") - last = gsub(last,"/%./","/") - last = gsub(last,"^/*","") - first = gsub(first,"/*$","") - if last == "" or last == "." then - return first - else - return first .. "/" .. last - end - end -else + else - function dir.expandname(str) -- will be merged with cleanpath and collapsepath - if not find(str,"^/") then - str = currentdir() .. "/" .. str + function dir.expandname(str) -- will be merged with cleanpath and collapsepath + if not find(str,"^/") then + str = currentdir() .. "/" .. str + end + str = gsub(str,"//","/") + str = gsub(str,"/%./","/") + str = gsub(str,"(.)/%.$","%1") + return str end - str = gsub(str,"//","/") - str = gsub(str,"/%./","/") - str = gsub(str,"(.)/%.$","%1") - return str + end end diff --git a/tex/context/base/l-file.lua b/tex/context/base/l-file.lua index 2742e99b3..2c471d727 100644 --- a/tex/context/base/l-file.lua +++ b/tex/context/base/l-file.lua @@ -15,51 +15,53 @@ if not lfs then lfs = optionalrequire("lfs") end -if not lfs then - - lfs = { - getcurrentdir = function() - return "." - end, - attributes = function() - return nil - end, - isfile = function(name) - local f = io.open(name,'rb') - if f then - f:close() - return true - end - end, - isdir = function(name) - print("you need to load lfs") - return false - end - } - -elseif not lfs.isfile then - - local attributes = lfs.attributes - - function lfs.isdir(name) - return attributes(name,"mode") == "directory" - end - - function lfs.isfile(name) - return attributes(name,"mode") == "file" - end - - -- function lfs.isdir(name) - -- local a = attributes(name) - -- return a and a.mode == "directory" - -- end - - -- function lfs.isfile(name) - -- local a = attributes(name) - -- return a and a.mode == "file" - -- end - -end +-- -- see later +-- +-- if not lfs then +-- +-- lfs = { +-- getcurrentdir = function() +-- return "." +-- end, +-- attributes = function() +-- return nil +-- end, +-- isfile = function(name) +-- local f = io.open(name,'rb') +-- if f then +-- f:close() +-- return true +-- end +-- end, +-- isdir = function(name) +-- print("you need to load lfs") +-- return false +-- end +-- } +-- +-- elseif not lfs.isfile then +-- +-- local attributes = lfs.attributes +-- +-- function lfs.isdir(name) +-- return attributes(name,"mode") == "directory" +-- end +-- +-- function lfs.isfile(name) +-- return attributes(name,"mode") == "file" +-- end +-- +-- -- function lfs.isdir(name) +-- -- local a = attributes(name) +-- -- return a and a.mode == "directory" +-- -- end +-- +-- -- function lfs.isfile(name) +-- -- local a = attributes(name) +-- -- return a and a.mode == "file" +-- -- end +-- +-- end local insert, concat = table.insert, table.concat local match, find, gmatch = string.match, string.find, string.gmatch @@ -72,6 +74,28 @@ local checkedsplit = string.checkedsplit local P, R, S, C, Cs, Cp, Cc, Ct = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cs, lpeg.Cp, lpeg.Cc, lpeg.Ct +-- better this way: + +local tricky = S("/\\") * P(-1) +local attributes = lfs.attributes + +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir, "lfs.isdir") +end + +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode") == "directory" + else + return attributes(name.."/.","mode") == "directory" + end +end + +function lfs.isfile(name) + return attributes(name,"mode") == "file" +end + local colon = P(":") local period = P(".") local periods = P("..") @@ -554,23 +578,6 @@ function file.collapsepath(str,anchor) -- anchor: false|nil, true, "." end end --- better this way: - -local tricky = S("/\\") * P(-1) -local attributes = lfs.attributes - -function lfs.isdir(name) - if lpegmatch(tricky,name) then - return attributes(name,"mode") == "directory" - else - return attributes(name.."/.","mode") == "directory" - end -end - -function lfs.isfile(name) - return attributes(name,"mode") == "file" -end - -- local function test(str,...) -- print(string.format("%-20s %-15s %-30s %-20s",str,file.collapsepath(str),file.collapsepath(str,true),file.collapsepath(str,"."))) -- end diff --git a/tex/context/base/l-io.lua b/tex/context/base/l-io.lua index 020e811bf..a91d44d87 100644 --- a/tex/context/base/l-io.lua +++ b/tex/context/base/l-io.lua @@ -339,11 +339,6 @@ function io.readstring(f,n,m) return str end --- - -if not io.i_limiter then function io.i_limiter() end end -- dummy so we can test safely -if not io.o_limiter then function io.o_limiter() end end -- dummy so we can test safely - -- This works quite ok: -- -- function io.piped(command,writer) diff --git a/tex/context/base/l-lua.lua b/tex/context/base/l-lua.lua index 9565f484a..1a2a98723 100644 --- a/tex/context/base/l-lua.lua +++ b/tex/context/base/l-lua.lua @@ -165,3 +165,14 @@ end if lua then lua.mask = load([[τεχ = 1]]) and "utf" or "ascii" end + +local flush = io.flush + +if flush then + + local execute = os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec = os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn = os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen = io.popen if popen then function io.popen (...) flush() return popen (...) end end + +end diff --git a/tex/context/base/l-os.lua b/tex/context/base/l-os.lua index 1dff79cd3..f44b31662 100644 --- a/tex/context/base/l-os.lua +++ b/tex/context/base/l-os.lua @@ -25,8 +25,6 @@ if not modules then modules = { } end modules ['l-os'] = { -- os.sleep() => socket.sleep() -- math.randomseed(tonumber(string.sub(string.reverse(tostring(math.floor(socket.gettime()*10000))),1,6))) --- maybe build io.flush in os.execute - local os = os local date, time = os.date, os.time local find, format, gsub, upper, gmatch = string.find, string.format, string.gsub, string.upper, string.gmatch @@ -118,15 +116,11 @@ end -- end of environment hack -local execute, spawn, exec, iopopen, ioflush = os.execute, os.spawn or os.execute, os.exec or os.execute, io.popen, io.flush - -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end +local execute = os.execute +local iopopen = io.popen function os.resultof(command) - local handle = io.popen(command,"r") + local handle = iopopen(command,"r") -- already has flush if handle then local result = handle:read("*all") or "" handle:close() @@ -160,7 +154,7 @@ local launchers = { } function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) + execute(format(launchers[os.name] or launchers.unix,str)) end if not os.times then -- ? diff --git a/tex/context/base/l-sandbox.lua b/tex/context/base/l-sandbox.lua new file mode 100644 index 000000000..f7901379c --- /dev/null +++ b/tex/context/base/l-sandbox.lua @@ -0,0 +1,271 @@ +if not modules then modules = { } end modules ['l-sandbox'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- We use string instead of function variables, so 'io.open' instead of io.open. That +-- way we can still intercept repetetive overloads. One complication is that when we use +-- sandboxed function sin helpers in the sanbox checkers, we can get a recursion loop +-- so for that vreason we need to keep originals around till we enable the sandbox. + +-- if sandbox then return end + +local global = _G +local next = next +local unpack = unpack or table.unpack +local type = type +local tprint = texio.write_nl or print +local tostring = tostring +local format = string.format -- no formatters yet +local concat = table.concat +local sort = table.sort +local gmatch = string.gmatch + +sandbox = { } +local sandboxed = false +local overloads = { } +local skiploads = { } +local initializers = { } +local finalizers = { } +local originals = { } +local comments = { } +local trace = false +local logger = false + +-- this comes real early, so that we can still alias + +local function report(...) + tprint("sandbox ! " .. format(...)) -- poor mans tracer +end + +sandbox.report = report + +function sandbox.setreporter(r) + report = r + sandbox.report = r +end + +function sandbox.settrace(v) + trace = v +end + +function sandbox.setlogger(l) + logger = type(l) == "function" and l or false +end + +local function register(func,overload,comment) + if type(func) == "function" then + if type(overload) == "string" then + comment = overload + overload = nil + end + local function f(...) + if sandboxed then + local overload = overloads[f] + if overload then + if logger then + local result = { overload(func,...) } + logger { + comment = comments[f] or tostring(f), + arguments = { ... }, + result = result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + -- ignored, maybe message + end + else + return func(...) + end + end + if comment then + comments[f] = comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f] = overload or false + originals[f] = func + return f + end +end + +local function redefine(func,comment) + if type(func) == "function" then + skiploads[func] = comment or comments[func] or "unknown" + if overloads[func] == false then + overloads[func] = nil -- not initialized anyway + end + end +end + +sandbox.register = register +sandbox.redefine = redefine + +function sandbox.original(func) + return originals and originals[func] or func +end + +function sandbox.overload(func,overload,comment) + comment = comment or comments[func] or "?" + if type(func) ~= "function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload) ~= "function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func] == nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func] = overload + end + return func +end + +function sandbox.initializer(f) + if not sandboxed then + initializers[#initializers+1] = f + elseif trace then + report("already enabled, discarding initializer") + end +end + +function sandbox.finalizer(f) + if not sandboxed then + finalizers[#finalizers+1] = f + elseif trace then + report("already enabled, discarding finalizer") + end +end + +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i]() + end + for i=1,#finalizers do + finalizers[i]() + end + local nnot = 0 + local nyes = 0 + local cnot = { } + local cyes = { } + local skip = { } + for k, v in next, overloads do + local c = comments[k] + if v then + if c then + cyes[#cyes+1] = c + else -- if not skiploads[k] then + nyes = nyes + 1 + end + else + if c then + cnot[#cnot+1] = c + else -- if not skiploads[k] then + nnot = nnot + 1 + end + end + end + for k, v in next, skiploads do + skip[#skip+1] = v + end + if #cyes > 0 then + sort(cyes) + report(" overloaded known : %s",concat(cyes," | ")) + end + if nyes > 0 then + report(" overloaded unknown : %s",nyes) + end + if #cnot > 0 then + sort(cnot) + report("not overloaded known : %s",concat(cnot," | ")) + end + if nnot > 0 then + report("not overloaded unknown : %s",nnot) + end + if #skip > 0 then + sort(skip) + report("not overloaded redefined : %s",concat(skip," | ")) + end + initializers = nil + finalizers = nil + originals = nil + sandboxed = true + end +end + +-- we sandbox some of the built-in functions now: + +-- todo: require +-- todo: load + +local function supported(library) + local l = _G[library] + -- if l then + -- for k, v in next, l do + -- report("%s.%s",library,k) + -- end + -- end + return l +end + +-- io.tmpfile : we don't know where that one ends up but probably is user temp space +-- os.tmpname : no need to deal with this: outputs rubish anyway (\s9v0. \s9v0.1 \s9v0.2 etc) +-- os.tmpdir : not needed either (luatex.vob000 luatex.vob000 etc) + +-- os.setenv : maybe +-- require : maybe (normally taken from tree) +-- http etc : maybe (all schemes that go outside) + +loadfile = register(loadfile,"loadfile") + +if supported("io") then + io.open = register(io.open, "io.open") + io.popen = register(io.popen, "io.popen") -- needs checking + io.lines = register(io.lines, "io.lines") + io.output = register(io.output, "io.output") + io.input = register(io.input, "io.input") +end + +if supported("os") then + os.execute = register(os.execute, "os.execute") + os.spawn = register(os.spawn, "os.spawn") + os.exec = register(os.exec, "os.exec") + os.rename = register(os.rename, "os.rename") + os.remove = register(os.remove, "os.remove") +end + +if supported("lfs") then + lfs.chdir = register(lfs.chdir, "lfs.chdir") + lfs.mkdir = register(lfs.mkdir, "lfs.mkdir") + lfs.rmdir = register(lfs.rmdir, "lfs.rmdir") + lfs.isfile = register(lfs.isfile, "lfs.isfile") + lfs.isdir = register(lfs.isdir, "lfs.isdir") + lfs.attributes = register(lfs.attributes, "lfs.attributes") + lfs.dir = register(lfs.dir, "lfs.dir") + lfs.lock_dir = register(lfs.lock_dir, "lfs.lock_dir") + lfs.touch = register(lfs.touch, "lfs.touch") + lfs.link = register(lfs.link, "lfs.link") + lfs.setmode = register(lfs.setmode, "lfs.setmode") + lfs.readlink = register(lfs.readlink, "lfs.readlink") + lfs.shortname = register(lfs.shortname, "lfs.shortname") + lfs.symlinkattributes = register(lfs.symlinkattributes,"lfs.symlinkattributes") +end diff --git a/tex/context/base/l-table.lua b/tex/context/base/l-table.lua index 3eb8b8514..97e0441eb 100644 --- a/tex/context/base/l-table.lua +++ b/tex/context/base/l-table.lua @@ -49,9 +49,19 @@ function table.keys(t) end end +-- local function compare(a,b) +-- local ta, tb = type(a), type(b) -- needed, else 11 < 2 +-- if ta == tb then +-- return a < b +-- else +-- return tostring(a) < tostring(b) -- not that efficient +-- end +-- end + local function compare(a,b) - local ta, tb = type(a), type(b) -- needed, else 11 < 2 - if ta == tb then + local ta = type(a) -- needed, else 11 < 2 + local tb = type(b) -- needed, else 11 < 2 + if ta == tb and ta == "number" then return a < b else return tostring(a) < tostring(b) -- not that efficient @@ -469,7 +479,7 @@ local function do_serialize(root,name,depth,level,indexed) end end -- we could check for k (index) being number (cardinal) - if root and next(root) then + if root and next(root) ~= nil then -- local first, last = nil, 0 -- #root cannot be trusted here (will be ok in 5.2 when ipairs is gone) -- if compact then -- -- NOT: for k=1,#root do (we need to quit at nil) @@ -513,7 +523,7 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s %q,",depth,v)) end elseif tv == "table" then - if not next(v) then + if next(v) == nil then handle(format("%s {},",depth)) elseif inline then -- and #t > 0 local st = simple_table(v) @@ -597,7 +607,7 @@ local function do_serialize(root,name,depth,level,indexed) end end elseif tv == "table" then - if not next(v) then + if next(v) == nil then if tk == "number" then if hexify then handle(format("%s [0x%X]={},",depth,k)) @@ -683,7 +693,7 @@ local function do_serialize(root,name,depth,level,indexed) --~ end end end - if level > 0 then + if level > 0 then handle(format("%s},",depth)) end end @@ -748,7 +758,7 @@ local function serialize(_handle,root,name,specification) -- handle wins root._w_h_a_t_e_v_e_r_ = nil end -- Let's forget about empty tables. - if next(root) then + if next(root) ~= nil then do_serialize(root,name,"",0) end end @@ -928,7 +938,7 @@ local function sparse(old,nest,keeptables) if not (v == "" or v == false) then if nest and type(v) == "table" then v = sparse(v,nest) - if keeptables or next(v) then + if keeptables or next(v) ~= nil then new[k] = v end else @@ -1066,11 +1076,11 @@ end -- slower than #t on indexed tables (#t only returns the size of the numerically indexed slice) function table.is_empty(t) - return not t or not next(t) + return not t or next(t) == nil end function table.has_one_entry(t) - return t and not next(t,next(t)) + return t and next(t,next(t)) == nil end -- new @@ -1157,7 +1167,7 @@ function table.filtered(t,pattern,sort,cmp) else local n = next(t) local function iterator() - while n do + while n ~= nil do local k = n n = next(t,k) if find(k,pattern) then diff --git a/tex/context/base/lpdf-epa.lua b/tex/context/base/lpdf-epa.lua index 8ca568b76..dd5ecc609 100644 --- a/tex/context/base/lpdf-epa.lua +++ b/tex/context/base/lpdf-epa.lua @@ -21,10 +21,13 @@ local trace_outlines = false trackers.register("figures.outliness", function(v) local report_link = logs.reporter("backend","link") local report_outline = logs.reporter("backend","outline") +local epdf = epdf local backends = backends local lpdf = lpdf local context = context +local loadpdffile = lpdf.epdf.load + local nameonly = file.nameonly local variables = interfaces.variables @@ -141,7 +144,7 @@ function codeinjections.mergereferences(specification) end if specification then local fullname = specification.fullname - local document = lpdf.epdf.load(fullname) -- costs time + local document = loadpdffile(fullname) -- costs time if document then local pagenumber = specification.page or 1 local xscale = specification.yscale or 1 @@ -221,7 +224,7 @@ function codeinjections.mergeviewerlayers(specification) end if specification then local fullname = specification.fullname - local document = lpdf.epdf.load(fullname) + local document = loadpdffile(fullname) if document then local namespace = makenamespace(fullname) local layers = document.layers @@ -273,7 +276,7 @@ function codeinjections.getbookmarks(filename) local document = nil if lfs.isfile(filename) then - document = lpdf.epdf.load(filename) + document = loadpdffile(filename) else report_outline("unknown file %a",filename) bookmarks.extras.register(filename,list) diff --git a/tex/context/base/lpdf-epd.lua b/tex/context/base/lpdf-epd.lua index d5cb15839..5ffd62bd6 100644 --- a/tex/context/base/lpdf-epd.lua +++ b/tex/context/base/lpdf-epd.lua @@ -44,7 +44,8 @@ local P, C, S, R, Ct, Cc, V, Carg, Cs, Cf, Cg = lpeg.P, lpeg.C, lpeg.S, lpeg.R, local epdf = epdf lpdf = lpdf or { } local lpdf = lpdf -lpdf.epdf = { } +local lpdf_epdf = { } +lpdf.epdf = lpdf_epdf local report_epdf = logs.reporter("epdf") @@ -89,8 +90,6 @@ local function initialize_methods(xref) arrayGetNF = array.getNF arrayGet = array.get -- - -- report_epdf("initializing lpdf.epdf library") - -- initialize_methods = function() -- already done end @@ -125,18 +124,6 @@ local function fatal_error(...) os.exit() end -local limited = false -- a bit of protection - -directives.register("system.inputmode", function(v) - if not limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - epdf.open = i_limiter.protect(epdf.open) - limited = true - end - end -end) - -- epdf is the built-in library function epdf.type(o) @@ -529,11 +516,11 @@ end local loaded = { } -function lpdf.epdf.load(filename) +function lpdf_epdf.load(filename) local document = loaded[filename] if not document then - statistics.starttiming(lpdf.epdf) - local __data__ = epdf.open(filename) -- maybe resolvers.find_file + statistics.starttiming(lpdf_epdf) + local __data__ = pdf_open(filename) -- maybe resolvers.find_file if __data__ then local __xref__ = __data__:getXRef() document = { @@ -565,13 +552,13 @@ function lpdf.epdf.load(filename) end loaded[filename] = document loaded[document] = document - statistics.stoptiming(lpdf.epdf) - -- print(statistics.elapsedtime(lpdf.epdf)) + statistics.stoptiming(lpdf_epdf) + -- print(statistics.elapsedtime(lpdf_epdf)) end return document or nil end -function lpdf.epdf.unload(filename) +function lpdf_epdf.unload(filename) local document = loaded[filename] if document then loaded[document] = nil @@ -597,8 +584,8 @@ local function expanded(t) return next, t end -lpdf.epdf.expand = expand -lpdf.epdf.expanded = expanded +lpdf_epdf.expand = expand +lpdf_epdf.expanded = expanded -- we could resolve the text stream in one pass if we directly handle the -- font but why should we complicate things @@ -707,7 +694,7 @@ end local p_hex_to_utf = P(true) / function() more = 0 end * Cs(p_hex_to_utf^1) local p_dec_to_utf = P(true) / function() more = 0 end * Cs(p_dec_to_utf^1) -function lpdf.epdf.getpagecontent(document,pagenumber) +function lpdf_epdf.getpagecontent(document,pagenumber) local page = document.pages[pagenumber] @@ -765,7 +752,7 @@ end local softhyphen = utfchar(0xAD) .. "$" local linefactor = 1.3 -function lpdf.epdf.contenttotext(document,list) -- maybe signal fonts +function lpdf_epdf.contenttotext(document,list) -- maybe signal fonts local last_y = 0 local last_f = 0 local text = { } @@ -813,7 +800,7 @@ function lpdf.epdf.contenttotext(document,list) -- maybe signal fonts return concat(text) end -function lpdf.epdf.getstructure(document,list) -- just a test +function lpdf_epdf.getstructure(document,list) -- just a test local depth = 0 for i=1,#list do local entry = list[i] @@ -844,7 +831,7 @@ end -- helpers --- function lpdf.epdf.getdestinationpage(document,name) +-- function lpdf_epdf.getdestinationpage(document,name) -- local destination = document.__data__:findDest(name) -- return destination and destination.number -- end diff --git a/tex/context/base/luat-bas.mkiv b/tex/context/base/luat-bas.mkiv index a38912716..cb00d8f55 100644 --- a/tex/context/base/luat-bas.mkiv +++ b/tex/context/base/luat-bas.mkiv @@ -13,7 +13,8 @@ \writestatus{loading}{ConTeXt Lua Macros / Basic Lua Libraries} -\registerctxluafile{l-lua} {1.001} +\registerctxluafile{l-lua} {1.001} % before sandbox +\registerctxluafile{l-sandbox} {1.001} \registerctxluafile{l-package} {1.001} \registerctxluafile{l-lpeg} {1.001} \registerctxluafile{l-function}{1.001} diff --git a/tex/context/base/luat-exe.lua b/tex/context/base/luat-exe.lua index a57a5a006..d8d954a30 100644 --- a/tex/context/base/luat-exe.lua +++ b/tex/context/base/luat-exe.lua @@ -6,121 +6,68 @@ if not modules then modules = { } end modules ['luat-exe'] = { license = "see context related readme files" } --- this module needs checking (very old and never really used, not even enabled) +if not sandbox then require("l-sandbox") require("util-sbx") end -- for testing -local match, find, gmatch = string.match, string.find, string.gmatch -local concat = table.concat -local select = select +local type = type -local report_executers = logs.reporter("system","executers") +local executers = resolvers.executers or { } +resolvers.executers = executers -resolvers.executers = resolvers.executers or { } -local executers = resolvers.executers +local disablerunners = sandbox.disablerunners +local registerbinary = sandbox.registerbinary +local registerroot = sandbox.registerroot -local permitted = { } +local lpegmatch = lpeg.match -local osexecute = os.execute -local osexec = os.exec -local osspawn = os.spawn -local iopopen = io.popen +local sc_splitter = lpeg.tsplitat(";") +local cm_splitter = lpeg.tsplitat(",") -local execute = osexecute -local exec = osexec -local spawn = osspawn -local popen = iopopen - -local function register(...) - for k=1,select("#",...) do - local v = select(k,...) - permitted[#permitted+1] = v == "*" and ".*" or v - end -end +local execution_mode directives.register("system.executionmode", function(v) execution_mode = v end) +local execution_list directives.register("system.executionlist", function(v) execution_list = v end) +local root_list directives.register("system.rootlist", function(v) root_list = v end) -local function prepare(...) - -- todo: make more clever first split - local t = { ... } - local n = #n - local one = t[1] - if n == 1 then - if type(one) == 'table' then - return one, concat(t," ",2,n) - else - local name, arguments = match(one,"^(.-)%s+(.+)$") - if name and arguments then - return name, arguments - else - return one, "" +sandbox.initializer(function() + if execution_mode == "none" then + -- will be done later + elseif execution_mode == "list" then + if type(execution_list) == "string" then + execution_list = lpegmatch(cm_splitter,execution_list) + end + if type(execution_list) == "table" then + for i=1,#execution_list do + registerbinary(execution_list[i]) end end else - return one, concat(t," ",2,n) + -- whatever else we have configured end -end +end) -local function executer(action) - return function(...) - local name, arguments = prepare(...) - for k=1,#permitted do - local v = permitted[k] - if find(name,v) then - return action(name .. " " .. arguments) - else - report_executers("not permitted: %s %s",name,arguments) +sandbox.initializer(function() + if type(root_list) == "string" then + root_list = lpegmatch(sc_splitter,root_list) + end + if type(root_list) == "table" then + for i=1,#root_list do + local entry = root_list[i] + if entry ~= "" then + registerroot(entry) end end - return action("") end -end +end) -local function finalize() -- todo: os.exec, todo: report ipv print - execute = executer(osexecute) - exec = executer(osexec) - spawn = executer(osspawn) - popen = executer(iopopen) - finalize = function() - report_executers("already finalized") - end - register = function() - report_executers("already finalized, no registration permitted") - end - os.execute = execute - os.exec = exec - os.spawn = spawn - io.popen = popen -end - -executers.finalize = function(...) return finalize(...) end -executers.register = function(...) return register(...) end -executers.execute = function(...) return execute (...) end -executers.exec = function(...) return exec (...) end -executers.spawn = function(...) return spawn (...) end -executers.popen = function(...) return popen (...) end - -local execution_mode directives.register("system.executionmode", function(v) execution_mode = v end) -local execution_list directives.register("system.executionlist", function(v) execution_list = v end) - -function executers.check() +sandbox.finalizer(function() if execution_mode == "none" then - finalize() - elseif execution_mode == "list" and execution_list ~= "" then - for s in gmatch("[^%s,]",execution_list) do - register(s) - end - finalize() - else - -- all + disablerunners() end -end - ---~ resolvers.executers.register('.*') ---~ resolvers.executers.register('*') ---~ resolvers.executers.register('dir','ls') ---~ resolvers.executers.register('dir') +end) ---~ resolvers.executers.finalize() ---~ resolvers.executers.execute('dir',"*.tex") ---~ resolvers.executers.execute("dir *.tex") ---~ resolvers.executers.execute("ls *.tex") ---~ os.execute('ls') +-- Let's prevent abuse of these libraries (built-in support still works). ---~ resolvers.executers.check() +sandbox.finalizer(function() + mplib = nil + epdf = nil + zip = nil + fontloader = nil +end) diff --git a/tex/context/base/luat-fmt.lua b/tex/context/base/luat-fmt.lua index 20a4a8fcd..92c1dd6c4 100644 --- a/tex/context/base/luat-fmt.lua +++ b/tex/context/base/luat-fmt.lua @@ -95,7 +95,7 @@ function environment.make_format(name) -- generate format local command = format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform == "unix" and "\\\\" or "\\") report_format("running command: %s\n",command) - os.spawn(command) + os.execute(command) -- remove related mem files local pattern = file.removesuffix(file.basename(usedluastub)).."-*.mem" -- report_format("removing related mplib format with pattern %a", pattern) @@ -133,7 +133,7 @@ function environment.run_format(name,data,more) else local command = format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more ~= "" and quoted(more) or "") report_format("running command: %s",command) - os.spawn(command) + os.execute(command) end end end diff --git a/tex/context/base/luat-iop.lua b/tex/context/base/luat-iop.lua index 52f14683e..efb383c57 100644 --- a/tex/context/base/luat-iop.lua +++ b/tex/context/base/luat-iop.lua @@ -6,190 +6,45 @@ if not modules then modules = { } end modules ['luat-iop'] = { license = "see context related readme files" } --- this paranoid stuff in web2c ... we cannot hook checks into the --- input functions because one can always change the callback but --- we can feed back specific patterns and paths into the next --- mechanism - --- os.execute os.exec os.spawn io.fopen --- os.remove lfs.chdir lfs.mkdir --- io.open zip.open epdf.open mlib.new - --- cache - -local topattern, find = string.topattern, string.find - -local report_limiter = logs.reporter("system","limiter") - --- the basic methods - -local function match(ruleset,name) - local n = #ruleset - if n > 0 then - for i=1,n do - local r = ruleset[i] - if find(name,r[1]) then - return r[2] - end - end - return false - else - -- nothing defined (or any) - return true - end -end - -local function protect(ruleset,proc) - return function(name,...) - if name == "" then - -- report_limiter("no access permitted: <no name>") -- can happen in mplib code - return nil, "no name given" - elseif match(ruleset,name) then - return proc(name,...) - else - report_limiter("no access permitted for %a",name) - return nil, name .. ": no access permitted" - end - end -end - -function io.limiter(preset) - preset = preset or { } - local ruleset = { } - for i=1,#preset do - local p = preset[i] - local what, spec = p[1] or "", p[2] or "" - if spec == "" then - -- skip 'm - elseif what == "tree" then - resolvers.dowithpath(spec, function(r) - local spec = resolvers.resolve(r) or "" - if spec ~= "" then - ruleset[#ruleset+1] = { topattern(spec,true), true } - end - end) - elseif what == "permit" then - ruleset[#ruleset+1] = { topattern(spec,true), true } - elseif what == "forbid" then - ruleset[#ruleset+1] = { topattern(spec,true), false } - end - end - if #ruleset > 0 then - return { - match = function(name) return match (ruleset,name) end, - protect = function(proc) return protect(ruleset,proc) end, - } - else - return { - match = function(name) return true end, - protect = proc, - } - end -end - --- a few handlers - -io.i_limiters = { } -io.o_limiters = { } - -function io.i_limiter(v) - local i = io.i_limiters[v] - if i then - local i_limiter = io.limiter(i) - function io.i_limiter() - return i_limiter - end - return i_limiter - end -end - -function io.o_limiter(v) - local o = io.o_limiters[v] - if o then - local o_limiter = io.limiter(o) - function io.o_limiter() - return o_limiter - end - return o_limiter - end -end - --- the real thing (somewhat fuzzy as we need to know what gets done) - -local i_opener, i_limited = io.open, false -local o_opener, o_limited = io.open, false - -local function i_register(v) - if not i_limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - local protect = i_limiter.protect - i_opener = protect(i_opener) - i_limited = true - report_limiter("input mode set to %a",v) - end - end -end - -local function o_register(v) - if not o_limited then - local o_limiter = io.o_limiter(v) - if o_limiter then - local protect = o_limiter.protect - o_opener = protect(o_opener) - o_limited = true - report_limiter("output mode set to %a",v) - end - end -end - -function io.open(name,method) - if method and find(method,"[wa]") then - return o_opener(name,method) - else - return i_opener(name,method) - end -end - -directives.register("system.inputmode", i_register) -directives.register("system.outputmode", o_register) - -local i_limited = false -local o_limited = false - -local function i_register(v) - if not i_limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - local protect = i_limiter.protect - lfs.chdir = protect(lfs.chdir) -- needs checking - i_limited = true - end - end -end - -local function o_register(v) - if not o_limited then - local o_limiter = io.o_limiter(v) - if o_limiter then - local protect = o_limiter.protect - os.remove = protect(os.remove) -- rather okay - lfs.chdir = protect(lfs.chdir) -- needs checking - lfs.mkdir = protect(lfs.mkdir) -- needs checking - o_limited = true - end - end -end - -directives.register("system.inputmode", i_register) -directives.register("system.outputmode", o_register) - --- the definitions - -local limiters = resolvers.variable("limiters") - -if limiters then - io.i_limiters = limiters.input or { } - io.o_limiters = limiters.output or { } -end - +-- local input_mode directives.register("system.inputmode", function(v) input_mode = v end) +-- local output_mode directives.register("system.outputmode",function(v) output_mode = v end) + +-- limiters = { +-- input = { +-- paranoid = { +-- { "permit", "^[^/]+$" }, +-- { "permit", "^./" }, +-- { "forbid", ".." }, +-- { "tree" , "TEXMF" }, +-- { "tree" , "MPINPUTS" }, +-- { "tree" , "TEXINPUTS" }, +-- { "forbid", "^/.." }, +-- { "forbid", "^[a-c]:/.." }, +-- }, +-- }, +-- output = { +-- paranoid = { +-- { "permit", "^[^/]+$" }, +-- { "permit", "^./" }, +-- }, +-- } +-- } + +-- sandbox.registerroot(".","write") -- always ok + +local cleanedpathlist = resolvers.cleanedpathlist +local registerroot = sandbox.registerroot + +sandbox.initializer(function() + local function register(str,mode) + local trees = cleanedpathlist(str) + for i=1,#trees do + registerroot(trees[i],mode) + end + end + register("TEXMF","read") + register("TEXINPUTS","read") + register("MPINPUTS","read") + -- register("TEXMFCACHE","write") + registerroot(".","write") +end) diff --git a/tex/context/base/luat-lib.mkiv b/tex/context/base/luat-lib.mkiv index 3f72e780e..24f9da415 100644 --- a/tex/context/base/luat-lib.mkiv +++ b/tex/context/base/luat-lib.mkiv @@ -36,6 +36,8 @@ \registerctxluafile{util-sta}{1.001} +\registerctxluafile{util-sbx}{1.001} % needs tracker and templates + \registerctxluafile{data-ini}{1.001} \registerctxluafile{data-exp}{1.001} \registerctxluafile{data-env}{1.001} @@ -71,8 +73,8 @@ \registerctxluafile{luat-ini}{1.001} \registerctxluafile{util-env}{1.001} \registerctxluafile{luat-env}{1.001} -\registerctxluafile{luat-exe}{1.001} -\registerctxluafile{luat-iop}{1.001} +\registerctxluafile{luat-exe}{1.001} % simplified +\registerctxluafile{luat-iop}{1.001} % simplified \registerctxluafile{luat-bwc}{1.001} \registerctxluafile{trac-lmx}{1.001} % might become l-lmx or luat-lmx \registerctxluafile{luat-mac}{1.001} diff --git a/tex/context/base/luat-run.lua b/tex/context/base/luat-run.lua index 607c3528a..e71215f13 100644 --- a/tex/context/base/luat-run.lua +++ b/tex/context/base/luat-run.lua @@ -230,3 +230,12 @@ directives.register("system.reportfiles", function(v) register("stop_file", report_none) end end) + +-- start_run doesn't work + +-- luatex.registerstartactions(function() +-- if environment.arguments.sandbox then +-- sandbox.enable() +-- end +-- end) + diff --git a/tex/context/base/lxml-sor.mkiv b/tex/context/base/lxml-sor.mkiv index 0ee1f16f3..0d8eb6ba1 100644 --- a/tex/context/base/lxml-sor.mkiv +++ b/tex/context/base/lxml-sor.mkiv @@ -19,10 +19,13 @@ \unprotect +% the flusher is unexpandable so that it can be used in tables (noalign +% interferences) + \unexpanded\def\xmlresetsorter #1{\ctxlxml{sorters.reset("#1")}} \unexpanded\def\xmladdsortentry#1#2#3{\ctxlxml{sorters.add("#1","#2",\!!bs#3\!!es)}} \unexpanded\def\xmlshowsorter #1{\ctxlxml{sorters.show("#1")}} -\unexpanded\def\xmlflushsorter #1#2{\ctxlxml{sorters.flush("#1","#2")}} + \def\xmlflushsorter #1#2{\ctxlxml{sorters.flush("#1","#2")}} \unexpanded\def\xmlsortentries #1{\ctxlxml{sorters.sort("#1")}} \protect \endinput diff --git a/tex/context/base/mlib-pdf.lua b/tex/context/base/mlib-pdf.lua index 025bfc144..11325b9fe 100644 --- a/tex/context/base/mlib-pdf.lua +++ b/tex/context/base/mlib-pdf.lua @@ -19,13 +19,17 @@ local report_metapost = logs.reporter("metapost") local trace_variables = false trackers.register("metapost.variables",function(v) trace_variables = v end) -local mplib, context = mplib, context +local mplib = mplib +local context = context local allocate = utilities.storage.allocate local copy_node = node.copy local write_node = node.write +local pen_info = mplib.pen_info +local object_fields = mplib.fields + metapost = metapost or { } local metapost = metapost @@ -167,8 +171,6 @@ local bend_tolerance = 131/65536 local rx, sx, sy, ry, tx, ty, divider = 1, 0, 0, 1, 0, 0, 1 -local pen_info = mplib.pen_info - local function pen_characteristics(object) local t = pen_info(object) rx, ry, sx, sy, tx, ty = t.rx, t.ry, t.sx, t.sy, t.tx, t.ty @@ -640,7 +642,7 @@ function metapost.totable(result) for o=1,#objects do local object = objects[o] local result = { } - local fields = mplib.fields(object) -- hm, is this the whole list, if so, we can get it once + local fields = object_fields(object) -- hm, is this the whole list, if so, we can get it once for f=1,#fields do local field = fields[f] result[field] = object[field] diff --git a/tex/context/base/mlib-run.lua b/tex/context/base/mlib-run.lua index bd00cc260..61089e7e6 100644 --- a/tex/context/base/mlib-run.lua +++ b/tex/context/base/mlib-run.lua @@ -48,8 +48,6 @@ local mplib = mplib metapost = metapost or { } local metapost = metapost -local mplibone = tonumber(mplib.version()) <= 1.50 - metapost.showlog = false metapost.lastlog = "" metapost.collapse = true -- currently mplib cannot deal with begingroup/endgroup mismatch in stepwise processing @@ -84,87 +82,74 @@ local mpbasepath = lpeg.instringchecker(P("/metapost/") * (P("context") + P("bas -- mplib has no real io interface so we have a different mechanism than -- tex (as soon as we have more control, we will use the normal code) - -local finders = { } -mplib.finders = finders - +-- -- for some reason mp sometimes calls this function twice which is inefficient -- but we cannot catch this -local function preprocessed(name) - if not mpbasepath(name) then - -- we could use the via file but we don't have a complete io interface yet - local data, found, forced = metapost.checktexts(io.loaddata(name) or "") - if found then - local temp = luatex.registertempfile(name,true) - io.savedata(temp,data) - return temp +do + + local finders = { } + mplib.finders = finders -- also used in meta-lua.lua + + local new_instance = mplib.new + local resolved_file = resolvers.findfile + + local function preprocessed(name) + if not mpbasepath(name) then + -- we could use the via file but we don't have a complete io interface yet + local data, found, forced = metapost.checktexts(io.loaddata(name) or "") + if found then + local temp = luatex.registertempfile(name,true) + io.savedata(temp,data) + return temp + end end + return name end - return name -end -mplib.preprocessed = preprocessed -- helper + mplib.preprocessed = preprocessed -- helper -local function validftype(ftype) - if ftype == "" then - -- whatever - elseif ftype == 0 then - -- mplib bug - else - return ftype + local function validftype(ftype) + if ftype == "" then + -- whatever + elseif ftype == 0 then + -- mplib bug + else + return ftype + end end -end - -finders.file = function(specification,name,mode,ftype) - return preprocessed(resolvers.findfile(name,validftype(ftype))) -end -local function i_finder(name,mode,ftype) -- fake message for mpost.map and metafun.mpvi - local specification = url.hashed(name) - local finder = finders[specification.scheme] or finders.file - return finder(specification,name,mode,validftype(ftype)) -end + finders.file = function(specification,name,mode,ftype) + return preprocessed(resolvers.findfile(name,validftype(ftype))) + end -local function o_finder(name,mode,ftype) - -- report_metapost("output file %a, mode %a, ftype %a",name,mode,ftype) - return name -end + local function i_finder(name,mode,ftype) -- fake message for mpost.map and metafun.mpvi + local specification = url.hashed(name) + local finder = finders[specification.scheme] or finders.file + return finder(specification,name,mode,validftype(ftype)) + end -local function finder(name,mode,ftype) - if mode == "w" then - return o_finder(name,mode,validftype(ftype)) - else - return i_finder(name,mode,validftype(ftype)) + local function o_finder(name,mode,ftype) + return name end -end -local i_limited = false -local o_limited = false + o_finder = sandbox.register(o_finder,sandbox.filehandlerone,"mplib output finder") -directives.register("system.inputmode", function(v) - if not i_limited then - local i_limiter = io.i_limiter(v) - if i_limiter then - i_finder = i_limiter.protect(i_finder) - i_limited = true - end + local function finder(name,mode,ftype) + return (mode == "w" and o_finder or i_finder)(name,mode,validftype(ftype)) end -end) - -directives.register("system.outputmode", function(v) - if not o_limited then - local o_limiter = io.o_limiter(v) - if o_limiter then - o_finder = o_limiter.protect(o_finder) - o_limited = true - end + + function mplib.new(specification) + specification.find_file = finder -- so we block an overload + return new_instance(specification) end -end) --- -- -- + mplib.finder = finder + +end -metapost.finder = finder +local new_instance = mplib.new +local find_file = mplib.finder function metapost.reporterror(result) if not result then @@ -192,184 +177,77 @@ function metapost.reporterror(result) return true end -if mplibone then - - report_metapost("fatal error: mplib is too old") - - os.exit() - - -- local preamble = [[ - -- boolean mplib ; mplib := true ; - -- string mp_parent_version ; mp_parent_version := "%s" ; - -- input "%s" ; dump ; - -- ]] - -- - -- metapost.parameters = { - -- hash_size = 100000, - -- main_memory = 4000000, - -- max_in_open = 50, - -- param_size = 100000, - -- } - -- - -- function metapost.make(name, target, version) - -- starttiming(mplib) - -- target = file.replacesuffix(target or name, "mem") -- redundant - -- local mpx = mplib.new ( table.merged ( - -- metapost.parameters, - -- { - -- ini_version = true, - -- find_file = finder, - -- job_name = file.removesuffix(target), - -- } - -- ) ) - -- if mpx then - -- starttiming(metapost.exectime) - -- local result = mpx:execute(format(preamble,version or "unknown",name)) - -- stoptiming(metapost.exectime) - -- mpx:finish() - -- end - -- stoptiming(mplib) - -- end - -- - -- function metapost.load(name) - -- starttiming(mplib) - -- local mpx = mplib.new ( table.merged ( - -- metapost.parameters, - -- { - -- ini_version = false, - -- mem_name = file.replacesuffix(name,"mem"), - -- find_file = finder, - -- -- job_name = "mplib", - -- } - -- ) ) - -- local result - -- if not mpx then - -- result = { status = 99, error = "out of memory"} - -- end - -- stoptiming(mplib) - -- return mpx, result - -- end - -- - -- function metapost.checkformat(mpsinput) - -- local mpsversion = environment.version or "unset version" - -- local mpsinput = file.addsuffix(mpsinput or "metafun", "mp") - -- local mpsformat = file.removesuffix(file.basename(texconfig.formatname or (tex and tex.formatname) or mpsinput)) - -- local mpsbase = file.removesuffix(file.basename(mpsinput)) - -- if mpsbase ~= mpsformat then - -- mpsformat = mpsformat .. "-" .. mpsbase - -- end - -- mpsformat = file.addsuffix(mpsformat, "mem") - -- local mpsformatfullname = caches.getfirstreadablefile(mpsformat,"formats","metapost") or "" - -- if mpsformatfullname ~= "" then - -- report_metapost("loading %a from %a", mpsinput, mpsformatfullname) - -- local mpx, result = metapost.load(mpsformatfullname) - -- if mpx then - -- local result = mpx:execute("show mp_parent_version ;") - -- if not result.log then - -- metapost.reporterror(result) - -- else - -- local version = match(result.log,">> *(.-)[\n\r]") or "unknown" - -- version = gsub(version,"[\'\"]","") - -- if version ~= mpsversion then - -- report_metapost("version mismatch: %s <> %s", version or "unknown", mpsversion) - -- else - -- return mpx - -- end - -- end - -- else - -- report_metapost("error in loading %a from %a", mpsinput, mpsformatfullname) - -- metapost.reporterror(result) - -- end - -- end - -- local mpsformatfullname = caches.setfirstwritablefile(mpsformat,"formats") - -- report_metapost("making %a into %a", mpsinput, mpsformatfullname) - -- metapost.make(mpsinput,mpsformatfullname,mpsversion) -- somehow return ... fails here - -- if lfs.isfile(mpsformatfullname) then - -- report_metapost("loading %a from %a", mpsinput, mpsformatfullname) - -- return metapost.load(mpsformatfullname) - -- else - -- report_metapost("problems with %a from %a", mpsinput, mpsformatfullname) - -- end - -- end - -else - - -- let end = relax ; - - local preamble = [[ - boolean mplib ; mplib := true ; - let dump = endinput ; - input "%s" ; - ]] - - local methods = { - double = "double", - scaled = "scaled", - binary = "binary", - decimal = "decimal", - default = "scaled", - } +local preamble = [[ + boolean mplib ; mplib := true ; + let dump = endinput ; + input "%s" ; +]] + +local methods = { + double = "double", + scaled = "scaled", + binary = "binary", + decimal = "decimal", + default = "scaled", +} - function metapost.runscript(code) - return code - end +function metapost.runscript(code) + return code +end - function metapost.scripterror(str) - report_metapost("script error: %s",str) - end +function metapost.scripterror(str) + report_metapost("script error: %s",str) +end - function metapost.load(name,method) - starttiming(mplib) - method = method and methods[method] or "scaled" - local mpx = mplib.new { - ini_version = true, - find_file = finder, - math_mode = method, - run_script = metapost.runscript, - script_error = metapost.scripterror, - } - report_metapost("initializing number mode %a",method) - local result - if not mpx then - result = { status = 99, error = "out of memory"} - else - result = mpx:execute(format(preamble, file.addsuffix(name,"mp"))) -- addsuffix is redundant - end - stoptiming(mplib) - metapost.reporterror(result) - return mpx, result +function metapost.load(name,method) + starttiming(mplib) + method = method and methods[method] or "scaled" + local mpx = new_instance { + ini_version = true, + math_mode = method, + run_script = metapost.runscript, + script_error = metapost.scripterror, + } + report_metapost("initializing number mode %a",method) + local result + if not mpx then + result = { status = 99, error = "out of memory"} + else + result = mpx:execute(format(preamble, file.addsuffix(name,"mp"))) -- addsuffix is redundant end + stoptiming(mplib) + metapost.reporterror(result) + return mpx, result +end - function metapost.checkformat(mpsinput,method) - local mpsversion = environment.version or "unset version" - local mpsinput = mpsinput or "metafun" - local foundfile = "" - if file.suffix(mpsinput) ~= "" then - foundfile = finder(mpsinput) or "" - end - if foundfile == "" then - foundfile = finder(file.replacesuffix(mpsinput,"mpvi")) or "" - end - if foundfile == "" then - foundfile = finder(file.replacesuffix(mpsinput,"mpiv")) or "" - end - if foundfile == "" then - foundfile = finder(file.replacesuffix(mpsinput,"mp")) or "" - end - if foundfile == "" then - report_metapost("loading %a fails, format not found",mpsinput) +function metapost.checkformat(mpsinput,method) + local mpsversion = environment.version or "unset version" + local mpsinput = mpsinput or "metafun" + local foundfile = "" + if file.suffix(mpsinput) ~= "" then + foundfile = find_file(mpsinput) or "" + end + if foundfile == "" then + foundfile = find_file(file.replacesuffix(mpsinput,"mpvi")) or "" + end + if foundfile == "" then + foundfile = find_file(file.replacesuffix(mpsinput,"mpiv")) or "" + end + if foundfile == "" then + foundfile = find_file(file.replacesuffix(mpsinput,"mp")) or "" + end + if foundfile == "" then + report_metapost("loading %a fails, format not found",mpsinput) + else + report_metapost("loading %a as %a using method %a",mpsinput,foundfile,method or "default") + local mpx, result = metapost.load(foundfile,method) + if mpx then + return mpx else - report_metapost("loading %a as %a using method %a",mpsinput,foundfile,method or "default") - local mpx, result = metapost.load(foundfile,method) - if mpx then - return mpx - else - report_metapost("error in loading %a",mpsinput) - metapost.reporterror(result) - end + report_metapost("error in loading %a",mpsinput) + metapost.reporterror(result) end end - end function metapost.unload(mpx) diff --git a/tex/context/base/mtx-context-precache.tex b/tex/context/base/mtx-context-precache.tex new file mode 100644 index 000000000..9cbb46cf2 --- /dev/null +++ b/tex/context/base/mtx-context-precache.tex @@ -0,0 +1,161 @@ +%D \module +%D [ file=mtx-context-precache, +%D version=2014.12.24, +%D title=\CONTEXT\ Extra Trickry, +%D subtitle=Precaching Fonts, +%D author=Hans Hagen, +%D date=\currentdate, +%D copyright={PRAGMA ADE \& \CONTEXT\ Development Team}] +%C +%C This module is part of the \CONTEXT\ macro||package and is +%C therefore copyrighted by \PRAGMA. See mreadme.pdf for +%C details. + +% begin help +% +% usage: context --extra=precache [no options yet] +% +% example: context --extra=precache +% +% end help + +\startluacode + +local lower = string.lower +local filesuffix = file.suffix +local findfile = resolvers.find_file + +local report = logs.reporter("fonts","precache") + +function fonts.names.precache() + local handlers = fonts.handlers + if not handlers then + report("no handlers available") + return + end + local otfloader = handlers.otf and handlers.otf.load + local afmloader = handlers.afm and handlers.afm.load + if not (otfloader or afmloader) then + report("no otf or afm handler available") + return + end + fonts.names.load() + local data = fonts.names.data + if not data then + report("no font data available") + return + end + local specifications = data.specifications + if not specifications then + report("no font specifications available") + return + end + local n = 0 + for i=1,#specifications do + local specification = specifications[i] + local filename = specification.filename + local cleanfilename = specification.cleanfilename + local foundfile = findfile(filename) + if foundfile and foundfile ~= "" then + local suffix = lower(filesuffix(foundfile)) + if suffix == "otf" or suffix == "ttf" then + if otfloader then + report("caching otf file: %s",foundfile) + otfloader(foundfile) -- todo: ttc/sub + n = n + 1 + end + elseif suffix == "afm" then + if afmloader then + report("caching afm file: %s",foundfile) + afmloader(foundfile) + n = n + 1 + end + end + end + end + report("%s files out of %s cached",n,#specifications) +end + +\stopluacode + +\starttext + +\setuppapersize + [A4,landscape] + +\setuplayout + [width=middle, + height=middle, + footer=0pt, + header=1cm, + headerdistance=0cm, + backspace=5mm, + topspace=5mm] + +\setupbodyfont + [dejavu,6pt,tt] + +\startmode[*first] + \startluacode + fonts.names.precache() + \stopluacode +\stopmode + +\startluacode + fonts.names.load() + + local specifications = fonts.names.data.specifications + + local sorted = { } + local hashed = { } + + for i=1,#specifications do + local filename = specifications[i].cleanfilename + sorted[i] = filename + hashed[filename] = i + end + + table.sort(sorted) + + local context = context + local basename = file.basename + + local NC = context.NC + local NR = context.NR + local HL = context.HL + local bold = context.bold + + context.starttabulate { "||||||||||" } + HL() + NC() bold("format") + NC() bold("cleanfilename") + NC() bold("filename") + -- NC() bold("familyname") + -- NC() bold("fontname") + NC() bold("fullname") + NC() bold("rawname") + NC() bold("style") + NC() bold("variant") + NC() bold("weight") + NC() bold("width") + NC() NR() + HL() + for i=1,#sorted do + local specification = specifications[hashed[sorted[i]]] + NC() context(specification.format) + NC() context(specification.cleanfilename) + NC() context(basename(specification.filename)) + -- NC() context(specification.familyname) + -- NC() context(specification.fontname) + NC() context(specification.fullname) + NC() context(specification.rawname) + NC() context(specification.style) + NC() context(specification.variant) + NC() context(specification.weight) + NC() context(specification.width) + NC() NR() + end + context.stoptabulate() +\stopluacode + +\stoptext diff --git a/tex/context/base/publ-aut.lua b/tex/context/base/publ-aut.lua index 266740ea2..ea86e702a 100644 --- a/tex/context/base/publ-aut.lua +++ b/tex/context/base/publ-aut.lua @@ -229,8 +229,7 @@ local function the_initials(initials,symbol,connector) end r = r + 1 ; result[r] = concat(set) else - r = r + 1 ; result[r] = initial - r = r + 1 ; result[r] = symbol + r = r + 1 ; result[r] = initial .. symbol end end return result @@ -462,6 +461,7 @@ end local function indexer(dataset,list,method) local current = datasets[dataset] local luadata = current.luadata + local details = current.details local result = { } local splitted = newsplitter(splitter) -- saves mem local snippets = { } -- saves mem @@ -473,8 +473,9 @@ local function indexer(dataset,list,method) local index = tostring(i) local entry = luadata[tag] if entry then - local value = getcasted(current,entry,field) or "" + local value = getcasted(current,tag,field) or "" local mainkey = writer(value,snippets) + local detail = details[tag] result[i] = { index = i, split = { @@ -515,17 +516,22 @@ local function indexer(dataset,list,method) return result end -local function sorted(dataset,list,sorttype) -- experimental - local valid = indexer(dataset,list,sorttype) - if #valid == 0 or #valid ~= #list then - return list - else - sorters.sort(valid,function(a,b) return a ~= b and compare(a,b) == -1 end) - for i=1,#valid do - valid[i] = valid[i].index - end - return valid - end +-- local function sorted(dataset,list) -- experimental +-- local valid = indexer(dataset,list,sorttype) +-- if #valid == 0 or #valid ~= #list then +-- return list +-- else +-- sorters.sort(valid,function(a,b) return a ~= b and compare(a,b) == -1 end) +-- for i=1,#valid do +-- valid[i] = valid[i].index +-- end +-- return valid +-- end +-- end + +local function sorted(dataset,valid) -- experimental + sorters.sort(valid,compare) + return valid end -- made public diff --git a/tex/context/base/publ-ini.lua b/tex/context/base/publ-ini.lua index f58729554..43cc37fc2 100644 --- a/tex/context/base/publ-ini.lua +++ b/tex/context/base/publ-ini.lua @@ -1601,12 +1601,14 @@ do end end, [v_author] = function(dataset,rendering,list) + -- there is no real need to go vi aindex as the list itself can be sorted ... todo local valid = publications.indexers.author(dataset,list) if #valid == 0 or #valid ~= #list then -- nothing to sort else -- if needed we can wrap compare and use the list directly but this is cleaner - sorters.sort(valid,sortcomparer) +-- sorters.sort(valid,publications.sorters.author) + local valid = publications.sorters.author(dataset,valid) for i=1,#valid do local v = valid[i] valid[i] = list[v.index] @@ -1662,17 +1664,23 @@ do end end end - rendering.list = type(sorter) == "function" and sorter(dataset,rendering,newlist,sorttype) or newlist + if type(sorter) == "function" then + rendering.list = sorter(dataset,rendering,newlist,sorttype) + else + rendering.list = newlist + end end function lists.fetchentries(dataset) local rendering = renderings[dataset] local list = rendering.list - for i=1,#list do - local li = list[i] - ctx_btxsettag(li[1]) - ctx_btxsetnumber(li[3]) - ctx_btxchecklistentry() + if list then + for i=1,#list do + local li = list[i] + ctx_btxsettag(li[1]) + ctx_btxsetnumber(li[3]) + ctx_btxchecklistentry() + end end end @@ -1745,49 +1753,51 @@ do -- maybe a startflushing here ignoredfields = rendering.ignored or { } -- - for i=1,#list do - local li = list[i] - local tag = li[1] - local n = li[3] - local entry = luadata[tag] - local combined = entry.combined - local language = entry.language - if combined then - ctx_btxsetcombis(concat(combined,",")) - end - ctx_btxsetcategory(entry.category or "unknown") - ctx_btxsettag(tag) - ctx_btxsetnumber(n) - if language then - ctx_btxsetlanguage(language) - end - local bl = li[5] - if bl and bl ~= "" then - ctx_btxsetbacklink(bl) - ctx_btxsetbacktrace(concat(li," ",5)) - local uc = citetolist[tonumber(bl)] - if uc then - ctx_btxsetinternal(uc.references.internal or "") + if list then + for i=1,#list do + local li = list[i] + local tag = li[1] + local n = li[3] + local entry = luadata[tag] + local combined = entry.combined + local language = entry.language + if combined then + ctx_btxsetcombis(concat(combined,",")) end - else - -- nothing - end - local userdata = li[4] - if userdata then - local b = userdata.btxbtx - local a = userdata.btxatx - if b then - ctx_btxsetbefore(b) + ctx_btxsetcategory(entry.category or "unknown") + ctx_btxsettag(tag) + ctx_btxsetnumber(n) + if language then + ctx_btxsetlanguage(language) end - if a then - ctx_btxsetafter(a) + local bl = li[5] + if bl and bl ~= "" then + ctx_btxsetbacklink(bl) + ctx_btxsetbacktrace(concat(li," ",5)) + local uc = citetolist[tonumber(bl)] + if uc then + ctx_btxsetinternal(uc.references.internal or "") + end + else + -- nothing + end + local userdata = li[4] + if userdata then + local b = userdata.btxbtx + local a = userdata.btxatx + if b then + ctx_btxsetbefore(b) + end + if a then + ctx_btxsetafter(a) + end + end + rendering.userdata = userdata + if textmode then + ctx_btxhandlelisttextentry() + else + ctx_btxhandlelistentry() end - end - rendering.userdata = userdata - if textmode then - ctx_btxhandlelisttextentry() - else - ctx_btxhandlelistentry() end end context(function() diff --git a/tex/context/base/publ-ini.mkiv b/tex/context/base/publ-ini.mkiv index da1466c7e..aaee08d53 100644 --- a/tex/context/base/publ-ini.mkiv +++ b/tex/context/base/publ-ini.mkiv @@ -1514,6 +1514,12 @@ \definebtxlistvariant [doi] +\definebtxlistvariant % because we inherit + [invertedshort] + +\definebtxlistvariant % because we inherit + [short] + \setupbtxcitevariant [\c!specification=\btxparameter\c!specification, \c!alternative=num, diff --git a/tex/context/base/sort-ini.lua b/tex/context/base/sort-ini.lua index f9d6b5da2..995977613 100644 --- a/tex/context/base/sort-ini.lua +++ b/tex/context/base/sort-ini.lua @@ -753,7 +753,9 @@ function sorters.sort(entries,cmp) first = " " else s = first - report_sorters(">> %C (%C)",first,letter) + if first and letter then + report_sorters(">> %C (%C)",first,letter) + end end report_sorters(" %s | %s",packch(entry),packuc(entry)) end diff --git a/tex/context/base/status-files.pdf b/tex/context/base/status-files.pdf Binary files differindex 28a543e8f..d124c87c0 100644 --- a/tex/context/base/status-files.pdf +++ b/tex/context/base/status-files.pdf diff --git a/tex/context/base/status-lua.pdf b/tex/context/base/status-lua.pdf Binary files differindex 23bd6e0ec..860ccdb74 100644 --- a/tex/context/base/status-lua.pdf +++ b/tex/context/base/status-lua.pdf diff --git a/tex/context/base/syst-lua.lua b/tex/context/base/syst-lua.lua index cd7dcc062..95f8628ee 100644 --- a/tex/context/base/syst-lua.lua +++ b/tex/context/base/syst-lua.lua @@ -122,3 +122,14 @@ end function commands.ntimes(str,n) context(rep(str,n or 1)) end + +function commands.write(n,str) + if n == 18 then + os.execute(str) + elseif n == 16 then + logs.report(str) + else + -- at the tex end we can still drop the write + context.writeviatex(n,str) + end +end diff --git a/tex/context/base/syst-lua.mkiv b/tex/context/base/syst-lua.mkiv index 88a8c246e..c146b81b7 100644 --- a/tex/context/base/syst-lua.mkiv +++ b/tex/context/base/syst-lua.mkiv @@ -50,4 +50,37 @@ \def\ui_ft#1#2{#1} \def\ui_st#1#2{#2} +%D Let's bring this under \LUA\ (and therefore \MKIV\ sandbox) control: + +% \setnewconstant\c_syst_write 18 +% +% \unexpanded\def\write#1#% so we can handle \immediate +% {\ifnum#1=\c_syst_write +% \expandafter\syst_execute +% \else +% \normalwrite#1% +% \fi} +% +% \unexpanded\def\syst_execute#1% +% {\ctxlua{os.execute(\!!bs#1\!!es)}} + +%D But as we only use write 16 we could as well do all in \LUA\ +%D and ignore the rest. Okay, we still can do writes here but only +%D when not blocked. + +% Nicer would be if we could just disable write 18 and keep os.execute +% which in fact we can do by defining write18 as macro instead of +% primitive ... todo. + +\unexpanded\def\write#1#% + {\syst_write{#1}} + +\def\syst_write#1#2% + {\ctxcommand{write(\number#1,\!!bs#2\!!es)}} + +\unexpanded\def\writeviatex#1#2% + {\ifx\normalwrite\relax\else + \normalwrite#1{#2}% + \fi} + \protect \endinput diff --git a/tex/context/base/util-sbx.lua b/tex/context/base/util-sbx.lua new file mode 100644 index 000000000..260e8b3b5 --- /dev/null +++ b/tex/context/base/util-sbx.lua @@ -0,0 +1,415 @@ +if not modules then modules = { } end modules ['util-sbx'] = { + version = 1.001, + comment = "companion to luat-lib.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- Note: we use expandname and collapsepath and these use chdir +-- which is overloaded so we need to use originals there. Just +-- something to keep in mind. + +if not sandbox then require("l-sandbox") end -- for testing + +local next, type = next, type + +local replace = utilities.templates.replace +local collapsepath = file.collapsepath +local expandname = dir.expandname +local sortedhash = table.sortedhash +local lpegmatch = lpeg.match +local platform = os.type +local P, S, C = lpeg.P, lpeg.S, lpeg.C +local gsub = string.gsub +local lower = string.lower +local unquoted = string.unquoted +local optionalquoted = string.optionalquoted + +local sandbox = sandbox +local validroots = { } +local validrunners = { } +local validbinaries = { } +local validators = { } +local p_validroot = nil +local finalized = nil +local norunners = false +local trace = false +local p_split = lpeg.tsplitat(" ") -- more spaces? + +local report = logs.reporter("sandbox") + +trackers.register("sandbox",function(v) trace = v end) -- often too late anyway + +sandbox.setreporter(report) + +sandbox.finalizer(function() + finalized = true +end) + +local function registerroot(root,what) -- what == read|write + if finalized then + report("roots are already finalized") + else + root = collapsepath(expandname(root)) + if platform == "windows" then + root = lower(root) -- we assume ascii names + end + -- true: read & write | false: read + validroots[root] = what == "write" or false + end +end + +sandbox.finalizer(function() -- initializers can set the path + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") -- always ok + -- also register texmf as read + for name in sortedhash(validroots) do + if p_validroot then + p_validroot = P(name) + p_validroot + else + p_validroot = P(name) + end + end + p_validroot = p_validroot / validroots + end +end) + +local function registerrunner(specification) + if finalized then + report("runners are already finalized") + else + local name = specification.name + if not name then + report("no runner name specified") + return + end + local program = specification.program + if type(program) == "string" then + -- common for all platforms + elseif type(program) == "table" then + program = program[platform] + end + if type(program) ~= "string" or program == "" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + specification.program = program + validrunners[name] = specification + end +end + +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name) == "string" and name ~= "" then + validbinaries[name] = true + end +end + +-- begin of validators + +local p_write = S("wa") p_write = (1 - p_write)^0 * p_write +local p_path = S("\\/~$%:") p_path = (1 - p_path )^0 * p_path -- be easy on other arguments + +local function normalized(name) -- only used in executers + if platform == "windows" then + name = gsub(name,"/","\\") + end + return name +end + +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end + +local filenamelogger = false + +function sandbox.setfilenamelogger(l) + filenamelogger = type(l) == "function" and l or false +end + +local function validfilename(name,what) + if p_validroot and type(name) == "string" and lpegmatch(p_path,name) then + local asked = collapsepath(expandname(name)) + if platform == "windows" then + asked = lower(asked) -- we assume ascii names + end + local okay = lpegmatch(p_validroot,asked) + if okay == true then + -- read and write access + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay == false then + -- read only access + if not what then + -- no further argument to io.open so a readonly case + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return -- we want write access + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + else + if filenamelogger then + filenamelogger(name,"*",name,false) + end + end + else + return name + end +end + +local function readable(name) + if platform == "windows" then + name = lower(name) -- we assume ascii names + end + local valid = validfilename(name,"r") + if valid then + return normalized(valid) + end +end + +local function writeable(name) + if platform == "windows" then + name = lower(name) -- we assume ascii names + end + local valid = validfilename(name,"w") + if valid then + return normalized(valid) + end +end + +validators.readable = readable +validators.writeable = writeable +validators.filename = readable + +table.setmetatableindex(validators,function(t,k) + if k then + t[k] = readable + end + return readable +end) + +function validators.string(s) + return s -- can be used to prevent filename checking +end + +-- end of validators + +sandbox.registerroot = registerroot +sandbox.registerrunner = registerrunner +sandbox.registerbinary = registerbinary +sandbox.validfilename = validfilename + +local function filehandlerone(action,one,...) + local checkedone = validfilename(one) + if checkedone then + return action(one,...) + else +-- report("file %a is unreachable",one) + end +end + +local function filehandlertwo(action,one,two,...) + local checkedone = validfilename(one) + if checkedone then + local checkedtwo = validfilename(two) + if checkedtwo then + return action(one,two,...) + else +-- report("file %a is unreachable",two) + end + else +-- report("file %a is unreachable",one) + end +end + +local function iohandler(action,one,...) + if type(one) == "string" then + local checkedone = validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end + +-- runners can be strings or tables +-- +-- os.execute : string +-- os.exec : table with program in [0|1] +-- os.spawn : table with program in [0|1] +-- +-- our execute: registered program with specification + +local function runhandler(action,name,specification) + local kind = type(name) + if kind ~= "string" then + return + end + if norunners then + report("no runners permitted, ignoring command: %s",name) + return + end + local spec = validrunners[name] + if not spec then + report("unknown runner: %s",name) + return + end + -- specs are already checked + local program = spec.program + local variables = { } + local checkers = spec.checkers or { } + if specification then + -- we only handle runners that are defined before the sandbox is + -- closed so in principle we cannot have user runs with no files + -- while for context runners we assume a robust specification + for k, v in next, specification do + local checker = validators[checkers[k]] + local value = checker(unquoted(v)) -- todo: write checkers + if value then + variables[k] = optionalquoted(value) + else + report("suspicious argument found, run blocked: %s",v) + return + end + end + end + local command = replace(program,variables) + if trace then + report("executing runner: %s",command) + end + return action(command) +end + +-- only registered (from list) -- no checking on writable so let's assume harmless +-- runs + +local function binaryhandler(action,name) + local kind = type(name) + local list = name + if kind == "string" then + list = lpegmatch(p_split,name) + end + local program = name[0] or name[1] + if type(program) ~= "string" or program == "" then + return --silently ignore + end + if norunners then + report("no binaries permitted, ignoring command: %s",program) + return + end + if not validbinaries[program] then + report("binary is not permitted: %s",program) + return + end + for i=0,#list do + local n = list[i] + if n then + local v = readable(unquoted(n)) + if v then + list[i] = optionalquoted(v) + else + report("suspicious argument found, run blocked: %s",n) + return + end + end + end + return action(name) +end + +sandbox.filehandlerone = filehandlerone +sandbox.filehandlertwo = filehandlertwo +sandbox.iohandler = iohandler +sandbox.runhandler = runhandler +sandbox.binaryhandler = binaryhandler + +function sandbox.disablerunners() + norunners = true +end + +local execute = sandbox.original(os.execute) + +function sandbox.run(name,specification) + return runhandler(execute,name,specification) +end + +------------------- + +local overload = sandbox.overload +local register = sandbox.register + + overload(loadfile, filehandlerone,"loadfile") -- todo + +if io then + overload(io.open, filehandlerone,"io.open") + overload(io.popen, filehandlerone,"io.popen") + overload(io.input, iohandler, "io.input") + overload(io.output, iohandler, "io.output") + overload(io.lines, filehandlerone,"io.lines") +end + +if os then + overload(os.execute, binaryhandler, "os.execute") + overload(os.spawn, binaryhandler, "os.spawn") + overload(os.exec, binaryhandler, "os.exec") + overload(os.rename, filehandlertwo,"os.rename") + overload(os.remove, filehandlerone,"os.remove") +end + +if lfs then + overload(lfs.chdir, filehandlerone,"lfs.chdir") + overload(lfs.mkdir, filehandlerone,"lfs.mkdir") + overload(lfs.rmdir, filehandlerone,"lfs.rmdir") + overload(lfs.isfile, filehandlerone,"lfs.isfile") + overload(lfs.isdir, filehandlerone,"lfs.isdir") + overload(lfs.attributes, filehandlerone,"lfs.attributes") + overload(lfs.dir, filehandlerone,"lfs.dir") + overload(lfs.lock_dir, filehandlerone,"lfs.lock_dir") + overload(lfs.touch, filehandlerone,"lfs.touch") + overload(lfs.link, filehandlertwo,"lfs.link") + overload(lfs.setmode, filehandlerone,"lfs.setmode") + overload(lfs.readlink, filehandlerone,"lfs.readlink") + overload(lfs.shortname, filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end + +-- these are used later on + +if zip then + zip.open = register(zip.open, filehandlerone,"zip.open") +end + +if fontloader then + fontloader.open = register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info = register(fontloader.info,filehandlerone,"fontloader.info") +end + +if epdf then + epdf.open = register(epdf.open, filehandlerone,"epdf.open") +end + +-- not used in a normal mkiv run : os.spawn = os.execute +-- not used in a normal mkiv run : os.exec = os.exec + +-- print(io.open("test.log")) +-- sandbox.enable() +-- print(io.open("test.log")) +-- print(io.open("t:/test.log")) diff --git a/tex/context/base/util-str.lua b/tex/context/base/util-str.lua index a040b0113..a677a82ed 100644 --- a/tex/context/base/util-str.lua +++ b/tex/context/base/util-str.lua @@ -361,10 +361,10 @@ strings.tracers = tracedchars function string.tracedchar(b) -- todo: table if type(b) == "number" then - return tracedchars[b] or (utfchar(b) .. " (U+" .. format('%05X',b) .. ")") + return tracedchars[b] or (utfchar(b) .. " (U+" .. format("%05X",b) .. ")") else local c = utfbyte(b) - return tracedchars[c] or (b .. " (U+" .. format('%05X',c) .. ")") + return tracedchars[c] or (b .. " (U+" .. (c and format("%05X",c) or "?????") .. ")") end end diff --git a/tex/context/base/util-tab.lua b/tex/context/base/util-tab.lua index 077c90643..5eae0d5f6 100644 --- a/tex/context/base/util-tab.lua +++ b/tex/context/base/util-tab.lua @@ -623,7 +623,7 @@ function table.serialize(root,name,specification) depth = depth + 1 end -- we could check for k (index) being number (cardinal) - if root and next(root) then + if root and next(root) ~= nil then local first = nil local last = 0 last = #root @@ -648,7 +648,7 @@ function table.serialize(root,name,specification) elseif tv == "string" then n = n + 1 t[n] = f_val_str(depth,v) elseif tv == "table" then - if not next(v) then + if next(v) == nil then n = n + 1 t[n] = f_val_not(depth) else local st = simple_table(v) @@ -678,7 +678,7 @@ function table.serialize(root,name,specification) n = n + 1 t[n] = f_key_boo_value_str(depth,k,v) end elseif tv == "table" then - if not next(v) then + if next(v) == nil then if tk == "number" then n = n + 1 t[n] = f_key_num_value_not(depth,k,v) elseif tk == "string" then @@ -742,7 +742,7 @@ function table.serialize(root,name,specification) root._w_h_a_t_e_v_e_r_ = nil end -- Let's forget about empty tables. - if next(root) then + if next(root) ~= nil then do_serialize(root,name,1,0) end end diff --git a/tex/generic/context/luatex/luatex-fonts-merged.lua b/tex/generic/context/luatex/luatex-fonts-merged.lua index 96a7dd361..2159621ea 100644 --- a/tex/generic/context/luatex/luatex-fonts-merged.lua +++ b/tex/generic/context/luatex/luatex-fonts-merged.lua @@ -1,6 +1,6 @@ -- merged file : luatex-fonts-merged.lua -- parent file : luatex-fonts.lua --- merge date : 12/21/14 22:25:48 +-- merge date : 12/28/14 19:50:53 do -- begin closure to overcome local limits and interference @@ -85,6 +85,13 @@ end if lua then lua.mask=load([[τεχ = 1]]) and "utf" or "ascii" end +local flush=io.flush +if flush then + local execute=os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec=os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end +end end -- closure @@ -964,8 +971,9 @@ function table.keys(t) end end local function compare(a,b) - local ta,tb=type(a),type(b) - if ta==tb then + local ta=type(a) + local tb=type(b) + if ta==tb and ta=="number" then return a<b else return tostring(a)<tostring(b) @@ -1288,7 +1296,7 @@ local function do_serialize(root,name,depth,level,indexed) end end end - if root and next(root) then + if root and next(root)~=nil then local first,last=nil,0 if compact then last=#root @@ -1321,7 +1329,7 @@ local function do_serialize(root,name,depth,level,indexed) handle(format("%s %q,",depth,v)) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then handle(format("%s {},",depth)) elseif inline then local st=simple_table(v) @@ -1405,7 +1413,7 @@ local function do_serialize(root,name,depth,level,indexed) end end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then if hexify then handle(format("%s [0x%X]={},",depth,k)) @@ -1547,7 +1555,7 @@ local function serialize(_handle,root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,"",0) end end @@ -1682,7 +1690,7 @@ local function sparse(old,nest,keeptables) if not (v=="" or v==false) then if nest and type(v)=="table" then v=sparse(v,nest) - if keeptables or next(v) then + if keeptables or next(v)~=nil then new[k]=v end else @@ -1799,10 +1807,10 @@ function table.sub(t,i,j) return { unpack(t,i,j) } end function table.is_empty(t) - return not t or not next(t) + return not t or next(t)==nil end function table.has_one_entry(t) - return t and not next(t,next(t)) + return t and next(t,next(t))==nil end function table.loweredkeys(t) local l={} @@ -1871,7 +1879,7 @@ function table.filtered(t,pattern,sort,cmp) else local n=next(t) local function iterator() - while n do + while n~=nil do local k=n n=next(t,k) if find(k,pattern) then @@ -2195,8 +2203,6 @@ function io.readstring(f,n,m) local str=gsub(f:read(n),"\000","") return str end -if not io.i_limiter then function io.i_limiter() end end -if not io.o_limiter then function io.o_limiter() end end end -- closure @@ -2214,41 +2220,28 @@ local file=file if not lfs then lfs=optionalrequire("lfs") end -if not lfs then - lfs={ - getcurrentdir=function() - return "." - end, - attributes=function() - return nil - end, - isfile=function(name) - local f=io.open(name,'rb') - if f then - f:close() - return true - end - end, - isdir=function(name) - print("you need to load lfs") - return false - end - } -elseif not lfs.isfile then - local attributes=lfs.attributes - function lfs.isdir(name) - return attributes(name,"mode")=="directory" - end - function lfs.isfile(name) - return attributes(name,"mode")=="file" - end -end local insert,concat=table.insert,table.concat local match,find,gmatch=string.match,string.find,string.gmatch local lpegmatch=lpeg.match local getcurrentdir,attributes=lfs.currentdir,lfs.attributes local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local tricky=S("/\\")*P(-1) +local attributes=lfs.attributes +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir,"lfs.isdir") +end +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end +end +function lfs.isfile(name) + return attributes(name,"mode")=="file" +end local colon=P(":") local period=P(".") local periods=P("..") @@ -2535,18 +2528,6 @@ function file.collapsepath(str,anchor) end end end -local tricky=S("/\\")*P(-1) -local attributes=lfs.attributes -function lfs.isdir(name) - if lpegmatch(tricky,name) then - return attributes(name,"mode")=="directory" - else - return attributes(name.."/.","mode")=="directory" - end -end -function lfs.isfile(name) - return attributes(name,"mode")=="file" -end local validchars=R("az","09","AZ","--","..") local pattern_a=lpeg.replacer(1-validchars) local pattern_a=Cs((validchars+P(1)/"-")^1) @@ -2874,10 +2855,10 @@ string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) if type(b)=="number" then - return tracedchars[b] or (utfchar(b).." (U+"..format('%05X',b)..")") + return tracedchars[b] or (utfchar(b).." (U+"..format("%05X",b)..")") else local c=utfbyte(b) - return tracedchars[c] or (b.." (U+"..format('%05X',c)..")") + return tracedchars[c] or (b.." (U+"..(c and format("%05X",c) or "?????")..")") end end function number.signed(i) @@ -3981,7 +3962,7 @@ fonts.analyzers={} fonts.readers={} fonts.definers={ methods={} } fonts.loggers={ register=function() end } -fontloader.totable=fontloader.to_table +fontloader.totable=fontloader.to_table end -- closure @@ -5915,6 +5896,10 @@ local findbinfile=resolvers.findbinfile local definers=fonts.definers local readers=fonts.readers local constructors=fonts.constructors +local fontloader=fontloader +local font_to_table=fontloader.to_table +local open_font=fontloader.open +local close_font=fontloader.close local afm=constructors.newhandler("afm") local pfb=constructors.newhandler("pfb") local afmfeatures=constructors.newfeatures("afm") @@ -6030,10 +6015,10 @@ local function get_variables(data,fontmetrics) end local function get_indexes(data,pfbname) data.resources.filename=resolvers.unresolve(pfbname) - local pfbblob=fontloader.open(pfbname) + local pfbblob=open_font(pfbname) if pfbblob then local characters=data.characters - local pfbdata=fontloader.to_table(pfbblob) + local pfbdata=font_to_table(pfbblob) if pfbdata then local glyphs=pfbdata.glyphs if glyphs then @@ -6058,7 +6043,7 @@ local function get_indexes(data,pfbname) elseif trace_loading then report_afm("no data in pfb file %a",pfbname) end - fontloader.close(pfbblob) + close_font(pfbblob) elseif trace_loading then report_afm("invalid pfb file %a",pfbname) end @@ -7074,11 +7059,12 @@ local otf=fonts.handlers.otf otf.glists={ "gsub","gpos" } otf.version=2.802 otf.cache=containers.define("fonts","otf",otf.version,true) -local fontdata=fonts.hashes.identifiers -local chardata=characters and characters.data +local hashes=fonts.hashes local definers=fonts.definers local readers=fonts.readers local constructors=fonts.constructors +local fontdata=hashes and hashes.identifiers +local chardata=characters and characters.data local otffeatures=constructors.newfeatures("otf") local registerotffeature=otffeatures.register local enhancers=allocate() @@ -7095,7 +7081,11 @@ local overloadkerns=false local applyruntimefixes=fonts.treatments and fonts.treatments.applyfixes local wildcard="*" local default="dflt" -local fontloaderfields=fontloader.fields +local fontloader=fontloader +local open_font=fontloader.open +local close_font=fontloader.close +local font_fields=fontloader.fields +local apply_featurefile=fontloader.apply_featurefile local mainfields=nil local glyphfields=nil local formats=fonts.formats @@ -7136,7 +7126,7 @@ local function load_featurefile(raw,featurefile) if trace_loading then report_otf("using featurefile %a",featurefile) end - fontloader.apply_featurefile(raw,featurefile) + apply_featurefile(raw,featurefile) end end local function showfeatureorder(rawdata,filename) @@ -7387,12 +7377,12 @@ function otf.load(filename,sub,featurefile) report_otf("loading %a, hash %a",filename,hash) local fontdata,messages if sub then - fontdata,messages=fontloader.open(filename,sub) + fontdata,messages=open_font(filename,sub) else - fontdata,messages=fontloader.open(filename) + fontdata,messages=open_font(filename) end if fontdata then - mainfields=mainfields or (fontloaderfields and fontloaderfields(fontdata)) + mainfields=mainfields or (font_fields and font_fields(fontdata)) end if trace_loading and messages and #messages>0 then if type(messages)=="string" then @@ -7466,7 +7456,7 @@ function otf.load(filename,sub,featurefile) report_otf("preprocessing and caching time %s, packtime %s", elapsedtime(data),packdata and elapsedtime(packtime) or 0) end - fontloader.close(fontdata) + close_font(fontdata) if cleanup>3 then collectgarbage("collect") end |