diff options
author | Context Git Mirror Bot <phg42.2a@gmail.com> | 2015-05-16 00:15:04 +0200 |
---|---|---|
committer | Context Git Mirror Bot <phg42.2a@gmail.com> | 2015-05-16 00:15:04 +0200 |
commit | b55577d0998160c0174e250b542016ecd6ca9056 (patch) | |
tree | 27093212d5ca3e6ffe4ae434c3ec094233ed37ba /scripts | |
parent | 624cbb5da392e9403984dd1cf368c0d408b1c2a8 (diff) | |
download | context-b55577d0998160c0174e250b542016ecd6ca9056.tar.gz |
2015-05-15 23:06:00
Diffstat (limited to 'scripts')
59 files changed, 29966 insertions, 5082 deletions
diff --git a/scripts/context/lua/mtx-bibtex.lua b/scripts/context/lua/mtx-bibtex.lua new file mode 100644 index 000000000..92036e3a5 --- /dev/null +++ b/scripts/context/lua/mtx-bibtex.lua @@ -0,0 +1,152 @@ +if not modules then modules = { } end modules ['mtx-bibtex'] = { + version = 1.002, + comment = "this script is part of publication support", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE", + license = "see context related readme files" +} + +local helpinfo = [[ +<?xml version="1.0"?> +<application> + <metadata> + <entry name="name">mtx-bibtex</entry> + <entry name="detail">bibtex helpers</entry> + <entry name="version">1.00</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="toxml"><short>convert bibtex database(s) to xml</short></flag> + <flag name="tolua"><short>convert bibtex database(s) to lua</short></flag> + <flag name="search"><short>seatch bibtex database(s)</short></flag> + </subcategory> + </category> + </flags> + <examples> + <category> + <title>Example</title> + <subcategory> + <example><command>mtxrun --script bibtex --tolua bibl-001.bib</command></example> + <example><command>mtxrun --script bibtex --tolua --simple bibl-001.bib</command></example> + <example><command>mtxrun --script bibtex --toxml bibl-001.bib bibl-002.bib bibl-003.bib biblio.xml</command></example> + <example><command>mtxrun --script bibtex --search --list --pattern=match(author:foo) bar.bib</command></example> + </subcategory> + </category> + </examples> +</application> +]] + +local application = logs.application { + name = "mtx-bibtex", + banner = "bibtex helpers", + helpinfo = helpinfo, +} + +local report = application.report + +require("util-seq") +require("publ-dat") +require("publ-fnd") + +scripts = scripts or { } +scripts.bibtex = scripts.bibtex or { } + +function scripts.bibtex.toxml(files) + local instance = bibtex.new() + local target = "mtx-bibtex-output.xml" + for i=1,#files do + local filename = files[i] + local filetype = file.suffix(filename) + if filetype == "xml" then + target = filename + elseif filetype == "bib" then + bibtex.load { dataset = instance, filename = filename } + else + -- not supported + end + end + bibtex.converttoxml(instance,true) + instance.shortcuts = nil + instance.luadata = nil + xml.save(instance.xmldata,target) +end + +function scripts.bibtex.tolua(files) + local instance = bibtex.new() + local target = "mtx-bibtex-output.lua" + for i=1,#files do + local filename = files[i] + local filetype = file.suffix(filename) + if filetype == "lua" then + target = filename + elseif filetype == "bib" then + bibtex.load { dataset = instance, filename = filename } + + else + -- not supported + end + end + instance.shortcuts = nil + instance.xmldata = nil + bibtex.analyze(instance) + if environment.arguments.simple then + table.save(target,instance) + else + table.save(target,instance.luadata) + end +end + +function scripts.bibtex.search(files,pattern,list) + if pattern then + local dataset = publications.datasets["whatever"] + for i=1,#files do + local filename = resolvers.findfile(files[i]) + if filename and filename ~= "" then + publications.load { dataset = "whatever", filename = filename } + end + end + local found = publications.search(dataset,pattern) + local tags = table.sortedkeys(found) + if #tags == 0 then + report("no match") + elseif list then + report("%s matches:",#tags) + local result = { } + local luadata = dataset.luadata + for i=1,#tags do + local tag = tags[i] + local entry = luadata[tag] + result[i] = { + tag, + entry.year, + entry.author, + entry.title, + } + end + utilities.formatters.formatcolumns(result) + logs.newline() + for i=1,#result do + texio.write_nl(result[i]) + end + logs.newline() + else + report("%s matches: % t",#tags,tags) + end + end +end + +if environment.arguments.search then + scripts.bibtex.search(environment.files,environment.arguments.pattern,environment.arguments.list) +elseif environment.arguments.toxml then + scripts.bibtex.toxml(environment.files) +elseif environment.arguments.tolua then + scripts.bibtex.tolua(environment.files) +elseif environment.arguments.exporthelp then + application.export(environment.arguments.exporthelp,environment.files[1]) +else + application.help() +end + +-- scripts.bibtex.toxml { "tugboat.bib" } +-- scripts.bibtex.tolua { "tugboat.bib" } diff --git a/scripts/context/lua/mtx-check.lua b/scripts/context/lua/mtx-check.lua index 9f52509ec..c456b4414 100644 --- a/scripts/context/lua/mtx-check.lua +++ b/scripts/context/lua/mtx-check.lua @@ -21,7 +21,7 @@ local helpinfo = [[ <flags> <category name="basic"> <subcategory> - <flag name="convert"><short>check tex file for errors</short></flag> + <flag name="check"><short>check tex file for errors</short></flag> </subcategory> </category> </flags> @@ -34,17 +34,17 @@ local application = logs.application { helpinfo = helpinfo, } -local report = application.report +local report = application.report -scripts = scripts or { } -scripts.checker = scripts.checker or { } +scripts = scripts or { } +scripts.checker = scripts.checker or { } -local validator = { } +local validator = { } -validator.n = 1 -validator.errors = { } -validator.trace = false -validator.direct = false +validator.n = 1 +validator.errors = { } +validator.trace = false +validator.direct = false validator.printer = print validator.tracer = print @@ -68,27 +68,24 @@ local progress = function(position, data, kind) end end -local i_m, d_m = P("$"), P("$$") -local l_s, r_s = P("["), P("]") -local l_g, r_g = P("{"), P("}") +local i_m, d_m = P("$"), P("$$") +local l_s, r_s = P("["), P("]") +local l_g, r_g = P("{"), P("}") -local okay = lpeg.P("{[}") + lpeg.P("{]}") +local okay = lpeg.P("{[}") + lpeg.P("{]}") -local esc = P("\\") -local cr = P("\r") -local lf = P("\n") -local crlf = P("\r\n") -local space = S(" \t\f\v") -local newline = crlf + cr + lf +local esc = P("\\") +local space = S(" \t\f\v") +local newline = lpeg.patterns.newline -local line = newline / function() validator.n = validator.n + 1 end +local line = newline / function() validator.n = validator.n + 1 end local startluacode = P("\\startluacode") local stopluacode = P("\\stopluacode") -local somecode = startluacode * (1-stopluacode)^1 * stopluacode +local somecode = startluacode * (1-stopluacode)^1 * stopluacode -local stack = { } +local stack = { } local function push(p,s) -- print("start",p,s) @@ -117,7 +114,7 @@ local contextgrammar = P { "tokens", ["start"] = start, ["stop"] = stop, ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), - ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_g - r_g))^0 * r_g, + ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + line + (1 - l_g - r_g))^0 * r_g, ["setup"] = l_s * (okay + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, ["display"] = d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m, ["inline"] = i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m, diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 4c6672051..d624f6831 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -28,10 +28,13 @@ local fileaddsuffix = file.addsuffix local filenewsuffix = file.replacesuffix local removesuffix = file.removesuffix local validfile = lfs.isfile +local removefile = os.remove +local renamefile = os.rename +local formatters = string.formatters local application = logs.application { name = "mtx-context", - banner = "ConTeXt Process Management 0.60", + banner = "ConTeXt Process Management 0.61", -- helpinfo = helpinfo, -- table with { category_a = text_1, category_b = text_2 } or helpstring or xml_blob helpinfo = "mtx-context.xml", } @@ -87,14 +90,17 @@ scripts.context = scripts.context or { } -- for the moment here -if getargument("jit") or getargument("jiton") then +if jit then -- already luajittex + setargument("engine","luajittex") + setargument("jit",nil) +elseif getargument("jit") or getargument("jiton") then -- relaunch luajittex -- bonus shortcut, we assume than --jit also indicates the engine -- although --jit and --engine=luajittex are independent setargument("engine","luajittex") end -local engine_new = getargument("engine") or directives.value("system.engine") -local engine_old = environment.ownbin +local engine_new = file.nameonly(getargument("engine") or directives.value("system.engine")) +local engine_old = file.nameonly(environment.ownbin) local function restart(engine_old,engine_new) local command = format("%s --luaonly %q %s --redirected",engine_new,environment.ownname,environment.reconstructcommandline()) @@ -253,8 +259,9 @@ end -- multipass control -local multipass_suffixes = { ".tuc" } -local multipass_nofruns = 8 -- or 7 to test oscillation +local multipass_suffixes = { ".tuc" } +local multipass_nofruns = 9 -- better for tracing oscillation +local multipass_forcedruns = false local function multipass_hashfiles(jobname) local hash = { } @@ -275,11 +282,47 @@ local function multipass_changed(oldhash, newhash) return false end -local function multipass_copyluafile(jobname) +local f_tempfile = formatters["%s-%s-%02d.tmp"] + +local function backup(run,kind,filename) + if run == 1 then + for i=1,10 do + local tmpname = f_tempfile(jobname,kind,i) + if validfile(tmpname) then + removefile(tmpname) + report("removing %a",tmpname) + end + end + end + if validfile(filename) then + local tmpname = f_tempfile(jobname,kind,run or 1) + report("copying %a into %a",filename,tmpname) + file.copy(filename,tmpname) + else + report("no file %a, nothing kept",filename) + end +end + +local function multipass_copyluafile(jobname,run) local tuaname, tucname = jobname..".tua", jobname..".tuc" if validfile(tuaname) then - os.remove(tucname) - os.rename(tuaname,tucname) + if run then + backup(run,"tuc",tucname) + report("copying %a into %a",tuaname,tucname) + report() + end + removefile(tucname) + renamefile(tuaname,tucname) + end +end + +local function multipass_copylogfile(jobname,run) + local logname = jobname..".log" + if validfile(logname) then + if run then + backup(run,"log",logname) + report() + end end end @@ -344,8 +387,8 @@ local function result_push_purge(oldbase,newbase) for _, suffix in next, usedsuffixes.after do local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) - os.remove(newname) - os.remove(oldname) + removefile(newname) + removefile(oldname) end end @@ -354,10 +397,10 @@ local function result_push_keep(oldbase,newbase) local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) local tmpname = "keep-"..oldname - os.remove(tmpname) - os.rename(oldname,tmpname) - os.remove(oldname) - os.rename(newname,oldname) + removefile(tmpname) + renamefile(oldname,tmpname) + removefile(oldname) + renamefile(newname,oldname) end end @@ -365,8 +408,8 @@ local function result_save_error(oldbase,newbase) for _, suffix in next, usedsuffixes.keep do local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) - os.remove(newname) -- to be sure - os.rename(oldname,newname) + removefile(newname) -- to be sure + renamefile(oldname,newname) end end @@ -374,8 +417,8 @@ local function result_save_purge(oldbase,newbase) for _, suffix in next, usedsuffixes.after do local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) - os.remove(newname) -- to be sure - os.rename(oldname,newname) + removefile(newname) -- to be sure + renamefile(oldname,newname) end end @@ -384,9 +427,9 @@ local function result_save_keep(oldbase,newbase) local oldname = fileaddsuffix(oldbase,suffix) local newname = fileaddsuffix(newbase,suffix) local tmpname = "keep-"..oldname - os.remove(newname) - os.rename(oldname,newname) - os.rename(tmpname,oldname) + removefile(newname) + renamefile(oldname,newname) + renamefile(tmpname,oldname) end end @@ -536,15 +579,30 @@ function scripts.context.run(ctxdata,filename) local a_profile = getargument("profile") local a_batchmode = getargument("batchmode") local a_nonstopmode = getargument("nonstopmode") + local a_scollmode = getargument("scrollmode") local a_once = getargument("once") local a_synctex = getargument("synctex") local a_backend = getargument("backend") local a_arrange = getargument("arrange") local a_noarrange = getargument("noarrange") local a_jiton = getargument("jiton") + local a_jithash = getargument("jithash") local a_texformat = getargument("texformat") + local a_keeptuc = getargument("keeptuc") + local a_keeplog = getargument("keeplog") + + -- the following flag is not officially supported because i cannot forsee + -- side effects (so no bug reports please) .. we provide --sandbox that + -- does similar things but tries to ensure that context works as expected + + local a_safer = getargument("safer") + + if a_safer then + report("warning: using the luatex safer options, processing is not guaranteed") + end + -- - a_batchmode = (a_batchmode and "batchmode") or (a_nonstopmode and "nonstopmode") or nil + a_batchmode = (a_batchmode and "batchmode") or (a_nonstopmode and "nonstopmode") or (a_scrollmode and "scrollmode") or nil a_synctex = check_synctex(a_synctex) -- for i=1,#filelist do @@ -578,13 +636,30 @@ function scripts.context.run(ctxdata,filename) formatfile, scriptfile = resolvers.locateformat(formatname) end -- - a_jiton = (a_jiton or toboolean(analysis.jiton,true)) and true or nil + a_jiton = (a_jiton or toboolean(analysis.jiton,true)) and true or nil + a_jithash = validstring(a_jithash or analysis.jithash) or nil -- if not formatfile or not scriptfile then report("warning: no format found, forcing remake (source driven)") scripts.context.make(formatname,a_engine) formatfile, scriptfile = resolvers.locateformat(formatname) end + -- + local function combine(key) + local flag = validstring(environment[key]) + local plus = analysis[key] + if flag and plus then + return plus .. "," .. flag -- flag wins + else + return flag or plus -- flag wins + end + end + local a_trackers = analysis.trackers + local a_experiments = analysis.experiments + local directives = combine("directives") + local trackers = combine("trackers") + local experiments = combine("experiments") + -- if formatfile and scriptfile then local suffix = validstring(getargument("suffix")) local resultname = validstring(getargument("result")) @@ -632,9 +707,9 @@ function scripts.context.run(ctxdata,filename) local maxnofruns = once and 1 or multipass_nofruns -- local c_flags = { - directives = validstring(environment.directives), -- gets passed via mtxrun - trackers = validstring(environment.trackers), -- gets passed via mtxrun - experiments = validstring(environment.experiments), -- gets passed via mtxrun + directives = directives, -- gets passed via mtxrun + trackers = trackers, -- gets passed via mtxrun + experiments = experiments, -- gets passed via mtxrun -- result = validstring(resultname), input = validstring(getargument("input") or filename), -- alternative input @@ -655,18 +730,16 @@ function scripts.context.run(ctxdata,filename) ["interaction"] = a_batchmode, ["synctex"] = a_synctex, ["no-parse-first-line"] = true, + ["safer"] = a_safer, -- ["no-mktex"] = true, -- ["file-line-error-style"] = true, ["fmt"] = formatfile, ["lua"] = scriptfile, ["jobname"] = jobname, ["jiton"] = a_jiton, + ["jithash"] = a_jithash, } -- - if a_synctex then - report("warning: synctex is enabled") -- can add upto 5% runtime - end - -- if not a_timing then -- okay elseif c_flags.usemodule then @@ -683,6 +756,15 @@ function scripts.context.run(ctxdata,filename) c_flags.directives = "system.profile" end -- + if a_synctex then + report("warning: synctex is enabled") -- can add upto 5% runtime + if c_flags.directives then + c_flags.directives = format("system.synctex=%s,%s",a_synctex,c_flags.directives) + else + c_flags.directives = format("system.synctex=%s",a_synctex) + end + end + -- -- kindofrun: 1:first run, 2:successive run, 3:once, 4:last of maxruns -- for currentrun=1,maxnofruns do @@ -690,14 +772,16 @@ function scripts.context.run(ctxdata,filename) c_flags.final = false c_flags.kindofrun = (a_once and 3) or (currentrun==1 and 1) or (currentrun==maxnofruns and 4) or 2 c_flags.maxnofruns = maxnofruns + c_flags.forcedruns = multipass_forcedruns and multipass_forcedruns > 0 and multipass_forcedruns or nil c_flags.currentrun = currentrun c_flags.noarrange = a_noarrange or a_arrange or nil -- local command = luatex_command(l_flags,c_flags,mainfile,a_engine) -- - report("run %s: %s",i,command) + report("run %s: %s",currentrun,command) print("") -- cleaner, else continuation on same line local returncode, errorstring = os.spawn(command) + -- todo: remake format when no proper format is found if not returncode then report("fatal error: no return code, message: %s",errorstring or "?") if resultname then @@ -706,11 +790,17 @@ function scripts.context.run(ctxdata,filename) os.exit(1) break elseif returncode == 0 then - multipass_copyluafile(jobname) - newhash = multipass_hashfiles(jobname) - if multipass_changed(oldhash,newhash) then - oldhash = newhash - else + multipass_copyluafile(jobname,a_keeptuc and currentrun) + multipass_copylogfile(jobname,a_keeplog and currentrun) + if not multipass_forcedruns then + newhash = multipass_hashfiles(jobname) + if multipass_changed(oldhash,newhash) then + oldhash = newhash + else + break + end + elseif currentrun == multipass_forcedruns then + report("quitting after force %i runs",multipass_forcedruns) break end else @@ -773,6 +863,24 @@ function scripts.context.run(ctxdata,filename) pdf_open(resultname or jobname,pdfview) end -- + local epub = analysis.epub + if epub then + if type(epub) == "string" then + local t = settings_to_array(epub) + for i=1,#t do + t[i] = "--" .. gsub(t[i],"^%-*","") + end + epub = concat(t," ") + else + epub = "--make" + end + local command = "mtxrun --script epub " .. epub .. " " .. jobname + report() + report("making epub file: ",command) + report() + os.execute(command) + end + -- if a_timing then report() report("you can process (timing) statistics with:",jobname) @@ -838,7 +946,7 @@ function scripts.context.pipe() -- still used? scripts.context.purge_job(filename) elseif getargument("purgeall") then scripts.context.purge_job(filename,true) - os.remove(filename) + removefile(filename) end else if formatname then @@ -1032,11 +1140,11 @@ local special_runfiles = { local function purge_file(dfile,cfile) if cfile and validfile(cfile) then - if os.remove(dfile) then + if removefile(dfile) then return filebasename(dfile) end elseif dfile then - if os.remove(dfile) then + if removefile(dfile) then return filebasename(dfile) end end @@ -1130,8 +1238,8 @@ local function touch(path,name,versionpattern,kind,kindpattern) end if newdata ~= "" and (oldversion ~= newversion or oldkind ~= newkind or newdata ~= olddata) then local backup = filenewsuffix(name,"tmp") - os.remove(backup) - os.rename(name,backup) + removefile(backup) + renamefile(name,backup) io.savedata(name,newdata) return name, oldversion, newversion, oldkind, newkind end @@ -1478,9 +1586,12 @@ do end if getargument("once") then - multipass_nofruns = 1 -elseif getargument("runs") then - multipass_nofruns = tonumber(getargument("runs")) or nil + multipass_nofruns = 1 +else + if getargument("runs") then + multipass_nofruns = tonumber(getargument("runs")) or nil + end + multipass_forcedruns = tonumber(getargument("forcedruns")) or nil end if getargument("run") then diff --git a/scripts/context/lua/mtx-context.xml b/scripts/context/lua/mtx-context.xml index a3812288f..c41093289 100644 --- a/scripts/context/lua/mtx-context.xml +++ b/scripts/context/lua/mtx-context.xml @@ -108,6 +108,14 @@ <flag name="once"> <short>only run once (no multipass data file is produced)</short> </flag> + <flag name="runs"> + <short>process at most this many times</short> + </flag> + <flag name="forcedruns"> + <short>process this many times (permits for optimization trial runs)</short> + </flag> + </subcategory> + <subcategory> <flag name="batchmode"> <short>run without stopping and do not show messages on the console</short> </flag> @@ -117,7 +125,7 @@ <flag name="synctex"> <short>run with synctex enabled (optional value: zipped, unzipped, 1, -1)</short> </flag> - </subcategory> + </subcategory> <subcategory> <flag name="generate"> <short>generate file database etc. (as luatools does)</short> @@ -134,7 +142,7 @@ <short>assume given file present elsewhere</short> </flag> <flag name="nofile"> - <short>use dummy file as jobname</short> + <short>use dummy file as jobname</short> </flag> </subcategory> </category> @@ -144,16 +152,22 @@ <short>update context version number (also provide <ref name="expert"/>, optionally provide <ref name="basepath"/>)</short> </flag> <flag name="nostatistics"> - <short>omit runtime statistics at the end of the run</short> + <short>omit runtime statistics at the end of the run</short> </flag> <flag name="update"> - <short>update context from website (not to be confused with contextgarden)</short> + <short>update context from website (not to be confused with contextgarden)</short> </flag> - <flag name="profile"> - <short>profile job (use: mtxrun <ref name="script"/> profile <ref name="analyze"/>)</short> + <flag name="profile"> + <short>profile job (use: mtxrun <ref name="script"/> profile <ref name="analyze"/>)</short> </flag> - <flag name="timing"> - <short>generate timing and statistics overview</short> + <flag name="timing"> + <short>generate timing and statistics overview</short> + </flag> + <flag name="keeptuc"> + <short>keep previous tuc files (jobname-tuc-[run].tmp)</short> + </flag> + <flag name="keeplog"> + <short>keep previous log files (jobname-log-[run].tmp)</short> </flag> </subcategory> <subcategory> @@ -182,6 +196,11 @@ <short>do not check for file and enter scroll mode (<ref name="dummyfile"/>=whatever.tmp)</short> </flag> </subcategory> + <subcategory> + <flag name="sandbox"> + <short>process file in a limited environment</short> + </flag> + </subcategory> </category> </flags> </application> diff --git a/scripts/context/lua/mtx-convert.lua b/scripts/context/lua/mtx-convert.lua index b76b3baaf..d5dba075a 100644 --- a/scripts/context/lua/mtx-convert.lua +++ b/scripts/context/lua/mtx-convert.lua @@ -48,7 +48,7 @@ local convert = scripts.convert convert.converters = convert.converters or { } local converters = convert.converters -local gsprogram = os.type == "windows" and "gswin32c" or "gs" +local gsprogram = (os.type == "windows" and (os.which("gswin64c.exe") or os.which("gswin32c.exe"))) or "gs" local gstemplate_eps = "%s -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -dEPSCrop -dNOPAUSE -dSAFER -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" local gstemplate_ps = "%s -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -dNOPAUSE -dSAFER -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" diff --git a/scripts/context/lua/mtx-epub.lua b/scripts/context/lua/mtx-epub.lua index 11f0a2024..956ce4931 100644 --- a/scripts/context/lua/mtx-epub.lua +++ b/scripts/context/lua/mtx-epub.lua @@ -18,8 +18,89 @@ if not modules then modules = { } end modules ['mtx-epub'] = { -- first we need a decent strategy to export them. More information will be -- available on the wiki. -local format, gsub = string.format, string.gsub -local concat = table.concat +-- META-INF +-- container.xml +-- OEBPS +-- content.opf +-- toc.ncx +-- images +-- styles +-- mimetype + +-- todo: +-- +-- remove m_k_v_i prefixes +-- remap fonts %mono% in css so that we can replace +-- coverpage tests +-- split up + +-- todo: automated cover page: +-- +-- \startMPpage +-- StartPage ; +-- fill Page withcolor .5red ; +-- numeric n ; +-- for i=10 downto 1 : +-- n := i * PaperWidth/40 ; +-- draw +-- lrcorner Page shifted (0,n) +-- % -- lrcorner Page +-- -- lrcorner Page shifted (-n,0) +-- % -- cycle +-- withpen pencircle scaled 1mm +-- withcolor white ; +-- endfor ; +-- picture p ; p := image ( +-- draw +-- anchored.top( +-- textext.bot("\tttf Some Title") +-- xsized .8PaperWidth +-- ,center topboundary Page +-- ) +-- withcolor white ; +-- ) ; +-- picture q ; q := image ( +-- draw +-- anchored.top( +-- textext.bot("\tttf An Author") +-- xsized .4PaperWidth +-- shifted (0,-PaperHeight/40) +-- ,center bottomboundary p +-- ) +-- withcolor white ; +-- ) ; +-- draw p ; +-- draw q ; +-- StopPage ; +-- \stopMPpage + +local format, gsub, find = string.format, string.gsub, string.find +local concat, sortedhash = table.concat, table.sortedhash + +local formatters = string.formatters +local replacetemplate = utilities.templates.replace + +local addsuffix = file.addsuffix +local nameonly = file.nameonly +local basename = file.basename +local pathpart = file.pathpart +local joinfile = file.join +local suffix = file.suffix +local addsuffix = file.addsuffix +local removesuffix = file.removesuffix +local replacesuffix = file.replacesuffix + +local copyfile = file.copy +local removefile = os.remove + +local needsupdating = file.needsupdating + +local isdir = lfs.isdir +local isfile = lfs.isfile +local mkdir = lfs.mkdir + +local pushdir = dir.push +local popdir = dir.pop local helpinfo = [[ <?xml version="1.0"?> @@ -27,12 +108,17 @@ local helpinfo = [[ <metadata> <entry name="name">mtx-epub</entry> <entry name="detail">ConTeXt EPUB Helpers</entry> - <entry name="version">0.12</entry> + <entry name="version">1.10</entry> </metadata> <flags> <category name="basic"> <subcategory> <flag name="make"><short>create epub zip file</short></flag> + <flag name="purge"><short>remove obsolete files</short></flag> + <flag name="rename"><short>rename images to sane names</short></flag> + <flag name="svgmath"><short>convert mathml to svg</short></flag> + <flag name="svgstyle"><short>use given tex style for svg generation (overloads style in specification)</short></flag> + <flag name="all"><short>assume: --purge --rename --svgmath (for fast testing)</short></flag> </subcategory> </category> </flags> @@ -49,10 +135,12 @@ local helpinfo = [[ local application = logs.application { name = "mtx-epub", - banner = "ConTeXt EPUB Helpers 0.12", + banner = "ConTeXt EPUB Helpers 1.10", helpinfo = helpinfo, } +local report = application.report + -- script code scripts = scripts or { } @@ -60,91 +148,142 @@ scripts.epub = scripts.epub or { } local mimetype = "application/epub+zip" -local container = [[ +local t_container = [[ <?xml version="1.0" encoding="UTF-8"?> <container version="1.0" xmlns="urn:oasis:names:tc:opendocument:xmlns:container"> <rootfiles> - <rootfile full-path="OEBPS/%s" media-type="application/oebps-package+xml"/> + <rootfile full-path="OEBPS/%rootfile%" media-type="application/oebps-package+xml"/> </rootfiles> </container> ]] -local package = [[ +-- urn:uuid: + +-- <dc:identifier id="%identifier%" opf:scheme="UUID">%uuid%</dc:identifier> + +local t_package = [[ <?xml version="1.0" encoding="UTF-8"?> -<package version="2.0" xmlns="http://www.idpf.org/2007/opf" unique-identifier="%s"> +<package xmlns="http://www.idpf.org/2007/opf" unique-identifier="%identifier%" version="3.0"> <metadata xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns:opf="http://www.idpf.org/2007/opf"> - <dc:title>%s</dc:title> - <dc:language>%s</dc:language> - <dc:identifier id="%s" opf:scheme="UUID">urn:uuid:%s</dc:identifier> - <dc:creator>%s</dc:creator> - <dc:date>%s</dc:date> - <meta name="cover" content="%s" /> + <dc:title>%title%</dc:title> + <dc:language>%language%</dc:language> + <dc:identifier id="%identifier%">%uuid%</dc:identifier> + <dc:creator>%creator%</dc:creator> + <dc:date>%date%</dc:date> + <!-- + <dc:subject>%subject%</dc:subject> + <dc:description>%description%</dc:description> + <dc:publisher>%publisher%</dc:publisher> + <dc:source>%source%</dc:source> + <dc:relation>%relation%</dc:relation> + <dc:coverage>%coverage%</dc:coverage> + <dc:rights>%rights%</dc:rights> + --> + <meta name="cover" content="%coverpage%" /> + <meta name="generator" content="ConTeXt MkIV" /> + <meta property="dcterms:modified">%date%</meta> </metadata> <manifest> -%s +%manifest% </manifest> <spine toc="ncx"> <itemref idref="cover-xhtml" /> - <itemref idref="%s" /> + <itemref idref="%rootfile%" /> </spine> </package> ]] -local item = [[ <item id="%s" href="%s" media-type="%s"/>]] -local toc = [[ -<?xml version="1.0"?> +local t_item = [[ <item id="%id%" href="%filename%" media-type="%mime%" />]] +local t_prop = [[ <item id="%id%" href="%filename%" media-type="%mime%" properties="%properties%" />]] + +-- <!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd"> + +local t_toc = [[ +<?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE ncx PUBLIC "-//NISO//DTD ncx 2005-1//EN" "http://www.daisy.org/z3986/2005/ncx-2005-1.dtd"> +<!-- this is no longer needed in epub 3.0+ --> <ncx xmlns="http://www.daisy.org/z3986/2005/ncx/" version="2005-1"> <head> - <meta name="dtb:uid" content="%s" /> + <meta name="generator" content="ConTeXt MkIV" /> + <meta name="dtb:uid" content="%identifier%" /> <meta name="dtb:depth" content="2" /> <meta name="dtb:totalPgeCount" content="0" /> <meta name="dtb:maxPageNumber" content="0" /> </head> <docTitle> - <text>%s</text> + <text>%title%</text> </docTitle> + <docAuthor> + <text>%author%</text> + </docAuthor> + <navMap> <navPoint id="np-1" playOrder="1"> <navLabel> <text>start</text> </navLabel> - <content src="%s"/> + <content src="%root%"/> </navPoint> </navMap> </ncx> ]] -local coverxhtml = [[ +local t_navtoc = [[ <?xml version="1.0" encoding="UTF-8"?> -<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:epub="http://www.idpf.org/2007/ops"> + <head> + <meta charset="utf-8" /> + <title>navtoc</title> + </head> + <body> + <div class="navtoc"> + <!-- <nav epub:type="lot"> --> + <nav epub:type="toc" id="navtoc"> + <ol> + <li><a href="%root%">document</a></li> + </ol> + </nav> + </div> + </body> +</html> +]] + +-- <html xmlns="http://www.w3.org/1999/xhtml"> +-- <!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN" "http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd"> + +local t_coverxhtml = [[ +<?xml version="1.0" encoding="UTF-8"?> <html xmlns="http://www.w3.org/1999/xhtml"> <head> - <title>cover.xhtml</title> + <meta charset="utf-8" /> + <title>cover page</title> </head> <body> - <div> - <img src="%s" alt="The cover image" style="max-width: 100%%;" /> + <div class="coverpage"> + %content% </div> </body> </html> ]] +local t_coverimg = [[ + <img src="%image%" alt="The cover image" style="max-width: 100%%;" /> +]] + -- We need to figure out what is permitted. Numbers only seem to give -- problems is some applications as do names with dashes. Also the -- optional toc is supposed to be there and although id's are by @@ -154,12 +293,13 @@ local coverxhtml = [[ local function dumbid(filename) -- return (string.gsub(os.uuid(),"%-%","")) -- to be tested - return file.nameonly(filename) .. "-" .. file.suffix(filename) + return nameonly(filename) .. "-" .. suffix(filename) end local mimetypes = { xhtml = "application/xhtml+xml", xml = "application/xhtml+xml", + html = "application/html", css = "text/css", svg = "image/svg+xml", png = "image/png", @@ -175,204 +315,601 @@ local idmakers = { default = function(filename) return dumbid(filename) end, } --- specification = { --- name = "document", --- identifier = "123", --- root = "a.xhtml", --- files = { --- "a.xhtml", --- "b.css", --- "c.png", --- } --- } - -local function locateimages(oldname,newname,subpath) +local function relocateimages(imagedata,oldname,newname,subpath,rename) local data = io.loaddata(oldname) - local images = { } - local done = gsub(data,"(background%-image *: * url%()(.-)(%))", function(before,name,after) - if subpath then - name = file.join(subpath,name) + if data then + subpath = joinfile("..",subpath) + report("relocating images") + local n = 0 + local done = gsub(data,[[(id=")(.-)(".-background%-image *: *url%()(.-)(%))]], function(s1,id,s2,name,s3) + local data = imagedata[id] + if data then + local newname = data[id].newname + if newname then + if subpath then + name = joinfile(subpath,basename(newname)) + else + name = basename(newname) + end + -- name = url.addscheme(name) + end + if newname then + n = n + 1 + if rename then + name = joinfile(subpath,addsuffix(id,suffix(name))) + end + return s1 .. id .. s2 .. name .. s3 + end + end + end) + report("%s images relocated in %a",n,newname) + if newname then + io.savedata(newname,done) end - images[#images+1] = name - return before .. name .. after - end) - if newname then - io.savedata(done,newname) end return images end +function reportobsolete(oldfiles,newfiles,purge) + + for i=1,#oldfiles do oldfiles[i] = gsub(oldfiles[i],"^[%./]+","") end + for i=1,#newfiles do newfiles[i] = gsub(newfiles[i],"^[%./]+","") end + + local old = table.tohash(oldfiles) + local new = table.tohash(newfiles) + local done = false + + for name in sortedhash(old) do + if not new[name] then + if not done then + report() + if purge then + report("removing obsolete files:") + else + report("obsolete files:") + end + report() + done = true + end + report(" %s",name) + if purge then + removefile(name) + end + end + end + + if done then + report() + end + + return done + +end + + local zippers = { { name = "zip", + binary = "zip", uncompressed = "zip %s -X -0 %s", compressed = "zip %s -X -9 -r %s", }, { - name = "7zip (7z)", + name = "7z (7zip)", + binary = "7z", uncompressed = "7z a -tzip -mx0 %s %s", compressed = "7z a -tzip %s %s", }, } -function scripts.epub.make() +function scripts.epub.make(purge,rename,svgmath,svgstyle) + + -- one can enter a jobname or jobname-export but the simple jobname is + -- preferred local filename = environment.files[1] - if filename and filename ~= "" and type(filename) == "string" then + if not filename or filename == "" or type(filename) ~= "string" then + report("provide filename") + return + end + + local specpath, specname, specfull - filename = file.basename(filename) - local specfile = file.replacesuffix(filename,"specification") - local specification = lfs.isfile(specfile) and dofile(specfile) or { } + if isdir(filename) then + specpath = filename + specname = addsuffix(specpath,"lua") + specfull = joinfile(specpath,specname) + end + + if not specfull or not isfile(specfull) then + specpath = filename .. "-export" + specname = addsuffix(filename .. "-pub","lua") + specfull = joinfile(specpath,specname) + end + + if not specfull or not isfile(specfull) then + report("unknown specificaton file %a for %a",specfull or "?",filename) + return + end --- inspect(specification) + local specification = dofile(specfull) + + if not specification or not next(specification) then + report("invalid specificaton file %a",specfile) + return + end + + report("using specification file %a",specfull) + + -- images: { ... url = location ... } + + local defaultcoverpage = "cover.xhtml" + + local name = specification.name or nameonly(filename) + local identifier = specification.identifier or "" + local htmlfiles = specification.htmlfiles or { } + local styles = specification.styles or { } + local images = specification.images or { } + local htmlroot = specification.htmlroot or htmlfiles[1] or "" + local language = specification.language or "en" + local creator = specification.creator or "context mkiv" + local author = specification.author or "anonymous" + local title = specification.title or name + local subtitle = specification.subtitle or "" + local imagefile = specification.imagefile or "" + local imagepath = specification.imagepath or "images" + local stylepath = specification.stylepath or "styles" + local coverpage = specification.firstpage or defaultcoverpage + + if type(svgstyle) == "string" and not svgstyle then + svgstyle = specification.svgstyle or "" + end - local name = specification.name or file.removesuffix(filename) - local identifier = specification.identifier or os.uuid(true) - local files = specification.files or { file.addsuffix(filename,"xhtml") } - local images = specification.images or { } - local root = specification.root or files[1] - local language = specification.language or "en" - local creator = specification.author or "My Self" - local title = specification.title or "My Title" - local firstpage = specification.firstpage or "" - local lastpage = specification.lastpage or "" + local obsolete = false - -- identifier = gsub(identifier,"[^a-zA-z0-9]","") + if #htmlfiles == 0 then + report("no html files specified") + return + end + if htmlroot == "" then + report("no html root file specified") + return + end + + if subtitle ~= "" then + title = format("%s, %s",title,subtitle) + end + + local htmlsource = specpath + local imagesource = joinfile(specpath,imagepath) + local stylesource = joinfile(specpath,stylepath) + + -- once we're here we can start moving files to the right spot; first we deal + -- with images + + -- ["image-1"]={ + -- height = "7.056cm", + -- name = "file:///t:/sources/cow.svg", + -- page = "1", + -- width = "9.701cm", + -- } + + -- end of todo + + local pdftosvg = os.which("mudraw") and formatters[ [[mudraw -o "%s" "%s" %s]] ] + + local f_svgpage = formatters["%s-page-%s.svg"] + local f_svgname = formatters["%s.svg"] + + local notupdated = 0 + local updated = 0 + local skipped = 0 + local oldfiles = dir.glob(file.join(imagesource,"*")) + local newfiles = { } + + if not pdftosvg then + report("the %a binary is not present","mudraw") + end + + -- a coverpage file has to be in the root of the export tree + + if not coverpage then + report("no cover page (image) defined") + elseif suffix(coverpage) ~= "xhtml" then + report("using cover page %a",coverpage) + local source = coverpage + local target = joinfile(htmlsource,coverpage) + htmlfiles[#htmlfiles+1 ] = coverpage + report("copying coverpage %a to %a",source,target) + copyfile(source,target) + elseif isfile(coverpage) then + report("using cover page image %a",coverpage) + images.cover = { + height = "100%", + width = "100%", + page = "1", + name = url.filename(coverpage), + used = coverpage, + } + local data = replacetemplate(t_coverxhtml, { + content = replacetemplate(t_coverimg, { + image = coverpage, + }) + }) + coverpage = defaultcoverpage + local target = joinfile(htmlsource,coverpage) + report("saving coverpage to %a",target) + io.savedata(target,data) + htmlfiles[#htmlfiles+1 ] = coverpage + else + report("cover page image %a is not present",coverpage) + coverpage = false + end + + if not coverpage then + local data = replacetemplate(t_coverxhtml, { + content = "no cover page" + }) + coverpage = defaultcoverpage + local target = joinfile(htmlsource,coverpage) + report("saving dummy coverpage to %a",target) + io.savedata(target,data) + htmlfiles[#htmlfiles+1 ] = coverpage + end - if firstpage ~= "" then - images[firstpage] = firstpage + for id, data in sortedhash(images) do + local name = url.filename(data.name) + local used = url.filename(data.used) + local base = basename(used) + local page = tonumber(data.page) or 1 + -- todo : check timestamp and prefix, rename to image-* + if suffix(used) == "pdf" then + -- todo: pass svg name + if page > 1 then + name = f_svgpage(nameonly(name),page) + else + name = f_svgname(nameonly(name)) + end + local source = used + local target = joinfile(imagesource,name) + if needsupdating(source,target) then + if pdftosvg then + local command = pdftosvg(target,source,page) + report("running command %a",command) + os.execute(command) + updated = updated + 1 + else + skipped = skipped + 1 + end + else + notupdated = notupdated + 1 + end + newfiles[#newfiles+1] = target + else + name = basename(used) + local source = used + local target = joinfile(imagesource,name) + if needsupdating(source,target) then + report("copying %a to %a",source,target) + copyfile(source,target) + updated = updated + 1 + else + notupdated = notupdated + 1 + -- no message + end + newfiles[#newfiles+1] = target end - if lastpage ~= "" then - images[lastpage] = lastpage + local target = newfiles[#newfiles] + if suffix(target) == "svg" and isfile(target) then + local data = io.loaddata(target) + if data then + local done = gsub(data,"<!(DOCTYPE.-)>","<!-- %1 -->",1) + if data ~= done then + report("doctype fixed in %a",target) + io.savedata(target,data) + end + end end + data.newname = name -- without path + end + + report("%s images checked, %s updated, %s kept, %s skipped",updated + notupdated + skipped,updated,notupdated,skipped) - identifier = "BookId" -- weird requirement - - local epubname = name - local epubpath = file.replacesuffix(name,"tree") - local epubfile = file.replacesuffix(name,"epub") - local epubroot = file.replacesuffix(name,"opf") - local epubtoc = "toc.ncx" - local epubcover = "cover.xhtml" - - application.report("creating paths in tree %s",epubpath) - lfs.mkdir(epubpath) - lfs.mkdir(file.join(epubpath,"META-INF")) - lfs.mkdir(file.join(epubpath,"OEBPS")) - - local used = { } - - local function copyone(filename) - local suffix = file.suffix(filename) - local mime = mimetypes[suffix] - if mime then - local idmaker = idmakers[suffix] or idmakers.default - local target = file.join(epubpath,"OEBPS",filename) - file.copy(filename,target) - application.report("copying %s to %s",filename,target) - used[#used+1] = format(item,idmaker(filename),filename,mime) + if reportobsolete(oldfiles,newfiles,purge) then + obsolete = true + end + + -- here we can decide not to make an epub + + local uuid = format("urn:uuid:%s",os.uuid(true)) -- os.uuid() + local identifier = "bookid" -- for now + + local epubname = removesuffix(name) + local epubpath = name .. "-epub" + local epubfile = replacesuffix(name,"epub") + local epubroot = replacesuffix(name,"opf") + local epubtoc = "toc.ncx" + local epubmimetypes = "mimetype" + local epubcontainer = "container.xml" + local epubnavigator = "nav.xhtml" + + local metapath = "META-INF" + local datapath = "OEBPS" + + local oldfiles = dir.glob(file.join(epubpath,"**/*")) + local newfiles = { } + + report("creating paths in tree %a",epubpath) + + if not isdir(epubpath) then + mkdir(epubpath) + end + if not isdir(epubpath) then + report("unable to create path %a",epubpath) + return + end + + local metatarget = joinfile(epubpath,metapath) + local htmltarget = joinfile(epubpath,datapath) + local styletarget = joinfile(epubpath,datapath,stylepath) + local imagetarget = joinfile(epubpath,datapath,imagepath) + + mkdir(metatarget) + mkdir(htmltarget) + mkdir(styletarget) + mkdir(imagetarget) + + local used = { } + local notupdated = 0 + local updated = 0 + + local oldimagespecification = joinfile(htmlsource,imagefile) + local newimagespecification = joinfile(htmltarget,imagefile) + + report("removing %a",newimagespecification) + -- removefile(newimagespecification) -- because we update that one + + local function registerone(path,filename,mathml) + local suffix = suffix(filename) + local mime = mimetypes[suffix] + if mime then + local idmaker = idmakers[suffix] or idmakers.default + local fullname = path and joinfile(path,filename) or filename + if mathml then + used[#used+1] = replacetemplate(t_prop, { + id = idmaker(filename), + filename = fullname, + mime = mime, + properties = "mathml", + } ) + else + used[#used+1] = replacetemplate(t_item, { + id = idmaker(filename), + filename = fullname, + mime = mime, + } ) end + return true + end + end + + local function registerandcopyfile(check,path,name,sourcepath,targetpath,newname,image) + + if name == "" then + report("ignoring unknown image") + return + end + + if newname then + newname = replacesuffix(newname,suffix(name)) + else + newname = name end - copyone("cover.xhtml") - copyone("toc.ncx") + local source = joinfile(sourcepath,name) + local target = joinfile(targetpath,newname) + local mathml = false - local function copythem(files) - for i=1,#files do - local filename = files[i] - if type(filename) == "string" then - copyone(filename) + if suffix(source) == "xhtml" then + if find(io.loaddata(source),"MathML") then + mathml = true -- inbelievable: the property is only valid when there is mathml + end + else + report("checking image %a -> %a",source,target) + end + if registerone(path,newname,mathml) then + if not check or needsupdating(source,target) or mathml and svgmath then + report("copying %a to %a",source,target) + copyfile(source,target) + updated = updated + 1 + else + notupdated = notupdated + 1 + end + newfiles[#newfiles+1] = target + if mathml and svgmath then + report() + report("converting mathml into svg in %a",target) + report() + local status, total, unique = moduledata.svgmath.convert(target,svgstyle) + report() + if status then + report("%s formulas converted, %s are unique",total,unique) + else + report("warning: %a in %a",total,target) end + report() end end + end - copythem(files) + -- local nofdummies = 0 + -- local dummyname = formatters["dummy-figure-%03i"] + -- local makedummy = formatters["context --extra=dummies --noconsole --once --result=%s"] + -- + -- local function registerandcopydummy(targetpath,name) + -- nofdummies = nofdummies + 1 + -- local newname = dummyname(nofdummies) + -- local target = joinfile(targetpath,newname) + -- if not isfile(target) then + -- pushdir(targetpath) + -- report("generating dummy %a for %a",newname,name or "unknown") + -- os.execute(makedummy(newname)) + -- popdir() + -- end + -- return newname + -- end + + for image, data in sortedhash(images) do + -- if data.used == "" then + -- data.newname = registerandcopydummy(imagetarget,data.name) + -- end + registerandcopyfile(true,imagepath,data.newname,imagesource,imagetarget,rename and image,true) + end + for i=1,#styles do + registerandcopyfile(false,stylepath,styles[i],stylesource,styletarget) + end + for i=1,#htmlfiles do + registerandcopyfile(false,false,htmlfiles[i],htmlsource,htmltarget) + end - local theimages = { } + relocateimages(images,oldimagespecification,oldimagespecification,imagepath,rename) + relocateimages(images,oldimagespecification,newimagespecification,imagepath,rename) - for k, v in table.sortedpairs(images) do - theimages[#theimages+1] = k - if not lfs.isfile(k) and file.suffix(k) == "svg" and file.suffix(v) == "pdf" then - local command = format("inkscape --export-plain-svg=%s %s",k,v) - application.report("running command '%s'\n\n",command) - os.execute(command) - end + report("%s files registered, %s updated, %s kept",updated + notupdated,updated,notupdated) + + local function saveinfile(what,name,data) + report("saving %s in %a",what,name) + io.savedata(name,data) + newfiles[#newfiles+1] = name + end + + used[#used+1] = replacetemplate(t_prop, { + id = "nav", + filename = epubnavigator, + properties = "nav", + mime = "application/xhtml+xml", + }) + + registerone(false,epubtoc) + + saveinfile("navigation data",joinfile(htmltarget,epubnavigator),replacetemplate(t_navtoc, { -- version 3.0 + root = htmlroot, + } ) ) + + saveinfile("used mimetypes",joinfile(epubpath,epubmimetypes),mimetype) + + saveinfile("version 2.0 container",joinfile(metatarget,epubcontainer),replacetemplate(t_container, { + rootfile = epubroot + } ) ) + + local idmaker = idmakers[suffix(htmlroot)] or idmakers.default + + saveinfile("package specification",joinfile(htmltarget,epubroot),replacetemplate(t_package, { + identifier = identifier, + title = title, + language = language, + uuid = uuid, + creator = creator, + date = os.date("!%Y-%m-%dT%H:%M:%SZ"), + coverpage = idmaker(coverpage), + manifest = concat(used,"\n"), + rootfile = idmaker(htmlroot) + } ) ) + + -- t_toc is replaced by t_navtoc in >= 3 + + saveinfile("table of contents",joinfile(htmltarget,epubtoc), replacetemplate(t_toc, { + identifier = uuid, -- identifier, + title = title, + author = author, + root = htmlroot, + } ) ) + + report("creating archive\n\n") + + pushdir(epubpath) + + removefile(epubfile) + + local usedzipper = false + + local function zipped(zipper) + local ok = os.execute(format(zipper.uncompressed,epubfile,epubmimetypes)) + if ok == 0 then + os.execute(format(zipper.compressed,epubfile,metapath)) + os.execute(format(zipper.compressed,epubfile,datapath)) + usedzipper = zipper.name + return true end + end + + -- nice way + + for i=1,#zippers do + if os.which(zippers[i].binary) and zipped(zippers[i]) then + break + end + end - copythem(theimages) - - local idmaker = idmakers[file.suffix(root)] or idmakers.default - - container = format(container, - epubroot - ) - package = format(package, - identifier, - title, - language, - identifier, - os.uuid(), - creator, - os.date("!%Y-%m-%dT%H:%M:%SZ"), - idmaker(firstpage), - concat(used,"\n"), - idmaker(root) - ) - toc = format(toc, - identifier, - title, - root - ) - coverxhtml = format(coverxhtml, - firstpage - ) - - io.savedata(file.join(epubpath,"mimetype"),mimetype) - io.savedata(file.join(epubpath,"META-INF","container.xml"),container) - io.savedata(file.join(epubpath,"OEBPS",epubroot),package) - io.savedata(file.join(epubpath,"OEBPS",epubtoc),toc) - io.savedata(file.join(epubpath,"OEBPS",epubcover),coverxhtml) - - application.report("creating archive\n\n") - - lfs.chdir(epubpath) - os.remove(epubfile) - - local done = false + -- trial and error + if not usedzipper then for i=1,#zippers do - local zipper = zippers[i] - if os.execute(format(zipper.uncompressed,epubfile,"mimetype")) then - os.execute(format(zipper.compressed,epubfile,"META-INF")) - os.execute(format(zipper.compressed,epubfile,"OEBPS")) - done = zipper.name + if zipped(zippers[i]) then break end end + end - lfs.chdir("..") + popdir() - if done then - application.report("epub archive made using %s: %s",done,file.join(epubpath,epubfile)) - else - local list = { } - for i=1,#zippers do - list[#list+1] = zipper.name - end - application.report("no epub archive made, install one of: %s",concat(list," ")) + if usedzipper then + local treefile = joinfile(epubpath,epubfile) + removefile(epubfile) + copyfile(treefile,epubfile) + if isfile(epubfile) then + removefile(treefile) end + report("epub archive made using %s: %s",usedzipper,epubfile) + else + local list = { } + for i=1,#zippers do + list[#list+1] = zippers[i].name + end + report("no epub archive made, install one of: % | t",list) + end + + if reportobsolete(oldfiles,newfiles,purge) then + obsolete = true + end + if obsolete and not purge then + report("use --purge to remove obsolete files") end end -- -if environment.argument("make") then - scripts.epub.make() -elseif environment.argument("exporthelp") then - application.export(environment.argument("exporthelp"),environment.files[1]) +local a_exporthelp = environment.argument("exporthelp") +local a_make = environment.argument("make") +local a_all = environment.argument("all") +local a_purge = a_all or environment.argument("purge") +local a_rename = a_all or environment.argument("rename") +local a_svgmath = a_all or environment.argument("svgmath") +local a_svgstyle = environment.argument("svgstyle") + +if a_make and a_svgmath then + require("x-math-svg") +end + +if a_make then + scripts.epub.make(a_purge,a_rename,a_svgmath,a_svgstyle) +elseif a_exporthelp then + application.export(a_exporthelp,environment.files[1]) else application.help() end + +-- java -jar d:\epubcheck\epubcheck-3.0.1.jar -v 3.0 -mode xhtml mkiv-publications.tree\mkiv-publications.epub diff --git a/scripts/context/lua/mtx-fcd.lua b/scripts/context/lua/mtx-fcd.lua index 2fcb9a2c7..76087cc37 100644 --- a/scripts/context/lua/mtx-fcd.lua +++ b/scripts/context/lua/mtx-fcd.lua @@ -247,7 +247,7 @@ end local function fcd_find() found = { } - pattern = environment.files[1] or "" + pattern = lower(environment.files[1] or "") if pattern ~= "" then pattern = string.escapedpattern(pattern) local paths = hash.paths @@ -255,7 +255,7 @@ local function fcd_find() local paths = paths[i][2] for i=1,#paths do local path = paths[i] - if find(path,pattern) then + if find(lower(path),pattern) then found[#found+1] = path end end diff --git a/scripts/context/lua/mtx-flac.lua b/scripts/context/lua/mtx-flac.lua index 4e01abc99..116eeda34 100644 --- a/scripts/context/lua/mtx-flac.lua +++ b/scripts/context/lua/mtx-flac.lua @@ -8,7 +8,7 @@ if not modules then modules = { } end modules ['mtx-flac'] = { local sub, match, byte, lower = string.sub, string.match, string.byte, string.lower local readstring, readnumber = io.readstring, io.readnumber -local concat, sortedpairs = table.concat, table.sortedpairs +local concat, sortedpairs, sort, keys = table.concat, table.sortedpairs, table.sort, table.keys local tonumber = tonumber local tobitstring = number.tobitstring local lpegmatch = lpeg.match @@ -105,35 +105,62 @@ function flac.savecollection(pattern,filename) local files = dir.glob(pattern) flac.report("%s files found, analyzing files",#files) local music = { } - table.sort(files) + sort(files) for i=1,#files do - local data = flac.getmetadata(files[i]) + local name = files[i] + local data = flac.getmetadata(name) if data then local tags = data.tags local info = data.info - local artist = tags.artist or "no-artist" - local album = tags.album or "no-album" - local albums = music[artist] - if not albums then - albums = { } - music[artist] = albums - end - local albumx = albums[album] - if not albumx then - albumx = { - year = tags.date, - tracks = { }, + if tags and info then + local artist = tags.artist or "no-artist" + local album = tags.album or "no-album" + local albums = music[artist] + if not albums then + albums = { } + music[artist] = albums + end + local albumx = albums[album] + if not albumx then + albumx = { + year = tags.date, + tracks = { }, + } + albums[album] = albumx + end + albumx.tracks[tonumber(tags.tracknumber) or 0] = { + title = tags.title, + length = math.round((info.samples_in_stream/info.sample_rate_in_hz)), } - albums[album] = albumx + else + flac.report("unable to read file",name) end - albumx.tracks[tonumber(tags.tracknumber) or 0] = { - title = tags.title, - length = math.round((info.samples_in_stream/info.sample_rate_in_hz)), - } end end - -- inspect(music) - local nofartists, nofalbums, noftracks, noferrors = 0, 0, 0, 0 + -- + local nofartists = 0 + local nofalbums = 0 + local noftracks = 0 + local noferrors = 0 + -- + local allalbums + local function compare(a,b) + local ya = allalbums[a].year or 0 + local yb = allalbums[b].year or 0 + if ya == yb then + return a < b + else + return ya < yb + end + end + local function getlist(albums) + allalbums = albums + local list = keys(albums) + sort(list,compare) + return list + end + -- + filename = file.addsuffix(filename,"xml") local f = io.open(filename,"wb") if f then flac.report("saving data in file %q",filename) @@ -144,15 +171,8 @@ function flac.savecollection(pattern,filename) f:write("\t<artist>\n") f:write("\t\t<name>",lpegmatch(p_escaped,artist),"</name>\n") f:write("\t\t<albums>\n") - local list = table.keys(albums) - table.sort(list,function(a,b) - local ya, yb = albums[a].year or 0, albums[b].year or 0 - if ya == yb then - return a < b - else - return ya < yb - end - end) + local list = getlist(albums) + nofalbums = nofalbums + #list for nofalbums=1,#list do local album = list[nofalbums] local data = albums[album] @@ -180,6 +200,40 @@ function flac.savecollection(pattern,filename) f:write("</collection>\n") f:close() flac.report("%s tracks of %s albums of %s artists saved in %q (%s errors)",noftracks,nofalbums,nofartists,filename,noferrors) + -- a secret option for alan braslau + if environment.argument("bibtex") then + filename = file.replacesuffix(filename,"bib") + local f = io.open(filename,"wb") + if f then + local n = 0 + for artist, albums in sortedpairs(music) do + local list = getlist(albums) + for nofalbums=1,#list do + n = n + 1 + local album = list[nofalbums] + local data = albums[album] + local tracks = data.tracks + f:write("@cd{entry-",n,",\n") + f:write("\tartist = {",artist,"},\n") + f:write("\ttitle = {",album or "no title","},\n") + f:write("\tyear = {",data.year or 0,"},\n") + f:write("\ttracks = {",#tracks,"},\n") + for i=1,#tracks do + local track = tracks[i] + if track then + noftracks = noftracks + 1 + f:write("\ttrack:",i," = {",track.title,"},\n") + f:write("\tlength:",i," = {",track.length,"},\n") + end + end + f:write("}\n") + end + end + f:close() + flac.report("additional bibtex file generated: %s",filename) + end + end + -- else flac.report("unable to save data in file %q",filename) end diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua index 4340cb357..694e6a649 100644 --- a/scripts/context/lua/mtx-fonts.lua +++ b/scripts/context/lua/mtx-fonts.lua @@ -38,7 +38,7 @@ local helpinfo = [[ <flag name="filter" value="list"><short>key-value pairs</short></flag> <flag name="all"><short>show all found instances (combined with other flags)</short></flag> <flag name="info"><short>give more details</short></flag> - <flag name="track" value="list"><short>enable trackers</short></flag> + <flag name="trackers" value="list"><short>enable trackers</short></flag> <flag name="statistics"><short>some info about the database</short></flag> </subcategory> </category> @@ -413,8 +413,12 @@ function scripts.fonts.save() local sub = givenfiles[2] or "" local function save(savename,fontblob) if fontblob then + if fontblob.validation_state and table.contains(fontblob.validation_state,"bad_ps_fontname") then + report("ignoring bad fontname for %a",name) + savename = file.nameonly(name) .. "-bad-ps-name" + end savename = file.addsuffix(string.lower(savename),"lua") - report("fontsave, saving data in %s",savename) + report("fontsave, saving data in %a",savename) table.tofile(savename,fontloader.to_table(fontblob),"return") fontloader.close(fontblob) end @@ -426,7 +430,7 @@ function scripts.fonts.save() if suffix == 'ttf' or suffix == 'otf' or suffix == 'ttc' or suffix == "dfont" then local fontinfo = fontloader.info(filename) if fontinfo then - report("font: %s located as %s",name,filename) + report("font: %a located as %a",name,filename) if #fontinfo > 0 then for k=1,#fontinfo do local v = fontinfo[k] @@ -436,13 +440,13 @@ function scripts.fonts.save() save(fontinfo.fullname,fontloader.open(filename)) end else - report("font: %s cannot be read",filename) + report("font: %a cannot be read",filename) end else - report("font: %s not saved",filename) + report("font: %a not saved",filename) end else - report("font: %s not found",name) + report("font: %a not found",name) end else report("font: no name given") diff --git a/scripts/context/lua/mtx-interface.lua b/scripts/context/lua/mtx-interface.lua index 82cefd638..1640f0891 100644 --- a/scripts/context/lua/mtx-interface.lua +++ b/scripts/context/lua/mtx-interface.lua @@ -248,7 +248,7 @@ function scripts.interface.editor(editor,split,forcedinterfaces) local mappings = { } local environments = { } local x = xml.load(keyfile) - for e, d, k in xml.elements(x,"cd:command") do + for e, d, k in xml.elements(x,"/cd:interface/cd:commands/cd:command") do -- somehow this was variable local at = d[k].at local name, value = at.name, at.value if name and value then @@ -256,7 +256,7 @@ function scripts.interface.editor(editor,split,forcedinterfaces) end end local x = xml.load(xmlfile) - for e, d, k in xml.elements(x,"cd:command") do + for e, d, k in xml.elements(x,"/cd:interface/cd:command") do local at = d[k].at local name, type = at.name, at["type"] if name and name ~= "" then @@ -322,7 +322,7 @@ function scripts.interface.check() if f then f:write("\\starttext\n") local x = xml.load(xmlfile) - for e, d, k in xml.elements(x,"cd:command") do + for e, d, k in xml.elements(x,"/cd:interface/cd:command") do local dk = d[k] local at = dk.at if at then @@ -384,6 +384,7 @@ function scripts.interface.interfaces() return a .. b .. c .. b end) end + -- we could just replace attributes for language, _ in next, commands.setuplayout do local texresult, xmlresult = { }, { } texresult[#texresult+1] = format("%% this file is auto-generated, don't edit this file\n%%") @@ -403,6 +404,7 @@ function scripts.interface.interfaces() report("saving interface translations '%s'",xmlfilename) if language ~= "en" and xmldata ~= "" then local newdata = xmldata:gsub("(<cd:interface.*language=.)en(.)","%1"..language.."%2",1) +-- newdata = replace(newdata, 'cd:command', 'name', interface.commands, interface.elements, language) newdata = replace(newdata, 'cd:string', 'value', interface.commands, interface.elements, language) newdata = replace(newdata, 'cd:variable' , 'value', interface.variables, nil, language) newdata = replace(newdata, 'cd:parameter', 'name', interface.constants, nil, language) diff --git a/scripts/context/lua/mtx-metapost.lua b/scripts/context/lua/mtx-metapost.lua index 08daec978..6306125d4 100644 --- a/scripts/context/lua/mtx-metapost.lua +++ b/scripts/context/lua/mtx-metapost.lua @@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['mtx-metapost'] = { -- this was m license = "see context related readme files" } +-- todo: load map files + local helpinfo = [[ <?xml version="1.0"?> <application> @@ -60,24 +62,42 @@ local function assumes_latex(filename) return find(d,"\\documentstyle") or find(d,"\\documentclass") or find(d,"\\begin{document}") end +local basemaps = "original-base.map,original-ams-base.map,original-ams-euler.map,original-public-lm.map" + +local wrapper = "\\starttext\n%s\n%s\\stoptext" +local loadmap = "\\loadmapfile[%s]\n" local template = "\\startTEXpage\n\\convertMPtoPDF{%s}{1}{1}\n\\stopTEXpage" local texified = "\\starttext\n%s\n\\stoptext" local splitter = "\\startTEXpage\\externalfigure[%s][page=%s]\\stopTEXpage" local tempname = "mptopdf-temp.tex" -local function do_convert(filename) +local function do_mapfiles(mapfiles) + local maps = { } + for i=1,#mapfiles do + local mapfile = mapfiles[i] + application.report("using map file %a",mapfile) + maps[i] = format(loadmap,mapfile) + end + return table.concat(maps) +end + +local function do_convert(filename,mapfiles) if find(filename,".%d+$") or find(filename,"%.mps$") then - io.savedata(tempname,format(template,filename)) + local body = format(template,filename) + local maps = do_mapfiles(mapfiles) + io.savedata(tempname,format(wrapper,maps,body)) local resultname = format("%s-%s.pdf",file.nameonly(filename),file.suffix(filename)) local result = os.execute(format([[context --once --batch --purge --result=%s "%s"]],resultname,tempname)) return lfs.isfile(resultname) and resultname end end -local function do_split(filename,numbers) +local function do_split(filename,numbers,mapfiles) local name = file.nameonly(filename) + local maps = do_mapfiles(mapfiles) for i=1,#numbers do - io.savedata(tempname,format(splitter,file.addsuffix(name,"pdf"),i)) + local body = format(splitter,file.addsuffix(name,"pdf"),i) + io.savedata(tempname,format(wrapper,maps,body)) local resultname = format("%s-%s.pdf",name,numbers[i]) local result = os.execute(format([[context --once --batch --purge --result=%s "%s"]],resultname,tempname)) end @@ -99,12 +119,12 @@ local function do_texify(str) return format(texified,str), numbers end -local function do_convert_all(filename) +local function do_convert_all(filename,mapfiles) local results = dir.glob(file.nameonly(filename) .. ".*") -- reset local report = { } for i=1,#results do local filename = results[i] - local resultname = do_convert(filename) + local resultname = do_convert(filename,mapfiles) if resultname then report[#report+1] = { filename, resultname } end @@ -121,8 +141,8 @@ local function do_convert_all(filename) end end -local function do_convert_one(filename) - local resultname = do_convert(filename) +local function do_convert_one(filename,mapfiles) + local resultname = do_convert(filename,mapfiles) if resultname then report("%s => %s", filename,resultname) else @@ -131,17 +151,13 @@ local function do_convert_one(filename) end function scripts.mptopdf.convertall() - local rawmp = environment.arguments.rawmp or false - local metafun = environment.arguments.metafun or false - local latex = environment.arguments.latex or false - local pattern = environment.arguments.pattern or false - local split = environment.arguments.split or false - local files - if pattern then - files = dir.glob(file.nameonly(filename)) - else - files = environment.files - end + local rawmp = environment.arguments.rawmp or false + local metafun = environment.arguments.metafun or false + local latex = environment.arguments.latex or false + local pattern = environment.arguments.pattern or false + local split = environment.arguments.split or false + local files = pattern and dir.glob(file.nameonly(filename)) or environment.files + local mapfiles = utilities.parsers.settings_to_array(environment.arguments.mapfiles or basemaps) if #files > 0 then for i=1,#files do local filename = files[i] @@ -168,16 +184,16 @@ function scripts.mptopdf.convertall() local done = os.execute(command) if done then if convert then - do_convert_all(filename) + do_convert_all(filename,mapfiles) elseif split then - do_split(filename,numbers) + do_split(filename,numbers,mapfiles) -- already pdf, maybe optionally split end else report("error while processing mp file '%s'", filename) end else - do_convert_one(filename) + do_convert_one(filename,mapfiles) end end else diff --git a/scripts/context/lua/mtx-mk-help.lua b/scripts/context/lua/mtx-mk-help.lua index 794bbca37..083dbc3ec 100644 --- a/scripts/context/lua/mtx-mk-help.lua +++ b/scripts/context/lua/mtx-mk-help.lua @@ -403,7 +403,7 @@ local helpinfo = [[ <application> <metadata> <entry name="name">mptopdf</entry> - <entry name="detail">convert MetaPost to PDF</entry> + <entry name="detail">convert MetaPost figures to PDF</entry> <entry name="version">1.4.1</entry> </metadata> <flags> diff --git a/scripts/context/lua/mtx-patterns.lua b/scripts/context/lua/mtx-patterns.lua index 5e2b2d902..b7d41e2b2 100644 --- a/scripts/context/lua/mtx-patterns.lua +++ b/scripts/context/lua/mtx-patterns.lua @@ -29,6 +29,7 @@ local helpinfo = [[ <flag name="specification"><short>additional patterns: e.g.: =cy,hyph-cy,welsh</short></flag> <flag name="compress"><short>compress data</short></flag> <flag name="words"><short>update words in given file</short></flag> + <flag name="hyphenate"><short>show hypephenated words</short></flag> </subcategory> </category> </flags> @@ -40,6 +41,7 @@ local helpinfo = [[ <example><command>mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns</command></example> <example><command>mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns</command></example> <example><command>mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns</command></example> + <example><command>mtxrun --script pattern --hyphenate --language=nl --left=3 nogalwiedes inderdaad</command></example> </subcategory> </category> </examples> @@ -625,6 +627,49 @@ function scripts.patterns.words() end end +-- mtxrun --script patterns --hyphenate --language=nl nogalwiedes --left=3 +-- +-- hyphenator | +-- hyphenator | . n o g a l w i e d e s . . n o g a l w i e d e s . +-- hyphenator | .0n4 0 4 0 0 0 0 0 0 0 0 0 0 +-- hyphenator | 0o0g0a4l0 0 4 0 0 4 0 0 0 0 0 0 0 +-- hyphenator | 1g0a0 0 4 1 0 4 0 0 0 0 0 0 0 +-- hyphenator | 0l1w0 0 4 1 0 4 1 0 0 0 0 0 0 +-- hyphenator | 4i0e0 0 4 1 0 4 1 4 0 0 0 0 0 +-- hyphenator | 0i0e3d0e0 0 4 1 0 4 1 4 0 3 0 0 0 +-- hyphenator | 0e1d0 0 4 1 0 4 1 4 0 3 0 0 0 +-- hyphenator | 1d0e0 0 4 1 0 4 1 4 0 3 0 0 0 +-- hyphenator | 0d0e2s0 0 4 1 0 4 1 4 0 3 0 2 0 +-- hyphenator | 4s0. 0 4 1 0 4 1 4 0 3 0 4 0 +-- hyphenator | .0n4o1g0a4l1w4i0e3d0e4s0. . n o-g a l-w i e-d e s . +-- hyphenator | +-- mtx-patterns | nl 3 3 : nogalwiedes : nogal-wie-des + +function scripts.patterns.hyphenate() + require("lang-hyp") + local traditional = languages.hyphenators.traditional + local left = tonumber(environment.arguments.left) or 3 + local right = tonumber(environment.arguments.right) or 3 + local language = environment.arguments.language or "us" + local dictionary = traditional.loadpatterns(language) + local words = environment.files + local specification = { + leftcharmin = left, + rightcharmin = right, + leftchar = false, + rightchar = false, + } + trackers.enable("hyphenator.steps") + for i=1,#words do + local word = words[i] + report("%s %s %s : %s : %s", + language, left, right, + word, + traditional.injecthyphens(dictionary,word,specification) + ) + end +end + if environment.argument("check") then scripts.patterns.prepare() scripts.patterns.check() @@ -633,16 +678,21 @@ elseif environment.argument("convert") then scripts.patterns.convert() elseif environment.argument("words") then scripts.patterns.words() -- for the moment here +elseif environment.argument("hyphenate") then + scripts.patterns.hyphenate() -- for the moment here elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) else application.help() end --- mtxrun --script pattern --check hyph-*.tex --- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns +-- mtxrun --script pattern --check hyph-*.tex +-- mtxrun --script pattern --check --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/tex --destination=e:/tmp/patterns +-- +-- use this call: +-- +-- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns -- copy /Y *.hyp e:\tex-context\tex\texmf-context\tex\context\patterns -- copy /Y *.pat e:\tex-context\tex\texmf-context\tex\context\patterns diff --git a/scripts/context/lua/mtx-plain.lua b/scripts/context/lua/mtx-plain.lua index d10c21375..1076572fc 100644 --- a/scripts/context/lua/mtx-plain.lua +++ b/scripts/context/lua/mtx-plain.lua @@ -103,7 +103,11 @@ function scripts.plain.make(texengine,texformat) end function scripts.plain.run(texengine,texformat,filename) - execute('%s --fmt=%s "%s"',texengine,file.removesuffix(texformat),filename) + local t = { } + for k, v in next, environment.arguments do + t[#t+1] = string.format("--mtx:%s=%s",k,v) + end + execute('%s --fmt=%s %s "%s"',texengine,file.removesuffix(texformat),table.concat(t," "),filename) end function scripts.plain.fonts() @@ -114,7 +118,7 @@ local texformat = environment.arguments.texformat or environment.arguments.forma local texengine = environment.arguments.texengine or environment.arguments.engine if type(texengine) ~= "string" or texengine == "" then - texengine = environment.arguments.jit and "luajittex" or"luatex" + texengine = (jit or environment.arguments.jit) and "luajittex" or "luatex" end if type(texformat) ~= "string" or texformat == "" then diff --git a/scripts/context/lua/mtx-scite.lua b/scripts/context/lua/mtx-scite.lua index 972edbfe6..ae8c67387 100644 --- a/scripts/context/lua/mtx-scite.lua +++ b/scripts/context/lua/mtx-scite.lua @@ -6,6 +6,8 @@ if not modules then modules = { } end modules ['mtx-scite'] = { license = "see context related readme files" } +-- mtxrun --script scite --tree --source=t:/texmf/tex/context --target=e:/tmp/context --numbers + local P, R, S, C, Ct, Cf, Cc, Cg = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Ct, lpeg.Cf, lpeg.Cc, lpeg.Cg local lpegmatch = lpeg.match local format, lower, gmatch = string.format, string.lower, string.gmatch @@ -22,6 +24,8 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="words"><short>convert spell-*.txt into spell-*.lua</short></flag> + <flag name="tree"><short>converts a tree into an html tree (--source --target --numbers)</short></flag> + <flag name="file"><short>converts a file into an html tree (--source --target --numbers --lexer)</short></flag> </subcategory> </category> </flags> @@ -36,6 +40,8 @@ local application = logs.application { local report = application.report +local scite = require("util-sci") + scripts = scripts or { } scripts.scite = scripts.scite or { } @@ -241,6 +247,51 @@ function scripts.scite.words() report("you need to move the lua files to lexers/data") end +function scripts.scite.tree() + local source = environment.argument("source") + local target = environment.argument("target") + local numbers = environment.argument("numbers") + if not lfs.isdir(source) then + report("you need to pass a valid source path with --source") + return + end + if not lfs.isdir(target) then + report("you need to pass a valid target path with --target") + return + end + if source == target then + report("source and target paths must be different") + return + end + scite.converttree(source,target,numbers) +end + +function scripts.scite.file() + local source = environment.argument("source") + local target = environment.argument("target") + local lexer = environment.argument("lexer") + local numbers = environment.argument("numbers") + if source then + local target = target or file.replacesuffix(source,"html") + if source == target then + report("the source file cannot be the same as the target") + else + scite.filetohtml(source,lexer,target,numbers) + end + + else + for i=1,#environment.files do + local source = environment.files[i] + local target = file.replacesuffix(source,"html") + if source == target then + report("the source file cannot be the same as the target") + else + scite.filetohtml(source,nil,target,numbers) + end + end + end +end + -- if environment.argument("start") then -- scripts.scite.start(true) -- elseif environment.argument("test") then @@ -251,6 +302,10 @@ end if environment.argument("words") then scripts.scite.words() +elseif environment.argument("tree") then + scripts.scite.tree() +elseif environment.argument("file") then + scripts.scite.file() elseif environment.argument("exporthelp") then application.export(environment.argument("exporthelp"),environment.files[1]) else diff --git a/scripts/context/lua/mtx-server.lua b/scripts/context/lua/mtx-server.lua index 5466bfe80..dba07f1d5 100644 --- a/scripts/context/lua/mtx-server.lua +++ b/scripts/context/lua/mtx-server.lua @@ -278,6 +278,20 @@ handlers.html = handlers.htm local indices = { "index.htm", "index.html" } local portnumber = 31415 -- pi suits tex +local newline = lpeg.patterns.newline +local spacer = lpeg.patterns.spacer +local whitespace = lpeg.patterns.whitespace +local method = lpeg.P("GET") + + lpeg.P("POST") +local identify = (1-method)^0 + * lpeg.C(method) + * spacer^1 + * lpeg.C((1-spacer)^1) + * spacer^1 + * lpeg.P("HTTP/") + * (1-whitespace)^0 + * lpeg.C(lpeg.P(1)^0) + function scripts.webserver.run(configuration) -- check configuration configuration.port = tonumber(configuration.port or os.getenv("MTX_SERVER_PORT") or portnumber) or portnumber @@ -329,17 +343,24 @@ function scripts.webserver.run(configuration) local from = client:getpeername() report("request from: %s",tostring(from)) report("request data: %s",tostring(request)) - local fullurl = string.match(request,"GET (.+) HTTP/.*$") or "" -- todo: more clever / post - if fullurl == "" then + -- local fullurl = string.match(request,"(GET) (.+) HTTP/.*$") or "" -- todo: more clever / post + -- if fullurl == "" then +-- print("!!!!",request) + local method, fullurl, body = lpeg.match(identify,request) + if method == "" or fullurl == "" then report("no url") errormessage(client,configuration,404) else + + -- todo: method: POST + fullurl = url.unescapeget(fullurl) report("requested url: %s",fullurl) -- fullurl = socket.url.unescape(fullurl) -- happens later local hashed = url.hashed(fullurl) local query = url.query(hashed.query) local filename = hashed.path -- hm, not query? + hashed.body = body if script then filename = script report("forced script: %s",filename) diff --git a/scripts/context/lua/mtx-update.lua b/scripts/context/lua/mtx-update.lua index c7eb74395..daf4f5b16 100644 --- a/scripts/context/lua/mtx-update.lua +++ b/scripts/context/lua/mtx-update.lua @@ -11,13 +11,15 @@ if not modules then modules = { } end modules ['mtx-update'] = { -- Together with Arthur Reutenauer she made sure that it worked well on all -- platforms that matter. +-- LuaTeX and LuajitTeX are now always installed together. + local helpinfo = [[ <?xml version="1.0"?> <application> <metadata> <entry name="name">mtx-update</entry> <entry name="detail">ConTeXt Minimals Updater</entry> - <entry name="version">1.01</entry> + <entry name="version">1.02</entry> </metadata> <flags> <category name="basic"> @@ -48,7 +50,7 @@ local helpinfo = [[ local application = logs.application { name = "mtx-update", - banner = "ConTeXt Minimals Updater 1.01", + banner = "ConTeXt Minimals Updater 1.02", helpinfo = helpinfo, } @@ -124,7 +126,7 @@ scripts.update.engines = { ["luatex"] = { { "fonts/new/", "texmf" }, { "bin/luatex/<platform>/", "texmf-<platform>" }, - { "bin/luajittex/<platform>/","texmf-<platform>" }, + -- { "bin/luajittex/<platform>/","texmf-<platform>" }, }, ["xetex"] = { { "base/xetex/", "texmf" }, @@ -142,7 +144,7 @@ scripts.update.engines = { { "fonts/old/", "texmf" }, { "base/xetex/", "texmf" }, { "bin/luatex/<platform>/", "texmf-<platform>" }, - { "bin/luajittex/<platform>/","texmf-<platform>" }, + -- { "bin/luajittex/<platform>/","texmf-<platform>" }, { "bin/xetex/<platform>/", "texmf-<platform>" }, { "bin/pdftex/<platform>/", "texmf-<platform>" }, }, @@ -181,6 +183,8 @@ scripts.update.platforms = { ["linux-64"] = "linux-64", ["linux64"] = "linux-64", -- + ["linux-armhf"] = "linux-armhf", + -- ["freebsd"] = "freebsd", -- ["freebsd-amd64"] = "freebsd-amd64", @@ -561,9 +565,8 @@ function scripts.update.make() local formatlist = concat(table.fromhash(texformats), " ") if formatlist ~= "" then for engine in table.sortedhash(engines) do - if engine == "luatex" then + if engine == "luatex" or engine == "luajittex" then scripts.update.run(format('mtxrun --tree="%s" --script context --autogenerate --make',texroot)) - elseif engine == "luajittex" then scripts.update.run(format('mtxrun --tree="%s" --script context --autogenerate --make --engine=luajittex',texroot)) else scripts.update.run(format('mtxrun --tree="%s" --script texexec --make --all --%s %s',texroot,engine,formatlist)) diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 0ff2d2897..edfeba8dd 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3123, stripped down to: 1694 +-- original size: 3888, stripped down to: 2197 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -136,6 +136,16 @@ function optionalrequire(...) return result end end +if lua then + lua.mask=load([[τεχ = 1]]) and "utf" or "ascii" +end +local flush=io.flush +if flush then + local execute=os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec=os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end +end end -- of closure @@ -434,7 +444,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 29245, stripped down to: 15964 +-- original size: 36977, stripped down to: 20349 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -450,7 +460,9 @@ local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.forma local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print -setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +if setinspector then + setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns local anything=P(1) @@ -469,7 +481,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=crlf+S("\r\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -491,8 +503,10 @@ patterns.utfbom_32_le=utfbom_32_le patterns.utfbom_16_be=utfbom_16_be patterns.utfbom_16_le=utfbom_16_le patterns.utfbom_8=utfbom_8 -patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") -patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") +patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_32_be_nl=P("\000\000\000\r\000\000\000\n")+P("\000\000\000\r")+P("\000\000\000\n") +patterns.utf_32_le_nl=P("\r\000\000\000\n\000\000\000")+P("\r\000\000\000")+P("\n\000\000\000") patterns.utf8one=R("\000\127") patterns.utf8two=R("\194\223")*utf8next patterns.utf8three=R("\224\239")*utf8next*utf8next @@ -519,10 +533,24 @@ patterns.spacer=spacer patterns.whitespace=whitespace patterns.nonspacer=nonspacer patterns.nonwhitespace=nonwhitespace -local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) +local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) +local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) +local b_stripper=Cs(spacer^0/""*(nonspacer^1+spacer^1/" ")^0) +local e_stripper=Cs((spacer^1*P(-1)/""+nonspacer^1+spacer^1/" ")^0) +local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper +patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.b_collapser=b_collapser +patterns.m_collapser=m_collapser +patterns.e_collapser=e_collapser +patterns.b_stripper=b_stripper +patterns.m_stripper=m_stripper +patterns.e_stripper=e_stripper patterns.lowercase=lowercase patterns.uppercase=uppercase patterns.letter=patterns.lowercase+patterns.uppercase @@ -559,9 +587,12 @@ patterns.integer=sign^-1*digit^1 patterns.unsigned=digit^0*period*digit^1 patterns.float=sign^-1*patterns.unsigned patterns.cunsigned=digit^0*comma*digit^1 +patterns.cpunsigned=digit^0*(period+comma)*digit^1 patterns.cfloat=sign^-1*patterns.cunsigned +patterns.cpfloat=sign^-1*patterns.cpunsigned patterns.number=patterns.float+patterns.integer patterns.cnumber=patterns.cfloat+patterns.integer +patterns.cpnumber=patterns.cpfloat+patterns.integer patterns.oct=zero*octdigit^1 patterns.octal=patterns.oct patterns.HEX=zero*P("X")*(digit+uppercase)^1 @@ -744,7 +775,7 @@ function lpeg.replacer(one,two,makefunction,isutf) return pattern end end -function lpeg.finder(lst,makefunction) +function lpeg.finder(lst,makefunction,isutf) local pattern if type(lst)=="table" then pattern=P(false) @@ -760,7 +791,11 @@ function lpeg.finder(lst,makefunction) else pattern=P(lst) end - pattern=(1-pattern)^0*pattern + if isutf then + pattern=((utf8char or 1)-pattern)^0*pattern + else + pattern=(1-pattern)^0*pattern + end if makefunction then return function(str) return lpegmatch(pattern,str) @@ -974,37 +1009,139 @@ function lpeg.append(list,pp,delayed,checked) end return p end +local p_false=P(false) +local p_true=P(true) local function make(t) - local p + local function making(t) + local p=p_false + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then + else + p=p+P(k)*making(v) + end + end + end + if t[""] then + p=p+p_true + end + return p + end + local p=p_false local keys=sortedkeys(t) for i=1,#keys do local k=keys[i] - local v=t[k] - if not p then - if next(v) then - p=P(k)*make(v) + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then else - p=P(k) + p=p+P(k)*making(v) end - else - if next(v) then - p=p+P(k)*make(v) + end + end + return p +end +local function collapse(t,x) + if type(t)~="table" then + return t,x + else + local n=next(t) + if n==nil then + return t,x + elseif next(t,n)==nil then + local k=n + local v=t[k] + if type(v)=="table" then + return collapse(v,x..k) else - p=p+P(k) + return v,x..k + end + else + local tt={} + for k,v in next,t do + local vv,kk=collapse(v,k) + tt[kk]=vv end + return tt,x end end - return p end function lpeg.utfchartabletopattern(list) local tree={} - for i=1,#list do - local t=tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c]={} + local n=#list + if n==0 then + for s in next,list do + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true + end + end + else + for i=1,n do + local s=list[i] + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true end - t=t[c] end end return make(tree) @@ -1044,6 +1181,65 @@ local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") local number=digit^1*(case_1+case_2) local stripper=Cs((number+1)^0) lpeg.patterns.stripzeros=stripper +local byte_to_HEX={} +local byte_to_hex={} +local byte_to_dec={} +local hex_to_byte={} +for i=0,255 do + local H=format("%02X",i) + local h=format("%02x",i) + local d=format("%03i",i) + local c=char(i) + byte_to_HEX[c]=H + byte_to_hex[c]=h + byte_to_dec[c]=d + hex_to_byte[h]=c + hex_to_byte[H]=c +end +local hextobyte=P(2)/hex_to_byte +local bytetoHEX=P(1)/byte_to_HEX +local bytetohex=P(1)/byte_to_hex +local bytetodec=P(1)/byte_to_dec +local hextobytes=Cs(hextobyte^0) +local bytestoHEX=Cs(bytetoHEX^0) +local bytestohex=Cs(bytetohex^0) +local bytestodec=Cs(bytetodec^0) +patterns.hextobyte=hextobyte +patterns.bytetoHEX=bytetoHEX +patterns.bytetohex=bytetohex +patterns.bytetodec=bytetodec +patterns.hextobytes=hextobytes +patterns.bytestoHEX=bytestoHEX +patterns.bytestohex=bytestohex +patterns.bytestodec=bytestodec +function string.toHEX(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestoHEX,s) + end +end +function string.tohex(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestohex,s) + end +end +function string.todec(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestodec,s) + end +end +function string.tobytes(s) + if not s or s=="" then + return s + else + return lpegmatch(hextobytes,s) + end +end end -- of closure @@ -1071,7 +1267,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5547, stripped down to: 2708 +-- original size: 5694, stripped down to: 2827 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1107,11 +1303,15 @@ function string.limit(str,n,sentinel) end end local stripper=patterns.stripper +local fullstripper=patterns.fullstripper local collapser=patterns.collapser local longtostring=patterns.longtostring function string.strip(str) return lpegmatch(stripper,str) or "" end +function string.fullstrip(str) + return lpegmatch(fullstripper,str) or "" +end function string.collapsespaces(str) return lpegmatch(collapser,str) or "" end @@ -1172,7 +1372,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 31113, stripped down to: 20256 +-- original size: 35724, stripped down to: 21525 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1205,7 +1405,7 @@ end function table.keys(t) if t then local keys,k={},0 - for key,_ in next,t do + for key in next,t do k=k+1 keys[k]=key end @@ -1215,32 +1415,52 @@ function table.keys(t) end end local function compare(a,b) - local ta,tb=type(a),type(b) - if ta==tb then - return a<b - else - return tostring(a)<tostring(b) + local ta=type(a) + if ta=="number" then + local tb=type(b) + if ta==tb then + return a<b + elseif tb=="string" then + return tostring(a)<b + end + elseif ta=="string" then + local tb=type(b) + if ta==tb then + return a<b + else + return a<tostring(b) + end end + return tostring(a)<tostring(b) end local function sortedkeys(tab) if tab then local srt,category,s={},0,0 - for key,_ in next,tab do + for key in next,tab do s=s+1 srt[s]=key if category==3 then + elseif category==1 then + if type(key)~="string" then + category=3 + end + elseif category==2 then + if type(key)~="number" then + category=3 + end else local tkey=type(key) if tkey=="string" then - category=(category==2 and 3) or 1 + category=1 elseif tkey=="number" then - category=(category==1 and 3) or 2 + category=2 else category=3 end end end - if category==0 or category==3 then + if s<2 then + elseif category==3 then sort(srt,compare) else sort(srt) @@ -1250,16 +1470,52 @@ local function sortedkeys(tab) return {} end end +local function sortedhashonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="string" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end +local function sortedindexonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="number" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end local function sortedhashkeys(tab,cmp) if tab then local srt,s={},0 - for key,_ in next,tab do + for key in next,tab do if key then s=s+1 srt[s]=key end end - sort(srt,cmp) + if s>1 then + sort(srt,cmp) + end return srt else return {} @@ -1268,13 +1524,15 @@ end function table.allkeys(t) local keys={} for k,v in next,t do - for k,v in next,v do + for k in next,v do keys[k]=true end end return sortedkeys(keys) end table.sortedkeys=sortedkeys +table.sortedhashonly=sortedhashonly +table.sortedindexonly=sortedindexonly table.sortedhashkeys=sortedhashkeys local function nothing() end local function sortedhash(t,cmp) @@ -1285,19 +1543,21 @@ local function sortedhash(t,cmp) else s=sortedkeys(t) end - local n=0 local m=#s - local function kv(s) - if n<m then - n=n+1 - local k=s[n] - return k,t[k] + if m==1 then + return next,t + elseif m>0 then + local n=0 + return function() + if n<m then + n=n+1 + local k=s[n] + return k,t[k] + end end end - return kv,s - else - return nothing end + return nothing end table.sortedhash=sortedhash table.sortedpairs=sortedhash @@ -1439,39 +1699,36 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,reduce,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } local function simple_table(t) - if #t>0 then + local nt=#t + if nt>0 then local n=0 for _,v in next,t do n=n+1 end - if n==#t then - local tt,nt={},0 - for i=1,#t do + if n==nt then + local tt={} + for i=1,nt do local v=t[i] local tv=type(v) if tv=="number" then - nt=nt+1 if hexify then - tt[nt]=format("0x%04X",v) + tt[i]=format("0x%X",v) else - tt[nt]=tostring(v) + tt[i]=tostring(v) end elseif tv=="string" then - nt=nt+1 - tt[nt]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then - nt=nt+1 - tt[nt]=v and "true" or "false" + tt[i]=v and "true" or "false" else - tt=nil - break + return nil end end return tt @@ -1490,7 +1747,7 @@ local function do_serialize(root,name,depth,level,indexed) local tn=type(name) if tn=="number" then if hexify then - handle(format("%s[0x%04X]={",depth,name)) + handle(format("%s[0x%X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end @@ -1507,7 +1764,7 @@ local function do_serialize(root,name,depth,level,indexed) end end end - if root and next(root) then + if root and next(root)~=nil then local first,last=nil,0 if compact then last=#root @@ -1525,22 +1782,19 @@ local function do_serialize(root,name,depth,level,indexed) for i=1,#sk do local k=sk[i] local v=root[k] - local tv,tk=type(v),type(k) + local tv=type(v) + local tk=type(k) if compact and first and tk=="number" and k>=first and k<=last then if tv=="number" then if hexify then - handle(format("%s 0x%04X,",depth,v)) + handle(format("%s 0x%X,",depth,v)) else handle(format("%s %s,",depth,v)) end elseif tv=="string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end + handle(format("%s %q,",depth,v)) elseif tv=="table" then - if not next(v) then + if next(v)==nil then handle(format("%s {},",depth)) elseif inline then local st=simple_table(v) @@ -1570,64 +1824,48 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="number" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + handle(format("%s [0x%X]=0x%X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then - handle(format("%s [%s]=0x%04X,",depth,k and "true" or "false",v)) + handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) + handle(format("%s %s=0x%X,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) + handle(format("%s [%q]=0x%X,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end end elseif tv=="string" then - if reduce and tonumber(v) then - if tk=="number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk=="boolean" then - handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,v)) + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=%q,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) else - if tk=="number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk=="boolean" then - handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end + handle(format("%s [%q]=%q,",depth,k,v)) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={},",depth,k)) + handle(format("%s [0x%X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -1643,7 +1881,7 @@ local function do_serialize(root,name,depth,level,indexed) if st then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -1663,7 +1901,7 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="boolean" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v and "true" or "false")) + handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -1679,7 +1917,7 @@ local function do_serialize(root,name,depth,level,indexed) local f=getinfo(v).what=="C" and dump(dummy) or dump(v) if tk=="number" then if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) + handle(format("%s [0x%X]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -1694,7 +1932,7 @@ local function do_serialize(root,name,depth,level,indexed) else if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -1718,7 +1956,6 @@ local function serialize(_handle,root,name,specification) noquotes=specification.noquotes hexify=specification.hexify handle=_handle or specification.handle or print - reduce=specification.reduce or false functions=specification.functions compact=specification.compact inline=specification.inline and compact @@ -1735,7 +1972,6 @@ local function serialize(_handle,root,name,specification) noquotes=false hexify=false handle=_handle or print - reduce=false compact=true inline=true functions=true @@ -1748,7 +1984,7 @@ local function serialize(_handle,root,name,specification) end elseif tname=="number" then if hexify then - handle(format("[0x%04X]={",name)) + handle(format("[0x%X]={",name)) else handle("["..name.."]={") end @@ -1766,7 +2002,7 @@ local function serialize(_handle,root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,"",0) end end @@ -1895,14 +2131,25 @@ local function identical(a,b) end table.identical=identical table.are_equal=are_equal -function table.compact(t) - if t then - for k,v in next,t do - if not next(v) then - t[k]=nil +local function sparse(old,nest,keeptables) + local new={} + for k,v in next,old do + if not (v=="" or v==false) then + if nest and type(v)=="table" then + v=sparse(v,nest) + if keeptables or next(v)~=nil then + new[k]=v + end + else + new[k]=v end end end + return new +end +table.sparse=sparse +function table.compact(t) + return sparse(t,true,true) end function table.contains(t,v) if t then @@ -2000,15 +2247,17 @@ function table.print(t,...) serialize(print,t,...) end end -setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +if setinspector then + setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +end function table.sub(t,i,j) return { unpack(t,i,j) } end function table.is_empty(t) - return not t or not next(t) + return not t or next(t)==nil end function table.has_one_entry(t) - return t and not next(t,next(t)) + return t and next(t,next(t))==nil end function table.loweredkeys(t) local l={} @@ -2053,6 +2302,44 @@ function table.values(t,s) return {} end end +function table.filtered(t,pattern,sort,cmp) + if t and type(pattern)=="string" then + if sort then + local s + if cmp then + s=sortedhashkeys(t,function(a,b) return cmp(t,a,b) end) + else + s=sortedkeys(t) + end + local n=0 + local m=#s + local function kv(s) + while n<m do + n=n+1 + local k=s[n] + if find(k,pattern) then + return k,t[k] + end + end + end + return kv,s + else + local n=next(t) + local function iterator() + while n~=nil do + local k=n + n=next(t,k) + if find(k,pattern) then + return k,t[k] + end + end + end + return iterator,t + end + else + return nothing + end +end end -- of closure @@ -2061,7 +2348,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8817, stripped down to: 6340 +-- original size: 8643, stripped down to: 6232 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2075,7 +2362,7 @@ local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor local type=type -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -2368,8 +2655,6 @@ function io.readstring(f,n,m) local str=gsub(f:read(n),"\000","") return str end -if not io.i_limiter then function io.i_limiter() end end -if not io.o_limiter then function io.o_limiter() end end end -- of closure @@ -2596,7 +2881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16023, stripped down to: 9634 +-- original size: 15832, stripped down to: 9456 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2670,13 +2955,10 @@ if not os.__getenv__ then setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) end end -local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end +local execute=os.execute +local iopopen=io.popen function os.resultof(command) - local handle=io.popen(command,"r") + local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" handle:close() @@ -2686,7 +2968,7 @@ function os.resultof(command) end end if not io.fileseparator then - if find(os.getenv("PATH"),";") then + if find(os.getenv("PATH"),";",1,true) then io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" @@ -2705,7 +2987,7 @@ local launchers={ unix="$BROWSER %s &> /dev/null &", } function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) + execute(format(launchers[os.name] or launchers.unix,str)) end if not os.times then function os.times() @@ -2746,7 +3028,7 @@ if platform~="" then elseif os.type=="windows" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then + if find(architecture,"AMD64",1,true) then platform="win64" else platform="mswin" @@ -2758,9 +3040,9 @@ elseif os.type=="windows" then elseif name=="linux" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then + if find(architecture,"x86_64",1,true) then platform="linux-64" - elseif find(architecture,"ppc") then + elseif find(architecture,"ppc",1,true) then platform="linux-ppc" else platform="linux" @@ -2774,9 +3056,9 @@ elseif name=="macosx" then local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" - elseif find(architecture,"i386") then + elseif find(architecture,"i386",1,true) then platform="osx-intel" - elseif find(architecture,"x86_64") then + elseif find(architecture,"x86_64",1,true) then platform="osx-64" else platform="osx-ppc" @@ -2788,7 +3070,7 @@ elseif name=="macosx" then elseif name=="sunos" then function resolvers.platform(t,k) local platform,architecture="",os.resultof("uname -m") or "" - if find(architecture,"sparc") then + if find(architecture,"sparc",1,true) then platform="solaris-sparc" else platform="solaris-intel" @@ -2800,7 +3082,7 @@ elseif name=="sunos" then elseif name=="freebsd" then function resolvers.platform(t,k) local platform,architecture="",os.resultof("uname -m") or "" - if find(architecture,"amd64") then + if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else platform="freebsd" @@ -2812,7 +3094,7 @@ elseif name=="freebsd" then elseif name=="kfreebsd" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then + if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else platform="kfreebsd-i386" @@ -2829,8 +3111,9 @@ else return platform end end +os.newline=name=="windows" and "\013\010" or "\010" function resolvers.bits(t,k) - local bits=find(os.platform,"64") and 64 or 32 + local bits=find(os.platform,"64",1,true) and 64 or 32 os.bits=bits return bits end @@ -2980,7 +3263,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 18308, stripped down to: 9948 +-- original size: 20949, stripped down to: 9945 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -2994,41 +3277,28 @@ local file=file if not lfs then lfs=optionalrequire("lfs") end -if not lfs then - lfs={ - getcurrentdir=function() - return "." - end, - attributes=function() - return nil - end, - isfile=function(name) - local f=io.open(name,'rb') - if f then - f:close() - return true - end - end, - isdir=function(name) - print("you need to load lfs") - return false - end - } -elseif not lfs.isfile then - local attributes=lfs.attributes - function lfs.isdir(name) - return attributes(name,"mode")=="directory" - end - function lfs.isfile(name) - return attributes(name,"mode")=="file" - end -end local insert,concat=table.insert,table.concat local match,find,gmatch=string.match,string.find,string.gmatch local lpegmatch=lpeg.match local getcurrentdir,attributes=lfs.currentdir,lfs.attributes local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local tricky=S("/\\")*P(-1) +local attributes=lfs.attributes +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir,"lfs.isdir") +end +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end +end +function lfs.isfile(name) + return attributes(name,"mode")=="file" +end local colon=P(":") local period=P(".") local periods=P("..") @@ -3230,28 +3500,30 @@ local isroot=fwslash^1*-1 local hasroot=fwslash^1 local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") -function file.join(...) - local lst={... } - local one=lst[1] +function file.join(one,two,three,...) + if not two then + return one=="" and one or lpegmatch(stripper,one) + end + if one=="" then + return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) + end if lpegmatch(isnetwork,one) then local one=lpegmatch(reslasher,one) - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return one..two else return one.."/"..two end elseif lpegmatch(isroot,one) then - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return two else return "/"..two end - elseif one=="" then - return lpegmatch(stripper,concat(lst,"/",2)) else - return lpegmatch(deslasher,concat(lst,"/")) + return lpegmatch(deslasher,concat({ one,two,three,... },"/")) end end local drivespec=R("az","AZ")^1*colon @@ -3425,7 +3697,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3760, stripped down to: 2088 +-- original size: 3248, stripped down to: 2266 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3443,14 +3715,20 @@ if not md5 then } end local md5,file=md5,file -local gsub,format,byte=string.gsub,string.format,string.byte -local md5sum=md5.sum -local function convert(str,fmt) - return (gsub(md5sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +local gsub=string.gsub +do + local patterns=lpeg and lpeg.patterns + if patterns then + local bytestoHEX=patterns.bytestoHEX + local bytestohex=patterns.bytestohex + local bytestodec=patterns.bytestodec + local lpegmatch=lpeg.match + local md5sum=md5.sum + if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end + if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end + if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + end +end function file.needsupdating(oldname,newname,threshold) local oldtime=lfs.attributes(oldname,"modification") if oldtime then @@ -3507,7 +3785,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 11993, stripped down to: 5584 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -3534,7 +3812,7 @@ local hexdigit=R("09","AF","af") local plus=P("+") local nothing=Cc("") local escapedchar=(percent*C(hexdigit*hexdigit))/tochar -local escaped=(plus/" ")+escapedchar +local escaped=(plus/" ")+escapedchar local noslash=P("/")/"" local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) @@ -3593,19 +3871,25 @@ local splitquery=Cf (Ct("")*P { "sequence", pair=Cg(key*equal*value), },rawset) local function hashed(str) - if str=="" then + if not str or str=="" then return { scheme="invalid", original=str, } end - local s=split(str) - local rawscheme=s[1] - local rawquery=s[4] - local somescheme=rawscheme~="" - local somequery=rawquery~="" + local detailed=split(str) + local rawscheme="" + local rawquery="" + local somescheme=false + local somequery=false + if detailed then + rawscheme=detailed[1] + rawquery=detailed[4] + somescheme=rawscheme~="" + somequery=rawquery~="" + end if not somescheme and not somequery then - s={ + return { scheme="file", authority="", path=str, @@ -3615,28 +3899,28 @@ local function hashed(str) noscheme=true, filename=str, } - else - local authority,path,filename=s[2],s[3] - if authority=="" then - filename=path - elseif path=="" then - filename="" - else - filename=authority.."/"..path - end - s={ - scheme=rawscheme, - authority=authority, - path=path, - query=lpegmatch(unescaper,rawquery), - queries=lpegmatch(splitquery,rawquery), - fragment=s[5], - original=str, - noscheme=false, - filename=filename, - } end - return s + local authority=detailed[2] + local path=detailed[3] + local filename=nil + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + return { + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=detailed[5], + original=str, + noscheme=false, + filename=filename, + } end url.split=split url.hasscheme=hasscheme @@ -3670,7 +3954,7 @@ function url.construct(hash) end return lpegmatch(escaper,concat(fullurl)) end -local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +local pattern=Cs(slash^-1/""*R("az","AZ")*((S(":|")/":")+P(":"))*slash*P(1)^0) function url.filename(filename) local spec=hashed(filename) local path=spec.path @@ -3718,7 +4002,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 14229, stripped down to: 8740 +-- original size: 16765, stripped down to: 11003 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -3728,7 +4012,7 @@ if not modules then modules={} end modules ['l-dir']={ license="see context related readme files" } local type,select=type,select -local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local find,gmatch,match,gsub,sub=string.find,string.gmatch,string.match,string.gsub,string.sub local concat,insert,remove,unpack=table.concat,table.insert,table.remove,table.unpack local lpegmatch=lpeg.match local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V @@ -3737,53 +4021,127 @@ local dir=dir local lfs=lfs local attributes=lfs.attributes local walkdir=lfs.dir -local isdir=lfs.isdir -local isfile=lfs.isfile +local isdir=lfs.isdir +local isfile=lfs.isfile local currentdir=lfs.currentdir local chdir=lfs.chdir -local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") -if not isdir then - function isdir(name) - local a=attributes(name) - return a and a.mode=="directory" +local mkdir=lfs.mkdir +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";",1,true) +if onwindows then + local tricky=S("/\\")*P(-1) + isdir=function(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end + end + isfile=function(name) + return attributes(name,"mode")=="file" end lfs.isdir=isdir -end -if not isfile then - function isfile(name) - local a=attributes(name) - return a and a.mode=="file" + lfs.isfile=isfile +else + isdir=function(name) + return attributes(name,"mode")=="directory" end + isfile=function(name) + return attributes(name,"mode")=="file" + end + lfs.isdir=isdir lfs.isfile=isfile end function dir.current() return (gsub(currentdir(),"\\","/")) end -local lfsisdir=isdir -local function isdir(path) - path=gsub(path,"[/\\]+$","") - return lfsisdir(path) +local function glob_pattern_function(path,patt,recurse,action) + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + action(full) + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end + end + end + end + if dirs then + for i=1,#dirs do + glob_pattern_function(dirs[i],patt,recurse,action) + end + end + end end -lfs.isdir=isdir -local function globpattern(path,patt,recurse,action) - if path=="/" then - path=path.."." - elseif not find(path,"/$") then - path=path..'/' - end - if isdir(path) then - for name in walkdir(path) do - local full=path..name - local mode=attributes(full,'mode') - if mode=='file' then - if find(full,patt) then - action(full) +local function glob_pattern_table(path,patt,recurse,result) + if not result then + result={} + end + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + result[#result+1]=full + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - globpattern(full,patt,recurse,action) + end + end + if dirs then + for i=1,#dirs do + glob_pattern_table(dirs[i],patt,recurse,result) end end end + return result +end +local function globpattern(path,patt,recurse,method) + local kind=type(method) + if patt and sub(patt,1,-3)==path then + patt=false + end + if kind=="function" then + return glob_pattern_function(path,patt,recurse,method) + elseif kind=="table" then + return glob_pattern_table(path,patt,recurse,method) + else + return glob_pattern_table(path,patt,recurse,{}) + end end dir.globpattern=globpattern local function collectpattern(path,patt,recurse,result) @@ -3795,34 +4153,40 @@ local function collectpattern(path,patt,recurse,result) ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) end if ok and type(scanner)=="function" then - if not find(path,"/$") then path=path..'/' end + if not find(path,"/$") then + path=path..'/' + end for name in scanner,first do - local full=path..name - local attr=attributes(full) - local mode=attr.mode - if mode=='file' then - if find(full,patt) then + if name=="." then + elseif name==".." then + else + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and mode=="directory" then + attr.list=collectpattern(full,patt,recurse) result[name]=attr end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - attr.list=collectpattern(full,patt,recurse) - result[name]=attr end end end return result end dir.collectpattern=collectpattern -local separator -if onwindows then +local separator,pattern +if onwindows then local slash=S("/\\")/"/" - pattern=Ct { + pattern={ [1]=(Cs(P(".")+slash^1)+Cs(R("az","AZ")*P(":")*slash^0)+Cc("./"))*V(2)*V(3), [2]=Cs(((1-S("*?/\\"))^0*slash)^0), [3]=Cs(P(1)^0) } -else - pattern=Ct { +else + pattern={ [1]=(C(P(".")+P("/")^1)+Cc("./"))*V(2)*V(3), [2]=C(((1-S("*?/"))^0*P("/"))^0), [3]=C(P(1)^0) @@ -3840,10 +4204,9 @@ local function glob(str,t) elseif isfile(str) then t(str) else - local split=lpegmatch(pattern,str) - if split then - local root,path,base=split[1],split[2],split[3] - local recurse=find(base,"%*%*") + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) globpattern(start,result,recurse,t) @@ -3864,16 +4227,12 @@ local function glob(str,t) return { str } end else - local split=lpegmatch(pattern,str) - if split then - local t=t or {} - local action=action or function(name) t[#t+1]=name end - local root,path,base=split[1],split[2],split[3] - local recurse=find(base,"%*%*") + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) - globpattern(start,result,recurse,action) - return t + return globpattern(start,result,recurse,t) else return {} end @@ -3913,16 +4272,26 @@ end local make_indeed=true if onwindows then function dir.mkdirs(...) - local str,pth="","" - for i=1,select("#",...) do - local s=select(i,...) - if s=="" then - elseif str=="" then - str=s - else - str=str.."/"..s + local n=select("#",...) + local str + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end end end + local pth="" local drive=false local first,middle,last=match(str,"^(//)(//*)(.*)$") if first then @@ -3957,21 +4326,30 @@ if onwindows then pth=pth.."/"..s end if make_indeed and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end return pth,(isdir(pth)==true) end else function dir.mkdirs(...) - local str,pth="","" - for i=1,select("#",...) do - local s=select(i,...) - if s and s~="" then - if str~="" then - str=str.."/"..s - else - str=s + local n=select("#",...) + local str,pth + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s + else + str=s + end end end end @@ -3986,7 +4364,7 @@ else pth=pth.."/"..s end if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end else @@ -3994,7 +4372,7 @@ else for s in gmatch(str,"[^/]+") do pth=pth.."/"..s if make_indeed and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end end @@ -4002,47 +4380,51 @@ else end end dir.makedirs=dir.mkdirs -if onwindows then - function dir.expandname(str) - local first,nothing,last=match(str,"^(//)(//*)(.*)$") - if first then - first=dir.current().."/" - end - if not first then - first,last=match(str,"^(//)/*(.*)$") - end - if not first then - first,last=match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d=currentdir() - if chdir(first) then - first=dir.current() +do + local chdir=sandbox and sandbox.original(chdir) or chdir + if onwindows then + local xcurrentdir=dir.current + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=xcurrentdir().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if chdir(first) then + first=xcurrentdir() + end + chdir(d) end - chdir(d) + end + if not first then + first,last=xcurrentdir(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first + else + return first.."/"..last end end - if not first then - first,last=dir.current(),str - end - last=gsub(last,"//","/") - last=gsub(last,"/%./","/") - last=gsub(last,"^/*","") - first=gsub(first,"/*$","") - if last=="" or last=="." then - return first - else - return first.."/"..last - end - end -else - function dir.expandname(str) - if not find(str,"^/") then - str=currentdir().."/"..str + else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str + end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str end - str=gsub(str,"//","/") - str=gsub(str,"/%./","/") - str=gsub(str,"(.)/%.$","%1") - return str end end file.expandname=dir.expandname @@ -4085,7 +4467,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1809, stripped down to: 1527 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4139,11 +4521,11 @@ function string.booleanstring(str) return str=="yes" or str=="on" or str=="t" end end -function string.is_boolean(str,default) +function string.is_boolean(str,default,strict) if type(str)=="string" then - if str=="true" or str=="yes" or str=="on" or str=="t" or str=="1" then + if str=="true" or str=="yes" or str=="on" or str=="t" or (not strict and str=="1") then return true - elseif str=="false" or str=="no" or str=="off" or str=="f" or str=="0" then + elseif str=="false" or str=="no" or str=="off" or str=="f" or (not strict and str=="0") then return false end end @@ -4157,7 +4539,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 33473, stripped down to: 14938 +-- original size: 37388, stripped down to: 15817 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4173,7 +4555,9 @@ local type=type local char,byte,format,sub,gmatch=string.char,string.byte,string.format,string.sub,string.gmatch local concat=table.concat local P,C,R,Cs,Ct,Cmt,Cc,Carg,Cp=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg,lpeg.Cp -local lpegmatch,patterns=lpeg.match,lpeg.patterns +local lpegmatch=lpeg.match +local patterns=lpeg.patterns +local tabletopattern=lpeg.utfchartabletopattern local bytepairs=string.bytepairs local finder=lpeg.finder local replacer=lpeg.replacer @@ -4182,7 +4566,7 @@ local utfgmatch=utf.gmatch local p_utftype=patterns.utftype local p_utfstricttype=patterns.utfstricttype local p_utfoffset=patterns.utfoffset -local p_utf8char=patterns.utf8char +local p_utf8char=patterns.utf8character local p_utf8byte=patterns.utf8byte local p_utfbom=patterns.utfbom local p_newline=patterns.newline @@ -4321,6 +4705,7 @@ if not utf.sub then local pattern_zero=Cmt(p_utf8char,slide_zero)^0 local pattern_one=Cmt(p_utf8char,slide_one )^0 local pattern_two=Cmt(p_utf8char,slide_two )^0 + local pattern_first=C(patterns.utf8character) function utf.sub(str,start,stop) if not start then return str @@ -4362,7 +4747,9 @@ if not utf.sub then end end end - if start>stop then + if start==1 and stop==1 then + return lpegmatch(pattern_first,str) or "" + elseif start>stop then return "" elseif start>1 then b,e,n,first,last=0,0,0,start-1,stop @@ -4381,15 +4768,52 @@ if not utf.sub then end end end -function utf.remapper(mapping) - local pattern=Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str=="" then - return "" +function utf.remapper(mapping,option) + local variant=type(mapping) + if variant=="table" then + if option=="dynamic" then + local pattern=false + table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) + return function(str) + if not str or str=="" then + return "" + else + if not pattern then + pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + end + return lpegmatch(pattern,str) + end + end + elseif option=="pattern" then + return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) else - return lpegmatch(pattern,str) + local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + elseif variant=="function" then + if option=="pattern" then + return Cs((p_utf8char/mapping+p_utf8char)^0) + else + local pattern=Cs((p_utf8char/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern end - end,pattern + else + return function(str) + return str or "" + end + end end function utf.replacer(t) local r=replacer(t,false,false,true) @@ -4439,190 +4863,157 @@ function utf.magic(f) end local utf16_to_utf8_be,utf16_to_utf8_le local utf32_to_utf8_be,utf32_to_utf8_le -local utf_16_be_linesplitter=patterns.utfbom_16_be^-1*lpeg.tsplitat(patterns.utf_16_be_nl) -local utf_16_le_linesplitter=patterns.utfbom_16_le^-1*lpeg.tsplitat(patterns.utf_16_le_nl) -if bytepairs then - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*left+right - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +local utf_16_be_getbom=patterns.utfbom_16_be^-1 +local utf_16_le_getbom=patterns.utfbom_16_le^-1 +local utf_32_be_getbom=patterns.utfbom_32_be^-1 +local utf_32_le_getbom=patterns.utfbom_32_le^-1 +local utf_16_be_linesplitter=utf_16_be_getbom*lpeg.tsplitat(patterns.utf_16_be_nl) +local utf_16_le_linesplitter=utf_16_le_getbom*lpeg.tsplitat(patterns.utf_16_le_nl) +local utf_32_be_linesplitter=utf_32_be_getbom*lpeg.tsplitat(patterns.utf_32_be_nl) +local utf_32_le_linesplitter=utf_32_le_getbom*lpeg.tsplitat(patterns.utf_32_le_nl) +local more=0 +local p_utf16_to_utf8_be=C(1)*C(1)/function(left,right) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf16_to_utf8_le=C(1)*C(1)/function(right,left) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf32_to_utf8_be=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(a)+256*256*byte(b)+256*byte(c)+byte(d)) +end +local p_utf32_to_utf8_le=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(d)+256*256*byte(c)+256*byte(b)+byte(a)) +end +p_utf16_to_utf8_be=P(true)/function() more=0 end*utf_16_be_getbom*Cs(p_utf16_to_utf8_be^0) +p_utf16_to_utf8_le=P(true)/function() more=0 end*utf_16_le_getbom*Cs(p_utf16_to_utf8_le^0) +p_utf32_to_utf8_be=P(true)/function() more=0 end*utf_32_be_getbom*Cs(p_utf32_to_utf8_be^0) +p_utf32_to_utf8_le=P(true)/function() more=0 end*utf_32_le_getbom*Cs(p_utf32_to_utf8_le^0) +patterns.utf16_to_utf8_be=p_utf16_to_utf8_be +patterns.utf16_to_utf8_le=p_utf16_to_utf8_le +patterns.utf32_to_utf8_be=p_utf32_to_utf8_be +patterns.utf32_to_utf8_le=p_utf32_to_utf8_le +utf16_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_be,s) + else + return s end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*right+left - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +end +local utf16_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_be_linesplitter,t) end - utf32_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*256*256*a+256*256*b - else - r=r+1 - result[t]=utfchar(more+256*a+b) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_be,s) end - return t end - utf32_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*b+a - else - r=r+1 - result[t]=utfchar(more+256*256*256*b+256*256*a) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) - end - return t + return t +end +utf16_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_le,s) + else + return s end -else - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if left=="\000" then - r=r+1 - result[r]=utfchar(byte(right)) - elseif right then - local now=256*byte(left)+byte(right) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) +end +local utf16_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_le,s) end - return t end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) + return t +end +utf32_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_be,s) + else + return s + end +end +local utf32_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_be_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_be,s) end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if right=="\000" then - r=r+1 - result[r]=utfchar(byte(left)) - elseif right then - local now=256*byte(right)+byte(left) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) + end + return t +end +utf32_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_le,s) + else + return s + end +end +local utf32_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_le,s) end - return t end - utf32_to_utf8_le=function() return {} end - utf32_to_utf8_be=function() return {} end + return t end +utf.utf16_to_utf8_le_t=utf16_to_utf8_le_t +utf.utf16_to_utf8_be_t=utf16_to_utf8_be_t +utf.utf32_to_utf8_le_t=utf32_to_utf8_le_t +utf.utf32_to_utf8_be_t=utf32_to_utf8_be_t utf.utf16_to_utf8_le=utf16_to_utf8_le utf.utf16_to_utf8_be=utf16_to_utf8_be utf.utf32_to_utf8_le=utf32_to_utf8_le utf.utf32_to_utf8_be=utf32_to_utf8_be -function utf.utf8_to_utf8(t) +function utf.utf8_to_utf8_t(t) return type(t)=="string" and lpegmatch(utflinesplitter,t) or t end -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +function utf.utf16_to_utf8_t(t,endian) + return endian and utf16_to_utf8_be_t(t) or utf16_to_utf8_le_t(t) or t end -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +function utf.utf32_to_utf8_t(t,endian) + return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(c) - local b=byte(c) +local function little(b) if b<0x10000 then return char(b%256,b/256) else @@ -4631,8 +5022,7 @@ local function little(c) return char(b1%256,b1/256,b2%256,b2/256) end end -local function big(c) - local b=byte(c) +local function big(b) if b<0x10000 then return char(b/256,b%256) else @@ -4641,27 +5031,29 @@ local function big(c) return char(b1/256,b1%256,b2/256,b2%256) end end -local _,l_remap=utf.remapper(little) -local _,b_remap=utf.remapper(big) -function utf.utf8_to_utf16_be(str,nobom) +local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) +local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) +local function utf8_to_utf16_be(str,nobom) if nobom then return lpegmatch(b_remap,str) else return char(254,255)..lpegmatch(b_remap,str) end end -function utf.utf8_to_utf16_le(str,nobom) +local function utf8_to_utf16_le(str,nobom) if nobom then return lpegmatch(l_remap,str) else return char(255,254)..lpegmatch(l_remap,str) end end +utf.utf8_to_utf16_be=utf8_to_utf16_be +utf.utf8_to_utf16_le=utf8_to_utf16_le function utf.utf8_to_utf16(str,littleendian,nobom) if littleendian then - return utf.utf8_to_utf16_le(str,nobom) + return utf8_to_utf16_le(str,nobom) else - return utf.utf8_to_utf16_be(str,nobom) + return utf8_to_utf16_be(str,nobom) end end local pattern=Cs ( @@ -4677,16 +5069,16 @@ function utf.xstring(s) return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end function utf.toeight(str) - if not str then + if not str or str=="" then return nil end local utftype=lpegmatch(p_utfstricttype,str) if utftype=="utf-8" then - return sub(str,4) - elseif utftype=="utf-16-le" then - return utf16_to_utf8_le(str) + return sub(str,4) elseif utftype=="utf-16-be" then - return utf16_to_utf8_ne(str) + return utf16_to_utf8_be(str) + elseif utftype=="utf-16-le" then + return utf16_to_utf8_le(str) else return str end @@ -4765,7 +5157,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 915, stripped down to: 836 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -4775,6 +5167,9 @@ if not modules then modules={} end modules ['l-math']={ license="see context related readme files" } local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.ceiling then + math.ceiling=math.ceil +end if not math.round then function math.round(x) return floor(x+0.5) end end @@ -4802,7 +5197,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 26857, stripped down to: 15062 +-- original size: 34503, stripped down to: 18933 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -4821,25 +5216,43 @@ local unpack,concat=table.unpack,table.concat local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc local patterns,lpegmatch=lpeg.patterns,lpeg.match local utfchar,utfbyte=utf.char,utf.byte -local loadstripped=_LUAVERSION<5.2 and load or function(str) - return load(dump(load(str),true)) +local loadstripped=nil +if _LUAVERSION<5.2 then + loadstripped=function(str,shortcuts) + return load(str) + end +else + loadstripped=function(str,shortcuts) + if shortcuts then + return load(dump(load(str),true),nil,nil,shortcuts) + else + return load(dump(load(str),true)) + end + end end if not number then number={} end local stripper=patterns.stripzeros +local newline=patterns.newline +local endofstring=patterns.endofstring +local whitespace=patterns.whitespace +local spacer=patterns.spacer +local spaceortab=patterns.spaceortab local function points(n) + n=tonumber(n) return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) end local function basepoints(n) + n=tonumber(n) return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) end number.points=points number.basepoints=basepoints -local rubish=patterns.spaceortab^0*patterns.newline -local anyrubish=patterns.spaceortab+patterns.newline +local rubish=spaceortab^0*newline +local anyrubish=spaceortab+newline local anything=patterns.anything -local stripped=(patterns.spaceortab^1/"")*patterns.newline +local stripped=(spaceortab^1/"")*newline local leading=rubish^0/"" -local trailing=(anyrubish^1*patterns.endofstring)/"" +local trailing=(anyrubish^1*endofstring)/"" local redundant=rubish^3/"\n" local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) function strings.collapsecrlf(str) @@ -4885,18 +5298,44 @@ local pattern=Carg(1)/function(t) else return "" end - end+patterns.newline*Cp()/function(position) + end+newline*Cp()/function(position) extra,start=0,position end+patterns.anything )^1) function strings.tabtospace(str,tab) return lpegmatch(pattern,str,1,tab or 7) end -function strings.striplong(str) - str=gsub(str,"^%s*","") - str=gsub(str,"[\n\r]+ *","\n") - return str +local space=spacer^0 +local nospace=space/"" +local endofline=nospace*newline +local stripend=(whitespace^1*endofstring)/"" +local normalline=(nospace*((1-space*(newline+endofstring))^1)*nospace) +local stripempty=endofline^1/"" +local normalempty=endofline^1 +local singleempty=endofline*(endofline^0/"") +local doubleempty=endofline*endofline^-1*(endofline^0/"") +local stripstart=stripempty^0 +local p_prune_normal=Cs (stripstart*(stripend+normalline+normalempty )^0 ) +local p_prune_collapse=Cs (stripstart*(stripend+normalline+doubleempty )^0 ) +local p_prune_noempty=Cs (stripstart*(stripend+normalline+singleempty )^0 ) +local p_retain_normal=Cs ((normalline+normalempty )^0 ) +local p_retain_collapse=Cs ((normalline+doubleempty )^0 ) +local p_retain_noempty=Cs ((normalline+singleempty )^0 ) +local striplinepatterns={ + ["prune"]=p_prune_normal, + ["prune and collapse"]=p_prune_collapse, + ["prune and no empty"]=p_prune_noempty, + ["retain"]=p_retain_normal, + ["retain and collapse"]=p_retain_collapse, + ["retain and no empty"]=p_retain_noempty, + ["collapse"]=patterns.collapser, +} +setmetatable(striplinepatterns,{ __index=function(t,k) return p_prune_collapse end }) +strings.striplinepatterns=striplinepatterns +function strings.striplines(str,how) + return str and lpegmatch(striplinepatterns[how],str) or str end +strings.striplong=strings.striplines function strings.nice(str) str=gsub(str,"[:%-+_]+"," ") return str @@ -4934,10 +5373,10 @@ string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) if type(b)=="number" then - return tracedchars[b] or (utfchar(b).." (U+"..format('%05X',b)..")") + return tracedchars[b] or (utfchar(b).." (U+"..format("%05X",b)..")") else local c=utfbyte(b) - return tracedchars[c] or (b.." (U+"..format('%05X',c)..")") + return tracedchars[c] or (b.." (U+"..(c and format("%05X",c) or "?????")..")") end end function number.signed(i) @@ -4972,31 +5411,58 @@ function number.sparseexponent(f,n) end return tostring(n) end -local preamble=[[ -local type = type -local tostring = tostring -local tonumber = tonumber -local format = string.format -local concat = table.concat -local signed = number.signed -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -local lpegmatch = lpeg.match -local nspaces = string.nspaces -local tracedchar = string.tracedchar -local autosingle = string.autosingle -local autodouble = string.autodouble -local sequenced = table.sequenced -local formattednumber = number.formatted -local sparseexponent = number.sparseexponent -]] local template=[[ %s %s return function(%s) return %s end ]] +local preamble,environment="",{} +if _LUAVERSION<5.2 then + preamble=[[ +local lpeg=lpeg +local type=type +local tostring=tostring +local tonumber=tonumber +local format=string.format +local concat=table.concat +local signed=number.signed +local points=number.points +local basepoints= number.basepoints +local utfchar=utf.char +local utfbyte=utf.byte +local lpegmatch=lpeg.match +local nspaces=string.nspaces +local tracedchar=string.tracedchar +local autosingle=string.autosingle +local autodouble=string.autodouble +local sequenced=table.sequenced +local formattednumber=number.formatted +local sparseexponent=number.sparseexponent + ]] +else + environment={ + global=global or _G, + lpeg=lpeg, + type=type, + tostring=tostring, + tonumber=tonumber, + format=string.format, + concat=table.concat, + signed=number.signed, + points=number.points, + basepoints=number.basepoints, + utfchar=utf.char, + utfbyte=utf.byte, + lpegmatch=lpeg.match, + nspaces=string.nspaces, + tracedchar=string.tracedchar, + autosingle=string.autosingle, + autodouble=string.autodouble, + sequenced=table.sequenced, + formattednumber=number.formatted, + sparseexponent=number.sparseexponent, + } +end local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) local v=t[k-1]..",a"..k @@ -5035,7 +5501,7 @@ local format_i=function(f) if f and f~="" then return format("format('%%%si',a%s)",f,n) else - return format("format('%%i',a%s)",n) + return format("format('%%i',a%s)",n) end end local format_d=format_i @@ -5047,6 +5513,14 @@ local format_f=function(f) n=n+1 return format("format('%%%sf',a%s)",f,n) end +local format_F=function(f) + n=n+1 + if not f or f=="" then + return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or format((a%s %% 1 == 0) and '%%i' or '%%.9f',a%s))",n,n,n,n) + else + return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) + end +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5261,7 +5735,7 @@ local builder=Cs { "start", ( P("%")/""*( V("!") -+V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") +V("N") @@ -5272,7 +5746,6 @@ local builder=Cs { "start", +V("j")+V("J") +V("m")+V("M") +V("z") -+V("*") )+V("*") )*(P(-1)+Carg(1)) )^0, @@ -5281,6 +5754,7 @@ local builder=Cs { "start", ["i"]=(prefix_any*P("i"))/format_i, ["d"]=(prefix_any*P("d"))/format_d, ["f"]=(prefix_any*P("f"))/format_f, + ["F"]=(prefix_any*P("F"))/format_F, ["g"]=(prefix_any*P("g"))/format_g, ["G"]=(prefix_any*P("G"))/format_G, ["e"]=(prefix_any*P("e"))/format_e, @@ -5315,11 +5789,12 @@ local builder=Cs { "start", ["a"]=(prefix_any*P("a"))/format_a, ["A"]=(prefix_any*P("A"))/format_A, ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, + ["?"]=Cs(((1-P("%"))^1 )^1)/format_rest, ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, } local direct=Cs ( - P("%")/""*Cc([[local format = string.format return function(str) return format("%]])*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*Cc([[",str) end]])*P(-1) - ) + P("%")*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*P(-1)/[[local format = string.format return function(str) return format("%0",str) end]] +) local function make(t,str) local f local p @@ -5328,10 +5803,10 @@ local function make(t,str) f=loadstripped(p)() else n=0 - p=lpegmatch(builder,str,1,"..",t._extensions_) + p=lpegmatch(builder,str,1,t._connector_,t._extensions_) if n>0 then p=format(template,preamble,t._preamble_,arguments[n],p) - f=loadstripped(p)() + f=loadstripped(p,t._environment_)() else f=function() return str end end @@ -5343,10 +5818,22 @@ local function use(t,fmt,...) return t[fmt](...) end strings.formatters={} -function strings.formatters.new() - local t={ _extensions_={},_preamble_="",_type_="formatter" } - setmetatable(t,{ __index=make,__call=use }) - return t +if _LUAVERSION<5.2 then + function strings.formatters.new(noconcat) + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_=preamble,_environment_={} } + setmetatable(t,{ __index=make,__call=use }) + return t + end +else + function strings.formatters.new(noconcat) + local e={} + for k,v in next,environment do + e[k]=v + end + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_="",_environment_=e } + setmetatable(t,{ __index=make,__call=use }) + return t + end end local formatters=strings.formatters.new() string.formatters=formatters @@ -5354,8 +5841,12 @@ string.formatter=function(str,...) return formatters[str](...) end local function add(t,name,template,preamble) if type(t)=="table" and t._type_=="formatter" then t._extensions_[name]=template or "%s" - if preamble then + if type(preamble)=="string" then t._preamble_=preamble.."\n"..t._preamble_ + elseif type(preamble)=="table" then + for k,v in next,preamble do + t._environment_[k]=v + end end end end @@ -5364,9 +5855,28 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+P(1))^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]]) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]]) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]]) +if _LUAVERSION<5.2 then + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],"local xmlescape = lpeg.patterns.xmlescape") + add(formatters,"tex",[[lpegmatch(texescape,%s)]],"local texescape = lpeg.patterns.texescape") + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],"local luaescape = lpeg.patterns.luaescape") +else + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) + add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +end +local dquote=patterns.dquote +local equote=patterns.escaped+dquote/'\\"'+1 +local space=patterns.space +local cquote=Cc('"') +local pattern=Cs(dquote*(equote-P(-2))^0*dquote) ++Cs(cquote*(equote-space)^0*space*equote^0*cquote) +function string.optionalquoted(str) + return lpegmatch(pattern,str) or str +end +local pattern=Cs((newline/os.newline+1)^0) +function string.replacenewlines(str) + return lpegmatch(pattern,str) +end end -- of closure @@ -5375,7 +5885,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 23952, stripped down to: 16092 +-- original size: 25338, stripped down to: 16247 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5388,7 +5898,7 @@ utilities=utilities or {} utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub -local concat,insert,remove=table.concat,table.insert,table.remove +local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc @@ -5396,27 +5906,29 @@ local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs local formatters=string.formatters local utftoeight=utf.toeight local splitter=lpeg.tsplitat(".") -function tables.definetable(target,nofirst,nolast) - local composed,shortcut,t=nil,nil,{} +function utilities.tables.definetable(target,nofirst,nolast) + local composed,t=nil,{} local snippets=lpegmatch(splitter,target) for i=1,#snippets-(nolast and 1 or 0) do local name=snippets[i] if composed then - composed=shortcut.."."..name - shortcut=shortcut.."_"..name - t[#t+1]=formatters["local %s = %s if not %s then %s = { } %s = %s end"](shortcut,composed,shortcut,shortcut,composed,shortcut) + composed=composed.."."..name + t[#t+1]=formatters["if not %s then %s = { } end"](composed,composed) else composed=name - shortcut=name if not nofirst then t[#t+1]=formatters["%s = %s or { }"](composed,composed) end end end - if nolast then - composed=shortcut.."."..snippets[#snippets] + if composed then + if nolast then + composed=composed.."."..snippets[#snippets] + end + return concat(t,"\n"),composed + else + return "",target end - return concat(t,"\n"),composed end function tables.definedtable(...) local t=_G @@ -5443,7 +5955,7 @@ function tables.accesstable(target,root) end function tables.migratetable(target,v,root) local t=root or _G - local names=string.split(target,".") + local names=lpegmatch(splitter,target) for i=1,#names-1 do local name=names[i] t[name]=t[name] or {} @@ -5463,6 +5975,15 @@ function tables.removevalue(t,value) end end end +function tables.replacevalue(t,oldvalue,newvalue) + if oldvalue and newvalue then + for i=1,#t do + if t[i]==oldvalue then + t[i]=newvalue + end + end + end +end function tables.insertbeforevalue(t,value,extra) for i=1,#t do if t[i]==extra then @@ -5610,7 +6131,7 @@ local f_ordered_string=formatters["%q,"] local f_ordered_number=formatters["%s,"] local f_ordered_boolean=formatters["%l,"] function table.fastserialize(t,prefix) - local r={ prefix or "return" } + local r={ type(prefix)=="string" and prefix or "return" } local m=1 local function fastserialize(t,outer) local n=#t @@ -5807,7 +6328,8 @@ function table.serialize(root,name,specification) local t local n=1 local function simple_table(t) - if #t>0 then + local nt=#t + if nt>0 then local n=0 for _,v in next,t do n=n+1 @@ -5815,19 +6337,17 @@ function table.serialize(root,name,specification) return nil end end - if n==#t then + if n==nt then local tt={} - local nt=0 - for i=1,#t do + for i=1,nt do local v=t[i] local tv=type(v) - nt=nt+1 if tv=="number" then - tt[nt]=v + tt[i]=v elseif tv=="string" then - tt[nt]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then - tt[nt]=v and "true" or "false" + tt[i]=v and "true" or "false" else return nil end @@ -5856,7 +6376,7 @@ function table.serialize(root,name,specification) end depth=depth+1 end - if root and next(root) then + if root and next(root)~=nil then local first=nil local last=0 last=#root @@ -5875,13 +6395,13 @@ function table.serialize(root,name,specification) local v=root[k] local tv=type(v) local tk=type(k) - if first and tk=="number" and k>=first and k<=last then + if first and tk=="number" and k<=last and k>=first then if tv=="number" then n=n+1 t[n]=f_val_num(depth,v) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if not next(v) then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -5911,13 +6431,13 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_boo_value_str(depth,k,v) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then - n=n+1 t[n]=f_key_num_value_not(depth,k,v) + n=n+1 t[n]=f_key_num_value_not(depth,k) elseif tk=="string" then - n=n+1 t[n]=f_key_str_value_not(depth,k,v) + n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then - n=n+1 t[n]=f_key_boo_value_not(depth,k,v) + n=n+1 t[n]=f_key_boo_value_not(depth,k) end else local st=simple_table(v) @@ -5969,7 +6489,7 @@ function table.serialize(root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,1,0) end end @@ -6132,7 +6652,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 19604, stripped down to: 13998 +-- original size: 21780, stripped down to: 15121 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6154,6 +6674,8 @@ local patterns=parsers.patterns or {} parsers.patterns=patterns local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash +local sortedkeys=table.sortedkeys +local tohash=table.tohash local digit=R("09") local space=P(' ') local equal=P("=") @@ -6203,9 +6725,7 @@ patterns.settings_to_hash_a=pattern_a_s patterns.settings_to_hash_b=pattern_b_s patterns.settings_to_hash_c=pattern_c_s function parsers.make_settings_to_hash_pattern(set,how) - if type(str)=="table" then - return set - elseif how=="strict" then + if how=="strict" then return (pattern_c/set)^1 elseif how=="tolerant" then return (pattern_b/set)^1 @@ -6214,7 +6734,9 @@ function parsers.make_settings_to_hash_pattern(set,how) end end function parsers.settings_to_hash(str,existing) - if type(str)=="table" then + if not str or str=="" then + return {} + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6223,16 +6745,16 @@ function parsers.settings_to_hash(str,existing) else return str end - elseif str and str~="" then + else hash=existing or {} lpegmatch(pattern_a_s,str) return hash - else - return {} end end function parsers.settings_to_hash_tolerant(str,existing) - if type(str)=="table" then + if not str or str=="" then + return {} + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6241,16 +6763,16 @@ function parsers.settings_to_hash_tolerant(str,existing) else return str end - elseif str and str~="" then + else hash=existing or {} lpegmatch(pattern_b_s,str) return hash - else - return {} end end function parsers.settings_to_hash_strict(str,existing) - if type(str)=="table" then + if not str or str=="" then + return nil + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6263,8 +6785,6 @@ function parsers.settings_to_hash_strict(str,existing) hash=existing or {} lpegmatch(pattern_c_s,str) return next(hash) and hash - else - return nil end end local separator=comma*space^0 @@ -6272,27 +6792,46 @@ local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comm local pattern=spaces*Ct(value*(separator*value)^0) patterns.settings_to_array=pattern function parsers.settings_to_array(str,strict) - if type(str)=="table" then - return str - elseif not str or str=="" then + if not str or str=="" then return {} + elseif type(str)=="table" then + return str elseif strict then - if find(str,"{") then + if find(str,"{",1,true) then return lpegmatch(pattern,str) else return { str } end - elseif find(str,",") then + elseif find(str,",",1,true) then return lpegmatch(pattern,str) else return { str } end end -local separator=space^0*comma*space^0 -local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(comma+P(-1)))))^0) -local withvalue=Carg(1)*value/function(f,s) return f(s) end -local pattern_a=spaces*Ct(value*(separator*value)^0) -local pattern_b=spaces*withvalue*(separator*withvalue)^0 +local cache_a={} +local cache_b={} +function parsers.groupedsplitat(symbol,withaction) + if not symbol then + symbol="," + end + local pattern=(withaction and cache_b or cache_a)[symbol] + if not pattern then + local symbols=S(symbol) + local separator=space^0*symbols*space^0 + local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(symbols+P(-1)))))^0) + if withaction then + local withvalue=Carg(1)*value/function(f,s) return f(s) end + pattern=spaces*withvalue*(separator*withvalue)^0 + cache_b[symbol]=pattern + else + pattern=spaces*Ct(value*(separator*value)^0) + cache_a[symbol]=pattern + end + end + return pattern +end +local pattern_a=parsers.groupedsplitat(",",false) +local pattern_b=parsers.groupedsplitat(",",true) function parsers.stripped_settings_to_array(str) if not str or str=="" then return {} @@ -6317,8 +6856,8 @@ function parsers.add_settings_to_array(t,str) end function parsers.hash_to_string(h,separator,yes,no,strict,omit) if h then - local t,tn,s={},0,table.sortedkeys(h) - omit=omit and table.tohash(omit) + local t,tn,s={},0,sortedkeys(h) + omit=omit and tohash(omit) for i=1,#s do local key=s[i] if not omit or not omit[key] then @@ -6354,12 +6893,9 @@ function parsers.array_to_string(a,separator) return "" end end -function parsers.settings_to_set(str,t) - t=t or {} - for s in gmatch(str,"[^, ]+") do - t[s]=true - end - return t +local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) +function utilities.parsers.settings_to_set(str,t) + return str and lpegmatch(pattern,str) or {} end function parsers.simple_hash_to_string(h,separator) local t,tn={},0 @@ -6371,12 +6907,16 @@ function parsers.simple_hash_to_string(h,separator) end return concat(t,separator or ",") end -local str=C((1-whitespace-equal)^1) +local str=Cs(lpegpatterns.unquoted)+C((1-whitespace-equal)^1) local setting=Cf(Carg(1)*(whitespace^0*Cg(str*whitespace^0*(equal*whitespace^0*str+Cc(""))))^1,rawset) local splitter=setting^1 function utilities.parsers.options_to_hash(str,target) return str and lpegmatch(splitter,str,1,target or {}) or {} end +local splitter=lpeg.tsplitat(" ") +function utilities.parsers.options_to_array(str) + return str and lpegmatch(splitter,str) or {} +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) local pattern_a=spaces*Ct(value*(separator*value)^0) local function repeater(n,str) @@ -6463,7 +7003,7 @@ function parsers.keq_to_hash(str) end local defaultspecification={ separator=",",quote='"' } function parsers.csvsplitter(specification) - specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification local separator=specification.separator local quotechar=specification.quote local separator=S(separator~="" and separator or ",") @@ -6487,7 +7027,7 @@ function parsers.csvsplitter(specification) end end function parsers.rfc4180splitter(specification) - specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification local separator=specification.separator local quotechar=P(specification.quote) local dquotechar=quotechar*quotechar @@ -6498,7 +7038,7 @@ function parsers.rfc4180splitter(specification) local field=escaped+non_escaped+Cc("") local record=Ct(field*(separator*field)^1) local headerline=record*Cp() - local wholeblob=Ct((newline^-1*record)^0) + local wholeblob=Ct((newline^(specification.strict and -1 or 1)*record)^0) return function(data,getheader) if getheader then local header,position=lpegmatch(headerline,data) @@ -6535,20 +7075,20 @@ function parsers.stepper(str,n,action) lpegmatch(stepper,str,1,n,action or print) end end -local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) -local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) +local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) +local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) patterns.unittotex=pattern function parsers.unittotex(str,textmode) return lpegmatch(textmode and pattern_text or pattern_math,str) end -local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+P(1))^0) +local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+anything)^0) function parsers.unittoxml(str) return lpegmatch(pattern,str) end local cache={} -local spaces=lpeg.patterns.space^0 +local spaces=lpegpatterns.space^0 local dummy=function() end -table.setmetatableindex(cache,function(t,k) +setmetatableindex(cache,function(t,k) local separator=P(k) local value=(1-separator)^0 local pattern=spaces*C(value)*separator^0*Cp() @@ -6613,6 +7153,18 @@ function utilities.parsers.runtime(time) local seconds=mod(time,60) return days,hours,minutes,seconds end +local spacing=whitespace^0 +local apply=P("->") +local method=C((1-apply)^1) +local token=lbrace*C((1-rbrace)^1)*rbrace+C(anything^1) +local pattern=spacing*(method*spacing*apply+Carg(1))*spacing*token +function utilities.parsers.splitmethod(str,default) + if str then + return lpegmatch(pattern,str,1,default or false) + else + return default or false,"" + end +end end -- of closure @@ -6702,7 +7254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12365, stripped down to: 8799 +-- original size: 12482, stripped down to: 8864 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -6730,7 +7282,7 @@ function setters.initialize(filename,name,values) local data=setter.data if data then for key,newvalue in next,values do - local newvalue=is_boolean(newvalue,newvalue) + local newvalue=is_boolean(newvalue,newvalue,true) local functions=data[key] if functions then local oldvalue=functions.value @@ -6784,7 +7336,7 @@ local function set(t,what,newvalue) elseif not value then value=false else - value=is_boolean(value,value) + value=is_boolean(value,value,true) end w=topattern(w,true,true) for name,functions in next,data do @@ -6923,6 +7475,7 @@ function setters.new(name) report=function(...) setters.report (setter,...) end, enable=function(...) enable (setter,...) end, disable=function(...) disable (setter,...) end, + reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, list=function(...) list (setter,...) end, show=function(...) show (setter,...) end, @@ -7014,7 +7567,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 25391, stripped down to: 16561 +-- original size: 29359, stripped down to: 20483 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7023,15 +7576,18 @@ if not modules then modules={} end modules ['trac-log']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local next,type,select,print=next,type,select,print local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write local format,gmatch,find=string.format,string.gmatch,string.find local concat,insert,remove=table.concat,table.insert,table.remove local topattern=string.topattern -local next,type,select=next,type,select local utfchar=utf.char +local datetime=os.date +local openfile=io.open local setmetatableindex=table.setmetatableindex local formatters=string.formatters local texgetcount=tex and tex.getcount +local variant="default" logs=logs or {} local logs=logs local moreinfo=[[ @@ -7041,32 +7597,122 @@ maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context webpage : http://www.pragma-ade.nl / http://tex.aanhet.net wiki : http://contextgarden.net ]] -utilities.strings.formatters.add ( +formatters.add ( formatters,"unichr", [["U+" .. format("%%05X",%s) .. " (" .. utfchar(%s) .. ")"]] ) -utilities.strings.formatters.add ( +formatters.add ( formatters,"chruni", [[utfchar(%s) .. " (U+" .. format("%%05X",%s) .. ")"]] ) local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations -local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters +local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then - local valueiskey={ __index=function(t,k) t[k]=k return k end } - local target="term and log" + local function useluawrites() + local texio_write_nl=texio.write_nl + local texio_write=texio.write + local io_write=io.write + write_nl=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write_nl("log",...) + texio_write_nl("term","") + io_write(...) + elseif target=="log" then + texio_write_nl("log",...) + elseif target=="term" then + texio_write_nl("term","") + io_write(...) + elseif target~="none" then + texio_write_nl("log",target,...) + texio_write_nl("term","") + io_write(target,...) + end + end + write=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write("log",...) + io_write(...) + elseif target=="log" then + texio_write("log",...) + elseif target=="term" then + io_write(...) + elseif target~="none" then + texio_write("log",target,...) + io_write(target,...) + end + end + texio.write=write + texio.write_nl=write_nl + useluawrites=ignore + end + local whereto="both" + local target=nil + local targets=nil + local formats=table.setmetatableindex("self") + local translations=table.setmetatableindex("self") + local report_yes,subreport_yes,direct_yes,subdirect_yes,status_yes + local report_nop,subreport_nop,direct_nop,subdirect_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s > %s\n"], + report_nop=formatters["%-15s >\n"], + direct_yes=formatters["%-15s > %s"], + direct_nop=formatters["%-15s >"], + subreport_yes=formatters["%-15s > %s > %s\n"], + subreport_nop=formatters["%-15s > %s >\n"], + subdirect_yes=formatters["%-15s > %s > %s"], + subdirect_nop=formatters["%-15s > %s >"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;33m%-15s [0;1m>[0m %s\n"], + report_nop=formatters["[0;33m%-15s [0;1m>[0m\n"], + direct_yes=formatters["[0;33m%-15s [0;1m>[0m %s"], + direct_nop=formatters["[0;33m%-15s [0;1m>[0m"], + subreport_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s\n"], + subreport_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m\n"], + subdirect_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s"], + subdirect_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m"], + status_yes=formatters["[0;33m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;33m%-15s [0;1m:[0m\n"], + }, + targets={ + logfile="none", + log="none", + file="none", + console="term", + terminal="term", + both="term", + }, + } + } logs.flush=io.flush - local formats={} setmetatable(formats,valueiskey) - local translations={} setmetatable(translations,valueiskey) writer=function(...) write_nl(target,...) end newline=function() write_nl(target,"\n") end - local report_yes=formatters["%-15s > %s\n"] - local report_nop=formatters["%-15s >\n"] report=function(a,b,c,...) if c then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) @@ -7078,8 +7724,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local direct_yes=formatters["%-15s > %s"] - local direct_nop=formatters["%-15s >"] direct=function(a,b,c,...) if c then return direct_yes(translations[a],formatters[formats[b]](c,...)) @@ -7091,8 +7735,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local subreport_yes=formatters["%-15s > %s > %s\n"] - local subreport_nop=formatters["%-15s > %s >\n"] subreport=function(a,s,b,c,...) if c then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) @@ -7104,8 +7746,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local subdirect_yes=formatters["%-15s > %s > %s"] - local subdirect_nop=formatters["%-15s > %s >"] subdirect=function(a,s,b,c,...) if c then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) @@ -7117,8 +7757,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) @@ -7130,16 +7768,13 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local targets={ - logfile="log", - log="log", - file="log", - console="term", - terminal="term", - both="term and log", - } - settarget=function(whereto) - target=targets[whereto or "both"] or targets.both + settarget=function(askedwhereto) + whereto=askedwhereto or whereto or "both" + target=targets[whereto] + if not target then + whereto="both" + target=targets[whereto] + end if target=="term" or target=="term and log" then logs.flush=io.flush else @@ -7168,21 +7803,74 @@ if tex and (tex.jobname or tex.formatname) then writeline(target,f(...)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - direct_yes=f.direct_yes or direct_yes - direct_nop=f.direct_nop or direct_nop - subdirect_yes=f.subdirect_yes or subdirect_yes - subdirect_nop=f.subdirect_nop or subdirect_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local t=nil + local f=nil + local d=variants.default + if not specification then + elseif type(specification)=="table" then + t=specification.targets + f=specification.formats or specification + else + local v=variants[specification] + if v then + t=v.targets + f=v.formats + variant=specification + end + end + targets=t or d.targets + target=targets[whereto] or target + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + direct_yes=f.direct_yes + direct_nop=f.direct_nop + subdirect_yes=f.subdirect_yes + subdirect_nop=f.subdirect_nop + status_yes=f.status_yes + status_nop=f.status_nop + if variant=="ansi" then + useluawrites() + end + settarget(whereto) + end + setformatters(variant) setlogfile=ignore settimedlog=ignore else + local report_yes,subreport_yes,status_yes + local report_nop,subreport_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s | %s"], + report_nop=formatters["%-15s |"], + subreport_yes=formatters["%-15s | %s | %s"], + subreport_nop=formatters["%-15s | %s |"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;32m%-15s [0;1m|[0m %s"], + report_nop=formatters["[0;32m%-15s [0;1m|[0m"], + subreport_yes=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m %s"], + subreport_nop=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m"], + status_yes=formatters["[0;32m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;32m%-15s [0;1m:[0m\n"], + }, + }, + } logs.flush=ignore writer=function(s) write_nl(s) @@ -7190,8 +7878,6 @@ else newline=function() write_nl("\n") end - local report_yes=formatters["%-15s | %s"] - local report_nop=formatters["%-15s |"] report=function(a,b,c,...) if c then write_nl(report_yes(a,formatters[b](c,...))) @@ -7203,8 +7889,6 @@ else write_nl("") end end - local subreport_yes=formatters["%-15s | %s | %s"] - local subreport_nop=formatters["%-15s | %s |"] subreport=function(a,sub,b,c,...) if c then write_nl(subreport_yes(a,sub,formatters[b](c,...))) @@ -7216,8 +7900,6 @@ else write_nl("") end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(status_yes(a,formatters[b](c,...))) @@ -7242,14 +7924,34 @@ else writeline(f(s)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local f=nil + local d=variants.default + if specification then + if type(specification)=="table" then + f=specification.formats or specification + else + local v=variants[specification] + if v then + f=v.formats + end + end + end + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + status_yes=f.status_yes + status_nop=f.status_nop + end + setformatters(variant) setlogfile=function(name,keepopen) if name and name~="" then local localtime=os.localtime @@ -7368,9 +8070,10 @@ local function setblocked(category,value) v.state=value end else - states=utilities.parsers.settings_to_hash(category) + states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) for c,_ in next,states do - if data[c] then + local v=data[c] + if v then v.state=value else c=topattern(c,true,true) @@ -7501,13 +8204,13 @@ end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple -function logs.setprogram () end -function logs.extendbanner() end -function logs.reportlines () end -function logs.reportbanner() end -function logs.reportline () end -function logs.simplelines () end -function logs.help () end +logs.setprogram=ignore +logs.extendbanner=ignore +logs.reportlines=ignore +logs.reportbanner=ignore +logs.reportline=ignore +logs.simplelines=ignore +logs.help=ignore local Carg,C,lpegmatch=lpeg.Carg,lpeg.C,lpeg.match local p_newline=lpeg.patterns.newline local linewise=( @@ -7584,10 +8287,11 @@ function logs.application(t) end return t end -function logs.system(whereto,process,jobname,category,...) - local message=formatters["%s %s => %s => %s => %s\r"](os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) +local f_syslog=formatters["%s %s => %s => %s => %s\r"] +function logs.system(whereto,process,jobname,category,fmt,arg,...) + local message=f_syslog(datetime("%d/%m/%y %H:%m:%S"),process,jobname,category,arg==nil and fmt or format(fmt,arg,...)) for i=1,10 do - local f=io.open(whereto,"a") + local f=openfile(whereto,"a") if f then f:write(message) f:close() @@ -7649,7 +8353,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6501, stripped down to: 5156 +-- original size: 6704, stripped down to: 5343 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -7659,7 +8363,7 @@ if not modules then modules={} end modules ['trac-inf']={ license="see context related readme files" } local type,tonumber,select=type,tonumber,select -local format,lower=string.format,string.lower +local format,lower,find=string.format,string.lower,string.find local concat=table.concat local clock=os.gettimeofday or os.clock local setmetatableindex=table.setmetatableindex @@ -7750,7 +8454,8 @@ function statistics.show() if statistics.enable then local register=statistics.register register("used platform",function() - return format("%s, type: %s, binary subtree: %s",os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") + return format("%s, type: %s, binary subtree: %s", + os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) register("luatex banner",function() return lower(status.banner) @@ -7763,14 +8468,23 @@ function statistics.show() return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) if jit then - local status={ jit.status() } - if status[1] then - register("luajit status",function() - return concat(status," ",2) - end) - end - end - register("current memory usage",statistics.memused) + local jitstatus={ jit.status() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end + end + register("lua properties",function() + local list=status.list() + local hashchar=tonumber(list.luatex_hashchars) + local mask=lua.mask or "ascii" + return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", + jit and "luajit" or "lua", + statistics.memused(), + list.luatex_hashtype or "default", + hashchar and 2^hashchar or "unknown", + mask, + mask=="utf" and "τεχ" or "tex") + end) register("runtime",statistics.runtime) logs.newline() for i=1,#statusinfo do @@ -7812,15 +8526,6 @@ function statistics.tracefunction(base,tag,...) statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end) end end -commands=commands or {} -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) -end end -- of closure @@ -7829,7 +8534,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5773, stripped down to: 3453 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -7846,14 +8551,16 @@ local namespaces=namespaces local registered={} local function report_index(k,name) if trace_namespaces then - report_system("reference to %a in protected namespace %a: %s",k,name,debug.traceback()) + report_system("reference to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) else report_system("reference to %a in protected namespace %a",k,name) end end local function report_newindex(k,name) if trace_namespaces then - report_system("assignment to %a in protected namespace %a: %s",k,name,debug.traceback()) + report_system("assignment to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) else report_system("assignment to %a in protected namespace %a",k,name) end @@ -8104,7 +8811,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3708, stripped down to: 2568 +-- original size: 3898, stripped down to: 2644 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8184,20 +8891,22 @@ end function debugger.disable() debug.sethook() end -function traceback() - local level=1 +local function showtraceback(rep) + local level=2 + local reporter=rep or report while true do - local info=debug.getinfo(level,"Sl") + local info=getinfo(level,"Sl") if not info then break elseif info.what=="C" then - print(format("%3i : C function",level)) + reporter("%2i : %s",level-1,"C function") else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + reporter("%2i : %s : %s",level-1,info.short_src,info.currentline) end level=level+1 end end +debugger.showtraceback=showtraceback end -- of closure @@ -8383,7 +9092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 6251, stripped down to: 3488 +-- original size: 7100, stripped down to: 3978 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -8425,7 +9134,7 @@ local sqlescape=lpeg.replacer { { "\r\n","\\n" }, { "\r","\\n" }, } -local sqlquoted=lpeg.Cs(lpeg.Cc("'")*sqlescape*lpeg.Cc("'")) +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) lpegpatterns.sqlescape=sqlescape lpegpatterns.sqlquoted=sqlquoted local luaescape=lpegpatterns.luaescape @@ -8448,12 +9157,24 @@ local quotedescapers={ local luaescaper=escapers.lua local quotedluaescaper=quotedescapers.lua local function replacekeyunquoted(s,t,how,recurse) - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end end local function replacekeyquoted(s,t,how,recurse) - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end local single=P("%") local double=P("%%") @@ -8468,11 +9189,16 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' -local key=nosingle*((C((1-nosingle )^1)*Carg(1)*Carg(2)*Carg(3))/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional local any=P(1) - replacer=Cs((unquoted+quoted+escape+key+any)^0) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -8511,7 +9237,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8807, stripped down to: 5085 +-- original size: 8022, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -8522,7 +9248,7 @@ if not modules then modules={} end modules ['util-env']={ } local allocate,mark=utilities.storage.allocate,utilities.storage.mark local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find -local unquoted,quoted=string.unquoted,string.quoted +local unquoted,quoted,optionalquoted=string.unquoted,string.quoted,string.optionalquoted local concat,insert,remove=table.concat,table.insert,table.remove environment=environment or {} local environment=environment @@ -8635,24 +9361,14 @@ function environment.splitarguments(separator) return before,after end function environment.reconstructcommandline(arg,noquote) + local resolveprefix=resolvers.resolve arg=arg or environment.originalarguments if noquote and #arg==1 then - local a=arg[1] - a=resolvers.resolve(a) - a=unquoted(a) - return a + return unquoted(resolveprefix and resolveprefix(arg[1]) or arg[1]) elseif #arg>0 then local result={} for i=1,#arg do - local a=arg[i] - a=resolvers.resolve(a) - a=unquoted(a) - a=gsub(a,'"','\\"') - if find(a," ") then - result[#result+1]=quoted(a) - else - result[#result+1]=a - end + result[i]=optionalquoted(resolveprefix and resolveprefix(arg[i]) or resolveprefix) end return concat(result," ") else @@ -8708,7 +9424,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 5930, stripped down to: 4235 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -8786,15 +9502,13 @@ function environment.luafilechunk(filename,silent) filename=file.replacesuffix(filename,"lua") local fullname=environment.luafile(filename) if fullname and fullname~="" then - local data=luautilities.loadedluacode(fullname,strippable,filename) - if trace_locating then + local data=luautilities.loadedluacode(fullname,strippable,filename) + if not silent then report_lua("loading file %a %s",fullname,not data and "failed" or "succeeded") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") end return data else - if trace_locating then + if not silent then report_lua("unknown file %a",filename) end return nil @@ -8863,7 +9577,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 42447, stripped down to: 26589 +-- original size: 45683, stripped down to: 27866 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -8878,10 +9592,10 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local utfchar=utf.char -local lpegmatch=lpeg.match +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters xml.xmlns=xml.xmlns or {} @@ -8976,8 +9690,10 @@ local function add_end(spacing,namespace,tag) top=stack[#stack] if #stack<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) elseif toclose.tg~=tag then errorstr=formatters["unable to close %s with %s %s"](toclose.tg,tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) end dt=top.dt dt[#dt+1]=toclose @@ -8986,10 +9702,29 @@ local function add_end(spacing,namespace,tag) end end local function add_text(text) + local n=#dt if cleanup and #text>0 then - dt[#dt+1]=cleanup(text) + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..cleanup(text) + else + dt[n+1]=cleanup(text) + end + else + dt[1]=cleanup(text) + end else - dt[#dt+1]=text + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..text + else + dt[n+1]=text + end + else + dt[1]=text + end end end local function add_special(what,spacing,text) @@ -9021,8 +9756,10 @@ local function attribute_specification_error(str) end return str end +local badentity="&error;" +local badentity="&" xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and "&error;" or formatters["&%s;"](str) end, + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, } @@ -9043,9 +9780,10 @@ local function fromdec(s) return formatters["d:%s"](s),true end end -local rest=(1-P(";"))^0 -local many=P(1)^0 -local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local p_rest=(1-P(";"))^0 +local p_many=P(1)^0 +local p_char=lpegpatterns.utf8character +local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) local predefined_unified={ [38]="&", [42]=""", @@ -9071,7 +9809,9 @@ local privates_u={ local privates_p={} local privates_n={ } -local escaped=utf.remapper(privates_u) +local escaped=utf.remapper(privates_u,"dynamic") +local unprivatized=utf.remapper(privates_p,"dynamic") +xml.unprivatized=unprivatized local function unescaped(s) local p=privates_n[s] if not p then @@ -9084,9 +9824,7 @@ local function unescaped(s) end return p end -local unprivatized=utf.remapper(privates_p) xml.privatetoken=unescaped -xml.unprivatized=unprivatized xml.privatecodes=privates_n local function handle_hex_entity(str) local h=hcache[str] @@ -9181,7 +9919,7 @@ local function handle_any_entity(str) report_xml("keeping entity &%s;",str) end if str=="" then - a="&error;" + a=badentity else a="&"..str..";" end @@ -9209,7 +9947,7 @@ local function handle_any_entity(str) if trace_entities then report_xml("invalid entity &%s;",str) end - a="&error;" + a=badentity acache[str]=a else if trace_entities then @@ -9222,8 +9960,14 @@ local function handle_any_entity(str) return a end end -local function handle_end_entity(chr) - report_xml("error in entity, %a found instead of %a",chr,";") +local function handle_end_entity(str) + report_xml("error in entity, %a found without ending %a",str,";") + return str +end +local function handle_crap_error(chr) + report_xml("error in parsing, unexpected %a found ",chr) + add_text(chr) + return chr end local space=S(' \r\n\t') local open=P('<') @@ -9239,15 +9983,15 @@ local valid=R('az','AZ','09')+S('_-.') local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop -local utfbom=lpeg.patterns.utfbom +local utfbom=lpegpatterns.utfbom local spacing=C(space^0) -local anyentitycontent=(1-open-semicolon-space-close)^0 +local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 local hexentitycontent=R("AF","af","09")^0 local decentitycontent=R("09")^0 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) )+(anyentitycontent/handle_any_entity) -local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) local somespace=space^1 @@ -9298,16 +10042,20 @@ local instruction=(spacing*begininstruction*someinstruction*endinstruction)/func local comment=(spacing*begincomment*somecomment*endcomment )/function(...) add_special("@cm@",...) end local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special("@cd@",...) end local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end +local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand +local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata +local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 local grammar_parsed_text=P { "preamble", preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, parent=beginelement*V("children")^0*endelement, - children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction, + children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } local grammar_unparsed_text=P { "preamble", preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, parent=beginelement*V("children")^0*endelement, - children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction, + children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) settings=settings or {} @@ -9341,7 +10089,6 @@ local function _xmlconvert_(data,settings) errorstr="empty xml file" elseif utfize or resolve then if lpegmatch(grammar_parsed_text,data) then - errorstr="" else errorstr="invalid xml file - parsed text" end @@ -9357,6 +10104,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } +setmetatable(result,mt) +setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -9389,8 +10138,11 @@ local function _xmlconvert_(data,settings) end if errorstr and errorstr~="" then result.error=true + else + errorstr=nil end result.statistics={ + errormessage=errorstr, entities={ decimals=dcache, hexadecimals=hcache, @@ -9404,7 +10156,7 @@ local function _xmlconvert_(data,settings) reported_attribute_errors,mt,errorhandler=nil,nil,nil return result end -function xmlconvert(data,settings) +local function xmlconvert(data,settings) local ok,result=pcall(function() return _xmlconvert_(data,settings) end) if ok then return result @@ -9496,14 +10248,17 @@ function xml.checkbom(root) insert(dt,2,"\n" ) end end -local function verbose_element(e,handlers) +local f_attribute=formatters['%s=%q'] +local function verbose_element(e,handlers,escape) local handle=handlers.handle local serialize=handlers.serialize local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn local ats=eat and next(eat) and {} if ats then + local n=0 for k,v in next,eat do - ats[#ats+1]=formatters['%s=%q'](k,escaped(v)) + n=n+1 + ats[n]=f_attribute(k,escaped(v)) end end if ern and trace_entities and ern~=ens then @@ -9588,23 +10343,25 @@ local function verbose_document(e,handlers) end end local function serialize(e,handlers,...) - local initialize=handlers.initialize - local finalize=handlers.finalize - local functions=handlers.functions - if initialize then - local state=initialize(...) - if not state==true then - return state + if e then + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() end - end - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) - end - if finalize then - return finalize() end end local function xserialize(e,handlers) @@ -9845,7 +10602,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48956, stripped down to: 30516 +-- original size: 48229, stripped down to: 30684 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10230,8 +10987,8 @@ local lp_builtin=P ( P("ns")/"ll.ns" )*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") -lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" -lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" local lp_fastpos=lp_fastpos_n+lp_fastpos_p local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" @@ -10410,7 +11167,7 @@ local function nodesettostring(set,nodetest) if not ns or ns=="" then ns="*" end if not tg or tg=="" then tg="*" end tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i]=(directive and tg) or format("not(%s)",tg) + t[#t+1]=(directive and tg) or format("not(%s)",tg) end if nodetest==false then return format("not(%s)",concat(t,"|")) @@ -10676,7 +11433,6 @@ expressions.print=function(...) print(...) return true end -expressions.contains=find expressions.find=find expressions.upper=upper expressions.lower=lower @@ -10698,6 +11454,9 @@ function expressions.contains(str,pattern) end return false end +function xml.expressions.idstring(str) + return type(str)=="string" and gsub(str,"^#","") or "" +end local function traverse(root,pattern,handle) local collected=applylpath(root,pattern) if collected then @@ -10826,8 +11585,13 @@ function xml.elements(root,pattern,reverse) local collected=applylpath(root,pattern) if not collected then return dummy - elseif reverse then - local c=#collected+1 + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 return function() if c>1 then c=c-1 @@ -10837,7 +11601,7 @@ function xml.elements(root,pattern,reverse) end end else - local n,c=#collected,0 + local c=0 return function() if c<n then c=c+1 @@ -10852,8 +11616,13 @@ function xml.collected(root,pattern,reverse) local collected=applylpath(root,pattern) if not collected then return dummy - elseif reverse then - local c=#collected+1 + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 return function() if c>1 then c=c-1 @@ -10861,7 +11630,7 @@ function xml.collected(root,pattern,reverse) end end else - local n,c=#collected,0 + local c=0 return function() if c<n then c=c+1 @@ -10876,7 +11645,7 @@ function xml.inspect(collection,pattern) report_lpath("pattern: %s\n\n%s\n",pattern,xml.tostring(e)) end end -local function split(e) +local function split(e) local dt=e.dt if dt then for i=1,#dt do @@ -10975,7 +11744,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 23804, stripped down to: 16817 +-- original size: 28786, stripped down to: 20578 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -10985,16 +11754,19 @@ if not modules then modules={} end modules ['lxml-aux']={ license="see context related readme files" } local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local trace_inclusions=false trackers.register("lxml.inclusions",function(v) trace_inclusions=v end) local report_xml=logs.reporter("xml") local xml=xml -local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath local xmlfilter=xml.filter -local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip local utfbyte=utf.byte +local lpegmatch=lpeg.match +local striplinepatterns=utilities.strings.striplinepatterns local function report(what,pattern,c,e) report_xml("%s element %a, root %a, position %a, index %a, pattern %a",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end @@ -11049,13 +11821,15 @@ end function xml.each(root,pattern,handle,reverse) local collected=xmlapplylpath(root,pattern) if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) + if handle then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end end return collected @@ -11111,6 +11885,7 @@ local function redo_ni(d) end end end +xml.reindex=redo_ni local function xmltoelement(whatever,root) if not whatever then return nil @@ -11162,8 +11937,16 @@ function xml.delete(root,pattern) report('deleting',pattern,c,e) end local d=p.dt - remove(d,e.ni) - redo_ni(d) + local ni=e.ni + if ni<=#d then + if false then + p.dt[ni]="" + else + remove(d,ni) + redo_ni(d) + end + else + end end end end @@ -11283,28 +12066,40 @@ xml.insertafter=insert_element xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end xml.injectafter=inject_element xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end -local function include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata,level) pattern=pattern or 'include' loaddata=loaddata or io.loaddata local collected=xmlapplylpath(xmldata,pattern) if collected then + if not level then + level=1 + end for c=1,#collected do local ek=collected[c] local name=nil local ekdt=ek.dt local ekat=ek.at - local epdt=ek.__p__.dt + local ekrt=ek.__p__ + local epdt=ekrt.dt if not attribute or attribute=="" then name=(type(ekdt)=="table" and ekdt[1]) or ekdt end if not name then for a in gmatch(attribute or "href","([^|]+)") do name=ekat[a] - if name then break end + if name then + break + end + end + end + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") end end - local data=(name and name~="" and loaddata(name)) or "" - if data=="" then + if not data or data=="" then epdt[ek.ni]="" elseif ekat["parse"]=="text" then epdt[ek.ni]=xml.escaped(data) @@ -11314,70 +12109,127 @@ local function include(xmldata,pattern,attribute,recursive,loaddata) epdt[ek.ni]="" else if recursive then - include(xi,pattern,attribute,recursive,loaddata) + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local inclusions=xmldata.settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + else + xmldata.settings.inclusions={ name } + end + if child.er then + local badinclusions=xmldata.settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + xmldata.settings.badinclusions={ name } + end end - epdt[ek.ni]=xml.body(xi) end end end end end xml.include=include +function xml.inclusion(e,default) + while e do + local f=e.__f__ + if f then + return f + else + e=e.__p__ + end + end + return default +end +local function getinclusions(key,e,sorted) + while e do + local settings=e.settings + if settings then + local inclusions=settings[key] + if inclusions then + inclusions=table.unique(inclusions) + if sorted then + table.sort(inclusions) + end + return inclusions + else + e=e.__p__ + end + else + e=e.__p__ + end + end +end +function xml.inclusions(e,sorted) + return getinclusions("inclusions",e,sorted) +end +function xml.badinclusions(e,sorted) + return getinclusions("badinclusions",e,sorted) +end +local b_collapser=lpeg.patterns.b_collapser +local m_collapser=lpeg.patterns.m_collapser +local e_collapser=lpeg.patterns.e_collapser +local b_stripper=lpeg.patterns.b_stripper +local m_stripper=lpeg.patterns.m_stripper +local e_stripper=lpeg.patterns.e_stripper +local lpegmatch=lpeg.match local function stripelement(e,nolines,anywhere) local edt=e.dt if edt then - if anywhere then - local t,n={},0 - for e=1,#edt do + local n=#edt + if n==0 then + return e + elseif anywhere then + local t={} + local m=0 + for e=1,n do local str=edt[e] if type(str)~="string" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str elseif str~="" then if nolines then - str=gsub(str,"%s+"," ") + str=lpegmatch((n==1 and b_collapser) or (n==m and e_collapser) or m_collapser,str) + else + str=lpegmatch((n==1 and b_stripper) or (n==m and e_stripper) or m_stripper,str) end - str=gsub(str,"^%s*(.-)%s*$","%1") if str~="" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str end end end e.dt=t else - if #edt>0 then - local str=edt[1] - if type(str)~="string" then - elseif str=="" then + local str=edt[1] + if type(str)=="string" then + if str~="" then + str=lpegmatch(nolines and b_collapser or b_stripper,str) + end + if str=="" then remove(edt,1) + n=n-1 else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"^%s+","") - if str=="" then - remove(edt,1) - else - edt[1]=str - end + edt[1]=str end end - local nedt=#edt - if nedt>0 then - local str=edt[nedt] - if type(str)~="string" then - elseif str=="" then - remove(edt) - else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"%s+$","") + if n>0 then + str=edt[n] + if type(str)=="string" then if str=="" then remove(edt) else - edt[nedt]=str + str=lpegmatch(nolines and e_collapser or e_stripper,str) + if str=="" then + remove(edt) + else + edt[n]=str + end end end end @@ -11563,8 +12415,8 @@ function xml.finalizers.xml.cdata(collected) end return "" end -function xml.insertcomment(e,str,n) - table.insert(e.dt,n or 1,{ +function xml.insertcomment(e,str,n) + insert(e.dt,n or 1,{ tg="@cm@", ns="", special=true, @@ -11572,7 +12424,25 @@ function xml.insertcomment(e,str,n) dt={ str }, }) end -function xml.setcdata(e,str) +function xml.insertcdata(e,str,n) + insert(e.dt,n or 1,{ + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcomment(e,str,n) + e.dt={ { + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.setcdata(e,str) e.dt={ { tg="@cd@", ns="", @@ -11642,7 +12512,7 @@ local function recurse(e,action) for i=1,#edt do local str=edt[i] if type(str)~="string" then - recurse(str,action,recursive) + recurse(str,action) elseif str~="" then edt[i]=action(str) end @@ -11660,6 +12530,65 @@ function helpers.recursetext(collected,action,recursive) end end end +local specials={ + ["@rt@"]="root", + ["@pi@"]="instruction", + ["@cm@"]="comment", + ["@dt@"]="declaration", + ["@cd@"]="cdata", +} +local function convert(x,strip,flat) + local ns=x.ns + local tg=x.tg + local at=x.at + local dt=x.dt + local node=flat and { + [0]=(not x.special and (ns~="" and ns..":"..tg or tg)) or nil, + } or { + _namespace=ns~="" and ns or nil, + _tag=not x.special and tg or nil, + _type=specials[tg] or "_element", + } + if at then + for k,v in next,at do + node[k]=v + end + end + local n=0 + for i=1,#dt do + local di=dt[i] + if type(di)=="table" then + if flat and di.special then + else + di=convert(di,strip,flat) + if di then + n=n+1 + node[n]=di + end + end + elseif strip then + di=lpegmatch(strip,di) + if di~="" then + n=n+1 + node[n]=di + end + else + n=n+1 + node[n]=di + end + end + if next(node) then + return node + end +end +function xml.totable(x,strip,flat) + if type(x)=="table" then + if strip then + strip=striplinepatterns[strip] + end + return convert(x,strip,flat) + end +end end -- of closure @@ -12216,7 +13145,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 7898, stripped down to: 5501 +-- original size: 11085, stripped down to: 7662 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -12225,14 +13154,15 @@ if not modules then modules={} end modules ['data-ini']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } +local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char -local next,type=next,type local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") -local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv resolvers=resolvers or {} local resolvers=resolvers texconfig.kpse_init=false @@ -12360,15 +13290,108 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if profiler then +if type(profiler)=="table" and not jit then directives.register("system.profile",function() profiler.start("luatex-profile.log") end) end -if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local resolved={} +local abstract={} +local dynamic={} +function resolvers.resetresolve(str) + resolved,abstract={},{} +end +function resolvers.allprefixes(separator) + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" + end + end + return all +end +local function _resolve_(method,target) + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +function resolvers.unresolve(str) + return abstract[str] or str +end +function resolvers.setdynamic(str) + dynamic[str]=true +end +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local prefix=C(R("az")^2)*P(":") +local target=C((1-S(" \"\';,"))^1) +local notarget=(#S(";,")+P(-1))*Cc("") +local p_resolve=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) +local p_simple=prefix*P(-1) +local function resolve(str) + if type(str)=="table" then + local res={} + for i=1,#str do + res[i]=resolve(str[i]) + end + return res + end + local res=resolved[str] + if res then + return res + end + local simple=lpegmatch(p_simple,str) + local action=prefixes[simple] + if action then + local res=action(res) + if not dynamic[simple] then + resolved[simple]=res + abstract[res]=simple + end + return res + end + res=lpegmatch(p_resolve,str) + resolved[str]=res + abstract[res]=str + return res +end +resolvers.resolve=resolve +if type(osuname)=="function" then + for k,v in next,osuname() do + if not prefixes[k] then + prefixes[k]=function() return v end + end + end +end +if ostype=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end + end + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + table.setmetatablenewindex(prefixes,makepattern) + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -12378,7 +13401,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 15303, stripped down to: 9716 +-- original size: 17216, stripped down to: 10657 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -12392,12 +13415,16 @@ local concat,sort=table.concat,table.sort local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next +local isdir=lfs.isdir local ostype=os.type -local collapsepath=file.collapsepath +local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_globbing=true trackers.register("resolvers.globbing",function(v) trace_globbing=v end) local report_expansions=logs.reporter("resolvers","expansions") +local report_globbing=logs.reporter("resolvers","globbing") local resolvers=resolvers +local resolveprefix=resolvers.resolve local function f_both(a,b) local t,n={},0 for sb in gmatch(b,"[^,]+") do @@ -12487,35 +13514,27 @@ function resolvers.expandedpathfromlist(pathlist) end return newlist end -local cleanup=lpeg.replacer { - { "!","" }, - { "\\","/" }, -} -function resolvers.cleanpath(str) - local doslashes=(P("\\")/"/"+1)^0 - local donegation=(P("!")/"" )^0 - local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") - if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" - else - return lpegmatch(cleanup,str) +local usedhomedir=nil +local donegation=(P("!")/"" )^0 +local doslashes=(P("\\")/"/"+1)^0 +local function expandedhome() + if not usedhomedir then + usedhomedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if usedhomedir=="~" or usedhomedir=="" or not isdir(usedhomedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent path using current path") end - end - else - local dohome=((P("~")+P("$HOME"))/homedir)^0 - local cleanup=Cs(donegation*dohome*doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" + usedhomedir="." end end - return resolvers.cleanpath(str) + return usedhomedir end -local expandhome=P("~")/"$HOME" +local dohome=((P("~")+P("$HOME")+P("%HOME%"))/expandedhome)^0 +local cleanup=Cs(donegation*dohome*doslashes) +resolvers.cleanpath=function(str) + return str and lpegmatch(cleanup,str) or "" +end +local expandhome=P("~")/"$HOME" local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" local dostring=(expandhome+1 )^0 @@ -12567,46 +13586,67 @@ function resolvers.splitpath(str) end function resolvers.joinpath(str) if type(str)=='table' then - return file.joinpath(str) + return joinpath(str) else return str end end local attributes,directory=lfs.attributes,lfs.dir local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local lessweird=P(".")^1+lpeg.anywhere(S("~`#$%^&*:;\"\'||<>,?\n\r\t")) local timer={} local scanned={} local nofscans=0 local scancache={} -local function scan(files,spec,path,n,m,r) - local full=(path=="" and spec) or (spec..path..'/') +local fullcache={} +local nofsharedscans=0 +local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) + local full=path=="" and spec or (spec..path..'/') local dirs={} local nofdirs=0 + local pattern=tolerant and lessweird or weird for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then + if not lpegmatch(pattern,name) then + local mode=attributes(full..name,"mode") + if mode=="file" then n=n+1 - local f=files[name] - if f then - if type(f)=='string' then - files[name]={ f,path } + local lower=lower(name) + local paths=files[lower] + if paths then + if onlyone then else - f[#f+1]=path + if type(paths)=="string" then + files[lower]={ paths,path } + else + paths[#paths+1]=path + end + if name~=lower then + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end + end end else - files[name]=path - local lower=lower(name) + files[lower]=path if name~=lower then - files["remap:"..lower]=name - r=r+1 + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end end end - elseif mode=='directory' then + elseif mode=="directory" then m=m+1 nofdirs=nofdirs+1 if path~="" then - dirs[nofdirs]=path..'/'..name + dirs[nofdirs]=path.."/"..name else dirs[nofdirs]=name end @@ -12616,107 +13656,69 @@ local function scan(files,spec,path,n,m,r) if nofdirs>0 then sort(dirs) for i=1,nofdirs do - files,n,m,r=scan(files,spec,dirs[i],n,m,r) + files,remap,n,m,r=scan(files,remap,spec,dirs[i],n,m,r,onlyonce,tolerant) end end scancache[sub(full,1,-2)]=files - return files,n,m,r + return files,remap,n,m,r end -local fullcache={} -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) +function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) + local realpath=resolveprefix(path) if usecache then - local files=fullcache[realpath] - if files then + local content=fullcache[realpath] + if content then if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) + report_expansions("using cached scan of path %a, branch %a",path,branch or path) end - return files + nofsharedscans=nofsharedscans+1 + return content end end + statistics.starttiming(timer) if trace_locating then report_expansions("scanning path %a, branch %a",path,branch or path) end - local files,n,m,r=scan({},realpath..'/',"",0,0,0) - files.__path__=path - files.__files__=n - files.__directories__=m - files.__remappings__=r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1]=realpath - fullcache[realpath]=files - end - nofscans=nofscans+1 - statistics.stoptiming(timer) - return files -end -local function simplescan(files,spec,path) - local full=(path=="" and spec) or (spec..path..'/') - local dirs={} - local nofdirs=0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then - if not files[name] then - files[name]=path - end - elseif mode=='directory' then - nofdirs=nofdirs+1 - if path~="" then - dirs[nofdirs]=path..'/'..name - else - dirs[nofdirs]=name - end - end - end - end - if nofdirs>0 then - sort(dirs) - for i=1,nofdirs do - files=simplescan(files,spec,dirs[i]) - end - end - return files -end -local simplecache={} -local nofsharedscans=0 -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) - if usecache then - local files=simplecache[realpath] - if not files then - files=scancache[realpath] - if files then - nofsharedscans=nofsharedscans+1 - end + local content + if isdir(realpath) then + local files,remap,n,m,r=scan({},{},realpath..'/',"",0,0,0,onlyonce,tolerant) + content={ + metadata={ + path=path, + files=n, + directories=m, + remappings=r, + }, + files=files, + remap=remap, + } + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) end - if files then - if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) - end - return files + else + content={ + metadata={ + path=path, + files=0, + directories=0, + remappings=0, + }, + files={}, + remap={}, + } + if trace_locating then + report_expansions("invalid path %a",realpath) end end - if trace_locating then - report_expansions("scanning path %a, branch %a",path,branch or path) - end - local files=simplescan({},realpath..'/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) - end if usecache then scanned[#scanned+1]=realpath - simplecache[realpath]=files + fullcache[realpath]=content end nofscans=nofscans+1 statistics.stoptiming(timer) - return files + return content +end +function resolvers.simplescanfiles(path,branch,usecache) + return resolvers.scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -12727,6 +13729,52 @@ function resolvers.scandata() paths=scanned, } end +function resolvers.get_from_content(content,path,name) + if not content then + return + end + local files=content.files + if not files then + return + end + local remap=content.remap + if not remap then + return + end + if name then + local used=lower(name) + return path,remap[used] or used + else + local name=path + local used=lower(name) + local path=files[used] + if path then + return path,remap[used] or used + end + end +end +local nothing=function() end +function resolvers.filtered_from_content(content,pattern) + if content and type(pattern)=="string" then + local pattern=lower(pattern) + local files=content.files + local remap=content.remap + if files and remap then + local n=next(files) + local function iterator() + while n do + local k=n + n=next(files,k) + if find(k,pattern) then + return files[k],remap and remap[k] or k + end + end + end + return iterator + end + end + return nothing +end end -- of closure @@ -12735,7 +13783,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 8769, stripped down to: 6490 +-- original size: 9216, stripped down to: 6798 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -12753,10 +13801,12 @@ local formats=allocate() local suffixes=allocate() local dangerous=allocate() local suffixmap=allocate() +local usertypes=allocate() resolvers.formats=formats resolvers.suffixes=suffixes resolvers.dangerous=dangerous resolvers.suffixmap=suffixmap +resolvers.usertypes=usertypes local luasuffixes=utilities.lua.suffixes local relations=allocate { core={ @@ -12824,11 +13874,13 @@ local relations=allocate { names={ "mp" }, variable='MPINPUTS', suffixes={ 'mp','mpvi','mpiv','mpii' }, + usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ 'tex',"mkvi","mkiv","mkii" }, + suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + usertype=true, }, icc={ names={ "icc","icc profile","icc profiles" }, @@ -12844,6 +13896,7 @@ local relations=allocate { names={ "lua" }, variable='LUAINPUTS', suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + usertype=true, }, lib={ names={ "lib" }, @@ -12852,11 +13905,15 @@ local relations=allocate { }, bib={ names={ 'bib' }, + variable='BIBINPUTS', suffixes={ 'bib' }, + usertype=true, }, bst={ names={ 'bst' }, + variable='BSTINPUTS', suffixes={ 'bst' }, + usertype=true, }, fontconfig={ names={ 'fontconfig','fontconfig file','fontconfig files' }, @@ -12938,8 +13995,9 @@ function resolvers.updaterelations() for name,relation in next,categories do local rn=relation.names local rv=relation.variable - local rs=relation.suffixes if rn and rv then + local rs=relation.suffixes + local ru=relation.usertype for i=1,#rn do local rni=lower(gsub(rn[i]," ","")) formats[rni]=rv @@ -12951,8 +14009,9 @@ function resolvers.updaterelations() end end end - end - if rs then + if ru then + usertypes[name]=true + end end end end @@ -13003,7 +14062,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15532, stripped down to: 11648 +-- original size: 15618, stripped down to: 11629 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -13013,7 +14072,7 @@ if not modules then modules={} end modules ['data-tmp']={ license="see context related readme files" } local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat,serialize,serializetofile=table.concat,table.serialize,table.tofile +local concat=table.concat local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable local formatters=string.formatters @@ -13022,6 +14081,7 @@ local trace_cache=false trackers.register("resolvers.cache",function(v) trace_ca local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers +local cleanpath=resolvers.cleanpath local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) local compile=utilities.lua.compile @@ -13043,7 +14103,7 @@ caches.relocate=false caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } local writable,readables,usedreadables=nil,{},{} local function identify() - local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] @@ -13281,15 +14341,11 @@ end local saveoptions={ compact=true } function caches.savedata(filepath,filename,data,raw) local tmaname,tmcname=caches.setluanames(filepath,filename) - local reduce,simplify=true,true - if raw then - reduce,simplify=false,false - end data.cache_uuid=os.uuid() if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) + file.savedata(tmaname,table.serialize(data,true,saveoptions)) else - serializetofile(tmaname,data,true,saveoptions) + table.tofile(tmaname,data,true,saveoptions) end utilities.lua.compile(tmaname,tmcname) end @@ -13297,10 +14353,12 @@ local content_state={} function caches.contentstate() return content_state or {} end -function caches.loadcontent(cachename,dataname) - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.loadcontent(cachename,dataname,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() @@ -13332,10 +14390,12 @@ function caches.collapsecontent(content) end end end -function caches.savecontent(cachename,dataname,content) - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.savecontent(cachename,dataname,content,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) if trace_locating then @@ -13350,7 +14410,7 @@ function caches.savecontent(cachename,dataname,content) content=content, uuid=os.uuid(), } - local ok=io.savedata(luaname,serialize(data,true)) + local ok=io.savedata(luaname,table.serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) @@ -13378,7 +14438,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5453, stripped down to: 4007 +-- original size: 5347, stripped down to: 4015 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -13406,8 +14466,8 @@ local function splitmethod(filename) if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") - if not find(filename,"://") then + filename=file.collapsepath(filename,".") + if not find(filename,"://",1,true) then return { scheme="file",path=filename,original=filename,filename=filename } end local specification=url.hashed(filename) @@ -13497,7 +14557,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 61799, stripped down to: 42957 +-- original size: 67003, stripped down to: 46291 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -13507,7 +14567,7 @@ if not modules then modules={} end modules ['data-res']={ license="see context related readme files", } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch -local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local concat,insert,remove,sortedkeys,sortedhash=table.concat,table.insert,table.remove,table.sortedkeys,table.sortedhash local next,type,rawget=next,type,rawget local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg @@ -13516,27 +14576,38 @@ local formatters=string.formatters local filedirname=file.dirname local filebasename=file.basename local suffixonly=file.suffixonly +local addsuffix=file.addsuffix +local removesuffix=file.removesuffix local filejoin=file.join local collapsepath=file.collapsepath local joinpath=file.joinpath +local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local getcurrentdir=lfs.currentdir +local isfile=lfs.isfile +local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes -local getcurrentdir=lfs.currentdir -local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) -local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) +local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) local report_resolving=logs.reporter("resolvers","resolving") local resolvers=resolvers local expandedpathfromlist=resolvers.expandedpathfromlist local checkedvariable=resolvers.checkedvariable local splitconfigurationpath=resolvers.splitconfigurationpath local methodhandler=resolvers.methodhandler +local filtered=resolvers.filtered_from_content +local lookup=resolvers.get_from_content +local cleanpath=resolvers.cleanpath +local resolveprefix=resolvers.resolve local initializesetter=utilities.setters.initialize local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv -resolvers.cacheversion='1.0.1' -resolvers.configbanner='' +resolvers.cacheversion="1.100" +resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" @@ -13555,6 +14626,7 @@ end local unset_variable="unset" local formats=resolvers.formats local suffixes=resolvers.suffixes +local usertypes=resolvers.usertypes local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } @@ -13563,7 +14635,7 @@ local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) if instance then instance.environment[key]=value - ossetenv(key,raw and value or resolvers.resolve(value)) + ossetenv(key,raw and value or resolveprefix(value)) end end local function getenv(key) @@ -13577,7 +14649,7 @@ local function getenv(key) end resolvers.getenv=getenv resolvers.env=getenv -local function resolve(k) +local function resolvevariable(k) return instance.expansions[k] end local dollarstripper=lpeg.stripper("$") @@ -13586,19 +14658,19 @@ local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolve +local somevariable=R("az","AZ","09","__","--")^1/resolvevariable local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) return lpegmatch(variableexpander,var) or var end -function resolvers.newinstance() - if trace_locating then +function resolvers.newinstance() + if trace_locating then report_resolving("creating instance") - end + end local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() local newinstance={ environment=environment, @@ -13611,6 +14683,7 @@ function resolvers.newinstance() foundintrees=allocate(), hashes=allocate(), hashed=allocate(), + pathlists=false, specification=allocate(), lists=allocate(), data=allocate(), @@ -13623,6 +14696,7 @@ function resolvers.newinstance() savelists=true, pattern=nil, force_suffixes=true, + pathstack={}, } setmetatableindex(variables,function(t,k) local v @@ -13672,8 +14746,13 @@ function resolvers.reset() end local function reset_hashes() instance.lists={} + instance.pathlists=false instance.found={} end +local function reset_caches() + instance.lists={} + instance.pathlists=false +end local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( @@ -13725,13 +14804,13 @@ local function identify_configuration_files() for i=1,#cnfpaths do local filepath=cnfpaths[i] local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) if trace_locating then - local fullpath=gsub(resolvers.resolve(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c") + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) end - if lfs.isfile(realname) then + if isfile(realname) then specification[#specification+1]=filename if trace_locating then report_resolving("found configuration file %a",realname) @@ -13753,7 +14832,7 @@ local function load_configuration_files() local filename=specification[i] local pathname=filedirname(filename) local filename=filejoin(pathname,luacnfname) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local setups=instance.setups @@ -13761,7 +14840,7 @@ local function load_configuration_files() local parent=data and data.parent if parent then local filename=filejoin(pathname,parent) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local parentdata=blob() @@ -13786,7 +14865,7 @@ local function load_configuration_files() elseif variables[k]==nil then if trace_locating and not warning then report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable", - k,resolvers.resolve(filename)) + k,resolveprefix(filename)) warning=true end variables[k]=v @@ -13846,7 +14925,7 @@ local function locate_file_databases() local stripped=lpegmatch(inhibitstripper,path) if stripped~="" then local runtime=stripped==path - path=resolvers.cleanpath(path) + path=cleanpath(path) local spec=resolvers.splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped @@ -13909,8 +14988,8 @@ function resolvers.renew(hashname) report_resolving("identifying tree %a",hashname) end end - local realpath=resolvers.resolve(hashname) - if lfs.isdir(realpath) then + local realpath=resolveprefix(hashname) + if isdir(realpath) then if trace_locating then report_resolving("using path %a",realpath) end @@ -14011,19 +15090,53 @@ end function resolvers.unexpandedpath(str) return joinpath(resolvers.unexpandedpathlist(str)) end +function resolvers.pushpath(name) + local pathstack=instance.pathstack + local lastpath=pathstack[#pathstack] + local pluspath=filedirname(name) + if lastpath then + lastpath=collapsepath(filejoin(lastpath,pluspath)) + else + lastpath=collapsepath(pluspath) + end + insert(pathstack,lastpath) + if trace_paths then + report_resolving("pushing path %a",lastpath) + end +end +function resolvers.poppath() + local pathstack=instance.pathstack + if trace_paths and #pathstack>0 then + report_resolving("popping path %a",pathstack[#pathstack]) + end + remove(pathstack) +end +function resolvers.stackpath() + local pathstack=instance.pathstack + local currentpath=pathstack[#pathstack] + return currentpath~="" and currentpath or nil +end local done={} function resolvers.resetextrapath() local ep=instance.extra_paths if not ep then - ep,done={},{} - instance.extra_paths=ep + done={} + instance.extra_paths={} elseif #ep>0 then - instance.lists,done={},{} + done={} + reset_caches() end end function resolvers.registerextrapath(paths,subpaths) - paths=settings_to_array(paths) - subpaths=settings_to_array(subpaths) + if not subpaths or subpaths=="" then + if not paths or path=="" then + return + elseif done[paths] then + return + end + end + local paths=settings_to_array(paths) + local subpaths=settings_to_array(subpaths) local ep=instance.extra_paths or {} local oldn=#ep local newn=oldn @@ -14038,7 +15151,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14048,7 +15161,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=resolvers.cleanpath(p) + ep[newn]=cleanpath(p) done[p]=true end end @@ -14060,7 +15173,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=ep[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14069,52 +15182,70 @@ function resolvers.registerextrapath(paths,subpaths) if newn>0 then instance.extra_paths=ep end - if newn>oldn then - instance.lists={} + if newn~=oldn then + reset_caches() end end -local function made_list(instance,list) - local ep=instance.extra_paths - if not ep or #ep==0 then - return list +function resolvers.pushextrapath(path) + local paths=settings_to_array(path) + if instance.extra_stack then + insert(instance.extra_stack,1,paths) else - local done,new,newn={},{},0 - for k=1,#list do - local v=list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v]=true - newn=newn+1 - new[newn]=v - else - break - end - end - end - for k=1,#ep do - local v=ep[k] + instance.extra_stack={ paths } + end + reset_caches() +end +function resolvers.popextrapath() + if instance.extra_stack then + reset_caches() + return remove(instance.extra_stack,1) + end +end +local function made_list(instance,list,extra_too) + local done={} + local new={} + local newn=0 + local function add(p) + for k=1,#p do + local v=p[k] if not done[v] then done[v]=true newn=newn+1 new[newn]=v end end - for k=1,#list do - local v=list[k] - if not done[v] then - done[v]=true - newn=newn+1 - new[newn]=v + end + for k=1,#list do + local v=list[k] + if done[v] then + elseif find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break + end + end + if extra_too then + local es=instance.extra_stack + if es and #es>0 then + for k=1,#es do + add(es[k]) end end - return new + local ep=instance.extra_paths + if ep and #ep>0 then + add(ep) + end end + add(list) + return new end function resolvers.cleanpathlist(str) local t=resolvers.expandedpathlist(str) if t then for i=1,#t do - t[i]=collapsepath(resolvers.cleanpath(t[i])) + t[i]=collapsepath(cleanpath(t[i])) end end return t @@ -14122,22 +15253,22 @@ end function resolvers.expandpath(str) return joinpath(resolvers.expandedpathlist(str)) end -function resolvers.expandedpathlist(str) +function resolvers.expandedpathlist(str,extra_too) if not str then return {} - elseif instance.savelists then + elseif instance.savelists then str=lpegmatch(dollarstripper,str) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else local lst=resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + return made_list(instance,expandedpathfromlist(lst),extra_too) end end function resolvers.expandedpathlistfromvariable(str) @@ -14148,6 +15279,13 @@ end function resolvers.expandpathfromvariable(str) return joinpath(resolvers.expandedpathlistfromvariable(str)) end +function resolvers.cleanedpathlist(v) + local t=resolvers.expandedpathlist(v) + for i=1,#t do + t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + end + return t +end function resolvers.expandbraces(str) local ori=str local pth=expandedpathfromlist(resolvers.splitpath(ori)) @@ -14164,7 +15302,7 @@ function resolvers.registerfilehash(name,content,someerror) end end local function isreadable(name) - local readable=lfs.isfile(name) + local readable=isfile(name) if trace_detail then if readable then report_resolving("file %a is readable",name) @@ -14174,70 +15312,57 @@ local function isreadable(name) end return readable end -local function collect_files(names) - local filelist,noffiles={},0 +local function collect_files(names) + local filelist={} + local noffiles=0 + local function check(hash,root,pathname,path,name) + if not pathname or find(path,pathname) then + local variant=hash.type + local search=filejoin(root,path,name) + local result=methodhandler('concatinators',variant,root,path,name) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + end for k=1,#names do - local fname=names[k] + local filename=names[k] if trace_detail then - report_resolving("checking name %a",fname) + report_resolving("checking name %a",filename) end - local bname=filebasename(fname) - local dname=filedirname(fname) - if dname=="" or find(dname,"^%.") then - dname=false + local basename=filebasename(filename) + local pathname=filedirname(filename) + if pathname=="" or find(pathname,"^%.") then + pathname=false else - dname=gsub(dname,"%*",".*") - dname="/"..dname.."$" + pathname=gsub(pathname,"%*",".*") + pathname="/"..pathname.."$" end local hashes=instance.hashes for h=1,#hashes do local hash=hashes[h] - local blobpath=hash.name - local files=blobpath and instance.files[blobpath] - if files then + local hashname=hash.name + local content=hashname and instance.files[hashname] + if content then if trace_detail then - report_resolving("deep checking %a, base %a, pattern %a",blobpath,bname,dname) + report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end - local blobfile=files[bname] - if not blobfile then - local rname="remap:"..bname - blobfile=files[rname] - if blobfile then - bname=files[rname] - blobfile=files[bname] - end - end - if blobfile then - local blobroot=files.__path__ or blobpath - if type(blobfile)=='string' then - if not dname or find(blobfile,dname) then - local variant=hash.type - local search=filejoin(blobroot,blobfile,bname) - local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + local path,name=lookup(content,basename) + if path then + local metadata=content.metadata + local realroot=metadata and metadata.path or hashname + if type(path)=="string" then + check(hash,realroot,pathname,path,name) else - for kk=1,#blobfile do - local vv=blobfile[kk] - if not dname or find(vv,dname) then - local variant=hash.type - local search=filejoin(blobroot,vv,bname) - local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + for i=1,#path do + check(hash,realroot,pathname,path[i],name) end end end elseif trace_locating then - report_resolving("no match in %a (%s)",blobpath,bname) + report_resolving("no match in %a (%s)",hashname,basename) end end end @@ -14262,7 +15387,7 @@ end local function can_be_dir(name) local fakepaths=instance.fakepaths if not fakepaths[name] then - if lfs.isdir(name) then + if isdir(name) then fakepaths[name]=1 else fakepaths[name]=2 @@ -14278,10 +15403,11 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if ext=="" or not suffixmap[ext] then local defaultsuffixes=resolvers.defaultsuffixes + local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname - filetype=resolvers.formatofsuffix(forcedname) + filetype=formatofsuffix(forcedname) if trace_locating then report_resolving("forcing filetype %a",filetype) end @@ -14317,18 +15443,18 @@ local function find_direct(filename,allresults) end end local function find_wildcard(filename,allresults) - if find(filename,'%*') then + if find(filename,'*',1,true) then if trace_locating then report_resolving("checking wildcard %a",filename) end - local method,result=resolvers.findwildcardfiles(filename) + local result=resolvers.findwildcardfiles(filename) if result then return "wildcard",result end end end local function find_qualified(filename,allresults,askedformat,alsostripped) - if not file.is_qualified_path(filename) then + if not is_qualified_path(filename) then return end if trace_locating then @@ -14402,33 +15528,66 @@ local function check_subpath(fname) return fname end end -local function find_intree(filename,filetype,wantedfiles,allresults) +local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec) - local method="intree" + local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local entry={} if pathlist and #pathlist>0 then - local filelist=collect_files(wantedfiles) + for k=1,#pathlist do + local path=pathlist[k] + local prescanned=find(path,'^!!') + local resursive=find(path,'//$') + local pathname=lpegmatch(inhibitstripper,path) + local expression=makepathexpression(pathname) + local barename=gsub(pathname,"/+$","") + barename=resolveprefix(barename) + local scheme=url.hasscheme(barename) + local schemename=gsub(barename,"%.%*$",'') + entry[k]={ + path=path, + pathname=pathname, + prescanned=prescanned, + recursive=recursive, + expression=expression, + barename=barename, + scheme=scheme, + schemename=schemename, + } + end + entry.typespec=typespec + list[filetype]=entry + else + list[filetype]=false + end + return entry +end +local function find_intree(filename,filetype,wantedfiles,allresults) + local pathlists=instance.pathlists + if not pathlists then + pathlists=setmetatableindex(allocate(),makepathlist) + instance.pathlists=pathlists + end + local pathlist=pathlists[filetype] + if pathlist then + local method="intree" + local filelist=collect_files(wantedfiles) local dirlist={} + local result={} if filelist then for i=1,#filelist do dirlist[i]=filedirname(filelist[i][3]).."/" end end if trace_detail then - report_resolving("checking filename %a",filename) + report_resolving("checking filename %a in tree",filename) end - local resolve=resolvers.resolve - local result={} for k=1,#pathlist do - local path=pathlist[k] - local pathname=lpegmatch(inhibitstripper,path) - local doscan=path==pathname - if not find (pathname,'//$') then - doscan=false - end + local entry=pathlist[k] + local path=entry.path + local pathname=entry.pathname local done=false if filelist then - local expression=makepathexpression(pathname) + local expression=entry.expression if trace_detail then report_resolving("using pattern %a for path %a",expression,pathname) end @@ -14436,8 +15595,8 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local fl=filelist[k] local f=fl[2] local d=dirlist[k] - if find(d,expression) or find(resolve(d),expression) then - result[#result+1]=resolve(fl[3]) + if find(d,expression) or find(resolveprefix(d),expression) then + result[#result+1]=resolveprefix(fl[3]) done=true if allresults then if trace_detail then @@ -14458,56 +15617,62 @@ local function find_intree(filename,filetype,wantedfiles,allresults) method="database" else method="filesystem" - pathname=gsub(pathname,"/+$","") - pathname=resolve(pathname) - local scheme=url.hasscheme(pathname) + local scheme=entry.scheme if not scheme or scheme=="file" then - local pname=gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then + local pname=entry.schemename + if not find(pname,"*",1,true) then if can_be_dir(pname) then - for k=1,#wantedfiles do - local w=wantedfiles[k] - local fname=check_subpath(filejoin(pname,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + if not done and not entry.prescanned then + if trace_detail then + report_resolving("quick root scan for %a",pname) end - end - if not done and doscan then - local files=resolvers.simplescanfiles(pname,false,true) for k=1,#wantedfiles do local w=wantedfiles[k] - local subpath=files[w] - if not subpath or subpath=="" then - elseif type(subpath)=="string" then - local fname=check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break end - else - for i=1,#subpath do - local sp=subpath[i] - if sp=="" then - else - local fname=check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break + end + end + if not done and entry.recursive then + if trace_detail then + report_resolving("scanning filesystem for %a",pname) + end + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end end end end - end - if done and not allresults then - break + if done and not allresults then + break + end end end end @@ -14515,6 +15680,18 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end else end + else + for k=1,#wantedfiles do + local pname=entry.barename + local fname=methodhandler('finders',pname.."/"..wantedfiles[k]) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end end end if done and not allresults then @@ -14549,10 +15726,13 @@ local function find_otherwise(filename,filetype,wantedfiles,allresults) local filelist=collect_files(wantedfiles) local fl=filelist and filelist[1] if fl then - return "otherwise",{ resolvers.resolve(fl[3]) } + return "otherwise",{ resolveprefix(fl[3]) } end end collect_instance_files=function(filename,askedformat,allresults) + if not filename or filename=="" then + return {} + end askedformat=askedformat or "" filename=collapsepath(filename,".") filename=gsub(filename,"^%./",getcurrentdir().."/") @@ -14587,7 +15767,11 @@ collect_instance_files=function(filename,askedformat,allresults) else local method,result,stamp,filetype,wantedfiles if instance.remember then - stamp=formatters["%s--%s"](filename,askedformat) + if askedformat=="" then + stamp=formatters["%s::%s"](suffixonly(filename),filename) + else + stamp=formatters["%s::%s"](askedformat,filename) + end result=stamp and instance.found[stamp] if result then if trace_locating then @@ -14606,7 +15790,7 @@ collect_instance_files=function(filename,askedformat,allresults) method,result=find_intree(filename,filetype,wantedfiles) if not result then method,result=find_onpath(filename,filetype,wantedfiles) - if not result then + if resolve_otherwise and not result then method,result=find_otherwise(filename,filetype,wantedfiles) end end @@ -14622,7 +15806,7 @@ collect_instance_files=function(filename,askedformat,allresults) end if stamp then if trace_locating then - report_resolving("remembering file %a",filename) + report_resolving("remembering file %a using hash %a",filename,stamp) end instance.found[stamp]=result end @@ -14630,6 +15814,9 @@ collect_instance_files=function(filename,askedformat,allresults) end end local function findfiles(filename,filetype,allresults) + if not filename or filename=="" then + return {} + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -14649,39 +15836,30 @@ function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) - local bname,result=filebasename(filename),{} + local base=filebasename(filename) + local result={} local hashes=instance.hashes - local noffound=0 + local function okay(hash,path,name) + local found=methodhandler('concatinators',hash.type,hash.name,path,name) + if found and found~="" then + result[#result+1]=resolveprefix(found) + return not allresults + end + end for k=1,#hashes do local hash=hashes[k] - local files=instance.files[hash.name] or {} - local blist=files[bname] - if not blist then - local rname="remap:"..bname - blist=files[rname] - if blist then - bname=files[rname] - blist=files[bname] - end - end - if blist then - if type(blist)=='string' then - local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then - break - end + local content=instance.files[hash.name] + if content then + local path,name=lookup(content,base) + if not path then + elseif type(path)=="string" then + if okay(hash,path,name) then + return result end else - for kk=1,#blist do - local vv=blist[kk] - local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then break end + for i=1,#path do + if okay(hash,path[i],name) then + return result end end end @@ -14695,64 +15873,80 @@ end function resolvers.findgivenfile(filename) return findgivenfiles(filename,false)[1] or "" end -local function doit(path,blist,bname,tag,variant,result,allresults) - local done=false - if blist and variant then - local resolve=resolvers.resolve - if type(blist)=='string' then - if find(lower(blist),path) then - local full=methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1]=resolve(full) - done=true - end - else - for kk=1,#blist do - local vv=blist[kk] - if find(lower(vv),path) then - local full=methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1]=resolve(full) - done=true - if not allresults then break end - end - end - end - end - return done -end local makewildcard=Cs( (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end -local function findwildcardfiles(filename,allresults,result) - result=result or {} +local function findwildcardfiles(filename,allresults,result) + local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files,done=instance.files,false - if find(name,"%*") then + local files=instance.files + if find(name,"*",1,true) then local hashes=instance.hashes + local function okay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - for kk,hh in next,files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + for found,base in filtered(files[hashname],name) do + if type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end end end end end else + local function okayokay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - if doit(path,files[hashname][base],base,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + local found,base=lookup(content,base) + if not found then + elseif type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end + end + end end end return result @@ -14825,7 +16019,7 @@ end function resolvers.dowithpath(name,func) local pathlist=resolvers.expandedpathlist(name) for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) + func("^"..cleanpath(pathlist[i])) end end function resolvers.dowithvariable(name,func) @@ -14833,23 +16027,23 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fullname=file.addsuffix(barename,"fmt") + local barename=removesuffix(name) + local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then fmtname=resolvers.findfile(fullname) - fmtname=resolvers.cleanpath(fmtname) + fmtname=cleanpath(fmtname) end if fmtname~="" then - local barename=file.removesuffix(fmtname) - local luaname=file.addsuffix(barename,luasuffixes.lua) - local lucname=file.addsuffix(barename,luasuffixes.luc) - local luiname=file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then + local barename=removesuffix(fmtname) + local luaname=addsuffix(barename,luasuffixes.lua) + local lucname=addsuffix(barename,luasuffixes.luc) + local luiname=addsuffix(barename,luasuffixes.lui) + if isfile(luiname) then return barename,luiname - elseif lfs.isfile(lucname) then + elseif isfile(lucname) then return barename,lucname - elseif lfs.isfile(luaname) then + elseif isfile(luaname) then return barename,luaname end end @@ -14871,29 +16065,24 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) local hash=hashes[i] local blobtype=hash.type local blobpath=hash.name - if blobpath then + if blobtype and blobpath then + local total=0 + local checked=0 + local done=0 if before then before(blobtype,blobpath,pattern) end - local files=instance.files[blobpath] - local total,checked,done=0,0,0 - if files then - for k,v in table.sortedhash(files) do - total=total+1 - if find(k,"^remap:") then - elseif find(k,pattern) then - if type(v)=="string" then - checked=checked+1 - if handle(blobtype,blobpath,v,k) then - done=done+1 - end - else - checked=checked+#v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done=done+1 - end - end + for path,name in filtered(instance.files[blobpath],pattern) do + if type(path)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,path,name) then + done=done+1 + end + else + checked=checked+#path + for i=1,#path do + if handle(blobtype,blobpath,path[i],name) then + done=done+1 end end end @@ -14904,8 +16093,8 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -resolvers.obsolete=resolvers.obsolete or {} -local obsolete=resolvers.obsolete +local obsolete=resolvers.obsolete or {} +resolvers.obsolete=obsolete resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles @@ -14916,7 +16105,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 6643, stripped down to: 4401 +-- original size: 3950, stripped down to: 2935 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -14926,44 +16115,51 @@ if not modules then modules={} end modules ['data-pre']={ license="see context related readme files" } local resolvers=resolvers -local prefixes=utilities.storage.allocate() -resolvers.prefixes=prefixes -local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local prefixes=resolvers.prefixes +local cleanpath=resolvers.cleanpath +local findgivenfile=resolvers.findgivenfile +local expansion=resolvers.expansion local getenv=resolvers.getenv -local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match -local joinpath,basename,dirname=file.join,file.basename,file.dirname -local getmetatable,rawset,type=getmetatable,rawset,type +local basename=file.basename +local dirname=file.dirname +local joinpath=file.join +local isfile=lfs.isfile prefixes.environment=function(str) return cleanpath(expansion(str)) end -prefixes.relative=function(str,n) - if io.exists(str) then - elseif io.exists("./"..str) then - str="./"..str - else - local p="../" - for i=1,n or 2 do - if io.exists(p..str) then - str=p..str - break - else - p=p.."../" +local function relative(str,n) + if not isfile(str) then + local pstr="./"..str + if isfile(pstr) then + str=pstr + else + local p="../" + for i=1,n or 2 do + local pstr=p..str + if isfile(pstr) then + str=pstr + break + else + p=p.."../" + end end end end return cleanpath(str) end +local function locate(str) + local fullname=findgivenfile(str) or "" + return cleanpath(fullname~="" and fullname or str) +end +prefixes.relative=relative +prefixes.locate=locate prefixes.auto=function(str) - local fullname=prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname=prefixes.locate(str) + local fullname=relative(str) + if not isfile(fullname) then + fullname=locate(str) end return fullname end -prefixes.locate=function(str) - local fullname=findgivenfile(str) or "" - return cleanpath((fullname~="" and fullname) or str) -end prefixes.filename=function(str) local fullname=findgivenfile(str) or "" return cleanpath(basename((fullname~="" and fullname) or str)) @@ -14984,6 +16180,13 @@ end prefixes.home=function(str) return cleanpath(joinpath(getenv('HOME'),str)) end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname local function toppath() local inputstack=resolvers.inputstack if not inputstack then @@ -14996,98 +16199,22 @@ local function toppath() return pathname end end -resolvers.toppath=toppath -prefixes.toppath=function(str) - return cleanpath(joinpath(toppath(),str)) -end -prefixes.env=prefixes.environment -prefixes.rel=prefixes.relative -prefixes.loc=prefixes.locate -prefixes.kpse=prefixes.locate -prefixes.full=prefixes.locate -prefixes.file=prefixes.filename -prefixes.path=prefixes.pathname -function resolvers.allprefixes(separator) - local all=table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i]=all[i]..":" - end - end - return all -end -local function _resolve_(method,target) - local action=prefixes[method] - if action then - return action(target) - else - return method..":"..target - end -end -local resolved,abstract={},{} -function resolvers.resetresolve(str) - resolved,abstract={},{} -end -local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) -local prefix=C(R("az")^2)*P(":") -local target=C((1-S(" \"\';,"))^1) -local notarget=(#S(";,")+P(-1))*Cc("") -local pattern=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) -local function resolve(str) - if type(str)=="table" then - local t={} - for i=1,#str do - t[i]=resolve(str[i]) - end - return t +local function jobpath() + local path=resolvers.stackpath() + if not path or path=="" then + return "." else - local res=resolved[str] - if not res then - res=lpegmatch(pattern,str) - resolved[str]=res - abstract[res]=str - end - return res - end -end -local function unresolve(str) - return abstract[str] or str -end -resolvers.resolve=resolve -resolvers.unresolve=unresolve -if type(os.uname)=="function" then - for k,v in next,os.uname() do - if not prefixes[k] then - prefixes[k]=function() return v end - end - end -end -if os.type=="unix" then - local pattern - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon=P(":") - for k,v in table.sortedpairs(prefixes) do - if p then - p=P(k)+p - else - p=P(k) - end - end - pattern=Cs((p*colon+colon/";"+P(1))^0) - end - makepattern() - getmetatable(prefixes).__newindex=makepattern - function resolvers.repath(str) - return lpegmatch(pattern,str) - end -else - function resolvers.repath(str) - return str + return path end end +resolvers.toppath=toppath +resolvers.jobpath=jobpath +prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end +prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end +resolvers.setdynamic("toppath") +resolvers.setdynamic("jobpath") +prefixes.jobfile=prefixes.jobpath +resolvers.setdynamic("jobfile") end -- of closure @@ -15149,7 +16276,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3801, stripped down to: 3231 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -15161,30 +16288,31 @@ if not modules then modules={} end modules ['data-fil']={ local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers +local resolveprefix=resolvers.resolve local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name=specification.filename - local realname=resolvers.resolve(name) + local filename=specification.filename + local realname=resolveprefix(filename) if realname and realname~='' and lfs.isdir(realname) then if trace_locating then - report_files("file locator %a found as %a",name,realname) + report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',name,true) + resolvers.appendhash('file',filename,true) elseif trace_locating then - report_files("file locator %a not found",name) + report_files("file locator %a not found",filename) end end function hashers.file(specification) - local name=specification.filename - local content=caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local pathname=specification.filename + local content=caches.loadcontent(pathname,'files') + resolvers.registerfilehash(pathname,content,content==nil) end function generators.file(specification) - local path=specification.filename - local content=resolvers.scanfiles(path,false,true) - resolvers.registerfilehash(path,content,true) + local pathname=specification.filename + local content=resolvers.scanfiles(pathname,false,true) + resolvers.registerfilehash(pathname,content,true) end concatinators.file=file.join function finders.file(specification,filetype) @@ -15375,7 +16503,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3913, stripped down to: 2998 +-- original size: 3899, stripped down to: 2984 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -15421,7 +16549,7 @@ end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile) - local enginebanner=status.list().banner + local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") local luvdata={ @@ -15434,7 +16562,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) end end function statistics.checkfmtstatus(texname) - local enginebanner=status.list().banner + local enginebanner=status.banner if enginebanner and texname then local luvname=file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then @@ -15466,7 +16594,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8489, stripped down to: 6757 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -15485,16 +16613,6 @@ zip.archives=zip.archives or {} local archives=zip.archives zip.registeredfiles=zip.registeredfiles or {} local registeredfiles=zip.registeredfiles -local limited=false -directives.register("system.inputmode",function(v) - if not limited then - local i_limiter=io.i_limiter(v) - if i_limiter then - zip.open=i_limiter.protect(zip.open) - limited=true - end - end -end) local function validzip(str) if not find(str,"^zip://") then return "zip:///"..str @@ -15509,7 +16627,7 @@ function zip.openarchive(name) local arch=archives[name] if not arch then local full=resolvers.findfile(name) or "" - arch=(full~="" and zip.open(full)) or false + arch=full~="" and zip.open(full) or false archives[name]=arch end return arch @@ -15668,31 +16786,42 @@ function resolvers.usezipfile(archive) end end function resolvers.registerzipfile(z,tree) - local files,filter={},"" - if tree=="" then - filter="^(.+)/(.-)$" - else - filter=format("^%s/(.+)/(.-)$",tree) - end + local names={} + local files={} + local remap={} + local n=0 + local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) + local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end - local register,n=resolvers.registerfile,0 for i in z:files() do - local path,name=match(i.filename,filter) - if path then - if name and name~='' then - register(files,name,path) - n=n+1 - else + local filename=i.filename + local path,name=match(filename,filter) + if not path then + n=n+1 + register(names,filename,"") + local usedname=lower(filename) + files[usedname]="" + if usedname~=filename then + remap[usedname]=filename end - else - register(files,i.filename,'') + elseif name and name~="" then n=n+1 + register(names,name,path) + local usedname=lower(name) + files[usedname]=path + if usedname~=name then + remap[usedname]=name + end + else end end report_zip("registering: %s files registered",n) - return files + return { + files=files, + remap=remap, + } end @@ -15702,7 +16831,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 2508, stripped down to: 2074 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -15711,42 +16840,64 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,gsub,format=string.find,string.gsub,string.format +local find,gsub,lower=string.find,string.gsub,string.lower +local basename,dirname,joinname=file.basename,file.dirname,file .join +local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile +local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local done,found,notfound={},{},resolvers.finders.notfound -function resolvers.finders.tree(specification) +local resolveprefix=resolvers.resolve +local notfound=resolvers.finders.notfound +local lookup=resolvers.get_from_content +local collectors={} +local found={} +function resolvers.finders.tree(specification) local spec=specification.filename - local fnd=found[spec] - if fnd==nil then + local okay=found[spec] + if okay==nil then if spec~="" then - local path,name=file.dirname(spec),file.basename(spec) - if path=="" then path="." end - local hash=done[path] - if not hash then - local pattern=path.."/*" - hash=dir.glob(pattern) - done[path]=hash + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names end local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for k=1,#hash do - local v=hash[k] - if find(v,pattern) then - found[spec]=v - return v + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then + found[spec]=fullname + return fullname + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end - fnd=notfound() - found[spec]=fnd + okay=notfound() + found[spec]=okay end - return fnd + return okay end function resolvers.locators.tree(specification) local name=specification.filename - local realname=resolvers.resolve(name) - if realname and realname~='' and lfs.isdir(realname) then + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then if trace_locating then report_trees("locator %a found",realname) end @@ -15757,16 +16908,110 @@ function resolvers.locators.tree(specification) end function resolvers.hashers.tree(specification) local name=specification.filename - if trace_locating then - report_trees("analysing %a",name) - end + report_trees("analyzing %a",name) resolvers.methodhandler("hashers",name) resolvers.generators.file(specification) end -resolvers.concatinators.tree=resolvers.concatinators.file -resolvers.generators.tree=resolvers.generators.file -resolvers.openers.tree=resolvers.openers.file -resolvers.loaders.tree=resolvers.loaders.file +local collectors={} +local splitter=lpeg.splitat("/**/") +local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } +table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=caches.loadcontent(dataname,"files",dataname) + if not content then + content=resolvers.scanfiles(rootname,nil,nil,false,true) + caches.savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content +end) +local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + end + return notfound() +end +local function resolve(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end + end + end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() + end + end + end + local path,name=dirname(filename),basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) + end + end + return notfound() +end +resolvers.finders .dirlist=resolve +resolvers.locators .dirlist=resolvers.locators .tree +resolvers.hashers .dirlist=resolvers.hashers .tree +resolvers.generators.dirlist=resolvers.generators.file +resolvers.openers .dirlist=resolvers.openers .file +resolvers.loaders .dirlist=resolvers.loaders .file +function resolvers.finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } + end + return resolve(specification) +end +resolvers.locators .dirfile=resolvers.locators .dirlist +resolvers.hashers .dirfile=resolvers.hashers .dirlist +resolvers.generators.dirfile=resolvers.generators.dirlist +resolvers.openers .dirfile=resolvers.openers .dirlist +resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -15775,7 +17020,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6202, stripped down to: 5149 +-- original size: 6569, stripped down to: 5304 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -15801,8 +17046,13 @@ directives.register("schemes.threshold",function(v) threshold=tonumber(v) or thr function cleaners.none(specification) return specification.original end -function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) +function cleaners.strip(specification) + local path,name=file.splitbase(specification.original) + if path=="" then + return (gsub(name,"[^%a%d%.]+","-")) + else + return (gsub((gsub(path,"%.","-").."-"..name),"[^%a%d%.]+","-")) + end end function cleaners.md5(specification) return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) @@ -15818,8 +17068,8 @@ function resolvers.schemes.cleanname(specification) end local cached,loaded,reused,thresholds,handlers={},{},{},{},{} local function runcurl(name,cachename) - local command="curl --silent --create-dirs --output "..cachename.." "..name - os.spawn(command) + local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name + os.execute(command) end local function fetch(specification) local original=specification.original @@ -15951,7 +17201,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4237, stripped down to: 3177 +-- original size: 4313, stripped down to: 3227 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -15960,7 +17210,7 @@ if not modules then modules={} end modules ['data-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local resolvers,package=resolvers,package +local package,lpeg=package,lpeg local gsub=string.gsub local concat=table.concat local addsuffix=file.addsuffix @@ -15971,9 +17221,11 @@ local luaformats={ 'TEXINPUTS','LUAINPUTS' } local libformats={ 'CLUAINPUTS' } local helpers=package.helpers or {} local methods=helpers.methods or {} +local resolvers=resolvers +local resolveprefix=resolvers.resolve +helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) -helpers.report=logs.reporter("resolvers","libraries") helpers.sequence={ "already loaded", "preload table", @@ -15988,7 +17240,7 @@ helpers.sequence={ } local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) function helpers.cleanpath(path) - return resolvers.resolve(lpegmatch(pattern,path)) + return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib local getextraluapaths=package.extraluapaths @@ -16058,7 +17310,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2394, stripped down to: 2005 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -16072,8 +17324,8 @@ local type,next=type,next local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers local report_scripts=logs.reporter("resolvers","scripts") -function resolvers.updatescript(oldname,newname) - local scriptpath="scripts/context/lua" +function resolvers.updatescript(oldname,newname) + local scriptpath="context/lua" newname=file.addsuffix(newname,"lua") local oldscript=resolvers.cleanpath(oldname) if trace_locating then @@ -16125,7 +17377,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2600, stripped down to: 1627 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -16181,7 +17433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2654, stripped down to: 2301 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -16190,10 +17442,13 @@ if not modules then modules={} end modules ['data-lst']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local rawget,type,next=rawget,type,next +local find,concat,upper=string.find,table.concat,string.upper local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs -resolvers.listers=resolvers.listers or {} local resolvers=resolvers +local listers=resolvers.listers or {} +resolvers.listers=listers +local resolveprefix=resolvers.resolve local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) if type(str)=='table' then @@ -16202,7 +17457,7 @@ local function tabstr(str) return str end end -function resolvers.listers.variables(pattern) +function listers.variables(pattern) local instance=resolvers.instance local environment=instance.environment local variables=instance.variables @@ -16223,10 +17478,10 @@ function resolvers.listers.variables(pattern) for key,value in sortedpairs(configured) do if key~="" and (pattern=="" or find(upper(key),pattern)) then report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolveprefix(expansions[key])) or "unset") end end instance.environment=fastcopy(env) @@ -16234,15 +17489,15 @@ function resolvers.listers.variables(pattern) instance.expansions=fastcopy(exp) end local report_resolved=logs.reporter("system","resolved") -function resolvers.listers.configurations() +function listers.configurations() local configurations=resolvers.instance.specification for i=1,#configurations do - report_resolved("file : %s",resolvers.resolve(configurations[i])) + report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) for i=1,#list do - local li=resolvers.resolve(list[i]) + local li=resolveprefix(list[i]) if lfs.isdir(li) then report_resolved("path - %s",li) else @@ -16547,7 +17802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5951, stripped down to: 4922 +-- original size: 5955, stripped down to: 4926 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -16635,7 +17890,7 @@ function environment.make_format(name) end local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") report_format("running command: %s\n",command) - os.spawn(command) + os.execute(command) local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -16670,7 +17925,7 @@ function environment.run_format(name,data,more) else local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") report_format("running command: %s",command) - os.spawn(command) + os.execute(command) end end end @@ -16681,8 +17936,8 @@ end -- of closure -- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 685064 --- stripped bytes : 242353 +-- original bytes : 745618 +-- stripped bytes : 269191 -- end library merge @@ -16781,17 +18036,18 @@ local ownlibs = { -- order can be made better } +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/context/sources/data-tmf.lua + local ownlist = { - '.', - ownpath , - ownpath .. "/../sources", -- HH's development path + -- '.', + -- ownpath , + owntree .. "/../../../../context/sources", -- HH's development path owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", - owntree .. "/../../texmf-dist/tex/context/base", owntree .. "/../../texmf/tex/context/base", owntree .. "/../../../texmf-local/tex/context/base", owntree .. "/../../../texmf-context/tex/context/base", - owntree .. "/../../../texmf-dist/tex/context/base", owntree .. "/../../../texmf/tex/context/base", } @@ -16907,6 +18163,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -16932,6 +18189,7 @@ local helpinfo = [[ <flag name="verbose"><short>give a bit more info</short></flag> <flag name="trackers" value="list"><short>enable given trackers</short></flag> <flag name="progname" value="str"><short>format or backend</short></flag> + <flag name="systeminfo" value="str"><short>show current operating system, processor, etc</short></flag> </subcategory> <subcategory> <flag name="edit"><short>launch editor with found file</short></flag> @@ -17561,6 +18819,39 @@ function runners.associate(filename) os.launch(filename) end +function runners.evaluate(code,filename) -- for Luigi + if code == "loop" then + while true do + io.write("> ") + local code = io.read() + if code ~= "" then + local temp = string.match(code,"^= (.*)$") + if temp then + code = "print("..temp..")" + end + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("! " .. (message or code).."\n") + else + io.write(compiled()) + end + end + end + else + if type(code) ~= "string" or code == "" then + code = filename + end + if code ~= "" then + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("invalid lua code: " .. (message or code)) + return + end + io.write(compiled()) + end + end +end + function runners.gethelp(filename) local url = environment.argument("url") if url and url ~= "" then @@ -17572,6 +18863,15 @@ function runners.gethelp(filename) end end +function runners.systeminfo() + report("architecture : %s",os.platform or "<unset>") + report("operating system : %s",os.name or "<unset>") + report("file architecture : %s",os.type or "<unset>") + report("binary path : %s",os.selfdir or "<unset>") + report("binary suffix : %s",os.binsuffix or "<unset>") + report("library suffix : %s",os.libsuffix or "<unset>") +end + -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script -- ... later we will use the second half @@ -17687,16 +18987,13 @@ end if e_argument("ansi") then - local formatters = string.formatters + logs.setformatters("ansi") - logs.setformatters { - report_yes = formatters["[1;32m%-15s [0;1m|[0m %s"], - report_nop = formatters["[1;32m%-15s [0;1m|[0m"], - subreport_yes = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m %s"], - subreport_nop = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m"], - status_yes = formatters["[1;32m%-15s [0;1m:[0m %s\n"], - status_nop = formatters["[1;32m%-15s [0;1m:[0m\n"], - } + local script = e_argument("script") or e_argument("scripts") + + if type(script) == "string" then + logs.writer("]0;"..script.."") -- for Alan to test + end end @@ -17715,14 +19012,26 @@ if e_argument("script") or e_argument("scripts") then ok = runners.execute_ctx_script(filename) end +elseif e_argument("evaluate") then + + runners.evaluate(e_argument("evaluate"),filename) + elseif e_argument("selfmerge") then -- embed used libraries runners.loadbase() local found = locate_libs() + if found then - utilities.merger.selfmerge(own.name,own.libs,{ found }) + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfmerge(mtxrun,own.libs,{ found }) + application.report("runner updated on resolved path: %s",mtxrun) + else + utilities.merger.selfmerge(own.name,own.libs,{ found }) + application.report("runner updated on relative path: %s",own.name) + end end elseif e_argument("selfclean") then @@ -17730,7 +19039,15 @@ elseif e_argument("selfclean") then -- remove embedded libraries runners.loadbase() - utilities.merger.selfclean(own.name) + + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfclean(mtxrun) + application.report("runner cleaned on resolved path: %s",mtxrun) + else + utilities.merger.selfclean(own.name) + application.report("runner cleaned on relative path: %s",own.name) + end elseif e_argument("selfupdate") then @@ -17972,6 +19289,8 @@ elseif e_argument("version") then application.version() + application.report("source path",environment.ownbin) + elseif e_argument("directives") then directives.show() @@ -17989,6 +19308,10 @@ elseif e_argument("exporthelp") then runners.loadbase() application.export(e_argument("exporthelp"),filename) +elseif e_argument("systeminfo") then + + runners.systeminfo() + elseif e_argument("help") or filename=='help' or filename == "" then application.help() diff --git a/scripts/context/ruby/texexec.rb b/scripts/context/ruby/texexec.rb index c673cb46b..7f8298c09 100644 --- a/scripts/context/ruby/texexec.rb +++ b/scripts/context/ruby/texexec.rb @@ -685,22 +685,22 @@ end # so far for compatibility, will move to tex -@@extrastringvars = [ +extrastringvars = [ 'pages', 'background', 'backspace', 'topspace', 'boxtype', 'tempdir','bannerheight', 'printformat', 'method', 'scale', 'selection', 'combination', 'textwidth', 'addempty', 'logfile', 'startline', 'endline', 'startcolumn', 'endcolumn', 'scale' ] -@@extrabooleanvars = [ +extrabooleanvars = [ 'centerpage', 'noduplex', 'color', 'pretty', 'fullscreen', 'screensaver', 'markings' ] if job = TEX.new(logger) then - job.setextrastringvars(@@extrastringvars) - job.setextrabooleanvars(@@extrabooleanvars) + job.setextrastringvars(extrastringvars) + job.setextrabooleanvars(extrabooleanvars) job.booleanvars.each do |k| commandline.registerflag(k) diff --git a/scripts/context/stubs/mswin/first-setup.bat b/scripts/context/stubs/install/first-setup.bat index f06ad0e6b..f06ad0e6b 100644 --- a/scripts/context/stubs/mswin/first-setup.bat +++ b/scripts/context/stubs/install/first-setup.bat diff --git a/scripts/context/stubs/install/first-setup.sh b/scripts/context/stubs/install/first-setup.sh new file mode 100644 index 000000000..9249fd2e0 --- /dev/null +++ b/scripts/context/stubs/install/first-setup.sh @@ -0,0 +1,120 @@ +#!/bin/sh + +# Takes the same arguments as mtx-update + +# you may change this if you want ... +CONTEXTROOT="$PWD/tex" + +# suggested by Tobias Florek to check for ruby & rsync +if [ ! -x "`which rsync`" ]; then + echo "You need to install rsync first." + exit 1 +fi +if [ ! -x "`which ruby`" ]; then + echo "You might want to install Ruby first if you want to use pdfTeX or XeTeX." +fi + +system=`uname -s` +cpu=`uname -m` + +case "$system" in + # linux + Linux) + case "$cpu" in + i*86) platform="linux" ;; + x86_64|ia64) platform="linux-64" ;; + # a little bit of cheating with ppc64 (won't work on Gentoo) + ppc|ppc64) platform="linux-ppc" ;; + # we currently support just mipsel, but Debian is lying (reports mips64) + # we need more hacks to fix the situation, this is just a temporary solution + mips|mips64|mipsel|mips64el) platform="linux-mipsel" ;; + *) platform="unknown" ;; + esac ;; + # Mac OS X + Darwin) + case "$cpu" in + i*86) platform="osx-intel" ;; + x86_64) platform="osx-64" ;; + ppc*|powerpc|power*|Power*) platform="osx-ppc" ;; + *) platform="unknown" ;; + esac ;; + # FreeBSD + FreeBSD|freebsd) + case "$cpu" in + i*86) platform="freebsd" ;; + x86_64) platform="freebsd" ;; # no special binaries are available yet + amd64) platform="freebsd-amd64" ;; + *) platform="unknown" ;; + esac ;; + # kFreeBSD (debian) + GNU/kFreeBSD) + case "$cpu" in + i*86) platform="kfreebsd-i386" ;; + x86_64|amd64) platform="kfreebsd-amd64" ;; + *) platform="unknown" ;; + esac ;; + # cygwin + CYGWIN*) + case "$cpu" in + i*86) platform="cygwin" ;; + x86_64|ia64) platform="cygwin-64" ;; + *) platform="unknown" ;; + esac ;; + # SunOS/Solaris + SunOS) + case "$cpu" in + sparc) platform="solaris-sparc" ;; + i86pc) platform="solaris-intel" ;; + *) platform="unknown" ;; + esac ;; + *) platform="unknown" +esac + +# temporary patch for 64-bit Leopard with 32-bit kernel +if test "$platform" = "osx-intel"; then + # if running Snow Leopard or later + # better: /usr/bin/sw_vers -productVersion + if test `uname -r|cut -f1 -d"."` -ge 10 ; then + # if working on 64-bit hardware + if test `sysctl -n hw.cpu64bit_capable` = 1; then + # snowleopard32=TRUE + platform="osx-64" + fi + fi +fi + +if test "$platform" = "unknown" ; then + echo "Error: your system \"$system $cpu\" is not supported yet." + echo "Please report to the ConTeXt mailing-list (ntg-context@ntg.nl)" + exit +fi + +# if you want to enforce some specific platform +# (when 'uname' doesn't agree with true architecture), uncomment and modify next line: +# platform=linux + +# download or rsync the latest scripts first +rsync -rlptv rsync://contextgarden.net/minimals/setup/$platform/bin . + +# download or update the distribution +# you may remove the --context=beta switch if you want to use "current" +# you can use --engine=luatex if you want just mkiv +env PATH="$PWD/bin:$CONTEXTROOT/texmf-$platform/bin:$PATH" \ +mtxrun --script ./bin/mtx-update.lua --force --update --make --context=beta --platform=$platform --texroot="$CONTEXTROOT" $@ + +echo +echo "When you want to use context, you need to initialize the tree by typing:" +echo +echo " . $CONTEXTROOT/setuptex" +echo +echo "in your shell or add" +echo " \"$CONTEXTROOT/texmf-$platform/bin\"" +echo "to PATH variable if you want to set it permanently." +echo "This can usually be done in .bashrc, .bash_profile" +echo "(or whatever file is used to initialize your shell)." +echo + +if [ ! -x "`which ruby`" ]; then + echo "You might want to install Ruby first if you want to use pdfTeX or XeTeX." + echo +fi diff --git a/scripts/context/stubs/mswin/context.exe b/scripts/context/stubs/mswin/context.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/context.exe +++ b/scripts/context/stubs/mswin/context.exe diff --git a/scripts/context/stubs/mswin/ctxtools.exe b/scripts/context/stubs/mswin/ctxtools.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/ctxtools.exe +++ b/scripts/context/stubs/mswin/ctxtools.exe diff --git a/scripts/context/stubs/mswin/luatools.exe b/scripts/context/stubs/mswin/luatools.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/luatools.exe +++ b/scripts/context/stubs/mswin/luatools.exe diff --git a/scripts/context/stubs/mswin/metatex.exe b/scripts/context/stubs/mswin/metatex.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/metatex.exe +++ b/scripts/context/stubs/mswin/metatex.exe diff --git a/scripts/context/stubs/mswin/mptopdf.exe b/scripts/context/stubs/mswin/mptopdf.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/mptopdf.exe +++ b/scripts/context/stubs/mswin/mptopdf.exe diff --git a/scripts/context/stubs/mswin/mtxrun.dll b/scripts/context/stubs/mswin/mtxrun.dll Binary files differindex 5a79e1bad..3c4481c31 100644 --- a/scripts/context/stubs/mswin/mtxrun.dll +++ b/scripts/context/stubs/mswin/mtxrun.dll diff --git a/scripts/context/stubs/mswin/mtxrun.exe b/scripts/context/stubs/mswin/mtxrun.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/mtxrun.exe +++ b/scripts/context/stubs/mswin/mtxrun.exe diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 0ff2d2897..edfeba8dd 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3123, stripped down to: 1694 +-- original size: 3888, stripped down to: 2197 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -136,6 +136,16 @@ function optionalrequire(...) return result end end +if lua then + lua.mask=load([[τεχ = 1]]) and "utf" or "ascii" +end +local flush=io.flush +if flush then + local execute=os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec=os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end +end end -- of closure @@ -434,7 +444,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 29245, stripped down to: 15964 +-- original size: 36977, stripped down to: 20349 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -450,7 +460,9 @@ local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.forma local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print -setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +if setinspector then + setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns local anything=P(1) @@ -469,7 +481,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=crlf+S("\r\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -491,8 +503,10 @@ patterns.utfbom_32_le=utfbom_32_le patterns.utfbom_16_be=utfbom_16_be patterns.utfbom_16_le=utfbom_16_le patterns.utfbom_8=utfbom_8 -patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") -patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") +patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_32_be_nl=P("\000\000\000\r\000\000\000\n")+P("\000\000\000\r")+P("\000\000\000\n") +patterns.utf_32_le_nl=P("\r\000\000\000\n\000\000\000")+P("\r\000\000\000")+P("\n\000\000\000") patterns.utf8one=R("\000\127") patterns.utf8two=R("\194\223")*utf8next patterns.utf8three=R("\224\239")*utf8next*utf8next @@ -519,10 +533,24 @@ patterns.spacer=spacer patterns.whitespace=whitespace patterns.nonspacer=nonspacer patterns.nonwhitespace=nonwhitespace -local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) +local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) +local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) +local b_stripper=Cs(spacer^0/""*(nonspacer^1+spacer^1/" ")^0) +local e_stripper=Cs((spacer^1*P(-1)/""+nonspacer^1+spacer^1/" ")^0) +local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper +patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.b_collapser=b_collapser +patterns.m_collapser=m_collapser +patterns.e_collapser=e_collapser +patterns.b_stripper=b_stripper +patterns.m_stripper=m_stripper +patterns.e_stripper=e_stripper patterns.lowercase=lowercase patterns.uppercase=uppercase patterns.letter=patterns.lowercase+patterns.uppercase @@ -559,9 +587,12 @@ patterns.integer=sign^-1*digit^1 patterns.unsigned=digit^0*period*digit^1 patterns.float=sign^-1*patterns.unsigned patterns.cunsigned=digit^0*comma*digit^1 +patterns.cpunsigned=digit^0*(period+comma)*digit^1 patterns.cfloat=sign^-1*patterns.cunsigned +patterns.cpfloat=sign^-1*patterns.cpunsigned patterns.number=patterns.float+patterns.integer patterns.cnumber=patterns.cfloat+patterns.integer +patterns.cpnumber=patterns.cpfloat+patterns.integer patterns.oct=zero*octdigit^1 patterns.octal=patterns.oct patterns.HEX=zero*P("X")*(digit+uppercase)^1 @@ -744,7 +775,7 @@ function lpeg.replacer(one,two,makefunction,isutf) return pattern end end -function lpeg.finder(lst,makefunction) +function lpeg.finder(lst,makefunction,isutf) local pattern if type(lst)=="table" then pattern=P(false) @@ -760,7 +791,11 @@ function lpeg.finder(lst,makefunction) else pattern=P(lst) end - pattern=(1-pattern)^0*pattern + if isutf then + pattern=((utf8char or 1)-pattern)^0*pattern + else + pattern=(1-pattern)^0*pattern + end if makefunction then return function(str) return lpegmatch(pattern,str) @@ -974,37 +1009,139 @@ function lpeg.append(list,pp,delayed,checked) end return p end +local p_false=P(false) +local p_true=P(true) local function make(t) - local p + local function making(t) + local p=p_false + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then + else + p=p+P(k)*making(v) + end + end + end + if t[""] then + p=p+p_true + end + return p + end + local p=p_false local keys=sortedkeys(t) for i=1,#keys do local k=keys[i] - local v=t[k] - if not p then - if next(v) then - p=P(k)*make(v) + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then else - p=P(k) + p=p+P(k)*making(v) end - else - if next(v) then - p=p+P(k)*make(v) + end + end + return p +end +local function collapse(t,x) + if type(t)~="table" then + return t,x + else + local n=next(t) + if n==nil then + return t,x + elseif next(t,n)==nil then + local k=n + local v=t[k] + if type(v)=="table" then + return collapse(v,x..k) else - p=p+P(k) + return v,x..k + end + else + local tt={} + for k,v in next,t do + local vv,kk=collapse(v,k) + tt[kk]=vv end + return tt,x end end - return p end function lpeg.utfchartabletopattern(list) local tree={} - for i=1,#list do - local t=tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c]={} + local n=#list + if n==0 then + for s in next,list do + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true + end + end + else + for i=1,n do + local s=list[i] + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true end - t=t[c] end end return make(tree) @@ -1044,6 +1181,65 @@ local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") local number=digit^1*(case_1+case_2) local stripper=Cs((number+1)^0) lpeg.patterns.stripzeros=stripper +local byte_to_HEX={} +local byte_to_hex={} +local byte_to_dec={} +local hex_to_byte={} +for i=0,255 do + local H=format("%02X",i) + local h=format("%02x",i) + local d=format("%03i",i) + local c=char(i) + byte_to_HEX[c]=H + byte_to_hex[c]=h + byte_to_dec[c]=d + hex_to_byte[h]=c + hex_to_byte[H]=c +end +local hextobyte=P(2)/hex_to_byte +local bytetoHEX=P(1)/byte_to_HEX +local bytetohex=P(1)/byte_to_hex +local bytetodec=P(1)/byte_to_dec +local hextobytes=Cs(hextobyte^0) +local bytestoHEX=Cs(bytetoHEX^0) +local bytestohex=Cs(bytetohex^0) +local bytestodec=Cs(bytetodec^0) +patterns.hextobyte=hextobyte +patterns.bytetoHEX=bytetoHEX +patterns.bytetohex=bytetohex +patterns.bytetodec=bytetodec +patterns.hextobytes=hextobytes +patterns.bytestoHEX=bytestoHEX +patterns.bytestohex=bytestohex +patterns.bytestodec=bytestodec +function string.toHEX(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestoHEX,s) + end +end +function string.tohex(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestohex,s) + end +end +function string.todec(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestodec,s) + end +end +function string.tobytes(s) + if not s or s=="" then + return s + else + return lpegmatch(hextobytes,s) + end +end end -- of closure @@ -1071,7 +1267,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5547, stripped down to: 2708 +-- original size: 5694, stripped down to: 2827 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1107,11 +1303,15 @@ function string.limit(str,n,sentinel) end end local stripper=patterns.stripper +local fullstripper=patterns.fullstripper local collapser=patterns.collapser local longtostring=patterns.longtostring function string.strip(str) return lpegmatch(stripper,str) or "" end +function string.fullstrip(str) + return lpegmatch(fullstripper,str) or "" +end function string.collapsespaces(str) return lpegmatch(collapser,str) or "" end @@ -1172,7 +1372,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 31113, stripped down to: 20256 +-- original size: 35724, stripped down to: 21525 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1205,7 +1405,7 @@ end function table.keys(t) if t then local keys,k={},0 - for key,_ in next,t do + for key in next,t do k=k+1 keys[k]=key end @@ -1215,32 +1415,52 @@ function table.keys(t) end end local function compare(a,b) - local ta,tb=type(a),type(b) - if ta==tb then - return a<b - else - return tostring(a)<tostring(b) + local ta=type(a) + if ta=="number" then + local tb=type(b) + if ta==tb then + return a<b + elseif tb=="string" then + return tostring(a)<b + end + elseif ta=="string" then + local tb=type(b) + if ta==tb then + return a<b + else + return a<tostring(b) + end end + return tostring(a)<tostring(b) end local function sortedkeys(tab) if tab then local srt,category,s={},0,0 - for key,_ in next,tab do + for key in next,tab do s=s+1 srt[s]=key if category==3 then + elseif category==1 then + if type(key)~="string" then + category=3 + end + elseif category==2 then + if type(key)~="number" then + category=3 + end else local tkey=type(key) if tkey=="string" then - category=(category==2 and 3) or 1 + category=1 elseif tkey=="number" then - category=(category==1 and 3) or 2 + category=2 else category=3 end end end - if category==0 or category==3 then + if s<2 then + elseif category==3 then sort(srt,compare) else sort(srt) @@ -1250,16 +1470,52 @@ local function sortedkeys(tab) return {} end end +local function sortedhashonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="string" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end +local function sortedindexonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="number" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end local function sortedhashkeys(tab,cmp) if tab then local srt,s={},0 - for key,_ in next,tab do + for key in next,tab do if key then s=s+1 srt[s]=key end end - sort(srt,cmp) + if s>1 then + sort(srt,cmp) + end return srt else return {} @@ -1268,13 +1524,15 @@ end function table.allkeys(t) local keys={} for k,v in next,t do - for k,v in next,v do + for k in next,v do keys[k]=true end end return sortedkeys(keys) end table.sortedkeys=sortedkeys +table.sortedhashonly=sortedhashonly +table.sortedindexonly=sortedindexonly table.sortedhashkeys=sortedhashkeys local function nothing() end local function sortedhash(t,cmp) @@ -1285,19 +1543,21 @@ local function sortedhash(t,cmp) else s=sortedkeys(t) end - local n=0 local m=#s - local function kv(s) - if n<m then - n=n+1 - local k=s[n] - return k,t[k] + if m==1 then + return next,t + elseif m>0 then + local n=0 + return function() + if n<m then + n=n+1 + local k=s[n] + return k,t[k] + end end end - return kv,s - else - return nothing end + return nothing end table.sortedhash=sortedhash table.sortedpairs=sortedhash @@ -1439,39 +1699,36 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,reduce,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } local function simple_table(t) - if #t>0 then + local nt=#t + if nt>0 then local n=0 for _,v in next,t do n=n+1 end - if n==#t then - local tt,nt={},0 - for i=1,#t do + if n==nt then + local tt={} + for i=1,nt do local v=t[i] local tv=type(v) if tv=="number" then - nt=nt+1 if hexify then - tt[nt]=format("0x%04X",v) + tt[i]=format("0x%X",v) else - tt[nt]=tostring(v) + tt[i]=tostring(v) end elseif tv=="string" then - nt=nt+1 - tt[nt]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then - nt=nt+1 - tt[nt]=v and "true" or "false" + tt[i]=v and "true" or "false" else - tt=nil - break + return nil end end return tt @@ -1490,7 +1747,7 @@ local function do_serialize(root,name,depth,level,indexed) local tn=type(name) if tn=="number" then if hexify then - handle(format("%s[0x%04X]={",depth,name)) + handle(format("%s[0x%X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end @@ -1507,7 +1764,7 @@ local function do_serialize(root,name,depth,level,indexed) end end end - if root and next(root) then + if root and next(root)~=nil then local first,last=nil,0 if compact then last=#root @@ -1525,22 +1782,19 @@ local function do_serialize(root,name,depth,level,indexed) for i=1,#sk do local k=sk[i] local v=root[k] - local tv,tk=type(v),type(k) + local tv=type(v) + local tk=type(k) if compact and first and tk=="number" and k>=first and k<=last then if tv=="number" then if hexify then - handle(format("%s 0x%04X,",depth,v)) + handle(format("%s 0x%X,",depth,v)) else handle(format("%s %s,",depth,v)) end elseif tv=="string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end + handle(format("%s %q,",depth,v)) elseif tv=="table" then - if not next(v) then + if next(v)==nil then handle(format("%s {},",depth)) elseif inline then local st=simple_table(v) @@ -1570,64 +1824,48 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="number" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + handle(format("%s [0x%X]=0x%X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then - handle(format("%s [%s]=0x%04X,",depth,k and "true" or "false",v)) + handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) + handle(format("%s %s=0x%X,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) + handle(format("%s [%q]=0x%X,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end end elseif tv=="string" then - if reduce and tonumber(v) then - if tk=="number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk=="boolean" then - handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,v)) + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=%q,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) else - if tk=="number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk=="boolean" then - handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end + handle(format("%s [%q]=%q,",depth,k,v)) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={},",depth,k)) + handle(format("%s [0x%X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -1643,7 +1881,7 @@ local function do_serialize(root,name,depth,level,indexed) if st then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -1663,7 +1901,7 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="boolean" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v and "true" or "false")) + handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -1679,7 +1917,7 @@ local function do_serialize(root,name,depth,level,indexed) local f=getinfo(v).what=="C" and dump(dummy) or dump(v) if tk=="number" then if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) + handle(format("%s [0x%X]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -1694,7 +1932,7 @@ local function do_serialize(root,name,depth,level,indexed) else if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -1718,7 +1956,6 @@ local function serialize(_handle,root,name,specification) noquotes=specification.noquotes hexify=specification.hexify handle=_handle or specification.handle or print - reduce=specification.reduce or false functions=specification.functions compact=specification.compact inline=specification.inline and compact @@ -1735,7 +1972,6 @@ local function serialize(_handle,root,name,specification) noquotes=false hexify=false handle=_handle or print - reduce=false compact=true inline=true functions=true @@ -1748,7 +1984,7 @@ local function serialize(_handle,root,name,specification) end elseif tname=="number" then if hexify then - handle(format("[0x%04X]={",name)) + handle(format("[0x%X]={",name)) else handle("["..name.."]={") end @@ -1766,7 +2002,7 @@ local function serialize(_handle,root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,"",0) end end @@ -1895,14 +2131,25 @@ local function identical(a,b) end table.identical=identical table.are_equal=are_equal -function table.compact(t) - if t then - for k,v in next,t do - if not next(v) then - t[k]=nil +local function sparse(old,nest,keeptables) + local new={} + for k,v in next,old do + if not (v=="" or v==false) then + if nest and type(v)=="table" then + v=sparse(v,nest) + if keeptables or next(v)~=nil then + new[k]=v + end + else + new[k]=v end end end + return new +end +table.sparse=sparse +function table.compact(t) + return sparse(t,true,true) end function table.contains(t,v) if t then @@ -2000,15 +2247,17 @@ function table.print(t,...) serialize(print,t,...) end end -setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +if setinspector then + setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +end function table.sub(t,i,j) return { unpack(t,i,j) } end function table.is_empty(t) - return not t or not next(t) + return not t or next(t)==nil end function table.has_one_entry(t) - return t and not next(t,next(t)) + return t and next(t,next(t))==nil end function table.loweredkeys(t) local l={} @@ -2053,6 +2302,44 @@ function table.values(t,s) return {} end end +function table.filtered(t,pattern,sort,cmp) + if t and type(pattern)=="string" then + if sort then + local s + if cmp then + s=sortedhashkeys(t,function(a,b) return cmp(t,a,b) end) + else + s=sortedkeys(t) + end + local n=0 + local m=#s + local function kv(s) + while n<m do + n=n+1 + local k=s[n] + if find(k,pattern) then + return k,t[k] + end + end + end + return kv,s + else + local n=next(t) + local function iterator() + while n~=nil do + local k=n + n=next(t,k) + if find(k,pattern) then + return k,t[k] + end + end + end + return iterator,t + end + else + return nothing + end +end end -- of closure @@ -2061,7 +2348,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8817, stripped down to: 6340 +-- original size: 8643, stripped down to: 6232 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2075,7 +2362,7 @@ local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor local type=type -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -2368,8 +2655,6 @@ function io.readstring(f,n,m) local str=gsub(f:read(n),"\000","") return str end -if not io.i_limiter then function io.i_limiter() end end -if not io.o_limiter then function io.o_limiter() end end end -- of closure @@ -2596,7 +2881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16023, stripped down to: 9634 +-- original size: 15832, stripped down to: 9456 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2670,13 +2955,10 @@ if not os.__getenv__ then setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) end end -local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end +local execute=os.execute +local iopopen=io.popen function os.resultof(command) - local handle=io.popen(command,"r") + local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" handle:close() @@ -2686,7 +2968,7 @@ function os.resultof(command) end end if not io.fileseparator then - if find(os.getenv("PATH"),";") then + if find(os.getenv("PATH"),";",1,true) then io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" @@ -2705,7 +2987,7 @@ local launchers={ unix="$BROWSER %s &> /dev/null &", } function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) + execute(format(launchers[os.name] or launchers.unix,str)) end if not os.times then function os.times() @@ -2746,7 +3028,7 @@ if platform~="" then elseif os.type=="windows" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then + if find(architecture,"AMD64",1,true) then platform="win64" else platform="mswin" @@ -2758,9 +3040,9 @@ elseif os.type=="windows" then elseif name=="linux" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then + if find(architecture,"x86_64",1,true) then platform="linux-64" - elseif find(architecture,"ppc") then + elseif find(architecture,"ppc",1,true) then platform="linux-ppc" else platform="linux" @@ -2774,9 +3056,9 @@ elseif name=="macosx" then local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" - elseif find(architecture,"i386") then + elseif find(architecture,"i386",1,true) then platform="osx-intel" - elseif find(architecture,"x86_64") then + elseif find(architecture,"x86_64",1,true) then platform="osx-64" else platform="osx-ppc" @@ -2788,7 +3070,7 @@ elseif name=="macosx" then elseif name=="sunos" then function resolvers.platform(t,k) local platform,architecture="",os.resultof("uname -m") or "" - if find(architecture,"sparc") then + if find(architecture,"sparc",1,true) then platform="solaris-sparc" else platform="solaris-intel" @@ -2800,7 +3082,7 @@ elseif name=="sunos" then elseif name=="freebsd" then function resolvers.platform(t,k) local platform,architecture="",os.resultof("uname -m") or "" - if find(architecture,"amd64") then + if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else platform="freebsd" @@ -2812,7 +3094,7 @@ elseif name=="freebsd" then elseif name=="kfreebsd" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then + if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else platform="kfreebsd-i386" @@ -2829,8 +3111,9 @@ else return platform end end +os.newline=name=="windows" and "\013\010" or "\010" function resolvers.bits(t,k) - local bits=find(os.platform,"64") and 64 or 32 + local bits=find(os.platform,"64",1,true) and 64 or 32 os.bits=bits return bits end @@ -2980,7 +3263,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 18308, stripped down to: 9948 +-- original size: 20949, stripped down to: 9945 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -2994,41 +3277,28 @@ local file=file if not lfs then lfs=optionalrequire("lfs") end -if not lfs then - lfs={ - getcurrentdir=function() - return "." - end, - attributes=function() - return nil - end, - isfile=function(name) - local f=io.open(name,'rb') - if f then - f:close() - return true - end - end, - isdir=function(name) - print("you need to load lfs") - return false - end - } -elseif not lfs.isfile then - local attributes=lfs.attributes - function lfs.isdir(name) - return attributes(name,"mode")=="directory" - end - function lfs.isfile(name) - return attributes(name,"mode")=="file" - end -end local insert,concat=table.insert,table.concat local match,find,gmatch=string.match,string.find,string.gmatch local lpegmatch=lpeg.match local getcurrentdir,attributes=lfs.currentdir,lfs.attributes local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local tricky=S("/\\")*P(-1) +local attributes=lfs.attributes +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir,"lfs.isdir") +end +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end +end +function lfs.isfile(name) + return attributes(name,"mode")=="file" +end local colon=P(":") local period=P(".") local periods=P("..") @@ -3230,28 +3500,30 @@ local isroot=fwslash^1*-1 local hasroot=fwslash^1 local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") -function file.join(...) - local lst={... } - local one=lst[1] +function file.join(one,two,three,...) + if not two then + return one=="" and one or lpegmatch(stripper,one) + end + if one=="" then + return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) + end if lpegmatch(isnetwork,one) then local one=lpegmatch(reslasher,one) - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return one..two else return one.."/"..two end elseif lpegmatch(isroot,one) then - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return two else return "/"..two end - elseif one=="" then - return lpegmatch(stripper,concat(lst,"/",2)) else - return lpegmatch(deslasher,concat(lst,"/")) + return lpegmatch(deslasher,concat({ one,two,three,... },"/")) end end local drivespec=R("az","AZ")^1*colon @@ -3425,7 +3697,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3760, stripped down to: 2088 +-- original size: 3248, stripped down to: 2266 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3443,14 +3715,20 @@ if not md5 then } end local md5,file=md5,file -local gsub,format,byte=string.gsub,string.format,string.byte -local md5sum=md5.sum -local function convert(str,fmt) - return (gsub(md5sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +local gsub=string.gsub +do + local patterns=lpeg and lpeg.patterns + if patterns then + local bytestoHEX=patterns.bytestoHEX + local bytestohex=patterns.bytestohex + local bytestodec=patterns.bytestodec + local lpegmatch=lpeg.match + local md5sum=md5.sum + if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end + if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end + if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + end +end function file.needsupdating(oldname,newname,threshold) local oldtime=lfs.attributes(oldname,"modification") if oldtime then @@ -3507,7 +3785,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 11993, stripped down to: 5584 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -3534,7 +3812,7 @@ local hexdigit=R("09","AF","af") local plus=P("+") local nothing=Cc("") local escapedchar=(percent*C(hexdigit*hexdigit))/tochar -local escaped=(plus/" ")+escapedchar +local escaped=(plus/" ")+escapedchar local noslash=P("/")/"" local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) @@ -3593,19 +3871,25 @@ local splitquery=Cf (Ct("")*P { "sequence", pair=Cg(key*equal*value), },rawset) local function hashed(str) - if str=="" then + if not str or str=="" then return { scheme="invalid", original=str, } end - local s=split(str) - local rawscheme=s[1] - local rawquery=s[4] - local somescheme=rawscheme~="" - local somequery=rawquery~="" + local detailed=split(str) + local rawscheme="" + local rawquery="" + local somescheme=false + local somequery=false + if detailed then + rawscheme=detailed[1] + rawquery=detailed[4] + somescheme=rawscheme~="" + somequery=rawquery~="" + end if not somescheme and not somequery then - s={ + return { scheme="file", authority="", path=str, @@ -3615,28 +3899,28 @@ local function hashed(str) noscheme=true, filename=str, } - else - local authority,path,filename=s[2],s[3] - if authority=="" then - filename=path - elseif path=="" then - filename="" - else - filename=authority.."/"..path - end - s={ - scheme=rawscheme, - authority=authority, - path=path, - query=lpegmatch(unescaper,rawquery), - queries=lpegmatch(splitquery,rawquery), - fragment=s[5], - original=str, - noscheme=false, - filename=filename, - } end - return s + local authority=detailed[2] + local path=detailed[3] + local filename=nil + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + return { + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=detailed[5], + original=str, + noscheme=false, + filename=filename, + } end url.split=split url.hasscheme=hasscheme @@ -3670,7 +3954,7 @@ function url.construct(hash) end return lpegmatch(escaper,concat(fullurl)) end -local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +local pattern=Cs(slash^-1/""*R("az","AZ")*((S(":|")/":")+P(":"))*slash*P(1)^0) function url.filename(filename) local spec=hashed(filename) local path=spec.path @@ -3718,7 +4002,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 14229, stripped down to: 8740 +-- original size: 16765, stripped down to: 11003 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -3728,7 +4012,7 @@ if not modules then modules={} end modules ['l-dir']={ license="see context related readme files" } local type,select=type,select -local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local find,gmatch,match,gsub,sub=string.find,string.gmatch,string.match,string.gsub,string.sub local concat,insert,remove,unpack=table.concat,table.insert,table.remove,table.unpack local lpegmatch=lpeg.match local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V @@ -3737,53 +4021,127 @@ local dir=dir local lfs=lfs local attributes=lfs.attributes local walkdir=lfs.dir -local isdir=lfs.isdir -local isfile=lfs.isfile +local isdir=lfs.isdir +local isfile=lfs.isfile local currentdir=lfs.currentdir local chdir=lfs.chdir -local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") -if not isdir then - function isdir(name) - local a=attributes(name) - return a and a.mode=="directory" +local mkdir=lfs.mkdir +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";",1,true) +if onwindows then + local tricky=S("/\\")*P(-1) + isdir=function(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end + end + isfile=function(name) + return attributes(name,"mode")=="file" end lfs.isdir=isdir -end -if not isfile then - function isfile(name) - local a=attributes(name) - return a and a.mode=="file" + lfs.isfile=isfile +else + isdir=function(name) + return attributes(name,"mode")=="directory" end + isfile=function(name) + return attributes(name,"mode")=="file" + end + lfs.isdir=isdir lfs.isfile=isfile end function dir.current() return (gsub(currentdir(),"\\","/")) end -local lfsisdir=isdir -local function isdir(path) - path=gsub(path,"[/\\]+$","") - return lfsisdir(path) +local function glob_pattern_function(path,patt,recurse,action) + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + action(full) + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end + end + end + end + if dirs then + for i=1,#dirs do + glob_pattern_function(dirs[i],patt,recurse,action) + end + end + end end -lfs.isdir=isdir -local function globpattern(path,patt,recurse,action) - if path=="/" then - path=path.."." - elseif not find(path,"/$") then - path=path..'/' - end - if isdir(path) then - for name in walkdir(path) do - local full=path..name - local mode=attributes(full,'mode') - if mode=='file' then - if find(full,patt) then - action(full) +local function glob_pattern_table(path,patt,recurse,result) + if not result then + result={} + end + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + result[#result+1]=full + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - globpattern(full,patt,recurse,action) + end + end + if dirs then + for i=1,#dirs do + glob_pattern_table(dirs[i],patt,recurse,result) end end end + return result +end +local function globpattern(path,patt,recurse,method) + local kind=type(method) + if patt and sub(patt,1,-3)==path then + patt=false + end + if kind=="function" then + return glob_pattern_function(path,patt,recurse,method) + elseif kind=="table" then + return glob_pattern_table(path,patt,recurse,method) + else + return glob_pattern_table(path,patt,recurse,{}) + end end dir.globpattern=globpattern local function collectpattern(path,patt,recurse,result) @@ -3795,34 +4153,40 @@ local function collectpattern(path,patt,recurse,result) ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) end if ok and type(scanner)=="function" then - if not find(path,"/$") then path=path..'/' end + if not find(path,"/$") then + path=path..'/' + end for name in scanner,first do - local full=path..name - local attr=attributes(full) - local mode=attr.mode - if mode=='file' then - if find(full,patt) then + if name=="." then + elseif name==".." then + else + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and mode=="directory" then + attr.list=collectpattern(full,patt,recurse) result[name]=attr end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - attr.list=collectpattern(full,patt,recurse) - result[name]=attr end end end return result end dir.collectpattern=collectpattern -local separator -if onwindows then +local separator,pattern +if onwindows then local slash=S("/\\")/"/" - pattern=Ct { + pattern={ [1]=(Cs(P(".")+slash^1)+Cs(R("az","AZ")*P(":")*slash^0)+Cc("./"))*V(2)*V(3), [2]=Cs(((1-S("*?/\\"))^0*slash)^0), [3]=Cs(P(1)^0) } -else - pattern=Ct { +else + pattern={ [1]=(C(P(".")+P("/")^1)+Cc("./"))*V(2)*V(3), [2]=C(((1-S("*?/"))^0*P("/"))^0), [3]=C(P(1)^0) @@ -3840,10 +4204,9 @@ local function glob(str,t) elseif isfile(str) then t(str) else - local split=lpegmatch(pattern,str) - if split then - local root,path,base=split[1],split[2],split[3] - local recurse=find(base,"%*%*") + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) globpattern(start,result,recurse,t) @@ -3864,16 +4227,12 @@ local function glob(str,t) return { str } end else - local split=lpegmatch(pattern,str) - if split then - local t=t or {} - local action=action or function(name) t[#t+1]=name end - local root,path,base=split[1],split[2],split[3] - local recurse=find(base,"%*%*") + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) - globpattern(start,result,recurse,action) - return t + return globpattern(start,result,recurse,t) else return {} end @@ -3913,16 +4272,26 @@ end local make_indeed=true if onwindows then function dir.mkdirs(...) - local str,pth="","" - for i=1,select("#",...) do - local s=select(i,...) - if s=="" then - elseif str=="" then - str=s - else - str=str.."/"..s + local n=select("#",...) + local str + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end end end + local pth="" local drive=false local first,middle,last=match(str,"^(//)(//*)(.*)$") if first then @@ -3957,21 +4326,30 @@ if onwindows then pth=pth.."/"..s end if make_indeed and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end return pth,(isdir(pth)==true) end else function dir.mkdirs(...) - local str,pth="","" - for i=1,select("#",...) do - local s=select(i,...) - if s and s~="" then - if str~="" then - str=str.."/"..s - else - str=s + local n=select("#",...) + local str,pth + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s + else + str=s + end end end end @@ -3986,7 +4364,7 @@ else pth=pth.."/"..s end if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end else @@ -3994,7 +4372,7 @@ else for s in gmatch(str,"[^/]+") do pth=pth.."/"..s if make_indeed and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end end @@ -4002,47 +4380,51 @@ else end end dir.makedirs=dir.mkdirs -if onwindows then - function dir.expandname(str) - local first,nothing,last=match(str,"^(//)(//*)(.*)$") - if first then - first=dir.current().."/" - end - if not first then - first,last=match(str,"^(//)/*(.*)$") - end - if not first then - first,last=match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d=currentdir() - if chdir(first) then - first=dir.current() +do + local chdir=sandbox and sandbox.original(chdir) or chdir + if onwindows then + local xcurrentdir=dir.current + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=xcurrentdir().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if chdir(first) then + first=xcurrentdir() + end + chdir(d) end - chdir(d) + end + if not first then + first,last=xcurrentdir(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first + else + return first.."/"..last end end - if not first then - first,last=dir.current(),str - end - last=gsub(last,"//","/") - last=gsub(last,"/%./","/") - last=gsub(last,"^/*","") - first=gsub(first,"/*$","") - if last=="" or last=="." then - return first - else - return first.."/"..last - end - end -else - function dir.expandname(str) - if not find(str,"^/") then - str=currentdir().."/"..str + else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str + end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str end - str=gsub(str,"//","/") - str=gsub(str,"/%./","/") - str=gsub(str,"(.)/%.$","%1") - return str end end file.expandname=dir.expandname @@ -4085,7 +4467,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1809, stripped down to: 1527 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4139,11 +4521,11 @@ function string.booleanstring(str) return str=="yes" or str=="on" or str=="t" end end -function string.is_boolean(str,default) +function string.is_boolean(str,default,strict) if type(str)=="string" then - if str=="true" or str=="yes" or str=="on" or str=="t" or str=="1" then + if str=="true" or str=="yes" or str=="on" or str=="t" or (not strict and str=="1") then return true - elseif str=="false" or str=="no" or str=="off" or str=="f" or str=="0" then + elseif str=="false" or str=="no" or str=="off" or str=="f" or (not strict and str=="0") then return false end end @@ -4157,7 +4539,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 33473, stripped down to: 14938 +-- original size: 37388, stripped down to: 15817 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4173,7 +4555,9 @@ local type=type local char,byte,format,sub,gmatch=string.char,string.byte,string.format,string.sub,string.gmatch local concat=table.concat local P,C,R,Cs,Ct,Cmt,Cc,Carg,Cp=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg,lpeg.Cp -local lpegmatch,patterns=lpeg.match,lpeg.patterns +local lpegmatch=lpeg.match +local patterns=lpeg.patterns +local tabletopattern=lpeg.utfchartabletopattern local bytepairs=string.bytepairs local finder=lpeg.finder local replacer=lpeg.replacer @@ -4182,7 +4566,7 @@ local utfgmatch=utf.gmatch local p_utftype=patterns.utftype local p_utfstricttype=patterns.utfstricttype local p_utfoffset=patterns.utfoffset -local p_utf8char=patterns.utf8char +local p_utf8char=patterns.utf8character local p_utf8byte=patterns.utf8byte local p_utfbom=patterns.utfbom local p_newline=patterns.newline @@ -4321,6 +4705,7 @@ if not utf.sub then local pattern_zero=Cmt(p_utf8char,slide_zero)^0 local pattern_one=Cmt(p_utf8char,slide_one )^0 local pattern_two=Cmt(p_utf8char,slide_two )^0 + local pattern_first=C(patterns.utf8character) function utf.sub(str,start,stop) if not start then return str @@ -4362,7 +4747,9 @@ if not utf.sub then end end end - if start>stop then + if start==1 and stop==1 then + return lpegmatch(pattern_first,str) or "" + elseif start>stop then return "" elseif start>1 then b,e,n,first,last=0,0,0,start-1,stop @@ -4381,15 +4768,52 @@ if not utf.sub then end end end -function utf.remapper(mapping) - local pattern=Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str=="" then - return "" +function utf.remapper(mapping,option) + local variant=type(mapping) + if variant=="table" then + if option=="dynamic" then + local pattern=false + table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) + return function(str) + if not str or str=="" then + return "" + else + if not pattern then + pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + end + return lpegmatch(pattern,str) + end + end + elseif option=="pattern" then + return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) else - return lpegmatch(pattern,str) + local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + elseif variant=="function" then + if option=="pattern" then + return Cs((p_utf8char/mapping+p_utf8char)^0) + else + local pattern=Cs((p_utf8char/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern end - end,pattern + else + return function(str) + return str or "" + end + end end function utf.replacer(t) local r=replacer(t,false,false,true) @@ -4439,190 +4863,157 @@ function utf.magic(f) end local utf16_to_utf8_be,utf16_to_utf8_le local utf32_to_utf8_be,utf32_to_utf8_le -local utf_16_be_linesplitter=patterns.utfbom_16_be^-1*lpeg.tsplitat(patterns.utf_16_be_nl) -local utf_16_le_linesplitter=patterns.utfbom_16_le^-1*lpeg.tsplitat(patterns.utf_16_le_nl) -if bytepairs then - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*left+right - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +local utf_16_be_getbom=patterns.utfbom_16_be^-1 +local utf_16_le_getbom=patterns.utfbom_16_le^-1 +local utf_32_be_getbom=patterns.utfbom_32_be^-1 +local utf_32_le_getbom=patterns.utfbom_32_le^-1 +local utf_16_be_linesplitter=utf_16_be_getbom*lpeg.tsplitat(patterns.utf_16_be_nl) +local utf_16_le_linesplitter=utf_16_le_getbom*lpeg.tsplitat(patterns.utf_16_le_nl) +local utf_32_be_linesplitter=utf_32_be_getbom*lpeg.tsplitat(patterns.utf_32_be_nl) +local utf_32_le_linesplitter=utf_32_le_getbom*lpeg.tsplitat(patterns.utf_32_le_nl) +local more=0 +local p_utf16_to_utf8_be=C(1)*C(1)/function(left,right) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf16_to_utf8_le=C(1)*C(1)/function(right,left) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf32_to_utf8_be=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(a)+256*256*byte(b)+256*byte(c)+byte(d)) +end +local p_utf32_to_utf8_le=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(d)+256*256*byte(c)+256*byte(b)+byte(a)) +end +p_utf16_to_utf8_be=P(true)/function() more=0 end*utf_16_be_getbom*Cs(p_utf16_to_utf8_be^0) +p_utf16_to_utf8_le=P(true)/function() more=0 end*utf_16_le_getbom*Cs(p_utf16_to_utf8_le^0) +p_utf32_to_utf8_be=P(true)/function() more=0 end*utf_32_be_getbom*Cs(p_utf32_to_utf8_be^0) +p_utf32_to_utf8_le=P(true)/function() more=0 end*utf_32_le_getbom*Cs(p_utf32_to_utf8_le^0) +patterns.utf16_to_utf8_be=p_utf16_to_utf8_be +patterns.utf16_to_utf8_le=p_utf16_to_utf8_le +patterns.utf32_to_utf8_be=p_utf32_to_utf8_be +patterns.utf32_to_utf8_le=p_utf32_to_utf8_le +utf16_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_be,s) + else + return s end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*right+left - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +end +local utf16_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_be_linesplitter,t) end - utf32_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*256*256*a+256*256*b - else - r=r+1 - result[t]=utfchar(more+256*a+b) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_be,s) end - return t end - utf32_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*b+a - else - r=r+1 - result[t]=utfchar(more+256*256*256*b+256*256*a) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) - end - return t + return t +end +utf16_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_le,s) + else + return s end -else - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if left=="\000" then - r=r+1 - result[r]=utfchar(byte(right)) - elseif right then - local now=256*byte(left)+byte(right) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) +end +local utf16_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_le,s) end - return t end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) + return t +end +utf32_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_be,s) + else + return s + end +end +local utf32_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_be_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_be,s) end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if right=="\000" then - r=r+1 - result[r]=utfchar(byte(left)) - elseif right then - local now=256*byte(right)+byte(left) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) + end + return t +end +utf32_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_le,s) + else + return s + end +end +local utf32_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_le,s) end - return t end - utf32_to_utf8_le=function() return {} end - utf32_to_utf8_be=function() return {} end + return t end +utf.utf16_to_utf8_le_t=utf16_to_utf8_le_t +utf.utf16_to_utf8_be_t=utf16_to_utf8_be_t +utf.utf32_to_utf8_le_t=utf32_to_utf8_le_t +utf.utf32_to_utf8_be_t=utf32_to_utf8_be_t utf.utf16_to_utf8_le=utf16_to_utf8_le utf.utf16_to_utf8_be=utf16_to_utf8_be utf.utf32_to_utf8_le=utf32_to_utf8_le utf.utf32_to_utf8_be=utf32_to_utf8_be -function utf.utf8_to_utf8(t) +function utf.utf8_to_utf8_t(t) return type(t)=="string" and lpegmatch(utflinesplitter,t) or t end -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +function utf.utf16_to_utf8_t(t,endian) + return endian and utf16_to_utf8_be_t(t) or utf16_to_utf8_le_t(t) or t end -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +function utf.utf32_to_utf8_t(t,endian) + return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(c) - local b=byte(c) +local function little(b) if b<0x10000 then return char(b%256,b/256) else @@ -4631,8 +5022,7 @@ local function little(c) return char(b1%256,b1/256,b2%256,b2/256) end end -local function big(c) - local b=byte(c) +local function big(b) if b<0x10000 then return char(b/256,b%256) else @@ -4641,27 +5031,29 @@ local function big(c) return char(b1/256,b1%256,b2/256,b2%256) end end -local _,l_remap=utf.remapper(little) -local _,b_remap=utf.remapper(big) -function utf.utf8_to_utf16_be(str,nobom) +local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) +local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) +local function utf8_to_utf16_be(str,nobom) if nobom then return lpegmatch(b_remap,str) else return char(254,255)..lpegmatch(b_remap,str) end end -function utf.utf8_to_utf16_le(str,nobom) +local function utf8_to_utf16_le(str,nobom) if nobom then return lpegmatch(l_remap,str) else return char(255,254)..lpegmatch(l_remap,str) end end +utf.utf8_to_utf16_be=utf8_to_utf16_be +utf.utf8_to_utf16_le=utf8_to_utf16_le function utf.utf8_to_utf16(str,littleendian,nobom) if littleendian then - return utf.utf8_to_utf16_le(str,nobom) + return utf8_to_utf16_le(str,nobom) else - return utf.utf8_to_utf16_be(str,nobom) + return utf8_to_utf16_be(str,nobom) end end local pattern=Cs ( @@ -4677,16 +5069,16 @@ function utf.xstring(s) return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end function utf.toeight(str) - if not str then + if not str or str=="" then return nil end local utftype=lpegmatch(p_utfstricttype,str) if utftype=="utf-8" then - return sub(str,4) - elseif utftype=="utf-16-le" then - return utf16_to_utf8_le(str) + return sub(str,4) elseif utftype=="utf-16-be" then - return utf16_to_utf8_ne(str) + return utf16_to_utf8_be(str) + elseif utftype=="utf-16-le" then + return utf16_to_utf8_le(str) else return str end @@ -4765,7 +5157,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 915, stripped down to: 836 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -4775,6 +5167,9 @@ if not modules then modules={} end modules ['l-math']={ license="see context related readme files" } local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.ceiling then + math.ceiling=math.ceil +end if not math.round then function math.round(x) return floor(x+0.5) end end @@ -4802,7 +5197,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 26857, stripped down to: 15062 +-- original size: 34503, stripped down to: 18933 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -4821,25 +5216,43 @@ local unpack,concat=table.unpack,table.concat local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc local patterns,lpegmatch=lpeg.patterns,lpeg.match local utfchar,utfbyte=utf.char,utf.byte -local loadstripped=_LUAVERSION<5.2 and load or function(str) - return load(dump(load(str),true)) +local loadstripped=nil +if _LUAVERSION<5.2 then + loadstripped=function(str,shortcuts) + return load(str) + end +else + loadstripped=function(str,shortcuts) + if shortcuts then + return load(dump(load(str),true),nil,nil,shortcuts) + else + return load(dump(load(str),true)) + end + end end if not number then number={} end local stripper=patterns.stripzeros +local newline=patterns.newline +local endofstring=patterns.endofstring +local whitespace=patterns.whitespace +local spacer=patterns.spacer +local spaceortab=patterns.spaceortab local function points(n) + n=tonumber(n) return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) end local function basepoints(n) + n=tonumber(n) return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) end number.points=points number.basepoints=basepoints -local rubish=patterns.spaceortab^0*patterns.newline -local anyrubish=patterns.spaceortab+patterns.newline +local rubish=spaceortab^0*newline +local anyrubish=spaceortab+newline local anything=patterns.anything -local stripped=(patterns.spaceortab^1/"")*patterns.newline +local stripped=(spaceortab^1/"")*newline local leading=rubish^0/"" -local trailing=(anyrubish^1*patterns.endofstring)/"" +local trailing=(anyrubish^1*endofstring)/"" local redundant=rubish^3/"\n" local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) function strings.collapsecrlf(str) @@ -4885,18 +5298,44 @@ local pattern=Carg(1)/function(t) else return "" end - end+patterns.newline*Cp()/function(position) + end+newline*Cp()/function(position) extra,start=0,position end+patterns.anything )^1) function strings.tabtospace(str,tab) return lpegmatch(pattern,str,1,tab or 7) end -function strings.striplong(str) - str=gsub(str,"^%s*","") - str=gsub(str,"[\n\r]+ *","\n") - return str +local space=spacer^0 +local nospace=space/"" +local endofline=nospace*newline +local stripend=(whitespace^1*endofstring)/"" +local normalline=(nospace*((1-space*(newline+endofstring))^1)*nospace) +local stripempty=endofline^1/"" +local normalempty=endofline^1 +local singleempty=endofline*(endofline^0/"") +local doubleempty=endofline*endofline^-1*(endofline^0/"") +local stripstart=stripempty^0 +local p_prune_normal=Cs (stripstart*(stripend+normalline+normalempty )^0 ) +local p_prune_collapse=Cs (stripstart*(stripend+normalline+doubleempty )^0 ) +local p_prune_noempty=Cs (stripstart*(stripend+normalline+singleempty )^0 ) +local p_retain_normal=Cs ((normalline+normalempty )^0 ) +local p_retain_collapse=Cs ((normalline+doubleempty )^0 ) +local p_retain_noempty=Cs ((normalline+singleempty )^0 ) +local striplinepatterns={ + ["prune"]=p_prune_normal, + ["prune and collapse"]=p_prune_collapse, + ["prune and no empty"]=p_prune_noempty, + ["retain"]=p_retain_normal, + ["retain and collapse"]=p_retain_collapse, + ["retain and no empty"]=p_retain_noempty, + ["collapse"]=patterns.collapser, +} +setmetatable(striplinepatterns,{ __index=function(t,k) return p_prune_collapse end }) +strings.striplinepatterns=striplinepatterns +function strings.striplines(str,how) + return str and lpegmatch(striplinepatterns[how],str) or str end +strings.striplong=strings.striplines function strings.nice(str) str=gsub(str,"[:%-+_]+"," ") return str @@ -4934,10 +5373,10 @@ string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) if type(b)=="number" then - return tracedchars[b] or (utfchar(b).." (U+"..format('%05X',b)..")") + return tracedchars[b] or (utfchar(b).." (U+"..format("%05X",b)..")") else local c=utfbyte(b) - return tracedchars[c] or (b.." (U+"..format('%05X',c)..")") + return tracedchars[c] or (b.." (U+"..(c and format("%05X",c) or "?????")..")") end end function number.signed(i) @@ -4972,31 +5411,58 @@ function number.sparseexponent(f,n) end return tostring(n) end -local preamble=[[ -local type = type -local tostring = tostring -local tonumber = tonumber -local format = string.format -local concat = table.concat -local signed = number.signed -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -local lpegmatch = lpeg.match -local nspaces = string.nspaces -local tracedchar = string.tracedchar -local autosingle = string.autosingle -local autodouble = string.autodouble -local sequenced = table.sequenced -local formattednumber = number.formatted -local sparseexponent = number.sparseexponent -]] local template=[[ %s %s return function(%s) return %s end ]] +local preamble,environment="",{} +if _LUAVERSION<5.2 then + preamble=[[ +local lpeg=lpeg +local type=type +local tostring=tostring +local tonumber=tonumber +local format=string.format +local concat=table.concat +local signed=number.signed +local points=number.points +local basepoints= number.basepoints +local utfchar=utf.char +local utfbyte=utf.byte +local lpegmatch=lpeg.match +local nspaces=string.nspaces +local tracedchar=string.tracedchar +local autosingle=string.autosingle +local autodouble=string.autodouble +local sequenced=table.sequenced +local formattednumber=number.formatted +local sparseexponent=number.sparseexponent + ]] +else + environment={ + global=global or _G, + lpeg=lpeg, + type=type, + tostring=tostring, + tonumber=tonumber, + format=string.format, + concat=table.concat, + signed=number.signed, + points=number.points, + basepoints=number.basepoints, + utfchar=utf.char, + utfbyte=utf.byte, + lpegmatch=lpeg.match, + nspaces=string.nspaces, + tracedchar=string.tracedchar, + autosingle=string.autosingle, + autodouble=string.autodouble, + sequenced=table.sequenced, + formattednumber=number.formatted, + sparseexponent=number.sparseexponent, + } +end local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) local v=t[k-1]..",a"..k @@ -5035,7 +5501,7 @@ local format_i=function(f) if f and f~="" then return format("format('%%%si',a%s)",f,n) else - return format("format('%%i',a%s)",n) + return format("format('%%i',a%s)",n) end end local format_d=format_i @@ -5047,6 +5513,14 @@ local format_f=function(f) n=n+1 return format("format('%%%sf',a%s)",f,n) end +local format_F=function(f) + n=n+1 + if not f or f=="" then + return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or format((a%s %% 1 == 0) and '%%i' or '%%.9f',a%s))",n,n,n,n) + else + return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) + end +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5261,7 +5735,7 @@ local builder=Cs { "start", ( P("%")/""*( V("!") -+V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") +V("N") @@ -5272,7 +5746,6 @@ local builder=Cs { "start", +V("j")+V("J") +V("m")+V("M") +V("z") -+V("*") )+V("*") )*(P(-1)+Carg(1)) )^0, @@ -5281,6 +5754,7 @@ local builder=Cs { "start", ["i"]=(prefix_any*P("i"))/format_i, ["d"]=(prefix_any*P("d"))/format_d, ["f"]=(prefix_any*P("f"))/format_f, + ["F"]=(prefix_any*P("F"))/format_F, ["g"]=(prefix_any*P("g"))/format_g, ["G"]=(prefix_any*P("G"))/format_G, ["e"]=(prefix_any*P("e"))/format_e, @@ -5315,11 +5789,12 @@ local builder=Cs { "start", ["a"]=(prefix_any*P("a"))/format_a, ["A"]=(prefix_any*P("A"))/format_A, ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, + ["?"]=Cs(((1-P("%"))^1 )^1)/format_rest, ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, } local direct=Cs ( - P("%")/""*Cc([[local format = string.format return function(str) return format("%]])*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*Cc([[",str) end]])*P(-1) - ) + P("%")*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*P(-1)/[[local format = string.format return function(str) return format("%0",str) end]] +) local function make(t,str) local f local p @@ -5328,10 +5803,10 @@ local function make(t,str) f=loadstripped(p)() else n=0 - p=lpegmatch(builder,str,1,"..",t._extensions_) + p=lpegmatch(builder,str,1,t._connector_,t._extensions_) if n>0 then p=format(template,preamble,t._preamble_,arguments[n],p) - f=loadstripped(p)() + f=loadstripped(p,t._environment_)() else f=function() return str end end @@ -5343,10 +5818,22 @@ local function use(t,fmt,...) return t[fmt](...) end strings.formatters={} -function strings.formatters.new() - local t={ _extensions_={},_preamble_="",_type_="formatter" } - setmetatable(t,{ __index=make,__call=use }) - return t +if _LUAVERSION<5.2 then + function strings.formatters.new(noconcat) + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_=preamble,_environment_={} } + setmetatable(t,{ __index=make,__call=use }) + return t + end +else + function strings.formatters.new(noconcat) + local e={} + for k,v in next,environment do + e[k]=v + end + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_="",_environment_=e } + setmetatable(t,{ __index=make,__call=use }) + return t + end end local formatters=strings.formatters.new() string.formatters=formatters @@ -5354,8 +5841,12 @@ string.formatter=function(str,...) return formatters[str](...) end local function add(t,name,template,preamble) if type(t)=="table" and t._type_=="formatter" then t._extensions_[name]=template or "%s" - if preamble then + if type(preamble)=="string" then t._preamble_=preamble.."\n"..t._preamble_ + elseif type(preamble)=="table" then + for k,v in next,preamble do + t._environment_[k]=v + end end end end @@ -5364,9 +5855,28 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+P(1))^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]]) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]]) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]]) +if _LUAVERSION<5.2 then + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],"local xmlescape = lpeg.patterns.xmlescape") + add(formatters,"tex",[[lpegmatch(texescape,%s)]],"local texescape = lpeg.patterns.texescape") + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],"local luaescape = lpeg.patterns.luaescape") +else + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) + add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +end +local dquote=patterns.dquote +local equote=patterns.escaped+dquote/'\\"'+1 +local space=patterns.space +local cquote=Cc('"') +local pattern=Cs(dquote*(equote-P(-2))^0*dquote) ++Cs(cquote*(equote-space)^0*space*equote^0*cquote) +function string.optionalquoted(str) + return lpegmatch(pattern,str) or str +end +local pattern=Cs((newline/os.newline+1)^0) +function string.replacenewlines(str) + return lpegmatch(pattern,str) +end end -- of closure @@ -5375,7 +5885,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 23952, stripped down to: 16092 +-- original size: 25338, stripped down to: 16247 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5388,7 +5898,7 @@ utilities=utilities or {} utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub -local concat,insert,remove=table.concat,table.insert,table.remove +local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc @@ -5396,27 +5906,29 @@ local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs local formatters=string.formatters local utftoeight=utf.toeight local splitter=lpeg.tsplitat(".") -function tables.definetable(target,nofirst,nolast) - local composed,shortcut,t=nil,nil,{} +function utilities.tables.definetable(target,nofirst,nolast) + local composed,t=nil,{} local snippets=lpegmatch(splitter,target) for i=1,#snippets-(nolast and 1 or 0) do local name=snippets[i] if composed then - composed=shortcut.."."..name - shortcut=shortcut.."_"..name - t[#t+1]=formatters["local %s = %s if not %s then %s = { } %s = %s end"](shortcut,composed,shortcut,shortcut,composed,shortcut) + composed=composed.."."..name + t[#t+1]=formatters["if not %s then %s = { } end"](composed,composed) else composed=name - shortcut=name if not nofirst then t[#t+1]=formatters["%s = %s or { }"](composed,composed) end end end - if nolast then - composed=shortcut.."."..snippets[#snippets] + if composed then + if nolast then + composed=composed.."."..snippets[#snippets] + end + return concat(t,"\n"),composed + else + return "",target end - return concat(t,"\n"),composed end function tables.definedtable(...) local t=_G @@ -5443,7 +5955,7 @@ function tables.accesstable(target,root) end function tables.migratetable(target,v,root) local t=root or _G - local names=string.split(target,".") + local names=lpegmatch(splitter,target) for i=1,#names-1 do local name=names[i] t[name]=t[name] or {} @@ -5463,6 +5975,15 @@ function tables.removevalue(t,value) end end end +function tables.replacevalue(t,oldvalue,newvalue) + if oldvalue and newvalue then + for i=1,#t do + if t[i]==oldvalue then + t[i]=newvalue + end + end + end +end function tables.insertbeforevalue(t,value,extra) for i=1,#t do if t[i]==extra then @@ -5610,7 +6131,7 @@ local f_ordered_string=formatters["%q,"] local f_ordered_number=formatters["%s,"] local f_ordered_boolean=formatters["%l,"] function table.fastserialize(t,prefix) - local r={ prefix or "return" } + local r={ type(prefix)=="string" and prefix or "return" } local m=1 local function fastserialize(t,outer) local n=#t @@ -5807,7 +6328,8 @@ function table.serialize(root,name,specification) local t local n=1 local function simple_table(t) - if #t>0 then + local nt=#t + if nt>0 then local n=0 for _,v in next,t do n=n+1 @@ -5815,19 +6337,17 @@ function table.serialize(root,name,specification) return nil end end - if n==#t then + if n==nt then local tt={} - local nt=0 - for i=1,#t do + for i=1,nt do local v=t[i] local tv=type(v) - nt=nt+1 if tv=="number" then - tt[nt]=v + tt[i]=v elseif tv=="string" then - tt[nt]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then - tt[nt]=v and "true" or "false" + tt[i]=v and "true" or "false" else return nil end @@ -5856,7 +6376,7 @@ function table.serialize(root,name,specification) end depth=depth+1 end - if root and next(root) then + if root and next(root)~=nil then local first=nil local last=0 last=#root @@ -5875,13 +6395,13 @@ function table.serialize(root,name,specification) local v=root[k] local tv=type(v) local tk=type(k) - if first and tk=="number" and k>=first and k<=last then + if first and tk=="number" and k<=last and k>=first then if tv=="number" then n=n+1 t[n]=f_val_num(depth,v) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if not next(v) then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -5911,13 +6431,13 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_boo_value_str(depth,k,v) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then - n=n+1 t[n]=f_key_num_value_not(depth,k,v) + n=n+1 t[n]=f_key_num_value_not(depth,k) elseif tk=="string" then - n=n+1 t[n]=f_key_str_value_not(depth,k,v) + n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then - n=n+1 t[n]=f_key_boo_value_not(depth,k,v) + n=n+1 t[n]=f_key_boo_value_not(depth,k) end else local st=simple_table(v) @@ -5969,7 +6489,7 @@ function table.serialize(root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,1,0) end end @@ -6132,7 +6652,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 19604, stripped down to: 13998 +-- original size: 21780, stripped down to: 15121 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6154,6 +6674,8 @@ local patterns=parsers.patterns or {} parsers.patterns=patterns local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash +local sortedkeys=table.sortedkeys +local tohash=table.tohash local digit=R("09") local space=P(' ') local equal=P("=") @@ -6203,9 +6725,7 @@ patterns.settings_to_hash_a=pattern_a_s patterns.settings_to_hash_b=pattern_b_s patterns.settings_to_hash_c=pattern_c_s function parsers.make_settings_to_hash_pattern(set,how) - if type(str)=="table" then - return set - elseif how=="strict" then + if how=="strict" then return (pattern_c/set)^1 elseif how=="tolerant" then return (pattern_b/set)^1 @@ -6214,7 +6734,9 @@ function parsers.make_settings_to_hash_pattern(set,how) end end function parsers.settings_to_hash(str,existing) - if type(str)=="table" then + if not str or str=="" then + return {} + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6223,16 +6745,16 @@ function parsers.settings_to_hash(str,existing) else return str end - elseif str and str~="" then + else hash=existing or {} lpegmatch(pattern_a_s,str) return hash - else - return {} end end function parsers.settings_to_hash_tolerant(str,existing) - if type(str)=="table" then + if not str or str=="" then + return {} + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6241,16 +6763,16 @@ function parsers.settings_to_hash_tolerant(str,existing) else return str end - elseif str and str~="" then + else hash=existing or {} lpegmatch(pattern_b_s,str) return hash - else - return {} end end function parsers.settings_to_hash_strict(str,existing) - if type(str)=="table" then + if not str or str=="" then + return nil + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6263,8 +6785,6 @@ function parsers.settings_to_hash_strict(str,existing) hash=existing or {} lpegmatch(pattern_c_s,str) return next(hash) and hash - else - return nil end end local separator=comma*space^0 @@ -6272,27 +6792,46 @@ local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comm local pattern=spaces*Ct(value*(separator*value)^0) patterns.settings_to_array=pattern function parsers.settings_to_array(str,strict) - if type(str)=="table" then - return str - elseif not str or str=="" then + if not str or str=="" then return {} + elseif type(str)=="table" then + return str elseif strict then - if find(str,"{") then + if find(str,"{",1,true) then return lpegmatch(pattern,str) else return { str } end - elseif find(str,",") then + elseif find(str,",",1,true) then return lpegmatch(pattern,str) else return { str } end end -local separator=space^0*comma*space^0 -local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(comma+P(-1)))))^0) -local withvalue=Carg(1)*value/function(f,s) return f(s) end -local pattern_a=spaces*Ct(value*(separator*value)^0) -local pattern_b=spaces*withvalue*(separator*withvalue)^0 +local cache_a={} +local cache_b={} +function parsers.groupedsplitat(symbol,withaction) + if not symbol then + symbol="," + end + local pattern=(withaction and cache_b or cache_a)[symbol] + if not pattern then + local symbols=S(symbol) + local separator=space^0*symbols*space^0 + local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(symbols+P(-1)))))^0) + if withaction then + local withvalue=Carg(1)*value/function(f,s) return f(s) end + pattern=spaces*withvalue*(separator*withvalue)^0 + cache_b[symbol]=pattern + else + pattern=spaces*Ct(value*(separator*value)^0) + cache_a[symbol]=pattern + end + end + return pattern +end +local pattern_a=parsers.groupedsplitat(",",false) +local pattern_b=parsers.groupedsplitat(",",true) function parsers.stripped_settings_to_array(str) if not str or str=="" then return {} @@ -6317,8 +6856,8 @@ function parsers.add_settings_to_array(t,str) end function parsers.hash_to_string(h,separator,yes,no,strict,omit) if h then - local t,tn,s={},0,table.sortedkeys(h) - omit=omit and table.tohash(omit) + local t,tn,s={},0,sortedkeys(h) + omit=omit and tohash(omit) for i=1,#s do local key=s[i] if not omit or not omit[key] then @@ -6354,12 +6893,9 @@ function parsers.array_to_string(a,separator) return "" end end -function parsers.settings_to_set(str,t) - t=t or {} - for s in gmatch(str,"[^, ]+") do - t[s]=true - end - return t +local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) +function utilities.parsers.settings_to_set(str,t) + return str and lpegmatch(pattern,str) or {} end function parsers.simple_hash_to_string(h,separator) local t,tn={},0 @@ -6371,12 +6907,16 @@ function parsers.simple_hash_to_string(h,separator) end return concat(t,separator or ",") end -local str=C((1-whitespace-equal)^1) +local str=Cs(lpegpatterns.unquoted)+C((1-whitespace-equal)^1) local setting=Cf(Carg(1)*(whitespace^0*Cg(str*whitespace^0*(equal*whitespace^0*str+Cc(""))))^1,rawset) local splitter=setting^1 function utilities.parsers.options_to_hash(str,target) return str and lpegmatch(splitter,str,1,target or {}) or {} end +local splitter=lpeg.tsplitat(" ") +function utilities.parsers.options_to_array(str) + return str and lpegmatch(splitter,str) or {} +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) local pattern_a=spaces*Ct(value*(separator*value)^0) local function repeater(n,str) @@ -6463,7 +7003,7 @@ function parsers.keq_to_hash(str) end local defaultspecification={ separator=",",quote='"' } function parsers.csvsplitter(specification) - specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification local separator=specification.separator local quotechar=specification.quote local separator=S(separator~="" and separator or ",") @@ -6487,7 +7027,7 @@ function parsers.csvsplitter(specification) end end function parsers.rfc4180splitter(specification) - specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification local separator=specification.separator local quotechar=P(specification.quote) local dquotechar=quotechar*quotechar @@ -6498,7 +7038,7 @@ function parsers.rfc4180splitter(specification) local field=escaped+non_escaped+Cc("") local record=Ct(field*(separator*field)^1) local headerline=record*Cp() - local wholeblob=Ct((newline^-1*record)^0) + local wholeblob=Ct((newline^(specification.strict and -1 or 1)*record)^0) return function(data,getheader) if getheader then local header,position=lpegmatch(headerline,data) @@ -6535,20 +7075,20 @@ function parsers.stepper(str,n,action) lpegmatch(stepper,str,1,n,action or print) end end -local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) -local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) +local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) +local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) patterns.unittotex=pattern function parsers.unittotex(str,textmode) return lpegmatch(textmode and pattern_text or pattern_math,str) end -local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+P(1))^0) +local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+anything)^0) function parsers.unittoxml(str) return lpegmatch(pattern,str) end local cache={} -local spaces=lpeg.patterns.space^0 +local spaces=lpegpatterns.space^0 local dummy=function() end -table.setmetatableindex(cache,function(t,k) +setmetatableindex(cache,function(t,k) local separator=P(k) local value=(1-separator)^0 local pattern=spaces*C(value)*separator^0*Cp() @@ -6613,6 +7153,18 @@ function utilities.parsers.runtime(time) local seconds=mod(time,60) return days,hours,minutes,seconds end +local spacing=whitespace^0 +local apply=P("->") +local method=C((1-apply)^1) +local token=lbrace*C((1-rbrace)^1)*rbrace+C(anything^1) +local pattern=spacing*(method*spacing*apply+Carg(1))*spacing*token +function utilities.parsers.splitmethod(str,default) + if str then + return lpegmatch(pattern,str,1,default or false) + else + return default or false,"" + end +end end -- of closure @@ -6702,7 +7254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12365, stripped down to: 8799 +-- original size: 12482, stripped down to: 8864 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -6730,7 +7282,7 @@ function setters.initialize(filename,name,values) local data=setter.data if data then for key,newvalue in next,values do - local newvalue=is_boolean(newvalue,newvalue) + local newvalue=is_boolean(newvalue,newvalue,true) local functions=data[key] if functions then local oldvalue=functions.value @@ -6784,7 +7336,7 @@ local function set(t,what,newvalue) elseif not value then value=false else - value=is_boolean(value,value) + value=is_boolean(value,value,true) end w=topattern(w,true,true) for name,functions in next,data do @@ -6923,6 +7475,7 @@ function setters.new(name) report=function(...) setters.report (setter,...) end, enable=function(...) enable (setter,...) end, disable=function(...) disable (setter,...) end, + reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, list=function(...) list (setter,...) end, show=function(...) show (setter,...) end, @@ -7014,7 +7567,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 25391, stripped down to: 16561 +-- original size: 29359, stripped down to: 20483 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7023,15 +7576,18 @@ if not modules then modules={} end modules ['trac-log']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local next,type,select,print=next,type,select,print local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write local format,gmatch,find=string.format,string.gmatch,string.find local concat,insert,remove=table.concat,table.insert,table.remove local topattern=string.topattern -local next,type,select=next,type,select local utfchar=utf.char +local datetime=os.date +local openfile=io.open local setmetatableindex=table.setmetatableindex local formatters=string.formatters local texgetcount=tex and tex.getcount +local variant="default" logs=logs or {} local logs=logs local moreinfo=[[ @@ -7041,32 +7597,122 @@ maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context webpage : http://www.pragma-ade.nl / http://tex.aanhet.net wiki : http://contextgarden.net ]] -utilities.strings.formatters.add ( +formatters.add ( formatters,"unichr", [["U+" .. format("%%05X",%s) .. " (" .. utfchar(%s) .. ")"]] ) -utilities.strings.formatters.add ( +formatters.add ( formatters,"chruni", [[utfchar(%s) .. " (U+" .. format("%%05X",%s) .. ")"]] ) local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations -local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters +local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then - local valueiskey={ __index=function(t,k) t[k]=k return k end } - local target="term and log" + local function useluawrites() + local texio_write_nl=texio.write_nl + local texio_write=texio.write + local io_write=io.write + write_nl=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write_nl("log",...) + texio_write_nl("term","") + io_write(...) + elseif target=="log" then + texio_write_nl("log",...) + elseif target=="term" then + texio_write_nl("term","") + io_write(...) + elseif target~="none" then + texio_write_nl("log",target,...) + texio_write_nl("term","") + io_write(target,...) + end + end + write=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write("log",...) + io_write(...) + elseif target=="log" then + texio_write("log",...) + elseif target=="term" then + io_write(...) + elseif target~="none" then + texio_write("log",target,...) + io_write(target,...) + end + end + texio.write=write + texio.write_nl=write_nl + useluawrites=ignore + end + local whereto="both" + local target=nil + local targets=nil + local formats=table.setmetatableindex("self") + local translations=table.setmetatableindex("self") + local report_yes,subreport_yes,direct_yes,subdirect_yes,status_yes + local report_nop,subreport_nop,direct_nop,subdirect_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s > %s\n"], + report_nop=formatters["%-15s >\n"], + direct_yes=formatters["%-15s > %s"], + direct_nop=formatters["%-15s >"], + subreport_yes=formatters["%-15s > %s > %s\n"], + subreport_nop=formatters["%-15s > %s >\n"], + subdirect_yes=formatters["%-15s > %s > %s"], + subdirect_nop=formatters["%-15s > %s >"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;33m%-15s [0;1m>[0m %s\n"], + report_nop=formatters["[0;33m%-15s [0;1m>[0m\n"], + direct_yes=formatters["[0;33m%-15s [0;1m>[0m %s"], + direct_nop=formatters["[0;33m%-15s [0;1m>[0m"], + subreport_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s\n"], + subreport_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m\n"], + subdirect_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s"], + subdirect_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m"], + status_yes=formatters["[0;33m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;33m%-15s [0;1m:[0m\n"], + }, + targets={ + logfile="none", + log="none", + file="none", + console="term", + terminal="term", + both="term", + }, + } + } logs.flush=io.flush - local formats={} setmetatable(formats,valueiskey) - local translations={} setmetatable(translations,valueiskey) writer=function(...) write_nl(target,...) end newline=function() write_nl(target,"\n") end - local report_yes=formatters["%-15s > %s\n"] - local report_nop=formatters["%-15s >\n"] report=function(a,b,c,...) if c then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) @@ -7078,8 +7724,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local direct_yes=formatters["%-15s > %s"] - local direct_nop=formatters["%-15s >"] direct=function(a,b,c,...) if c then return direct_yes(translations[a],formatters[formats[b]](c,...)) @@ -7091,8 +7735,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local subreport_yes=formatters["%-15s > %s > %s\n"] - local subreport_nop=formatters["%-15s > %s >\n"] subreport=function(a,s,b,c,...) if c then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) @@ -7104,8 +7746,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local subdirect_yes=formatters["%-15s > %s > %s"] - local subdirect_nop=formatters["%-15s > %s >"] subdirect=function(a,s,b,c,...) if c then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) @@ -7117,8 +7757,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) @@ -7130,16 +7768,13 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local targets={ - logfile="log", - log="log", - file="log", - console="term", - terminal="term", - both="term and log", - } - settarget=function(whereto) - target=targets[whereto or "both"] or targets.both + settarget=function(askedwhereto) + whereto=askedwhereto or whereto or "both" + target=targets[whereto] + if not target then + whereto="both" + target=targets[whereto] + end if target=="term" or target=="term and log" then logs.flush=io.flush else @@ -7168,21 +7803,74 @@ if tex and (tex.jobname or tex.formatname) then writeline(target,f(...)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - direct_yes=f.direct_yes or direct_yes - direct_nop=f.direct_nop or direct_nop - subdirect_yes=f.subdirect_yes or subdirect_yes - subdirect_nop=f.subdirect_nop or subdirect_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local t=nil + local f=nil + local d=variants.default + if not specification then + elseif type(specification)=="table" then + t=specification.targets + f=specification.formats or specification + else + local v=variants[specification] + if v then + t=v.targets + f=v.formats + variant=specification + end + end + targets=t or d.targets + target=targets[whereto] or target + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + direct_yes=f.direct_yes + direct_nop=f.direct_nop + subdirect_yes=f.subdirect_yes + subdirect_nop=f.subdirect_nop + status_yes=f.status_yes + status_nop=f.status_nop + if variant=="ansi" then + useluawrites() + end + settarget(whereto) + end + setformatters(variant) setlogfile=ignore settimedlog=ignore else + local report_yes,subreport_yes,status_yes + local report_nop,subreport_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s | %s"], + report_nop=formatters["%-15s |"], + subreport_yes=formatters["%-15s | %s | %s"], + subreport_nop=formatters["%-15s | %s |"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;32m%-15s [0;1m|[0m %s"], + report_nop=formatters["[0;32m%-15s [0;1m|[0m"], + subreport_yes=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m %s"], + subreport_nop=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m"], + status_yes=formatters["[0;32m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;32m%-15s [0;1m:[0m\n"], + }, + }, + } logs.flush=ignore writer=function(s) write_nl(s) @@ -7190,8 +7878,6 @@ else newline=function() write_nl("\n") end - local report_yes=formatters["%-15s | %s"] - local report_nop=formatters["%-15s |"] report=function(a,b,c,...) if c then write_nl(report_yes(a,formatters[b](c,...))) @@ -7203,8 +7889,6 @@ else write_nl("") end end - local subreport_yes=formatters["%-15s | %s | %s"] - local subreport_nop=formatters["%-15s | %s |"] subreport=function(a,sub,b,c,...) if c then write_nl(subreport_yes(a,sub,formatters[b](c,...))) @@ -7216,8 +7900,6 @@ else write_nl("") end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(status_yes(a,formatters[b](c,...))) @@ -7242,14 +7924,34 @@ else writeline(f(s)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local f=nil + local d=variants.default + if specification then + if type(specification)=="table" then + f=specification.formats or specification + else + local v=variants[specification] + if v then + f=v.formats + end + end + end + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + status_yes=f.status_yes + status_nop=f.status_nop + end + setformatters(variant) setlogfile=function(name,keepopen) if name and name~="" then local localtime=os.localtime @@ -7368,9 +8070,10 @@ local function setblocked(category,value) v.state=value end else - states=utilities.parsers.settings_to_hash(category) + states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) for c,_ in next,states do - if data[c] then + local v=data[c] + if v then v.state=value else c=topattern(c,true,true) @@ -7501,13 +8204,13 @@ end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple -function logs.setprogram () end -function logs.extendbanner() end -function logs.reportlines () end -function logs.reportbanner() end -function logs.reportline () end -function logs.simplelines () end -function logs.help () end +logs.setprogram=ignore +logs.extendbanner=ignore +logs.reportlines=ignore +logs.reportbanner=ignore +logs.reportline=ignore +logs.simplelines=ignore +logs.help=ignore local Carg,C,lpegmatch=lpeg.Carg,lpeg.C,lpeg.match local p_newline=lpeg.patterns.newline local linewise=( @@ -7584,10 +8287,11 @@ function logs.application(t) end return t end -function logs.system(whereto,process,jobname,category,...) - local message=formatters["%s %s => %s => %s => %s\r"](os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) +local f_syslog=formatters["%s %s => %s => %s => %s\r"] +function logs.system(whereto,process,jobname,category,fmt,arg,...) + local message=f_syslog(datetime("%d/%m/%y %H:%m:%S"),process,jobname,category,arg==nil and fmt or format(fmt,arg,...)) for i=1,10 do - local f=io.open(whereto,"a") + local f=openfile(whereto,"a") if f then f:write(message) f:close() @@ -7649,7 +8353,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6501, stripped down to: 5156 +-- original size: 6704, stripped down to: 5343 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -7659,7 +8363,7 @@ if not modules then modules={} end modules ['trac-inf']={ license="see context related readme files" } local type,tonumber,select=type,tonumber,select -local format,lower=string.format,string.lower +local format,lower,find=string.format,string.lower,string.find local concat=table.concat local clock=os.gettimeofday or os.clock local setmetatableindex=table.setmetatableindex @@ -7750,7 +8454,8 @@ function statistics.show() if statistics.enable then local register=statistics.register register("used platform",function() - return format("%s, type: %s, binary subtree: %s",os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") + return format("%s, type: %s, binary subtree: %s", + os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) register("luatex banner",function() return lower(status.banner) @@ -7763,14 +8468,23 @@ function statistics.show() return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) if jit then - local status={ jit.status() } - if status[1] then - register("luajit status",function() - return concat(status," ",2) - end) - end - end - register("current memory usage",statistics.memused) + local jitstatus={ jit.status() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end + end + register("lua properties",function() + local list=status.list() + local hashchar=tonumber(list.luatex_hashchars) + local mask=lua.mask or "ascii" + return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", + jit and "luajit" or "lua", + statistics.memused(), + list.luatex_hashtype or "default", + hashchar and 2^hashchar or "unknown", + mask, + mask=="utf" and "τεχ" or "tex") + end) register("runtime",statistics.runtime) logs.newline() for i=1,#statusinfo do @@ -7812,15 +8526,6 @@ function statistics.tracefunction(base,tag,...) statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end) end end -commands=commands or {} -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) -end end -- of closure @@ -7829,7 +8534,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5773, stripped down to: 3453 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -7846,14 +8551,16 @@ local namespaces=namespaces local registered={} local function report_index(k,name) if trace_namespaces then - report_system("reference to %a in protected namespace %a: %s",k,name,debug.traceback()) + report_system("reference to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) else report_system("reference to %a in protected namespace %a",k,name) end end local function report_newindex(k,name) if trace_namespaces then - report_system("assignment to %a in protected namespace %a: %s",k,name,debug.traceback()) + report_system("assignment to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) else report_system("assignment to %a in protected namespace %a",k,name) end @@ -8104,7 +8811,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3708, stripped down to: 2568 +-- original size: 3898, stripped down to: 2644 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8184,20 +8891,22 @@ end function debugger.disable() debug.sethook() end -function traceback() - local level=1 +local function showtraceback(rep) + local level=2 + local reporter=rep or report while true do - local info=debug.getinfo(level,"Sl") + local info=getinfo(level,"Sl") if not info then break elseif info.what=="C" then - print(format("%3i : C function",level)) + reporter("%2i : %s",level-1,"C function") else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + reporter("%2i : %s : %s",level-1,info.short_src,info.currentline) end level=level+1 end end +debugger.showtraceback=showtraceback end -- of closure @@ -8383,7 +9092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 6251, stripped down to: 3488 +-- original size: 7100, stripped down to: 3978 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -8425,7 +9134,7 @@ local sqlescape=lpeg.replacer { { "\r\n","\\n" }, { "\r","\\n" }, } -local sqlquoted=lpeg.Cs(lpeg.Cc("'")*sqlescape*lpeg.Cc("'")) +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) lpegpatterns.sqlescape=sqlescape lpegpatterns.sqlquoted=sqlquoted local luaescape=lpegpatterns.luaescape @@ -8448,12 +9157,24 @@ local quotedescapers={ local luaescaper=escapers.lua local quotedluaescaper=quotedescapers.lua local function replacekeyunquoted(s,t,how,recurse) - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end end local function replacekeyquoted(s,t,how,recurse) - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end local single=P("%") local double=P("%%") @@ -8468,11 +9189,16 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' -local key=nosingle*((C((1-nosingle )^1)*Carg(1)*Carg(2)*Carg(3))/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional local any=P(1) - replacer=Cs((unquoted+quoted+escape+key+any)^0) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -8511,7 +9237,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8807, stripped down to: 5085 +-- original size: 8022, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -8522,7 +9248,7 @@ if not modules then modules={} end modules ['util-env']={ } local allocate,mark=utilities.storage.allocate,utilities.storage.mark local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find -local unquoted,quoted=string.unquoted,string.quoted +local unquoted,quoted,optionalquoted=string.unquoted,string.quoted,string.optionalquoted local concat,insert,remove=table.concat,table.insert,table.remove environment=environment or {} local environment=environment @@ -8635,24 +9361,14 @@ function environment.splitarguments(separator) return before,after end function environment.reconstructcommandline(arg,noquote) + local resolveprefix=resolvers.resolve arg=arg or environment.originalarguments if noquote and #arg==1 then - local a=arg[1] - a=resolvers.resolve(a) - a=unquoted(a) - return a + return unquoted(resolveprefix and resolveprefix(arg[1]) or arg[1]) elseif #arg>0 then local result={} for i=1,#arg do - local a=arg[i] - a=resolvers.resolve(a) - a=unquoted(a) - a=gsub(a,'"','\\"') - if find(a," ") then - result[#result+1]=quoted(a) - else - result[#result+1]=a - end + result[i]=optionalquoted(resolveprefix and resolveprefix(arg[i]) or resolveprefix) end return concat(result," ") else @@ -8708,7 +9424,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 5930, stripped down to: 4235 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -8786,15 +9502,13 @@ function environment.luafilechunk(filename,silent) filename=file.replacesuffix(filename,"lua") local fullname=environment.luafile(filename) if fullname and fullname~="" then - local data=luautilities.loadedluacode(fullname,strippable,filename) - if trace_locating then + local data=luautilities.loadedluacode(fullname,strippable,filename) + if not silent then report_lua("loading file %a %s",fullname,not data and "failed" or "succeeded") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") end return data else - if trace_locating then + if not silent then report_lua("unknown file %a",filename) end return nil @@ -8863,7 +9577,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 42447, stripped down to: 26589 +-- original size: 45683, stripped down to: 27866 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -8878,10 +9592,10 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local utfchar=utf.char -local lpegmatch=lpeg.match +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters xml.xmlns=xml.xmlns or {} @@ -8976,8 +9690,10 @@ local function add_end(spacing,namespace,tag) top=stack[#stack] if #stack<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) elseif toclose.tg~=tag then errorstr=formatters["unable to close %s with %s %s"](toclose.tg,tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) end dt=top.dt dt[#dt+1]=toclose @@ -8986,10 +9702,29 @@ local function add_end(spacing,namespace,tag) end end local function add_text(text) + local n=#dt if cleanup and #text>0 then - dt[#dt+1]=cleanup(text) + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..cleanup(text) + else + dt[n+1]=cleanup(text) + end + else + dt[1]=cleanup(text) + end else - dt[#dt+1]=text + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..text + else + dt[n+1]=text + end + else + dt[1]=text + end end end local function add_special(what,spacing,text) @@ -9021,8 +9756,10 @@ local function attribute_specification_error(str) end return str end +local badentity="&error;" +local badentity="&" xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and "&error;" or formatters["&%s;"](str) end, + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, } @@ -9043,9 +9780,10 @@ local function fromdec(s) return formatters["d:%s"](s),true end end -local rest=(1-P(";"))^0 -local many=P(1)^0 -local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local p_rest=(1-P(";"))^0 +local p_many=P(1)^0 +local p_char=lpegpatterns.utf8character +local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) local predefined_unified={ [38]="&", [42]=""", @@ -9071,7 +9809,9 @@ local privates_u={ local privates_p={} local privates_n={ } -local escaped=utf.remapper(privates_u) +local escaped=utf.remapper(privates_u,"dynamic") +local unprivatized=utf.remapper(privates_p,"dynamic") +xml.unprivatized=unprivatized local function unescaped(s) local p=privates_n[s] if not p then @@ -9084,9 +9824,7 @@ local function unescaped(s) end return p end -local unprivatized=utf.remapper(privates_p) xml.privatetoken=unescaped -xml.unprivatized=unprivatized xml.privatecodes=privates_n local function handle_hex_entity(str) local h=hcache[str] @@ -9181,7 +9919,7 @@ local function handle_any_entity(str) report_xml("keeping entity &%s;",str) end if str=="" then - a="&error;" + a=badentity else a="&"..str..";" end @@ -9209,7 +9947,7 @@ local function handle_any_entity(str) if trace_entities then report_xml("invalid entity &%s;",str) end - a="&error;" + a=badentity acache[str]=a else if trace_entities then @@ -9222,8 +9960,14 @@ local function handle_any_entity(str) return a end end -local function handle_end_entity(chr) - report_xml("error in entity, %a found instead of %a",chr,";") +local function handle_end_entity(str) + report_xml("error in entity, %a found without ending %a",str,";") + return str +end +local function handle_crap_error(chr) + report_xml("error in parsing, unexpected %a found ",chr) + add_text(chr) + return chr end local space=S(' \r\n\t') local open=P('<') @@ -9239,15 +9983,15 @@ local valid=R('az','AZ','09')+S('_-.') local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop -local utfbom=lpeg.patterns.utfbom +local utfbom=lpegpatterns.utfbom local spacing=C(space^0) -local anyentitycontent=(1-open-semicolon-space-close)^0 +local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 local hexentitycontent=R("AF","af","09")^0 local decentitycontent=R("09")^0 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) )+(anyentitycontent/handle_any_entity) -local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) local somespace=space^1 @@ -9298,16 +10042,20 @@ local instruction=(spacing*begininstruction*someinstruction*endinstruction)/func local comment=(spacing*begincomment*somecomment*endcomment )/function(...) add_special("@cm@",...) end local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special("@cd@",...) end local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end +local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand +local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata +local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 local grammar_parsed_text=P { "preamble", preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, parent=beginelement*V("children")^0*endelement, - children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction, + children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } local grammar_unparsed_text=P { "preamble", preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, parent=beginelement*V("children")^0*endelement, - children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction, + children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) settings=settings or {} @@ -9341,7 +10089,6 @@ local function _xmlconvert_(data,settings) errorstr="empty xml file" elseif utfize or resolve then if lpegmatch(grammar_parsed_text,data) then - errorstr="" else errorstr="invalid xml file - parsed text" end @@ -9357,6 +10104,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } +setmetatable(result,mt) +setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -9389,8 +10138,11 @@ local function _xmlconvert_(data,settings) end if errorstr and errorstr~="" then result.error=true + else + errorstr=nil end result.statistics={ + errormessage=errorstr, entities={ decimals=dcache, hexadecimals=hcache, @@ -9404,7 +10156,7 @@ local function _xmlconvert_(data,settings) reported_attribute_errors,mt,errorhandler=nil,nil,nil return result end -function xmlconvert(data,settings) +local function xmlconvert(data,settings) local ok,result=pcall(function() return _xmlconvert_(data,settings) end) if ok then return result @@ -9496,14 +10248,17 @@ function xml.checkbom(root) insert(dt,2,"\n" ) end end -local function verbose_element(e,handlers) +local f_attribute=formatters['%s=%q'] +local function verbose_element(e,handlers,escape) local handle=handlers.handle local serialize=handlers.serialize local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn local ats=eat and next(eat) and {} if ats then + local n=0 for k,v in next,eat do - ats[#ats+1]=formatters['%s=%q'](k,escaped(v)) + n=n+1 + ats[n]=f_attribute(k,escaped(v)) end end if ern and trace_entities and ern~=ens then @@ -9588,23 +10343,25 @@ local function verbose_document(e,handlers) end end local function serialize(e,handlers,...) - local initialize=handlers.initialize - local finalize=handlers.finalize - local functions=handlers.functions - if initialize then - local state=initialize(...) - if not state==true then - return state + if e then + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() end - end - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) - end - if finalize then - return finalize() end end local function xserialize(e,handlers) @@ -9845,7 +10602,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48956, stripped down to: 30516 +-- original size: 48229, stripped down to: 30684 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10230,8 +10987,8 @@ local lp_builtin=P ( P("ns")/"ll.ns" )*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") -lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" -lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" local lp_fastpos=lp_fastpos_n+lp_fastpos_p local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" @@ -10410,7 +11167,7 @@ local function nodesettostring(set,nodetest) if not ns or ns=="" then ns="*" end if not tg or tg=="" then tg="*" end tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i]=(directive and tg) or format("not(%s)",tg) + t[#t+1]=(directive and tg) or format("not(%s)",tg) end if nodetest==false then return format("not(%s)",concat(t,"|")) @@ -10676,7 +11433,6 @@ expressions.print=function(...) print(...) return true end -expressions.contains=find expressions.find=find expressions.upper=upper expressions.lower=lower @@ -10698,6 +11454,9 @@ function expressions.contains(str,pattern) end return false end +function xml.expressions.idstring(str) + return type(str)=="string" and gsub(str,"^#","") or "" +end local function traverse(root,pattern,handle) local collected=applylpath(root,pattern) if collected then @@ -10826,8 +11585,13 @@ function xml.elements(root,pattern,reverse) local collected=applylpath(root,pattern) if not collected then return dummy - elseif reverse then - local c=#collected+1 + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 return function() if c>1 then c=c-1 @@ -10837,7 +11601,7 @@ function xml.elements(root,pattern,reverse) end end else - local n,c=#collected,0 + local c=0 return function() if c<n then c=c+1 @@ -10852,8 +11616,13 @@ function xml.collected(root,pattern,reverse) local collected=applylpath(root,pattern) if not collected then return dummy - elseif reverse then - local c=#collected+1 + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 return function() if c>1 then c=c-1 @@ -10861,7 +11630,7 @@ function xml.collected(root,pattern,reverse) end end else - local n,c=#collected,0 + local c=0 return function() if c<n then c=c+1 @@ -10876,7 +11645,7 @@ function xml.inspect(collection,pattern) report_lpath("pattern: %s\n\n%s\n",pattern,xml.tostring(e)) end end -local function split(e) +local function split(e) local dt=e.dt if dt then for i=1,#dt do @@ -10975,7 +11744,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 23804, stripped down to: 16817 +-- original size: 28786, stripped down to: 20578 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -10985,16 +11754,19 @@ if not modules then modules={} end modules ['lxml-aux']={ license="see context related readme files" } local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local trace_inclusions=false trackers.register("lxml.inclusions",function(v) trace_inclusions=v end) local report_xml=logs.reporter("xml") local xml=xml -local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath local xmlfilter=xml.filter -local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip local utfbyte=utf.byte +local lpegmatch=lpeg.match +local striplinepatterns=utilities.strings.striplinepatterns local function report(what,pattern,c,e) report_xml("%s element %a, root %a, position %a, index %a, pattern %a",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end @@ -11049,13 +11821,15 @@ end function xml.each(root,pattern,handle,reverse) local collected=xmlapplylpath(root,pattern) if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) + if handle then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end end return collected @@ -11111,6 +11885,7 @@ local function redo_ni(d) end end end +xml.reindex=redo_ni local function xmltoelement(whatever,root) if not whatever then return nil @@ -11162,8 +11937,16 @@ function xml.delete(root,pattern) report('deleting',pattern,c,e) end local d=p.dt - remove(d,e.ni) - redo_ni(d) + local ni=e.ni + if ni<=#d then + if false then + p.dt[ni]="" + else + remove(d,ni) + redo_ni(d) + end + else + end end end end @@ -11283,28 +12066,40 @@ xml.insertafter=insert_element xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end xml.injectafter=inject_element xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end -local function include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata,level) pattern=pattern or 'include' loaddata=loaddata or io.loaddata local collected=xmlapplylpath(xmldata,pattern) if collected then + if not level then + level=1 + end for c=1,#collected do local ek=collected[c] local name=nil local ekdt=ek.dt local ekat=ek.at - local epdt=ek.__p__.dt + local ekrt=ek.__p__ + local epdt=ekrt.dt if not attribute or attribute=="" then name=(type(ekdt)=="table" and ekdt[1]) or ekdt end if not name then for a in gmatch(attribute or "href","([^|]+)") do name=ekat[a] - if name then break end + if name then + break + end + end + end + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") end end - local data=(name and name~="" and loaddata(name)) or "" - if data=="" then + if not data or data=="" then epdt[ek.ni]="" elseif ekat["parse"]=="text" then epdt[ek.ni]=xml.escaped(data) @@ -11314,70 +12109,127 @@ local function include(xmldata,pattern,attribute,recursive,loaddata) epdt[ek.ni]="" else if recursive then - include(xi,pattern,attribute,recursive,loaddata) + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local inclusions=xmldata.settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + else + xmldata.settings.inclusions={ name } + end + if child.er then + local badinclusions=xmldata.settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + xmldata.settings.badinclusions={ name } + end end - epdt[ek.ni]=xml.body(xi) end end end end end xml.include=include +function xml.inclusion(e,default) + while e do + local f=e.__f__ + if f then + return f + else + e=e.__p__ + end + end + return default +end +local function getinclusions(key,e,sorted) + while e do + local settings=e.settings + if settings then + local inclusions=settings[key] + if inclusions then + inclusions=table.unique(inclusions) + if sorted then + table.sort(inclusions) + end + return inclusions + else + e=e.__p__ + end + else + e=e.__p__ + end + end +end +function xml.inclusions(e,sorted) + return getinclusions("inclusions",e,sorted) +end +function xml.badinclusions(e,sorted) + return getinclusions("badinclusions",e,sorted) +end +local b_collapser=lpeg.patterns.b_collapser +local m_collapser=lpeg.patterns.m_collapser +local e_collapser=lpeg.patterns.e_collapser +local b_stripper=lpeg.patterns.b_stripper +local m_stripper=lpeg.patterns.m_stripper +local e_stripper=lpeg.patterns.e_stripper +local lpegmatch=lpeg.match local function stripelement(e,nolines,anywhere) local edt=e.dt if edt then - if anywhere then - local t,n={},0 - for e=1,#edt do + local n=#edt + if n==0 then + return e + elseif anywhere then + local t={} + local m=0 + for e=1,n do local str=edt[e] if type(str)~="string" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str elseif str~="" then if nolines then - str=gsub(str,"%s+"," ") + str=lpegmatch((n==1 and b_collapser) or (n==m and e_collapser) or m_collapser,str) + else + str=lpegmatch((n==1 and b_stripper) or (n==m and e_stripper) or m_stripper,str) end - str=gsub(str,"^%s*(.-)%s*$","%1") if str~="" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str end end end e.dt=t else - if #edt>0 then - local str=edt[1] - if type(str)~="string" then - elseif str=="" then + local str=edt[1] + if type(str)=="string" then + if str~="" then + str=lpegmatch(nolines and b_collapser or b_stripper,str) + end + if str=="" then remove(edt,1) + n=n-1 else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"^%s+","") - if str=="" then - remove(edt,1) - else - edt[1]=str - end + edt[1]=str end end - local nedt=#edt - if nedt>0 then - local str=edt[nedt] - if type(str)~="string" then - elseif str=="" then - remove(edt) - else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"%s+$","") + if n>0 then + str=edt[n] + if type(str)=="string" then if str=="" then remove(edt) else - edt[nedt]=str + str=lpegmatch(nolines and e_collapser or e_stripper,str) + if str=="" then + remove(edt) + else + edt[n]=str + end end end end @@ -11563,8 +12415,8 @@ function xml.finalizers.xml.cdata(collected) end return "" end -function xml.insertcomment(e,str,n) - table.insert(e.dt,n or 1,{ +function xml.insertcomment(e,str,n) + insert(e.dt,n or 1,{ tg="@cm@", ns="", special=true, @@ -11572,7 +12424,25 @@ function xml.insertcomment(e,str,n) dt={ str }, }) end -function xml.setcdata(e,str) +function xml.insertcdata(e,str,n) + insert(e.dt,n or 1,{ + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcomment(e,str,n) + e.dt={ { + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.setcdata(e,str) e.dt={ { tg="@cd@", ns="", @@ -11642,7 +12512,7 @@ local function recurse(e,action) for i=1,#edt do local str=edt[i] if type(str)~="string" then - recurse(str,action,recursive) + recurse(str,action) elseif str~="" then edt[i]=action(str) end @@ -11660,6 +12530,65 @@ function helpers.recursetext(collected,action,recursive) end end end +local specials={ + ["@rt@"]="root", + ["@pi@"]="instruction", + ["@cm@"]="comment", + ["@dt@"]="declaration", + ["@cd@"]="cdata", +} +local function convert(x,strip,flat) + local ns=x.ns + local tg=x.tg + local at=x.at + local dt=x.dt + local node=flat and { + [0]=(not x.special and (ns~="" and ns..":"..tg or tg)) or nil, + } or { + _namespace=ns~="" and ns or nil, + _tag=not x.special and tg or nil, + _type=specials[tg] or "_element", + } + if at then + for k,v in next,at do + node[k]=v + end + end + local n=0 + for i=1,#dt do + local di=dt[i] + if type(di)=="table" then + if flat and di.special then + else + di=convert(di,strip,flat) + if di then + n=n+1 + node[n]=di + end + end + elseif strip then + di=lpegmatch(strip,di) + if di~="" then + n=n+1 + node[n]=di + end + else + n=n+1 + node[n]=di + end + end + if next(node) then + return node + end +end +function xml.totable(x,strip,flat) + if type(x)=="table" then + if strip then + strip=striplinepatterns[strip] + end + return convert(x,strip,flat) + end +end end -- of closure @@ -12216,7 +13145,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 7898, stripped down to: 5501 +-- original size: 11085, stripped down to: 7662 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -12225,14 +13154,15 @@ if not modules then modules={} end modules ['data-ini']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } +local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char -local next,type=next,type local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") -local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv resolvers=resolvers or {} local resolvers=resolvers texconfig.kpse_init=false @@ -12360,15 +13290,108 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if profiler then +if type(profiler)=="table" and not jit then directives.register("system.profile",function() profiler.start("luatex-profile.log") end) end -if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local resolved={} +local abstract={} +local dynamic={} +function resolvers.resetresolve(str) + resolved,abstract={},{} +end +function resolvers.allprefixes(separator) + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" + end + end + return all +end +local function _resolve_(method,target) + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +function resolvers.unresolve(str) + return abstract[str] or str +end +function resolvers.setdynamic(str) + dynamic[str]=true +end +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local prefix=C(R("az")^2)*P(":") +local target=C((1-S(" \"\';,"))^1) +local notarget=(#S(";,")+P(-1))*Cc("") +local p_resolve=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) +local p_simple=prefix*P(-1) +local function resolve(str) + if type(str)=="table" then + local res={} + for i=1,#str do + res[i]=resolve(str[i]) + end + return res + end + local res=resolved[str] + if res then + return res + end + local simple=lpegmatch(p_simple,str) + local action=prefixes[simple] + if action then + local res=action(res) + if not dynamic[simple] then + resolved[simple]=res + abstract[res]=simple + end + return res + end + res=lpegmatch(p_resolve,str) + resolved[str]=res + abstract[res]=str + return res +end +resolvers.resolve=resolve +if type(osuname)=="function" then + for k,v in next,osuname() do + if not prefixes[k] then + prefixes[k]=function() return v end + end + end +end +if ostype=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end + end + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + table.setmetatablenewindex(prefixes,makepattern) + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -12378,7 +13401,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 15303, stripped down to: 9716 +-- original size: 17216, stripped down to: 10657 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -12392,12 +13415,16 @@ local concat,sort=table.concat,table.sort local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next +local isdir=lfs.isdir local ostype=os.type -local collapsepath=file.collapsepath +local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_globbing=true trackers.register("resolvers.globbing",function(v) trace_globbing=v end) local report_expansions=logs.reporter("resolvers","expansions") +local report_globbing=logs.reporter("resolvers","globbing") local resolvers=resolvers +local resolveprefix=resolvers.resolve local function f_both(a,b) local t,n={},0 for sb in gmatch(b,"[^,]+") do @@ -12487,35 +13514,27 @@ function resolvers.expandedpathfromlist(pathlist) end return newlist end -local cleanup=lpeg.replacer { - { "!","" }, - { "\\","/" }, -} -function resolvers.cleanpath(str) - local doslashes=(P("\\")/"/"+1)^0 - local donegation=(P("!")/"" )^0 - local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") - if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" - else - return lpegmatch(cleanup,str) +local usedhomedir=nil +local donegation=(P("!")/"" )^0 +local doslashes=(P("\\")/"/"+1)^0 +local function expandedhome() + if not usedhomedir then + usedhomedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if usedhomedir=="~" or usedhomedir=="" or not isdir(usedhomedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent path using current path") end - end - else - local dohome=((P("~")+P("$HOME"))/homedir)^0 - local cleanup=Cs(donegation*dohome*doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" + usedhomedir="." end end - return resolvers.cleanpath(str) + return usedhomedir end -local expandhome=P("~")/"$HOME" +local dohome=((P("~")+P("$HOME")+P("%HOME%"))/expandedhome)^0 +local cleanup=Cs(donegation*dohome*doslashes) +resolvers.cleanpath=function(str) + return str and lpegmatch(cleanup,str) or "" +end +local expandhome=P("~")/"$HOME" local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" local dostring=(expandhome+1 )^0 @@ -12567,46 +13586,67 @@ function resolvers.splitpath(str) end function resolvers.joinpath(str) if type(str)=='table' then - return file.joinpath(str) + return joinpath(str) else return str end end local attributes,directory=lfs.attributes,lfs.dir local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local lessweird=P(".")^1+lpeg.anywhere(S("~`#$%^&*:;\"\'||<>,?\n\r\t")) local timer={} local scanned={} local nofscans=0 local scancache={} -local function scan(files,spec,path,n,m,r) - local full=(path=="" and spec) or (spec..path..'/') +local fullcache={} +local nofsharedscans=0 +local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) + local full=path=="" and spec or (spec..path..'/') local dirs={} local nofdirs=0 + local pattern=tolerant and lessweird or weird for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then + if not lpegmatch(pattern,name) then + local mode=attributes(full..name,"mode") + if mode=="file" then n=n+1 - local f=files[name] - if f then - if type(f)=='string' then - files[name]={ f,path } + local lower=lower(name) + local paths=files[lower] + if paths then + if onlyone then else - f[#f+1]=path + if type(paths)=="string" then + files[lower]={ paths,path } + else + paths[#paths+1]=path + end + if name~=lower then + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end + end end else - files[name]=path - local lower=lower(name) + files[lower]=path if name~=lower then - files["remap:"..lower]=name - r=r+1 + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end end end - elseif mode=='directory' then + elseif mode=="directory" then m=m+1 nofdirs=nofdirs+1 if path~="" then - dirs[nofdirs]=path..'/'..name + dirs[nofdirs]=path.."/"..name else dirs[nofdirs]=name end @@ -12616,107 +13656,69 @@ local function scan(files,spec,path,n,m,r) if nofdirs>0 then sort(dirs) for i=1,nofdirs do - files,n,m,r=scan(files,spec,dirs[i],n,m,r) + files,remap,n,m,r=scan(files,remap,spec,dirs[i],n,m,r,onlyonce,tolerant) end end scancache[sub(full,1,-2)]=files - return files,n,m,r + return files,remap,n,m,r end -local fullcache={} -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) +function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) + local realpath=resolveprefix(path) if usecache then - local files=fullcache[realpath] - if files then + local content=fullcache[realpath] + if content then if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) + report_expansions("using cached scan of path %a, branch %a",path,branch or path) end - return files + nofsharedscans=nofsharedscans+1 + return content end end + statistics.starttiming(timer) if trace_locating then report_expansions("scanning path %a, branch %a",path,branch or path) end - local files,n,m,r=scan({},realpath..'/',"",0,0,0) - files.__path__=path - files.__files__=n - files.__directories__=m - files.__remappings__=r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1]=realpath - fullcache[realpath]=files - end - nofscans=nofscans+1 - statistics.stoptiming(timer) - return files -end -local function simplescan(files,spec,path) - local full=(path=="" and spec) or (spec..path..'/') - local dirs={} - local nofdirs=0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then - if not files[name] then - files[name]=path - end - elseif mode=='directory' then - nofdirs=nofdirs+1 - if path~="" then - dirs[nofdirs]=path..'/'..name - else - dirs[nofdirs]=name - end - end - end - end - if nofdirs>0 then - sort(dirs) - for i=1,nofdirs do - files=simplescan(files,spec,dirs[i]) - end - end - return files -end -local simplecache={} -local nofsharedscans=0 -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) - if usecache then - local files=simplecache[realpath] - if not files then - files=scancache[realpath] - if files then - nofsharedscans=nofsharedscans+1 - end + local content + if isdir(realpath) then + local files,remap,n,m,r=scan({},{},realpath..'/',"",0,0,0,onlyonce,tolerant) + content={ + metadata={ + path=path, + files=n, + directories=m, + remappings=r, + }, + files=files, + remap=remap, + } + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) end - if files then - if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) - end - return files + else + content={ + metadata={ + path=path, + files=0, + directories=0, + remappings=0, + }, + files={}, + remap={}, + } + if trace_locating then + report_expansions("invalid path %a",realpath) end end - if trace_locating then - report_expansions("scanning path %a, branch %a",path,branch or path) - end - local files=simplescan({},realpath..'/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) - end if usecache then scanned[#scanned+1]=realpath - simplecache[realpath]=files + fullcache[realpath]=content end nofscans=nofscans+1 statistics.stoptiming(timer) - return files + return content +end +function resolvers.simplescanfiles(path,branch,usecache) + return resolvers.scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -12727,6 +13729,52 @@ function resolvers.scandata() paths=scanned, } end +function resolvers.get_from_content(content,path,name) + if not content then + return + end + local files=content.files + if not files then + return + end + local remap=content.remap + if not remap then + return + end + if name then + local used=lower(name) + return path,remap[used] or used + else + local name=path + local used=lower(name) + local path=files[used] + if path then + return path,remap[used] or used + end + end +end +local nothing=function() end +function resolvers.filtered_from_content(content,pattern) + if content and type(pattern)=="string" then + local pattern=lower(pattern) + local files=content.files + local remap=content.remap + if files and remap then + local n=next(files) + local function iterator() + while n do + local k=n + n=next(files,k) + if find(k,pattern) then + return files[k],remap and remap[k] or k + end + end + end + return iterator + end + end + return nothing +end end -- of closure @@ -12735,7 +13783,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 8769, stripped down to: 6490 +-- original size: 9216, stripped down to: 6798 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -12753,10 +13801,12 @@ local formats=allocate() local suffixes=allocate() local dangerous=allocate() local suffixmap=allocate() +local usertypes=allocate() resolvers.formats=formats resolvers.suffixes=suffixes resolvers.dangerous=dangerous resolvers.suffixmap=suffixmap +resolvers.usertypes=usertypes local luasuffixes=utilities.lua.suffixes local relations=allocate { core={ @@ -12824,11 +13874,13 @@ local relations=allocate { names={ "mp" }, variable='MPINPUTS', suffixes={ 'mp','mpvi','mpiv','mpii' }, + usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ 'tex',"mkvi","mkiv","mkii" }, + suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + usertype=true, }, icc={ names={ "icc","icc profile","icc profiles" }, @@ -12844,6 +13896,7 @@ local relations=allocate { names={ "lua" }, variable='LUAINPUTS', suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + usertype=true, }, lib={ names={ "lib" }, @@ -12852,11 +13905,15 @@ local relations=allocate { }, bib={ names={ 'bib' }, + variable='BIBINPUTS', suffixes={ 'bib' }, + usertype=true, }, bst={ names={ 'bst' }, + variable='BSTINPUTS', suffixes={ 'bst' }, + usertype=true, }, fontconfig={ names={ 'fontconfig','fontconfig file','fontconfig files' }, @@ -12938,8 +13995,9 @@ function resolvers.updaterelations() for name,relation in next,categories do local rn=relation.names local rv=relation.variable - local rs=relation.suffixes if rn and rv then + local rs=relation.suffixes + local ru=relation.usertype for i=1,#rn do local rni=lower(gsub(rn[i]," ","")) formats[rni]=rv @@ -12951,8 +14009,9 @@ function resolvers.updaterelations() end end end - end - if rs then + if ru then + usertypes[name]=true + end end end end @@ -13003,7 +14062,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15532, stripped down to: 11648 +-- original size: 15618, stripped down to: 11629 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -13013,7 +14072,7 @@ if not modules then modules={} end modules ['data-tmp']={ license="see context related readme files" } local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat,serialize,serializetofile=table.concat,table.serialize,table.tofile +local concat=table.concat local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable local formatters=string.formatters @@ -13022,6 +14081,7 @@ local trace_cache=false trackers.register("resolvers.cache",function(v) trace_ca local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers +local cleanpath=resolvers.cleanpath local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) local compile=utilities.lua.compile @@ -13043,7 +14103,7 @@ caches.relocate=false caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } local writable,readables,usedreadables=nil,{},{} local function identify() - local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] @@ -13281,15 +14341,11 @@ end local saveoptions={ compact=true } function caches.savedata(filepath,filename,data,raw) local tmaname,tmcname=caches.setluanames(filepath,filename) - local reduce,simplify=true,true - if raw then - reduce,simplify=false,false - end data.cache_uuid=os.uuid() if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) + file.savedata(tmaname,table.serialize(data,true,saveoptions)) else - serializetofile(tmaname,data,true,saveoptions) + table.tofile(tmaname,data,true,saveoptions) end utilities.lua.compile(tmaname,tmcname) end @@ -13297,10 +14353,12 @@ local content_state={} function caches.contentstate() return content_state or {} end -function caches.loadcontent(cachename,dataname) - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.loadcontent(cachename,dataname,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() @@ -13332,10 +14390,12 @@ function caches.collapsecontent(content) end end end -function caches.savecontent(cachename,dataname,content) - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.savecontent(cachename,dataname,content,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) if trace_locating then @@ -13350,7 +14410,7 @@ function caches.savecontent(cachename,dataname,content) content=content, uuid=os.uuid(), } - local ok=io.savedata(luaname,serialize(data,true)) + local ok=io.savedata(luaname,table.serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) @@ -13378,7 +14438,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5453, stripped down to: 4007 +-- original size: 5347, stripped down to: 4015 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -13406,8 +14466,8 @@ local function splitmethod(filename) if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") - if not find(filename,"://") then + filename=file.collapsepath(filename,".") + if not find(filename,"://",1,true) then return { scheme="file",path=filename,original=filename,filename=filename } end local specification=url.hashed(filename) @@ -13497,7 +14557,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 61799, stripped down to: 42957 +-- original size: 67003, stripped down to: 46291 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -13507,7 +14567,7 @@ if not modules then modules={} end modules ['data-res']={ license="see context related readme files", } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch -local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local concat,insert,remove,sortedkeys,sortedhash=table.concat,table.insert,table.remove,table.sortedkeys,table.sortedhash local next,type,rawget=next,type,rawget local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg @@ -13516,27 +14576,38 @@ local formatters=string.formatters local filedirname=file.dirname local filebasename=file.basename local suffixonly=file.suffixonly +local addsuffix=file.addsuffix +local removesuffix=file.removesuffix local filejoin=file.join local collapsepath=file.collapsepath local joinpath=file.joinpath +local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local getcurrentdir=lfs.currentdir +local isfile=lfs.isfile +local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes -local getcurrentdir=lfs.currentdir -local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) -local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) +local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) local report_resolving=logs.reporter("resolvers","resolving") local resolvers=resolvers local expandedpathfromlist=resolvers.expandedpathfromlist local checkedvariable=resolvers.checkedvariable local splitconfigurationpath=resolvers.splitconfigurationpath local methodhandler=resolvers.methodhandler +local filtered=resolvers.filtered_from_content +local lookup=resolvers.get_from_content +local cleanpath=resolvers.cleanpath +local resolveprefix=resolvers.resolve local initializesetter=utilities.setters.initialize local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv -resolvers.cacheversion='1.0.1' -resolvers.configbanner='' +resolvers.cacheversion="1.100" +resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" @@ -13555,6 +14626,7 @@ end local unset_variable="unset" local formats=resolvers.formats local suffixes=resolvers.suffixes +local usertypes=resolvers.usertypes local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } @@ -13563,7 +14635,7 @@ local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) if instance then instance.environment[key]=value - ossetenv(key,raw and value or resolvers.resolve(value)) + ossetenv(key,raw and value or resolveprefix(value)) end end local function getenv(key) @@ -13577,7 +14649,7 @@ local function getenv(key) end resolvers.getenv=getenv resolvers.env=getenv -local function resolve(k) +local function resolvevariable(k) return instance.expansions[k] end local dollarstripper=lpeg.stripper("$") @@ -13586,19 +14658,19 @@ local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolve +local somevariable=R("az","AZ","09","__","--")^1/resolvevariable local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) return lpegmatch(variableexpander,var) or var end -function resolvers.newinstance() - if trace_locating then +function resolvers.newinstance() + if trace_locating then report_resolving("creating instance") - end + end local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() local newinstance={ environment=environment, @@ -13611,6 +14683,7 @@ function resolvers.newinstance() foundintrees=allocate(), hashes=allocate(), hashed=allocate(), + pathlists=false, specification=allocate(), lists=allocate(), data=allocate(), @@ -13623,6 +14696,7 @@ function resolvers.newinstance() savelists=true, pattern=nil, force_suffixes=true, + pathstack={}, } setmetatableindex(variables,function(t,k) local v @@ -13672,8 +14746,13 @@ function resolvers.reset() end local function reset_hashes() instance.lists={} + instance.pathlists=false instance.found={} end +local function reset_caches() + instance.lists={} + instance.pathlists=false +end local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( @@ -13725,13 +14804,13 @@ local function identify_configuration_files() for i=1,#cnfpaths do local filepath=cnfpaths[i] local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) if trace_locating then - local fullpath=gsub(resolvers.resolve(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c") + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) end - if lfs.isfile(realname) then + if isfile(realname) then specification[#specification+1]=filename if trace_locating then report_resolving("found configuration file %a",realname) @@ -13753,7 +14832,7 @@ local function load_configuration_files() local filename=specification[i] local pathname=filedirname(filename) local filename=filejoin(pathname,luacnfname) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local setups=instance.setups @@ -13761,7 +14840,7 @@ local function load_configuration_files() local parent=data and data.parent if parent then local filename=filejoin(pathname,parent) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local parentdata=blob() @@ -13786,7 +14865,7 @@ local function load_configuration_files() elseif variables[k]==nil then if trace_locating and not warning then report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable", - k,resolvers.resolve(filename)) + k,resolveprefix(filename)) warning=true end variables[k]=v @@ -13846,7 +14925,7 @@ local function locate_file_databases() local stripped=lpegmatch(inhibitstripper,path) if stripped~="" then local runtime=stripped==path - path=resolvers.cleanpath(path) + path=cleanpath(path) local spec=resolvers.splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped @@ -13909,8 +14988,8 @@ function resolvers.renew(hashname) report_resolving("identifying tree %a",hashname) end end - local realpath=resolvers.resolve(hashname) - if lfs.isdir(realpath) then + local realpath=resolveprefix(hashname) + if isdir(realpath) then if trace_locating then report_resolving("using path %a",realpath) end @@ -14011,19 +15090,53 @@ end function resolvers.unexpandedpath(str) return joinpath(resolvers.unexpandedpathlist(str)) end +function resolvers.pushpath(name) + local pathstack=instance.pathstack + local lastpath=pathstack[#pathstack] + local pluspath=filedirname(name) + if lastpath then + lastpath=collapsepath(filejoin(lastpath,pluspath)) + else + lastpath=collapsepath(pluspath) + end + insert(pathstack,lastpath) + if trace_paths then + report_resolving("pushing path %a",lastpath) + end +end +function resolvers.poppath() + local pathstack=instance.pathstack + if trace_paths and #pathstack>0 then + report_resolving("popping path %a",pathstack[#pathstack]) + end + remove(pathstack) +end +function resolvers.stackpath() + local pathstack=instance.pathstack + local currentpath=pathstack[#pathstack] + return currentpath~="" and currentpath or nil +end local done={} function resolvers.resetextrapath() local ep=instance.extra_paths if not ep then - ep,done={},{} - instance.extra_paths=ep + done={} + instance.extra_paths={} elseif #ep>0 then - instance.lists,done={},{} + done={} + reset_caches() end end function resolvers.registerextrapath(paths,subpaths) - paths=settings_to_array(paths) - subpaths=settings_to_array(subpaths) + if not subpaths or subpaths=="" then + if not paths or path=="" then + return + elseif done[paths] then + return + end + end + local paths=settings_to_array(paths) + local subpaths=settings_to_array(subpaths) local ep=instance.extra_paths or {} local oldn=#ep local newn=oldn @@ -14038,7 +15151,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14048,7 +15161,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=resolvers.cleanpath(p) + ep[newn]=cleanpath(p) done[p]=true end end @@ -14060,7 +15173,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=ep[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14069,52 +15182,70 @@ function resolvers.registerextrapath(paths,subpaths) if newn>0 then instance.extra_paths=ep end - if newn>oldn then - instance.lists={} + if newn~=oldn then + reset_caches() end end -local function made_list(instance,list) - local ep=instance.extra_paths - if not ep or #ep==0 then - return list +function resolvers.pushextrapath(path) + local paths=settings_to_array(path) + if instance.extra_stack then + insert(instance.extra_stack,1,paths) else - local done,new,newn={},{},0 - for k=1,#list do - local v=list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v]=true - newn=newn+1 - new[newn]=v - else - break - end - end - end - for k=1,#ep do - local v=ep[k] + instance.extra_stack={ paths } + end + reset_caches() +end +function resolvers.popextrapath() + if instance.extra_stack then + reset_caches() + return remove(instance.extra_stack,1) + end +end +local function made_list(instance,list,extra_too) + local done={} + local new={} + local newn=0 + local function add(p) + for k=1,#p do + local v=p[k] if not done[v] then done[v]=true newn=newn+1 new[newn]=v end end - for k=1,#list do - local v=list[k] - if not done[v] then - done[v]=true - newn=newn+1 - new[newn]=v + end + for k=1,#list do + local v=list[k] + if done[v] then + elseif find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break + end + end + if extra_too then + local es=instance.extra_stack + if es and #es>0 then + for k=1,#es do + add(es[k]) end end - return new + local ep=instance.extra_paths + if ep and #ep>0 then + add(ep) + end end + add(list) + return new end function resolvers.cleanpathlist(str) local t=resolvers.expandedpathlist(str) if t then for i=1,#t do - t[i]=collapsepath(resolvers.cleanpath(t[i])) + t[i]=collapsepath(cleanpath(t[i])) end end return t @@ -14122,22 +15253,22 @@ end function resolvers.expandpath(str) return joinpath(resolvers.expandedpathlist(str)) end -function resolvers.expandedpathlist(str) +function resolvers.expandedpathlist(str,extra_too) if not str then return {} - elseif instance.savelists then + elseif instance.savelists then str=lpegmatch(dollarstripper,str) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else local lst=resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + return made_list(instance,expandedpathfromlist(lst),extra_too) end end function resolvers.expandedpathlistfromvariable(str) @@ -14148,6 +15279,13 @@ end function resolvers.expandpathfromvariable(str) return joinpath(resolvers.expandedpathlistfromvariable(str)) end +function resolvers.cleanedpathlist(v) + local t=resolvers.expandedpathlist(v) + for i=1,#t do + t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + end + return t +end function resolvers.expandbraces(str) local ori=str local pth=expandedpathfromlist(resolvers.splitpath(ori)) @@ -14164,7 +15302,7 @@ function resolvers.registerfilehash(name,content,someerror) end end local function isreadable(name) - local readable=lfs.isfile(name) + local readable=isfile(name) if trace_detail then if readable then report_resolving("file %a is readable",name) @@ -14174,70 +15312,57 @@ local function isreadable(name) end return readable end -local function collect_files(names) - local filelist,noffiles={},0 +local function collect_files(names) + local filelist={} + local noffiles=0 + local function check(hash,root,pathname,path,name) + if not pathname or find(path,pathname) then + local variant=hash.type + local search=filejoin(root,path,name) + local result=methodhandler('concatinators',variant,root,path,name) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + end for k=1,#names do - local fname=names[k] + local filename=names[k] if trace_detail then - report_resolving("checking name %a",fname) + report_resolving("checking name %a",filename) end - local bname=filebasename(fname) - local dname=filedirname(fname) - if dname=="" or find(dname,"^%.") then - dname=false + local basename=filebasename(filename) + local pathname=filedirname(filename) + if pathname=="" or find(pathname,"^%.") then + pathname=false else - dname=gsub(dname,"%*",".*") - dname="/"..dname.."$" + pathname=gsub(pathname,"%*",".*") + pathname="/"..pathname.."$" end local hashes=instance.hashes for h=1,#hashes do local hash=hashes[h] - local blobpath=hash.name - local files=blobpath and instance.files[blobpath] - if files then + local hashname=hash.name + local content=hashname and instance.files[hashname] + if content then if trace_detail then - report_resolving("deep checking %a, base %a, pattern %a",blobpath,bname,dname) + report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end - local blobfile=files[bname] - if not blobfile then - local rname="remap:"..bname - blobfile=files[rname] - if blobfile then - bname=files[rname] - blobfile=files[bname] - end - end - if blobfile then - local blobroot=files.__path__ or blobpath - if type(blobfile)=='string' then - if not dname or find(blobfile,dname) then - local variant=hash.type - local search=filejoin(blobroot,blobfile,bname) - local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + local path,name=lookup(content,basename) + if path then + local metadata=content.metadata + local realroot=metadata and metadata.path or hashname + if type(path)=="string" then + check(hash,realroot,pathname,path,name) else - for kk=1,#blobfile do - local vv=blobfile[kk] - if not dname or find(vv,dname) then - local variant=hash.type - local search=filejoin(blobroot,vv,bname) - local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + for i=1,#path do + check(hash,realroot,pathname,path[i],name) end end end elseif trace_locating then - report_resolving("no match in %a (%s)",blobpath,bname) + report_resolving("no match in %a (%s)",hashname,basename) end end end @@ -14262,7 +15387,7 @@ end local function can_be_dir(name) local fakepaths=instance.fakepaths if not fakepaths[name] then - if lfs.isdir(name) then + if isdir(name) then fakepaths[name]=1 else fakepaths[name]=2 @@ -14278,10 +15403,11 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if ext=="" or not suffixmap[ext] then local defaultsuffixes=resolvers.defaultsuffixes + local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname - filetype=resolvers.formatofsuffix(forcedname) + filetype=formatofsuffix(forcedname) if trace_locating then report_resolving("forcing filetype %a",filetype) end @@ -14317,18 +15443,18 @@ local function find_direct(filename,allresults) end end local function find_wildcard(filename,allresults) - if find(filename,'%*') then + if find(filename,'*',1,true) then if trace_locating then report_resolving("checking wildcard %a",filename) end - local method,result=resolvers.findwildcardfiles(filename) + local result=resolvers.findwildcardfiles(filename) if result then return "wildcard",result end end end local function find_qualified(filename,allresults,askedformat,alsostripped) - if not file.is_qualified_path(filename) then + if not is_qualified_path(filename) then return end if trace_locating then @@ -14402,33 +15528,66 @@ local function check_subpath(fname) return fname end end -local function find_intree(filename,filetype,wantedfiles,allresults) +local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec) - local method="intree" + local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local entry={} if pathlist and #pathlist>0 then - local filelist=collect_files(wantedfiles) + for k=1,#pathlist do + local path=pathlist[k] + local prescanned=find(path,'^!!') + local resursive=find(path,'//$') + local pathname=lpegmatch(inhibitstripper,path) + local expression=makepathexpression(pathname) + local barename=gsub(pathname,"/+$","") + barename=resolveprefix(barename) + local scheme=url.hasscheme(barename) + local schemename=gsub(barename,"%.%*$",'') + entry[k]={ + path=path, + pathname=pathname, + prescanned=prescanned, + recursive=recursive, + expression=expression, + barename=barename, + scheme=scheme, + schemename=schemename, + } + end + entry.typespec=typespec + list[filetype]=entry + else + list[filetype]=false + end + return entry +end +local function find_intree(filename,filetype,wantedfiles,allresults) + local pathlists=instance.pathlists + if not pathlists then + pathlists=setmetatableindex(allocate(),makepathlist) + instance.pathlists=pathlists + end + local pathlist=pathlists[filetype] + if pathlist then + local method="intree" + local filelist=collect_files(wantedfiles) local dirlist={} + local result={} if filelist then for i=1,#filelist do dirlist[i]=filedirname(filelist[i][3]).."/" end end if trace_detail then - report_resolving("checking filename %a",filename) + report_resolving("checking filename %a in tree",filename) end - local resolve=resolvers.resolve - local result={} for k=1,#pathlist do - local path=pathlist[k] - local pathname=lpegmatch(inhibitstripper,path) - local doscan=path==pathname - if not find (pathname,'//$') then - doscan=false - end + local entry=pathlist[k] + local path=entry.path + local pathname=entry.pathname local done=false if filelist then - local expression=makepathexpression(pathname) + local expression=entry.expression if trace_detail then report_resolving("using pattern %a for path %a",expression,pathname) end @@ -14436,8 +15595,8 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local fl=filelist[k] local f=fl[2] local d=dirlist[k] - if find(d,expression) or find(resolve(d),expression) then - result[#result+1]=resolve(fl[3]) + if find(d,expression) or find(resolveprefix(d),expression) then + result[#result+1]=resolveprefix(fl[3]) done=true if allresults then if trace_detail then @@ -14458,56 +15617,62 @@ local function find_intree(filename,filetype,wantedfiles,allresults) method="database" else method="filesystem" - pathname=gsub(pathname,"/+$","") - pathname=resolve(pathname) - local scheme=url.hasscheme(pathname) + local scheme=entry.scheme if not scheme or scheme=="file" then - local pname=gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then + local pname=entry.schemename + if not find(pname,"*",1,true) then if can_be_dir(pname) then - for k=1,#wantedfiles do - local w=wantedfiles[k] - local fname=check_subpath(filejoin(pname,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + if not done and not entry.prescanned then + if trace_detail then + report_resolving("quick root scan for %a",pname) end - end - if not done and doscan then - local files=resolvers.simplescanfiles(pname,false,true) for k=1,#wantedfiles do local w=wantedfiles[k] - local subpath=files[w] - if not subpath or subpath=="" then - elseif type(subpath)=="string" then - local fname=check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break end - else - for i=1,#subpath do - local sp=subpath[i] - if sp=="" then - else - local fname=check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break + end + end + if not done and entry.recursive then + if trace_detail then + report_resolving("scanning filesystem for %a",pname) + end + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end end end end - end - if done and not allresults then - break + if done and not allresults then + break + end end end end @@ -14515,6 +15680,18 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end else end + else + for k=1,#wantedfiles do + local pname=entry.barename + local fname=methodhandler('finders',pname.."/"..wantedfiles[k]) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end end end if done and not allresults then @@ -14549,10 +15726,13 @@ local function find_otherwise(filename,filetype,wantedfiles,allresults) local filelist=collect_files(wantedfiles) local fl=filelist and filelist[1] if fl then - return "otherwise",{ resolvers.resolve(fl[3]) } + return "otherwise",{ resolveprefix(fl[3]) } end end collect_instance_files=function(filename,askedformat,allresults) + if not filename or filename=="" then + return {} + end askedformat=askedformat or "" filename=collapsepath(filename,".") filename=gsub(filename,"^%./",getcurrentdir().."/") @@ -14587,7 +15767,11 @@ collect_instance_files=function(filename,askedformat,allresults) else local method,result,stamp,filetype,wantedfiles if instance.remember then - stamp=formatters["%s--%s"](filename,askedformat) + if askedformat=="" then + stamp=formatters["%s::%s"](suffixonly(filename),filename) + else + stamp=formatters["%s::%s"](askedformat,filename) + end result=stamp and instance.found[stamp] if result then if trace_locating then @@ -14606,7 +15790,7 @@ collect_instance_files=function(filename,askedformat,allresults) method,result=find_intree(filename,filetype,wantedfiles) if not result then method,result=find_onpath(filename,filetype,wantedfiles) - if not result then + if resolve_otherwise and not result then method,result=find_otherwise(filename,filetype,wantedfiles) end end @@ -14622,7 +15806,7 @@ collect_instance_files=function(filename,askedformat,allresults) end if stamp then if trace_locating then - report_resolving("remembering file %a",filename) + report_resolving("remembering file %a using hash %a",filename,stamp) end instance.found[stamp]=result end @@ -14630,6 +15814,9 @@ collect_instance_files=function(filename,askedformat,allresults) end end local function findfiles(filename,filetype,allresults) + if not filename or filename=="" then + return {} + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -14649,39 +15836,30 @@ function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) - local bname,result=filebasename(filename),{} + local base=filebasename(filename) + local result={} local hashes=instance.hashes - local noffound=0 + local function okay(hash,path,name) + local found=methodhandler('concatinators',hash.type,hash.name,path,name) + if found and found~="" then + result[#result+1]=resolveprefix(found) + return not allresults + end + end for k=1,#hashes do local hash=hashes[k] - local files=instance.files[hash.name] or {} - local blist=files[bname] - if not blist then - local rname="remap:"..bname - blist=files[rname] - if blist then - bname=files[rname] - blist=files[bname] - end - end - if blist then - if type(blist)=='string' then - local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then - break - end + local content=instance.files[hash.name] + if content then + local path,name=lookup(content,base) + if not path then + elseif type(path)=="string" then + if okay(hash,path,name) then + return result end else - for kk=1,#blist do - local vv=blist[kk] - local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then break end + for i=1,#path do + if okay(hash,path[i],name) then + return result end end end @@ -14695,64 +15873,80 @@ end function resolvers.findgivenfile(filename) return findgivenfiles(filename,false)[1] or "" end -local function doit(path,blist,bname,tag,variant,result,allresults) - local done=false - if blist and variant then - local resolve=resolvers.resolve - if type(blist)=='string' then - if find(lower(blist),path) then - local full=methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1]=resolve(full) - done=true - end - else - for kk=1,#blist do - local vv=blist[kk] - if find(lower(vv),path) then - local full=methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1]=resolve(full) - done=true - if not allresults then break end - end - end - end - end - return done -end local makewildcard=Cs( (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end -local function findwildcardfiles(filename,allresults,result) - result=result or {} +local function findwildcardfiles(filename,allresults,result) + local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files,done=instance.files,false - if find(name,"%*") then + local files=instance.files + if find(name,"*",1,true) then local hashes=instance.hashes + local function okay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - for kk,hh in next,files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + for found,base in filtered(files[hashname],name) do + if type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end end end end end else + local function okayokay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - if doit(path,files[hashname][base],base,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + local found,base=lookup(content,base) + if not found then + elseif type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end + end + end end end return result @@ -14825,7 +16019,7 @@ end function resolvers.dowithpath(name,func) local pathlist=resolvers.expandedpathlist(name) for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) + func("^"..cleanpath(pathlist[i])) end end function resolvers.dowithvariable(name,func) @@ -14833,23 +16027,23 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fullname=file.addsuffix(barename,"fmt") + local barename=removesuffix(name) + local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then fmtname=resolvers.findfile(fullname) - fmtname=resolvers.cleanpath(fmtname) + fmtname=cleanpath(fmtname) end if fmtname~="" then - local barename=file.removesuffix(fmtname) - local luaname=file.addsuffix(barename,luasuffixes.lua) - local lucname=file.addsuffix(barename,luasuffixes.luc) - local luiname=file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then + local barename=removesuffix(fmtname) + local luaname=addsuffix(barename,luasuffixes.lua) + local lucname=addsuffix(barename,luasuffixes.luc) + local luiname=addsuffix(barename,luasuffixes.lui) + if isfile(luiname) then return barename,luiname - elseif lfs.isfile(lucname) then + elseif isfile(lucname) then return barename,lucname - elseif lfs.isfile(luaname) then + elseif isfile(luaname) then return barename,luaname end end @@ -14871,29 +16065,24 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) local hash=hashes[i] local blobtype=hash.type local blobpath=hash.name - if blobpath then + if blobtype and blobpath then + local total=0 + local checked=0 + local done=0 if before then before(blobtype,blobpath,pattern) end - local files=instance.files[blobpath] - local total,checked,done=0,0,0 - if files then - for k,v in table.sortedhash(files) do - total=total+1 - if find(k,"^remap:") then - elseif find(k,pattern) then - if type(v)=="string" then - checked=checked+1 - if handle(blobtype,blobpath,v,k) then - done=done+1 - end - else - checked=checked+#v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done=done+1 - end - end + for path,name in filtered(instance.files[blobpath],pattern) do + if type(path)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,path,name) then + done=done+1 + end + else + checked=checked+#path + for i=1,#path do + if handle(blobtype,blobpath,path[i],name) then + done=done+1 end end end @@ -14904,8 +16093,8 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -resolvers.obsolete=resolvers.obsolete or {} -local obsolete=resolvers.obsolete +local obsolete=resolvers.obsolete or {} +resolvers.obsolete=obsolete resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles @@ -14916,7 +16105,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 6643, stripped down to: 4401 +-- original size: 3950, stripped down to: 2935 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -14926,44 +16115,51 @@ if not modules then modules={} end modules ['data-pre']={ license="see context related readme files" } local resolvers=resolvers -local prefixes=utilities.storage.allocate() -resolvers.prefixes=prefixes -local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local prefixes=resolvers.prefixes +local cleanpath=resolvers.cleanpath +local findgivenfile=resolvers.findgivenfile +local expansion=resolvers.expansion local getenv=resolvers.getenv -local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match -local joinpath,basename,dirname=file.join,file.basename,file.dirname -local getmetatable,rawset,type=getmetatable,rawset,type +local basename=file.basename +local dirname=file.dirname +local joinpath=file.join +local isfile=lfs.isfile prefixes.environment=function(str) return cleanpath(expansion(str)) end -prefixes.relative=function(str,n) - if io.exists(str) then - elseif io.exists("./"..str) then - str="./"..str - else - local p="../" - for i=1,n or 2 do - if io.exists(p..str) then - str=p..str - break - else - p=p.."../" +local function relative(str,n) + if not isfile(str) then + local pstr="./"..str + if isfile(pstr) then + str=pstr + else + local p="../" + for i=1,n or 2 do + local pstr=p..str + if isfile(pstr) then + str=pstr + break + else + p=p.."../" + end end end end return cleanpath(str) end +local function locate(str) + local fullname=findgivenfile(str) or "" + return cleanpath(fullname~="" and fullname or str) +end +prefixes.relative=relative +prefixes.locate=locate prefixes.auto=function(str) - local fullname=prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname=prefixes.locate(str) + local fullname=relative(str) + if not isfile(fullname) then + fullname=locate(str) end return fullname end -prefixes.locate=function(str) - local fullname=findgivenfile(str) or "" - return cleanpath((fullname~="" and fullname) or str) -end prefixes.filename=function(str) local fullname=findgivenfile(str) or "" return cleanpath(basename((fullname~="" and fullname) or str)) @@ -14984,6 +16180,13 @@ end prefixes.home=function(str) return cleanpath(joinpath(getenv('HOME'),str)) end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname local function toppath() local inputstack=resolvers.inputstack if not inputstack then @@ -14996,98 +16199,22 @@ local function toppath() return pathname end end -resolvers.toppath=toppath -prefixes.toppath=function(str) - return cleanpath(joinpath(toppath(),str)) -end -prefixes.env=prefixes.environment -prefixes.rel=prefixes.relative -prefixes.loc=prefixes.locate -prefixes.kpse=prefixes.locate -prefixes.full=prefixes.locate -prefixes.file=prefixes.filename -prefixes.path=prefixes.pathname -function resolvers.allprefixes(separator) - local all=table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i]=all[i]..":" - end - end - return all -end -local function _resolve_(method,target) - local action=prefixes[method] - if action then - return action(target) - else - return method..":"..target - end -end -local resolved,abstract={},{} -function resolvers.resetresolve(str) - resolved,abstract={},{} -end -local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) -local prefix=C(R("az")^2)*P(":") -local target=C((1-S(" \"\';,"))^1) -local notarget=(#S(";,")+P(-1))*Cc("") -local pattern=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) -local function resolve(str) - if type(str)=="table" then - local t={} - for i=1,#str do - t[i]=resolve(str[i]) - end - return t +local function jobpath() + local path=resolvers.stackpath() + if not path or path=="" then + return "." else - local res=resolved[str] - if not res then - res=lpegmatch(pattern,str) - resolved[str]=res - abstract[res]=str - end - return res - end -end -local function unresolve(str) - return abstract[str] or str -end -resolvers.resolve=resolve -resolvers.unresolve=unresolve -if type(os.uname)=="function" then - for k,v in next,os.uname() do - if not prefixes[k] then - prefixes[k]=function() return v end - end - end -end -if os.type=="unix" then - local pattern - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon=P(":") - for k,v in table.sortedpairs(prefixes) do - if p then - p=P(k)+p - else - p=P(k) - end - end - pattern=Cs((p*colon+colon/";"+P(1))^0) - end - makepattern() - getmetatable(prefixes).__newindex=makepattern - function resolvers.repath(str) - return lpegmatch(pattern,str) - end -else - function resolvers.repath(str) - return str + return path end end +resolvers.toppath=toppath +resolvers.jobpath=jobpath +prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end +prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end +resolvers.setdynamic("toppath") +resolvers.setdynamic("jobpath") +prefixes.jobfile=prefixes.jobpath +resolvers.setdynamic("jobfile") end -- of closure @@ -15149,7 +16276,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3801, stripped down to: 3231 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -15161,30 +16288,31 @@ if not modules then modules={} end modules ['data-fil']={ local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers +local resolveprefix=resolvers.resolve local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name=specification.filename - local realname=resolvers.resolve(name) + local filename=specification.filename + local realname=resolveprefix(filename) if realname and realname~='' and lfs.isdir(realname) then if trace_locating then - report_files("file locator %a found as %a",name,realname) + report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',name,true) + resolvers.appendhash('file',filename,true) elseif trace_locating then - report_files("file locator %a not found",name) + report_files("file locator %a not found",filename) end end function hashers.file(specification) - local name=specification.filename - local content=caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local pathname=specification.filename + local content=caches.loadcontent(pathname,'files') + resolvers.registerfilehash(pathname,content,content==nil) end function generators.file(specification) - local path=specification.filename - local content=resolvers.scanfiles(path,false,true) - resolvers.registerfilehash(path,content,true) + local pathname=specification.filename + local content=resolvers.scanfiles(pathname,false,true) + resolvers.registerfilehash(pathname,content,true) end concatinators.file=file.join function finders.file(specification,filetype) @@ -15375,7 +16503,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3913, stripped down to: 2998 +-- original size: 3899, stripped down to: 2984 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -15421,7 +16549,7 @@ end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile) - local enginebanner=status.list().banner + local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") local luvdata={ @@ -15434,7 +16562,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) end end function statistics.checkfmtstatus(texname) - local enginebanner=status.list().banner + local enginebanner=status.banner if enginebanner and texname then local luvname=file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then @@ -15466,7 +16594,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8489, stripped down to: 6757 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -15485,16 +16613,6 @@ zip.archives=zip.archives or {} local archives=zip.archives zip.registeredfiles=zip.registeredfiles or {} local registeredfiles=zip.registeredfiles -local limited=false -directives.register("system.inputmode",function(v) - if not limited then - local i_limiter=io.i_limiter(v) - if i_limiter then - zip.open=i_limiter.protect(zip.open) - limited=true - end - end -end) local function validzip(str) if not find(str,"^zip://") then return "zip:///"..str @@ -15509,7 +16627,7 @@ function zip.openarchive(name) local arch=archives[name] if not arch then local full=resolvers.findfile(name) or "" - arch=(full~="" and zip.open(full)) or false + arch=full~="" and zip.open(full) or false archives[name]=arch end return arch @@ -15668,31 +16786,42 @@ function resolvers.usezipfile(archive) end end function resolvers.registerzipfile(z,tree) - local files,filter={},"" - if tree=="" then - filter="^(.+)/(.-)$" - else - filter=format("^%s/(.+)/(.-)$",tree) - end + local names={} + local files={} + local remap={} + local n=0 + local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) + local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end - local register,n=resolvers.registerfile,0 for i in z:files() do - local path,name=match(i.filename,filter) - if path then - if name and name~='' then - register(files,name,path) - n=n+1 - else + local filename=i.filename + local path,name=match(filename,filter) + if not path then + n=n+1 + register(names,filename,"") + local usedname=lower(filename) + files[usedname]="" + if usedname~=filename then + remap[usedname]=filename end - else - register(files,i.filename,'') + elseif name and name~="" then n=n+1 + register(names,name,path) + local usedname=lower(name) + files[usedname]=path + if usedname~=name then + remap[usedname]=name + end + else end end report_zip("registering: %s files registered",n) - return files + return { + files=files, + remap=remap, + } end @@ -15702,7 +16831,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 2508, stripped down to: 2074 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -15711,42 +16840,64 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,gsub,format=string.find,string.gsub,string.format +local find,gsub,lower=string.find,string.gsub,string.lower +local basename,dirname,joinname=file.basename,file.dirname,file .join +local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile +local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local done,found,notfound={},{},resolvers.finders.notfound -function resolvers.finders.tree(specification) +local resolveprefix=resolvers.resolve +local notfound=resolvers.finders.notfound +local lookup=resolvers.get_from_content +local collectors={} +local found={} +function resolvers.finders.tree(specification) local spec=specification.filename - local fnd=found[spec] - if fnd==nil then + local okay=found[spec] + if okay==nil then if spec~="" then - local path,name=file.dirname(spec),file.basename(spec) - if path=="" then path="." end - local hash=done[path] - if not hash then - local pattern=path.."/*" - hash=dir.glob(pattern) - done[path]=hash + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names end local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for k=1,#hash do - local v=hash[k] - if find(v,pattern) then - found[spec]=v - return v + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then + found[spec]=fullname + return fullname + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end - fnd=notfound() - found[spec]=fnd + okay=notfound() + found[spec]=okay end - return fnd + return okay end function resolvers.locators.tree(specification) local name=specification.filename - local realname=resolvers.resolve(name) - if realname and realname~='' and lfs.isdir(realname) then + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then if trace_locating then report_trees("locator %a found",realname) end @@ -15757,16 +16908,110 @@ function resolvers.locators.tree(specification) end function resolvers.hashers.tree(specification) local name=specification.filename - if trace_locating then - report_trees("analysing %a",name) - end + report_trees("analyzing %a",name) resolvers.methodhandler("hashers",name) resolvers.generators.file(specification) end -resolvers.concatinators.tree=resolvers.concatinators.file -resolvers.generators.tree=resolvers.generators.file -resolvers.openers.tree=resolvers.openers.file -resolvers.loaders.tree=resolvers.loaders.file +local collectors={} +local splitter=lpeg.splitat("/**/") +local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } +table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=caches.loadcontent(dataname,"files",dataname) + if not content then + content=resolvers.scanfiles(rootname,nil,nil,false,true) + caches.savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content +end) +local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + end + return notfound() +end +local function resolve(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end + end + end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() + end + end + end + local path,name=dirname(filename),basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) + end + end + return notfound() +end +resolvers.finders .dirlist=resolve +resolvers.locators .dirlist=resolvers.locators .tree +resolvers.hashers .dirlist=resolvers.hashers .tree +resolvers.generators.dirlist=resolvers.generators.file +resolvers.openers .dirlist=resolvers.openers .file +resolvers.loaders .dirlist=resolvers.loaders .file +function resolvers.finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } + end + return resolve(specification) +end +resolvers.locators .dirfile=resolvers.locators .dirlist +resolvers.hashers .dirfile=resolvers.hashers .dirlist +resolvers.generators.dirfile=resolvers.generators.dirlist +resolvers.openers .dirfile=resolvers.openers .dirlist +resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -15775,7 +17020,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6202, stripped down to: 5149 +-- original size: 6569, stripped down to: 5304 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -15801,8 +17046,13 @@ directives.register("schemes.threshold",function(v) threshold=tonumber(v) or thr function cleaners.none(specification) return specification.original end -function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) +function cleaners.strip(specification) + local path,name=file.splitbase(specification.original) + if path=="" then + return (gsub(name,"[^%a%d%.]+","-")) + else + return (gsub((gsub(path,"%.","-").."-"..name),"[^%a%d%.]+","-")) + end end function cleaners.md5(specification) return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) @@ -15818,8 +17068,8 @@ function resolvers.schemes.cleanname(specification) end local cached,loaded,reused,thresholds,handlers={},{},{},{},{} local function runcurl(name,cachename) - local command="curl --silent --create-dirs --output "..cachename.." "..name - os.spawn(command) + local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name + os.execute(command) end local function fetch(specification) local original=specification.original @@ -15951,7 +17201,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4237, stripped down to: 3177 +-- original size: 4313, stripped down to: 3227 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -15960,7 +17210,7 @@ if not modules then modules={} end modules ['data-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local resolvers,package=resolvers,package +local package,lpeg=package,lpeg local gsub=string.gsub local concat=table.concat local addsuffix=file.addsuffix @@ -15971,9 +17221,11 @@ local luaformats={ 'TEXINPUTS','LUAINPUTS' } local libformats={ 'CLUAINPUTS' } local helpers=package.helpers or {} local methods=helpers.methods or {} +local resolvers=resolvers +local resolveprefix=resolvers.resolve +helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) -helpers.report=logs.reporter("resolvers","libraries") helpers.sequence={ "already loaded", "preload table", @@ -15988,7 +17240,7 @@ helpers.sequence={ } local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) function helpers.cleanpath(path) - return resolvers.resolve(lpegmatch(pattern,path)) + return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib local getextraluapaths=package.extraluapaths @@ -16058,7 +17310,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2394, stripped down to: 2005 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -16072,8 +17324,8 @@ local type,next=type,next local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers local report_scripts=logs.reporter("resolvers","scripts") -function resolvers.updatescript(oldname,newname) - local scriptpath="scripts/context/lua" +function resolvers.updatescript(oldname,newname) + local scriptpath="context/lua" newname=file.addsuffix(newname,"lua") local oldscript=resolvers.cleanpath(oldname) if trace_locating then @@ -16125,7 +17377,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2600, stripped down to: 1627 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -16181,7 +17433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2654, stripped down to: 2301 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -16190,10 +17442,13 @@ if not modules then modules={} end modules ['data-lst']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local rawget,type,next=rawget,type,next +local find,concat,upper=string.find,table.concat,string.upper local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs -resolvers.listers=resolvers.listers or {} local resolvers=resolvers +local listers=resolvers.listers or {} +resolvers.listers=listers +local resolveprefix=resolvers.resolve local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) if type(str)=='table' then @@ -16202,7 +17457,7 @@ local function tabstr(str) return str end end -function resolvers.listers.variables(pattern) +function listers.variables(pattern) local instance=resolvers.instance local environment=instance.environment local variables=instance.variables @@ -16223,10 +17478,10 @@ function resolvers.listers.variables(pattern) for key,value in sortedpairs(configured) do if key~="" and (pattern=="" or find(upper(key),pattern)) then report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolveprefix(expansions[key])) or "unset") end end instance.environment=fastcopy(env) @@ -16234,15 +17489,15 @@ function resolvers.listers.variables(pattern) instance.expansions=fastcopy(exp) end local report_resolved=logs.reporter("system","resolved") -function resolvers.listers.configurations() +function listers.configurations() local configurations=resolvers.instance.specification for i=1,#configurations do - report_resolved("file : %s",resolvers.resolve(configurations[i])) + report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) for i=1,#list do - local li=resolvers.resolve(list[i]) + local li=resolveprefix(list[i]) if lfs.isdir(li) then report_resolved("path - %s",li) else @@ -16547,7 +17802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5951, stripped down to: 4922 +-- original size: 5955, stripped down to: 4926 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -16635,7 +17890,7 @@ function environment.make_format(name) end local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") report_format("running command: %s\n",command) - os.spawn(command) + os.execute(command) local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -16670,7 +17925,7 @@ function environment.run_format(name,data,more) else local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") report_format("running command: %s",command) - os.spawn(command) + os.execute(command) end end end @@ -16681,8 +17936,8 @@ end -- of closure -- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 685064 --- stripped bytes : 242353 +-- original bytes : 745618 +-- stripped bytes : 269191 -- end library merge @@ -16781,17 +18036,18 @@ local ownlibs = { -- order can be made better } +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/context/sources/data-tmf.lua + local ownlist = { - '.', - ownpath , - ownpath .. "/../sources", -- HH's development path + -- '.', + -- ownpath , + owntree .. "/../../../../context/sources", -- HH's development path owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", - owntree .. "/../../texmf-dist/tex/context/base", owntree .. "/../../texmf/tex/context/base", owntree .. "/../../../texmf-local/tex/context/base", owntree .. "/../../../texmf-context/tex/context/base", - owntree .. "/../../../texmf-dist/tex/context/base", owntree .. "/../../../texmf/tex/context/base", } @@ -16907,6 +18163,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -16932,6 +18189,7 @@ local helpinfo = [[ <flag name="verbose"><short>give a bit more info</short></flag> <flag name="trackers" value="list"><short>enable given trackers</short></flag> <flag name="progname" value="str"><short>format or backend</short></flag> + <flag name="systeminfo" value="str"><short>show current operating system, processor, etc</short></flag> </subcategory> <subcategory> <flag name="edit"><short>launch editor with found file</short></flag> @@ -17561,6 +18819,39 @@ function runners.associate(filename) os.launch(filename) end +function runners.evaluate(code,filename) -- for Luigi + if code == "loop" then + while true do + io.write("> ") + local code = io.read() + if code ~= "" then + local temp = string.match(code,"^= (.*)$") + if temp then + code = "print("..temp..")" + end + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("! " .. (message or code).."\n") + else + io.write(compiled()) + end + end + end + else + if type(code) ~= "string" or code == "" then + code = filename + end + if code ~= "" then + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("invalid lua code: " .. (message or code)) + return + end + io.write(compiled()) + end + end +end + function runners.gethelp(filename) local url = environment.argument("url") if url and url ~= "" then @@ -17572,6 +18863,15 @@ function runners.gethelp(filename) end end +function runners.systeminfo() + report("architecture : %s",os.platform or "<unset>") + report("operating system : %s",os.name or "<unset>") + report("file architecture : %s",os.type or "<unset>") + report("binary path : %s",os.selfdir or "<unset>") + report("binary suffix : %s",os.binsuffix or "<unset>") + report("library suffix : %s",os.libsuffix or "<unset>") +end + -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script -- ... later we will use the second half @@ -17687,16 +18987,13 @@ end if e_argument("ansi") then - local formatters = string.formatters + logs.setformatters("ansi") - logs.setformatters { - report_yes = formatters["[1;32m%-15s [0;1m|[0m %s"], - report_nop = formatters["[1;32m%-15s [0;1m|[0m"], - subreport_yes = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m %s"], - subreport_nop = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m"], - status_yes = formatters["[1;32m%-15s [0;1m:[0m %s\n"], - status_nop = formatters["[1;32m%-15s [0;1m:[0m\n"], - } + local script = e_argument("script") or e_argument("scripts") + + if type(script) == "string" then + logs.writer("]0;"..script.."") -- for Alan to test + end end @@ -17715,14 +19012,26 @@ if e_argument("script") or e_argument("scripts") then ok = runners.execute_ctx_script(filename) end +elseif e_argument("evaluate") then + + runners.evaluate(e_argument("evaluate"),filename) + elseif e_argument("selfmerge") then -- embed used libraries runners.loadbase() local found = locate_libs() + if found then - utilities.merger.selfmerge(own.name,own.libs,{ found }) + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfmerge(mtxrun,own.libs,{ found }) + application.report("runner updated on resolved path: %s",mtxrun) + else + utilities.merger.selfmerge(own.name,own.libs,{ found }) + application.report("runner updated on relative path: %s",own.name) + end end elseif e_argument("selfclean") then @@ -17730,7 +19039,15 @@ elseif e_argument("selfclean") then -- remove embedded libraries runners.loadbase() - utilities.merger.selfclean(own.name) + + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfclean(mtxrun) + application.report("runner cleaned on resolved path: %s",mtxrun) + else + utilities.merger.selfclean(own.name) + application.report("runner cleaned on relative path: %s",own.name) + end elseif e_argument("selfupdate") then @@ -17972,6 +19289,8 @@ elseif e_argument("version") then application.version() + application.report("source path",environment.ownbin) + elseif e_argument("directives") then directives.show() @@ -17989,6 +19308,10 @@ elseif e_argument("exporthelp") then runners.loadbase() application.export(e_argument("exporthelp"),filename) +elseif e_argument("systeminfo") then + + runners.systeminfo() + elseif e_argument("help") or filename=='help' or filename == "" then application.help() diff --git a/scripts/context/stubs/mswin/mtxrunjit.exe b/scripts/context/stubs/mswin/mtxrunjit.exe Binary files differnew file mode 100644 index 000000000..0e7882cf9 --- /dev/null +++ b/scripts/context/stubs/mswin/mtxrunjit.exe diff --git a/scripts/context/stubs/mswin/mtxworks.exe b/scripts/context/stubs/mswin/mtxworks.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/mtxworks.exe +++ b/scripts/context/stubs/mswin/mtxworks.exe diff --git a/scripts/context/stubs/mswin/pstopdf.exe b/scripts/context/stubs/mswin/pstopdf.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/pstopdf.exe +++ b/scripts/context/stubs/mswin/pstopdf.exe diff --git a/scripts/context/stubs/mswin/texexec.exe b/scripts/context/stubs/mswin/texexec.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/texexec.exe +++ b/scripts/context/stubs/mswin/texexec.exe diff --git a/scripts/context/stubs/mswin/texmfstart.exe b/scripts/context/stubs/mswin/texmfstart.exe Binary files differindex faae5caa7..0e7882cf9 100644 --- a/scripts/context/stubs/mswin/texmfstart.exe +++ b/scripts/context/stubs/mswin/texmfstart.exe diff --git a/scripts/context/stubs/setup/setuptex b/scripts/context/stubs/setup/setuptex new file mode 100644 index 000000000..d41e36707 --- /dev/null +++ b/scripts/context/stubs/setup/setuptex @@ -0,0 +1,167 @@ +# Example setup file for ConTeXt distribution +# +# Author: Hans Hagen +# Patches: Arthur R. & Mojca M. +# +# Usage: +# . setuptex [texroot] +# +# On the first run also execute: +# mktexlsr +# texexec --make --alone + +# +# PLATFORM +# + +# we will try to guess the platform first +# (needs to be kept in sync with first-setup.sh and mtxrun) +# if yours is missing, let us know + +system=`uname -s` +cpu=`uname -m` + +case "$system" in + # linux + Linux) + case "$cpu" in + i*86) platform="linux" ;; + x86_64|ia64) platform="linux-64" ;; + # a little bit of cheating with ppc64 (won't work on Gentoo) + ppc|ppc64) platform="linux-ppc" ;; + *) platform="unknown" ;; + esac ;; + # Mac OS X + Darwin) + case "$cpu" in + i*86) platform="osx-intel" ;; + x86_64) platform="osx-64" ;; + ppc*|powerpc|power*|Power*) platform="osx-ppc" ;; + *) platform="unknown" ;; + esac ;; + # FreeBSD + FreeBSD|freebsd) + case "$cpu" in + i*86) platform="freebsd" ;; + x86_64) platform="freebsd" ;; + amd64) platform="freebsd-amd64" ;; + *) platform="unknown" ;; + esac ;; + # kFreeBSD (Debian) + GNU/kFreeBSD) + case "$cpu" in + i*86) platform="kfreebsd-i386" ;; + x86_64|amd64) platform="kfreebsd-amd64" ;; + *) platform="unknown" ;; + esac ;; + # cygwin + CYGWIN) + case "$cpu" in + i*86) platform="cygwin" ;; + x86_64|ia64) platform="cygwin-64" ;; + *) platform="unknown" ;; + esac ;; + # SunOS/Solaris + SunOS) + case "$cpu" in + sparc) platform="solaris-sparc" ;; + i86pc) platform="solaris-intel" ;; + *) platform="unknown" ;; + esac ;; + *) platform="unknown" +esac + +# temporary fix for Snow Leopard +if test "$platform" = "osx-intel"; then + # running Snow Leopard or later + if test `uname -r|cut -f1 -d"."` -ge 10 ; then + # working on 64-bit hardware + if test `sysctl -n hw.cpu64bit_capable` = 1; then + platform="osx-64" + fi + fi +fi + +if test "$platform" = "unknown" ; then + echo "Error: your system \"$system $cpu\" is not supported yet." + echo "Please report to the ConTeXt mailing-list (ntg-context@ntg.nl)" +fi + +# +# PATH +# + +# this resolves to path of the setuptex script +# We use $0 for determine the path to the script, except for: +# * bash where $0 always is bash; here we use BASH_SOURCE +# * ksh93 where we use ${.sh.file} +# Thanks to Vasile Gaburici and Alessandro Perucchi for reporting this +# * http://www.ntg.nl/pipermail/ntg-context/2008/033953.html +# * http://www.ntg.nl/pipermail/ntg-context/2012/068658.html +if [ z"$BASH_SOURCE" != z ]; then + SCRIPTPATH="$BASH_SOURCE" +elif [ z"$KSH_VERSION" != z ]; then + SCRIPTPATH="${.sh.file}" +else + SCRIPTPATH="$0" +fi + +OWNPATH=$(cd -P -- "$(dirname -- "$SCRIPTPATH")" && pwd -P) + +# but one can also call +# . setuptex path-to-tree + +TEXROOT="" +# first check if any path has been provided in the argument, and try to use that one +if [ $# -ne 0 ] ; then + # TODO: resolve any errors + ARGPATH=$(cd -P -- "$(dirname -- "$1")" && pwd -P) && ARGPATH=$ARGPATH/$(basename -- "$1") + if test -f "$ARGPATH/texmf/tex/plain/base/plain.tex" ; then + if [ -d "$ARGPATH/texmf-$platform/bin" ]; then + TEXROOT="$ARGPATH" + else + echo "Binaries for platform '$platform' are missing." + echo "(There is no folder \"$ARGPATH/texmf-$platform/bin\")" + fi + else + echo "The argument \"$ARGPATH\" is not a valid TEXROOT path." + echo "(There is no file \"$ARGPATH/texmf/tex/plain/base/plain.tex\")" + + if [ -f "$OWNPATH/texmf/tex/plain/base/plain.tex" ]; then + TEXROOT="$OWNPATH" + fi + fi +else + if [ -f "$OWNPATH/texmf/tex/plain/base/plain.tex" ]; then + if [ -d "$OWNPATH/texmf-$platform/bin" ]; then + TEXROOT="$OWNPATH" + else + echo "Binaries for platform '$platform' are missing." + echo "(There is no folder \"$OWNPATH/texmf-$platform/bin\")" + fi + else + echo "\"$OWNPATH\" is not a valid TEXROOT path." + echo "(There is no file \"$OWNPATH/texmf/tex/plain/base/plain.tex\")" + fi +fi + +if [ "$TEXROOT" != "" ]; then + # for Alan Braslau's server :) + if [ "x$PS1" != "x" ] ; then + echo "Setting \"$TEXROOT\" as ConTeXt root." + fi + +# ConTeXt binaries have to be added to PATH +TEXMFOS=$TEXROOT/texmf-$platform +export PATH=$TEXMFOS/bin:$PATH + +# unset variables that won't be used lately +unset platform cpu system OWNPATH SCRIPTPATH ARGPATH TEXMFOS + +# not sure why this would be needed +# export CTXMINIMAL=yes + +else + echo "provide a proper tex root (like '. setuptex /something/tex')" ; +fi + diff --git a/scripts/context/stubs/mswin/setuptex.bat b/scripts/context/stubs/setup/setuptex.bat index b61fd4494..b61fd4494 100644 --- a/scripts/context/stubs/mswin/setuptex.bat +++ b/scripts/context/stubs/setup/setuptex.bat diff --git a/scripts/context/stubs/setup/setuptex.csh b/scripts/context/stubs/setup/setuptex.csh new file mode 100644 index 000000000..c1160675f --- /dev/null +++ b/scripts/context/stubs/setup/setuptex.csh @@ -0,0 +1,164 @@ +# Example setup file for ConTeXt distribution +# +# Author: Hans Hagen +# Patches: Arthur R. & Mojca M. +# (t)csh version: Alan B. +# +# Usage : +# source setuptex.csh [texroot] +# +# On the first run also execute: +# mktexlsr +# texexec --make --alone + +echo "We are considering removing setuptex.csh in case that nobody uses it." +echo "If you still use this file please drop us some mail at" +echo " gardeners (at) contextgarden (dot) net" +echo "If we don't get any response, we will delete it in near future." + +# +# PLATFORM +# + +# we will try to guess the platform first +# (needs to be kept in sync with first-setup.sh and mtxrun) +# if yours is missing, let us know + +set system=`uname -s` +set cpu=`uname -m` + +switch ( $system ) + # linux + case Linux: + switch ( $cpu ) + case i*86: + set platform="linux" + breaksw + case x86_64: + case ia64: + set platform="linux-64" + breaksw + case ppc: + case ppc64: + set platform="linux-ppc" + breaksw + default: + set platform="unknown" + endsw + breaksw + # Mac OS X + case Darwin: + switch ( $cpu ) + case i*86: + set platform="osx-intel" + breaksw + case x86_64: + set platform="osx-64" + breaksw + case ppc*: + case powerpc: + case power*: + case Power*: + set platform="osx-ppc" + breaksw + default: + set platform="unknown" + endsw + breaksw + # FreeBSD + case FreeBSD: + case freebsd: + switch ( $cpu ) + case i*86: + set platform="freebsd" + breaksw + case x86_64: + set platform="freebsd" + breaksw + case amd64: + set platform="freebsd-amd64" + breaksw + default: + set platform="unknown" + endsw + breaksw + # cygwin + case CYGWIN: + switch ( $cpu ) + case i*86: + set platform="cygwin" + breaksw + case x86_64: + case ia64: + set platform="cygwin-64" + breaksw + default: + set platform="unknown" + endsw + breaksw + # SunOS/Solaris + case SunOS: + switch ( $cpu ) + case sparc: + set platform="solaris-sparc" + breaksw + case i86pc: + set platform="solaris-intel" + default: + set platform="unknown" + endsw + breaksw + # Other + default: + set platform="unknown" +endsw + +if ( $platform == "unknown" ) then + echo Error: your system \"$system $cpu\" is not supported yet. + echo Please report to the ConTeXt mailing-list (ntg-context@ntg.nl). +endif + +# +# PATH +# + +# this resolves to path of the setuptex script +# We use $0 for determine the path to the script, except for bash and (t)csh where $0 +# always is bash or (t)csh. + +# but one can also call +# . setuptex path-to-tex-tree + +# first check if any path has been provided in the argument, and try to use that one +if ( $# > 0 ) then + setenv TEXROOT $1 +else + # $_ should be `history -h 1` but doesn't seem to work... + set cmd=`history -h 1` + if ( $cmd[2]:h == $cmd[2]:t ) then + setenv TEXROOT $cwd + else + setenv TEXROOT $cmd[2]:h + endif + unset cmd +endif +cd $TEXROOT; setenv TEXROOT $cwd; cd - + +if ( -f "$TEXROOT/texmf/tex/plain/base/plain.tex" ) then + echo Setting \"$TEXROOT\" as TEXROOT. +else + echo \"$TEXROOT\" is not a valid TEXROOT path. + echo There is no file \"$TEXROOT/texmf/tex/plain/base/plain.tex\". + echo Please provide a proper tex root (like \"source setuptex /path/tex\") + unsetenv TEXROOT + exit +endif + +unsetenv TEXINPUTS MPINPUTS MFINPUTS + +# ConTeXt binaries have to be added to PATH +setenv TEXMFOS $TEXROOT/texmf-$platform +setenv PATH $TEXMFOS/bin:$PATH +# TODO: we could set OSFONTDIR on Mac for example + +# setenv CTXMINIMAL yes diff --git a/scripts/context/stubs/source/mtxrun_dll.c b/scripts/context/stubs/source/mtxrun_dll.c index 400ed6778..fc2e260f5 100644 --- a/scripts/context/stubs/source/mtxrun_dll.c +++ b/scripts/context/stubs/source/mtxrun_dll.c @@ -55,7 +55,6 @@ return 1; \ } -char texlua_name[] = "texlua"; // just a bare name, luatex strips the rest anyway static char cmdline[MAX_CMD]; static char dirpath[MAX_PATH]; static char progname[MAX_PATH]; @@ -70,12 +69,12 @@ int main( int argc, char *argv[] ) __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) #endif { - char *s, *luatexfname, *argstr, **lua_argv; + char *binary, *s, *luatexfname, *argstr, **lua_argv; int k, quoted, lua_argc; int passprogname = 0; + unsigned char is_jit=0; // directory of this module/executable - HMODULE module_handle = GetModuleHandle( "mtxrun.dll" ); // if ( module_handle == NULL ) exe path will be used, which is OK too k = (int) GetModuleFileName( module_handle, dirpath, MAX_PATH ); @@ -86,13 +85,20 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) *(++s) = '\0'; //remove file name, leave trailing backslash // program name - k = strlen(argv[0]); while ( k && (argv[0][k-1] != '/') && (argv[0][k-1] != '\\') ) k--; strcpy(progname, &argv[0][k]); s = progname; if ( s = strrchr(s, '.') ) *s = '\0'; // remove file extension part + /* check "jit" : strlen("jit") = 3 */ + if (strncmp(progname + strlen(progname) - 3, "jit", 3) == 0) { + is_jit = 1; + progname[strlen(progname) - 3]='\0'; + } + else + is_jit = 0; + // script path strcpy( scriptpath, dirpath ); @@ -114,44 +120,110 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) strcat( scriptpath, "mtxrun.lua" ); passprogname = 1; } - if ( GetFileAttributes(scriptpath) == INVALID_FILE_ATTRIBUTES ) DIE( "file not found: %s\n", scriptpath ); - // find texlua.exe - - if ( !SearchPath( - getenv( "PATH" ), // path to search (optional) - "texlua.exe", // file name to search - NULL, // file extension to add (optional) - MAX_PATH, // output buffer size - luatexpath, // output buffer pointer - &luatexfname ) // pointer to a file part in the output buffer (optional) - ) - if ( !SearchPath( - dirpath, // path to search (optional) - "texlua.exe", // file name to search - NULL, // file extension to add (optional) - MAX_PATH, // output buffer size - luatexpath, // output buffer pointer - &luatexfname ) // pointer to a file part in the output buffer (optional) - ) - DIE( "unable to locate texlua.exe on the search path" ); + // find luatex.exe /luajittex.exe + if ( SearchPath( + dirpath, // was getenv( "PATH" ), // path to search (optional) + (is_jit ? "luajittex.exe":"luatex.exe"), // file name to search + NULL, // file extension to add (optional) + MAX_PATH, // output buffer size + luatexpath, // output buffer pointer + &luatexfname ) // pointer to a file part in the output buffer (optional) + ) { + binary = (is_jit ? "luajittex.exe":"luatex.exe"); + } else if ( SearchPath( + dirpath, // was getenv( "PATH" ), // path to search (optional) + (is_jit ? "texluajit.exe":"texlua.exe"), // file name to search + NULL, // file extension to add (optional) + MAX_PATH, // output buffer size + luatexpath, // output buffer pointer + &luatexfname ) // pointer to a file part in the output buffer (optional) + ) { + binary = (is_jit ? "texluajit.exe":"texlua.exe"); + } else if ( SearchPath( + getenv("PATH"), // was dirpath, // path to search (optional) + (is_jit ? "luajittex.exe":"luatex.exe"), // file name to search + NULL, // file extension to add (optional) + MAX_PATH, // output buffer size + luatexpath, // output buffer pointer + &luatexfname ) // pointer to a file part in the output buffer (optional) + ) { + binary = (is_jit ? "luajittex.exe":"luatex.exe"); + } else if ( SearchPath( + getenv("PATH") , // was dirpath, // path to search (optional) + (is_jit ? "texluajit.exe":"texlua.exe"), // file name to search + NULL, // file extension to add (optional) + MAX_PATH, // output buffer size + luatexpath, // output buffer pointer + &luatexfname ) // pointer to a file part in the output buffer (optional) + ) { + binary = (is_jit ? "texluajit.exe":"texlua.exe"); + }else { + DIE( "unable to locate texlua.exe on the search path" ); + } + + /* if ( SearchPath( */ + /* dirpath, // was getenv( "PATH" ), // path to search (optional) */ + /* (is_jit ? "luajittex.exe":"luatex.exe"), // file name to search */ + /* NULL, // file extension to add (optional) */ + /* MAX_PATH, // output buffer size */ + /* luatexpath, // output buffer pointer */ + /* &luatexfname ) // pointer to a file part in the output buffer (optional) */ + /* ) { */ + /* binary = (is_jit ? "luajittex.exe":"luatex.exe"); */ + /* }else if ( SearchPath( */ + /* getenv("PATH"), // was dirpath, // path to search (optional) */ + /* (is_jit ? "luajittex.exe":"luatex.exe"), // file name to search */ + /* NULL, // file extension to add (optional) */ + /* MAX_PATH, // output buffer size */ + /* luatexpath, // output buffer pointer */ + /* &luatexfname ) // pointer to a file part in the output buffer (optional) */ + /* ) { */ + /* binary = (is_jit ? "luajittex.exe":"luatex.exe"); */ + /* }else if ( SearchPath( */ + /* dirpath, // was getenv( "PATH" ), // path to search (optional) */ + /* (is_jit ? "texluajit.exe":"texlua.exe"), // file name to search */ + /* NULL, // file extension to add (optional) */ + /* MAX_PATH, // output buffer size */ + /* luatexpath, // output buffer pointer */ + /* &luatexfname ) // pointer to a file part in the output buffer (optional) */ + /* ) { */ + /* binary = (is_jit ? "texluajit.exe":"texlua.exe"); */ + /* }else if ( SearchPath( */ + /* getenv("PATH") , // was dirpath, // path to search (optional) */ + /* (is_jit ? "texluajit.exe":"texlua.exe"), // file name to search */ + /* NULL, // file extension to add (optional) */ + /* MAX_PATH, // output buffer size */ + /* luatexpath, // output buffer pointer */ + /* &luatexfname ) // pointer to a file part in the output buffer (optional) */ + /* ) { */ + /* binary = (is_jit ? "texluajit.exe":"texlua.exe"); */ + /* }else { */ + /* DIE( "unable to locate texlua.exe on the search path" ); */ + /* } */ + - // link directly with luatex.dll if available in texlua's dir - strcpy( luatexfname, "luatex.dll" ); + + // link directly with luatex.dll if available in texlua's dir + strcpy( luatexfname, (is_jit ? "luajittex.dll":"luatex.dll") ); if ( dllluatex = LoadLibrary(luatexpath) ) { - mainlikeproc dllluatexmain = (mainlikeproc) GetProcAddress( dllluatex, "dllluatexmain" ); + mainlikeproc dllluatexmain = (mainlikeproc) GetProcAddress( dllluatex, (is_jit ? "dllluajittexmain": "dllluatexmain" )); if ( dllluatexmain == NULL ) - DIE( "unable to locate dllluatexmain procedure in luatex.dll" ); + if (is_jit) + DIE( "unable to locate dllluatexmain procedure in luajittex.dll" ) + else + DIE( "unable to locate dllluatexmain procedure in luatex.dll" ); // set up argument list for texlua script - lua_argv = (char **)malloc( (argc + 4) * sizeof(char *) ); + lua_argv = (char **)malloc( (argc + 5) * sizeof(char *) ); if ( lua_argv == NULL ) DIE( "out of memory\n" ); - lua_argv[lua_argc=0] = texlua_name; + lua_argv[lua_argc=0] = luatexfname; + lua_argv[++lua_argc] = "--luaonly"; lua_argv[++lua_argc] = scriptpath; // script to execute if (passprogname) { lua_argv[++lua_argc] = "--script"; @@ -162,15 +234,15 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) // call texlua interpreter // dllluatexmain never returns, but we pretend that it does - + k = dllluatexmain( lua_argc, lua_argv ); if (lua_argv) free( lua_argv ); return k; } - // we are still here, so no luatex.dll; spawn texlua.exe instead - strcpy( luatexfname, "texlua.exe" ); + strcpy( luatexfname,binary); + strcpy( cmdline, " --luaonly " ); strcpy( cmdline, "\"" ); strcat( cmdline, luatexpath ); strcat( cmdline, "\" \"" ); @@ -180,7 +252,6 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) strcat( cmdline, " --script " ); strcat( cmdline, progname ); } - argstr = GetCommandLine(); // get the command line of this process if ( argstr == NULL ) DIE( "unable to retrieve the command line string\n" ); @@ -209,7 +280,6 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) si.hStdOutput = GetStdHandle( STD_OUTPUT_HANDLE ); si.hStdError = GetStdHandle( STD_ERROR_HANDLE ); ZeroMemory( &pi, sizeof(pi) ); - if( !CreateProcess( NULL, // module name (uses command line if NULL) cmdline, // command line @@ -222,7 +292,6 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) &si, // STARTUPINFO structure &pi ) // PROCESS_INFORMATION structure ) DIE( "command execution failed: %s\n", cmdline ); - DWORD ret = 0; CloseHandle( pi.hThread ); // thread handle is not needed if ( WaitForSingleObject( pi.hProcess, INFINITE ) == WAIT_OBJECT_0 ) { @@ -232,7 +301,6 @@ __declspec(dllexport) int dllrunscript( int argc, char *argv[] ) CloseHandle( pi.hProcess ); // propagate exit code from the child process - return ret; } diff --git a/scripts/context/stubs/source/readme.txt b/scripts/context/stubs/source/readme.txt index 354d85b09..72892ee2f 100644 --- a/scripts/context/stubs/source/readme.txt +++ b/scripts/context/stubs/source/readme.txt @@ -1,36 +1,40 @@ Copyright: -The originally 'runscript' program was written by in 2009 by -T.M.Trzeciak and is public domain. This derived mtxrun program -is an adapted version by Hans Hagen. +The originally 'runscript' program was written by in 2009 by T.M.Trzeciak and is +public domain. This derived mtxrun program is an adapted version by Hans Hagen and +Luigi Scarso. Comment: -In ConTeXt MkIV we have two core scripts: luatools.lua and -mtxrun.lua where the second one is used to launch other scripts. +In ConTeXt MkIV we have two core scripts: luatools.lua and mtxrun.lua where the +second one is used to launch other scripts. The mtxrun.exe program calls luatex.exe. + Normally a user will use a call like: -mtxrun --script font --reload + mtxrun --script font --reload + +Here mtxrun is a lua script. In order to avoid the usage of a cmd file on windows this +runner will start texlua directly. In TeXlive a runner is added for each cmd file but +we don't want that overhead (and extra files). By using an exe we can call these +scripts in batch files without the need for using call. + +The mtxrun.exe file can be copied to a mtxrunjit.exe file in which case luajittex.exe +is called. -Here mtxrun is a lua script. In order to avoid the usage of a cmd -file on windows this runner will start texlua directly. In TeXlive -a runner is added for each cmd file but we don't want that overhead -(and extra files). By using an exe we can call these scripts in -batch files without the need for using call. + mtxrunjit --script font --reload -We also don't want to use other runners, like those that use kpse -to locate the script as this is exactly what mtxrun itself is doing -already. Therefore the runscript program is adapted to a more direct -approach suitable for mtxrun. +We also don't want to use other runners, like those that use kpse to locate the script +as this is exactly what mtxrun itself is doing already. Therefore the runscript program +is adapted to a more direct approach suitable for mtxrun. Compilation: with gcc (size optimized): -gcc -Os -s -shared -o mtxrun.dll mtxrun_dll.c -gcc -Os -s -o mtxrun.exe mtxrun_exe.c -L./ -lmtxrun + gcc -Os -s -shared -o mtxrun.dll mtxrun_dll.c + gcc -Os -s -o mtxrun.exe mtxrun_exe.c -L./ -lmtxrun with tcc (ver. 0.9.24), extra small size -tcc -shared -o runscript.dll runscript_dll.c -tcc -o runscript.exe runscript_exe.c runscript.def + tcc -shared -o runscript.dll runscript_dll.c + tcc -o runscript.exe runscript_exe.c runscript.def diff --git a/scripts/context/stubs/unix/contextjit b/scripts/context/stubs/unix/contextjit new file mode 100644 index 000000000..5ac1947c7 --- /dev/null +++ b/scripts/context/stubs/unix/contextjit @@ -0,0 +1,5 @@ +#!/bin/sh + +luajittex --luaonly $(dirname $0)/mtxrun --script context "$@" + +# luajittex --luaonly ${0%contextjit}mtxrun --script context "$@" diff --git a/scripts/context/stubs/unix/ctxtools b/scripts/context/stubs/unix/ctxtools deleted file mode 100644 index 2e6bd4afa..000000000 --- a/scripts/context/stubs/unix/ctxtools +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --script ctxtools "$@" diff --git a/scripts/context/stubs/unix/mptopdf b/scripts/context/stubs/unix/mptopdf deleted file mode 100644 index 147333740..000000000 --- a/scripts/context/stubs/unix/mptopdf +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --script mptopdf "$@" diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 0ff2d2897..edfeba8dd 100644 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3123, stripped down to: 1694 +-- original size: 3888, stripped down to: 2197 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -136,6 +136,16 @@ function optionalrequire(...) return result end end +if lua then + lua.mask=load([[τεχ = 1]]) and "utf" or "ascii" +end +local flush=io.flush +if flush then + local execute=os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec=os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end +end end -- of closure @@ -434,7 +444,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 29245, stripped down to: 15964 +-- original size: 36977, stripped down to: 20349 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -450,7 +460,9 @@ local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.forma local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print -setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +if setinspector then + setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns local anything=P(1) @@ -469,7 +481,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=crlf+S("\r\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -491,8 +503,10 @@ patterns.utfbom_32_le=utfbom_32_le patterns.utfbom_16_be=utfbom_16_be patterns.utfbom_16_le=utfbom_16_le patterns.utfbom_8=utfbom_8 -patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") -patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") +patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_32_be_nl=P("\000\000\000\r\000\000\000\n")+P("\000\000\000\r")+P("\000\000\000\n") +patterns.utf_32_le_nl=P("\r\000\000\000\n\000\000\000")+P("\r\000\000\000")+P("\n\000\000\000") patterns.utf8one=R("\000\127") patterns.utf8two=R("\194\223")*utf8next patterns.utf8three=R("\224\239")*utf8next*utf8next @@ -519,10 +533,24 @@ patterns.spacer=spacer patterns.whitespace=whitespace patterns.nonspacer=nonspacer patterns.nonwhitespace=nonwhitespace -local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) +local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) +local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) +local b_stripper=Cs(spacer^0/""*(nonspacer^1+spacer^1/" ")^0) +local e_stripper=Cs((spacer^1*P(-1)/""+nonspacer^1+spacer^1/" ")^0) +local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper +patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.b_collapser=b_collapser +patterns.m_collapser=m_collapser +patterns.e_collapser=e_collapser +patterns.b_stripper=b_stripper +patterns.m_stripper=m_stripper +patterns.e_stripper=e_stripper patterns.lowercase=lowercase patterns.uppercase=uppercase patterns.letter=patterns.lowercase+patterns.uppercase @@ -559,9 +587,12 @@ patterns.integer=sign^-1*digit^1 patterns.unsigned=digit^0*period*digit^1 patterns.float=sign^-1*patterns.unsigned patterns.cunsigned=digit^0*comma*digit^1 +patterns.cpunsigned=digit^0*(period+comma)*digit^1 patterns.cfloat=sign^-1*patterns.cunsigned +patterns.cpfloat=sign^-1*patterns.cpunsigned patterns.number=patterns.float+patterns.integer patterns.cnumber=patterns.cfloat+patterns.integer +patterns.cpnumber=patterns.cpfloat+patterns.integer patterns.oct=zero*octdigit^1 patterns.octal=patterns.oct patterns.HEX=zero*P("X")*(digit+uppercase)^1 @@ -744,7 +775,7 @@ function lpeg.replacer(one,two,makefunction,isutf) return pattern end end -function lpeg.finder(lst,makefunction) +function lpeg.finder(lst,makefunction,isutf) local pattern if type(lst)=="table" then pattern=P(false) @@ -760,7 +791,11 @@ function lpeg.finder(lst,makefunction) else pattern=P(lst) end - pattern=(1-pattern)^0*pattern + if isutf then + pattern=((utf8char or 1)-pattern)^0*pattern + else + pattern=(1-pattern)^0*pattern + end if makefunction then return function(str) return lpegmatch(pattern,str) @@ -974,37 +1009,139 @@ function lpeg.append(list,pp,delayed,checked) end return p end +local p_false=P(false) +local p_true=P(true) local function make(t) - local p + local function making(t) + local p=p_false + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then + else + p=p+P(k)*making(v) + end + end + end + if t[""] then + p=p+p_true + end + return p + end + local p=p_false local keys=sortedkeys(t) for i=1,#keys do local k=keys[i] - local v=t[k] - if not p then - if next(v) then - p=P(k)*make(v) + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then else - p=P(k) + p=p+P(k)*making(v) end - else - if next(v) then - p=p+P(k)*make(v) + end + end + return p +end +local function collapse(t,x) + if type(t)~="table" then + return t,x + else + local n=next(t) + if n==nil then + return t,x + elseif next(t,n)==nil then + local k=n + local v=t[k] + if type(v)=="table" then + return collapse(v,x..k) else - p=p+P(k) + return v,x..k + end + else + local tt={} + for k,v in next,t do + local vv,kk=collapse(v,k) + tt[kk]=vv end + return tt,x end end - return p end function lpeg.utfchartabletopattern(list) local tree={} - for i=1,#list do - local t=tree - for c in gmatch(list[i],".") do - if not t[c] then - t[c]={} + local n=#list + if n==0 then + for s in next,list do + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true + end + end + else + for i=1,n do + local s=list[i] + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true end - t=t[c] end end return make(tree) @@ -1044,6 +1181,65 @@ local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") local number=digit^1*(case_1+case_2) local stripper=Cs((number+1)^0) lpeg.patterns.stripzeros=stripper +local byte_to_HEX={} +local byte_to_hex={} +local byte_to_dec={} +local hex_to_byte={} +for i=0,255 do + local H=format("%02X",i) + local h=format("%02x",i) + local d=format("%03i",i) + local c=char(i) + byte_to_HEX[c]=H + byte_to_hex[c]=h + byte_to_dec[c]=d + hex_to_byte[h]=c + hex_to_byte[H]=c +end +local hextobyte=P(2)/hex_to_byte +local bytetoHEX=P(1)/byte_to_HEX +local bytetohex=P(1)/byte_to_hex +local bytetodec=P(1)/byte_to_dec +local hextobytes=Cs(hextobyte^0) +local bytestoHEX=Cs(bytetoHEX^0) +local bytestohex=Cs(bytetohex^0) +local bytestodec=Cs(bytetodec^0) +patterns.hextobyte=hextobyte +patterns.bytetoHEX=bytetoHEX +patterns.bytetohex=bytetohex +patterns.bytetodec=bytetodec +patterns.hextobytes=hextobytes +patterns.bytestoHEX=bytestoHEX +patterns.bytestohex=bytestohex +patterns.bytestodec=bytestodec +function string.toHEX(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestoHEX,s) + end +end +function string.tohex(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestohex,s) + end +end +function string.todec(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestodec,s) + end +end +function string.tobytes(s) + if not s or s=="" then + return s + else + return lpegmatch(hextobytes,s) + end +end end -- of closure @@ -1071,7 +1267,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5547, stripped down to: 2708 +-- original size: 5694, stripped down to: 2827 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1107,11 +1303,15 @@ function string.limit(str,n,sentinel) end end local stripper=patterns.stripper +local fullstripper=patterns.fullstripper local collapser=patterns.collapser local longtostring=patterns.longtostring function string.strip(str) return lpegmatch(stripper,str) or "" end +function string.fullstrip(str) + return lpegmatch(fullstripper,str) or "" +end function string.collapsespaces(str) return lpegmatch(collapser,str) or "" end @@ -1172,7 +1372,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 31113, stripped down to: 20256 +-- original size: 35724, stripped down to: 21525 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1205,7 +1405,7 @@ end function table.keys(t) if t then local keys,k={},0 - for key,_ in next,t do + for key in next,t do k=k+1 keys[k]=key end @@ -1215,32 +1415,52 @@ function table.keys(t) end end local function compare(a,b) - local ta,tb=type(a),type(b) - if ta==tb then - return a<b - else - return tostring(a)<tostring(b) + local ta=type(a) + if ta=="number" then + local tb=type(b) + if ta==tb then + return a<b + elseif tb=="string" then + return tostring(a)<b + end + elseif ta=="string" then + local tb=type(b) + if ta==tb then + return a<b + else + return a<tostring(b) + end end + return tostring(a)<tostring(b) end local function sortedkeys(tab) if tab then local srt,category,s={},0,0 - for key,_ in next,tab do + for key in next,tab do s=s+1 srt[s]=key if category==3 then + elseif category==1 then + if type(key)~="string" then + category=3 + end + elseif category==2 then + if type(key)~="number" then + category=3 + end else local tkey=type(key) if tkey=="string" then - category=(category==2 and 3) or 1 + category=1 elseif tkey=="number" then - category=(category==1 and 3) or 2 + category=2 else category=3 end end end - if category==0 or category==3 then + if s<2 then + elseif category==3 then sort(srt,compare) else sort(srt) @@ -1250,16 +1470,52 @@ local function sortedkeys(tab) return {} end end +local function sortedhashonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="string" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end +local function sortedindexonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="number" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end local function sortedhashkeys(tab,cmp) if tab then local srt,s={},0 - for key,_ in next,tab do + for key in next,tab do if key then s=s+1 srt[s]=key end end - sort(srt,cmp) + if s>1 then + sort(srt,cmp) + end return srt else return {} @@ -1268,13 +1524,15 @@ end function table.allkeys(t) local keys={} for k,v in next,t do - for k,v in next,v do + for k in next,v do keys[k]=true end end return sortedkeys(keys) end table.sortedkeys=sortedkeys +table.sortedhashonly=sortedhashonly +table.sortedindexonly=sortedindexonly table.sortedhashkeys=sortedhashkeys local function nothing() end local function sortedhash(t,cmp) @@ -1285,19 +1543,21 @@ local function sortedhash(t,cmp) else s=sortedkeys(t) end - local n=0 local m=#s - local function kv(s) - if n<m then - n=n+1 - local k=s[n] - return k,t[k] + if m==1 then + return next,t + elseif m>0 then + local n=0 + return function() + if n<m then + n=n+1 + local k=s[n] + return k,t[k] + end end end - return kv,s - else - return nothing end + return nothing end table.sortedhash=sortedhash table.sortedpairs=sortedhash @@ -1439,39 +1699,36 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,reduce,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } local function simple_table(t) - if #t>0 then + local nt=#t + if nt>0 then local n=0 for _,v in next,t do n=n+1 end - if n==#t then - local tt,nt={},0 - for i=1,#t do + if n==nt then + local tt={} + for i=1,nt do local v=t[i] local tv=type(v) if tv=="number" then - nt=nt+1 if hexify then - tt[nt]=format("0x%04X",v) + tt[i]=format("0x%X",v) else - tt[nt]=tostring(v) + tt[i]=tostring(v) end elseif tv=="string" then - nt=nt+1 - tt[nt]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then - nt=nt+1 - tt[nt]=v and "true" or "false" + tt[i]=v and "true" or "false" else - tt=nil - break + return nil end end return tt @@ -1490,7 +1747,7 @@ local function do_serialize(root,name,depth,level,indexed) local tn=type(name) if tn=="number" then if hexify then - handle(format("%s[0x%04X]={",depth,name)) + handle(format("%s[0x%X]={",depth,name)) else handle(format("%s[%s]={",depth,name)) end @@ -1507,7 +1764,7 @@ local function do_serialize(root,name,depth,level,indexed) end end end - if root and next(root) then + if root and next(root)~=nil then local first,last=nil,0 if compact then last=#root @@ -1525,22 +1782,19 @@ local function do_serialize(root,name,depth,level,indexed) for i=1,#sk do local k=sk[i] local v=root[k] - local tv,tk=type(v),type(k) + local tv=type(v) + local tk=type(k) if compact and first and tk=="number" and k>=first and k<=last then if tv=="number" then if hexify then - handle(format("%s 0x%04X,",depth,v)) + handle(format("%s 0x%X,",depth,v)) else handle(format("%s %s,",depth,v)) end elseif tv=="string" then - if reduce and tonumber(v) then - handle(format("%s %s,",depth,v)) - else - handle(format("%s %q,",depth,v)) - end + handle(format("%s %q,",depth,v)) elseif tv=="table" then - if not next(v) then + if next(v)==nil then handle(format("%s {},",depth)) elseif inline then local st=simple_table(v) @@ -1570,64 +1824,48 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="number" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=0x%04X,",depth,k,v)) + handle(format("%s [0x%X]=0x%X,",depth,k,v)) else handle(format("%s [%s]=%s,",depth,k,v)) end elseif tk=="boolean" then if hexify then - handle(format("%s [%s]=0x%04X,",depth,k and "true" or "false",v)) + handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then - handle(format("%s %s=0x%04X,",depth,k,v)) + handle(format("%s %s=0x%X,",depth,k,v)) else handle(format("%s %s=%s,",depth,k,v)) end else if hexify then - handle(format("%s [%q]=0x%04X,",depth,k,v)) + handle(format("%s [%q]=0x%X,",depth,k,v)) else handle(format("%s [%q]=%s,",depth,k,v)) end end elseif tv=="string" then - if reduce and tonumber(v) then - if tk=="number" then - if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v)) - else - handle(format("%s [%s]=%s,",depth,k,v)) - end - elseif tk=="boolean" then - handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%s,",depth,k,v)) + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=%q,",depth,k,v)) else - handle(format("%s [%q]=%s,",depth,k,v)) + handle(format("%s [%s]=%q,",depth,k,v)) end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) else - if tk=="number" then - if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,v)) - else - handle(format("%s [%s]=%q,",depth,k,v)) - end - elseif tk=="boolean" then - handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) - elseif noquotes and not reserved[k] and lpegmatch(propername,k) then - handle(format("%s %s=%q,",depth,k,v)) - else - handle(format("%s [%q]=%q,",depth,k,v)) - end + handle(format("%s [%q]=%q,",depth,k,v)) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={},",depth,k)) + handle(format("%s [0x%X]={},",depth,k)) else handle(format("%s [%s]={},",depth,k)) end @@ -1643,7 +1881,7 @@ local function do_serialize(root,name,depth,level,indexed) if st then if tk=="number" then if hexify then - handle(format("%s [0x%04X]={ %s },",depth,k,concat(st,", "))) + handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) else handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) end @@ -1663,7 +1901,7 @@ local function do_serialize(root,name,depth,level,indexed) elseif tv=="boolean" then if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%s,",depth,k,v and "true" or "false")) + handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) else handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) end @@ -1679,7 +1917,7 @@ local function do_serialize(root,name,depth,level,indexed) local f=getinfo(v).what=="C" and dump(dummy) or dump(v) if tk=="number" then if hexify then - handle(format("%s [0x%04X]=load(%q),",depth,k,f)) + handle(format("%s [0x%X]=load(%q),",depth,k,f)) else handle(format("%s [%s]=load(%q),",depth,k,f)) end @@ -1694,7 +1932,7 @@ local function do_serialize(root,name,depth,level,indexed) else if tk=="number" then if hexify then - handle(format("%s [0x%04X]=%q,",depth,k,tostring(v))) + handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) else handle(format("%s [%s]=%q,",depth,k,tostring(v))) end @@ -1718,7 +1956,6 @@ local function serialize(_handle,root,name,specification) noquotes=specification.noquotes hexify=specification.hexify handle=_handle or specification.handle or print - reduce=specification.reduce or false functions=specification.functions compact=specification.compact inline=specification.inline and compact @@ -1735,7 +1972,6 @@ local function serialize(_handle,root,name,specification) noquotes=false hexify=false handle=_handle or print - reduce=false compact=true inline=true functions=true @@ -1748,7 +1984,7 @@ local function serialize(_handle,root,name,specification) end elseif tname=="number" then if hexify then - handle(format("[0x%04X]={",name)) + handle(format("[0x%X]={",name)) else handle("["..name.."]={") end @@ -1766,7 +2002,7 @@ local function serialize(_handle,root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,"",0) end end @@ -1895,14 +2131,25 @@ local function identical(a,b) end table.identical=identical table.are_equal=are_equal -function table.compact(t) - if t then - for k,v in next,t do - if not next(v) then - t[k]=nil +local function sparse(old,nest,keeptables) + local new={} + for k,v in next,old do + if not (v=="" or v==false) then + if nest and type(v)=="table" then + v=sparse(v,nest) + if keeptables or next(v)~=nil then + new[k]=v + end + else + new[k]=v end end end + return new +end +table.sparse=sparse +function table.compact(t) + return sparse(t,true,true) end function table.contains(t,v) if t then @@ -2000,15 +2247,17 @@ function table.print(t,...) serialize(print,t,...) end end -setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +if setinspector then + setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +end function table.sub(t,i,j) return { unpack(t,i,j) } end function table.is_empty(t) - return not t or not next(t) + return not t or next(t)==nil end function table.has_one_entry(t) - return t and not next(t,next(t)) + return t and next(t,next(t))==nil end function table.loweredkeys(t) local l={} @@ -2053,6 +2302,44 @@ function table.values(t,s) return {} end end +function table.filtered(t,pattern,sort,cmp) + if t and type(pattern)=="string" then + if sort then + local s + if cmp then + s=sortedhashkeys(t,function(a,b) return cmp(t,a,b) end) + else + s=sortedkeys(t) + end + local n=0 + local m=#s + local function kv(s) + while n<m do + n=n+1 + local k=s[n] + if find(k,pattern) then + return k,t[k] + end + end + end + return kv,s + else + local n=next(t) + local function iterator() + while n~=nil do + local k=n + n=next(t,k) + if find(k,pattern) then + return k,t[k] + end + end + end + return iterator,t + end + else + return nothing + end +end end -- of closure @@ -2061,7 +2348,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8817, stripped down to: 6340 +-- original size: 8643, stripped down to: 6232 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2075,7 +2362,7 @@ local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor local type=type -if string.find(os.getenv("PATH"),";") then +if string.find(os.getenv("PATH"),";",1,true) then io.fileseparator,io.pathseparator="\\",";" else io.fileseparator,io.pathseparator="/",":" @@ -2368,8 +2655,6 @@ function io.readstring(f,n,m) local str=gsub(f:read(n),"\000","") return str end -if not io.i_limiter then function io.i_limiter() end end -if not io.o_limiter then function io.o_limiter() end end end -- of closure @@ -2596,7 +2881,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16023, stripped down to: 9634 +-- original size: 15832, stripped down to: 9456 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2670,13 +2955,10 @@ if not os.__getenv__ then setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) end end -local execute,spawn,exec,iopopen,ioflush=os.execute,os.spawn or os.execute,os.exec or os.execute,io.popen,io.flush -function os.execute(...) ioflush() return execute(...) end -function os.spawn (...) ioflush() return spawn (...) end -function os.exec (...) ioflush() return exec (...) end -function io.popen (...) ioflush() return iopopen(...) end +local execute=os.execute +local iopopen=io.popen function os.resultof(command) - local handle=io.popen(command,"r") + local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" handle:close() @@ -2686,7 +2968,7 @@ function os.resultof(command) end end if not io.fileseparator then - if find(os.getenv("PATH"),";") then + if find(os.getenv("PATH"),";",1,true) then io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" @@ -2705,7 +2987,7 @@ local launchers={ unix="$BROWSER %s &> /dev/null &", } function os.launch(str) - os.execute(format(launchers[os.name] or launchers.unix,str)) + execute(format(launchers[os.name] or launchers.unix,str)) end if not os.times then function os.times() @@ -2746,7 +3028,7 @@ if platform~="" then elseif os.type=="windows" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" - if find(architecture,"AMD64") then + if find(architecture,"AMD64",1,true) then platform="win64" else platform="mswin" @@ -2758,9 +3040,9 @@ elseif os.type=="windows" then elseif name=="linux" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then + if find(architecture,"x86_64",1,true) then platform="linux-64" - elseif find(architecture,"ppc") then + elseif find(architecture,"ppc",1,true) then platform="linux-ppc" else platform="linux" @@ -2774,9 +3056,9 @@ elseif name=="macosx" then local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" - elseif find(architecture,"i386") then + elseif find(architecture,"i386",1,true) then platform="osx-intel" - elseif find(architecture,"x86_64") then + elseif find(architecture,"x86_64",1,true) then platform="osx-64" else platform="osx-ppc" @@ -2788,7 +3070,7 @@ elseif name=="macosx" then elseif name=="sunos" then function resolvers.platform(t,k) local platform,architecture="",os.resultof("uname -m") or "" - if find(architecture,"sparc") then + if find(architecture,"sparc",1,true) then platform="solaris-sparc" else platform="solaris-intel" @@ -2800,7 +3082,7 @@ elseif name=="sunos" then elseif name=="freebsd" then function resolvers.platform(t,k) local platform,architecture="",os.resultof("uname -m") or "" - if find(architecture,"amd64") then + if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else platform="freebsd" @@ -2812,7 +3094,7 @@ elseif name=="freebsd" then elseif name=="kfreebsd" then function resolvers.platform(t,k) local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" - if find(architecture,"x86_64") then + if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else platform="kfreebsd-i386" @@ -2829,8 +3111,9 @@ else return platform end end +os.newline=name=="windows" and "\013\010" or "\010" function resolvers.bits(t,k) - local bits=find(os.platform,"64") and 64 or 32 + local bits=find(os.platform,"64",1,true) and 64 or 32 os.bits=bits return bits end @@ -2980,7 +3263,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 18308, stripped down to: 9948 +-- original size: 20949, stripped down to: 9945 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -2994,41 +3277,28 @@ local file=file if not lfs then lfs=optionalrequire("lfs") end -if not lfs then - lfs={ - getcurrentdir=function() - return "." - end, - attributes=function() - return nil - end, - isfile=function(name) - local f=io.open(name,'rb') - if f then - f:close() - return true - end - end, - isdir=function(name) - print("you need to load lfs") - return false - end - } -elseif not lfs.isfile then - local attributes=lfs.attributes - function lfs.isdir(name) - return attributes(name,"mode")=="directory" - end - function lfs.isfile(name) - return attributes(name,"mode")=="file" - end -end local insert,concat=table.insert,table.concat local match,find,gmatch=string.match,string.find,string.gmatch local lpegmatch=lpeg.match local getcurrentdir,attributes=lfs.currentdir,lfs.attributes local checkedsplit=string.checkedsplit local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local tricky=S("/\\")*P(-1) +local attributes=lfs.attributes +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir,"lfs.isdir") +end +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end +end +function lfs.isfile(name) + return attributes(name,"mode")=="file" +end local colon=P(":") local period=P(".") local periods=P("..") @@ -3230,28 +3500,30 @@ local isroot=fwslash^1*-1 local hasroot=fwslash^1 local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") -function file.join(...) - local lst={... } - local one=lst[1] +function file.join(one,two,three,...) + if not two then + return one=="" and one or lpegmatch(stripper,one) + end + if one=="" then + return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) + end if lpegmatch(isnetwork,one) then local one=lpegmatch(reslasher,one) - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return one..two else return one.."/"..two end elseif lpegmatch(isroot,one) then - local two=lpegmatch(deslasher,concat(lst,"/",2)) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) if lpegmatch(hasroot,two) then return two else return "/"..two end - elseif one=="" then - return lpegmatch(stripper,concat(lst,"/",2)) else - return lpegmatch(deslasher,concat(lst,"/")) + return lpegmatch(deslasher,concat({ one,two,three,... },"/")) end end local drivespec=R("az","AZ")^1*colon @@ -3425,7 +3697,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3760, stripped down to: 2088 +-- original size: 3248, stripped down to: 2266 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3443,14 +3715,20 @@ if not md5 then } end local md5,file=md5,file -local gsub,format,byte=string.gsub,string.format,string.byte -local md5sum=md5.sum -local function convert(str,fmt) - return (gsub(md5sum(str),".",function(chr) return format(fmt,byte(chr)) end)) -end -if not md5.HEX then function md5.HEX(str) return convert(str,"%02X") end end -if not md5.hex then function md5.hex(str) return convert(str,"%02x") end end -if not md5.dec then function md5.dec(str) return convert(str,"%03i") end end +local gsub=string.gsub +do + local patterns=lpeg and lpeg.patterns + if patterns then + local bytestoHEX=patterns.bytestoHEX + local bytestohex=patterns.bytestohex + local bytestodec=patterns.bytestodec + local lpegmatch=lpeg.match + local md5sum=md5.sum + if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end + if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end + if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + end +end function file.needsupdating(oldname,newname,threshold) local oldtime=lfs.attributes(oldname,"modification") if oldtime then @@ -3507,7 +3785,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 11993, stripped down to: 5584 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -3534,7 +3812,7 @@ local hexdigit=R("09","AF","af") local plus=P("+") local nothing=Cc("") local escapedchar=(percent*C(hexdigit*hexdigit))/tochar -local escaped=(plus/" ")+escapedchar +local escaped=(plus/" ")+escapedchar local noslash=P("/")/"" local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) @@ -3593,19 +3871,25 @@ local splitquery=Cf (Ct("")*P { "sequence", pair=Cg(key*equal*value), },rawset) local function hashed(str) - if str=="" then + if not str or str=="" then return { scheme="invalid", original=str, } end - local s=split(str) - local rawscheme=s[1] - local rawquery=s[4] - local somescheme=rawscheme~="" - local somequery=rawquery~="" + local detailed=split(str) + local rawscheme="" + local rawquery="" + local somescheme=false + local somequery=false + if detailed then + rawscheme=detailed[1] + rawquery=detailed[4] + somescheme=rawscheme~="" + somequery=rawquery~="" + end if not somescheme and not somequery then - s={ + return { scheme="file", authority="", path=str, @@ -3615,28 +3899,28 @@ local function hashed(str) noscheme=true, filename=str, } - else - local authority,path,filename=s[2],s[3] - if authority=="" then - filename=path - elseif path=="" then - filename="" - else - filename=authority.."/"..path - end - s={ - scheme=rawscheme, - authority=authority, - path=path, - query=lpegmatch(unescaper,rawquery), - queries=lpegmatch(splitquery,rawquery), - fragment=s[5], - original=str, - noscheme=false, - filename=filename, - } end - return s + local authority=detailed[2] + local path=detailed[3] + local filename=nil + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + return { + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=detailed[5], + original=str, + noscheme=false, + filename=filename, + } end url.split=split url.hasscheme=hasscheme @@ -3670,7 +3954,7 @@ function url.construct(hash) end return lpegmatch(escaper,concat(fullurl)) end -local pattern=Cs(noslash*R("az","AZ")*(S(":|")/":")*noslash*P(1)^0) +local pattern=Cs(slash^-1/""*R("az","AZ")*((S(":|")/":")+P(":"))*slash*P(1)^0) function url.filename(filename) local spec=hashed(filename) local path=spec.path @@ -3718,7 +4002,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 14229, stripped down to: 8740 +-- original size: 16765, stripped down to: 11003 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -3728,7 +4012,7 @@ if not modules then modules={} end modules ['l-dir']={ license="see context related readme files" } local type,select=type,select -local find,gmatch,match,gsub=string.find,string.gmatch,string.match,string.gsub +local find,gmatch,match,gsub,sub=string.find,string.gmatch,string.match,string.gsub,string.sub local concat,insert,remove,unpack=table.concat,table.insert,table.remove,table.unpack local lpegmatch=lpeg.match local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V @@ -3737,53 +4021,127 @@ local dir=dir local lfs=lfs local attributes=lfs.attributes local walkdir=lfs.dir -local isdir=lfs.isdir -local isfile=lfs.isfile +local isdir=lfs.isdir +local isfile=lfs.isfile local currentdir=lfs.currentdir local chdir=lfs.chdir -local onwindows=os.type=="windows" or find(os.getenv("PATH"),";") -if not isdir then - function isdir(name) - local a=attributes(name) - return a and a.mode=="directory" +local mkdir=lfs.mkdir +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";",1,true) +if onwindows then + local tricky=S("/\\")*P(-1) + isdir=function(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end + end + isfile=function(name) + return attributes(name,"mode")=="file" end lfs.isdir=isdir -end -if not isfile then - function isfile(name) - local a=attributes(name) - return a and a.mode=="file" + lfs.isfile=isfile +else + isdir=function(name) + return attributes(name,"mode")=="directory" end + isfile=function(name) + return attributes(name,"mode")=="file" + end + lfs.isdir=isdir lfs.isfile=isfile end function dir.current() return (gsub(currentdir(),"\\","/")) end -local lfsisdir=isdir -local function isdir(path) - path=gsub(path,"[/\\]+$","") - return lfsisdir(path) +local function glob_pattern_function(path,patt,recurse,action) + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + action(full) + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end + end + end + end + if dirs then + for i=1,#dirs do + glob_pattern_function(dirs[i],patt,recurse,action) + end + end + end end -lfs.isdir=isdir -local function globpattern(path,patt,recurse,action) - if path=="/" then - path=path.."." - elseif not find(path,"/$") then - path=path..'/' - end - if isdir(path) then - for name in walkdir(path) do - local full=path..name - local mode=attributes(full,'mode') - if mode=='file' then - if find(full,patt) then - action(full) +local function glob_pattern_table(path,patt,recurse,result) + if not result then + result={} + end + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + result[#result+1]=full + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - globpattern(full,patt,recurse,action) + end + end + if dirs then + for i=1,#dirs do + glob_pattern_table(dirs[i],patt,recurse,result) end end end + return result +end +local function globpattern(path,patt,recurse,method) + local kind=type(method) + if patt and sub(patt,1,-3)==path then + patt=false + end + if kind=="function" then + return glob_pattern_function(path,patt,recurse,method) + elseif kind=="table" then + return glob_pattern_table(path,patt,recurse,method) + else + return glob_pattern_table(path,patt,recurse,{}) + end end dir.globpattern=globpattern local function collectpattern(path,patt,recurse,result) @@ -3795,34 +4153,40 @@ local function collectpattern(path,patt,recurse,result) ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) end if ok and type(scanner)=="function" then - if not find(path,"/$") then path=path..'/' end + if not find(path,"/$") then + path=path..'/' + end for name in scanner,first do - local full=path..name - local attr=attributes(full) - local mode=attr.mode - if mode=='file' then - if find(full,patt) then + if name=="." then + elseif name==".." then + else + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and mode=="directory" then + attr.list=collectpattern(full,patt,recurse) result[name]=attr end - elseif recurse and (mode=="directory") and (name~='.') and (name~="..") then - attr.list=collectpattern(full,patt,recurse) - result[name]=attr end end end return result end dir.collectpattern=collectpattern -local separator -if onwindows then +local separator,pattern +if onwindows then local slash=S("/\\")/"/" - pattern=Ct { + pattern={ [1]=(Cs(P(".")+slash^1)+Cs(R("az","AZ")*P(":")*slash^0)+Cc("./"))*V(2)*V(3), [2]=Cs(((1-S("*?/\\"))^0*slash)^0), [3]=Cs(P(1)^0) } -else - pattern=Ct { +else + pattern={ [1]=(C(P(".")+P("/")^1)+Cc("./"))*V(2)*V(3), [2]=C(((1-S("*?/"))^0*P("/"))^0), [3]=C(P(1)^0) @@ -3840,10 +4204,9 @@ local function glob(str,t) elseif isfile(str) then t(str) else - local split=lpegmatch(pattern,str) - if split then - local root,path,base=split[1],split[2],split[3] - local recurse=find(base,"%*%*") + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) globpattern(start,result,recurse,t) @@ -3864,16 +4227,12 @@ local function glob(str,t) return { str } end else - local split=lpegmatch(pattern,str) - if split then - local t=t or {} - local action=action or function(name) t[#t+1]=name end - local root,path,base=split[1],split[2],split[3] - local recurse=find(base,"%*%*") + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) local start=root..path local result=lpegmatch(filter,start..base) - globpattern(start,result,recurse,action) - return t + return globpattern(start,result,recurse,t) else return {} end @@ -3913,16 +4272,26 @@ end local make_indeed=true if onwindows then function dir.mkdirs(...) - local str,pth="","" - for i=1,select("#",...) do - local s=select(i,...) - if s=="" then - elseif str=="" then - str=s - else - str=str.."/"..s + local n=select("#",...) + local str + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end end end + local pth="" local drive=false local first,middle,last=match(str,"^(//)(//*)(.*)$") if first then @@ -3957,21 +4326,30 @@ if onwindows then pth=pth.."/"..s end if make_indeed and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end return pth,(isdir(pth)==true) end else function dir.mkdirs(...) - local str,pth="","" - for i=1,select("#",...) do - local s=select(i,...) - if s and s~="" then - if str~="" then - str=str.."/"..s - else - str=s + local n=select("#",...) + local str,pth + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s + else + str=s + end end end end @@ -3986,7 +4364,7 @@ else pth=pth.."/"..s end if make_indeed and not first and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end else @@ -3994,7 +4372,7 @@ else for s in gmatch(str,"[^/]+") do pth=pth.."/"..s if make_indeed and not isdir(pth) then - lfs.mkdir(pth) + mkdir(pth) end end end @@ -4002,47 +4380,51 @@ else end end dir.makedirs=dir.mkdirs -if onwindows then - function dir.expandname(str) - local first,nothing,last=match(str,"^(//)(//*)(.*)$") - if first then - first=dir.current().."/" - end - if not first then - first,last=match(str,"^(//)/*(.*)$") - end - if not first then - first,last=match(str,"^([a-zA-Z]:)(.*)$") - if first and not find(last,"^/") then - local d=currentdir() - if chdir(first) then - first=dir.current() +do + local chdir=sandbox and sandbox.original(chdir) or chdir + if onwindows then + local xcurrentdir=dir.current + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=xcurrentdir().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if chdir(first) then + first=xcurrentdir() + end + chdir(d) end - chdir(d) + end + if not first then + first,last=xcurrentdir(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first + else + return first.."/"..last end end - if not first then - first,last=dir.current(),str - end - last=gsub(last,"//","/") - last=gsub(last,"/%./","/") - last=gsub(last,"^/*","") - first=gsub(first,"/*$","") - if last=="" or last=="." then - return first - else - return first.."/"..last - end - end -else - function dir.expandname(str) - if not find(str,"^/") then - str=currentdir().."/"..str + else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str + end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str end - str=gsub(str,"//","/") - str=gsub(str,"/%./","/") - str=gsub(str,"(.)/%.$","%1") - return str end end file.expandname=dir.expandname @@ -4085,7 +4467,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1809, stripped down to: 1527 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4139,11 +4521,11 @@ function string.booleanstring(str) return str=="yes" or str=="on" or str=="t" end end -function string.is_boolean(str,default) +function string.is_boolean(str,default,strict) if type(str)=="string" then - if str=="true" or str=="yes" or str=="on" or str=="t" or str=="1" then + if str=="true" or str=="yes" or str=="on" or str=="t" or (not strict and str=="1") then return true - elseif str=="false" or str=="no" or str=="off" or str=="f" or str=="0" then + elseif str=="false" or str=="no" or str=="off" or str=="f" or (not strict and str=="0") then return false end end @@ -4157,7 +4539,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 33473, stripped down to: 14938 +-- original size: 37388, stripped down to: 15817 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4173,7 +4555,9 @@ local type=type local char,byte,format,sub,gmatch=string.char,string.byte,string.format,string.sub,string.gmatch local concat=table.concat local P,C,R,Cs,Ct,Cmt,Cc,Carg,Cp=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg,lpeg.Cp -local lpegmatch,patterns=lpeg.match,lpeg.patterns +local lpegmatch=lpeg.match +local patterns=lpeg.patterns +local tabletopattern=lpeg.utfchartabletopattern local bytepairs=string.bytepairs local finder=lpeg.finder local replacer=lpeg.replacer @@ -4182,7 +4566,7 @@ local utfgmatch=utf.gmatch local p_utftype=patterns.utftype local p_utfstricttype=patterns.utfstricttype local p_utfoffset=patterns.utfoffset -local p_utf8char=patterns.utf8char +local p_utf8char=patterns.utf8character local p_utf8byte=patterns.utf8byte local p_utfbom=patterns.utfbom local p_newline=patterns.newline @@ -4321,6 +4705,7 @@ if not utf.sub then local pattern_zero=Cmt(p_utf8char,slide_zero)^0 local pattern_one=Cmt(p_utf8char,slide_one )^0 local pattern_two=Cmt(p_utf8char,slide_two )^0 + local pattern_first=C(patterns.utf8character) function utf.sub(str,start,stop) if not start then return str @@ -4362,7 +4747,9 @@ if not utf.sub then end end end - if start>stop then + if start==1 and stop==1 then + return lpegmatch(pattern_first,str) or "" + elseif start>stop then return "" elseif start>1 then b,e,n,first,last=0,0,0,start-1,stop @@ -4381,15 +4768,52 @@ if not utf.sub then end end end -function utf.remapper(mapping) - local pattern=Cs((p_utf8char/mapping)^0) - return function(str) - if not str or str=="" then - return "" +function utf.remapper(mapping,option) + local variant=type(mapping) + if variant=="table" then + if option=="dynamic" then + local pattern=false + table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) + return function(str) + if not str or str=="" then + return "" + else + if not pattern then + pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + end + return lpegmatch(pattern,str) + end + end + elseif option=="pattern" then + return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) else - return lpegmatch(pattern,str) + local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + elseif variant=="function" then + if option=="pattern" then + return Cs((p_utf8char/mapping+p_utf8char)^0) + else + local pattern=Cs((p_utf8char/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern end - end,pattern + else + return function(str) + return str or "" + end + end end function utf.replacer(t) local r=replacer(t,false,false,true) @@ -4439,190 +4863,157 @@ function utf.magic(f) end local utf16_to_utf8_be,utf16_to_utf8_le local utf32_to_utf8_be,utf32_to_utf8_le -local utf_16_be_linesplitter=patterns.utfbom_16_be^-1*lpeg.tsplitat(patterns.utf_16_be_nl) -local utf_16_le_linesplitter=patterns.utfbom_16_le^-1*lpeg.tsplitat(patterns.utf_16_le_nl) -if bytepairs then - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*left+right - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +local utf_16_be_getbom=patterns.utfbom_16_be^-1 +local utf_16_le_getbom=patterns.utfbom_16_le^-1 +local utf_32_be_getbom=patterns.utfbom_32_be^-1 +local utf_32_le_getbom=patterns.utfbom_32_le^-1 +local utf_16_be_linesplitter=utf_16_be_getbom*lpeg.tsplitat(patterns.utf_16_be_nl) +local utf_16_le_linesplitter=utf_16_le_getbom*lpeg.tsplitat(patterns.utf_16_le_nl) +local utf_32_be_linesplitter=utf_32_be_getbom*lpeg.tsplitat(patterns.utf_32_be_nl) +local utf_32_le_linesplitter=utf_32_le_getbom*lpeg.tsplitat(patterns.utf_32_le_nl) +local more=0 +local p_utf16_to_utf8_be=C(1)*C(1)/function(left,right) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf16_to_utf8_le=C(1)*C(1)/function(right,left) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf32_to_utf8_be=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(a)+256*256*byte(b)+256*byte(c)+byte(d)) +end +local p_utf32_to_utf8_le=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(d)+256*256*byte(c)+256*byte(b)+byte(a)) +end +p_utf16_to_utf8_be=P(true)/function() more=0 end*utf_16_be_getbom*Cs(p_utf16_to_utf8_be^0) +p_utf16_to_utf8_le=P(true)/function() more=0 end*utf_16_le_getbom*Cs(p_utf16_to_utf8_le^0) +p_utf32_to_utf8_be=P(true)/function() more=0 end*utf_32_be_getbom*Cs(p_utf32_to_utf8_be^0) +p_utf32_to_utf8_le=P(true)/function() more=0 end*utf_32_le_getbom*Cs(p_utf32_to_utf8_le^0) +patterns.utf16_to_utf8_be=p_utf16_to_utf8_be +patterns.utf16_to_utf8_le=p_utf16_to_utf8_le +patterns.utf32_to_utf8_be=p_utf32_to_utf8_be +patterns.utf32_to_utf8_le=p_utf32_to_utf8_le +utf16_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_be,s) + else + return s end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in bytepairs(t[i]) do - if right then - local now=256*right+left - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) - end - return t +end +local utf16_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_be_linesplitter,t) end - utf32_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*256*256*a+256*256*b - else - r=r+1 - result[t]=utfchar(more+256*a+b) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_be,s) end - return t end - utf32_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utflinesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,-1 - for a,b in bytepairs(t[i]) do - if a and b then - if more<0 then - more=256*b+a - else - r=r+1 - result[t]=utfchar(more+256*256*256*b+256*256*a) - more=-1 - end - else - break - end - end - t[i]=concat(result,"",1,r) - end - return t + return t +end +utf16_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_le,s) + else + return s end -else - utf16_to_utf8_be=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_be_linesplitter,t) - end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if left=="\000" then - r=r+1 - result[r]=utfchar(byte(right)) - elseif right then - local now=256*byte(left)+byte(right) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) +end +local utf16_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_le,s) end - return t end - utf16_to_utf8_le=function(t) - if type(t)=="string" then - t=lpegmatch(utf_16_le_linesplitter,t) + return t +end +utf32_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_be,s) + else + return s + end +end +local utf32_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_be_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_be,s) end - local result={} - for i=1,#t do - local r,more=0,0 - for left,right in gmatch(t[i],"(.)(.)") do - if right=="\000" then - r=r+1 - result[r]=utfchar(byte(left)) - elseif right then - local now=256*byte(right)+byte(left) - if more>0 then - now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 - more=0 - r=r+1 - result[r]=utfchar(now) - elseif now>=0xD800 and now<=0xDBFF then - more=now - else - r=r+1 - result[r]=utfchar(now) - end - end - end - t[i]=concat(result,"",1,r) + end + return t +end +utf32_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_le,s) + else + return s + end +end +local utf32_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_le,s) end - return t end - utf32_to_utf8_le=function() return {} end - utf32_to_utf8_be=function() return {} end + return t end +utf.utf16_to_utf8_le_t=utf16_to_utf8_le_t +utf.utf16_to_utf8_be_t=utf16_to_utf8_be_t +utf.utf32_to_utf8_le_t=utf32_to_utf8_le_t +utf.utf32_to_utf8_be_t=utf32_to_utf8_be_t utf.utf16_to_utf8_le=utf16_to_utf8_le utf.utf16_to_utf8_be=utf16_to_utf8_be utf.utf32_to_utf8_le=utf32_to_utf8_le utf.utf32_to_utf8_be=utf32_to_utf8_be -function utf.utf8_to_utf8(t) +function utf.utf8_to_utf8_t(t) return type(t)=="string" and lpegmatch(utflinesplitter,t) or t end -function utf.utf16_to_utf8(t,endian) - return endian and utf16_to_utf8_be(t) or utf16_to_utf8_le(t) or t +function utf.utf16_to_utf8_t(t,endian) + return endian and utf16_to_utf8_be_t(t) or utf16_to_utf8_le_t(t) or t end -function utf.utf32_to_utf8(t,endian) - return endian and utf32_to_utf8_be(t) or utf32_to_utf8_le(t) or t +function utf.utf32_to_utf8_t(t,endian) + return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t end -local function little(c) - local b=byte(c) +local function little(b) if b<0x10000 then return char(b%256,b/256) else @@ -4631,8 +5022,7 @@ local function little(c) return char(b1%256,b1/256,b2%256,b2/256) end end -local function big(c) - local b=byte(c) +local function big(b) if b<0x10000 then return char(b/256,b%256) else @@ -4641,27 +5031,29 @@ local function big(c) return char(b1/256,b1%256,b2/256,b2%256) end end -local _,l_remap=utf.remapper(little) -local _,b_remap=utf.remapper(big) -function utf.utf8_to_utf16_be(str,nobom) +local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) +local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) +local function utf8_to_utf16_be(str,nobom) if nobom then return lpegmatch(b_remap,str) else return char(254,255)..lpegmatch(b_remap,str) end end -function utf.utf8_to_utf16_le(str,nobom) +local function utf8_to_utf16_le(str,nobom) if nobom then return lpegmatch(l_remap,str) else return char(255,254)..lpegmatch(l_remap,str) end end +utf.utf8_to_utf16_be=utf8_to_utf16_be +utf.utf8_to_utf16_le=utf8_to_utf16_le function utf.utf8_to_utf16(str,littleendian,nobom) if littleendian then - return utf.utf8_to_utf16_le(str,nobom) + return utf8_to_utf16_le(str,nobom) else - return utf.utf8_to_utf16_be(str,nobom) + return utf8_to_utf16_be(str,nobom) end end local pattern=Cs ( @@ -4677,16 +5069,16 @@ function utf.xstring(s) return format("0x%05X",type(s)=="number" and s or utfbyte(s)) end function utf.toeight(str) - if not str then + if not str or str=="" then return nil end local utftype=lpegmatch(p_utfstricttype,str) if utftype=="utf-8" then - return sub(str,4) - elseif utftype=="utf-16-le" then - return utf16_to_utf8_le(str) + return sub(str,4) elseif utftype=="utf-16-be" then - return utf16_to_utf8_ne(str) + return utf16_to_utf8_be(str) + elseif utftype=="utf-16-le" then + return utf16_to_utf8_le(str) else return str end @@ -4765,7 +5157,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 915, stripped down to: 836 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -4775,6 +5167,9 @@ if not modules then modules={} end modules ['l-math']={ license="see context related readme files" } local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.ceiling then + math.ceiling=math.ceil +end if not math.round then function math.round(x) return floor(x+0.5) end end @@ -4802,7 +5197,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 26857, stripped down to: 15062 +-- original size: 34503, stripped down to: 18933 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -4821,25 +5216,43 @@ local unpack,concat=table.unpack,table.concat local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc local patterns,lpegmatch=lpeg.patterns,lpeg.match local utfchar,utfbyte=utf.char,utf.byte -local loadstripped=_LUAVERSION<5.2 and load or function(str) - return load(dump(load(str),true)) +local loadstripped=nil +if _LUAVERSION<5.2 then + loadstripped=function(str,shortcuts) + return load(str) + end +else + loadstripped=function(str,shortcuts) + if shortcuts then + return load(dump(load(str),true),nil,nil,shortcuts) + else + return load(dump(load(str),true)) + end + end end if not number then number={} end local stripper=patterns.stripzeros +local newline=patterns.newline +local endofstring=patterns.endofstring +local whitespace=patterns.whitespace +local spacer=patterns.spacer +local spaceortab=patterns.spaceortab local function points(n) + n=tonumber(n) return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) end local function basepoints(n) + n=tonumber(n) return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) end number.points=points number.basepoints=basepoints -local rubish=patterns.spaceortab^0*patterns.newline -local anyrubish=patterns.spaceortab+patterns.newline +local rubish=spaceortab^0*newline +local anyrubish=spaceortab+newline local anything=patterns.anything -local stripped=(patterns.spaceortab^1/"")*patterns.newline +local stripped=(spaceortab^1/"")*newline local leading=rubish^0/"" -local trailing=(anyrubish^1*patterns.endofstring)/"" +local trailing=(anyrubish^1*endofstring)/"" local redundant=rubish^3/"\n" local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) function strings.collapsecrlf(str) @@ -4885,18 +5298,44 @@ local pattern=Carg(1)/function(t) else return "" end - end+patterns.newline*Cp()/function(position) + end+newline*Cp()/function(position) extra,start=0,position end+patterns.anything )^1) function strings.tabtospace(str,tab) return lpegmatch(pattern,str,1,tab or 7) end -function strings.striplong(str) - str=gsub(str,"^%s*","") - str=gsub(str,"[\n\r]+ *","\n") - return str +local space=spacer^0 +local nospace=space/"" +local endofline=nospace*newline +local stripend=(whitespace^1*endofstring)/"" +local normalline=(nospace*((1-space*(newline+endofstring))^1)*nospace) +local stripempty=endofline^1/"" +local normalempty=endofline^1 +local singleempty=endofline*(endofline^0/"") +local doubleempty=endofline*endofline^-1*(endofline^0/"") +local stripstart=stripempty^0 +local p_prune_normal=Cs (stripstart*(stripend+normalline+normalempty )^0 ) +local p_prune_collapse=Cs (stripstart*(stripend+normalline+doubleempty )^0 ) +local p_prune_noempty=Cs (stripstart*(stripend+normalline+singleempty )^0 ) +local p_retain_normal=Cs ((normalline+normalempty )^0 ) +local p_retain_collapse=Cs ((normalline+doubleempty )^0 ) +local p_retain_noempty=Cs ((normalline+singleempty )^0 ) +local striplinepatterns={ + ["prune"]=p_prune_normal, + ["prune and collapse"]=p_prune_collapse, + ["prune and no empty"]=p_prune_noempty, + ["retain"]=p_retain_normal, + ["retain and collapse"]=p_retain_collapse, + ["retain and no empty"]=p_retain_noempty, + ["collapse"]=patterns.collapser, +} +setmetatable(striplinepatterns,{ __index=function(t,k) return p_prune_collapse end }) +strings.striplinepatterns=striplinepatterns +function strings.striplines(str,how) + return str and lpegmatch(striplinepatterns[how],str) or str end +strings.striplong=strings.striplines function strings.nice(str) str=gsub(str,"[:%-+_]+"," ") return str @@ -4934,10 +5373,10 @@ string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) if type(b)=="number" then - return tracedchars[b] or (utfchar(b).." (U+"..format('%05X',b)..")") + return tracedchars[b] or (utfchar(b).." (U+"..format("%05X",b)..")") else local c=utfbyte(b) - return tracedchars[c] or (b.." (U+"..format('%05X',c)..")") + return tracedchars[c] or (b.." (U+"..(c and format("%05X",c) or "?????")..")") end end function number.signed(i) @@ -4972,31 +5411,58 @@ function number.sparseexponent(f,n) end return tostring(n) end -local preamble=[[ -local type = type -local tostring = tostring -local tonumber = tonumber -local format = string.format -local concat = table.concat -local signed = number.signed -local points = number.points -local basepoints = number.basepoints -local utfchar = utf.char -local utfbyte = utf.byte -local lpegmatch = lpeg.match -local nspaces = string.nspaces -local tracedchar = string.tracedchar -local autosingle = string.autosingle -local autodouble = string.autodouble -local sequenced = table.sequenced -local formattednumber = number.formatted -local sparseexponent = number.sparseexponent -]] local template=[[ %s %s return function(%s) return %s end ]] +local preamble,environment="",{} +if _LUAVERSION<5.2 then + preamble=[[ +local lpeg=lpeg +local type=type +local tostring=tostring +local tonumber=tonumber +local format=string.format +local concat=table.concat +local signed=number.signed +local points=number.points +local basepoints= number.basepoints +local utfchar=utf.char +local utfbyte=utf.byte +local lpegmatch=lpeg.match +local nspaces=string.nspaces +local tracedchar=string.tracedchar +local autosingle=string.autosingle +local autodouble=string.autodouble +local sequenced=table.sequenced +local formattednumber=number.formatted +local sparseexponent=number.sparseexponent + ]] +else + environment={ + global=global or _G, + lpeg=lpeg, + type=type, + tostring=tostring, + tonumber=tonumber, + format=string.format, + concat=table.concat, + signed=number.signed, + points=number.points, + basepoints=number.basepoints, + utfchar=utf.char, + utfbyte=utf.byte, + lpegmatch=lpeg.match, + nspaces=string.nspaces, + tracedchar=string.tracedchar, + autosingle=string.autosingle, + autodouble=string.autodouble, + sequenced=table.sequenced, + formattednumber=number.formatted, + sparseexponent=number.sparseexponent, + } +end local arguments={ "a1" } setmetatable(arguments,{ __index=function(t,k) local v=t[k-1]..",a"..k @@ -5035,7 +5501,7 @@ local format_i=function(f) if f and f~="" then return format("format('%%%si',a%s)",f,n) else - return format("format('%%i',a%s)",n) + return format("format('%%i',a%s)",n) end end local format_d=format_i @@ -5047,6 +5513,14 @@ local format_f=function(f) n=n+1 return format("format('%%%sf',a%s)",f,n) end +local format_F=function(f) + n=n+1 + if not f or f=="" then + return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or format((a%s %% 1 == 0) and '%%i' or '%%.9f',a%s))",n,n,n,n) + else + return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) + end +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5261,7 +5735,7 @@ local builder=Cs { "start", ( P("%")/""*( V("!") -+V("s")+V("q")+V("i")+V("d")+V("f")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") +V("N") @@ -5272,7 +5746,6 @@ local builder=Cs { "start", +V("j")+V("J") +V("m")+V("M") +V("z") -+V("*") )+V("*") )*(P(-1)+Carg(1)) )^0, @@ -5281,6 +5754,7 @@ local builder=Cs { "start", ["i"]=(prefix_any*P("i"))/format_i, ["d"]=(prefix_any*P("d"))/format_d, ["f"]=(prefix_any*P("f"))/format_f, + ["F"]=(prefix_any*P("F"))/format_F, ["g"]=(prefix_any*P("g"))/format_g, ["G"]=(prefix_any*P("G"))/format_G, ["e"]=(prefix_any*P("e"))/format_e, @@ -5315,11 +5789,12 @@ local builder=Cs { "start", ["a"]=(prefix_any*P("a"))/format_a, ["A"]=(prefix_any*P("A"))/format_A, ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, + ["?"]=Cs(((1-P("%"))^1 )^1)/format_rest, ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, } local direct=Cs ( - P("%")/""*Cc([[local format = string.format return function(str) return format("%]])*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*Cc([[",str) end]])*P(-1) - ) + P("%")*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*P(-1)/[[local format = string.format return function(str) return format("%0",str) end]] +) local function make(t,str) local f local p @@ -5328,10 +5803,10 @@ local function make(t,str) f=loadstripped(p)() else n=0 - p=lpegmatch(builder,str,1,"..",t._extensions_) + p=lpegmatch(builder,str,1,t._connector_,t._extensions_) if n>0 then p=format(template,preamble,t._preamble_,arguments[n],p) - f=loadstripped(p)() + f=loadstripped(p,t._environment_)() else f=function() return str end end @@ -5343,10 +5818,22 @@ local function use(t,fmt,...) return t[fmt](...) end strings.formatters={} -function strings.formatters.new() - local t={ _extensions_={},_preamble_="",_type_="formatter" } - setmetatable(t,{ __index=make,__call=use }) - return t +if _LUAVERSION<5.2 then + function strings.formatters.new(noconcat) + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_=preamble,_environment_={} } + setmetatable(t,{ __index=make,__call=use }) + return t + end +else + function strings.formatters.new(noconcat) + local e={} + for k,v in next,environment do + e[k]=v + end + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_="",_environment_=e } + setmetatable(t,{ __index=make,__call=use }) + return t + end end local formatters=strings.formatters.new() string.formatters=formatters @@ -5354,8 +5841,12 @@ string.formatter=function(str,...) return formatters[str](...) end local function add(t,name,template,preamble) if type(t)=="table" and t._type_=="formatter" then t._extensions_[name]=template or "%s" - if preamble then + if type(preamble)=="string" then t._preamble_=preamble.."\n"..t._preamble_ + elseif type(preamble)=="table" then + for k,v in next,preamble do + t._environment_[k]=v + end end end end @@ -5364,9 +5855,28 @@ patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"" patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+P(1))^0) patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) -add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],[[local xmlescape = lpeg.patterns.xmlescape]]) -add(formatters,"tex",[[lpegmatch(texescape,%s)]],[[local texescape = lpeg.patterns.texescape]]) -add(formatters,"lua",[[lpegmatch(luaescape,%s)]],[[local luaescape = lpeg.patterns.luaescape]]) +if _LUAVERSION<5.2 then + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],"local xmlescape = lpeg.patterns.xmlescape") + add(formatters,"tex",[[lpegmatch(texescape,%s)]],"local texescape = lpeg.patterns.texescape") + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],"local luaescape = lpeg.patterns.luaescape") +else + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) + add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +end +local dquote=patterns.dquote +local equote=patterns.escaped+dquote/'\\"'+1 +local space=patterns.space +local cquote=Cc('"') +local pattern=Cs(dquote*(equote-P(-2))^0*dquote) ++Cs(cquote*(equote-space)^0*space*equote^0*cquote) +function string.optionalquoted(str) + return lpegmatch(pattern,str) or str +end +local pattern=Cs((newline/os.newline+1)^0) +function string.replacenewlines(str) + return lpegmatch(pattern,str) +end end -- of closure @@ -5375,7 +5885,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 23952, stripped down to: 16092 +-- original size: 25338, stripped down to: 16247 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5388,7 +5898,7 @@ utilities=utilities or {} utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub -local concat,insert,remove=table.concat,table.insert,table.remove +local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc @@ -5396,27 +5906,29 @@ local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs local formatters=string.formatters local utftoeight=utf.toeight local splitter=lpeg.tsplitat(".") -function tables.definetable(target,nofirst,nolast) - local composed,shortcut,t=nil,nil,{} +function utilities.tables.definetable(target,nofirst,nolast) + local composed,t=nil,{} local snippets=lpegmatch(splitter,target) for i=1,#snippets-(nolast and 1 or 0) do local name=snippets[i] if composed then - composed=shortcut.."."..name - shortcut=shortcut.."_"..name - t[#t+1]=formatters["local %s = %s if not %s then %s = { } %s = %s end"](shortcut,composed,shortcut,shortcut,composed,shortcut) + composed=composed.."."..name + t[#t+1]=formatters["if not %s then %s = { } end"](composed,composed) else composed=name - shortcut=name if not nofirst then t[#t+1]=formatters["%s = %s or { }"](composed,composed) end end end - if nolast then - composed=shortcut.."."..snippets[#snippets] + if composed then + if nolast then + composed=composed.."."..snippets[#snippets] + end + return concat(t,"\n"),composed + else + return "",target end - return concat(t,"\n"),composed end function tables.definedtable(...) local t=_G @@ -5443,7 +5955,7 @@ function tables.accesstable(target,root) end function tables.migratetable(target,v,root) local t=root or _G - local names=string.split(target,".") + local names=lpegmatch(splitter,target) for i=1,#names-1 do local name=names[i] t[name]=t[name] or {} @@ -5463,6 +5975,15 @@ function tables.removevalue(t,value) end end end +function tables.replacevalue(t,oldvalue,newvalue) + if oldvalue and newvalue then + for i=1,#t do + if t[i]==oldvalue then + t[i]=newvalue + end + end + end +end function tables.insertbeforevalue(t,value,extra) for i=1,#t do if t[i]==extra then @@ -5610,7 +6131,7 @@ local f_ordered_string=formatters["%q,"] local f_ordered_number=formatters["%s,"] local f_ordered_boolean=formatters["%l,"] function table.fastserialize(t,prefix) - local r={ prefix or "return" } + local r={ type(prefix)=="string" and prefix or "return" } local m=1 local function fastserialize(t,outer) local n=#t @@ -5807,7 +6328,8 @@ function table.serialize(root,name,specification) local t local n=1 local function simple_table(t) - if #t>0 then + local nt=#t + if nt>0 then local n=0 for _,v in next,t do n=n+1 @@ -5815,19 +6337,17 @@ function table.serialize(root,name,specification) return nil end end - if n==#t then + if n==nt then local tt={} - local nt=0 - for i=1,#t do + for i=1,nt do local v=t[i] local tv=type(v) - nt=nt+1 if tv=="number" then - tt[nt]=v + tt[i]=v elseif tv=="string" then - tt[nt]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then - tt[nt]=v and "true" or "false" + tt[i]=v and "true" or "false" else return nil end @@ -5856,7 +6376,7 @@ function table.serialize(root,name,specification) end depth=depth+1 end - if root and next(root) then + if root and next(root)~=nil then local first=nil local last=0 last=#root @@ -5875,13 +6395,13 @@ function table.serialize(root,name,specification) local v=root[k] local tv=type(v) local tk=type(k) - if first and tk=="number" and k>=first and k<=last then + if first and tk=="number" and k<=last and k>=first then if tv=="number" then n=n+1 t[n]=f_val_num(depth,v) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if not next(v) then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -5911,13 +6431,13 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_boo_value_str(depth,k,v) end elseif tv=="table" then - if not next(v) then + if next(v)==nil then if tk=="number" then - n=n+1 t[n]=f_key_num_value_not(depth,k,v) + n=n+1 t[n]=f_key_num_value_not(depth,k) elseif tk=="string" then - n=n+1 t[n]=f_key_str_value_not(depth,k,v) + n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then - n=n+1 t[n]=f_key_boo_value_not(depth,k,v) + n=n+1 t[n]=f_key_boo_value_not(depth,k) end else local st=simple_table(v) @@ -5969,7 +6489,7 @@ function table.serialize(root,name,specification) local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end - if next(root) then + if next(root)~=nil then do_serialize(root,name,1,0) end end @@ -6132,7 +6652,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 19604, stripped down to: 13998 +-- original size: 21780, stripped down to: 15121 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6154,6 +6674,8 @@ local patterns=parsers.patterns or {} parsers.patterns=patterns local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash +local sortedkeys=table.sortedkeys +local tohash=table.tohash local digit=R("09") local space=P(' ') local equal=P("=") @@ -6203,9 +6725,7 @@ patterns.settings_to_hash_a=pattern_a_s patterns.settings_to_hash_b=pattern_b_s patterns.settings_to_hash_c=pattern_c_s function parsers.make_settings_to_hash_pattern(set,how) - if type(str)=="table" then - return set - elseif how=="strict" then + if how=="strict" then return (pattern_c/set)^1 elseif how=="tolerant" then return (pattern_b/set)^1 @@ -6214,7 +6734,9 @@ function parsers.make_settings_to_hash_pattern(set,how) end end function parsers.settings_to_hash(str,existing) - if type(str)=="table" then + if not str or str=="" then + return {} + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6223,16 +6745,16 @@ function parsers.settings_to_hash(str,existing) else return str end - elseif str and str~="" then + else hash=existing or {} lpegmatch(pattern_a_s,str) return hash - else - return {} end end function parsers.settings_to_hash_tolerant(str,existing) - if type(str)=="table" then + if not str or str=="" then + return {} + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6241,16 +6763,16 @@ function parsers.settings_to_hash_tolerant(str,existing) else return str end - elseif str and str~="" then + else hash=existing or {} lpegmatch(pattern_b_s,str) return hash - else - return {} end end function parsers.settings_to_hash_strict(str,existing) - if type(str)=="table" then + if not str or str=="" then + return nil + elseif type(str)=="table" then if existing then for k,v in next,str do existing[k]=v @@ -6263,8 +6785,6 @@ function parsers.settings_to_hash_strict(str,existing) hash=existing or {} lpegmatch(pattern_c_s,str) return next(hash) and hash - else - return nil end end local separator=comma*space^0 @@ -6272,27 +6792,46 @@ local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comm local pattern=spaces*Ct(value*(separator*value)^0) patterns.settings_to_array=pattern function parsers.settings_to_array(str,strict) - if type(str)=="table" then - return str - elseif not str or str=="" then + if not str or str=="" then return {} + elseif type(str)=="table" then + return str elseif strict then - if find(str,"{") then + if find(str,"{",1,true) then return lpegmatch(pattern,str) else return { str } end - elseif find(str,",") then + elseif find(str,",",1,true) then return lpegmatch(pattern,str) else return { str } end end -local separator=space^0*comma*space^0 -local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(comma+P(-1)))))^0) -local withvalue=Carg(1)*value/function(f,s) return f(s) end -local pattern_a=spaces*Ct(value*(separator*value)^0) -local pattern_b=spaces*withvalue*(separator*withvalue)^0 +local cache_a={} +local cache_b={} +function parsers.groupedsplitat(symbol,withaction) + if not symbol then + symbol="," + end + local pattern=(withaction and cache_b or cache_a)[symbol] + if not pattern then + local symbols=S(symbol) + local separator=space^0*symbols*space^0 + local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(symbols+P(-1)))))^0) + if withaction then + local withvalue=Carg(1)*value/function(f,s) return f(s) end + pattern=spaces*withvalue*(separator*withvalue)^0 + cache_b[symbol]=pattern + else + pattern=spaces*Ct(value*(separator*value)^0) + cache_a[symbol]=pattern + end + end + return pattern +end +local pattern_a=parsers.groupedsplitat(",",false) +local pattern_b=parsers.groupedsplitat(",",true) function parsers.stripped_settings_to_array(str) if not str or str=="" then return {} @@ -6317,8 +6856,8 @@ function parsers.add_settings_to_array(t,str) end function parsers.hash_to_string(h,separator,yes,no,strict,omit) if h then - local t,tn,s={},0,table.sortedkeys(h) - omit=omit and table.tohash(omit) + local t,tn,s={},0,sortedkeys(h) + omit=omit and tohash(omit) for i=1,#s do local key=s[i] if not omit or not omit[key] then @@ -6354,12 +6893,9 @@ function parsers.array_to_string(a,separator) return "" end end -function parsers.settings_to_set(str,t) - t=t or {} - for s in gmatch(str,"[^, ]+") do - t[s]=true - end - return t +local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) +function utilities.parsers.settings_to_set(str,t) + return str and lpegmatch(pattern,str) or {} end function parsers.simple_hash_to_string(h,separator) local t,tn={},0 @@ -6371,12 +6907,16 @@ function parsers.simple_hash_to_string(h,separator) end return concat(t,separator or ",") end -local str=C((1-whitespace-equal)^1) +local str=Cs(lpegpatterns.unquoted)+C((1-whitespace-equal)^1) local setting=Cf(Carg(1)*(whitespace^0*Cg(str*whitespace^0*(equal*whitespace^0*str+Cc(""))))^1,rawset) local splitter=setting^1 function utilities.parsers.options_to_hash(str,target) return str and lpegmatch(splitter,str,1,target or {}) or {} end +local splitter=lpeg.tsplitat(" ") +function utilities.parsers.options_to_array(str) + return str and lpegmatch(splitter,str) or {} +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) local pattern_a=spaces*Ct(value*(separator*value)^0) local function repeater(n,str) @@ -6463,7 +7003,7 @@ function parsers.keq_to_hash(str) end local defaultspecification={ separator=",",quote='"' } function parsers.csvsplitter(specification) - specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification local separator=specification.separator local quotechar=specification.quote local separator=S(separator~="" and separator or ",") @@ -6487,7 +7027,7 @@ function parsers.csvsplitter(specification) end end function parsers.rfc4180splitter(specification) - specification=specification and table.setmetatableindex(specification,defaultspecification) or defaultspecification + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification local separator=specification.separator local quotechar=P(specification.quote) local dquotechar=quotechar*quotechar @@ -6498,7 +7038,7 @@ function parsers.rfc4180splitter(specification) local field=escaped+non_escaped+Cc("") local record=Ct(field*(separator*field)^1) local headerline=record*Cp() - local wholeblob=Ct((newline^-1*record)^0) + local wholeblob=Ct((newline^(specification.strict and -1 or 1)*record)^0) return function(data,getheader) if getheader then local header,position=lpegmatch(headerline,data) @@ -6535,20 +7075,20 @@ function parsers.stepper(str,n,action) lpegmatch(stepper,str,1,n,action or print) end end -local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) -local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+P(1))^0) +local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) +local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) patterns.unittotex=pattern function parsers.unittotex(str,textmode) return lpegmatch(textmode and pattern_text or pattern_math,str) end -local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+P(1))^0) +local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+anything)^0) function parsers.unittoxml(str) return lpegmatch(pattern,str) end local cache={} -local spaces=lpeg.patterns.space^0 +local spaces=lpegpatterns.space^0 local dummy=function() end -table.setmetatableindex(cache,function(t,k) +setmetatableindex(cache,function(t,k) local separator=P(k) local value=(1-separator)^0 local pattern=spaces*C(value)*separator^0*Cp() @@ -6613,6 +7153,18 @@ function utilities.parsers.runtime(time) local seconds=mod(time,60) return days,hours,minutes,seconds end +local spacing=whitespace^0 +local apply=P("->") +local method=C((1-apply)^1) +local token=lbrace*C((1-rbrace)^1)*rbrace+C(anything^1) +local pattern=spacing*(method*spacing*apply+Carg(1))*spacing*token +function utilities.parsers.splitmethod(str,default) + if str then + return lpegmatch(pattern,str,1,default or false) + else + return default or false,"" + end +end end -- of closure @@ -6702,7 +7254,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12365, stripped down to: 8799 +-- original size: 12482, stripped down to: 8864 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -6730,7 +7282,7 @@ function setters.initialize(filename,name,values) local data=setter.data if data then for key,newvalue in next,values do - local newvalue=is_boolean(newvalue,newvalue) + local newvalue=is_boolean(newvalue,newvalue,true) local functions=data[key] if functions then local oldvalue=functions.value @@ -6784,7 +7336,7 @@ local function set(t,what,newvalue) elseif not value then value=false else - value=is_boolean(value,value) + value=is_boolean(value,value,true) end w=topattern(w,true,true) for name,functions in next,data do @@ -6923,6 +7475,7 @@ function setters.new(name) report=function(...) setters.report (setter,...) end, enable=function(...) enable (setter,...) end, disable=function(...) disable (setter,...) end, + reset=function(...) reset (setter,...) end, register=function(...) register(setter,...) end, list=function(...) list (setter,...) end, show=function(...) show (setter,...) end, @@ -7014,7 +7567,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 25391, stripped down to: 16561 +-- original size: 29359, stripped down to: 20483 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7023,15 +7576,18 @@ if not modules then modules={} end modules ['trac-log']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } +local next,type,select,print=next,type,select,print local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write local format,gmatch,find=string.format,string.gmatch,string.find local concat,insert,remove=table.concat,table.insert,table.remove local topattern=string.topattern -local next,type,select=next,type,select local utfchar=utf.char +local datetime=os.date +local openfile=io.open local setmetatableindex=table.setmetatableindex local formatters=string.formatters local texgetcount=tex and tex.getcount +local variant="default" logs=logs or {} local logs=logs local moreinfo=[[ @@ -7041,32 +7597,122 @@ maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context webpage : http://www.pragma-ade.nl / http://tex.aanhet.net wiki : http://contextgarden.net ]] -utilities.strings.formatters.add ( +formatters.add ( formatters,"unichr", [["U+" .. format("%%05X",%s) .. " (" .. utfchar(%s) .. ")"]] ) -utilities.strings.formatters.add ( +formatters.add ( formatters,"chruni", [[utfchar(%s) .. " (U+" .. format("%%05X",%s) .. ")"]] ) local function ignore() end setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations -local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters +local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then - local valueiskey={ __index=function(t,k) t[k]=k return k end } - local target="term and log" + local function useluawrites() + local texio_write_nl=texio.write_nl + local texio_write=texio.write + local io_write=io.write + write_nl=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write_nl("log",...) + texio_write_nl("term","") + io_write(...) + elseif target=="log" then + texio_write_nl("log",...) + elseif target=="term" then + texio_write_nl("term","") + io_write(...) + elseif target~="none" then + texio_write_nl("log",target,...) + texio_write_nl("term","") + io_write(target,...) + end + end + write=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write("log",...) + io_write(...) + elseif target=="log" then + texio_write("log",...) + elseif target=="term" then + io_write(...) + elseif target~="none" then + texio_write("log",target,...) + io_write(target,...) + end + end + texio.write=write + texio.write_nl=write_nl + useluawrites=ignore + end + local whereto="both" + local target=nil + local targets=nil + local formats=table.setmetatableindex("self") + local translations=table.setmetatableindex("self") + local report_yes,subreport_yes,direct_yes,subdirect_yes,status_yes + local report_nop,subreport_nop,direct_nop,subdirect_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s > %s\n"], + report_nop=formatters["%-15s >\n"], + direct_yes=formatters["%-15s > %s"], + direct_nop=formatters["%-15s >"], + subreport_yes=formatters["%-15s > %s > %s\n"], + subreport_nop=formatters["%-15s > %s >\n"], + subdirect_yes=formatters["%-15s > %s > %s"], + subdirect_nop=formatters["%-15s > %s >"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;33m%-15s [0;1m>[0m %s\n"], + report_nop=formatters["[0;33m%-15s [0;1m>[0m\n"], + direct_yes=formatters["[0;33m%-15s [0;1m>[0m %s"], + direct_nop=formatters["[0;33m%-15s [0;1m>[0m"], + subreport_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s\n"], + subreport_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m\n"], + subdirect_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s"], + subdirect_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m"], + status_yes=formatters["[0;33m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;33m%-15s [0;1m:[0m\n"], + }, + targets={ + logfile="none", + log="none", + file="none", + console="term", + terminal="term", + both="term", + }, + } + } logs.flush=io.flush - local formats={} setmetatable(formats,valueiskey) - local translations={} setmetatable(translations,valueiskey) writer=function(...) write_nl(target,...) end newline=function() write_nl(target,"\n") end - local report_yes=formatters["%-15s > %s\n"] - local report_nop=formatters["%-15s >\n"] report=function(a,b,c,...) if c then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) @@ -7078,8 +7724,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local direct_yes=formatters["%-15s > %s"] - local direct_nop=formatters["%-15s >"] direct=function(a,b,c,...) if c then return direct_yes(translations[a],formatters[formats[b]](c,...)) @@ -7091,8 +7735,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local subreport_yes=formatters["%-15s > %s > %s\n"] - local subreport_nop=formatters["%-15s > %s >\n"] subreport=function(a,s,b,c,...) if c then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) @@ -7104,8 +7746,6 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local subdirect_yes=formatters["%-15s > %s > %s"] - local subdirect_nop=formatters["%-15s > %s >"] subdirect=function(a,s,b,c,...) if c then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) @@ -7117,8 +7757,6 @@ if tex and (tex.jobname or tex.formatname) then return "" end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) @@ -7130,16 +7768,13 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end end - local targets={ - logfile="log", - log="log", - file="log", - console="term", - terminal="term", - both="term and log", - } - settarget=function(whereto) - target=targets[whereto or "both"] or targets.both + settarget=function(askedwhereto) + whereto=askedwhereto or whereto or "both" + target=targets[whereto] + if not target then + whereto="both" + target=targets[whereto] + end if target=="term" or target=="term and log" then logs.flush=io.flush else @@ -7168,21 +7803,74 @@ if tex and (tex.jobname or tex.formatname) then writeline(target,f(...)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - direct_yes=f.direct_yes or direct_yes - direct_nop=f.direct_nop or direct_nop - subdirect_yes=f.subdirect_yes or subdirect_yes - subdirect_nop=f.subdirect_nop or subdirect_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local t=nil + local f=nil + local d=variants.default + if not specification then + elseif type(specification)=="table" then + t=specification.targets + f=specification.formats or specification + else + local v=variants[specification] + if v then + t=v.targets + f=v.formats + variant=specification + end + end + targets=t or d.targets + target=targets[whereto] or target + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + direct_yes=f.direct_yes + direct_nop=f.direct_nop + subdirect_yes=f.subdirect_yes + subdirect_nop=f.subdirect_nop + status_yes=f.status_yes + status_nop=f.status_nop + if variant=="ansi" then + useluawrites() + end + settarget(whereto) + end + setformatters(variant) setlogfile=ignore settimedlog=ignore else + local report_yes,subreport_yes,status_yes + local report_nop,subreport_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s | %s"], + report_nop=formatters["%-15s |"], + subreport_yes=formatters["%-15s | %s | %s"], + subreport_nop=formatters["%-15s | %s |"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;32m%-15s [0;1m|[0m %s"], + report_nop=formatters["[0;32m%-15s [0;1m|[0m"], + subreport_yes=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m %s"], + subreport_nop=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m"], + status_yes=formatters["[0;32m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;32m%-15s [0;1m:[0m\n"], + }, + }, + } logs.flush=ignore writer=function(s) write_nl(s) @@ -7190,8 +7878,6 @@ else newline=function() write_nl("\n") end - local report_yes=formatters["%-15s | %s"] - local report_nop=formatters["%-15s |"] report=function(a,b,c,...) if c then write_nl(report_yes(a,formatters[b](c,...))) @@ -7203,8 +7889,6 @@ else write_nl("") end end - local subreport_yes=formatters["%-15s | %s | %s"] - local subreport_nop=formatters["%-15s | %s |"] subreport=function(a,sub,b,c,...) if c then write_nl(subreport_yes(a,sub,formatters[b](c,...))) @@ -7216,8 +7900,6 @@ else write_nl("") end end - local status_yes=formatters["%-15s : %s\n"] - local status_nop=formatters["%-15s :\n"] status=function(a,b,c,...) if c then write_nl(status_yes(a,formatters[b](c,...))) @@ -7242,14 +7924,34 @@ else writeline(f(s)) end end - setformatters=function(f) - report_yes=f.report_yes or report_yes - report_nop=f.report_nop or report_nop - subreport_yes=f.subreport_yes or subreport_yes - subreport_nop=f.subreport_nop or subreport_nop - status_yes=f.status_yes or status_yes - status_nop=f.status_nop or status_nop - end + setformatters=function(specification) + local f=nil + local d=variants.default + if specification then + if type(specification)=="table" then + f=specification.formats or specification + else + local v=variants[specification] + if v then + f=v.formats + end + end + end + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + status_yes=f.status_yes + status_nop=f.status_nop + end + setformatters(variant) setlogfile=function(name,keepopen) if name and name~="" then local localtime=os.localtime @@ -7368,9 +8070,10 @@ local function setblocked(category,value) v.state=value end else - states=utilities.parsers.settings_to_hash(category) + states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) for c,_ in next,states do - if data[c] then + local v=data[c] + if v then v.state=value else c=topattern(c,true,true) @@ -7501,13 +8204,13 @@ end local simple=logs.reporter("comment") logs.simple=simple logs.simpleline=simple -function logs.setprogram () end -function logs.extendbanner() end -function logs.reportlines () end -function logs.reportbanner() end -function logs.reportline () end -function logs.simplelines () end -function logs.help () end +logs.setprogram=ignore +logs.extendbanner=ignore +logs.reportlines=ignore +logs.reportbanner=ignore +logs.reportline=ignore +logs.simplelines=ignore +logs.help=ignore local Carg,C,lpegmatch=lpeg.Carg,lpeg.C,lpeg.match local p_newline=lpeg.patterns.newline local linewise=( @@ -7584,10 +8287,11 @@ function logs.application(t) end return t end -function logs.system(whereto,process,jobname,category,...) - local message=formatters["%s %s => %s => %s => %s\r"](os.date("%d/%m/%y %H:%m:%S"),process,jobname,category,format(...)) +local f_syslog=formatters["%s %s => %s => %s => %s\r"] +function logs.system(whereto,process,jobname,category,fmt,arg,...) + local message=f_syslog(datetime("%d/%m/%y %H:%m:%S"),process,jobname,category,arg==nil and fmt or format(fmt,arg,...)) for i=1,10 do - local f=io.open(whereto,"a") + local f=openfile(whereto,"a") if f then f:write(message) f:close() @@ -7649,7 +8353,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6501, stripped down to: 5156 +-- original size: 6704, stripped down to: 5343 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -7659,7 +8363,7 @@ if not modules then modules={} end modules ['trac-inf']={ license="see context related readme files" } local type,tonumber,select=type,tonumber,select -local format,lower=string.format,string.lower +local format,lower,find=string.format,string.lower,string.find local concat=table.concat local clock=os.gettimeofday or os.clock local setmetatableindex=table.setmetatableindex @@ -7750,7 +8454,8 @@ function statistics.show() if statistics.enable then local register=statistics.register register("used platform",function() - return format("%s, type: %s, binary subtree: %s",os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") + return format("%s, type: %s, binary subtree: %s", + os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") end) register("luatex banner",function() return lower(status.banner) @@ -7763,14 +8468,23 @@ function statistics.show() return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) if jit then - local status={ jit.status() } - if status[1] then - register("luajit status",function() - return concat(status," ",2) - end) - end - end - register("current memory usage",statistics.memused) + local jitstatus={ jit.status() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end + end + register("lua properties",function() + local list=status.list() + local hashchar=tonumber(list.luatex_hashchars) + local mask=lua.mask or "ascii" + return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", + jit and "luajit" or "lua", + statistics.memused(), + list.luatex_hashtype or "default", + hashchar and 2^hashchar or "unknown", + mask, + mask=="utf" and "τεχ" or "tex") + end) register("runtime",statistics.runtime) logs.newline() for i=1,#statusinfo do @@ -7812,15 +8526,6 @@ function statistics.tracefunction(base,tag,...) statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end) end end -commands=commands or {} -function commands.resettimer(name) - resettiming(name or "whatever") - starttiming(name or "whatever") -end -function commands.elapsedtime(name) - stoptiming(name or "whatever") - context(elapsedtime(name or "whatever")) -end end -- of closure @@ -7829,7 +8534,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5773, stripped down to: 3453 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -7846,14 +8551,16 @@ local namespaces=namespaces local registered={} local function report_index(k,name) if trace_namespaces then - report_system("reference to %a in protected namespace %a: %s",k,name,debug.traceback()) + report_system("reference to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) else report_system("reference to %a in protected namespace %a",k,name) end end local function report_newindex(k,name) if trace_namespaces then - report_system("assignment to %a in protected namespace %a: %s",k,name,debug.traceback()) + report_system("assignment to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) else report_system("assignment to %a in protected namespace %a",k,name) end @@ -8104,7 +8811,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3708, stripped down to: 2568 +-- original size: 3898, stripped down to: 2644 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8184,20 +8891,22 @@ end function debugger.disable() debug.sethook() end -function traceback() - local level=1 +local function showtraceback(rep) + local level=2 + local reporter=rep or report while true do - local info=debug.getinfo(level,"Sl") + local info=getinfo(level,"Sl") if not info then break elseif info.what=="C" then - print(format("%3i : C function",level)) + reporter("%2i : %s",level-1,"C function") else - print(format("%3i : [%s]:%d",level,info.short_src,info.currentline)) + reporter("%2i : %s : %s",level-1,info.short_src,info.currentline) end level=level+1 end end +debugger.showtraceback=showtraceback end -- of closure @@ -8383,7 +9092,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 6251, stripped down to: 3488 +-- original size: 7100, stripped down to: 3978 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -8425,7 +9134,7 @@ local sqlescape=lpeg.replacer { { "\r\n","\\n" }, { "\r","\\n" }, } -local sqlquoted=lpeg.Cs(lpeg.Cc("'")*sqlescape*lpeg.Cc("'")) +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) lpegpatterns.sqlescape=sqlescape lpegpatterns.sqlquoted=sqlquoted local luaescape=lpegpatterns.luaescape @@ -8448,12 +9157,24 @@ local quotedescapers={ local luaescaper=escapers.lua local quotedluaescaper=quotedescapers.lua local function replacekeyunquoted(s,t,how,recurse) - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end end local function replacekeyquoted(s,t,how,recurse) - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" end local single=P("%") local double=P("%%") @@ -8468,11 +9189,16 @@ local nolquoted=lquoted/'' local norquoted=rquoted/'' local nolquotedq=lquotedq/'' local norquotedq=rquotedq/'' -local key=nosingle*((C((1-nosingle )^1)*Carg(1)*Carg(2)*Carg(3))/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq)^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*Carg(1)*Carg(2)*Carg(3))/replacekeyunquoted)*norquoted +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional local any=P(1) - replacer=Cs((unquoted+quoted+escape+key+any)^0) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) local function replace(str,mapping,how,recurse) if mapping and str then return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str @@ -8511,7 +9237,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8807, stripped down to: 5085 +-- original size: 8022, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -8522,7 +9248,7 @@ if not modules then modules={} end modules ['util-env']={ } local allocate,mark=utilities.storage.allocate,utilities.storage.mark local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find -local unquoted,quoted=string.unquoted,string.quoted +local unquoted,quoted,optionalquoted=string.unquoted,string.quoted,string.optionalquoted local concat,insert,remove=table.concat,table.insert,table.remove environment=environment or {} local environment=environment @@ -8635,24 +9361,14 @@ function environment.splitarguments(separator) return before,after end function environment.reconstructcommandline(arg,noquote) + local resolveprefix=resolvers.resolve arg=arg or environment.originalarguments if noquote and #arg==1 then - local a=arg[1] - a=resolvers.resolve(a) - a=unquoted(a) - return a + return unquoted(resolveprefix and resolveprefix(arg[1]) or arg[1]) elseif #arg>0 then local result={} for i=1,#arg do - local a=arg[i] - a=resolvers.resolve(a) - a=unquoted(a) - a=gsub(a,'"','\\"') - if find(a," ") then - result[#result+1]=quoted(a) - else - result[#result+1]=a - end + result[i]=optionalquoted(resolveprefix and resolveprefix(arg[i]) or resolveprefix) end return concat(result," ") else @@ -8708,7 +9424,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 5930, stripped down to: 4235 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -8786,15 +9502,13 @@ function environment.luafilechunk(filename,silent) filename=file.replacesuffix(filename,"lua") local fullname=environment.luafile(filename) if fullname and fullname~="" then - local data=luautilities.loadedluacode(fullname,strippable,filename) - if trace_locating then + local data=luautilities.loadedluacode(fullname,strippable,filename) + if not silent then report_lua("loading file %a %s",fullname,not data and "failed" or "succeeded") - elseif not silent then - texio.write("<",data and "+ " or "- ",fullname,">") end return data else - if trace_locating then + if not silent then report_lua("unknown file %a",filename) end return nil @@ -8863,7 +9577,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 42447, stripped down to: 26589 +-- original size: 45683, stripped down to: 27866 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -8878,10 +9592,10 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber=type,next,setmetatable,getmetatable,tonumber +local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local utfchar=utf.char -local lpegmatch=lpeg.match +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters xml.xmlns=xml.xmlns or {} @@ -8976,8 +9690,10 @@ local function add_end(spacing,namespace,tag) top=stack[#stack] if #stack<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) elseif toclose.tg~=tag then errorstr=formatters["unable to close %s with %s %s"](toclose.tg,tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) end dt=top.dt dt[#dt+1]=toclose @@ -8986,10 +9702,29 @@ local function add_end(spacing,namespace,tag) end end local function add_text(text) + local n=#dt if cleanup and #text>0 then - dt[#dt+1]=cleanup(text) + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..cleanup(text) + else + dt[n+1]=cleanup(text) + end + else + dt[1]=cleanup(text) + end else - dt[#dt+1]=text + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..text + else + dt[n+1]=text + end + else + dt[1]=text + end end end local function add_special(what,spacing,text) @@ -9021,8 +9756,10 @@ local function attribute_specification_error(str) end return str end +local badentity="&error;" +local badentity="&" xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and "&error;" or formatters["&%s;"](str) end, + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, } @@ -9043,9 +9780,10 @@ local function fromdec(s) return formatters["d:%s"](s),true end end -local rest=(1-P(";"))^0 -local many=P(1)^0 -local parsedentity=P("&")*(P("#x")*(rest/fromhex)+P("#")*(rest/fromdec))*P(";")*P(-1)+(P("#x")*(many/fromhex)+P("#")*(many/fromdec)) +local p_rest=(1-P(";"))^0 +local p_many=P(1)^0 +local p_char=lpegpatterns.utf8character +local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) local predefined_unified={ [38]="&", [42]=""", @@ -9071,7 +9809,9 @@ local privates_u={ local privates_p={} local privates_n={ } -local escaped=utf.remapper(privates_u) +local escaped=utf.remapper(privates_u,"dynamic") +local unprivatized=utf.remapper(privates_p,"dynamic") +xml.unprivatized=unprivatized local function unescaped(s) local p=privates_n[s] if not p then @@ -9084,9 +9824,7 @@ local function unescaped(s) end return p end -local unprivatized=utf.remapper(privates_p) xml.privatetoken=unescaped -xml.unprivatized=unprivatized xml.privatecodes=privates_n local function handle_hex_entity(str) local h=hcache[str] @@ -9181,7 +9919,7 @@ local function handle_any_entity(str) report_xml("keeping entity &%s;",str) end if str=="" then - a="&error;" + a=badentity else a="&"..str..";" end @@ -9209,7 +9947,7 @@ local function handle_any_entity(str) if trace_entities then report_xml("invalid entity &%s;",str) end - a="&error;" + a=badentity acache[str]=a else if trace_entities then @@ -9222,8 +9960,14 @@ local function handle_any_entity(str) return a end end -local function handle_end_entity(chr) - report_xml("error in entity, %a found instead of %a",chr,";") +local function handle_end_entity(str) + report_xml("error in entity, %a found without ending %a",str,";") + return str +end +local function handle_crap_error(chr) + report_xml("error in parsing, unexpected %a found ",chr) + add_text(chr) + return chr end local space=S(' \r\n\t') local open=P('<') @@ -9239,15 +9983,15 @@ local valid=R('az','AZ','09')+S('_-.') local name_yes=C(valid^1)*colon*C(valid^1) local name_nop=C(P(true))*C(valid^1) local name=name_yes+name_nop -local utfbom=lpeg.patterns.utfbom +local utfbom=lpegpatterns.utfbom local spacing=C(space^0) -local anyentitycontent=(1-open-semicolon-space-close)^0 +local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 local hexentitycontent=R("AF","af","09")^0 local decentitycontent=R("09")^0 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) )+(anyentitycontent/handle_any_entity) -local entity=ampersand/""*parsedentity*((semicolon/"")+#(P(1)/handle_end_entity)) +local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) local somespace=space^1 @@ -9298,16 +10042,20 @@ local instruction=(spacing*begininstruction*someinstruction*endinstruction)/func local comment=(spacing*begincomment*somecomment*endcomment )/function(...) add_special("@cm@",...) end local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special("@cd@",...) end local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end +local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand +local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata +local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 local grammar_parsed_text=P { "preamble", preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, parent=beginelement*V("children")^0*endelement, - children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction, + children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } local grammar_unparsed_text=P { "preamble", preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, parent=beginelement*V("children")^0*endelement, - children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction, + children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) settings=settings or {} @@ -9341,7 +10089,6 @@ local function _xmlconvert_(data,settings) errorstr="empty xml file" elseif utfize or resolve then if lpegmatch(grammar_parsed_text,data) then - errorstr="" else errorstr="invalid xml file - parsed text" end @@ -9357,6 +10104,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } +setmetatable(result,mt) +setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -9389,8 +10138,11 @@ local function _xmlconvert_(data,settings) end if errorstr and errorstr~="" then result.error=true + else + errorstr=nil end result.statistics={ + errormessage=errorstr, entities={ decimals=dcache, hexadecimals=hcache, @@ -9404,7 +10156,7 @@ local function _xmlconvert_(data,settings) reported_attribute_errors,mt,errorhandler=nil,nil,nil return result end -function xmlconvert(data,settings) +local function xmlconvert(data,settings) local ok,result=pcall(function() return _xmlconvert_(data,settings) end) if ok then return result @@ -9496,14 +10248,17 @@ function xml.checkbom(root) insert(dt,2,"\n" ) end end -local function verbose_element(e,handlers) +local f_attribute=formatters['%s=%q'] +local function verbose_element(e,handlers,escape) local handle=handlers.handle local serialize=handlers.serialize local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn local ats=eat and next(eat) and {} if ats then + local n=0 for k,v in next,eat do - ats[#ats+1]=formatters['%s=%q'](k,escaped(v)) + n=n+1 + ats[n]=f_attribute(k,escaped(v)) end end if ern and trace_entities and ern~=ens then @@ -9588,23 +10343,25 @@ local function verbose_document(e,handlers) end end local function serialize(e,handlers,...) - local initialize=handlers.initialize - local finalize=handlers.finalize - local functions=handlers.functions - if initialize then - local state=initialize(...) - if not state==true then - return state + if e then + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() end - end - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) - end - if finalize then - return finalize() end end local function xserialize(e,handlers) @@ -9845,7 +10602,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48956, stripped down to: 30516 +-- original size: 48229, stripped down to: 30684 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10230,8 +10987,8 @@ local lp_builtin=P ( P("ns")/"ll.ns" )*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") -lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" -lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" local lp_fastpos=lp_fastpos_n+lp_fastpos_p local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" @@ -10410,7 +11167,7 @@ local function nodesettostring(set,nodetest) if not ns or ns=="" then ns="*" end if not tg or tg=="" then tg="*" end tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) - t[i]=(directive and tg) or format("not(%s)",tg) + t[#t+1]=(directive and tg) or format("not(%s)",tg) end if nodetest==false then return format("not(%s)",concat(t,"|")) @@ -10676,7 +11433,6 @@ expressions.print=function(...) print(...) return true end -expressions.contains=find expressions.find=find expressions.upper=upper expressions.lower=lower @@ -10698,6 +11454,9 @@ function expressions.contains(str,pattern) end return false end +function xml.expressions.idstring(str) + return type(str)=="string" and gsub(str,"^#","") or "" +end local function traverse(root,pattern,handle) local collected=applylpath(root,pattern) if collected then @@ -10826,8 +11585,13 @@ function xml.elements(root,pattern,reverse) local collected=applylpath(root,pattern) if not collected then return dummy - elseif reverse then - local c=#collected+1 + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 return function() if c>1 then c=c-1 @@ -10837,7 +11601,7 @@ function xml.elements(root,pattern,reverse) end end else - local n,c=#collected,0 + local c=0 return function() if c<n then c=c+1 @@ -10852,8 +11616,13 @@ function xml.collected(root,pattern,reverse) local collected=applylpath(root,pattern) if not collected then return dummy - elseif reverse then - local c=#collected+1 + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 return function() if c>1 then c=c-1 @@ -10861,7 +11630,7 @@ function xml.collected(root,pattern,reverse) end end else - local n,c=#collected,0 + local c=0 return function() if c<n then c=c+1 @@ -10876,7 +11645,7 @@ function xml.inspect(collection,pattern) report_lpath("pattern: %s\n\n%s\n",pattern,xml.tostring(e)) end end -local function split(e) +local function split(e) local dt=e.dt if dt then for i=1,#dt do @@ -10975,7 +11744,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 23804, stripped down to: 16817 +-- original size: 28786, stripped down to: 20578 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -10985,16 +11754,19 @@ if not modules then modules={} end modules ['lxml-aux']={ license="see context related readme files" } local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local trace_inclusions=false trackers.register("lxml.inclusions",function(v) trace_inclusions=v end) local report_xml=logs.reporter("xml") local xml=xml -local xmlconvert,xmlcopy,xmlname=xml.convert,xml.copy,xml.name +local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath local xmlfilter=xml.filter -local type,setmetatable,getmetatable=type,setmetatable,getmetatable +local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip local utfbyte=utf.byte +local lpegmatch=lpeg.match +local striplinepatterns=utilities.strings.striplinepatterns local function report(what,pattern,c,e) report_xml("%s element %a, root %a, position %a, index %a, pattern %a",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) end @@ -11049,13 +11821,15 @@ end function xml.each(root,pattern,handle,reverse) local collected=xmlapplylpath(root,pattern) if collected then - if reverse then - for c=#collected,1,-1 do - handle(collected[c]) - end - else - for c=1,#collected do - handle(collected[c]) + if handle then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end end end return collected @@ -11111,6 +11885,7 @@ local function redo_ni(d) end end end +xml.reindex=redo_ni local function xmltoelement(whatever,root) if not whatever then return nil @@ -11162,8 +11937,16 @@ function xml.delete(root,pattern) report('deleting',pattern,c,e) end local d=p.dt - remove(d,e.ni) - redo_ni(d) + local ni=e.ni + if ni<=#d then + if false then + p.dt[ni]="" + else + remove(d,ni) + redo_ni(d) + end + else + end end end end @@ -11283,28 +12066,40 @@ xml.insertafter=insert_element xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end xml.injectafter=inject_element xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end -local function include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata,level) pattern=pattern or 'include' loaddata=loaddata or io.loaddata local collected=xmlapplylpath(xmldata,pattern) if collected then + if not level then + level=1 + end for c=1,#collected do local ek=collected[c] local name=nil local ekdt=ek.dt local ekat=ek.at - local epdt=ek.__p__.dt + local ekrt=ek.__p__ + local epdt=ekrt.dt if not attribute or attribute=="" then name=(type(ekdt)=="table" and ekdt[1]) or ekdt end if not name then for a in gmatch(attribute or "href","([^|]+)") do name=ekat[a] - if name then break end + if name then + break + end + end + end + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") end end - local data=(name and name~="" and loaddata(name)) or "" - if data=="" then + if not data or data=="" then epdt[ek.ni]="" elseif ekat["parse"]=="text" then epdt[ek.ni]=xml.escaped(data) @@ -11314,70 +12109,127 @@ local function include(xmldata,pattern,attribute,recursive,loaddata) epdt[ek.ni]="" else if recursive then - include(xi,pattern,attribute,recursive,loaddata) + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local inclusions=xmldata.settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + else + xmldata.settings.inclusions={ name } + end + if child.er then + local badinclusions=xmldata.settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + xmldata.settings.badinclusions={ name } + end end - epdt[ek.ni]=xml.body(xi) end end end end end xml.include=include +function xml.inclusion(e,default) + while e do + local f=e.__f__ + if f then + return f + else + e=e.__p__ + end + end + return default +end +local function getinclusions(key,e,sorted) + while e do + local settings=e.settings + if settings then + local inclusions=settings[key] + if inclusions then + inclusions=table.unique(inclusions) + if sorted then + table.sort(inclusions) + end + return inclusions + else + e=e.__p__ + end + else + e=e.__p__ + end + end +end +function xml.inclusions(e,sorted) + return getinclusions("inclusions",e,sorted) +end +function xml.badinclusions(e,sorted) + return getinclusions("badinclusions",e,sorted) +end +local b_collapser=lpeg.patterns.b_collapser +local m_collapser=lpeg.patterns.m_collapser +local e_collapser=lpeg.patterns.e_collapser +local b_stripper=lpeg.patterns.b_stripper +local m_stripper=lpeg.patterns.m_stripper +local e_stripper=lpeg.patterns.e_stripper +local lpegmatch=lpeg.match local function stripelement(e,nolines,anywhere) local edt=e.dt if edt then - if anywhere then - local t,n={},0 - for e=1,#edt do + local n=#edt + if n==0 then + return e + elseif anywhere then + local t={} + local m=0 + for e=1,n do local str=edt[e] if type(str)~="string" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str elseif str~="" then if nolines then - str=gsub(str,"%s+"," ") + str=lpegmatch((n==1 and b_collapser) or (n==m and e_collapser) or m_collapser,str) + else + str=lpegmatch((n==1 and b_stripper) or (n==m and e_stripper) or m_stripper,str) end - str=gsub(str,"^%s*(.-)%s*$","%1") if str~="" then - n=n+1 - t[n]=str + m=m+1 + t[m]=str end end end e.dt=t else - if #edt>0 then - local str=edt[1] - if type(str)~="string" then - elseif str=="" then + local str=edt[1] + if type(str)=="string" then + if str~="" then + str=lpegmatch(nolines and b_collapser or b_stripper,str) + end + if str=="" then remove(edt,1) + n=n-1 else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"^%s+","") - if str=="" then - remove(edt,1) - else - edt[1]=str - end + edt[1]=str end end - local nedt=#edt - if nedt>0 then - local str=edt[nedt] - if type(str)~="string" then - elseif str=="" then - remove(edt) - else - if nolines then - str=gsub(str,"%s+"," ") - end - str=gsub(str,"%s+$","") + if n>0 then + str=edt[n] + if type(str)=="string" then if str=="" then remove(edt) else - edt[nedt]=str + str=lpegmatch(nolines and e_collapser or e_stripper,str) + if str=="" then + remove(edt) + else + edt[n]=str + end end end end @@ -11563,8 +12415,8 @@ function xml.finalizers.xml.cdata(collected) end return "" end -function xml.insertcomment(e,str,n) - table.insert(e.dt,n or 1,{ +function xml.insertcomment(e,str,n) + insert(e.dt,n or 1,{ tg="@cm@", ns="", special=true, @@ -11572,7 +12424,25 @@ function xml.insertcomment(e,str,n) dt={ str }, }) end -function xml.setcdata(e,str) +function xml.insertcdata(e,str,n) + insert(e.dt,n or 1,{ + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcomment(e,str,n) + e.dt={ { + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.setcdata(e,str) e.dt={ { tg="@cd@", ns="", @@ -11642,7 +12512,7 @@ local function recurse(e,action) for i=1,#edt do local str=edt[i] if type(str)~="string" then - recurse(str,action,recursive) + recurse(str,action) elseif str~="" then edt[i]=action(str) end @@ -11660,6 +12530,65 @@ function helpers.recursetext(collected,action,recursive) end end end +local specials={ + ["@rt@"]="root", + ["@pi@"]="instruction", + ["@cm@"]="comment", + ["@dt@"]="declaration", + ["@cd@"]="cdata", +} +local function convert(x,strip,flat) + local ns=x.ns + local tg=x.tg + local at=x.at + local dt=x.dt + local node=flat and { + [0]=(not x.special and (ns~="" and ns..":"..tg or tg)) or nil, + } or { + _namespace=ns~="" and ns or nil, + _tag=not x.special and tg or nil, + _type=specials[tg] or "_element", + } + if at then + for k,v in next,at do + node[k]=v + end + end + local n=0 + for i=1,#dt do + local di=dt[i] + if type(di)=="table" then + if flat and di.special then + else + di=convert(di,strip,flat) + if di then + n=n+1 + node[n]=di + end + end + elseif strip then + di=lpegmatch(strip,di) + if di~="" then + n=n+1 + node[n]=di + end + else + n=n+1 + node[n]=di + end + end + if next(node) then + return node + end +end +function xml.totable(x,strip,flat) + if type(x)=="table" then + if strip then + strip=striplinepatterns[strip] + end + return convert(x,strip,flat) + end +end end -- of closure @@ -12216,7 +13145,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 7898, stripped down to: 5501 +-- original size: 11085, stripped down to: 7662 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -12225,14 +13154,15 @@ if not modules then modules={} end modules ['data-ini']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } +local next,type,getmetatable,rawset=next,type,getmetatable,rawset local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char -local next,type=next,type local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) local report_initialization=logs.reporter("resolvers","initialization") -local ostype,osname,ossetenv,osgetenv=os.type,os.name,os.setenv,os.getenv resolvers=resolvers or {} local resolvers=resolvers texconfig.kpse_init=false @@ -12360,15 +13290,108 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if profiler then +if type(profiler)=="table" and not jit then directives.register("system.profile",function() profiler.start("luatex-profile.log") end) end -if not resolvers.resolve then - function resolvers.resolve (s) return s end - function resolvers.unresolve(s) return s end - function resolvers.repath (s) return s end +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local resolved={} +local abstract={} +local dynamic={} +function resolvers.resetresolve(str) + resolved,abstract={},{} +end +function resolvers.allprefixes(separator) + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" + end + end + return all +end +local function _resolve_(method,target) + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +function resolvers.unresolve(str) + return abstract[str] or str +end +function resolvers.setdynamic(str) + dynamic[str]=true +end +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local prefix=C(R("az")^2)*P(":") +local target=C((1-S(" \"\';,"))^1) +local notarget=(#S(";,")+P(-1))*Cc("") +local p_resolve=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) +local p_simple=prefix*P(-1) +local function resolve(str) + if type(str)=="table" then + local res={} + for i=1,#str do + res[i]=resolve(str[i]) + end + return res + end + local res=resolved[str] + if res then + return res + end + local simple=lpegmatch(p_simple,str) + local action=prefixes[simple] + if action then + local res=action(res) + if not dynamic[simple] then + resolved[simple]=res + abstract[res]=simple + end + return res + end + res=lpegmatch(p_resolve,str) + resolved[str]=res + abstract[res]=str + return res +end +resolvers.resolve=resolve +if type(osuname)=="function" then + for k,v in next,osuname() do + if not prefixes[k] then + prefixes[k]=function() return v end + end + end +end +if ostype=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end + end + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + table.setmetatablenewindex(prefixes,makepattern) + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end end @@ -12378,7 +13401,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 15303, stripped down to: 9716 +-- original size: 17216, stripped down to: 10657 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -12392,12 +13415,16 @@ local concat,sort=table.concat,table.sort local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next +local isdir=lfs.isdir local ostype=os.type -local collapsepath=file.collapsepath +local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_globbing=true trackers.register("resolvers.globbing",function(v) trace_globbing=v end) local report_expansions=logs.reporter("resolvers","expansions") +local report_globbing=logs.reporter("resolvers","globbing") local resolvers=resolvers +local resolveprefix=resolvers.resolve local function f_both(a,b) local t,n={},0 for sb in gmatch(b,"[^,]+") do @@ -12487,35 +13514,27 @@ function resolvers.expandedpathfromlist(pathlist) end return newlist end -local cleanup=lpeg.replacer { - { "!","" }, - { "\\","/" }, -} -function resolvers.cleanpath(str) - local doslashes=(P("\\")/"/"+1)^0 - local donegation=(P("!")/"" )^0 - local homedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") - if homedir=="~" or homedir=="" or not lfs.isdir(homedir) then - if trace_expansions then - report_expansions("no home dir set, ignoring dependent paths") - end - function resolvers.cleanpath(str) - if not str or find(str,"~") then - return "" - else - return lpegmatch(cleanup,str) +local usedhomedir=nil +local donegation=(P("!")/"" )^0 +local doslashes=(P("\\")/"/"+1)^0 +local function expandedhome() + if not usedhomedir then + usedhomedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if usedhomedir=="~" or usedhomedir=="" or not isdir(usedhomedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent path using current path") end - end - else - local dohome=((P("~")+P("$HOME"))/homedir)^0 - local cleanup=Cs(donegation*dohome*doslashes) - function resolvers.cleanpath(str) - return str and lpegmatch(cleanup,str) or "" + usedhomedir="." end end - return resolvers.cleanpath(str) + return usedhomedir end -local expandhome=P("~")/"$HOME" +local dohome=((P("~")+P("$HOME")+P("%HOME%"))/expandedhome)^0 +local cleanup=Cs(donegation*dohome*doslashes) +resolvers.cleanpath=function(str) + return str and lpegmatch(cleanup,str) or "" +end +local expandhome=P("~")/"$HOME" local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" local dostring=(expandhome+1 )^0 @@ -12567,46 +13586,67 @@ function resolvers.splitpath(str) end function resolvers.joinpath(str) if type(str)=='table' then - return file.joinpath(str) + return joinpath(str) else return str end end local attributes,directory=lfs.attributes,lfs.dir local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local lessweird=P(".")^1+lpeg.anywhere(S("~`#$%^&*:;\"\'||<>,?\n\r\t")) local timer={} local scanned={} local nofscans=0 local scancache={} -local function scan(files,spec,path,n,m,r) - local full=(path=="" and spec) or (spec..path..'/') +local fullcache={} +local nofsharedscans=0 +local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) + local full=path=="" and spec or (spec..path..'/') local dirs={} local nofdirs=0 + local pattern=tolerant and lessweird or weird for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then + if not lpegmatch(pattern,name) then + local mode=attributes(full..name,"mode") + if mode=="file" then n=n+1 - local f=files[name] - if f then - if type(f)=='string' then - files[name]={ f,path } + local lower=lower(name) + local paths=files[lower] + if paths then + if onlyone then else - f[#f+1]=path + if type(paths)=="string" then + files[lower]={ paths,path } + else + paths[#paths+1]=path + end + if name~=lower then + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end + end end else - files[name]=path - local lower=lower(name) + files[lower]=path if name~=lower then - files["remap:"..lower]=name - r=r+1 + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end end end - elseif mode=='directory' then + elseif mode=="directory" then m=m+1 nofdirs=nofdirs+1 if path~="" then - dirs[nofdirs]=path..'/'..name + dirs[nofdirs]=path.."/"..name else dirs[nofdirs]=name end @@ -12616,107 +13656,69 @@ local function scan(files,spec,path,n,m,r) if nofdirs>0 then sort(dirs) for i=1,nofdirs do - files,n,m,r=scan(files,spec,dirs[i],n,m,r) + files,remap,n,m,r=scan(files,remap,spec,dirs[i],n,m,r,onlyonce,tolerant) end end scancache[sub(full,1,-2)]=files - return files,n,m,r + return files,remap,n,m,r end -local fullcache={} -function resolvers.scanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) +function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) + local realpath=resolveprefix(path) if usecache then - local files=fullcache[realpath] - if files then + local content=fullcache[realpath] + if content then if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) + report_expansions("using cached scan of path %a, branch %a",path,branch or path) end - return files + nofsharedscans=nofsharedscans+1 + return content end end + statistics.starttiming(timer) if trace_locating then report_expansions("scanning path %a, branch %a",path,branch or path) end - local files,n,m,r=scan({},realpath..'/',"",0,0,0) - files.__path__=path - files.__files__=n - files.__directories__=m - files.__remappings__=r - if trace_locating then - report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) - end - if usecache then - scanned[#scanned+1]=realpath - fullcache[realpath]=files - end - nofscans=nofscans+1 - statistics.stoptiming(timer) - return files -end -local function simplescan(files,spec,path) - local full=(path=="" and spec) or (spec..path..'/') - local dirs={} - local nofdirs=0 - for name in directory(full) do - if not lpegmatch(weird,name) then - local mode=attributes(full..name,'mode') - if mode=='file' then - if not files[name] then - files[name]=path - end - elseif mode=='directory' then - nofdirs=nofdirs+1 - if path~="" then - dirs[nofdirs]=path..'/'..name - else - dirs[nofdirs]=name - end - end - end - end - if nofdirs>0 then - sort(dirs) - for i=1,nofdirs do - files=simplescan(files,spec,dirs[i]) - end - end - return files -end -local simplecache={} -local nofsharedscans=0 -function resolvers.simplescanfiles(path,branch,usecache) - statistics.starttiming(timer) - local realpath=resolvers.resolve(path) - if usecache then - local files=simplecache[realpath] - if not files then - files=scancache[realpath] - if files then - nofsharedscans=nofsharedscans+1 - end + local content + if isdir(realpath) then + local files,remap,n,m,r=scan({},{},realpath..'/',"",0,0,0,onlyonce,tolerant) + content={ + metadata={ + path=path, + files=n, + directories=m, + remappings=r, + }, + files=files, + remap=remap, + } + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) end - if files then - if trace_locating then - report_expansions("using caches scan of path %a, branch %a",path,branch or path) - end - return files + else + content={ + metadata={ + path=path, + files=0, + directories=0, + remappings=0, + }, + files={}, + remap={}, + } + if trace_locating then + report_expansions("invalid path %a",realpath) end end - if trace_locating then - report_expansions("scanning path %a, branch %a",path,branch or path) - end - local files=simplescan({},realpath..'/',"") - if trace_locating then - report_expansions("%s files found",table.count(files)) - end if usecache then scanned[#scanned+1]=realpath - simplecache[realpath]=files + fullcache[realpath]=content end nofscans=nofscans+1 statistics.stoptiming(timer) - return files + return content +end +function resolvers.simplescanfiles(path,branch,usecache) + return resolvers.scanfiles(path,branch,usecache,true,true) end function resolvers.scandata() table.sort(scanned) @@ -12727,6 +13729,52 @@ function resolvers.scandata() paths=scanned, } end +function resolvers.get_from_content(content,path,name) + if not content then + return + end + local files=content.files + if not files then + return + end + local remap=content.remap + if not remap then + return + end + if name then + local used=lower(name) + return path,remap[used] or used + else + local name=path + local used=lower(name) + local path=files[used] + if path then + return path,remap[used] or used + end + end +end +local nothing=function() end +function resolvers.filtered_from_content(content,pattern) + if content and type(pattern)=="string" then + local pattern=lower(pattern) + local files=content.files + local remap=content.remap + if files and remap then + local n=next(files) + local function iterator() + while n do + local k=n + n=next(files,k) + if find(k,pattern) then + return files[k],remap and remap[k] or k + end + end + end + return iterator + end + end + return nothing +end end -- of closure @@ -12735,7 +13783,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 8769, stripped down to: 6490 +-- original size: 9216, stripped down to: 6798 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -12753,10 +13801,12 @@ local formats=allocate() local suffixes=allocate() local dangerous=allocate() local suffixmap=allocate() +local usertypes=allocate() resolvers.formats=formats resolvers.suffixes=suffixes resolvers.dangerous=dangerous resolvers.suffixmap=suffixmap +resolvers.usertypes=usertypes local luasuffixes=utilities.lua.suffixes local relations=allocate { core={ @@ -12824,11 +13874,13 @@ local relations=allocate { names={ "mp" }, variable='MPINPUTS', suffixes={ 'mp','mpvi','mpiv','mpii' }, + usertype=true, }, tex={ names={ "tex" }, variable='TEXINPUTS', - suffixes={ 'tex',"mkvi","mkiv","mkii" }, + suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + usertype=true, }, icc={ names={ "icc","icc profile","icc profiles" }, @@ -12844,6 +13896,7 @@ local relations=allocate { names={ "lua" }, variable='LUAINPUTS', suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + usertype=true, }, lib={ names={ "lib" }, @@ -12852,11 +13905,15 @@ local relations=allocate { }, bib={ names={ 'bib' }, + variable='BIBINPUTS', suffixes={ 'bib' }, + usertype=true, }, bst={ names={ 'bst' }, + variable='BSTINPUTS', suffixes={ 'bst' }, + usertype=true, }, fontconfig={ names={ 'fontconfig','fontconfig file','fontconfig files' }, @@ -12938,8 +13995,9 @@ function resolvers.updaterelations() for name,relation in next,categories do local rn=relation.names local rv=relation.variable - local rs=relation.suffixes if rn and rv then + local rs=relation.suffixes + local ru=relation.usertype for i=1,#rn do local rni=lower(gsub(rn[i]," ","")) formats[rni]=rv @@ -12951,8 +14009,9 @@ function resolvers.updaterelations() end end end - end - if rs then + if ru then + usertypes[name]=true + end end end end @@ -13003,7 +14062,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15532, stripped down to: 11648 +-- original size: 15618, stripped down to: 11629 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -13013,7 +14072,7 @@ if not modules then modules={} end modules ['data-tmp']={ license="see context related readme files" } local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat -local concat,serialize,serializetofile=table.concat,table.serialize,table.tofile +local concat=table.concat local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable local formatters=string.formatters @@ -13022,6 +14081,7 @@ local trace_cache=false trackers.register("resolvers.cache",function(v) trace_ca local report_caches=logs.reporter("resolvers","caches") local report_resolvers=logs.reporter("resolvers","caching") local resolvers=resolvers +local cleanpath=resolvers.cleanpath local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) local compile=utilities.lua.compile @@ -13043,7 +14103,7 @@ caches.relocate=false caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } local writable,readables,usedreadables=nil,{},{} local function identify() - local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") if texmfcaches then for k=1,#texmfcaches do local cachepath=texmfcaches[k] @@ -13281,15 +14341,11 @@ end local saveoptions={ compact=true } function caches.savedata(filepath,filename,data,raw) local tmaname,tmcname=caches.setluanames(filepath,filename) - local reduce,simplify=true,true - if raw then - reduce,simplify=false,false - end data.cache_uuid=os.uuid() if caches.direct then - file.savedata(tmaname,serialize(data,true,saveoptions)) + file.savedata(tmaname,table.serialize(data,true,saveoptions)) else - serializetofile(tmaname,data,true,saveoptions) + table.tofile(tmaname,data,true,saveoptions) end utilities.lua.compile(tmaname,tmcname) end @@ -13297,10 +14353,12 @@ local content_state={} function caches.contentstate() return content_state or {} end -function caches.loadcontent(cachename,dataname) - local name=caches.hashed(cachename) - local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.loadcontent(cachename,dataname,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) if blob then local data=blob() @@ -13332,10 +14390,12 @@ function caches.collapsecontent(content) end end end -function caches.savecontent(cachename,dataname,content) - local name=caches.hashed(cachename) - local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") - local filename=file.join(path,name) +function caches.savecontent(cachename,dataname,content,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end local luaname=addsuffix(filename,luasuffixes.lua) local lucname=addsuffix(filename,luasuffixes.luc) if trace_locating then @@ -13350,7 +14410,7 @@ function caches.savecontent(cachename,dataname,content) content=content, uuid=os.uuid(), } - local ok=io.savedata(luaname,serialize(data,true)) + local ok=io.savedata(luaname,table.serialize(data,true)) if ok then if trace_locating then report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) @@ -13378,7 +14438,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5453, stripped down to: 4007 +-- original size: 5347, stripped down to: 4015 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -13406,8 +14466,8 @@ local function splitmethod(filename) if type(filename)=="table" then return filename end - filename=file.collapsepath(filename,".") - if not find(filename,"://") then + filename=file.collapsepath(filename,".") + if not find(filename,"://",1,true) then return { scheme="file",path=filename,original=filename,filename=filename } end local specification=url.hashed(filename) @@ -13497,7 +14557,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 61799, stripped down to: 42957 +-- original size: 67003, stripped down to: 46291 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -13507,7 +14567,7 @@ if not modules then modules={} end modules ['data-res']={ license="see context related readme files", } local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch -local concat,insert,sortedkeys=table.concat,table.insert,table.sortedkeys +local concat,insert,remove,sortedkeys,sortedhash=table.concat,table.insert,table.remove,table.sortedkeys,table.sortedhash local next,type,rawget=next,type,rawget local os=os local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg @@ -13516,27 +14576,38 @@ local formatters=string.formatters local filedirname=file.dirname local filebasename=file.basename local suffixonly=file.suffixonly +local addsuffix=file.addsuffix +local removesuffix=file.removesuffix local filejoin=file.join local collapsepath=file.collapsepath local joinpath=file.joinpath +local is_qualified_path=file.is_qualified_path local allocate=utilities.storage.allocate local settings_to_array=utilities.parsers.settings_to_array +local getcurrentdir=lfs.currentdir +local isfile=lfs.isfile +local isdir=lfs.isdir local setmetatableindex=table.setmetatableindex local luasuffixes=utilities.lua.suffixes -local getcurrentdir=lfs.currentdir -local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) -local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) -local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) +local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) local report_resolving=logs.reporter("resolvers","resolving") local resolvers=resolvers local expandedpathfromlist=resolvers.expandedpathfromlist local checkedvariable=resolvers.checkedvariable local splitconfigurationpath=resolvers.splitconfigurationpath local methodhandler=resolvers.methodhandler +local filtered=resolvers.filtered_from_content +local lookup=resolvers.get_from_content +local cleanpath=resolvers.cleanpath +local resolveprefix=resolvers.resolve local initializesetter=utilities.setters.initialize local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv -resolvers.cacheversion='1.0.1' -resolvers.configbanner='' +resolvers.cacheversion="1.100" +resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" @@ -13555,6 +14626,7 @@ end local unset_variable="unset" local formats=resolvers.formats local suffixes=resolvers.suffixes +local usertypes=resolvers.usertypes local dangerous=resolvers.dangerous local suffixmap=resolvers.suffixmap resolvers.defaultsuffixes={ "tex" } @@ -13563,7 +14635,7 @@ local instance=resolvers.instance or nil function resolvers.setenv(key,value,raw) if instance then instance.environment[key]=value - ossetenv(key,raw and value or resolvers.resolve(value)) + ossetenv(key,raw and value or resolveprefix(value)) end end local function getenv(key) @@ -13577,7 +14649,7 @@ local function getenv(key) end resolvers.getenv=getenv resolvers.env=getenv -local function resolve(k) +local function resolvevariable(k) return instance.expansions[k] end local dollarstripper=lpeg.stripper("$") @@ -13586,19 +14658,19 @@ local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) -local variableexpander=Cs((somevariable*(somekey/resolve)+somethingelse)^1 ) +local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" local variablecleaner=Cs((cleaner+P(1))^0) -local somevariable=R("az","AZ","09","__","--")^1/resolve +local somevariable=R("az","AZ","09","__","--")^1/resolvevariable local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) local variableresolver=Cs((variable+P(1))^0) local function expandedvariable(var) return lpegmatch(variableexpander,var) or var end -function resolvers.newinstance() - if trace_locating then +function resolvers.newinstance() + if trace_locating then report_resolving("creating instance") - end + end local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() local newinstance={ environment=environment, @@ -13611,6 +14683,7 @@ function resolvers.newinstance() foundintrees=allocate(), hashes=allocate(), hashed=allocate(), + pathlists=false, specification=allocate(), lists=allocate(), data=allocate(), @@ -13623,6 +14696,7 @@ function resolvers.newinstance() savelists=true, pattern=nil, force_suffixes=true, + pathstack={}, } setmetatableindex(variables,function(t,k) local v @@ -13672,8 +14746,13 @@ function resolvers.reset() end local function reset_hashes() instance.lists={} + instance.pathlists=false instance.found={} end +local function reset_caches() + instance.lists={} + instance.pathlists=false +end local slash=P("/") local pathexpressionpattern=Cs ( Cc("^")*( @@ -13725,13 +14804,13 @@ local function identify_configuration_files() for i=1,#cnfpaths do local filepath=cnfpaths[i] local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) if trace_locating then - local fullpath=gsub(resolvers.resolve(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c") + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) end - if lfs.isfile(realname) then + if isfile(realname) then specification[#specification+1]=filename if trace_locating then report_resolving("found configuration file %a",realname) @@ -13753,7 +14832,7 @@ local function load_configuration_files() local filename=specification[i] local pathname=filedirname(filename) local filename=filejoin(pathname,luacnfname) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local setups=instance.setups @@ -13761,7 +14840,7 @@ local function load_configuration_files() local parent=data and data.parent if parent then local filename=filejoin(pathname,parent) - local realname=resolvers.resolve(filename) + local realname=resolveprefix(filename) local blob=loadfile(realname) if blob then local parentdata=blob() @@ -13786,7 +14865,7 @@ local function load_configuration_files() elseif variables[k]==nil then if trace_locating and not warning then report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable", - k,resolvers.resolve(filename)) + k,resolveprefix(filename)) warning=true end variables[k]=v @@ -13846,7 +14925,7 @@ local function locate_file_databases() local stripped=lpegmatch(inhibitstripper,path) if stripped~="" then local runtime=stripped==path - path=resolvers.cleanpath(path) + path=cleanpath(path) local spec=resolvers.splitmethod(stripped) if runtime and (spec.noscheme or spec.scheme=="file") then stripped="tree:///"..stripped @@ -13909,8 +14988,8 @@ function resolvers.renew(hashname) report_resolving("identifying tree %a",hashname) end end - local realpath=resolvers.resolve(hashname) - if lfs.isdir(realpath) then + local realpath=resolveprefix(hashname) + if isdir(realpath) then if trace_locating then report_resolving("using path %a",realpath) end @@ -14011,19 +15090,53 @@ end function resolvers.unexpandedpath(str) return joinpath(resolvers.unexpandedpathlist(str)) end +function resolvers.pushpath(name) + local pathstack=instance.pathstack + local lastpath=pathstack[#pathstack] + local pluspath=filedirname(name) + if lastpath then + lastpath=collapsepath(filejoin(lastpath,pluspath)) + else + lastpath=collapsepath(pluspath) + end + insert(pathstack,lastpath) + if trace_paths then + report_resolving("pushing path %a",lastpath) + end +end +function resolvers.poppath() + local pathstack=instance.pathstack + if trace_paths and #pathstack>0 then + report_resolving("popping path %a",pathstack[#pathstack]) + end + remove(pathstack) +end +function resolvers.stackpath() + local pathstack=instance.pathstack + local currentpath=pathstack[#pathstack] + return currentpath~="" and currentpath or nil +end local done={} function resolvers.resetextrapath() local ep=instance.extra_paths if not ep then - ep,done={},{} - instance.extra_paths=ep + done={} + instance.extra_paths={} elseif #ep>0 then - instance.lists,done={},{} + done={} + reset_caches() end end function resolvers.registerextrapath(paths,subpaths) - paths=settings_to_array(paths) - subpaths=settings_to_array(subpaths) + if not subpaths or subpaths=="" then + if not paths or path=="" then + return + elseif done[paths] then + return + end + end + local paths=settings_to_array(paths) + local subpaths=settings_to_array(subpaths) local ep=instance.extra_paths or {} local oldn=#ep local newn=oldn @@ -14038,7 +15151,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=p.."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14048,7 +15161,7 @@ function resolvers.registerextrapath(paths,subpaths) local p=paths[i] if not done[p] then newn=newn+1 - ep[newn]=resolvers.cleanpath(p) + ep[newn]=cleanpath(p) done[p]=true end end @@ -14060,7 +15173,7 @@ function resolvers.registerextrapath(paths,subpaths) local ps=ep[i].."/"..s if not done[ps] then newn=newn+1 - ep[newn]=resolvers.cleanpath(ps) + ep[newn]=cleanpath(ps) done[ps]=true end end @@ -14069,52 +15182,70 @@ function resolvers.registerextrapath(paths,subpaths) if newn>0 then instance.extra_paths=ep end - if newn>oldn then - instance.lists={} + if newn~=oldn then + reset_caches() end end -local function made_list(instance,list) - local ep=instance.extra_paths - if not ep or #ep==0 then - return list +function resolvers.pushextrapath(path) + local paths=settings_to_array(path) + if instance.extra_stack then + insert(instance.extra_stack,1,paths) else - local done,new,newn={},{},0 - for k=1,#list do - local v=list[k] - if not done[v] then - if find(v,"^[%.%/]$") then - done[v]=true - newn=newn+1 - new[newn]=v - else - break - end - end - end - for k=1,#ep do - local v=ep[k] + instance.extra_stack={ paths } + end + reset_caches() +end +function resolvers.popextrapath() + if instance.extra_stack then + reset_caches() + return remove(instance.extra_stack,1) + end +end +local function made_list(instance,list,extra_too) + local done={} + local new={} + local newn=0 + local function add(p) + for k=1,#p do + local v=p[k] if not done[v] then done[v]=true newn=newn+1 new[newn]=v end end - for k=1,#list do - local v=list[k] - if not done[v] then - done[v]=true - newn=newn+1 - new[newn]=v + end + for k=1,#list do + local v=list[k] + if done[v] then + elseif find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break + end + end + if extra_too then + local es=instance.extra_stack + if es and #es>0 then + for k=1,#es do + add(es[k]) end end - return new + local ep=instance.extra_paths + if ep and #ep>0 then + add(ep) + end end + add(list) + return new end function resolvers.cleanpathlist(str) local t=resolvers.expandedpathlist(str) if t then for i=1,#t do - t[i]=collapsepath(resolvers.cleanpath(t[i])) + t[i]=collapsepath(cleanpath(t[i])) end end return t @@ -14122,22 +15253,22 @@ end function resolvers.expandpath(str) return joinpath(resolvers.expandedpathlist(str)) end -function resolvers.expandedpathlist(str) +function resolvers.expandedpathlist(str,extra_too) if not str then return {} - elseif instance.savelists then + elseif instance.savelists then str=lpegmatch(dollarstripper,str) local lists=instance.lists local lst=lists[str] if not lst then - local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str))) + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) lst=expandedpathfromlist(l) lists[str]=lst end return lst else local lst=resolvers.splitpath(resolvers.expansion(str)) - return made_list(instance,expandedpathfromlist(lst)) + return made_list(instance,expandedpathfromlist(lst),extra_too) end end function resolvers.expandedpathlistfromvariable(str) @@ -14148,6 +15279,13 @@ end function resolvers.expandpathfromvariable(str) return joinpath(resolvers.expandedpathlistfromvariable(str)) end +function resolvers.cleanedpathlist(v) + local t=resolvers.expandedpathlist(v) + for i=1,#t do + t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + end + return t +end function resolvers.expandbraces(str) local ori=str local pth=expandedpathfromlist(resolvers.splitpath(ori)) @@ -14164,7 +15302,7 @@ function resolvers.registerfilehash(name,content,someerror) end end local function isreadable(name) - local readable=lfs.isfile(name) + local readable=isfile(name) if trace_detail then if readable then report_resolving("file %a is readable",name) @@ -14174,70 +15312,57 @@ local function isreadable(name) end return readable end -local function collect_files(names) - local filelist,noffiles={},0 +local function collect_files(names) + local filelist={} + local noffiles=0 + local function check(hash,root,pathname,path,name) + if not pathname or find(path,pathname) then + local variant=hash.type + local search=filejoin(root,path,name) + local result=methodhandler('concatinators',variant,root,path,name) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + end for k=1,#names do - local fname=names[k] + local filename=names[k] if trace_detail then - report_resolving("checking name %a",fname) + report_resolving("checking name %a",filename) end - local bname=filebasename(fname) - local dname=filedirname(fname) - if dname=="" or find(dname,"^%.") then - dname=false + local basename=filebasename(filename) + local pathname=filedirname(filename) + if pathname=="" or find(pathname,"^%.") then + pathname=false else - dname=gsub(dname,"%*",".*") - dname="/"..dname.."$" + pathname=gsub(pathname,"%*",".*") + pathname="/"..pathname.."$" end local hashes=instance.hashes for h=1,#hashes do local hash=hashes[h] - local blobpath=hash.name - local files=blobpath and instance.files[blobpath] - if files then + local hashname=hash.name + local content=hashname and instance.files[hashname] + if content then if trace_detail then - report_resolving("deep checking %a, base %a, pattern %a",blobpath,bname,dname) + report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) end - local blobfile=files[bname] - if not blobfile then - local rname="remap:"..bname - blobfile=files[rname] - if blobfile then - bname=files[rname] - blobfile=files[bname] - end - end - if blobfile then - local blobroot=files.__path__ or blobpath - if type(blobfile)=='string' then - if not dname or find(blobfile,dname) then - local variant=hash.type - local search=filejoin(blobroot,blobfile,bname) - local result=methodhandler('concatinators',hash.type,blobroot,blobfile,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + local path,name=lookup(content,basename) + if path then + local metadata=content.metadata + local realroot=metadata and metadata.path or hashname + if type(path)=="string" then + check(hash,realroot,pathname,path,name) else - for kk=1,#blobfile do - local vv=blobfile[kk] - if not dname or find(vv,dname) then - local variant=hash.type - local search=filejoin(blobroot,vv,bname) - local result=methodhandler('concatinators',hash.type,blobroot,vv,bname) - if trace_detail then - report_resolving("match: variant %a, search %a, result %a",variant,search,result) - end - noffiles=noffiles+1 - filelist[noffiles]={ variant,search,result } - end + for i=1,#path do + check(hash,realroot,pathname,path[i],name) end end end elseif trace_locating then - report_resolving("no match in %a (%s)",blobpath,bname) + report_resolving("no match in %a (%s)",hashname,basename) end end end @@ -14262,7 +15387,7 @@ end local function can_be_dir(name) local fakepaths=instance.fakepaths if not fakepaths[name] then - if lfs.isdir(name) then + if isdir(name) then fakepaths[name]=1 else fakepaths[name]=2 @@ -14278,10 +15403,11 @@ local function find_analyze(filename,askedformat,allresults) if askedformat=="" then if ext=="" or not suffixmap[ext] then local defaultsuffixes=resolvers.defaultsuffixes + local formatofsuffix=resolvers.formatofsuffix for i=1,#defaultsuffixes do local forcedname=filename..'.'..defaultsuffixes[i] wantedfiles[#wantedfiles+1]=forcedname - filetype=resolvers.formatofsuffix(forcedname) + filetype=formatofsuffix(forcedname) if trace_locating then report_resolving("forcing filetype %a",filetype) end @@ -14317,18 +15443,18 @@ local function find_direct(filename,allresults) end end local function find_wildcard(filename,allresults) - if find(filename,'%*') then + if find(filename,'*',1,true) then if trace_locating then report_resolving("checking wildcard %a",filename) end - local method,result=resolvers.findwildcardfiles(filename) + local result=resolvers.findwildcardfiles(filename) if result then return "wildcard",result end end end local function find_qualified(filename,allresults,askedformat,alsostripped) - if not file.is_qualified_path(filename) then + if not is_qualified_path(filename) then return end if trace_locating then @@ -14402,33 +15528,66 @@ local function check_subpath(fname) return fname end end -local function find_intree(filename,filetype,wantedfiles,allresults) +local function makepathlist(list,filetype) local typespec=resolvers.variableofformat(filetype) - local pathlist=resolvers.expandedpathlist(typespec) - local method="intree" + local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local entry={} if pathlist and #pathlist>0 then - local filelist=collect_files(wantedfiles) + for k=1,#pathlist do + local path=pathlist[k] + local prescanned=find(path,'^!!') + local resursive=find(path,'//$') + local pathname=lpegmatch(inhibitstripper,path) + local expression=makepathexpression(pathname) + local barename=gsub(pathname,"/+$","") + barename=resolveprefix(barename) + local scheme=url.hasscheme(barename) + local schemename=gsub(barename,"%.%*$",'') + entry[k]={ + path=path, + pathname=pathname, + prescanned=prescanned, + recursive=recursive, + expression=expression, + barename=barename, + scheme=scheme, + schemename=schemename, + } + end + entry.typespec=typespec + list[filetype]=entry + else + list[filetype]=false + end + return entry +end +local function find_intree(filename,filetype,wantedfiles,allresults) + local pathlists=instance.pathlists + if not pathlists then + pathlists=setmetatableindex(allocate(),makepathlist) + instance.pathlists=pathlists + end + local pathlist=pathlists[filetype] + if pathlist then + local method="intree" + local filelist=collect_files(wantedfiles) local dirlist={} + local result={} if filelist then for i=1,#filelist do dirlist[i]=filedirname(filelist[i][3]).."/" end end if trace_detail then - report_resolving("checking filename %a",filename) + report_resolving("checking filename %a in tree",filename) end - local resolve=resolvers.resolve - local result={} for k=1,#pathlist do - local path=pathlist[k] - local pathname=lpegmatch(inhibitstripper,path) - local doscan=path==pathname - if not find (pathname,'//$') then - doscan=false - end + local entry=pathlist[k] + local path=entry.path + local pathname=entry.pathname local done=false if filelist then - local expression=makepathexpression(pathname) + local expression=entry.expression if trace_detail then report_resolving("using pattern %a for path %a",expression,pathname) end @@ -14436,8 +15595,8 @@ local function find_intree(filename,filetype,wantedfiles,allresults) local fl=filelist[k] local f=fl[2] local d=dirlist[k] - if find(d,expression) or find(resolve(d),expression) then - result[#result+1]=resolve(fl[3]) + if find(d,expression) or find(resolveprefix(d),expression) then + result[#result+1]=resolveprefix(fl[3]) done=true if allresults then if trace_detail then @@ -14458,56 +15617,62 @@ local function find_intree(filename,filetype,wantedfiles,allresults) method="database" else method="filesystem" - pathname=gsub(pathname,"/+$","") - pathname=resolve(pathname) - local scheme=url.hasscheme(pathname) + local scheme=entry.scheme if not scheme or scheme=="file" then - local pname=gsub(pathname,"%.%*$",'') - if not find(pname,"%*") then + local pname=entry.schemename + if not find(pname,"*",1,true) then if can_be_dir(pname) then - for k=1,#wantedfiles do - local w=wantedfiles[k] - local fname=check_subpath(filejoin(pname,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + if not done and not entry.prescanned then + if trace_detail then + report_resolving("quick root scan for %a",pname) end - end - if not done and doscan then - local files=resolvers.simplescanfiles(pname,false,true) for k=1,#wantedfiles do local w=wantedfiles[k] - local subpath=files[w] - if not subpath or subpath=="" then - elseif type(subpath)=="string" then - local fname=check_subpath(filejoin(pname,subpath,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break - end + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break end - else - for i=1,#subpath do - local sp=subpath[i] - if sp=="" then - else - local fname=check_subpath(filejoin(pname,sp,w)) - if fname then - result[#result+1]=fname - done=true - if not allresults then - break + end + end + if not done and entry.recursive then + if trace_detail then + report_resolving("scanning filesystem for %a",pname) + end + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end end end end - end - if done and not allresults then - break + if done and not allresults then + break + end end end end @@ -14515,6 +15680,18 @@ local function find_intree(filename,filetype,wantedfiles,allresults) end else end + else + for k=1,#wantedfiles do + local pname=entry.barename + local fname=methodhandler('finders',pname.."/"..wantedfiles[k]) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end end end if done and not allresults then @@ -14549,10 +15726,13 @@ local function find_otherwise(filename,filetype,wantedfiles,allresults) local filelist=collect_files(wantedfiles) local fl=filelist and filelist[1] if fl then - return "otherwise",{ resolvers.resolve(fl[3]) } + return "otherwise",{ resolveprefix(fl[3]) } end end collect_instance_files=function(filename,askedformat,allresults) + if not filename or filename=="" then + return {} + end askedformat=askedformat or "" filename=collapsepath(filename,".") filename=gsub(filename,"^%./",getcurrentdir().."/") @@ -14587,7 +15767,11 @@ collect_instance_files=function(filename,askedformat,allresults) else local method,result,stamp,filetype,wantedfiles if instance.remember then - stamp=formatters["%s--%s"](filename,askedformat) + if askedformat=="" then + stamp=formatters["%s::%s"](suffixonly(filename),filename) + else + stamp=formatters["%s::%s"](askedformat,filename) + end result=stamp and instance.found[stamp] if result then if trace_locating then @@ -14606,7 +15790,7 @@ collect_instance_files=function(filename,askedformat,allresults) method,result=find_intree(filename,filetype,wantedfiles) if not result then method,result=find_onpath(filename,filetype,wantedfiles) - if not result then + if resolve_otherwise and not result then method,result=find_otherwise(filename,filetype,wantedfiles) end end @@ -14622,7 +15806,7 @@ collect_instance_files=function(filename,askedformat,allresults) end if stamp then if trace_locating then - report_resolving("remembering file %a",filename) + report_resolving("remembering file %a using hash %a",filename,stamp) end instance.found[stamp]=result end @@ -14630,6 +15814,9 @@ collect_instance_files=function(filename,askedformat,allresults) end end local function findfiles(filename,filetype,allresults) + if not filename or filename=="" then + return {} + end local result,status=collect_instance_files(filename,filetype or "",allresults) if not result or #result==0 then local lowered=lower(filename) @@ -14649,39 +15836,30 @@ function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") end local function findgivenfiles(filename,allresults) - local bname,result=filebasename(filename),{} + local base=filebasename(filename) + local result={} local hashes=instance.hashes - local noffound=0 + local function okay(hash,path,name) + local found=methodhandler('concatinators',hash.type,hash.name,path,name) + if found and found~="" then + result[#result+1]=resolveprefix(found) + return not allresults + end + end for k=1,#hashes do local hash=hashes[k] - local files=instance.files[hash.name] or {} - local blist=files[bname] - if not blist then - local rname="remap:"..bname - blist=files[rname] - if blist then - bname=files[rname] - blist=files[bname] - end - end - if blist then - if type(blist)=='string' then - local found=methodhandler('concatinators',hash.type,hash.name,blist,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then - break - end + local content=instance.files[hash.name] + if content then + local path,name=lookup(content,base) + if not path then + elseif type(path)=="string" then + if okay(hash,path,name) then + return result end else - for kk=1,#blist do - local vv=blist[kk] - local found=methodhandler('concatinators',hash.type,hash.name,vv,bname) or "" - if found~="" then - noffound=noffound+1 - result[noffound]=resolvers.resolve(found) - if not allresults then break end + for i=1,#path do + if okay(hash,path[i],name) then + return result end end end @@ -14695,64 +15873,80 @@ end function resolvers.findgivenfile(filename) return findgivenfiles(filename,false)[1] or "" end -local function doit(path,blist,bname,tag,variant,result,allresults) - local done=false - if blist and variant then - local resolve=resolvers.resolve - if type(blist)=='string' then - if find(lower(blist),path) then - local full=methodhandler('concatinators',variant,tag,blist,bname) or "" - result[#result+1]=resolve(full) - done=true - end - else - for kk=1,#blist do - local vv=blist[kk] - if find(lower(vv),path) then - local full=methodhandler('concatinators',variant,tag,vv,bname) or "" - result[#result+1]=resolve(full) - done=true - if not allresults then break end - end - end - end - end - return done -end local makewildcard=Cs( (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 ) function resolvers.wildcardpattern(pattern) return lpegmatch(makewildcard,pattern) or pattern end -local function findwildcardfiles(filename,allresults,result) - result=result or {} +local function findwildcardfiles(filename,allresults,result) + local result=result or {} local base=filebasename(filename) local dirn=filedirname(filename) local path=lower(lpegmatch(makewildcard,dirn) or dirn) local name=lower(lpegmatch(makewildcard,base) or base) - local files,done=instance.files,false - if find(name,"%*") then + local files=instance.files + if find(name,"*",1,true) then local hashes=instance.hashes + local function okay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - for kk,hh in next,files[hashname] do - if not find(kk,"^remap:") then - if find(lower(kk),name) then - if doit(path,hh,kk,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + for found,base in filtered(files[hashname],name) do + if type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end end end end end else + local function okayokay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end local hashes=instance.hashes for k=1,#hashes do local hash=hashes[k] - local hashname,hashtype=hash.name,hash.type - if doit(path,files[hashname][base],base,hashname,hashtype,result,allresults) then done=true end - if done and not allresults then break end + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + local found,base=lookup(content,base) + if not found then + elseif type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end + end + end end end return result @@ -14825,7 +16019,7 @@ end function resolvers.dowithpath(name,func) local pathlist=resolvers.expandedpathlist(name) for i=1,#pathlist do - func("^"..resolvers.cleanpath(pathlist[i])) + func("^"..cleanpath(pathlist[i])) end end function resolvers.dowithvariable(name,func) @@ -14833,23 +16027,23 @@ function resolvers.dowithvariable(name,func) end function resolvers.locateformat(name) local engine=environment.ownmain or "luatex" - local barename=file.removesuffix(name) - local fullname=file.addsuffix(barename,"fmt") + local barename=removesuffix(name) + local fullname=addsuffix(barename,"fmt") local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" if fmtname=="" then fmtname=resolvers.findfile(fullname) - fmtname=resolvers.cleanpath(fmtname) + fmtname=cleanpath(fmtname) end if fmtname~="" then - local barename=file.removesuffix(fmtname) - local luaname=file.addsuffix(barename,luasuffixes.lua) - local lucname=file.addsuffix(barename,luasuffixes.luc) - local luiname=file.addsuffix(barename,luasuffixes.lui) - if lfs.isfile(luiname) then + local barename=removesuffix(fmtname) + local luaname=addsuffix(barename,luasuffixes.lua) + local lucname=addsuffix(barename,luasuffixes.luc) + local luiname=addsuffix(barename,luasuffixes.lui) + if isfile(luiname) then return barename,luiname - elseif lfs.isfile(lucname) then + elseif isfile(lucname) then return barename,lucname - elseif lfs.isfile(luaname) then + elseif isfile(luaname) then return barename,luaname end end @@ -14871,29 +16065,24 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) local hash=hashes[i] local blobtype=hash.type local blobpath=hash.name - if blobpath then + if blobtype and blobpath then + local total=0 + local checked=0 + local done=0 if before then before(blobtype,blobpath,pattern) end - local files=instance.files[blobpath] - local total,checked,done=0,0,0 - if files then - for k,v in table.sortedhash(files) do - total=total+1 - if find(k,"^remap:") then - elseif find(k,pattern) then - if type(v)=="string" then - checked=checked+1 - if handle(blobtype,blobpath,v,k) then - done=done+1 - end - else - checked=checked+#v - for i=1,#v do - if handle(blobtype,blobpath,v[i],k) then - done=done+1 - end - end + for path,name in filtered(instance.files[blobpath],pattern) do + if type(path)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,path,name) then + done=done+1 + end + else + checked=checked+#path + for i=1,#path do + if handle(blobtype,blobpath,path[i],name) then + done=done+1 end end end @@ -14904,8 +16093,8 @@ function resolvers.dowithfilesintree(pattern,handle,before,after) end end end -resolvers.obsolete=resolvers.obsolete or {} -local obsolete=resolvers.obsolete +local obsolete=resolvers.obsolete or {} +resolvers.obsolete=obsolete resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles @@ -14916,7 +16105,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 6643, stripped down to: 4401 +-- original size: 3950, stripped down to: 2935 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -14926,44 +16115,51 @@ if not modules then modules={} end modules ['data-pre']={ license="see context related readme files" } local resolvers=resolvers -local prefixes=utilities.storage.allocate() -resolvers.prefixes=prefixes -local cleanpath,findgivenfile,expansion=resolvers.cleanpath,resolvers.findgivenfile,resolvers.expansion +local prefixes=resolvers.prefixes +local cleanpath=resolvers.cleanpath +local findgivenfile=resolvers.findgivenfile +local expansion=resolvers.expansion local getenv=resolvers.getenv -local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match -local joinpath,basename,dirname=file.join,file.basename,file.dirname -local getmetatable,rawset,type=getmetatable,rawset,type +local basename=file.basename +local dirname=file.dirname +local joinpath=file.join +local isfile=lfs.isfile prefixes.environment=function(str) return cleanpath(expansion(str)) end -prefixes.relative=function(str,n) - if io.exists(str) then - elseif io.exists("./"..str) then - str="./"..str - else - local p="../" - for i=1,n or 2 do - if io.exists(p..str) then - str=p..str - break - else - p=p.."../" +local function relative(str,n) + if not isfile(str) then + local pstr="./"..str + if isfile(pstr) then + str=pstr + else + local p="../" + for i=1,n or 2 do + local pstr=p..str + if isfile(pstr) then + str=pstr + break + else + p=p.."../" + end end end end return cleanpath(str) end +local function locate(str) + local fullname=findgivenfile(str) or "" + return cleanpath(fullname~="" and fullname or str) +end +prefixes.relative=relative +prefixes.locate=locate prefixes.auto=function(str) - local fullname=prefixes.relative(str) - if not lfs.isfile(fullname) then - fullname=prefixes.locate(str) + local fullname=relative(str) + if not isfile(fullname) then + fullname=locate(str) end return fullname end -prefixes.locate=function(str) - local fullname=findgivenfile(str) or "" - return cleanpath((fullname~="" and fullname) or str) -end prefixes.filename=function(str) local fullname=findgivenfile(str) or "" return cleanpath(basename((fullname~="" and fullname) or str)) @@ -14984,6 +16180,13 @@ end prefixes.home=function(str) return cleanpath(joinpath(getenv('HOME'),str)) end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname local function toppath() local inputstack=resolvers.inputstack if not inputstack then @@ -14996,98 +16199,22 @@ local function toppath() return pathname end end -resolvers.toppath=toppath -prefixes.toppath=function(str) - return cleanpath(joinpath(toppath(),str)) -end -prefixes.env=prefixes.environment -prefixes.rel=prefixes.relative -prefixes.loc=prefixes.locate -prefixes.kpse=prefixes.locate -prefixes.full=prefixes.locate -prefixes.file=prefixes.filename -prefixes.path=prefixes.pathname -function resolvers.allprefixes(separator) - local all=table.sortedkeys(prefixes) - if separator then - for i=1,#all do - all[i]=all[i]..":" - end - end - return all -end -local function _resolve_(method,target) - local action=prefixes[method] - if action then - return action(target) - else - return method..":"..target - end -end -local resolved,abstract={},{} -function resolvers.resetresolve(str) - resolved,abstract={},{} -end -local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) -local prefix=C(R("az")^2)*P(":") -local target=C((1-S(" \"\';,"))^1) -local notarget=(#S(";,")+P(-1))*Cc("") -local pattern=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) -local function resolve(str) - if type(str)=="table" then - local t={} - for i=1,#str do - t[i]=resolve(str[i]) - end - return t +local function jobpath() + local path=resolvers.stackpath() + if not path or path=="" then + return "." else - local res=resolved[str] - if not res then - res=lpegmatch(pattern,str) - resolved[str]=res - abstract[res]=str - end - return res - end -end -local function unresolve(str) - return abstract[str] or str -end -resolvers.resolve=resolve -resolvers.unresolve=unresolve -if type(os.uname)=="function" then - for k,v in next,os.uname() do - if not prefixes[k] then - prefixes[k]=function() return v end - end - end -end -if os.type=="unix" then - local pattern - local function makepattern(t,k,v) - if t then - rawset(t,k,v) - end - local colon=P(":") - for k,v in table.sortedpairs(prefixes) do - if p then - p=P(k)+p - else - p=P(k) - end - end - pattern=Cs((p*colon+colon/";"+P(1))^0) - end - makepattern() - getmetatable(prefixes).__newindex=makepattern - function resolvers.repath(str) - return lpegmatch(pattern,str) - end -else - function resolvers.repath(str) - return str + return path end end +resolvers.toppath=toppath +resolvers.jobpath=jobpath +prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end +prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end +resolvers.setdynamic("toppath") +resolvers.setdynamic("jobpath") +prefixes.jobfile=prefixes.jobpath +resolvers.setdynamic("jobfile") end -- of closure @@ -15149,7 +16276,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3801, stripped down to: 3231 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -15161,30 +16288,31 @@ if not modules then modules={} end modules ['data-fil']={ local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_files=logs.reporter("resolvers","files") local resolvers=resolvers +local resolveprefix=resolvers.resolve local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check function locators.file(specification) - local name=specification.filename - local realname=resolvers.resolve(name) + local filename=specification.filename + local realname=resolveprefix(filename) if realname and realname~='' and lfs.isdir(realname) then if trace_locating then - report_files("file locator %a found as %a",name,realname) + report_files("file locator %a found as %a",filename,realname) end - resolvers.appendhash('file',name,true) + resolvers.appendhash('file',filename,true) elseif trace_locating then - report_files("file locator %a not found",name) + report_files("file locator %a not found",filename) end end function hashers.file(specification) - local name=specification.filename - local content=caches.loadcontent(name,'files') - resolvers.registerfilehash(name,content,content==nil) + local pathname=specification.filename + local content=caches.loadcontent(pathname,'files') + resolvers.registerfilehash(pathname,content,content==nil) end function generators.file(specification) - local path=specification.filename - local content=resolvers.scanfiles(path,false,true) - resolvers.registerfilehash(path,content,true) + local pathname=specification.filename + local content=resolvers.scanfiles(pathname,false,true) + resolvers.registerfilehash(pathname,content,true) end concatinators.file=file.join function finders.file(specification,filetype) @@ -15375,7 +16503,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3913, stripped down to: 2998 +-- original size: 3899, stripped down to: 2984 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -15421,7 +16549,7 @@ end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) function statistics.savefmtstatus(texname,formatbanner,sourcefile) - local enginebanner=status.list().banner + local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") local luvdata={ @@ -15434,7 +16562,7 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) end end function statistics.checkfmtstatus(texname) - local enginebanner=status.list().banner + local enginebanner=status.banner if enginebanner and texname then local luvname=file.replacesuffix(texname,"luv") if lfs.isfile(luvname) then @@ -15466,7 +16594,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8489, stripped down to: 6757 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -15485,16 +16613,6 @@ zip.archives=zip.archives or {} local archives=zip.archives zip.registeredfiles=zip.registeredfiles or {} local registeredfiles=zip.registeredfiles -local limited=false -directives.register("system.inputmode",function(v) - if not limited then - local i_limiter=io.i_limiter(v) - if i_limiter then - zip.open=i_limiter.protect(zip.open) - limited=true - end - end -end) local function validzip(str) if not find(str,"^zip://") then return "zip:///"..str @@ -15509,7 +16627,7 @@ function zip.openarchive(name) local arch=archives[name] if not arch then local full=resolvers.findfile(name) or "" - arch=(full~="" and zip.open(full)) or false + arch=full~="" and zip.open(full) or false archives[name]=arch end return arch @@ -15668,31 +16786,42 @@ function resolvers.usezipfile(archive) end end function resolvers.registerzipfile(z,tree) - local files,filter={},"" - if tree=="" then - filter="^(.+)/(.-)$" - else - filter=format("^%s/(.+)/(.-)$",tree) - end + local names={} + local files={} + local remap={} + local n=0 + local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) + local register=resolvers.registerfile if trace_locating then report_zip("registering: using filter %a",filter) end - local register,n=resolvers.registerfile,0 for i in z:files() do - local path,name=match(i.filename,filter) - if path then - if name and name~='' then - register(files,name,path) - n=n+1 - else + local filename=i.filename + local path,name=match(filename,filter) + if not path then + n=n+1 + register(names,filename,"") + local usedname=lower(filename) + files[usedname]="" + if usedname~=filename then + remap[usedname]=filename end - else - register(files,i.filename,'') + elseif name and name~="" then n=n+1 + register(names,name,path) + local usedname=lower(name) + files[usedname]=path + if usedname~=name then + remap[usedname]=name + end + else end end report_zip("registering: %s files registered",n) - return files + return { + files=files, + remap=remap, + } end @@ -15702,7 +16831,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 2508, stripped down to: 2074 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -15711,42 +16840,64 @@ if not modules then modules={} end modules ['data-tre']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,gsub,format=string.find,string.gsub,string.format +local find,gsub,lower=string.find,string.gsub,string.lower +local basename,dirname,joinname=file.basename,file.dirname,file .join +local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile +local P,lpegmatch=lpeg.P,lpeg.match local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local report_trees=logs.reporter("resolvers","trees") local resolvers=resolvers -local done,found,notfound={},{},resolvers.finders.notfound -function resolvers.finders.tree(specification) +local resolveprefix=resolvers.resolve +local notfound=resolvers.finders.notfound +local lookup=resolvers.get_from_content +local collectors={} +local found={} +function resolvers.finders.tree(specification) local spec=specification.filename - local fnd=found[spec] - if fnd==nil then + local okay=found[spec] + if okay==nil then if spec~="" then - local path,name=file.dirname(spec),file.basename(spec) - if path=="" then path="." end - local hash=done[path] - if not hash then - local pattern=path.."/*" - hash=dir.glob(pattern) - done[path]=hash + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names end local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" - for k=1,#hash do - local v=hash[k] - if find(v,pattern) then - found[spec]=v - return v + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then + found[spec]=fullname + return fullname + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end end end end - fnd=notfound() - found[spec]=fnd + okay=notfound() + found[spec]=okay end - return fnd + return okay end function resolvers.locators.tree(specification) local name=specification.filename - local realname=resolvers.resolve(name) - if realname and realname~='' and lfs.isdir(realname) then + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then if trace_locating then report_trees("locator %a found",realname) end @@ -15757,16 +16908,110 @@ function resolvers.locators.tree(specification) end function resolvers.hashers.tree(specification) local name=specification.filename - if trace_locating then - report_trees("analysing %a",name) - end + report_trees("analyzing %a",name) resolvers.methodhandler("hashers",name) resolvers.generators.file(specification) end -resolvers.concatinators.tree=resolvers.concatinators.file -resolvers.generators.tree=resolvers.generators.file -resolvers.openers.tree=resolvers.openers.file -resolvers.loaders.tree=resolvers.loaders.file +local collectors={} +local splitter=lpeg.splitat("/**/") +local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } +table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=caches.loadcontent(dataname,"files",dataname) + if not content then + content=resolvers.scanfiles(rootname,nil,nil,false,true) + caches.savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content +end) +local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + end + return notfound() +end +local function resolve(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end + end + end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() + end + end + end + local path,name=dirname(filename),basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) + end + end + return notfound() +end +resolvers.finders .dirlist=resolve +resolvers.locators .dirlist=resolvers.locators .tree +resolvers.hashers .dirlist=resolvers.hashers .tree +resolvers.generators.dirlist=resolvers.generators.file +resolvers.openers .dirlist=resolvers.openers .file +resolvers.loaders .dirlist=resolvers.loaders .file +function resolvers.finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } + end + return resolve(specification) +end +resolvers.locators .dirfile=resolvers.locators .dirlist +resolvers.hashers .dirfile=resolvers.hashers .dirlist +resolvers.generators.dirfile=resolvers.generators.dirlist +resolvers.openers .dirfile=resolvers.openers .dirlist +resolvers.loaders .dirfile=resolvers.loaders .dirlist end -- of closure @@ -15775,7 +17020,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6202, stripped down to: 5149 +-- original size: 6569, stripped down to: 5304 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -15801,8 +17046,13 @@ directives.register("schemes.threshold",function(v) threshold=tonumber(v) or thr function cleaners.none(specification) return specification.original end -function cleaners.strip(specification) - return (gsub(specification.original,"[^%a%d%.]+","-")) +function cleaners.strip(specification) + local path,name=file.splitbase(specification.original) + if path=="" then + return (gsub(name,"[^%a%d%.]+","-")) + else + return (gsub((gsub(path,"%.","-").."-"..name),"[^%a%d%.]+","-")) + end end function cleaners.md5(specification) return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) @@ -15818,8 +17068,8 @@ function resolvers.schemes.cleanname(specification) end local cached,loaded,reused,thresholds,handlers={},{},{},{},{} local function runcurl(name,cachename) - local command="curl --silent --create-dirs --output "..cachename.." "..name - os.spawn(command) + local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name + os.execute(command) end local function fetch(specification) local original=specification.original @@ -15951,7 +17201,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4237, stripped down to: 3177 +-- original size: 4313, stripped down to: 3227 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -15960,7 +17210,7 @@ if not modules then modules={} end modules ['data-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local resolvers,package=resolvers,package +local package,lpeg=package,lpeg local gsub=string.gsub local concat=table.concat local addsuffix=file.addsuffix @@ -15971,9 +17221,11 @@ local luaformats={ 'TEXINPUTS','LUAINPUTS' } local libformats={ 'CLUAINPUTS' } local helpers=package.helpers or {} local methods=helpers.methods or {} +local resolvers=resolvers +local resolveprefix=resolvers.resolve +helpers.report=logs.reporter("resolvers","libraries") trackers.register("resolvers.libraries",function(v) helpers.trace=v end) trackers.register("resolvers.locating",function(v) helpers.trace=v end) -helpers.report=logs.reporter("resolvers","libraries") helpers.sequence={ "already loaded", "preload table", @@ -15988,7 +17240,7 @@ helpers.sequence={ } local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) function helpers.cleanpath(path) - return resolvers.resolve(lpegmatch(pattern,path)) + return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib local getextraluapaths=package.extraluapaths @@ -16058,7 +17310,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2394, stripped down to: 2005 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -16072,8 +17324,8 @@ local type,next=type,next local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local resolvers=resolvers local report_scripts=logs.reporter("resolvers","scripts") -function resolvers.updatescript(oldname,newname) - local scriptpath="scripts/context/lua" +function resolvers.updatescript(oldname,newname) + local scriptpath="context/lua" newname=file.addsuffix(newname,"lua") local oldscript=resolvers.cleanpath(oldname) if trace_locating then @@ -16125,7 +17377,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2600, stripped down to: 1627 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -16181,7 +17433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2654, stripped down to: 2301 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -16190,10 +17442,13 @@ if not modules then modules={} end modules ['data-lst']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local find,concat,upper,format=string.find,table.concat,string.upper,string.format +local rawget,type,next=rawget,type,next +local find,concat,upper=string.find,table.concat,string.upper local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs -resolvers.listers=resolvers.listers or {} local resolvers=resolvers +local listers=resolvers.listers or {} +resolvers.listers=listers +local resolveprefix=resolvers.resolve local report_lists=logs.reporter("resolvers","lists") local function tabstr(str) if type(str)=='table' then @@ -16202,7 +17457,7 @@ local function tabstr(str) return str end end -function resolvers.listers.variables(pattern) +function listers.variables(pattern) local instance=resolvers.instance local environment=instance.environment local variables=instance.variables @@ -16223,10 +17478,10 @@ function resolvers.listers.variables(pattern) for key,value in sortedpairs(configured) do if key~="" and (pattern=="" or find(upper(key),pattern)) then report_lists(key) - report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") - report_lists(" var: %s",tabstr(configured[key]) or "unset") - report_lists(" exp: %s",tabstr(expansions[key]) or "unset") - report_lists(" res: %s",tabstr(resolvers.resolve(expansions[key])) or "unset") + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolveprefix(expansions[key])) or "unset") end end instance.environment=fastcopy(env) @@ -16234,15 +17489,15 @@ function resolvers.listers.variables(pattern) instance.expansions=fastcopy(exp) end local report_resolved=logs.reporter("system","resolved") -function resolvers.listers.configurations() +function listers.configurations() local configurations=resolvers.instance.specification for i=1,#configurations do - report_resolved("file : %s",resolvers.resolve(configurations[i])) + report_resolved("file : %s",resolveprefix(configurations[i])) end report_resolved("") local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) for i=1,#list do - local li=resolvers.resolve(list[i]) + local li=resolveprefix(list[i]) if lfs.isdir(li) then report_resolved("path - %s",li) else @@ -16547,7 +17802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5951, stripped down to: 4922 +-- original size: 5955, stripped down to: 4926 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -16635,7 +17890,7 @@ function environment.make_format(name) end local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") report_format("running command: %s\n",command) - os.spawn(command) + os.execute(command) local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -16670,7 +17925,7 @@ function environment.run_format(name,data,more) else local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") report_format("running command: %s",command) - os.spawn(command) + os.execute(command) end end end @@ -16681,8 +17936,8 @@ end -- of closure -- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 685064 --- stripped bytes : 242353 +-- original bytes : 745618 +-- stripped bytes : 269191 -- end library merge @@ -16781,17 +18036,18 @@ local ownlibs = { -- order can be made better } +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/context/sources/data-tmf.lua + local ownlist = { - '.', - ownpath , - ownpath .. "/../sources", -- HH's development path + -- '.', + -- ownpath , + owntree .. "/../../../../context/sources", -- HH's development path owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", - owntree .. "/../../texmf-dist/tex/context/base", owntree .. "/../../texmf/tex/context/base", owntree .. "/../../../texmf-local/tex/context/base", owntree .. "/../../../texmf-context/tex/context/base", - owntree .. "/../../../texmf-dist/tex/context/base", owntree .. "/../../../texmf/tex/context/base", } @@ -16907,6 +18163,7 @@ local helpinfo = [[ <category name="basic"> <subcategory> <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> <flag name="resolve"><short>resolve prefixed arguments</short></flag> <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> @@ -16932,6 +18189,7 @@ local helpinfo = [[ <flag name="verbose"><short>give a bit more info</short></flag> <flag name="trackers" value="list"><short>enable given trackers</short></flag> <flag name="progname" value="str"><short>format or backend</short></flag> + <flag name="systeminfo" value="str"><short>show current operating system, processor, etc</short></flag> </subcategory> <subcategory> <flag name="edit"><short>launch editor with found file</short></flag> @@ -17561,6 +18819,39 @@ function runners.associate(filename) os.launch(filename) end +function runners.evaluate(code,filename) -- for Luigi + if code == "loop" then + while true do + io.write("> ") + local code = io.read() + if code ~= "" then + local temp = string.match(code,"^= (.*)$") + if temp then + code = "print("..temp..")" + end + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("! " .. (message or code).."\n") + else + io.write(compiled()) + end + end + end + else + if type(code) ~= "string" or code == "" then + code = filename + end + if code ~= "" then + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("invalid lua code: " .. (message or code)) + return + end + io.write(compiled()) + end + end +end + function runners.gethelp(filename) local url = environment.argument("url") if url and url ~= "" then @@ -17572,6 +18863,15 @@ function runners.gethelp(filename) end end +function runners.systeminfo() + report("architecture : %s",os.platform or "<unset>") + report("operating system : %s",os.name or "<unset>") + report("file architecture : %s",os.type or "<unset>") + report("binary path : %s",os.selfdir or "<unset>") + report("binary suffix : %s",os.binsuffix or "<unset>") + report("library suffix : %s",os.libsuffix or "<unset>") +end + -- this is a bit dirty ... first we store the first filename and next we -- split the arguments so that we only see the ones meant for this script -- ... later we will use the second half @@ -17687,16 +18987,13 @@ end if e_argument("ansi") then - local formatters = string.formatters + logs.setformatters("ansi") - logs.setformatters { - report_yes = formatters["[1;32m%-15s [0;1m|[0m %s"], - report_nop = formatters["[1;32m%-15s [0;1m|[0m"], - subreport_yes = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m %s"], - subreport_nop = formatters["[1;32m%-15s [0;1m|[1;31m %s [0;1m|[0m"], - status_yes = formatters["[1;32m%-15s [0;1m:[0m %s\n"], - status_nop = formatters["[1;32m%-15s [0;1m:[0m\n"], - } + local script = e_argument("script") or e_argument("scripts") + + if type(script) == "string" then + logs.writer("]0;"..script.."") -- for Alan to test + end end @@ -17715,14 +19012,26 @@ if e_argument("script") or e_argument("scripts") then ok = runners.execute_ctx_script(filename) end +elseif e_argument("evaluate") then + + runners.evaluate(e_argument("evaluate"),filename) + elseif e_argument("selfmerge") then -- embed used libraries runners.loadbase() local found = locate_libs() + if found then - utilities.merger.selfmerge(own.name,own.libs,{ found }) + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfmerge(mtxrun,own.libs,{ found }) + application.report("runner updated on resolved path: %s",mtxrun) + else + utilities.merger.selfmerge(own.name,own.libs,{ found }) + application.report("runner updated on relative path: %s",own.name) + end end elseif e_argument("selfclean") then @@ -17730,7 +19039,15 @@ elseif e_argument("selfclean") then -- remove embedded libraries runners.loadbase() - utilities.merger.selfclean(own.name) + + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfclean(mtxrun) + application.report("runner cleaned on resolved path: %s",mtxrun) + else + utilities.merger.selfclean(own.name) + application.report("runner cleaned on relative path: %s",own.name) + end elseif e_argument("selfupdate") then @@ -17972,6 +19289,8 @@ elseif e_argument("version") then application.version() + application.report("source path",environment.ownbin) + elseif e_argument("directives") then directives.show() @@ -17989,6 +19308,10 @@ elseif e_argument("exporthelp") then runners.loadbase() application.export(e_argument("exporthelp"),filename) +elseif e_argument("systeminfo") then + + runners.systeminfo() + elseif e_argument("help") or filename=='help' or filename == "" then application.help() diff --git a/scripts/context/stubs/unix/mtxrunjit b/scripts/context/stubs/unix/mtxrunjit new file mode 100644 index 000000000..117105aa5 --- /dev/null +++ b/scripts/context/stubs/unix/mtxrunjit @@ -0,0 +1,5 @@ +#!/bin/sh + +luajittex --luaonly $(dirname $0)/mtxrun "$@" + +# luajittex --luaonly ${0%jit} "$@" diff --git a/scripts/context/stubs/unix/pstopdf b/scripts/context/stubs/unix/pstopdf deleted file mode 100644 index 116f5f4a3..000000000 --- a/scripts/context/stubs/unix/pstopdf +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -mtxrun --script pstopdf "$@" diff --git a/scripts/context/stubs/win64/context.exe b/scripts/context/stubs/win64/context.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/context.exe diff --git a/scripts/context/stubs/win64/contextjit.exe b/scripts/context/stubs/win64/contextjit.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/contextjit.exe diff --git a/scripts/context/stubs/win64/ctxtools.exe b/scripts/context/stubs/win64/ctxtools.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/ctxtools.exe diff --git a/scripts/context/stubs/win64/luatools.exe b/scripts/context/stubs/win64/luatools.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/luatools.exe diff --git a/scripts/context/stubs/win64/metatex.exe b/scripts/context/stubs/win64/metatex.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/metatex.exe diff --git a/scripts/context/stubs/win64/mptopdf.exe b/scripts/context/stubs/win64/mptopdf.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/mptopdf.exe diff --git a/scripts/context/stubs/win64/mtxrun.dll b/scripts/context/stubs/win64/mtxrun.dll Binary files differnew file mode 100644 index 000000000..910502735 --- /dev/null +++ b/scripts/context/stubs/win64/mtxrun.dll diff --git a/scripts/context/stubs/win64/mtxrun.exe b/scripts/context/stubs/win64/mtxrun.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/mtxrun.exe diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua new file mode 100644 index 000000000..edfeba8dd --- /dev/null +++ b/scripts/context/stubs/win64/mtxrun.lua @@ -0,0 +1,19363 @@ +#!/usr/bin/env texlua + +-- for k, v in next, _G.string do +-- local tv = type(v) +-- if tv == "table" then +-- for kk, vv in next, v do +-- print(k,kk,vv) +-- end +-- else +-- print(tv,k,v) +-- end +-- end + +if not modules then modules = { } end modules ['mtxrun'] = { + version = 1.001, + comment = "runner, lua replacement for texmfstart.rb", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- one can make a stub: +-- +-- #!/bin/sh +-- env LUATEXDIR=/....../texmf/scripts/context/lua luatex --luaonly mtxrun.lua "$@" + +-- filename : mtxrun.lua +-- comment : companion to context.tex +-- author : Hans Hagen, PRAGMA-ADE, Hasselt NL +-- copyright: PRAGMA ADE / ConTeXt Development Team +-- license : see context related readme files + +-- This script is based on texmfstart.rb but does not use kpsewhich to +-- locate files. Although kpse is a library it never came to opening up +-- its interface to other programs (esp scripting languages) and so we +-- do it ourselves. The lua variant evolved out of an experimental ruby +-- one. Interesting is that using a scripting language instead of c does +-- not have a speed penalty. Actually the lua variant is more efficient, +-- especially when multiple calls to kpsewhich are involved. The lua +-- library also gives way more control. + +-- to be done / considered +-- +-- support for --exec or make it default +-- support for jar files (or maybe not, never used, too messy) +-- support for $RUBYINPUTS cum suis (if still needed) +-- remember for subruns: _CTX_K_V_#{original}_ +-- remember for subruns: _CTX_K_S_#{original}_ +-- remember for subruns: TEXMFSTART.#{original} [tex.rb texmfstart.rb] + +-- begin library merge + + + +do -- create closure to overcome 200 locals limit + +package.loaded["l-lua"] = package.loaded["l-lua"] or true + +-- original size: 3888, stripped down to: 2197 + +if not modules then modules={} end modules ['l-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(major) or 5 +_MINORVERSION=tonumber(minor) or 1 +_LUAVERSION=_MAJORVERSION+_MINORVERSION/10 +if not lpeg then + lpeg=require("lpeg") +end +if loadstring then + local loadnormal=load + function load(first,...) + if type(first)=="string" then + return loadstring(first,...) + else + return loadnormal(first,...) + end + end +else + loadstring=load +end +if not ipairs then + local function iterate(a,i) + i=i+1 + local v=a[i] + if v~=nil then + return i,v + end + end + function ipairs(a) + return iterate,a,0 + end +end +if not pairs then + function pairs(t) + return next,t + end +end +if not table.unpack then + table.unpack=_G.unpack +elseif not unpack then + _G.unpack=table.unpack +end +if not package.loaders then + package.loaders=package.searchers +end +local print,select,tostring=print,select,tostring +local inspectors={} +function setinspector(inspector) + inspectors[#inspectors+1]=inspector +end +function inspect(...) + for s=1,select("#",...) do + local value=select(s,...) + local done=false + for i=1,#inspectors do + done=inspectors[i](value) + if done then + break + end + end + if not done then + print(tostring(value)) + end + end +end +local dummy=function() end +function optionalrequire(...) + local ok,result=xpcall(require,dummy,...) + if ok then + return result + end +end +if lua then + lua.mask=load([[τεχ = 1]]) and "utf" or "ascii" +end +local flush=io.flush +if flush then + local execute=os.execute if execute then function os.execute(...) flush() return execute(...) end end + local exec=os.exec if exec then function os.exec (...) flush() return exec (...) end end + local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end + local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-package"] = package.loaded["l-package"] or true + +-- original size: 10587, stripped down to: 7815 + +if not modules then modules={} end modules ['l-package']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type=type +local gsub,format,find=string.gsub,string.format,string.find +local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match +local package=package +local searchers=package.searchers or package.loaders +local filejoin=file and file.join or function(path,name) return path.."/"..name end +local isreadable=file and file.is_readable or function(name) local f=io.open(name) if f then f:close() return true end end +local addsuffix=file and file.addsuffix or function(name,suffix) return name.."."..suffix end +local function cleanpath(path) + return path +end +local pattern=Cs((((1-S("\\/"))^0*(S("\\/")^1/"/"))^0*(P(".")^1/"/"+P(1))^1)*-1) +local function lualibfile(name) + return lpegmatch(pattern,name) or name +end +local offset=luarocks and 1 or 0 +local helpers=package.helpers or { + cleanpath=cleanpath, + lualibfile=lualibfile, + trace=false, + report=function(...) print(format(...)) end, + builtin={ + ["preload table"]=searchers[1+offset], + ["path specification"]=searchers[2+offset], + ["cpath specification"]=searchers[3+offset], + ["all in one fallback"]=searchers[4+offset], + }, + methods={}, + sequence={ + "already loaded", + "preload table", + "qualified path", + "lua extra list", + "lib extra list", + "path specification", + "cpath specification", + "all in one fallback", + "not loaded", + } +} +package.helpers=helpers +local methods=helpers.methods +local builtin=helpers.builtin +local extraluapaths={} +local extralibpaths={} +local luapaths=nil +local libpaths=nil +local oldluapath=nil +local oldlibpath=nil +local nofextralua=-1 +local nofextralib=-1 +local nofpathlua=-1 +local nofpathlib=-1 +local function listpaths(what,paths) + local nofpaths=#paths + if nofpaths>0 then + for i=1,nofpaths do + helpers.report("using %s path %i: %s",what,i,paths[i]) + end + else + helpers.report("no %s paths defined",what) + end + return nofpaths +end +local function getextraluapaths() + if helpers.trace and #extraluapaths~=nofextralua then + nofextralua=listpaths("extra lua",extraluapaths) + end + return extraluapaths +end +local function getextralibpaths() + if helpers.trace and #extralibpaths~=nofextralib then + nofextralib=listpaths("extra lib",extralibpaths) + end + return extralibpaths +end +local function getluapaths() + local luapath=package.path or "" + if oldluapath~=luapath then + luapaths=file.splitpath(luapath,";") + oldluapath=luapath + nofpathlua=-1 + end + if helpers.trace and #luapaths~=nofpathlua then + nofpathlua=listpaths("builtin lua",luapaths) + end + return luapaths +end +local function getlibpaths() + local libpath=package.cpath or "" + if oldlibpath~=libpath then + libpaths=file.splitpath(libpath,";") + oldlibpath=libpath + nofpathlib=-1 + end + if helpers.trace and #libpaths~=nofpathlib then + nofpathlib=listpaths("builtin lib",libpaths) + end + return libpaths +end +package.luapaths=getluapaths +package.libpaths=getlibpaths +package.extraluapaths=getextraluapaths +package.extralibpaths=getextralibpaths +local hashes={ + lua={}, + lib={}, +} +local function registerpath(tag,what,target,...) + local pathlist={... } + local cleanpath=helpers.cleanpath + local trace=helpers.trace + local report=helpers.report + local hash=hashes[what] + local function add(path) + local path=cleanpath(path) + if not hash[path] then + target[#target+1]=path + hash[path]=true + if trace then + report("registered %s path %s: %s",tag,#target,path) + end + else + if trace then + report("duplicate %s path: %s",tag,path) + end + end + end + for p=1,#pathlist do + local path=pathlist[p] + if type(path)=="table" then + for i=1,#path do + add(path[i]) + end + else + add(path) + end + end + return paths +end +helpers.registerpath=registerpath +function package.extraluapath(...) + registerpath("extra lua","lua",extraluapaths,...) +end +function package.extralibpath(...) + registerpath("extra lib","lib",extralibpaths,...) +end +local function loadedaslib(resolved,rawname) + local base=gsub(rawname,"%.","_") + local init="luaopen_"..gsub(base,"%.","_") + if helpers.trace then + helpers.report("calling loadlib with '%s' with init '%s'",resolved,init) + end + return package.loadlib(resolved,init) +end +helpers.loadedaslib=loadedaslib +local function loadedbypath(name,rawname,paths,islib,what) + local trace=helpers.trace + for p=1,#paths do + local path=paths[p] + local resolved=filejoin(path,name) + if trace then + helpers.report("%s path, identifying '%s' on '%s'",what,name,path) + end + if isreadable(resolved) then + if trace then + helpers.report("%s path, '%s' found on '%s'",what,name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end + end + end +end +helpers.loadedbypath=loadedbypath +local function loadedbyname(name,rawname) + if find(name,"^/") or find(name,"^[a-zA-Z]:/") then + local trace=helpers.trace + if trace then + helpers.report("qualified name, identifying '%s'",what,name) + end + if isreadable(name) then + if trace then + helpers.report("qualified name, '%s' found",what,name) + end + return loadfile(name) + end + end +end +helpers.loadedbyname=loadedbyname +methods["already loaded"]=function(name) + return package.loaded[name] +end +methods["preload table"]=function(name) + return builtin["preload table"](name) +end +methods["qualified path"]=function(name) + return loadedbyname(addsuffix(lualibfile(name),"lua"),name) +end +methods["lua extra list"]=function(name) + return loadedbypath(addsuffix(lualibfile(name),"lua"),name,getextraluapaths(),false,"lua") +end +methods["lib extra list"]=function(name) + return loadedbypath(addsuffix(lualibfile(name),os.libsuffix),name,getextralibpaths(),true,"lib") +end +methods["path specification"]=function(name) + getluapaths() + return builtin["path specification"](name) +end +methods["cpath specification"]=function(name) + getlibpaths() + return builtin["cpath specification"](name) +end +methods["all in one fallback"]=function(name) + return builtin["all in one fallback"](name) +end +methods["not loaded"]=function(name) + if helpers.trace then + helpers.report("unable to locate '%s'",name or "?") + end + return nil +end +local level=0 +local used={} +helpers.traceused=false +function helpers.loaded(name) + local sequence=helpers.sequence + level=level+1 + for i=1,#sequence do + local method=sequence[i] + if helpers.trace then + helpers.report("%s, level '%s', method '%s', name '%s'","locating",level,method,name) + end + local result,rest=methods[method](name) + if type(result)=="function" then + if helpers.trace then + helpers.report("%s, level '%s', method '%s', name '%s'","found",level,method,name) + end + if helpers.traceused then + used[#used+1]={ level=level,name=name } + end + level=level-1 + return result,rest + end + end + level=level-1 + return nil +end +function helpers.showused() + local n=#used + if n>0 then + helpers.report("%s libraries loaded:",n) + helpers.report() + for i=1,n do + local u=used[i] + helpers.report("%i %a",u.level,u.name) + end + helpers.report() + end +end +function helpers.unload(name) + if helpers.trace then + if package.loaded[name] then + helpers.report("unloading, name '%s', %s",name,"done") + else + helpers.report("unloading, name '%s', %s",name,"not loaded") + end + end + package.loaded[name]=nil +end +table.insert(searchers,1,helpers.loaded) + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true + +-- original size: 36977, stripped down to: 20349 + +if not modules then modules={} end modules ['l-lpeg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +lpeg=require("lpeg") +if not lpeg.print then function lpeg.print(...) print(lpeg.pcode(...)) end end +local type,next,tostring=type,next,tostring +local byte,char,gmatch,format=string.byte,string.char,string.gmatch,string.format +local floor=math.floor +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt +local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print +if setinspector then + setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) +end +lpeg.patterns=lpeg.patterns or {} +local patterns=lpeg.patterns +local anything=P(1) +local endofstring=P(-1) +local alwaysmatched=P(true) +patterns.anything=anything +patterns.endofstring=endofstring +patterns.beginofstring=alwaysmatched +patterns.alwaysmatched=alwaysmatched +local sign=S('+-') +local zero=P('0') +local digit=R('09') +local octdigit=R("07") +local lowercase=R("az") +local uppercase=R("AZ") +local underscore=P("_") +local hexdigit=digit+lowercase+uppercase +local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") +local escaped=P("\\")*anything +local squote=P("'") +local dquote=P('"') +local space=P(" ") +local period=P(".") +local comma=P(",") +local utfbom_32_be=P('\000\000\254\255') +local utfbom_32_le=P('\255\254\000\000') +local utfbom_16_be=P('\254\255') +local utfbom_16_le=P('\255\254') +local utfbom_8=P('\239\187\191') +local utfbom=utfbom_32_be+utfbom_32_le+utfbom_16_be+utfbom_16_le+utfbom_8 +local utftype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8")+alwaysmatched*Cc("utf-8") +local utfstricttype=utfbom_32_be*Cc("utf-32-be")+utfbom_32_le*Cc("utf-32-le")+utfbom_16_be*Cc("utf-16-be")+utfbom_16_le*Cc("utf-16-le")+utfbom_8*Cc("utf-8") +local utfoffset=utfbom_32_be*Cc(4)+utfbom_32_le*Cc(4)+utfbom_16_be*Cc(2)+utfbom_16_le*Cc(2)+utfbom_8*Cc(3)+Cc(0) +local utf8next=R("\128\191") +patterns.utfbom_32_be=utfbom_32_be +patterns.utfbom_32_le=utfbom_32_le +patterns.utfbom_16_be=utfbom_16_be +patterns.utfbom_16_le=utfbom_16_le +patterns.utfbom_8=utfbom_8 +patterns.utf_16_be_nl=P("\000\r\000\n")+P("\000\r")+P("\000\n") +patterns.utf_16_le_nl=P("\r\000\n\000")+P("\r\000")+P("\n\000") +patterns.utf_32_be_nl=P("\000\000\000\r\000\000\000\n")+P("\000\000\000\r")+P("\000\000\000\n") +patterns.utf_32_le_nl=P("\r\000\000\000\n\000\000\000")+P("\r\000\000\000")+P("\n\000\000\000") +patterns.utf8one=R("\000\127") +patterns.utf8two=R("\194\223")*utf8next +patterns.utf8three=R("\224\239")*utf8next*utf8next +patterns.utf8four=R("\240\244")*utf8next*utf8next*utf8next +patterns.utfbom=utfbom +patterns.utftype=utftype +patterns.utfstricttype=utfstricttype +patterns.utfoffset=utfoffset +local utf8char=patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four +local validutf8char=utf8char^0*endofstring*Cc(true)+Cc(false) +local utf8character=P(1)*R("\128\191")^0 +patterns.utf8=utf8char +patterns.utf8char=utf8char +patterns.utf8character=utf8character +patterns.validutf8=validutf8char +patterns.validutf8char=validutf8char +local eol=S("\n\r") +local spacer=S(" \t\f\v") +local whitespace=eol+spacer +local nonspacer=1-spacer +local nonwhitespace=1-whitespace +patterns.eol=eol +patterns.spacer=spacer +patterns.whitespace=whitespace +patterns.nonspacer=nonspacer +patterns.nonwhitespace=nonwhitespace +local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) +local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) +local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) +local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) +local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) +local b_stripper=Cs(spacer^0/""*(nonspacer^1+spacer^1/" ")^0) +local e_stripper=Cs((spacer^1*P(-1)/""+nonspacer^1+spacer^1/" ")^0) +local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) +patterns.stripper=stripper +patterns.fullstripper=fullstripper +patterns.collapser=collapser +patterns.b_collapser=b_collapser +patterns.m_collapser=m_collapser +patterns.e_collapser=e_collapser +patterns.b_stripper=b_stripper +patterns.m_stripper=m_stripper +patterns.e_stripper=e_stripper +patterns.lowercase=lowercase +patterns.uppercase=uppercase +patterns.letter=patterns.lowercase+patterns.uppercase +patterns.space=space +patterns.tab=P("\t") +patterns.spaceortab=patterns.space+patterns.tab +patterns.newline=newline +patterns.emptyline=newline^1 +patterns.equal=P("=") +patterns.comma=comma +patterns.commaspacer=comma*spacer^0 +patterns.period=period +patterns.colon=P(":") +patterns.semicolon=P(";") +patterns.underscore=underscore +patterns.escaped=escaped +patterns.squote=squote +patterns.dquote=dquote +patterns.nosquote=(escaped+(1-squote))^0 +patterns.nodquote=(escaped+(1-dquote))^0 +patterns.unsingle=(squote/"")*patterns.nosquote*(squote/"") +patterns.undouble=(dquote/"")*patterns.nodquote*(dquote/"") +patterns.unquoted=patterns.undouble+patterns.unsingle +patterns.unspacer=((patterns.spacer^1)/"")^0 +patterns.singlequoted=squote*patterns.nosquote*squote +patterns.doublequoted=dquote*patterns.nodquote*dquote +patterns.quoted=patterns.doublequoted+patterns.singlequoted +patterns.digit=digit +patterns.octdigit=octdigit +patterns.hexdigit=hexdigit +patterns.sign=sign +patterns.cardinal=digit^1 +patterns.integer=sign^-1*digit^1 +patterns.unsigned=digit^0*period*digit^1 +patterns.float=sign^-1*patterns.unsigned +patterns.cunsigned=digit^0*comma*digit^1 +patterns.cpunsigned=digit^0*(period+comma)*digit^1 +patterns.cfloat=sign^-1*patterns.cunsigned +patterns.cpfloat=sign^-1*patterns.cpunsigned +patterns.number=patterns.float+patterns.integer +patterns.cnumber=patterns.cfloat+patterns.integer +patterns.cpnumber=patterns.cpfloat+patterns.integer +patterns.oct=zero*octdigit^1 +patterns.octal=patterns.oct +patterns.HEX=zero*P("X")*(digit+uppercase)^1 +patterns.hex=zero*P("x")*(digit+lowercase)^1 +patterns.hexadecimal=zero*S("xX")*hexdigit^1 +patterns.hexafloat=sign^-1*zero*S("xX")*(hexdigit^0*period*hexdigit^1+hexdigit^1*period*hexdigit^0+hexdigit^1)*(S("pP")*sign^-1*hexdigit^1)^-1 +patterns.decafloat=sign^-1*(digit^0*period*digit^1+digit^1*period*digit^0+digit^1)*S("eE")*sign^-1*digit^1 +patterns.propername=(uppercase+lowercase+underscore)*(uppercase+lowercase+underscore+digit)^0*endofstring +patterns.somecontent=(anything-newline-space)^1 +patterns.beginline=#(1-newline) +patterns.longtostring=Cs(whitespace^0/""*((patterns.quoted+nonwhitespace^1+whitespace^1/""*(P(-1)+Cc(" ")))^0)) +local function anywhere(pattern) + return P { P(pattern)+1*V(1) } +end +lpeg.anywhere=anywhere +function lpeg.instringchecker(p) + p=anywhere(p) + return function(str) + return lpegmatch(p,str) and true or false + end +end +function lpeg.splitter(pattern,action) + return (((1-P(pattern))^1)/action+1)^0 +end +function lpeg.tsplitter(pattern,action) + return Ct((((1-P(pattern))^1)/action+1)^0) +end +local splitters_s,splitters_m,splitters_t={},{},{} +local function splitat(separator,single) + local splitter=(single and splitters_s[separator]) or splitters_m[separator] + if not splitter then + separator=P(separator) + local other=C((1-separator)^0) + if single then + local any=anything + splitter=other*(separator*C(any^0)+"") + splitters_s[separator]=splitter + else + splitter=other*(separator*other)^0 + splitters_m[separator]=splitter + end + end + return splitter +end +local function tsplitat(separator) + local splitter=splitters_t[separator] + if not splitter then + splitter=Ct(splitat(separator)) + splitters_t[separator]=splitter + end + return splitter +end +lpeg.splitat=splitat +lpeg.tsplitat=tsplitat +function string.splitup(str,separator) + if not separator then + separator="," + end + return lpegmatch(splitters_m[separator] or splitat(separator),str) +end +local cache={} +function lpeg.split(separator,str) + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) +end +function string.split(str,separator) + if separator then + local c=cache[separator] + if not c then + c=tsplitat(separator) + cache[separator]=c + end + return lpegmatch(c,str) + else + return { str } + end +end +local spacing=patterns.spacer^0*newline +local empty=spacing*Cc("") +local nonempty=Cs((1-spacing)^1)*spacing^-1 +local content=(empty+nonempty)^1 +patterns.textline=content +local linesplitter=tsplitat(newline) +patterns.linesplitter=linesplitter +function string.splitlines(str) + return lpegmatch(linesplitter,str) +end +local cache={} +function lpeg.checkedsplit(separator,str) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +function string.checkedsplit(str,separator) + local c=cache[separator] + if not c then + separator=P(separator) + local other=C((1-separator)^1) + c=Ct(separator^0*other*(separator^1*other)^0) + cache[separator]=c + end + return lpegmatch(c,str) +end +local function f2(s) local c1,c2=byte(s,1,2) return c1*64+c2-12416 end +local function f3(s) local c1,c2,c3=byte(s,1,3) return (c1*64+c2)*64+c3-925824 end +local function f4(s) local c1,c2,c3,c4=byte(s,1,4) return ((c1*64+c2)*64+c3)*64+c4-63447168 end +local utf8byte=patterns.utf8one/byte+patterns.utf8two/f2+patterns.utf8three/f3+patterns.utf8four/f4 +patterns.utf8byte=utf8byte +local cache={} +function lpeg.stripper(str) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs(((S(str)^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs(((str^1)/""+1)^0) + end +end +local cache={} +function lpeg.keeper(str) + if type(str)=="string" then + local s=cache[str] + if not s then + s=Cs((((1-S(str))^1)/""+1)^0) + cache[str]=s + end + return s + else + return Cs((((1-str)^1)/""+1)^0) + end +end +function lpeg.frontstripper(str) + return (P(str)+P(true))*Cs(anything^0) +end +function lpeg.endstripper(str) + return Cs((1-P(str)*endofstring)^0) +end +function lpeg.replacer(one,two,makefunction,isutf) + local pattern + local u=isutf and utf8char or 1 + if type(one)=="table" then + local no=#one + local p=P(false) + if no==0 then + for k,v in next,one do + p=p+P(k)/v + end + pattern=Cs((p+u)^0) + elseif no==1 then + local o=one[1] + one,two=P(o[1]),o[2] + pattern=Cs((one/two+u)^0) + else + for i=1,no do + local o=one[i] + p=p+P(o[1])/o[2] + end + pattern=Cs((p+u)^0) + end + else + pattern=Cs((P(one)/(two or "")+u)^0) + end + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +function lpeg.finder(lst,makefunction,isutf) + local pattern + if type(lst)=="table" then + pattern=P(false) + if #lst==0 then + for k,v in next,lst do + pattern=pattern+P(k) + end + else + for i=1,#lst do + pattern=pattern+P(lst[i]) + end + end + else + pattern=P(lst) + end + if isutf then + pattern=((utf8char or 1)-pattern)^0*pattern + else + pattern=(1-pattern)^0*pattern + end + if makefunction then + return function(str) + return lpegmatch(pattern,str) + end + else + return pattern + end +end +local splitters_f,splitters_s={},{} +function lpeg.firstofsplit(separator) + local splitter=splitters_f[separator] + if not splitter then + local pattern=P(separator) + splitter=C((1-pattern)^0) + splitters_f[separator]=splitter + end + return splitter +end +function lpeg.secondofsplit(separator) + local splitter=splitters_s[separator] + if not splitter then + local pattern=P(separator) + splitter=(1-pattern)^0*pattern*C(anything^0) + splitters_s[separator]=splitter + end + return splitter +end +local splitters_s,splitters_p={},{} +function lpeg.beforesuffix(separator) + local splitter=splitters_s[separator] + if not splitter then + local pattern=P(separator) + splitter=C((1-pattern)^0)*pattern*endofstring + splitters_s[separator]=splitter + end + return splitter +end +function lpeg.afterprefix(separator) + local splitter=splitters_p[separator] + if not splitter then + local pattern=P(separator) + splitter=pattern*C(anything^0) + splitters_p[separator]=splitter + end + return splitter +end +function lpeg.balancer(left,right) + left,right=P(left),P(right) + return P { left*((1-left-right)+V(1))^0*right } +end +local nany=utf8char/"" +function lpeg.counter(pattern) + pattern=Cs((P(pattern)/" "+nany)^0) + return function(str) + return #lpegmatch(pattern,str) + end +end +utf=utf or (unicode and unicode.utf8) or {} +local utfcharacters=utf and utf.characters or string.utfcharacters +local utfgmatch=utf and utf.gmatch +local utfchar=utf and utf.char +lpeg.UP=lpeg.P +if utfcharacters then + function lpeg.US(str) + local p=P(false) + for uc in utfcharacters(str) do + p=p+P(uc) + end + return p + end +elseif utfgmatch then + function lpeg.US(str) + local p=P(false) + for uc in utfgmatch(str,".") do + p=p+P(uc) + end + return p + end +else + function lpeg.US(str) + local p=P(false) + local f=function(uc) + p=p+P(uc) + end + lpegmatch((utf8char/f)^0,str) + return p + end +end +local range=utf8byte*utf8byte+Cc(false) +function lpeg.UR(str,more) + local first,last + if type(str)=="number" then + first=str + last=more or first + else + first,last=lpegmatch(range,str) + if not last then + return P(str) + end + end + if first==last then + return P(str) + elseif utfchar and (last-first<8) then + local p=P(false) + for i=first,last do + p=p+P(utfchar(i)) + end + return p + else + local f=function(b) + return b>=first and b<=last + end + return utf8byte/f + end +end +function lpeg.is_lpeg(p) + return p and lpegtype(p)=="pattern" +end +function lpeg.oneof(list,...) + if type(list)~="table" then + list={ list,... } + end + local p=P(list[1]) + for l=2,#list do + p=p+P(list[l]) + end + return p +end +local sort=table.sort +local function copyindexed(old) + local new={} + for i=1,#old do + new[i]=old + end + return new +end +local function sortedkeys(tab) + local keys,s={},0 + for key,_ in next,tab do + s=s+1 + keys[s]=key + end + sort(keys) + return keys +end +function lpeg.append(list,pp,delayed,checked) + local p=pp + if #list>0 then + local keys=copyindexed(list) + sort(keys) + for i=#keys,1,-1 do + local k=keys[i] + if p then + p=P(k)+p + else + p=P(k) + end + end + elseif delayed then + local keys=sortedkeys(list) + if p then + for i=1,#keys,1 do + local k=keys[i] + local v=list[k] + p=P(k)/list+p + end + else + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)+p + else + p=P(k) + end + end + if p then + p=p/list + end + end + elseif checked then + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + if k==v then + p=P(k)+p + else + p=P(k)/v+p + end + else + if k==v then + p=P(k) + else + p=P(k)/v + end + end + end + else + local keys=sortedkeys(list) + for i=1,#keys do + local k=keys[i] + local v=list[k] + if p then + p=P(k)/v+p + else + p=P(k)/v + end + end + end + return p +end +local p_false=P(false) +local p_true=P(true) +local function make(t) + local function making(t) + local p=p_false + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then + else + p=p+P(k)*making(v) + end + end + end + if t[""] then + p=p+p_true + end + return p + end + local p=p_false + local keys=sortedkeys(t) + for i=1,#keys do + local k=keys[i] + if k~="" then + local v=t[k] + if v==true then + p=p+P(k)*p_true + elseif v==false then + else + p=p+P(k)*making(v) + end + end + end + return p +end +local function collapse(t,x) + if type(t)~="table" then + return t,x + else + local n=next(t) + if n==nil then + return t,x + elseif next(t,n)==nil then + local k=n + local v=t[k] + if type(v)=="table" then + return collapse(v,x..k) + else + return v,x..k + end + else + local tt={} + for k,v in next,t do + local vv,kk=collapse(v,k) + tt[kk]=vv + end + return tt,x + end + end +end +function lpeg.utfchartabletopattern(list) + local tree={} + local n=#list + if n==0 then + for s in next,list do + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true + end + end + else + for i=1,n do + local s=list[i] + local t=tree + local p,pk + for c in gmatch(s,".") do + if t==true then + t={ [c]=true,[""]=true } + p[pk]=t + p=t + t=false + elseif t==false then + t={ [c]=false } + p[pk]=t + p=t + t=false + else + local tc=t[c] + if not tc then + tc=false + t[c]=false + end + p=t + t=tc + end + pk=c + end + if t==false then + p[pk]=true + elseif t==true then + else + t[""]=true + end + end + end + return make(tree) +end +patterns.containseol=lpeg.finder(eol) +local function nextstep(n,step,result) + local m=n%step + local d=floor(n/step) + if d>0 then + local v=V(tostring(step)) + local s=result.start + for i=1,d do + if s then + s=v*s + else + s=v + end + end + result.start=s + end + if step>1 and result.start then + local v=V(tostring(step/2)) + result[tostring(step)]=v*v + end + if step>0 then + return nextstep(m,step/2,result) + else + return result + end +end +function lpeg.times(pattern,n) + return P(nextstep(n,2^16,{ "start",["1"]=pattern })) +end +local trailingzeros=zero^0*-digit +local case_1=period*trailingzeros/"" +local case_2=period*(digit-trailingzeros)^1*(trailingzeros/"") +local number=digit^1*(case_1+case_2) +local stripper=Cs((number+1)^0) +lpeg.patterns.stripzeros=stripper +local byte_to_HEX={} +local byte_to_hex={} +local byte_to_dec={} +local hex_to_byte={} +for i=0,255 do + local H=format("%02X",i) + local h=format("%02x",i) + local d=format("%03i",i) + local c=char(i) + byte_to_HEX[c]=H + byte_to_hex[c]=h + byte_to_dec[c]=d + hex_to_byte[h]=c + hex_to_byte[H]=c +end +local hextobyte=P(2)/hex_to_byte +local bytetoHEX=P(1)/byte_to_HEX +local bytetohex=P(1)/byte_to_hex +local bytetodec=P(1)/byte_to_dec +local hextobytes=Cs(hextobyte^0) +local bytestoHEX=Cs(bytetoHEX^0) +local bytestohex=Cs(bytetohex^0) +local bytestodec=Cs(bytetodec^0) +patterns.hextobyte=hextobyte +patterns.bytetoHEX=bytetoHEX +patterns.bytetohex=bytetohex +patterns.bytetodec=bytetodec +patterns.hextobytes=hextobytes +patterns.bytestoHEX=bytestoHEX +patterns.bytestohex=bytestohex +patterns.bytestodec=bytestodec +function string.toHEX(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestoHEX,s) + end +end +function string.tohex(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestohex,s) + end +end +function string.todec(s) + if not s or s=="" then + return s + else + return lpegmatch(bytestodec,s) + end +end +function string.tobytes(s) + if not s or s=="" then + return s + else + return lpegmatch(hextobytes,s) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-function"] = package.loaded["l-function"] or true + +-- original size: 361, stripped down to: 322 + +if not modules then modules={} end modules ['l-functions']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +functions=functions or {} +function functions.dummy() end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-string"] = package.loaded["l-string"] or true + +-- original size: 5694, stripped down to: 2827 + +if not modules then modules={} end modules ['l-string']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local string=string +local sub,gmatch,format,char,byte,rep,lower=string.sub,string.gmatch,string.format,string.char,string.byte,string.rep,string.lower +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local P,S,C,Ct,Cc,Cs=lpeg.P,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cc,lpeg.Cs +local unquoted=patterns.squote*C(patterns.nosquote)*patterns.squote+patterns.dquote*C(patterns.nodquote)*patterns.dquote +function string.unquoted(str) + return lpegmatch(unquoted,str) or str +end +function string.quoted(str) + return format("%q",str) +end +function string.count(str,pattern) + local n=0 + for _ in gmatch(str,pattern) do + n=n+1 + end + return n +end +function string.limit(str,n,sentinel) + if #str>n then + sentinel=sentinel or "..." + return sub(str,1,(n-#sentinel))..sentinel + else + return str + end +end +local stripper=patterns.stripper +local fullstripper=patterns.fullstripper +local collapser=patterns.collapser +local longtostring=patterns.longtostring +function string.strip(str) + return lpegmatch(stripper,str) or "" +end +function string.fullstrip(str) + return lpegmatch(fullstripper,str) or "" +end +function string.collapsespaces(str) + return lpegmatch(collapser,str) or "" +end +function string.longtostring(str) + return lpegmatch(longtostring,str) or "" +end +local pattern=P(" ")^0*P(-1) +function string.is_empty(str) + if str=="" then + return true + else + return lpegmatch(pattern,str) and true or false + end +end +local anything=patterns.anything +local allescapes=Cc("%")*S(".-+%?()[]*") +local someescapes=Cc("%")*S(".-+%()[]") +local matchescapes=Cc(".")*S("*?") +local pattern_a=Cs ((allescapes+anything )^0 ) +local pattern_b=Cs ((someescapes+matchescapes+anything )^0 ) +local pattern_c=Cs (Cc("^")*(someescapes+matchescapes+anything )^0*Cc("$") ) +function string.escapedpattern(str,simple) + return lpegmatch(simple and pattern_b or pattern_a,str) +end +function string.topattern(str,lowercase,strict) + if str=="" or type(str)~="string" then + return ".*" + elseif strict then + str=lpegmatch(pattern_c,str) + else + str=lpegmatch(pattern_b,str) + end + if lowercase then + return lower(str) + else + return str + end +end +function string.valid(str,default) + return (type(str)=="string" and str~="" and str) or default or nil +end +string.itself=function(s) return s end +local pattern=Ct(C(1)^0) +function string.totable(str) + return lpegmatch(pattern,str) +end +local replacer=lpeg.replacer("@","%%") +function string.tformat(fmt,...) + return format(lpegmatch(replacer,fmt),...) +end +string.quote=string.quoted +string.unquote=string.unquoted + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-table"] = package.loaded["l-table"] or true + +-- original size: 35724, stripped down to: 21525 + +if not modules then modules={} end modules ['l-table']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring,tonumber,ipairs,select=type,next,tostring,tonumber,ipairs,select +local table,string=table,string +local concat,sort,insert,remove=table.concat,table.sort,table.insert,table.remove +local format,lower,dump=string.format,string.lower,string.dump +local getmetatable,setmetatable=getmetatable,setmetatable +local getinfo=debug.getinfo +local lpegmatch,patterns=lpeg.match,lpeg.patterns +local floor=math.floor +local stripper=patterns.stripper +function table.strip(tab) + local lst,l={},0 + for i=1,#tab do + local s=lpegmatch(stripper,tab[i]) or "" + if s=="" then + else + l=l+1 + lst[l]=s + end + end + return lst +end +function table.keys(t) + if t then + local keys,k={},0 + for key in next,t do + k=k+1 + keys[k]=key + end + return keys + else + return {} + end +end +local function compare(a,b) + local ta=type(a) + if ta=="number" then + local tb=type(b) + if ta==tb then + return a<b + elseif tb=="string" then + return tostring(a)<b + end + elseif ta=="string" then + local tb=type(b) + if ta==tb then + return a<b + else + return a<tostring(b) + end + end + return tostring(a)<tostring(b) +end +local function sortedkeys(tab) + if tab then + local srt,category,s={},0,0 + for key in next,tab do + s=s+1 + srt[s]=key + if category==3 then + elseif category==1 then + if type(key)~="string" then + category=3 + end + elseif category==2 then + if type(key)~="number" then + category=3 + end + else + local tkey=type(key) + if tkey=="string" then + category=1 + elseif tkey=="number" then + category=2 + else + category=3 + end + end + end + if s<2 then + elseif category==3 then + sort(srt,compare) + else + sort(srt) + end + return srt + else + return {} + end +end +local function sortedhashonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="string" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end +local function sortedindexonly(tab) + if tab then + local srt,s={},0 + for key in next,tab do + if type(key)=="number" then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt) + end + return srt + else + return {} + end +end +local function sortedhashkeys(tab,cmp) + if tab then + local srt,s={},0 + for key in next,tab do + if key then + s=s+1 + srt[s]=key + end + end + if s>1 then + sort(srt,cmp) + end + return srt + else + return {} + end +end +function table.allkeys(t) + local keys={} + for k,v in next,t do + for k in next,v do + keys[k]=true + end + end + return sortedkeys(keys) +end +table.sortedkeys=sortedkeys +table.sortedhashonly=sortedhashonly +table.sortedindexonly=sortedindexonly +table.sortedhashkeys=sortedhashkeys +local function nothing() end +local function sortedhash(t,cmp) + if t then + local s + if cmp then + s=sortedhashkeys(t,function(a,b) return cmp(t,a,b) end) + else + s=sortedkeys(t) + end + local m=#s + if m==1 then + return next,t + elseif m>0 then + local n=0 + return function() + if n<m then + n=n+1 + local k=s[n] + return k,t[k] + end + end + end + end + return nothing +end +table.sortedhash=sortedhash +table.sortedpairs=sortedhash +function table.append(t,list) + local n=#t + for i=1,#list do + n=n+1 + t[n]=list[i] + end + return t +end +function table.prepend(t,list) + local nl=#list + local nt=nl+#t + for i=#t,1,-1 do + t[nt]=t[i] + nt=nt-1 + end + for i=1,#list do + t[i]=list[i] + end + return t +end +function table.merge(t,...) + t=t or {} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v + end + end + return t +end +function table.merged(...) + local t={} + for i=1,select("#",...) do + for k,v in next,(select(i,...)) do + t[k]=v + end + end + return t +end +function table.imerge(t,...) + local nt=#t + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + nt=nt+1 + t[nt]=nst[j] + end + end + return t +end +function table.imerged(...) + local tmp,ntmp={},0 + for i=1,select("#",...) do + local nst=select(i,...) + for j=1,#nst do + ntmp=ntmp+1 + tmp[ntmp]=nst[j] + end + end + return tmp +end +local function fastcopy(old,metatabletoo) + if old then + local new={} + for k,v in next,old do + if type(v)=="table" then + new[k]=fastcopy(v,metatabletoo) + else + new[k]=v + end + end + if metatabletoo then + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) + end + end + return new + else + return {} + end +end +local function copy(t,tables) + tables=tables or {} + local tcopy={} + if not tables[t] then + tables[t]=tcopy + end + for i,v in next,t do + if type(i)=="table" then + if tables[i] then + i=tables[i] + else + i=copy(i,tables) + end + end + if type(v)~="table" then + tcopy[i]=v + elseif tables[v] then + tcopy[i]=tables[v] + else + tcopy[i]=copy(v,tables) + end + end + local mt=getmetatable(t) + if mt then + setmetatable(tcopy,mt) + end + return tcopy +end +table.fastcopy=fastcopy +table.copy=copy +function table.derive(parent) + local child={} + if parent then + setmetatable(child,{ __index=parent }) + end + return child +end +function table.tohash(t,value) + local h={} + if t then + if value==nil then value=true end + for _,v in next,t do + h[v]=value + end + end + return h +end +function table.fromhash(t) + local hsh,h={},0 + for k,v in next,t do + if v then + h=h+1 + hsh[h]=k + end + end + return hsh +end +local noquotes,hexify,handle,compact,inline,functions +local reserved=table.tohash { + 'and','break','do','else','elseif','end','false','for','function','if', + 'in','local','nil','not','or','repeat','return','then','true','until','while', + 'NaN','goto', +} +local function simple_table(t) + local nt=#t + if nt>0 then + local n=0 + for _,v in next,t do + n=n+1 + end + if n==nt then + local tt={} + for i=1,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + if hexify then + tt[i]=format("0x%X",v) + else + tt[i]=tostring(v) + end + elseif tv=="string" then + tt[i]=format("%q",v) + elseif tv=="boolean" then + tt[i]=v and "true" or "false" + else + return nil + end + end + return tt + end + end + return nil +end +local propername=patterns.propername +local function dummy() end +local function do_serialize(root,name,depth,level,indexed) + if level>0 then + depth=depth.." " + if indexed then + handle(format("%s{",depth)) + else + local tn=type(name) + if tn=="number" then + if hexify then + handle(format("%s[0x%X]={",depth,name)) + else + handle(format("%s[%s]={",depth,name)) + end + elseif tn=="string" then + if noquotes and not reserved[name] and lpegmatch(propername,name) then + handle(format("%s%s={",depth,name)) + else + handle(format("%s[%q]={",depth,name)) + end + elseif tn=="boolean" then + handle(format("%s[%s]={",depth,name and "true" or "false")) + else + handle(format("%s{",depth)) + end + end + end + if root and next(root)~=nil then + local first,last=nil,0 + if compact then + last=#root + for k=1,last do + if root[k]==nil then + last=k-1 + break + end + end + if last>0 then + first=1 + end + end + local sk=sortedkeys(root) + for i=1,#sk do + local k=sk[i] + local v=root[k] + local tv=type(v) + local tk=type(k) + if compact and first and tk=="number" and k>=first and k<=last then + if tv=="number" then + if hexify then + handle(format("%s 0x%X,",depth,v)) + else + handle(format("%s %s,",depth,v)) + end + elseif tv=="string" then + handle(format("%s %q,",depth,v)) + elseif tv=="table" then + if next(v)==nil then + handle(format("%s {},",depth)) + elseif inline then + local st=simple_table(v) + if st then + handle(format("%s { %s },",depth,concat(st,", "))) + else + do_serialize(v,k,depth,level+1,true) + end + else + do_serialize(v,k,depth,level+1,true) + end + elseif tv=="boolean" then + handle(format("%s %s,",depth,v and "true" or "false")) + elseif tv=="function" then + if functions then + handle(format('%s load(%q),',depth,dump(v))) + else + handle(format('%s "function",',depth)) + end + else + handle(format("%s %q,",depth,tostring(v))) + end + elseif k=="__p__" then + if false then + handle(format("%s __p__=nil,",depth)) + end + elseif tv=="number" then + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=0x%X,",depth,k,v)) + else + handle(format("%s [%s]=%s,",depth,k,v)) + end + elseif tk=="boolean" then + if hexify then + handle(format("%s [%s]=0x%X,",depth,k and "true" or "false",v)) + else + handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) + end + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + if hexify then + handle(format("%s %s=0x%X,",depth,k,v)) + else + handle(format("%s %s=%s,",depth,k,v)) + end + else + if hexify then + handle(format("%s [%q]=0x%X,",depth,k,v)) + else + handle(format("%s [%q]=%s,",depth,k,v)) + end + end + elseif tv=="string" then + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=%q,",depth,k,v)) + else + handle(format("%s [%s]=%q,",depth,k,v)) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,v)) + else + handle(format("%s [%q]=%q,",depth,k,v)) + end + elseif tv=="table" then + if next(v)==nil then + if tk=="number" then + if hexify then + handle(format("%s [0x%X]={},",depth,k)) + else + handle(format("%s [%s]={},",depth,k)) + end + elseif tk=="boolean" then + handle(format("%s [%s]={},",depth,k and "true" or "false")) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={},",depth,k)) + else + handle(format("%s [%q]={},",depth,k)) + end + elseif inline then + local st=simple_table(v) + if st then + if tk=="number" then + if hexify then + handle(format("%s [0x%X]={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%s]={ %s },",depth,k,concat(st,", "))) + end + elseif tk=="boolean" then + handle(format("%s [%s]={ %s },",depth,k and "true" or "false",concat(st,", "))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s={ %s },",depth,k,concat(st,", "))) + else + handle(format("%s [%q]={ %s },",depth,k,concat(st,", "))) + end + else + do_serialize(v,k,depth,level+1) + end + else + do_serialize(v,k,depth,level+1) + end + elseif tv=="boolean" then + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=%s,",depth,k,v and "true" or "false")) + else + handle(format("%s [%s]=%s,",depth,k,v and "true" or "false")) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%s,",depth,tostring(k),v and "true" or "false")) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%s,",depth,k,v and "true" or "false")) + else + handle(format("%s [%q]=%s,",depth,k,v and "true" or "false")) + end + elseif tv=="function" then + if functions then + local f=getinfo(v).what=="C" and dump(dummy) or dump(v) + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=load(%q),",depth,k,f)) + else + handle(format("%s [%s]=load(%q),",depth,k,f)) + end + elseif tk=="boolean" then + handle(format("%s [%s]=load(%q),",depth,k and "true" or "false",f)) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=load(%q),",depth,k,f)) + else + handle(format("%s [%q]=load(%q),",depth,k,f)) + end + end + else + if tk=="number" then + if hexify then + handle(format("%s [0x%X]=%q,",depth,k,tostring(v))) + else + handle(format("%s [%s]=%q,",depth,k,tostring(v))) + end + elseif tk=="boolean" then + handle(format("%s [%s]=%q,",depth,k and "true" or "false",tostring(v))) + elseif noquotes and not reserved[k] and lpegmatch(propername,k) then + handle(format("%s %s=%q,",depth,k,tostring(v))) + else + handle(format("%s [%q]=%q,",depth,k,tostring(v))) + end + end + end + end + if level>0 then + handle(format("%s},",depth)) + end +end +local function serialize(_handle,root,name,specification) + local tname=type(name) + if type(specification)=="table" then + noquotes=specification.noquotes + hexify=specification.hexify + handle=_handle or specification.handle or print + functions=specification.functions + compact=specification.compact + inline=specification.inline and compact + if functions==nil then + functions=true + end + if compact==nil then + compact=true + end + if inline==nil then + inline=compact + end + else + noquotes=false + hexify=false + handle=_handle or print + compact=true + inline=true + functions=true + end + if tname=="string" then + if name=="return" then + handle("return {") + else + handle(name.."={") + end + elseif tname=="number" then + if hexify then + handle(format("[0x%X]={",name)) + else + handle("["..name.."]={") + end + elseif tname=="boolean" then + if name then + handle("return {") + else + handle("{") + end + else + handle("t={") + end + if root then + if getmetatable(root) then + local dummy=root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_=nil + end + if next(root)~=nil then + do_serialize(root,name,"",0) + end + end + handle("}") +end +function table.serialize(root,name,specification) + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + end + serialize(flush,root,name,specification) + return concat(t,"\n") +end +table.tohandle=serialize +local maxtab=2*1024 +function table.tofile(filename,root,name,specification) + local f=io.open(filename,'w') + if f then + if maxtab>1 then + local t,n={},0 + local function flush(s) + n=n+1 + t[n]=s + if n>maxtab then + f:write(concat(t,"\n"),"\n") + t,n={},0 + end + end + serialize(flush,root,name,specification) + f:write(concat(t,"\n"),"\n") + else + local function flush(s) + f:write(s,"\n") + end + serialize(flush,root,name,specification) + end + f:close() + io.flush() + end +end +local function flattened(t,f,depth) + if f==nil then + f={} + depth=0xFFFF + elseif tonumber(f) then + depth=f + f={} + elseif not depth then + depth=0xFFFF + end + for k,v in next,t do + if type(k)~="number" then + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + else + f[#f+1]=v + end + end + end + for k=1,#t do + local v=t[k] + if depth>0 and type(v)=="table" then + flattened(v,f,depth-1) + else + f[#f+1]=v + end + end + return f +end +table.flattened=flattened +local function unnest(t,f) + if not f then + f={} + end + for i=1,#t do + local v=t[i] + if type(v)=="table" then + if type(v[1])=="table" then + unnest(v,f) + else + f[#f+1]=v + end + else + f[#f+1]=v + end + end + return f +end +function table.unnest(t) + return unnest(t) +end +local function are_equal(a,b,n,m) + if a and b and #a==#b then + n=n or 1 + m=m or #a + for i=n,m do + local ai,bi=a[i],b[i] + if ai==bi then + elseif type(ai)=="table" and type(bi)=="table" then + if not are_equal(ai,bi) then + return false + end + else + return false + end + end + return true + else + return false + end +end +local function identical(a,b) + for ka,va in next,a do + local vb=b[ka] + if va==vb then + elseif type(va)=="table" and type(vb)=="table" then + if not identical(va,vb) then + return false + end + else + return false + end + end + return true +end +table.identical=identical +table.are_equal=are_equal +local function sparse(old,nest,keeptables) + local new={} + for k,v in next,old do + if not (v=="" or v==false) then + if nest and type(v)=="table" then + v=sparse(v,nest) + if keeptables or next(v)~=nil then + new[k]=v + end + else + new[k]=v + end + end + end + return new +end +table.sparse=sparse +function table.compact(t) + return sparse(t,true,true) +end +function table.contains(t,v) + if t then + for i=1,#t do + if t[i]==v then + return i + end + end + end + return false +end +function table.count(t) + local n=0 + for k,v in next,t do + n=n+1 + end + return n +end +function table.swapped(t,s) + local n={} + if s then + for k,v in next,s do + n[k]=v + end + end + for k,v in next,t do + n[v]=k + end + return n +end +function table.mirrored(t) + local n={} + for k,v in next,t do + n[v]=k + n[k]=v + end + return n +end +function table.reversed(t) + if t then + local tt,tn={},#t + if tn>0 then + local ttn=0 + for i=tn,1,-1 do + ttn=ttn+1 + tt[ttn]=t[i] + end + end + return tt + end +end +function table.reverse(t) + if t then + local n=#t + for i=1,floor(n/2) do + local j=n-i+1 + t[i],t[j]=t[j],t[i] + end + return t + end +end +function table.sequenced(t,sep,simple) + if not t then + return "" + end + local n=#t + local s={} + if n>0 then + for i=1,n do + s[i]=tostring(t[i]) + end + else + n=0 + for k,v in sortedhash(t) do + if simple then + if v==true then + n=n+1 + s[n]=k + elseif v and v~="" then + n=n+1 + s[n]=k.."="..tostring(v) + end + else + n=n+1 + s[n]=k.."="..tostring(v) + end + end + end + return concat(s,sep or " | ") +end +function table.print(t,...) + if type(t)~="table" then + print(tostring(t)) + else + serialize(print,t,...) + end +end +if setinspector then + setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) +end +function table.sub(t,i,j) + return { unpack(t,i,j) } +end +function table.is_empty(t) + return not t or next(t)==nil +end +function table.has_one_entry(t) + return t and next(t,next(t))==nil +end +function table.loweredkeys(t) + local l={} + for k,v in next,t do + l[lower(k)]=v + end + return l +end +function table.unique(old) + local hash={} + local new={} + local n=0 + for i=1,#old do + local oi=old[i] + if not hash[oi] then + n=n+1 + new[n]=oi + hash[oi]=true + end + end + return new +end +function table.sorted(t,...) + sort(t,...) + return t +end +function table.values(t,s) + if t then + local values,keys,v={},{},0 + for key,value in next,t do + if not keys[value] then + v=v+1 + values[v]=value + keys[k]=key + end + end + if s then + sort(values) + end + return values + else + return {} + end +end +function table.filtered(t,pattern,sort,cmp) + if t and type(pattern)=="string" then + if sort then + local s + if cmp then + s=sortedhashkeys(t,function(a,b) return cmp(t,a,b) end) + else + s=sortedkeys(t) + end + local n=0 + local m=#s + local function kv(s) + while n<m do + n=n+1 + local k=s[n] + if find(k,pattern) then + return k,t[k] + end + end + end + return kv,s + else + local n=next(t) + local function iterator() + while n~=nil do + local k=n + n=next(t,k) + if find(k,pattern) then + return k,t[k] + end + end + end + return iterator,t + end + else + return nothing + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-io"] = package.loaded["l-io"] or true + +-- original size: 8643, stripped down to: 6232 + +if not modules then modules={} end modules ['l-io']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local io=io +local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format +local concat=table.concat +local floor=math.floor +local type=type +if string.find(os.getenv("PATH"),";",1,true) then + io.fileseparator,io.pathseparator="\\",";" +else + io.fileseparator,io.pathseparator="/",":" +end +local function readall(f) + return f:read("*all") +end +local function readall(f) + local size=f:seek("end") + if size==0 then + return "" + elseif size<1024*1024 then + f:seek("set",0) + return f:read('*all') + else + local done=f:seek("set",0) + local step + if size<1024*1024 then + step=1024*1024 + elseif size>16*1024*1024 then + step=16*1024*1024 + else + step=floor(size/(1024*1024))*1024*1024/8 + end + local data={} + while true do + local r=f:read(step) + if not r then + return concat(data) + else + data[#data+1]=r + end + end + end +end +io.readall=readall +function io.loaddata(filename,textmode) + local f=io.open(filename,(textmode and 'r') or 'rb') + if f then + local data=readall(f) + f:close() + if #data>0 then + return data + end + end +end +function io.savedata(filename,data,joiner) + local f=io.open(filename,"wb") + if f then + if type(data)=="table" then + f:write(concat(data,joiner or "")) + elseif type(data)=="function" then + data(f) + else + f:write(data or "") + end + f:close() + io.flush() + return true + else + return false + end +end +function io.loadlines(filename,n) + local f=io.open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[#lines+1]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end + end +end +function io.loadchunk(filename,n) + local f=io.open(filename,'rb') + if f then + local data=f:read(n or 1024) + f:close() + if #data>0 then + return data + end + end +end +function io.exists(filename) + local f=io.open(filename) + if f==nil then + return false + else + f:close() + return true + end +end +function io.size(filename) + local f=io.open(filename) + if f==nil then + return 0 + else + local s=f:seek("end") + f:close() + return s + end +end +function io.noflines(f) + if type(f)=="string" then + local f=io.open(filename) + if f then + local n=f and io.noflines(f) or 0 + f:close() + return n + else + return 0 + end + else + local n=0 + for _ in f:lines() do + n=n+1 + end + f:seek('set',0) + return n + end +end +local nextchar={ + [ 4]=function(f) + return f:read(1,1,1,1) + end, + [ 2]=function(f) + return f:read(1,1) + end, + [ 1]=function(f) + return f:read(1) + end, + [-2]=function(f) + local a,b=f:read(1,1) + return b,a + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + return d,c,b,a + end +} +function io.characters(f,n) + if f then + return nextchar[n or 1],f + end +end +local nextbyte={ + [4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(a),byte(b),byte(c),byte(d) + end + end, + [3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(a),byte(b),byte(c) + end + end, + [2]=function(f) + local a,b=f:read(1,1) + if b then + return byte(a),byte(b) + end + end, + [1]=function (f) + local a=f:read(1) + if a then + return byte(a) + end + end, + [-2]=function (f) + local a,b=f:read(1,1) + if b then + return byte(b),byte(a) + end + end, + [-3]=function(f) + local a,b,c=f:read(1,1,1) + if b then + return byte(c),byte(b),byte(a) + end + end, + [-4]=function(f) + local a,b,c,d=f:read(1,1,1,1) + if d then + return byte(d),byte(c),byte(b),byte(a) + end + end +} +function io.bytes(f,n) + if f then + return nextbyte[n or 1],f + else + return nil,nil + end +end +function io.ask(question,default,options) + while true do + io.write(question) + if options then + io.write(format(" [%s]",concat(options,"|"))) + end + if default then + io.write(format(" [%s]",default)) + end + io.write(format(" ")) + io.flush() + local answer=io.read() + answer=gsub(answer,"^%s*(.*)%s*$","%1") + if answer=="" and default then + return default + elseif not options then + return answer + else + for k=1,#options do + if options[k]==answer then + return answer + end + end + local pattern="^"..answer + for k=1,#options do + local v=options[k] + if find(v,pattern) then + return v + end + end + end + end +end +local function readnumber(f,n,m) + if m then + f:seek("set",n) + n=m + end + if n==1 then + return byte(f:read(1)) + elseif n==2 then + local a,b=byte(f:read(2),1,2) + return 256*a+b + elseif n==3 then + local a,b,c=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==4 then + local a,b,c,d=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==8 then + local a,b=readnumber(f,4),readnumber(f,4) + return 256*a+b + elseif n==12 then + local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) + return 256*256*a+256*b+c + elseif n==-2 then + local b,a=byte(f:read(2),1,2) + return 256*a+b + elseif n==-3 then + local c,b,a=byte(f:read(3),1,3) + return 256*256*a+256*b+c + elseif n==-4 then + local d,c,b,a=byte(f:read(4),1,4) + return 256*256*256*a+256*256*b+256*c+d + elseif n==-8 then + local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) + return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + else + return 0 + end +end +io.readnumber=readnumber +function io.readstring(f,n,m) + if m then + f:seek("set",n) + n=m + end + local str=gsub(f:read(n),"\000","") + return str +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-number"] = package.loaded["l-number"] or true + +-- original size: 4939, stripped down to: 2830 + +if not modules then modules={} end modules ['l-number']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local tostring,tonumber=tostring,tonumber +local format,floor,match,rep=string.format,math.floor,string.match,string.rep +local concat,insert=table.concat,table.insert +local lpegmatch=lpeg.match +number=number or {} +local number=number +if bit32 then + local btest,bor=bit32.btest,bit32.bor + function number.bit(p) + return 2^(p-1) + end + number.hasbit=btest + number.setbit=bor + function number.setbit(x,p) + return btest(x,p) and x or x+p + end + function number.clearbit(x,p) + return btest(x,p) and x-p or x + end +else + function number.bit(p) + return 2^(p-1) + end + function number.hasbit(x,p) + return x%(p+p)>=p + end + function number.setbit(x,p) + return (x%(p+p)>=p) and x or x+p + end + function number.clearbit(x,p) + return (x%(p+p)>=p) and x-p or x + end +end +if bit32 then + local bextract=bit32.extract + local t={ + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + "0","0","0","0","0","0","0","0", + } + function number.tobitstring(b,m) + local n=32 + for i=0,31 do + local v=bextract(b,i) + local k=32-i + if v==1 then + n=k + t[k]="1" + else + t[k]="0" + end + end + if m then + m=33-m*8 + if m<1 then + m=1 + end + return concat(t,"",m) + elseif n<8 then + return concat(t) + elseif n<16 then + return concat(t,"",9) + elseif n<24 then + return concat(t,"",17) + else + return concat(t,"",25) + end + end +else + function number.tobitstring(n,m) + if n>0 then + local t={} + while n>0 do + insert(t,1,n%2>0 and 1 or 0) + n=floor(n/2) + end + local nn=8-#t%8 + if nn>0 and nn<8 then + for i=1,nn do + insert(t,1,0) + end + end + if m then + m=m*8-#t + if m>0 then + insert(t,1,rep("0",m)) + end + end + return concat(t) + elseif m then + rep("00000000",m) + else + return "00000000" + end + end +end +function number.valid(str,default) + return tonumber(str) or default or nil +end +function number.toevenhex(n) + local s=format("%X",n) + if #s%2==0 then + return s + else + return "0"..s + end +end +local one=lpeg.C(1-lpeg.S('')/tonumber)^1 +function number.toset(n) + return lpegmatch(one,tostring(n)) +end +local function bits(n,i,...) + if n>0 then + local m=n%2 + local n=floor(n/2) + if m>0 then + return bits(n,i+1,i,...) + else + return bits(n,i+1,...) + end + else + return... + end +end +function number.bits(n) + return { bits(n,1) } +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-set"] = package.loaded["l-set"] or true + +-- original size: 1923, stripped down to: 1133 + +if not modules then modules={} end modules ['l-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +set=set or {} +local nums={} +local tabs={} +local concat=table.concat +local next,type=next,type +set.create=table.tohash +function set.tonumber(t) + if next(t) then + local s="" + for k,v in next,t do + if v then + s=s.." "..k + end + end + local n=nums[s] + if not n then + n=#tabs+1 + tabs[n]=t + nums[s]=n + end + return n + else + return 0 + end +end +function set.totable(n) + if n==0 then + return {} + else + return tabs[n] or {} + end +end +function set.tolist(n) + if n==0 or not tabs[n] then + return "" + else + local t,n={},0 + for k,v in next,tabs[n] do + if v then + n=n+1 + t[n]=k + end + end + return concat(t," ") + end +end +function set.contains(n,s) + if type(n)=="table" then + return n[s] + elseif n==0 then + return false + else + local t=tabs[n] + return t and t[s] + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-os"] = package.loaded["l-os"] or true + +-- original size: 15832, stripped down to: 9456 + +if not modules then modules={} end modules ['l-os']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local os=os +local date,time=os.date,os.time +local find,format,gsub,upper,gmatch=string.find,string.format,string.gsub,string.upper,string.gmatch +local concat=table.concat +local random,ceil,randomseed=math.random,math.ceil,math.randomseed +local rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring=rawget,rawset,type,getmetatable,setmetatable,tonumber,tostring +math.initialseed=tonumber(string.sub(string.reverse(tostring(ceil(socket and socket.gettime()*10000 or time()))),1,6)) +randomseed(math.initialseed) +if not os.__getenv__ then + os.__getenv__=os.getenv + os.__setenv__=os.setenv + if os.env then + local osgetenv=os.getenv + local ossetenv=os.setenv + local osenv=os.env local _=osenv.PATH + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + if type(v)=="table" then + v=concat(v,";") + end + ossetenv(K,v) + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osenv[k] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + else + local ossetenv=os.setenv + local osgetenv=os.getenv + local osenv={} + function os.setenv(k,v) + if v==nil then + v="" + end + local K=upper(k) + osenv[K]=v + end + function os.getenv(k) + local K=upper(k) + local v=osenv[K] or osgetenv(K) or osgetenv(k) + if v=="" then + return nil + else + return v + end + end + local function __index(t,k) + return os.getenv(k) + end + local function __newindex(t,k,v) + os.setenv(k,v) + end + os.env={} + setmetatable(os.env,{ __index=__index,__newindex=__newindex } ) + end +end +local execute=os.execute +local iopopen=io.popen +function os.resultof(command) + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + else + return "" + end +end +if not io.fileseparator then + if find(os.getenv("PATH"),";",1,true) then + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + else + io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" + end +end +os.type=os.type or (io.pathseparator==";" and "windows") or "unix" +os.name=os.name or (os.type=="windows" and "mswin" ) or "linux" +if os.type=="windows" then + os.libsuffix,os.binsuffix,os.binsuffixes='dll','exe',{ 'exe','cmd','bat' } +else + os.libsuffix,os.binsuffix,os.binsuffixes='so','',{ '' } +end +local launchers={ + windows="start %s", + macosx="open %s", + unix="$BROWSER %s &> /dev/null &", +} +function os.launch(str) + execute(format(launchers[os.name] or launchers.unix,str)) +end +if not os.times then + function os.times() + return { + utime=os.gettimeofday(), + stime=0, + cutime=0, + cstime=0, + } + end +end +local gettimeofday=os.gettimeofday or os.clock +os.gettimeofday=gettimeofday +local startuptime=gettimeofday() +function os.runtime() + return gettimeofday()-startuptime +end +local resolvers=os.resolvers or {} +os.resolvers=resolvers +setmetatable(os,{ __index=function(t,k) + local r=resolvers[k] + return r and r(t,k) or nil +end }) +local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" +local function guess() + local architecture=os.resultof("uname -m") or "" + if architecture~="" then + return architecture + end + architecture=os.getenv("HOSTTYPE") or "" + if architecture~="" then + return architecture + end + return os.resultof("echo $HOSTTYPE") or "" +end +if platform~="" then + os.platform=platform +elseif os.type=="windows" then + function resolvers.platform(t,k) + local platform,architecture="",os.getenv("PROCESSOR_ARCHITECTURE") or "" + if find(architecture,"AMD64",1,true) then + platform="win64" + else + platform="mswin" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="linux" then + function resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64",1,true) then + platform="linux-64" + elseif find(architecture,"ppc",1,true) then + platform="linux-ppc" + else + platform="linux" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="macosx" then + function resolvers.platform(t,k) + local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + if architecture=="" then + platform="osx-intel" + elseif find(architecture,"i386",1,true) then + platform="osx-intel" + elseif find(architecture,"x86_64",1,true) then + platform="osx-64" + else + platform="osx-ppc" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="sunos" then + function resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"sparc",1,true) then + platform="solaris-sparc" + else + platform="solaris-intel" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="freebsd" then + function resolvers.platform(t,k) + local platform,architecture="",os.resultof("uname -m") or "" + if find(architecture,"amd64",1,true) then + platform="freebsd-amd64" + else + platform="freebsd" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +elseif name=="kfreebsd" then + function resolvers.platform(t,k) + local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + if find(architecture,"x86_64",1,true) then + platform="kfreebsd-amd64" + else + platform="kfreebsd-i386" + end + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +else + function resolvers.platform(t,k) + local platform="linux" + os.setenv("MTX_PLATFORM",platform) + os.platform=platform + return platform + end +end +os.newline=name=="windows" and "\013\010" or "\010" +function resolvers.bits(t,k) + local bits=find(os.platform,"64",1,true) and 64 or 32 + os.bits=bits + return bits +end +local t={ 8,9,"a","b" } +function os.uuid() + return format("%04x%04x-4%03x-%s%03x-%04x-%04x%04x%04x", + random(0xFFFF),random(0xFFFF), + random(0x0FFF), + t[ceil(random(4))] or 8,random(0x0FFF), + random(0xFFFF), + random(0xFFFF),random(0xFFFF),random(0xFFFF) + ) +end +local d +function os.timezone(delta) + d=d or tonumber(tonumber(date("%H")-date("!%H"))) + if delta then + if d>0 then + return format("+%02i:00",d) + else + return format("-%02i:00",-d) + end + else + return 1 + end +end +local timeformat=format("%%s%s",os.timezone(true)) +local dateformat="!%Y-%m-%d %H:%M:%S" +local lasttime=nil +local lastdate=nil +function os.fulltime(t,default) + t=t and tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=time() + end + if t~=lasttime then + lasttime=t + lastdate=format(timeformat,date(dateformat)) + end + return lastdate +end +local dateformat="%Y-%m-%d %H:%M:%S" +local lasttime=nil +local lastdate=nil +function os.localtime(t,default) + t=t and tonumber(t) or 0 + if t>0 then + elseif default then + return default + else + t=time() + end + if t~=lasttime then + lasttime=t + lastdate=date(dateformat,t) + end + return lastdate +end +function os.converttime(t,default) + local t=tonumber(t) + if t and t>0 then + return date(dateformat,t) + else + return default or "-" + end +end +local memory={} +local function which(filename) + local fullname=memory[filename] + if fullname==nil then + local suffix=file.suffix(filename) + local suffixes=suffix=="" and os.binsuffixes or { suffix } + for directory in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + local df=file.join(directory,filename) + for i=1,#suffixes do + local dfs=file.addsuffix(df,suffixes[i]) + if io.exists(dfs) then + fullname=dfs + break + end + end + end + if not fullname then + fullname=false + end + memory[filename]=fullname + end + return fullname +end +os.which=which +os.where=which +function os.today() + return date("!*t") +end +function os.now() + return date("!%Y-%m-%d %H:%M:%S") +end +if not os.sleep then + local socket=socket + function os.sleep(n) + if not socket then + socket=require("socket") + end + socket.sleep(n) + end +end +local function isleapyear(year) + return (year%400==0) or ((year%100~=0) and (year%4==0)) +end +os.isleapyear=isleapyear +local days={ 31,28,31,30,31,30,31,31,30,31,30,31 } +local function nofdays(year,month) + if not month then + return isleapyear(year) and 365 or 364 + else + return month==2 and isleapyear(year) and 29 or days[month] + end +end +os.nofdays=nofdays +function os.weekday(day,month,year) + return date("%w",time { year=year,month=month,day=day })+1 +end +function os.validdate(year,month,day) + if month<1 then + month=1 + elseif month>12 then + month=12 + end + if day<1 then + day=1 + else + local max=nofdays(year,month) + if day>max then + day=max + end + end + return year,month,day +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-file"] = package.loaded["l-file"] or true + +-- original size: 20949, stripped down to: 9945 + +if not modules then modules={} end modules ['l-file']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +file=file or {} +local file=file +if not lfs then + lfs=optionalrequire("lfs") +end +local insert,concat=table.insert,table.concat +local match,find,gmatch=string.match,string.find,string.gmatch +local lpegmatch=lpeg.match +local getcurrentdir,attributes=lfs.currentdir,lfs.attributes +local checkedsplit=string.checkedsplit +local P,R,S,C,Cs,Cp,Cc,Ct=lpeg.P,lpeg.R,lpeg.S,lpeg.C,lpeg.Cs,lpeg.Cp,lpeg.Cc,lpeg.Ct +local tricky=S("/\\")*P(-1) +local attributes=lfs.attributes +if sandbox then + sandbox.redefine(lfs.isfile,"lfs.isfile") + sandbox.redefine(lfs.isdir,"lfs.isdir") +end +function lfs.isdir(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end +end +function lfs.isfile(name) + return attributes(name,"mode")=="file" +end +local colon=P(":") +local period=P(".") +local periods=P("..") +local fwslash=P("/") +local bwslash=P("\\") +local slashes=S("\\/") +local noperiod=1-period +local noslashes=1-slashes +local name=noperiod^1 +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=C((1-(slashes^1*noslashes^1*-1))^1)*P(1) +local function pathpart(name,default) + return name and lpegmatch(pattern,name) or default or "" +end +local pattern=(noslashes^0*slashes)^1*C(noslashes^1)*-1 +local function basename(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes^1)^0*Cs((1-suffix)^1)*suffix^0 +local function nameonly(name) + return name and lpegmatch(pattern,name) or name +end +local pattern=(noslashes^0*slashes)^0*(noperiod^1*period)^1*C(noperiod^1)*-1 +local function suffixonly(name) + return name and lpegmatch(pattern,name) or "" +end +local pattern=(noslashes^0*slashes)^0*noperiod^1*((period*C(noperiod^1))^1)*-1+Cc("") +local function suffixesonly(name) + if name then + return lpegmatch(pattern,name) + else + return "" + end +end +file.pathpart=pathpart +file.basename=basename +file.nameonly=nameonly +file.suffixonly=suffixonly +file.suffix=suffixonly +file.suffixesonly=suffixesonly +file.suffixes=suffixesonly +file.dirname=pathpart +file.extname=suffixonly +local drive=C(R("az","AZ"))*colon +local path=C((noslashes^0*slashes)^0) +local suffix=period*C(P(1-period)^0*P(-1)) +local base=C((1-suffix)^0) +local rest=C(P(1)^0) +drive=drive+Cc("") +path=path+Cc("") +base=base+Cc("") +suffix=suffix+Cc("") +local pattern_a=drive*path*base*suffix +local pattern_b=path*base*suffix +local pattern_c=C(drive*path)*C(base*suffix) +local pattern_d=path*rest +function file.splitname(str,splitdrive) + if not str then + elseif splitdrive then + return lpegmatch(pattern_a,str) + else + return lpegmatch(pattern_b,str) + end +end +function file.splitbase(str) + if str then + return lpegmatch(pattern_d,str) + else + return "",str + end +end +function file.nametotable(str,splitdrive) + if str then + local path,drive,subpath,name,base,suffix=lpegmatch(pattern_c,str) + if splitdrive then + return { + path=path, + drive=drive, + subpath=subpath, + name=name, + base=base, + suffix=suffix, + } + else + return { + path=path, + name=name, + base=base, + suffix=suffix, + } + end + end +end +local pattern=Cs(((period*(1-period-slashes)^1*-1)/""+1)^1) +function file.removesuffix(name) + return name and lpegmatch(pattern,name) +end +local suffix=period/""*(1-period-slashes)^1*-1 +local pattern=Cs((noslashes^0*slashes^1)^0*((1-suffix)^1))*Cs(suffix) +function file.addsuffix(filename,suffix,criterium) + if not filename or not suffix or suffix=="" then + return filename + elseif criterium==true then + return filename.."."..suffix + elseif not criterium then + local n,s=lpegmatch(pattern,filename) + if not s or s=="" then + return filename.."."..suffix + else + return filename + end + else + local n,s=lpegmatch(pattern,filename) + if s and s~="" then + local t=type(criterium) + if t=="table" then + for i=1,#criterium do + if s==criterium[i] then + return filename + end + end + elseif t=="string" then + if s==criterium then + return filename + end + end + end + return (n or filename).."."..suffix + end +end +local suffix=period*(1-period-slashes)^1*-1 +local pattern=Cs((1-suffix)^0) +function file.replacesuffix(name,suffix) + if name and suffix and suffix~="" then + return lpegmatch(pattern,name).."."..suffix + else + return name + end +end +local reslasher=lpeg.replacer(P("\\"),"/") +function file.reslash(str) + return str and lpegmatch(reslasher,str) +end +function file.is_writable(name) + if not name then + elseif lfs.isdir(name) then + name=name.."/m_t_x_t_e_s_t.tmp" + local f=io.open(name,"wb") + if f then + f:close() + os.remove(name) + return true + end + elseif lfs.isfile(name) then + local f=io.open(name,"ab") + if f then + f:close() + return true + end + else + local f=io.open(name,"ab") + if f then + f:close() + os.remove(name) + return true + end + end + return false +end +local readable=P("r")*Cc(true) +function file.is_readable(name) + if name then + local a=attributes(name) + return a and lpegmatch(readable,a.permissions) or false + else + return false + end +end +file.isreadable=file.is_readable +file.iswritable=file.is_writable +function file.size(name) + if name then + local a=attributes(name) + return a and a.size or 0 + else + return 0 + end +end +function file.splitpath(str,separator) + return str and checkedsplit(lpegmatch(reslasher,str),separator or io.pathseparator) +end +function file.joinpath(tab,separator) + return tab and concat(tab,separator or io.pathseparator) +end +local someslash=S("\\/") +local stripper=Cs(P(fwslash)^0/""*reslasher) +local isnetwork=someslash*someslash*(1-someslash)+(1-fwslash-colon)^1*colon +local isroot=fwslash^1*-1 +local hasroot=fwslash^1 +local reslasher=lpeg.replacer(S("\\/"),"/") +local deslasher=lpeg.replacer(S("\\/")^1,"/") +function file.join(one,two,three,...) + if not two then + return one=="" and one or lpegmatch(stripper,one) + end + if one=="" then + return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) + end + if lpegmatch(isnetwork,one) then + local one=lpegmatch(reslasher,one) + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) + if lpegmatch(hasroot,two) then + return one..two + else + return one.."/"..two + end + elseif lpegmatch(isroot,one) then + local two=lpegmatch(deslasher,three and concat({ two,three,... },"/") or two) + if lpegmatch(hasroot,two) then + return two + else + return "/"..two + end + else + return lpegmatch(deslasher,concat({ one,two,three,... },"/")) + end +end +local drivespec=R("az","AZ")^1*colon +local anchors=fwslash+drivespec +local untouched=periods+(1-period)^1*P(-1) +local mswindrive=Cs(drivespec*(bwslash/"/"+fwslash)^0) +local mswinuncpath=(bwslash+fwslash)*(bwslash+fwslash)*Cc("//") +local splitstarter=(mswindrive+mswinuncpath+Cc(false))*Ct(lpeg.splitat(S("/\\")^1)) +local absolute=fwslash +function file.collapsepath(str,anchor) + if not str then + return + end + if anchor==true and not lpegmatch(anchors,str) then + str=getcurrentdir().."/"..str + end + if str=="" or str=="." then + return "." + elseif lpegmatch(untouched,str) then + return lpegmatch(reslasher,str) + end + local starter,oldelements=lpegmatch(splitstarter,str) + local newelements={} + local i=#oldelements + while i>0 do + local element=oldelements[i] + if element=='.' then + elseif element=='..' then + local n=i-1 + while n>0 do + local element=oldelements[n] + if element~='..' and element~='.' then + oldelements[n]='.' + break + else + n=n-1 + end + end + if n<1 then + insert(newelements,1,'..') + end + elseif element~="" then + insert(newelements,1,element) + end + i=i-1 + end + if #newelements==0 then + return starter or "." + elseif starter then + return starter..concat(newelements,'/') + elseif lpegmatch(absolute,str) then + return "/"..concat(newelements,'/') + else + newelements=concat(newelements,'/') + if anchor=="." and find(str,"^%./") then + return "./"..newelements + else + return newelements + end + end +end +local validchars=R("az","09","AZ","--","..") +local pattern_a=lpeg.replacer(1-validchars) +local pattern_a=Cs((validchars+P(1)/"-")^1) +local whatever=P("-")^0/"" +local pattern_b=Cs(whatever*(1-whatever*-1)^1) +function file.robustname(str,strict) + if str then + str=lpegmatch(pattern_a,str) or str + if strict then + return lpegmatch(pattern_b,str) or str + else + return str + end + end +end +file.readdata=io.loaddata +file.savedata=io.savedata +function file.copy(oldname,newname) + if oldname and newname then + local data=io.loaddata(oldname) + if data and data~="" then + file.savedata(newname,data) + end + end +end +local letter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=period^0*fwslash+letter*colon+letter^1*separator+letter^1*fwslash +local rootbased=fwslash+letter*colon +lpeg.patterns.qualified=qualified +lpeg.patterns.rootbased=rootbased +function file.is_qualified_path(filename) + return filename and lpegmatch(qualified,filename)~=nil +end +function file.is_rootbased_path(filename) + return filename and lpegmatch(rootbased,filename)~=nil +end +function file.strip(name,dir) + if name then + local b,a=match(name,"^(.-)"..dir.."(.*)$") + return a~="" and a or name + end +end +function lfs.mkdirs(path) + local full="" + for sub in gmatch(path,"(/*[^\\/]+)") do + full=full..sub + lfs.mkdir(full) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-gzip"] = package.loaded["l-gzip"] or true + +-- original size: 1211, stripped down to: 1002 + +if not modules then modules={} end modules ['l-gzip']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not gzip then + return +end +local suffix,suffixes=file.suffix,file.suffixes +function gzip.load(filename) + local f=io.open(filename,"rb") + if not f then + elseif suffix(filename)=="gz" then + f:close() + local g=gzip.open(filename,"rb") + if g then + local str=g:read("*all") + g:close() + return str + end + else + local str=f:read("*all") + f:close() + return str + end +end +function gzip.save(filename,data) + if suffix(filename)~="gz" then + filename=filename..".gz" + end + local f=io.open(filename,"wb") + if f then + local s=zlib.compress(data or "",9,nil,15+16) + f:write(s) + f:close() + return #s + end +end +function gzip.suffix(filename) + local suffix,extra=suffixes(filename) + local gzipped=extra=="gz" + return suffix,gzipped +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-md5"] = package.loaded["l-md5"] or true + +-- original size: 3248, stripped down to: 2266 + +if not modules then modules={} end modules ['l-md5']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not md5 then + md5=optionalrequire("md5") +end +if not md5 then + md5={ + sum=function(str) print("error: md5 is not loaded (sum ignored)") return str end, + sumhexa=function(str) print("error: md5 is not loaded (sumhexa ignored)") return str end, + } +end +local md5,file=md5,file +local gsub=string.gsub +do + local patterns=lpeg and lpeg.patterns + if patterns then + local bytestoHEX=patterns.bytestoHEX + local bytestohex=patterns.bytestohex + local bytestodec=patterns.bytestodec + local lpegmatch=lpeg.match + local md5sum=md5.sum + if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end + if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end + if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + end +end +function file.needsupdating(oldname,newname,threshold) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime then + local newtime=lfs.attributes(newname,"modification") + if not newtime then + return true + elseif newtime>=oldtime then + return false + elseif oldtime-newtime<(threshold or 1) then + return false + else + return true + end + else + return false + end +end +file.needs_updating=file.needsupdating +function file.syncmtimes(oldname,newname) + local oldtime=lfs.attributes(oldname,"modification") + if oldtime and lfs.isfile(newname) then + lfs.touch(newname,oldtime,oldtime) + end +end +function file.checksum(name) + if md5 then + local data=io.loaddata(name) + if data then + return md5.HEX(data) + end + end + return nil +end +function file.loadchecksum(name) + if md5 then + local data=io.loaddata(name..".md5") + return data and (gsub(data,"%s","")) + end + return nil +end +function file.savechecksum(name,checksum) + if not checksum then checksum=file.checksum(name) end + if checksum then + io.savedata(name..".md5",checksum) + return checksum + end + return nil +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-url"] = package.loaded["l-url"] or true + +-- original size: 12531, stripped down to: 5721 + +if not modules then modules={} end modules ['l-url']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local char,format,byte=string.char,string.format,string.byte +local concat=table.concat +local tonumber,type=tonumber,type +local P,C,R,S,Cs,Cc,Ct,Cf,Cg,V=lpeg.P,lpeg.C,lpeg.R,lpeg.S,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.Cf,lpeg.Cg,lpeg.V +local lpegmatch,lpegpatterns,replacer=lpeg.match,lpeg.patterns,lpeg.replacer +url=url or {} +local url=url +local tochar=function(s) return char(tonumber(s,16)) end +local colon=P(":") +local qmark=P("?") +local hash=P("#") +local slash=P("/") +local percent=P("%") +local endofstring=P(-1) +local hexdigit=R("09","AF","af") +local plus=P("+") +local nothing=Cc("") +local escapedchar=(percent*C(hexdigit*hexdigit))/tochar +local escaped=(plus/" ")+escapedchar +local noslash=P("/")/"" +local schemestr=Cs((escaped+(1-colon-slash-qmark-hash))^2) +local authoritystr=Cs((escaped+(1- slash-qmark-hash))^0) +local pathstr=Cs((escaped+(1- qmark-hash))^0) +local querystr=Cs(((1- hash))^0) +local fragmentstr=Cs((escaped+(1- endofstring))^0) +local scheme=schemestr*colon+nothing +local authority=slash*slash*authoritystr+nothing +local path=slash*pathstr+nothing +local query=qmark*querystr+nothing +local fragment=hash*fragmentstr+nothing +local validurl=scheme*authority*path*query*fragment +local parser=Ct(validurl) +lpegpatterns.url=validurl +lpegpatterns.urlsplitter=parser +local escapes={} +setmetatable(escapes,{ __index=function(t,k) + local v=format("%%%02X",byte(k)) + t[k]=v + return v +end }) +local escaper=Cs((R("09","AZ","az")^1+P(" ")/"%%20"+S("-./_")^1+P(1)/escapes)^0) +local unescaper=Cs((escapedchar+1)^0) +local getcleaner=Cs((P("+++")/"%%2B"+P("+")/"%%20"+P(1))^1) +lpegpatterns.urlunescaped=escapedchar +lpegpatterns.urlescaper=escaper +lpegpatterns.urlunescaper=unescaper +lpegpatterns.urlgetcleaner=getcleaner +function url.unescapeget(str) + return lpegmatch(getcleaner,str) +end +local function split(str) + return (type(str)=="string" and lpegmatch(parser,str)) or str +end +local isscheme=schemestr*colon*slash*slash +local function hasscheme(str) + if str then + local scheme=lpegmatch(isscheme,str) + return scheme~="" and scheme or false + else + return false + end +end +local rootletter=R("az","AZ")+S("_-+") +local separator=P("://") +local qualified=P(".")^0*P("/")+rootletter*P(":")+rootletter^1*separator+rootletter^1*P("/") +local rootbased=P("/")+rootletter*P(":") +local barswapper=replacer("|",":") +local backslashswapper=replacer("\\","/") +local equal=P("=") +local amp=P("&") +local key=Cs(((escapedchar+1)-equal )^0) +local value=Cs(((escapedchar+1)-amp -endofstring)^0) +local splitquery=Cf (Ct("")*P { "sequence", + sequence=V("pair")*(amp*V("pair"))^0, + pair=Cg(key*equal*value), +},rawset) +local function hashed(str) + if not str or str=="" then + return { + scheme="invalid", + original=str, + } + end + local detailed=split(str) + local rawscheme="" + local rawquery="" + local somescheme=false + local somequery=false + if detailed then + rawscheme=detailed[1] + rawquery=detailed[4] + somescheme=rawscheme~="" + somequery=rawquery~="" + end + if not somescheme and not somequery then + return { + scheme="file", + authority="", + path=str, + query="", + fragment="", + original=str, + noscheme=true, + filename=str, + } + end + local authority=detailed[2] + local path=detailed[3] + local filename=nil + if authority=="" then + filename=path + elseif path=="" then + filename="" + else + filename=authority.."/"..path + end + return { + scheme=rawscheme, + authority=authority, + path=path, + query=lpegmatch(unescaper,rawquery), + queries=lpegmatch(splitquery,rawquery), + fragment=detailed[5], + original=str, + noscheme=false, + filename=filename, + } +end +url.split=split +url.hasscheme=hasscheme +url.hashed=hashed +function url.addscheme(str,scheme) + if hasscheme(str) then + return str + elseif not scheme then + return "file:///"..str + else + return scheme..":///"..str + end +end +function url.construct(hash) + local fullurl,f={},0 + local scheme,authority,path,query,fragment=hash.scheme,hash.authority,hash.path,hash.query,hash.fragment + if scheme and scheme~="" then + f=f+1;fullurl[f]=scheme.."://" + end + if authority and authority~="" then + f=f+1;fullurl[f]=authority + end + if path and path~="" then + f=f+1;fullurl[f]="/"..path + end + if query and query~="" then + f=f+1;fullurl[f]="?"..query + end + if fragment and fragment~="" then + f=f+1;fullurl[f]="#"..fragment + end + return lpegmatch(escaper,concat(fullurl)) +end +local pattern=Cs(slash^-1/""*R("az","AZ")*((S(":|")/":")+P(":"))*slash*P(1)^0) +function url.filename(filename) + local spec=hashed(filename) + local path=spec.path + return (spec.scheme=="file" and path and lpegmatch(pattern,path)) or filename +end +local function escapestring(str) + return lpegmatch(escaper,str) +end +url.escape=escapestring +function url.query(str) + if type(str)=="string" then + return lpegmatch(splitquery,str) or "" + else + return str + end +end +function url.toquery(data) + local td=type(data) + if td=="string" then + return #str and escape(data) or nil + elseif td=="table" then + if next(data) then + local t={} + for k,v in next,data do + t[#t+1]=format("%s=%s",k,escapestring(v)) + end + return concat(t,"&") + end + else + end +end +local pattern=Cs(noslash^0*(1-noslash*P(-1))^0) +function url.barepath(path) + if not path or path=="" then + return "" + else + return lpegmatch(pattern,path) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-dir"] = package.loaded["l-dir"] or true + +-- original size: 16765, stripped down to: 11003 + +if not modules then modules={} end modules ['l-dir']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,select=type,select +local find,gmatch,match,gsub,sub=string.find,string.gmatch,string.match,string.gsub,string.sub +local concat,insert,remove,unpack=table.concat,table.insert,table.remove,table.unpack +local lpegmatch=lpeg.match +local P,S,R,C,Cc,Cs,Ct,Cv,V=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Cv,lpeg.V +dir=dir or {} +local dir=dir +local lfs=lfs +local attributes=lfs.attributes +local walkdir=lfs.dir +local isdir=lfs.isdir +local isfile=lfs.isfile +local currentdir=lfs.currentdir +local chdir=lfs.chdir +local mkdir=lfs.mkdir +local onwindows=os.type=="windows" or find(os.getenv("PATH"),";",1,true) +if onwindows then + local tricky=S("/\\")*P(-1) + isdir=function(name) + if lpegmatch(tricky,name) then + return attributes(name,"mode")=="directory" + else + return attributes(name.."/.","mode")=="directory" + end + end + isfile=function(name) + return attributes(name,"mode")=="file" + end + lfs.isdir=isdir + lfs.isfile=isfile +else + isdir=function(name) + return attributes(name,"mode")=="directory" + end + isfile=function(name) + return attributes(name,"mode")=="file" + end + lfs.isdir=isdir + lfs.isfile=isfile +end +function dir.current() + return (gsub(currentdir(),"\\","/")) +end +local function glob_pattern_function(path,patt,recurse,action) + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + action(full) + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end + end + end + end + if dirs then + for i=1,#dirs do + glob_pattern_function(dirs[i],patt,recurse,action) + end + end + end +end +local function glob_pattern_table(path,patt,recurse,result) + if not result then + result={} + end + if isdir(path) then + local usedpath + if path=="/" then + usedpath="/." + elseif not find(path,"/$") then + usedpath=path.."/." + path=path.."/" + else + usedpath=path + end + local dirs + for name in walkdir(usedpath) do + if name~="." and name~=".." then + local full=path..name + local mode=attributes(full,'mode') + if mode=='file' then + if not patt or find(full,patt) then + result[#result+1]=full + end + elseif recurse and mode=="directory" then + if not dirs then + dirs={ full } + else + dirs[#dirs+1]=full + end + end + end + end + if dirs then + for i=1,#dirs do + glob_pattern_table(dirs[i],patt,recurse,result) + end + end + end + return result +end +local function globpattern(path,patt,recurse,method) + local kind=type(method) + if patt and sub(patt,1,-3)==path then + patt=false + end + if kind=="function" then + return glob_pattern_function(path,patt,recurse,method) + elseif kind=="table" then + return glob_pattern_table(path,patt,recurse,method) + else + return glob_pattern_table(path,patt,recurse,{}) + end +end +dir.globpattern=globpattern +local function collectpattern(path,patt,recurse,result) + local ok,scanner + result=result or {} + if path=="/" then + ok,scanner,first=xpcall(function() return walkdir(path..".") end,function() end) + else + ok,scanner,first=xpcall(function() return walkdir(path) end,function() end) + end + if ok and type(scanner)=="function" then + if not find(path,"/$") then + path=path..'/' + end + for name in scanner,first do + if name=="." then + elseif name==".." then + else + local full=path..name + local attr=attributes(full) + local mode=attr.mode + if mode=='file' then + if find(full,patt) then + result[name]=attr + end + elseif recurse and mode=="directory" then + attr.list=collectpattern(full,patt,recurse) + result[name]=attr + end + end + end + end + return result +end +dir.collectpattern=collectpattern +local separator,pattern +if onwindows then + local slash=S("/\\")/"/" + pattern={ + [1]=(Cs(P(".")+slash^1)+Cs(R("az","AZ")*P(":")*slash^0)+Cc("./"))*V(2)*V(3), + [2]=Cs(((1-S("*?/\\"))^0*slash)^0), + [3]=Cs(P(1)^0) + } +else + pattern={ + [1]=(C(P(".")+P("/")^1)+Cc("./"))*V(2)*V(3), + [2]=C(((1-S("*?/"))^0*P("/"))^0), + [3]=C(P(1)^0) + } +end +local filter=Cs (( + P("**")/".*"+P("*")/"[^/]*"+P("?")/"[^/]"+P(".")/"%%."+P("+")/"%%+"+P("-")/"%%-"+P(1) +)^0 ) +local function glob(str,t) + if type(t)=="function" then + if type(str)=="table" then + for s=1,#str do + glob(str[s],t) + end + elseif isfile(str) then + t(str) + else + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) + local start=root..path + local result=lpegmatch(filter,start..base) + globpattern(start,result,recurse,t) + end + end + else + if type(str)=="table" then + local t=t or {} + for s=1,#str do + glob(str[s],t) + end + return t + elseif isfile(str) then + if t then + t[#t+1]=str + return t + else + return { str } + end + else + local root,path,base=lpegmatch(pattern,str) + if root and path and base then + local recurse=find(base,"**",1,true) + local start=root..path + local result=lpegmatch(filter,start..base) + return globpattern(start,result,recurse,t) + else + return {} + end + end + end +end +dir.glob=glob +local function globfiles(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if recurse then + globfiles(path.."/"..name,recurse,func,files) + end + elseif mode=="file" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + end + end + end + end + return files +end +dir.globfiles=globfiles +function dir.ls(pattern) + return concat(glob(pattern),"\n") +end +local make_indeed=true +if onwindows then + function dir.mkdirs(...) + local n=select("#",...) + local str + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s=="" then + elseif str=="" then + str=s + else + str=str.."/"..s + end + end + end + local pth="" + local drive=false + local first,middle,last=match(str,"^(//)(//*)(.*)$") + if first then + else + first,last=match(str,"^(//)/*(.-)$") + if first then + middle,last=match(str,"([^/]+)/+(.-)$") + if middle then + pth="//"..middle + else + pth="//"..last + last="" + end + else + first,middle,last=match(str,"^([a-zA-Z]:)(/*)(.-)$") + if first then + pth,drive=first..middle,true + else + middle,last=match(str,"^(/*)(.-)$") + if not middle then + last=str + end + end + end + end + for s in gmatch(last,"[^/]+") do + if pth=="" then + pth=s + elseif drive then + pth,drive=pth..s,false + else + pth=pth.."/"..s + end + if make_indeed and not isdir(pth) then + mkdir(pth) + end + end + return pth,(isdir(pth)==true) + end +else + function dir.mkdirs(...) + local n=select("#",...) + local str,pth + if n==1 then + str=select(1,...) + if isdir(str) then + return str,true + end + else + str="" + for i=1,n do + local s=select(i,...) + if s and s~="" then + if str~="" then + str=str.."/"..s + else + str=s + end + end + end + end + str=gsub(str,"/+","/") + if find(str,"^/") then + pth="/" + for s in gmatch(str,"[^/]+") do + local first=(pth=="/") + if first then + pth=pth..s + else + pth=pth.."/"..s + end + if make_indeed and not first and not isdir(pth) then + mkdir(pth) + end + end + else + pth="." + for s in gmatch(str,"[^/]+") do + pth=pth.."/"..s + if make_indeed and not isdir(pth) then + mkdir(pth) + end + end + end + return pth,(isdir(pth)==true) + end +end +dir.makedirs=dir.mkdirs +do + local chdir=sandbox and sandbox.original(chdir) or chdir + if onwindows then + local xcurrentdir=dir.current + function dir.expandname(str) + local first,nothing,last=match(str,"^(//)(//*)(.*)$") + if first then + first=xcurrentdir().."/" + end + if not first then + first,last=match(str,"^(//)/*(.*)$") + end + if not first then + first,last=match(str,"^([a-zA-Z]:)(.*)$") + if first and not find(last,"^/") then + local d=currentdir() + if chdir(first) then + first=xcurrentdir() + end + chdir(d) + end + end + if not first then + first,last=xcurrentdir(),str + end + last=gsub(last,"//","/") + last=gsub(last,"/%./","/") + last=gsub(last,"^/*","") + first=gsub(first,"/*$","") + if last=="" or last=="." then + return first + else + return first.."/"..last + end + end + else + function dir.expandname(str) + if not find(str,"^/") then + str=currentdir().."/"..str + end + str=gsub(str,"//","/") + str=gsub(str,"/%./","/") + str=gsub(str,"(.)/%.$","%1") + return str + end + end +end +file.expandname=dir.expandname +local stack={} +function dir.push(newdir) + insert(stack,currentdir()) + if newdir and newdir~="" then + chdir(newdir) + end +end +function dir.pop() + local d=remove(stack) + if d then + chdir(d) + end + return d +end +local function found(...) + for i=1,select("#",...) do + local path=select(i,...) + local kind=type(path) + if kind=="string" then + if isdir(path) then + return path + end + elseif kind=="table" then + local path=found(unpack(path)) + if path then + return path + end + end + end +end +dir.found=found + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-boolean"] = package.loaded["l-boolean"] or true + +-- original size: 1850, stripped down to: 1568 + +if not modules then modules={} end modules ['l-boolean']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber=type,tonumber +boolean=boolean or {} +local boolean=boolean +function boolean.tonumber(b) + if b then return 1 else return 0 end +end +function toboolean(str,tolerant) + if str==nil then + return false + elseif str==false then + return false + elseif str==true then + return true + elseif str=="true" then + return true + elseif str=="false" then + return false + elseif not tolerant then + return false + elseif str==0 then + return false + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +string.toboolean=toboolean +function string.booleanstring(str) + if str=="0" then + return false + elseif str=="1" then + return true + elseif str=="" then + return false + elseif str=="false" then + return false + elseif str=="true" then + return true + elseif (tonumber(str) or 0)>0 then + return true + else + return str=="yes" or str=="on" or str=="t" + end +end +function string.is_boolean(str,default,strict) + if type(str)=="string" then + if str=="true" or str=="yes" or str=="on" or str=="t" or (not strict and str=="1") then + return true + elseif str=="false" or str=="no" or str=="off" or str=="f" or (not strict and str=="0") then + return false + end + end + return default +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-unicode"] = package.loaded["l-unicode"] or true + +-- original size: 37388, stripped down to: 15817 + +if not modules then modules={} end modules ['l-unicode']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utf=utf or (unicode and unicode.utf8) or {} +utf.characters=utf.characters or string.utfcharacters +utf.values=utf.values or string.utfvalues +local type=type +local char,byte,format,sub,gmatch=string.char,string.byte,string.format,string.sub,string.gmatch +local concat=table.concat +local P,C,R,Cs,Ct,Cmt,Cc,Carg,Cp=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Ct,lpeg.Cmt,lpeg.Cc,lpeg.Carg,lpeg.Cp +local lpegmatch=lpeg.match +local patterns=lpeg.patterns +local tabletopattern=lpeg.utfchartabletopattern +local bytepairs=string.bytepairs +local finder=lpeg.finder +local replacer=lpeg.replacer +local utfvalues=utf.values +local utfgmatch=utf.gmatch +local p_utftype=patterns.utftype +local p_utfstricttype=patterns.utfstricttype +local p_utfoffset=patterns.utfoffset +local p_utf8char=patterns.utf8character +local p_utf8byte=patterns.utf8byte +local p_utfbom=patterns.utfbom +local p_newline=patterns.newline +local p_whitespace=patterns.whitespace +if not unicode then + unicode={ utf=utf } +end +if not utf.char then + local floor,char=math.floor,string.char + function utf.char(n) + if n<0x80 then + return char(n) + elseif n<0x800 then + return char( + 0xC0+floor(n/0x40), + 0x80+(n%0x40) + ) + elseif n<0x10000 then + return char( + 0xE0+floor(n/0x1000), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + elseif n<0x200000 then + return char( + 0xF0+floor(n/0x40000), + 0x80+(floor(n/0x1000)%0x40), + 0x80+(floor(n/0x40)%0x40), + 0x80+(n%0x40) + ) + else + return "" + end + end +end +if not utf.byte then + local utf8byte=patterns.utf8byte + function utf.byte(c) + return lpegmatch(utf8byte,c) + end +end +local utfchar,utfbyte=utf.char,utf.byte +function utf.filetype(data) + return data and lpegmatch(p_utftype,data) or "unknown" +end +local toentities=Cs ( + ( + patterns.utf8one+( + patterns.utf8two+patterns.utf8three+patterns.utf8four + )/function(s) local b=utfbyte(s) if b<127 then return s else return format("&#%X;",b) end end + )^0 +) +patterns.toentities=toentities +function utf.toentities(str) + return lpegmatch(toentities,str) +end +local one=P(1) +local two=C(1)*C(1) +local four=C(R(utfchar(0xD8),utfchar(0xFF)))*C(1)*C(1)*C(1) +local pattern=P("\254\255")*Cs(( + four/function(a,b,c,d) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(a,b) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 )+P("\255\254")*Cs(( + four/function(b,a,d,c) + local ab=0xFF*byte(a)+byte(b) + local cd=0xFF*byte(c)+byte(d) + return utfchar((ab-0xD800)*0x400+(cd-0xDC00)+0x10000) + end+two/function(b,a) + return utfchar(byte(a)*256+byte(b)) + end+one + )^1 ) +function string.toutf(s) + return lpegmatch(pattern,s) or s +end +local validatedutf=Cs ( + ( + patterns.utf8one+patterns.utf8two+patterns.utf8three+patterns.utf8four+P(1)/"�" + )^0 +) +patterns.validatedutf=validatedutf +function utf.is_valid(str) + return type(str)=="string" and lpegmatch(validatedutf,str) or false +end +if not utf.len then + local n,f=0,1 + local utfcharcounter=patterns.utfbom^-1*Cmt ( + Cc(1)*patterns.utf8one^1+Cc(2)*patterns.utf8two^1+Cc(3)*patterns.utf8three^1+Cc(4)*patterns.utf8four^1, + function(_,t,d) + n=n+(t-f)/d + f=t + return true + end + )^0 + function utf.len(str) + n,f=0,1 + lpegmatch(utfcharcounter,str or "") + return n + end +end +utf.length=utf.len +if not utf.sub then + local utflength=utf.length + local b,e,n,first,last=0,0,0,0,0 + local function slide_zero(s,p) + n=n+1 + if n>=last then + e=p-1 + else + return p + end + end + local function slide_one(s,p) + n=n+1 + if n==first then + b=p + end + if n>=last then + e=p-1 + else + return p + end + end + local function slide_two(s,p) + n=n+1 + if n==first then + b=p + else + return true + end + end + local pattern_zero=Cmt(p_utf8char,slide_zero)^0 + local pattern_one=Cmt(p_utf8char,slide_one )^0 + local pattern_two=Cmt(p_utf8char,slide_two )^0 + local pattern_first=C(patterns.utf8character) + function utf.sub(str,start,stop) + if not start then + return str + end + if start==0 then + start=1 + end + if not stop then + if start<0 then + local l=utflength(str) + start=l+start + else + start=start-1 + end + b,n,first=0,0,start + lpegmatch(pattern_two,str) + if n>=first then + return sub(str,b) + else + return "" + end + end + if start<0 or stop<0 then + local l=utf.length(str) + if start<0 then + start=l+start + if start<=0 then + start=1 + else + start=start+1 + end + end + if stop<0 then + stop=l+stop + if stop==0 then + stop=1 + else + stop=stop+1 + end + end + end + if start==1 and stop==1 then + return lpegmatch(pattern_first,str) or "" + elseif start>stop then + return "" + elseif start>1 then + b,e,n,first,last=0,0,0,start-1,stop + lpegmatch(pattern_one,str) + if n>=first and e==0 then + e=#str + end + return sub(str,b,e) + else + b,e,n,last=1,0,0,stop + lpegmatch(pattern_zero,str) + if e==0 then + e=#str + end + return sub(str,b,e) + end + end +end +function utf.remapper(mapping,option) + local variant=type(mapping) + if variant=="table" then + if option=="dynamic" then + local pattern=false + table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) + return function(str) + if not str or str=="" then + return "" + else + if not pattern then + pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + end + return lpegmatch(pattern,str) + end + end + elseif option=="pattern" then + return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + else + local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + elseif variant=="function" then + if option=="pattern" then + return Cs((p_utf8char/mapping+p_utf8char)^0) + else + local pattern=Cs((p_utf8char/mapping+p_utf8char)^0) + return function(str) + if not str or str=="" then + return "" + else + return lpegmatch(pattern,str) + end + end,pattern + end + else + return function(str) + return str or "" + end + end +end +function utf.replacer(t) + local r=replacer(t,false,false,true) + return function(str) + return lpegmatch(r,str) + end +end +function utf.subtituter(t) + local f=finder (t) + local r=replacer(t,false,false,true) + return function(str) + local i=lpegmatch(f,str) + if not i then + return str + elseif i>#str then + return str + else + return lpegmatch(r,str) + end + end +end +local utflinesplitter=p_utfbom^-1*lpeg.tsplitat(p_newline) +local utfcharsplitter_ows=p_utfbom^-1*Ct(C(p_utf8char)^0) +local utfcharsplitter_iws=p_utfbom^-1*Ct((p_whitespace^1+C(p_utf8char))^0) +local utfcharsplitter_raw=Ct(C(p_utf8char)^0) +patterns.utflinesplitter=utflinesplitter +function utf.splitlines(str) + return lpegmatch(utflinesplitter,str or "") +end +function utf.split(str,ignorewhitespace) + if ignorewhitespace then + return lpegmatch(utfcharsplitter_iws,str or "") + else + return lpegmatch(utfcharsplitter_ows,str or "") + end +end +function utf.totable(str) + return lpegmatch(utfcharsplitter_raw,str) +end +function utf.magic(f) + local str=f:read(4) or "" + local off=lpegmatch(p_utfoffset,str) + if off<4 then + f:seek('set',off) + end + return lpegmatch(p_utftype,str) +end +local utf16_to_utf8_be,utf16_to_utf8_le +local utf32_to_utf8_be,utf32_to_utf8_le +local utf_16_be_getbom=patterns.utfbom_16_be^-1 +local utf_16_le_getbom=patterns.utfbom_16_le^-1 +local utf_32_be_getbom=patterns.utfbom_32_be^-1 +local utf_32_le_getbom=patterns.utfbom_32_le^-1 +local utf_16_be_linesplitter=utf_16_be_getbom*lpeg.tsplitat(patterns.utf_16_be_nl) +local utf_16_le_linesplitter=utf_16_le_getbom*lpeg.tsplitat(patterns.utf_16_le_nl) +local utf_32_be_linesplitter=utf_32_be_getbom*lpeg.tsplitat(patterns.utf_32_be_nl) +local utf_32_le_linesplitter=utf_32_le_getbom*lpeg.tsplitat(patterns.utf_32_le_nl) +local more=0 +local p_utf16_to_utf8_be=C(1)*C(1)/function(left,right) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf16_to_utf8_le=C(1)*C(1)/function(right,left) + local now=256*byte(left)+byte(right) + if more>0 then + now=(more-0xD800)*0x400+(now-0xDC00)+0x10000 + more=0 + return utfchar(now) + elseif now>=0xD800 and now<=0xDBFF then + more=now + return "" + else + return utfchar(now) + end +end +local p_utf32_to_utf8_be=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(a)+256*256*byte(b)+256*byte(c)+byte(d)) +end +local p_utf32_to_utf8_le=C(1)*C(1)*C(1)*C(1)/function(a,b,c,d) + return utfchar(256*256*256*byte(d)+256*256*byte(c)+256*byte(b)+byte(a)) +end +p_utf16_to_utf8_be=P(true)/function() more=0 end*utf_16_be_getbom*Cs(p_utf16_to_utf8_be^0) +p_utf16_to_utf8_le=P(true)/function() more=0 end*utf_16_le_getbom*Cs(p_utf16_to_utf8_le^0) +p_utf32_to_utf8_be=P(true)/function() more=0 end*utf_32_be_getbom*Cs(p_utf32_to_utf8_be^0) +p_utf32_to_utf8_le=P(true)/function() more=0 end*utf_32_le_getbom*Cs(p_utf32_to_utf8_le^0) +patterns.utf16_to_utf8_be=p_utf16_to_utf8_be +patterns.utf16_to_utf8_le=p_utf16_to_utf8_le +patterns.utf32_to_utf8_be=p_utf32_to_utf8_be +patterns.utf32_to_utf8_le=p_utf32_to_utf8_le +utf16_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_be,s) + else + return s + end +end +local utf16_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_be_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_be,s) + end + end + return t +end +utf16_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf16_to_utf8_le,s) + else + return s + end +end +local utf16_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_16_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf16_to_utf8_le,s) + end + end + return t +end +utf32_to_utf8_be=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_be,s) + else + return s + end +end +local utf32_to_utf8_be_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_be_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_be,s) + end + end + return t +end +utf32_to_utf8_le=function(s) + if s and s~="" then + return lpegmatch(p_utf32_to_utf8_le,s) + else + return s + end +end +local utf32_to_utf8_le_t=function(t) + if not t then + return nil + elseif type(t)=="string" then + t=lpegmatch(utf_32_le_linesplitter,t) + end + for i=1,#t do + local s=t[i] + if s~="" then + t[i]=lpegmatch(p_utf32_to_utf8_le,s) + end + end + return t +end +utf.utf16_to_utf8_le_t=utf16_to_utf8_le_t +utf.utf16_to_utf8_be_t=utf16_to_utf8_be_t +utf.utf32_to_utf8_le_t=utf32_to_utf8_le_t +utf.utf32_to_utf8_be_t=utf32_to_utf8_be_t +utf.utf16_to_utf8_le=utf16_to_utf8_le +utf.utf16_to_utf8_be=utf16_to_utf8_be +utf.utf32_to_utf8_le=utf32_to_utf8_le +utf.utf32_to_utf8_be=utf32_to_utf8_be +function utf.utf8_to_utf8_t(t) + return type(t)=="string" and lpegmatch(utflinesplitter,t) or t +end +function utf.utf16_to_utf8_t(t,endian) + return endian and utf16_to_utf8_be_t(t) or utf16_to_utf8_le_t(t) or t +end +function utf.utf32_to_utf8_t(t,endian) + return endian and utf32_to_utf8_be_t(t) or utf32_to_utf8_le_t(t) or t +end +local function little(b) + if b<0x10000 then + return char(b%256,b/256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1%256,b1/256,b2%256,b2/256) + end +end +local function big(b) + if b<0x10000 then + return char(b/256,b%256) + else + b=b-0x10000 + local b1,b2=b/1024+0xD800,b%1024+0xDC00 + return char(b1/256,b1%256,b2/256,b2%256) + end +end +local l_remap=Cs((p_utf8byte/little+P(1)/"")^0) +local b_remap=Cs((p_utf8byte/big+P(1)/"")^0) +local function utf8_to_utf16_be(str,nobom) + if nobom then + return lpegmatch(b_remap,str) + else + return char(254,255)..lpegmatch(b_remap,str) + end +end +local function utf8_to_utf16_le(str,nobom) + if nobom then + return lpegmatch(l_remap,str) + else + return char(255,254)..lpegmatch(l_remap,str) + end +end +utf.utf8_to_utf16_be=utf8_to_utf16_be +utf.utf8_to_utf16_le=utf8_to_utf16_le +function utf.utf8_to_utf16(str,littleendian,nobom) + if littleendian then + return utf8_to_utf16_le(str,nobom) + else + return utf8_to_utf16_be(str,nobom) + end +end +local pattern=Cs ( + (p_utf8byte/function(unicode ) return format("0x%04X",unicode) end)*(p_utf8byte*Carg(1)/function(unicode,separator) return format("%s0x%04X",separator,unicode) end)^0 +) +function utf.tocodes(str,separator) + return lpegmatch(pattern,str,1,separator or " ") +end +function utf.ustring(s) + return format("U+%05X",type(s)=="number" and s or utfbyte(s)) +end +function utf.xstring(s) + return format("0x%05X",type(s)=="number" and s or utfbyte(s)) +end +function utf.toeight(str) + if not str or str=="" then + return nil + end + local utftype=lpegmatch(p_utfstricttype,str) + if utftype=="utf-8" then + return sub(str,4) + elseif utftype=="utf-16-be" then + return utf16_to_utf8_be(str) + elseif utftype=="utf-16-le" then + return utf16_to_utf8_le(str) + else + return str + end +end +local p_nany=p_utf8char/"" +if utfgmatch then + function utf.count(str,what) + if type(what)=="string" then + local n=0 + for _ in utfgmatch(str,what) do + n=n+1 + end + return n + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +else + local cache={} + function utf.count(str,what) + if type(what)=="string" then + local p=cache[what] + if not p then + p=Cs((P(what)/" "+p_nany)^0) + cache[p]=p + end + return #lpegmatch(p,str) + else + return #lpegmatch(Cs((P(what)/" "+p_nany)^0),str) + end + end +end +if not utf.characters then + function utf.characters(str) + return gmatch(str,".[\128-\191]*") + end + string.utfcharacters=utf.characters +end +if not utf.values then + local find=string.find + local dummy=function() + end + function utf.values(str) + local n=#str + if n==0 then + return dummy + elseif n==1 then + return function() return utfbyte(str) end + else + local p=1 + return function() + local b,e=find(str,".[\128-\191]*",p) + if b then + p=e+1 + return utfbyte(sub(str,b,e)) + end + end + end + end + string.utfvalues=utf.values +end +function utf.chrlen(u) + return + (u<0x80 and 1) or + (u<0xE0 and 2) or + (u<0xF0 and 3) or + (u<0xF8 and 4) or + (u<0xFC and 5) or + (u<0xFE and 6) or 0 +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-math"] = package.loaded["l-math"] or true + +-- original size: 974, stripped down to: 890 + +if not modules then modules={} end modules ['l-math']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local floor,sin,cos,tan=math.floor,math.sin,math.cos,math.tan +if not math.ceiling then + math.ceiling=math.ceil +end +if not math.round then + function math.round(x) return floor(x+0.5) end +end +if not math.div then + function math.div(n,m) return floor(n/m) end +end +if not math.mod then + function math.mod(n,m) return n%m end +end +local pipi=2*math.pi/360 +if not math.sind then + function math.sind(d) return sin(d*pipi) end + function math.cosd(d) return cos(d*pipi) end + function math.tand(d) return tan(d*pipi) end +end +if not math.odd then + function math.odd (n) return n%2~=0 end + function math.even(n) return n%2==0 end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-str"] = package.loaded["util-str"] or true + +-- original size: 34503, stripped down to: 18933 + +if not modules then modules={} end modules ['util-str']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.strings=utilities.strings or {} +local strings=utilities.strings +local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local load,dump=load,string.dump +local tonumber,type,tostring=tonumber,type,tostring +local unpack,concat=table.unpack,table.concat +local P,V,C,S,R,Ct,Cs,Cp,Carg,Cc=lpeg.P,lpeg.V,lpeg.C,lpeg.S,lpeg.R,lpeg.Ct,lpeg.Cs,lpeg.Cp,lpeg.Carg,lpeg.Cc +local patterns,lpegmatch=lpeg.patterns,lpeg.match +local utfchar,utfbyte=utf.char,utf.byte +local loadstripped=nil +if _LUAVERSION<5.2 then + loadstripped=function(str,shortcuts) + return load(str) + end +else + loadstripped=function(str,shortcuts) + if shortcuts then + return load(dump(load(str),true),nil,nil,shortcuts) + else + return load(dump(load(str),true)) + end + end +end +if not number then number={} end +local stripper=patterns.stripzeros +local newline=patterns.newline +local endofstring=patterns.endofstring +local whitespace=patterns.whitespace +local spacer=patterns.spacer +local spaceortab=patterns.spaceortab +local function points(n) + n=tonumber(n) + return (not n or n==0) and "0pt" or lpegmatch(stripper,format("%.5fpt",n/65536)) +end +local function basepoints(n) + n=tonumber(n) + return (not n or n==0) and "0bp" or lpegmatch(stripper,format("%.5fbp",n*(7200/7227)/65536)) +end +number.points=points +number.basepoints=basepoints +local rubish=spaceortab^0*newline +local anyrubish=spaceortab+newline +local anything=patterns.anything +local stripped=(spaceortab^1/"")*newline +local leading=rubish^0/"" +local trailing=(anyrubish^1*endofstring)/"" +local redundant=rubish^3/"\n" +local pattern=Cs(leading*(trailing+redundant+stripped+anything)^0) +function strings.collapsecrlf(str) + return lpegmatch(pattern,str) +end +local repeaters={} +function strings.newrepeater(str,offset) + offset=offset or 0 + local s=repeaters[str] + if not s then + s={} + repeaters[str]=s + end + local t=s[offset] + if t then + return t + end + t={} + setmetatable(t,{ __index=function(t,k) + if not k then + return "" + end + local n=k+offset + local s=n>0 and rep(str,n) or "" + t[k]=s + return s + end }) + s[offset]=t + return t +end +local extra,tab,start=0,0,4,0 +local nspaces=strings.newrepeater(" ") +string.nspaces=nspaces +local pattern=Carg(1)/function(t) + extra,tab,start=0,t or 7,1 + end*Cs(( + Cp()*patterns.tab/function(position) + local current=(position-start+1)+extra + local spaces=tab-(current-1)%tab + if spaces>0 then + extra=extra+spaces-1 + return nspaces[spaces] + else + return "" + end + end+newline*Cp()/function(position) + extra,start=0,position + end+patterns.anything + )^1) +function strings.tabtospace(str,tab) + return lpegmatch(pattern,str,1,tab or 7) +end +local space=spacer^0 +local nospace=space/"" +local endofline=nospace*newline +local stripend=(whitespace^1*endofstring)/"" +local normalline=(nospace*((1-space*(newline+endofstring))^1)*nospace) +local stripempty=endofline^1/"" +local normalempty=endofline^1 +local singleempty=endofline*(endofline^0/"") +local doubleempty=endofline*endofline^-1*(endofline^0/"") +local stripstart=stripempty^0 +local p_prune_normal=Cs (stripstart*(stripend+normalline+normalempty )^0 ) +local p_prune_collapse=Cs (stripstart*(stripend+normalline+doubleempty )^0 ) +local p_prune_noempty=Cs (stripstart*(stripend+normalline+singleempty )^0 ) +local p_retain_normal=Cs ((normalline+normalempty )^0 ) +local p_retain_collapse=Cs ((normalline+doubleempty )^0 ) +local p_retain_noempty=Cs ((normalline+singleempty )^0 ) +local striplinepatterns={ + ["prune"]=p_prune_normal, + ["prune and collapse"]=p_prune_collapse, + ["prune and no empty"]=p_prune_noempty, + ["retain"]=p_retain_normal, + ["retain and collapse"]=p_retain_collapse, + ["retain and no empty"]=p_retain_noempty, + ["collapse"]=patterns.collapser, +} +setmetatable(striplinepatterns,{ __index=function(t,k) return p_prune_collapse end }) +strings.striplinepatterns=striplinepatterns +function strings.striplines(str,how) + return str and lpegmatch(striplinepatterns[how],str) or str +end +strings.striplong=strings.striplines +function strings.nice(str) + str=gsub(str,"[:%-+_]+"," ") + return str +end +local n=0 +local sequenced=table.sequenced +function string.autodouble(s,sep) + if s==nil then + return '""' + end + local t=type(s) + if t=="number" then + return tostring(s) + end + if t=="table" then + return ('"'..sequenced(s,sep or ",")..'"') + end + return ('"'..tostring(s)..'"') +end +function string.autosingle(s,sep) + if s==nil then + return "''" + end + local t=type(s) + if t=="number" then + return tostring(s) + end + if t=="table" then + return ("'"..sequenced(s,sep or ",").."'") + end + return ("'"..tostring(s).."'") +end +local tracedchars={} +string.tracedchars=tracedchars +strings.tracers=tracedchars +function string.tracedchar(b) + if type(b)=="number" then + return tracedchars[b] or (utfchar(b).." (U+"..format("%05X",b)..")") + else + local c=utfbyte(b) + return tracedchars[c] or (b.." (U+"..(c and format("%05X",c) or "?????")..")") + end +end +function number.signed(i) + if i>0 then + return "+",i + else + return "-",-i + end +end +local zero=P("0")^1/"" +local plus=P("+")/"" +local minus=P("-") +local separator=S(".") +local digit=R("09") +local trailing=zero^1*#S("eE") +local exponent=(S("eE")*(plus+Cs((minus*zero^0*P(-1))/"")+minus)*zero^0*(P(-1)*Cc("0")+P(1)^1)) +local pattern_a=Cs(minus^0*digit^1*(separator/""*trailing+separator*(trailing+digit)^0)*exponent) +local pattern_b=Cs((exponent+P(1))^0) +function number.sparseexponent(f,n) + if not n then + n=f + f="%e" + end + local tn=type(n) + if tn=="string" then + local m=tonumber(n) + if m then + return lpegmatch((f=="%e" or f=="%E") and pattern_a or pattern_b,format(f,m)) + end + elseif tn=="number" then + return lpegmatch((f=="%e" or f=="%E") and pattern_a or pattern_b,format(f,n)) + end + return tostring(n) +end +local template=[[ +%s +%s +return function(%s) return %s end +]] +local preamble,environment="",{} +if _LUAVERSION<5.2 then + preamble=[[ +local lpeg=lpeg +local type=type +local tostring=tostring +local tonumber=tonumber +local format=string.format +local concat=table.concat +local signed=number.signed +local points=number.points +local basepoints= number.basepoints +local utfchar=utf.char +local utfbyte=utf.byte +local lpegmatch=lpeg.match +local nspaces=string.nspaces +local tracedchar=string.tracedchar +local autosingle=string.autosingle +local autodouble=string.autodouble +local sequenced=table.sequenced +local formattednumber=number.formatted +local sparseexponent=number.sparseexponent + ]] +else + environment={ + global=global or _G, + lpeg=lpeg, + type=type, + tostring=tostring, + tonumber=tonumber, + format=string.format, + concat=table.concat, + signed=number.signed, + points=number.points, + basepoints=number.basepoints, + utfchar=utf.char, + utfbyte=utf.byte, + lpegmatch=lpeg.match, + nspaces=string.nspaces, + tracedchar=string.tracedchar, + autosingle=string.autosingle, + autodouble=string.autodouble, + sequenced=table.sequenced, + formattednumber=number.formatted, + sparseexponent=number.sparseexponent, + } +end +local arguments={ "a1" } +setmetatable(arguments,{ __index=function(t,k) + local v=t[k-1]..",a"..k + t[k]=v + return v + end +}) +local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_tab=P("{")*C((1-P("}"))^0)*P("}")+C((1-R("az","AZ","09","%%"))^0) +local format_s=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',a%s)",f,n) + else + return format("(a%s or '')",n) + end +end +local format_S=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%ss',tostring(a%s))",f,n) + else + return format("tostring(a%s)",n) + end +end +local format_q=function() + n=n+1 + return format("(a%s and format('%%q',a%s) or '')",n,n) +end +local format_Q=function() + n=n+1 + return format("format('%%q',tostring(a%s))",n) +end +local format_i=function(f) + n=n+1 + if f and f~="" then + return format("format('%%%si',a%s)",f,n) + else + return format("format('%%i',a%s)",n) + end +end +local format_d=format_i +local format_I=function(f) + n=n+1 + return format("format('%%s%%%si',signed(a%s))",f,n) +end +local format_f=function(f) + n=n+1 + return format("format('%%%sf',a%s)",f,n) +end +local format_F=function(f) + n=n+1 + if not f or f=="" then + return format("(((a%s > -0.0000000005 and a%s < 0.0000000005) and '0') or format((a%s %% 1 == 0) and '%%i' or '%%.9f',a%s))",n,n,n,n) + else + return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) + end +end +local format_g=function(f) + n=n+1 + return format("format('%%%sg',a%s)",f,n) +end +local format_G=function(f) + n=n+1 + return format("format('%%%sG',a%s)",f,n) +end +local format_e=function(f) + n=n+1 + return format("format('%%%se',a%s)",f,n) +end +local format_E=function(f) + n=n+1 + return format("format('%%%sE',a%s)",f,n) +end +local format_j=function(f) + n=n+1 + return format("sparseexponent('%%%se',a%s)",f,n) +end +local format_J=function(f) + n=n+1 + return format("sparseexponent('%%%sE',a%s)",f,n) +end +local format_x=function(f) + n=n+1 + return format("format('%%%sx',a%s)",f,n) +end +local format_X=function(f) + n=n+1 + return format("format('%%%sX',a%s)",f,n) +end +local format_o=function(f) + n=n+1 + return format("format('%%%so',a%s)",f,n) +end +local format_c=function() + n=n+1 + return format("utfchar(a%s)",n) +end +local format_C=function() + n=n+1 + return format("tracedchar(a%s)",n) +end +local format_r=function(f) + n=n+1 + return format("format('%%%s.0f',a%s)",f,n) +end +local format_h=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('0x%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_H=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('0x%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_u=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('u+%%%sx',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_U=function(f) + n=n+1 + if f=="-" then + f=sub(f,2) + return format("format('%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + else + return format("format('U+%%%sX',type(a%s) == 'number' and a%s or utfbyte(a%s))",f=="" and "05" or f,n,n,n) + end +end +local format_p=function() + n=n+1 + return format("points(a%s)",n) +end +local format_b=function() + n=n+1 + return format("basepoints(a%s)",n) +end +local format_t=function(f) + n=n+1 + if f and f~="" then + return format("concat(a%s,%q)",n,f) + else + return format("concat(a%s)",n) + end +end +local format_T=function(f) + n=n+1 + if f and f~="" then + return format("sequenced(a%s,%q)",n,f) + else + return format("sequenced(a%s)",n) + end +end +local format_l=function() + n=n+1 + return format("(a%s and 'true' or 'false')",n) +end +local format_L=function() + n=n+1 + return format("(a%s and 'TRUE' or 'FALSE')",n) +end +local format_N=function() + n=n+1 + return format("tostring(tonumber(a%s) or a%s)",n,n) +end +local format_a=function(f) + n=n+1 + if f and f~="" then + return format("autosingle(a%s,%q)",n,f) + else + return format("autosingle(a%s)",n) + end +end +local format_A=function(f) + n=n+1 + if f and f~="" then + return format("autodouble(a%s,%q)",n,f) + else + return format("autodouble(a%s)",n) + end +end +local format_w=function(f) + n=n+1 + f=tonumber(f) + if f then + return format("nspaces[%s+a%s]",f,n) + else + return format("nspaces[a%s]",n) + end +end +local format_W=function(f) + return format("nspaces[%s]",tonumber(f) or 0) +end +local digit=patterns.digit +local period=patterns.period +local three=digit*digit*digit +local splitter=Cs ( + (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) +) +patterns.formattednumber=splitter +function number.formatted(n,sep1,sep2) + local s=type(s)=="string" and n or format("%0.2f",n) + if sep1==true then + return lpegmatch(splitter,s,1,".",",") + elseif sep1=="." then + return lpegmatch(splitter,s,1,sep1,sep2 or ",") + elseif sep1=="," then + return lpegmatch(splitter,s,1,sep1,sep2 or ".") + else + return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") + end +end +local format_m=function(f) + n=n+1 + if not f or f=="" then + f="," + end + return format([[formattednumber(a%s,%q,".")]],n,f) +end +local format_M=function(f) + n=n+1 + if not f or f=="" then + f="." + end + return format([[formattednumber(a%s,%q,",")]],n,f) +end +local format_z=function(f) + n=n+(tonumber(f) or 1) + return "''" +end +local format_rest=function(s) + return format("%q",s) +end +local format_extension=function(extensions,f,name) + local extension=extensions[name] or "tostring(%s)" + local f=tonumber(f) or 1 + if f==0 then + return extension + elseif f==1 then + n=n+1 + local a="a"..n + return format(extension,a,a) + elseif f<0 then + local a="a"..(n+f+1) + return format(extension,a,a) + else + local t={} + for i=1,f do + n=n+1 + t[#t+1]="a"..n + end + return format(extension,unpack(t)) + end +end +local builder=Cs { "start", + start=( + ( + P("%")/""*( + V("!") ++V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") ++V("c")+V("C")+V("S") ++V("Q") ++V("N") ++V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") ++V("W") ++V("a") ++V("A") ++V("j")+V("J") ++V("m")+V("M") ++V("z") + )+V("*") + )*(P(-1)+Carg(1)) + )^0, + ["s"]=(prefix_any*P("s"))/format_s, + ["q"]=(prefix_any*P("q"))/format_q, + ["i"]=(prefix_any*P("i"))/format_i, + ["d"]=(prefix_any*P("d"))/format_d, + ["f"]=(prefix_any*P("f"))/format_f, + ["F"]=(prefix_any*P("F"))/format_F, + ["g"]=(prefix_any*P("g"))/format_g, + ["G"]=(prefix_any*P("G"))/format_G, + ["e"]=(prefix_any*P("e"))/format_e, + ["E"]=(prefix_any*P("E"))/format_E, + ["x"]=(prefix_any*P("x"))/format_x, + ["X"]=(prefix_any*P("X"))/format_X, + ["o"]=(prefix_any*P("o"))/format_o, + ["S"]=(prefix_any*P("S"))/format_S, + ["Q"]=(prefix_any*P("Q"))/format_S, + ["N"]=(prefix_any*P("N"))/format_N, + ["c"]=(prefix_any*P("c"))/format_c, + ["C"]=(prefix_any*P("C"))/format_C, + ["r"]=(prefix_any*P("r"))/format_r, + ["h"]=(prefix_any*P("h"))/format_h, + ["H"]=(prefix_any*P("H"))/format_H, + ["u"]=(prefix_any*P("u"))/format_u, + ["U"]=(prefix_any*P("U"))/format_U, + ["p"]=(prefix_any*P("p"))/format_p, + ["b"]=(prefix_any*P("b"))/format_b, + ["t"]=(prefix_tab*P("t"))/format_t, + ["T"]=(prefix_tab*P("T"))/format_T, + ["l"]=(prefix_any*P("l"))/format_l, + ["L"]=(prefix_any*P("L"))/format_L, + ["I"]=(prefix_any*P("I"))/format_I, + ["w"]=(prefix_any*P("w"))/format_w, + ["W"]=(prefix_any*P("W"))/format_W, + ["j"]=(prefix_any*P("j"))/format_j, + ["J"]=(prefix_any*P("J"))/format_J, + ["m"]=(prefix_tab*P("m"))/format_m, + ["M"]=(prefix_tab*P("M"))/format_M, + ["z"]=(prefix_any*P("z"))/format_z, + ["a"]=(prefix_any*P("a"))/format_a, + ["A"]=(prefix_any*P("A"))/format_A, + ["*"]=Cs(((1-P("%"))^1+P("%%")/"%%")^1)/format_rest, + ["?"]=Cs(((1-P("%"))^1 )^1)/format_rest, + ["!"]=Carg(2)*prefix_any*P("!")*C((1-P("!"))^1)*P("!")/format_extension, +} +local direct=Cs ( + P("%")*(S("+- .")+R("09"))^0*S("sqidfgGeExXo")*P(-1)/[[local format = string.format return function(str) return format("%0",str) end]] +) +local function make(t,str) + local f + local p + local p=lpegmatch(direct,str) + if p then + f=loadstripped(p)() + else + n=0 + p=lpegmatch(builder,str,1,t._connector_,t._extensions_) + if n>0 then + p=format(template,preamble,t._preamble_,arguments[n],p) + f=loadstripped(p,t._environment_)() + else + f=function() return str end + end + end + t[str]=f + return f +end +local function use(t,fmt,...) + return t[fmt](...) +end +strings.formatters={} +if _LUAVERSION<5.2 then + function strings.formatters.new(noconcat) + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_=preamble,_environment_={} } + setmetatable(t,{ __index=make,__call=use }) + return t + end +else + function strings.formatters.new(noconcat) + local e={} + for k,v in next,environment do + e[k]=v + end + local t={ _type_="formatter",_connector_=noconcat and "," or "..",_extensions_={},_preamble_="",_environment_=e } + setmetatable(t,{ __index=make,__call=use }) + return t + end +end +local formatters=strings.formatters.new() +string.formatters=formatters +string.formatter=function(str,...) return formatters[str](...) end +local function add(t,name,template,preamble) + if type(t)=="table" and t._type_=="formatter" then + t._extensions_[name]=template or "%s" + if type(preamble)=="string" then + t._preamble_=preamble.."\n"..t._preamble_ + elseif type(preamble)=="table" then + for k,v in next,preamble do + t._environment_[k]=v + end + end + end +end +strings.formatters.add=add +patterns.xmlescape=Cs((P("<")/"<"+P(">")/">"+P("&")/"&"+P('"')/"""+P(1))^0) +patterns.texescape=Cs((C(S("#$%\\{}"))/"\\%1"+P(1))^0) +patterns.luaescape=Cs(((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0) +patterns.luaquoted=Cs(Cc('"')*((1-S('"\n'))^1+P('"')/'\\"'+P('\n')/'\\n"')^0*Cc('"')) +if _LUAVERSION<5.2 then + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],"local xmlescape = lpeg.patterns.xmlescape") + add(formatters,"tex",[[lpegmatch(texescape,%s)]],"local texescape = lpeg.patterns.texescape") + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],"local luaescape = lpeg.patterns.luaescape") +else + add(formatters,"xml",[[lpegmatch(xmlescape,%s)]],{ xmlescape=lpeg.patterns.xmlescape }) + add(formatters,"tex",[[lpegmatch(texescape,%s)]],{ texescape=lpeg.patterns.texescape }) + add(formatters,"lua",[[lpegmatch(luaescape,%s)]],{ luaescape=lpeg.patterns.luaescape }) +end +local dquote=patterns.dquote +local equote=patterns.escaped+dquote/'\\"'+1 +local space=patterns.space +local cquote=Cc('"') +local pattern=Cs(dquote*(equote-P(-2))^0*dquote) ++Cs(cquote*(equote-space)^0*space*equote^0*cquote) +function string.optionalquoted(str) + return lpegmatch(pattern,str) or str +end +local pattern=Cs((newline/os.newline+1)^0) +function string.replacenewlines(str) + return lpegmatch(pattern,str) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-tab"] = package.loaded["util-tab"] or true + +-- original size: 25338, stripped down to: 16247 + +if not modules then modules={} end modules ['util-tab']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.tables=utilities.tables or {} +local tables=utilities.tables +local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub +local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort +local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select +local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc +local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs +local formatters=string.formatters +local utftoeight=utf.toeight +local splitter=lpeg.tsplitat(".") +function utilities.tables.definetable(target,nofirst,nolast) + local composed,t=nil,{} + local snippets=lpegmatch(splitter,target) + for i=1,#snippets-(nolast and 1 or 0) do + local name=snippets[i] + if composed then + composed=composed.."."..name + t[#t+1]=formatters["if not %s then %s = { } end"](composed,composed) + else + composed=name + if not nofirst then + t[#t+1]=formatters["%s = %s or { }"](composed,composed) + end + end + end + if composed then + if nolast then + composed=composed.."."..snippets[#snippets] + end + return concat(t,"\n"),composed + else + return "",target + end +end +function tables.definedtable(...) + local t=_G + for i=1,select("#",...) do + local li=select(i,...) + local tl=t[li] + if not tl then + tl={} + t[li]=tl + end + t=tl + end + return t +end +function tables.accesstable(target,root) + local t=root or _G + for name in gmatch(target,"([^%.]+)") do + t=t[name] + if not t then + return + end + end + return t +end +function tables.migratetable(target,v,root) + local t=root or _G + local names=lpegmatch(splitter,target) + for i=1,#names-1 do + local name=names[i] + t[name]=t[name] or {} + t=t[name] + if not t then + return + end + end + t[names[#names]]=v +end +function tables.removevalue(t,value) + if value then + for i=1,#t do + if t[i]==value then + remove(t,i) + end + end + end +end +function tables.replacevalue(t,oldvalue,newvalue) + if oldvalue and newvalue then + for i=1,#t do + if t[i]==oldvalue then + t[i]=newvalue + end + end + end +end +function tables.insertbeforevalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end +function tables.insertaftervalue(t,value,extra) + for i=1,#t do + if t[i]==extra then + remove(t,i) + end + end + for i=1,#t do + if t[i]==value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end +local escape=Cs(Cc('"')*((P('"')/'""'+P(1))^0)*Cc('"')) +function table.tocsv(t,specification) + if t and #t>0 then + local result={} + local r={} + specification=specification or {} + local fields=specification.fields + if type(fields)~="string" then + fields=sortedkeys(t[1]) + end + local separator=specification.separator or "," + if specification.preamble==true then + for f=1,#fields do + r[f]=lpegmatch(escape,tostring(fields[f])) + end + result[1]=concat(r,separator) + end + for i=1,#t do + local ti=t[i] + for f=1,#fields do + local field=ti[fields[f]] + if type(field)=="string" then + r[f]=lpegmatch(escape,field) + else + r[f]=tostring(field) + end + end + result[#result+1]=concat(r,separator) + end + return concat(result,"\n") + else + return "" + end +end +local nspaces=utilities.strings.newrepeater(" ") +local function toxml(t,d,result,step) + for k,v in sortedpairs(t) do + local s=nspaces[d] + local tk=type(k) + local tv=type(v) + if tv=="table" then + if tk=="number" then + result[#result+1]=formatters["%s<entry n='%s'>"](s,k) + toxml(v,d+step,result,step) + result[#result+1]=formatters["%s</entry>"](s,k) + else + result[#result+1]=formatters["%s<%s>"](s,k) + toxml(v,d+step,result,step) + result[#result+1]=formatters["%s</%s>"](s,k) + end + elseif tv=="string" then + if tk=="number" then + result[#result+1]=formatters["%s<entry n='%s'>%!xml!</entry>"](s,k,v,k) + else + result[#result+1]=formatters["%s<%s>%!xml!</%s>"](s,k,v,k) + end + elseif tk=="number" then + result[#result+1]=formatters["%s<entry n='%s'>%S</entry>"](s,k,v,k) + else + result[#result+1]=formatters["%s<%s>%S</%s>"](s,k,v,k) + end + end +end +function table.toxml(t,specification) + specification=specification or {} + local name=specification.name + local noroot=name==false + local result=(specification.nobanner or noroot) and {} or { "<?xml version='1.0' standalone='yes' ?>" } + local indent=specification.indent or 0 + local spaces=specification.spaces or 1 + if noroot then + toxml(t,indent,result,spaces) + else + toxml({ [name or "data"]=t },indent,result,spaces) + end + return concat(result,"\n") +end +function tables.encapsulate(core,capsule,protect) + if type(capsule)~="table" then + protect=true + capsule={} + end + for key,value in next,core do + if capsule[key] then + print(formatters["\ninvalid %s %a in %a"]("inheritance",key,core)) + os.exit() + else + capsule[key]=value + end + end + if protect then + for key,value in next,core do + core[key]=nil + end + setmetatable(core,{ + __index=capsule, + __newindex=function(t,key,value) + if capsule[key] then + print(formatters["\ninvalid %s %a' in %a"]("overload",key,core)) + os.exit() + else + rawset(t,key,value) + end + end + } ) + end +end +local f_hashed_string=formatters["[%q]=%q,"] +local f_hashed_number=formatters["[%q]=%s,"] +local f_hashed_boolean=formatters["[%q]=%l,"] +local f_hashed_table=formatters["[%q]="] +local f_indexed_string=formatters["[%s]=%q,"] +local f_indexed_number=formatters["[%s]=%s,"] +local f_indexed_boolean=formatters["[%s]=%l,"] +local f_indexed_table=formatters["[%s]="] +local f_ordered_string=formatters["%q,"] +local f_ordered_number=formatters["%s,"] +local f_ordered_boolean=formatters["%l,"] +function table.fastserialize(t,prefix) + local r={ type(prefix)=="string" and prefix or "return" } + local m=1 + local function fastserialize(t,outer) + local n=#t + m=m+1 + r[m]="{" + if n>0 then + for i=0,n do + local v=t[i] + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_ordered_string(v) + elseif tv=="number" then + m=m+1 r[m]=f_ordered_number(v) + elseif tv=="table" then + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_ordered_boolean(v) + end + end + end + for k,v in next,t do + local tk=type(k) + if tk=="number" then + if k>n or k<0 then + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_indexed_string(k,v) + elseif tv=="number" then + m=m+1 r[m]=f_indexed_number(k,v) + elseif tv=="table" then + m=m+1 r[m]=f_indexed_table(k) + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_indexed_boolean(k,v) + end + end + else + local tv=type(v) + if tv=="string" then + m=m+1 r[m]=f_hashed_string(k,v) + elseif tv=="number" then + m=m+1 r[m]=f_hashed_number(k,v) + elseif tv=="table" then + m=m+1 r[m]=f_hashed_table(k) + fastserialize(v) + elseif tv=="boolean" then + m=m+1 r[m]=f_hashed_boolean(k,v) + end + end + end + m=m+1 + if outer then + r[m]="}" + else + r[m]="}," + end + return r + end + return concat(fastserialize(t,true)) +end +function table.deserialize(str) + if not str or str=="" then + return + end + local code=load(str) + if not code then + return + end + code=code() + if not code then + return + end + return code +end +function table.load(filename,loader) + if filename then + local t=(loader or io.loaddata)(filename) + if t and t~="" then + local t=utftoeight(t) + t=load(t) + if type(t)=="function" then + t=t() + if type(t)=="table" then + return t + end + end + end + end +end +function table.save(filename,t,n,...) + io.savedata(filename,table.serialize(t,n==nil and true or n,...)) +end +local f_key_value=formatters["%s=%q"] +local f_add_table=formatters[" {%t},\n"] +local f_return_table=formatters["return {\n%t}"] +local function slowdrop(t) + local r={} + local l={} + for i=1,#t do + local ti=t[i] + local j=0 + for k,v in next,ti do + j=j+1 + l[j]=f_key_value(k,v) + end + r[i]=f_add_table(l) + end + return f_return_table(r) +end +local function fastdrop(t) + local r={ "return {\n" } + local m=1 + for i=1,#t do + local ti=t[i] + m=m+1 r[m]=" {" + for k,v in next,ti do + m=m+1 r[m]=f_key_value(k,v) + end + m=m+1 r[m]="},\n" + end + m=m+1 + r[m]="}" + return concat(r) +end +function table.drop(t,slow) + if #t==0 then + return "return { }" + elseif slow==true then + return slowdrop(t) + else + return fastdrop(t) + end +end +function table.autokey(t,k) + local v={} + t[k]=v + return v +end +local selfmapper={ __index=function(t,k) t[k]=k return k end } +function table.twowaymapper(t) + if not t then + t={} + else + for i=0,#t do + local ti=t[i] + if ti then + local i=tostring(i) + t[i]=ti + t[ti]=i + end + end + t[""]=t[0] or "" + end + setmetatable(t,selfmapper) + return t +end +local f_start_key_idx=formatters["%w{"] +local f_start_key_num=formatters["%w[%s]={"] +local f_start_key_str=formatters["%w[%q]={"] +local f_start_key_boo=formatters["%w[%l]={"] +local f_start_key_nop=formatters["%w{"] +local f_stop=formatters["%w},"] +local f_key_num_value_num=formatters["%w[%s]=%s,"] +local f_key_str_value_num=formatters["%w[%q]=%s,"] +local f_key_boo_value_num=formatters["%w[%l]=%s,"] +local f_key_num_value_str=formatters["%w[%s]=%q,"] +local f_key_str_value_str=formatters["%w[%q]=%q,"] +local f_key_boo_value_str=formatters["%w[%l]=%q,"] +local f_key_num_value_boo=formatters["%w[%s]=%l,"] +local f_key_str_value_boo=formatters["%w[%q]=%l,"] +local f_key_boo_value_boo=formatters["%w[%l]=%l,"] +local f_key_num_value_not=formatters["%w[%s]={},"] +local f_key_str_value_not=formatters["%w[%q]={},"] +local f_key_boo_value_not=formatters["%w[%l]={},"] +local f_key_num_value_seq=formatters["%w[%s]={ %, t },"] +local f_key_str_value_seq=formatters["%w[%q]={ %, t },"] +local f_key_boo_value_seq=formatters["%w[%l]={ %, t },"] +local f_val_num=formatters["%w%s,"] +local f_val_str=formatters["%w%q,"] +local f_val_boo=formatters["%w%l,"] +local f_val_not=formatters["%w{},"] +local f_val_seq=formatters["%w{ %, t },"] +local f_table_return=formatters["return {"] +local f_table_name=formatters["%s={"] +local f_table_direct=formatters["{"] +local f_table_entry=formatters["[%q]={"] +local f_table_finish=formatters["}"] +local spaces=utilities.strings.newrepeater(" ") +local serialize=table.serialize +function table.serialize(root,name,specification) + if type(specification)=="table" then + return serialize(root,name,specification) + end + local t + local n=1 + local function simple_table(t) + local nt=#t + if nt>0 then + local n=0 + for _,v in next,t do + n=n+1 + if type(v)=="table" then + return nil + end + end + if n==nt then + local tt={} + for i=1,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + tt[i]=v + elseif tv=="string" then + tt[i]=format("%q",v) + elseif tv=="boolean" then + tt[i]=v and "true" or "false" + else + return nil + end + end + return tt + end + end + return nil + end + local function do_serialize(root,name,depth,level,indexed) + if level>0 then + n=n+1 + if indexed then + t[n]=f_start_key_idx(depth) + else + local tn=type(name) + if tn=="number" then + t[n]=f_start_key_num(depth,name) + elseif tn=="string" then + t[n]=f_start_key_str(depth,name) + elseif tn=="boolean" then + t[n]=f_start_key_boo(depth,name) + else + t[n]=f_start_key_nop(depth) + end + end + depth=depth+1 + end + if root and next(root)~=nil then + local first=nil + local last=0 + last=#root + for k=1,last do + if root[k]==nil then + last=k-1 + break + end + end + if last>0 then + first=1 + end + local sk=sortedkeys(root) + for i=1,#sk do + local k=sk[i] + local v=root[k] + local tv=type(v) + local tk=type(k) + if first and tk=="number" and k<=last and k>=first then + if tv=="number" then + n=n+1 t[n]=f_val_num(depth,v) + elseif tv=="string" then + n=n+1 t[n]=f_val_str(depth,v) + elseif tv=="table" then + if next(v)==nil then + n=n+1 t[n]=f_val_not(depth) + else + local st=simple_table(v) + if st then + n=n+1 t[n]=f_val_seq(depth,st) + else + do_serialize(v,k,depth,level+1,true) + end + end + elseif tv=="boolean" then + n=n+1 t[n]=f_val_boo(depth,v) + end + elseif tv=="number" then + if tk=="number" then + n=n+1 t[n]=f_key_num_value_num(depth,k,v) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_num(depth,k,v) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_num(depth,k,v) + end + elseif tv=="string" then + if tk=="number" then + n=n+1 t[n]=f_key_num_value_str(depth,k,v) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_str(depth,k,v) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_str(depth,k,v) + end + elseif tv=="table" then + if next(v)==nil then + if tk=="number" then + n=n+1 t[n]=f_key_num_value_not(depth,k) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_not(depth,k) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_not(depth,k) + end + else + local st=simple_table(v) + if not st then + do_serialize(v,k,depth,level+1) + elseif tk=="number" then + n=n+1 t[n]=f_key_num_value_seq(depth,k,st) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_seq(depth,k,st) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_seq(depth,k,st) + end + end + elseif tv=="boolean" then + if tk=="number" then + n=n+1 t[n]=f_key_num_value_boo(depth,k,v) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_boo(depth,k,v) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_boo(depth,k,v) + end + end + end + end + if level>0 then + n=n+1 t[n]=f_stop(depth-1) + end + end + local tname=type(name) + if tname=="string" then + if name=="return" then + t={ f_table_return() } + else + t={ f_table_name(name) } + end + elseif tname=="number" then + t={ f_table_entry(name) } + elseif tname=="boolean" then + if name then + t={ f_table_return() } + else + t={ f_table_direct() } + end + else + t={ f_table_name("t") } + end + if root then + if getmetatable(root) then + local dummy=root._w_h_a_t_e_v_e_r_ + root._w_h_a_t_e_v_e_r_=nil + end + if next(root)~=nil then + do_serialize(root,name,1,0) + end + end + n=n+1 + t[n]=f_table_finish() + return concat(t,"\n") +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sto"] = package.loaded["util-sto"] or true + +-- original size: 4172, stripped down to: 2953 + +if not modules then modules={} end modules ['util-sto']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local setmetatable,getmetatable,type=setmetatable,getmetatable,type +utilities=utilities or {} +utilities.storage=utilities.storage or {} +local storage=utilities.storage +function storage.mark(t) + if not t then + print("\nfatal error: storage cannot be marked\n") + os.exit() + return + end + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.allocate(t) + t=t or {} + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m.__storage__=true + return t +end +function storage.marked(t) + local m=getmetatable(t) + return m and m.__storage__ +end +function storage.checked(t) + if not t then + report("\nfatal error: storage has not been allocated\n") + os.exit() + return + end + return t +end +function storage.setinitializer(data,initialize) + local m=getmetatable(data) or {} + m.__index=function(data,k) + m.__index=nil + initialize() + return data[k] + end + setmetatable(data,m) +end +local keyisvalue={ __index=function(t,k) + t[k]=k + return k +end } +function storage.sparse(t) + t=t or {} + setmetatable(t,keyisvalue) + return t +end +local function f_empty () return "" end +local function f_self (t,k) t[k]=k return k end +local function f_table (t,k) local v={} t[k]=v return v end +local function f_number(t,k) t[k]=0 return 0 end +local function f_ignore() end +local f_index={ + ["empty"]=f_empty, + ["self"]=f_self, + ["table"]=f_table, + ["number"]=f_number, +} +local t_index={ + ["empty"]={ __index=f_empty }, + ["self"]={ __index=f_self }, + ["table"]={ __index=f_table }, + ["number"]={ __index=f_number }, +} +function table.setmetatableindex(t,f) + if type(t)~="table" then + f,t=t,{} + end + local m=getmetatable(t) + if m then + m.__index=f_index[f] or f + else + setmetatable(t,t_index[f] or { __index=f }) + end + return t +end +local f_index={ + ["ignore"]=f_ignore, +} +local t_index={ + ["ignore"]={ __newindex=f_ignore }, +} +function table.setmetatablenewindex(t,f) + if type(t)~="table" then + f,t=t,{} + end + local m=getmetatable(t) + if m then + m.__newindex=f_index[f] or f + else + setmetatable(t,t_index[f] or { __newindex=f }) + end + return t +end +function table.setmetatablecall(t,f) + if type(t)~="table" then + f,t=t,{} + end + local m=getmetatable(t) + if m then + m.__call=f + else + setmetatable(t,{ __call=f }) + end + return t +end +function table.setmetatablekey(t,key,value) + local m=getmetatable(t) + if not m then + m={} + setmetatable(t,m) + end + m[key]=value + return t +end +function table.getmetatablekey(t,key,value) + local m=getmetatable(t) + return m and m[key] +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-prs"] = package.loaded["util-prs"] or true + +-- original size: 21780, stripped down to: 15121 + +if not modules then modules={} end modules ['util-prs']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local lpeg,table,string=lpeg,table,string +local P,R,V,S,C,Ct,Cs,Carg,Cc,Cg,Cf,Cp=lpeg.P,lpeg.R,lpeg.V,lpeg.S,lpeg.C,lpeg.Ct,lpeg.Cs,lpeg.Carg,lpeg.Cc,lpeg.Cg,lpeg.Cf,lpeg.Cp +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local concat,gmatch,find=table.concat,string.gmatch,string.find +local tostring,type,next,rawset=tostring,type,next,rawset +local mod,div=math.mod,math.div +utilities=utilities or {} +local parsers=utilities.parsers or {} +utilities.parsers=parsers +local patterns=parsers.patterns or {} +parsers.patterns=patterns +local setmetatableindex=table.setmetatableindex +local sortedhash=table.sortedhash +local sortedkeys=table.sortedkeys +local tohash=table.tohash +local digit=R("09") +local space=P(' ') +local equal=P("=") +local comma=P(",") +local lbrace=P("{") +local rbrace=P("}") +local lparent=P("(") +local rparent=P(")") +local period=S(".") +local punctuation=S(".,:;") +local spacer=lpegpatterns.spacer +local whitespace=lpegpatterns.whitespace +local newline=lpegpatterns.newline +local anything=lpegpatterns.anything +local endofstring=lpegpatterns.endofstring +local nobrace=1-(lbrace+rbrace ) +local noparent=1-(lparent+rparent) +local escape,left,right=P("\\"),P('{'),P('}') +lpegpatterns.balanced=P { + [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, + [2]=left*V(1)*right +} +local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } +local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local spaces=space^0 +local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) +local content=(1-endofstring)^0 +lpegpatterns.nestedbraces=nestedbraces +lpegpatterns.nestedparents=nestedparents +lpegpatterns.nested=nestedbraces +lpegpatterns.argument=argument +lpegpatterns.content=content +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local key=C((1-equal-comma)^1) +local pattern_a=(space+comma)^0*(key*equal*value+key*C("")) +local pattern_c=(space+comma)^0*(key*equal*value) +local key=C((1-space-equal-comma)^1) +local pattern_b=spaces*comma^0*spaces*(key*((spaces*equal*spaces*value)+C(""))) +local hash={} +local function set(key,value) + hash[key]=value +end +local pattern_a_s=(pattern_a/set)^1 +local pattern_b_s=(pattern_b/set)^1 +local pattern_c_s=(pattern_c/set)^1 +patterns.settings_to_hash_a=pattern_a_s +patterns.settings_to_hash_b=pattern_b_s +patterns.settings_to_hash_c=pattern_c_s +function parsers.make_settings_to_hash_pattern(set,how) + if how=="strict" then + return (pattern_c/set)^1 + elseif how=="tolerant" then + return (pattern_b/set)^1 + else + return (pattern_a/set)^1 + end +end +function parsers.settings_to_hash(str,existing) + if not str or str=="" then + return {} + elseif type(str)=="table" then + if existing then + for k,v in next,str do + existing[k]=v + end + return exiting + else + return str + end + else + hash=existing or {} + lpegmatch(pattern_a_s,str) + return hash + end +end +function parsers.settings_to_hash_tolerant(str,existing) + if not str or str=="" then + return {} + elseif type(str)=="table" then + if existing then + for k,v in next,str do + existing[k]=v + end + return exiting + else + return str + end + else + hash=existing or {} + lpegmatch(pattern_b_s,str) + return hash + end +end +function parsers.settings_to_hash_strict(str,existing) + if not str or str=="" then + return nil + elseif type(str)=="table" then + if existing then + for k,v in next,str do + existing[k]=v + end + return exiting + else + return str + end + elseif str and str~="" then + hash=existing or {} + lpegmatch(pattern_c_s,str) + return next(hash) and hash + end +end +local separator=comma*space^0 +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +patterns.settings_to_array=pattern +function parsers.settings_to_array(str,strict) + if not str or str=="" then + return {} + elseif type(str)=="table" then + return str + elseif strict then + if find(str,"{",1,true) then + return lpegmatch(pattern,str) + else + return { str } + end + elseif find(str,",",1,true) then + return lpegmatch(pattern,str) + else + return { str } + end +end +local cache_a={} +local cache_b={} +function parsers.groupedsplitat(symbol,withaction) + if not symbol then + symbol="," + end + local pattern=(withaction and cache_b or cache_a)[symbol] + if not pattern then + local symbols=S(symbol) + local separator=space^0*symbols*space^0 + local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+(1-(space^0*(symbols+P(-1)))))^0) + if withaction then + local withvalue=Carg(1)*value/function(f,s) return f(s) end + pattern=spaces*withvalue*(separator*withvalue)^0 + cache_b[symbol]=pattern + else + pattern=spaces*Ct(value*(separator*value)^0) + cache_a[symbol]=pattern + end + end + return pattern +end +local pattern_a=parsers.groupedsplitat(",",false) +local pattern_b=parsers.groupedsplitat(",",true) +function parsers.stripped_settings_to_array(str) + if not str or str=="" then + return {} + else + return lpegmatch(pattern_a,str) + end +end +function parsers.process_stripped_settings(str,action) + if not str or str=="" then + return {} + else + return lpegmatch(pattern_b,str,1,action) + end +end +local function set(t,v) + t[#t+1]=v +end +local value=P(Carg(1)*value)/set +local pattern=value*(separator*value)^0*Carg(1) +function parsers.add_settings_to_array(t,str) + return lpegmatch(pattern,str,nil,t) +end +function parsers.hash_to_string(h,separator,yes,no,strict,omit) + if h then + local t,tn,s={},0,sortedkeys(h) + omit=omit and tohash(omit) + for i=1,#s do + local key=s[i] + if not omit or not omit[key] then + local value=h[key] + if type(value)=="boolean" then + if yes and no then + if value then + tn=tn+1 + t[tn]=key..'='..yes + elseif not strict then + tn=tn+1 + t[tn]=key..'='..no + end + elseif value or not strict then + tn=tn+1 + t[tn]=key..'='..tostring(value) + end + else + tn=tn+1 + t[tn]=key..'='..value + end + end + end + return concat(t,separator or ",") + else + return "" + end +end +function parsers.array_to_string(a,separator) + if a then + return concat(a,separator or ",") + else + return "" + end +end +local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) +function utilities.parsers.settings_to_set(str,t) + return str and lpegmatch(pattern,str) or {} +end +function parsers.simple_hash_to_string(h,separator) + local t,tn={},0 + for k,v in sortedhash(h) do + if v then + tn=tn+1 + t[tn]=k + end + end + return concat(t,separator or ",") +end +local str=Cs(lpegpatterns.unquoted)+C((1-whitespace-equal)^1) +local setting=Cf(Carg(1)*(whitespace^0*Cg(str*whitespace^0*(equal*whitespace^0*str+Cc(""))))^1,rawset) +local splitter=setting^1 +function utilities.parsers.options_to_hash(str,target) + return str and lpegmatch(splitter,str,1,target or {}) or {} +end +local splitter=lpeg.tsplitat(" ") +function utilities.parsers.options_to_array(str) + return str and lpegmatch(splitter,str) or {} +end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C(digit^1*lparent*(noparent+nestedparents)^1*rparent)+C((nestedbraces+(1-comma))^1) +local pattern_a=spaces*Ct(value*(separator*value)^0) +local function repeater(n,str) + if not n then + return str + else + local s=lpegmatch(pattern_a,str) + if n==1 then + return unpack(s) + else + local t,tn={},0 + for i=1,n do + for j=1,#s do + tn=tn+1 + t[tn]=s[j] + end + end + return unpack(t) + end + end +end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+(C(digit^1)/tonumber*lparent*Cs((noparent+nestedparents)^1)*rparent)/repeater+C((nestedbraces+(1-comma))^1) +local pattern_b=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_with_repeat(str,expand) + if expand then + return lpegmatch(pattern_b,str) or {} + else + return lpegmatch(pattern_a,str) or {} + end +end +local value=lbrace*C((nobrace+nestedbraces)^0)*rbrace +local pattern=Ct((space+value)^0) +function parsers.arguments_to_table(str) + return lpegmatch(pattern,str) +end +function parsers.getparameters(self,class,parentclass,settings) + local sc=self[class] + if not sc then + sc={} + self[class]=sc + if parentclass then + local sp=self[parentclass] + if not sp then + sp={} + self[parentclass]=sp + end + setmetatableindex(sc,sp) + end + end + parsers.settings_to_hash(settings,sc) +end +function parsers.listitem(str) + return gmatch(str,"[^, ]+") +end +local pattern=Cs { "start", + start=V("one")+V("two")+V("three"), + rest=(Cc(",")*V("thousand"))^0*(P(".")+endofstring)*anything^0, + thousand=digit*digit*digit, + one=digit*V("rest"), + two=digit*digit*V("rest"), + three=V("thousand")*V("rest"), +} +lpegpatterns.splitthousands=pattern +function parsers.splitthousands(str) + return lpegmatch(pattern,str) or str +end +local optionalwhitespace=whitespace^0 +lpegpatterns.words=Ct((Cs((1-punctuation-whitespace)^1)+anything)^1) +lpegpatterns.sentences=Ct((optionalwhitespace*Cs((1-period)^0*period))^1) +lpegpatterns.paragraphs=Ct((optionalwhitespace*Cs((whitespace^1*endofstring/""+1-(spacer^0*newline*newline))^1))^1) +local dquote=P('"') +local equal=P('=') +local escape=P('\\') +local separator=S(' ,') +local key=C((1-equal)^1) +local value=dquote*C((1-dquote-escape*dquote)^0)*dquote +local pattern=Cf(Ct("")*(Cg(key*equal*value)*separator^0)^1,rawset)^0*P(-1) +function parsers.keq_to_hash(str) + if str and str~="" then + return lpegmatch(pattern,str) + else + return {} + end +end +local defaultspecification={ separator=",",quote='"' } +function parsers.csvsplitter(specification) + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=specification.quote + local separator=S(separator~="" and separator or ",") + local whatever=C((1-separator-newline)^0) + if quotechar and quotechar~="" then + local quotedata=nil + for chr in gmatch(quotechar,".") do + local quotechar=P(chr) + local quoteword=quotechar*C((1-quotechar)^0)*quotechar + if quotedata then + quotedata=quotedata+quoteword + else + quotedata=quoteword + end + end + whatever=quotedata+whatever + end + local parser=Ct((Ct(whatever*(separator*whatever)^0)*S("\n\r")^1)^0 ) + return function(data) + return lpegmatch(parser,data) + end +end +function parsers.rfc4180splitter(specification) + specification=specification and setmetatableindex(specification,defaultspecification) or defaultspecification + local separator=specification.separator + local quotechar=P(specification.quote) + local dquotechar=quotechar*quotechar +/specification.quote + local separator=S(separator~="" and separator or ",") + local escaped=quotechar*Cs((dquotechar+(1-quotechar))^0)*quotechar + local non_escaped=C((1-quotechar-newline-separator)^1) + local field=escaped+non_escaped+Cc("") + local record=Ct(field*(separator*field)^1) + local headerline=record*Cp() + local wholeblob=Ct((newline^(specification.strict and -1 or 1)*record)^0) + return function(data,getheader) + if getheader then + local header,position=lpegmatch(headerline,data) + local data=lpegmatch(wholeblob,data,position) + return data,header + else + return lpegmatch(wholeblob,data) + end + end +end +local function ranger(first,last,n,action) + if not first then + elseif last==true then + for i=first,n or first do + action(i) + end + elseif last then + for i=first,last do + action(i) + end + else + action(first) + end +end +local cardinal=lpegpatterns.cardinal/tonumber +local spacers=lpegpatterns.spacer^0 +local endofstring=lpegpatterns.endofstring +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1 +local stepper=spacers*(C(cardinal)*(spacers*S(":-")*spacers*(C(cardinal)+(P("*")+endofstring)*Cc(true) )+Cc(false) )*Carg(1)*Carg(2)/ranger*S(", ")^0 )^1*endofstring +function parsers.stepper(str,n,action) + if type(n)=="function" then + lpegmatch(stepper,str,1,false,n or print) + else + lpegmatch(stepper,str,1,n,action or print) + end +end +local pattern_math=Cs((P("%")/"\\percent "+P("^")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) +local pattern_text=Cs((P("%")/"\\percent "+(P("^")/"\\high")*Cc("{")*lpegpatterns.integer*Cc("}")+anything)^0) +patterns.unittotex=pattern +function parsers.unittotex(str,textmode) + return lpegmatch(textmode and pattern_text or pattern_math,str) +end +local pattern=Cs((P("^")/"<sup>"*lpegpatterns.integer*Cc("</sup>")+anything)^0) +function parsers.unittoxml(str) + return lpegmatch(pattern,str) +end +local cache={} +local spaces=lpegpatterns.space^0 +local dummy=function() end +setmetatableindex(cache,function(t,k) + local separator=P(k) + local value=(1-separator)^0 + local pattern=spaces*C(value)*separator^0*Cp() + t[k]=pattern + return pattern +end) +local commalistiterator=cache[","] +function utilities.parsers.iterator(str,separator) + local n=#str + if n==0 then + return dummy + else + local pattern=separator and cache[separator] or commalistiterator + local p=1 + return function() + if p<=n then + local s,e=lpegmatch(pattern,str,p) + if e then + p=e + return s + end + end + end + end +end +local function initialize(t,name) + local source=t[name] + if source then + local result={} + for k,v in next,t[name] do + result[k]=v + end + return result + else + return {} + end +end +local function fetch(t,name) + return t[name] or {} +end +local function process(result,more) + for k,v in next,more do + result[k]=v + end + return result +end +local name=C((1-S(", "))^1) +local parser=(Carg(1)*name/initialize)*(S(", ")^1*(Carg(1)*name/fetch))^0 +local merge=Cf(parser,process) +function utilities.parsers.mergehashes(hash,list) + return lpegmatch(merge,list,1,hash) +end +function utilities.parsers.runtime(time) + if not time then + time=os.runtime() + end + local days=div(time,24*60*60) + time=mod(time,24*60*60) + local hours=div(time,60*60) + time=mod(time,60*60) + local minutes=div(time,60) + local seconds=mod(time,60) + return days,hours,minutes,seconds +end +local spacing=whitespace^0 +local apply=P("->") +local method=C((1-apply)^1) +local token=lbrace*C((1-rbrace)^1)*rbrace+C(anything^1) +local pattern=spacing*(method*spacing*apply+Carg(1))*spacing*token +function utilities.parsers.splitmethod(str,default) + if str then + return lpegmatch(pattern,str,1,default or false) + else + return default or false,"" + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-fmt"] = package.loaded["util-fmt"] or true + +-- original size: 2274, stripped down to: 1781 + +if not modules then modules={} end modules ['util-fmt']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities=utilities or {} +utilities.formatters=utilities.formatters or {} +local formatters=utilities.formatters +local concat,format=table.concat,string.format +local tostring,type=tostring,type +local strip=string.strip +local lpegmatch=lpeg.match +local stripper=lpeg.patterns.stripzeros +function formatters.stripzeros(str) + return lpegmatch(stripper,str) +end +function formatters.formatcolumns(result,between) + if result and #result>0 then + between=between or " " + local widths,numbers={},{} + local first=result[1] + local n=#first + for i=1,n do + widths[i]=0 + end + for i=1,#result do + local r=result[i] + for j=1,n do + local rj=r[j] + local tj=type(rj) + if tj=="number" then + numbers[j]=true + end + if tj~="string" then + rj=tostring(rj) + r[j]=rj + end + local w=#rj + if w>widths[j] then + widths[j]=w + end + end + end + for i=1,n do + local w=widths[i] + if numbers[i] then + if w>80 then + widths[i]="%s"..between + else + widths[i]="%0"..w.."i"..between + end + else + if w>80 then + widths[i]="%s"..between + elseif w>0 then + widths[i]="%-"..w.."s"..between + else + widths[i]="%s" + end + end + end + local template=strip(concat(widths)) + for i=1,#result do + local str=format(template,unpack(result[i])) + result[i]=strip(str) + end + end + return result +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["trac-set"] = package.loaded["trac-set"] or true + +-- original size: 12482, stripped down to: 8864 + +if not modules then modules={} end modules ['trac-set']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,next,tostring=type,next,tostring +local concat=table.concat +local format,find,lower,gsub,topattern=string.format,string.find,string.lower,string.gsub,string.topattern +local is_boolean=string.is_boolean +local settings_to_hash=utilities.parsers.settings_to_hash +local allocate=utilities.storage.allocate +utilities=utilities or {} +local utilities=utilities +local setters=utilities.setters or {} +utilities.setters=setters +local data={} +local trace_initialize=false +function setters.initialize(filename,name,values) + local setter=data[name] + if setter then + frozen=true + local data=setter.data + if data then + for key,newvalue in next,values do + local newvalue=is_boolean(newvalue,newvalue,true) + local functions=data[key] + if functions then + local oldvalue=functions.value + if functions.frozen then + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"frozen",oldvalue) + end + elseif #functions>0 and not oldvalue then + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"set",newvalue) + end + for i=1,#functions do + functions[i](newvalue) + end + functions.value=newvalue + functions.frozen=functions.frozen or frozen + else + if trace_initialize then + setter.report("%s: %a is %s as %a",filename,key,"kept",oldvalue) + end + end + else + functions={ default=newvalue,frozen=frozen } + data[key]=functions + if trace_initialize then + setter.report("%s: %a is %s to %a",filename,key,"defaulted",newvalue) + end + end + end + return true + end + end +end +local function set(t,what,newvalue) + local data=t.data + if not data.frozen then + local done=t.done + if type(what)=="string" then + what=settings_to_hash(what) + end + if type(what)~="table" then + return + end + if not done then + done={} + t.done=done + end + for w,value in next,what do + if value=="" then + value=newvalue + elseif not value then + value=false + else + value=is_boolean(value,value,true) + end + w=topattern(w,true,true) + for name,functions in next,data do + if done[name] then + elseif find(name,w) then + done[name]=true + for i=1,#functions do + functions[i](value) + end + functions.value=value + end + end + end + end +end +local function reset(t) + local data=t.data + if not data.frozen then + for name,functions in next,data do + for i=1,#functions do + functions[i](false) + end + functions.value=false + end + end +end +local function enable(t,what) + set(t,what,true) +end +local function disable(t,what) + local data=t.data + if not what or what=="" then + t.done={} + reset(t) + else + set(t,what,false) + end +end +function setters.register(t,what,...) + local data=t.data + what=lower(what) + local functions=data[what] + if not functions then + functions={} + data[what]=functions + if trace_initialize then + t.report("defining %a",what) + end + end + local default=functions.default + for i=1,select("#",...) do + local fnc=select(i,...) + local typ=type(fnc) + if typ=="string" then + if trace_initialize then + t.report("coupling %a to %a",what,fnc) + end + local s=fnc + fnc=function(value) set(t,s,value) end + elseif typ~="function" then + fnc=nil + end + if fnc then + functions[#functions+1]=fnc + local value=functions.value or default + if value~=nil then + fnc(value) + functions.value=value + end + end + end + return false +end +function setters.enable(t,what) + local e=t.enable + t.enable,t.done=enable,{} + enable(t,what) + t.enable,t.done=e,{} +end +function setters.disable(t,what) + local e=t.disable + t.disable,t.done=disable,{} + disable(t,what) + t.disable,t.done=e,{} +end +function setters.reset(t) + t.done={} + reset(t) +end +function setters.list(t) + local list=table.sortedkeys(t.data) + local user,system={},{} + for l=1,#list do + local what=list[l] + if find(what,"^%*") then + system[#system+1]=what + else + user[#user+1]=what + end + end + return user,system +end +function setters.show(t) + local category=t.name + local list=setters.list(t) + t.report() + for k=1,#list do + local name=list[k] + local functions=t.data[name] + if functions then + local value,default,modules=functions.value,functions.default,#functions + value=value==nil and "unset" or tostring(value) + default=default==nil and "unset" or tostring(default) + t.report("%-50s modules: %2i default: %-12s value: %-12s",name,modules,default,value) + end + end + t.report() +end +local enable,disable,register,list,show=setters.enable,setters.disable,setters.register,setters.list,setters.show +function setters.report(setter,...) + print(format("%-15s : %s\n",setter.name,format(...))) +end +local function default(setter,name) + local d=setter.data[name] + return d and d.default +end +local function value(setter,name) + local d=setter.data[name] + return d and (d.value or d.default) +end +function setters.new(name) + local setter + setter={ + data=allocate(), + name=name, + report=function(...) setters.report (setter,...) end, + enable=function(...) enable (setter,...) end, + disable=function(...) disable (setter,...) end, + reset=function(...) reset (setter,...) end, + register=function(...) register(setter,...) end, + list=function(...) list (setter,...) end, + show=function(...) show (setter,...) end, + default=function(...) return default (setter,...) end, + value=function(...) return value (setter,...) end, + } + data[name]=setter + return setter +end +trackers=setters.new("trackers") +directives=setters.new("directives") +experiments=setters.new("experiments") +local t_enable,t_disable=trackers .enable,trackers .disable +local d_enable,d_disable=directives .enable,directives .disable +local e_enable,e_disable=experiments.enable,experiments.disable +local trace_directives=false local trace_directives=false trackers.register("system.directives",function(v) trace_directives=v end) +local trace_experiments=false local trace_experiments=false trackers.register("system.experiments",function(v) trace_experiments=v end) +function directives.enable(...) + if trace_directives then + directives.report("enabling: % t",{...}) + end + d_enable(...) +end +function directives.disable(...) + if trace_directives then + directives.report("disabling: % t",{...}) + end + d_disable(...) +end +function experiments.enable(...) + if trace_experiments then + experiments.report("enabling: % t",{...}) + end + e_enable(...) +end +function experiments.disable(...) + if trace_experiments then + experiments.report("disabling: % t",{...}) + end + e_disable(...) +end +directives.register("system.nostatistics",function(v) + if statistics then + statistics.enable=not v + else + end +end) +directives.register("system.nolibraries",function(v) + if libraries then + libraries=nil + else + end +end) +if environment then + local engineflags=environment.engineflags + if engineflags then + local list=engineflags["c:trackers"] or engineflags["trackers"] + if type(list)=="string" then + setters.initialize("commandline flags","trackers",settings_to_hash(list)) + end + local list=engineflags["c:directives"] or engineflags["directives"] + if type(list)=="string" then + setters.initialize("commandline flags","directives",settings_to_hash(list)) + end + end +end +if texconfig then + local function set(k,v) + v=tonumber(v) + if v then + texconfig[k]=v + end + end + directives.register("luatex.expanddepth",function(v) set("expand_depth",v) end) + directives.register("luatex.hashextra",function(v) set("hash_extra",v) end) + directives.register("luatex.nestsize",function(v) set("nest_size",v) end) + directives.register("luatex.maxinopen",function(v) set("max_in_open",v) end) + directives.register("luatex.maxprintline",function(v) set("max_print_line",v) end) + directives.register("luatex.maxstrings",function(v) set("max_strings",v) end) + directives.register("luatex.paramsize",function(v) set("param_size",v) end) + directives.register("luatex.savesize",function(v) set("save_size",v) end) + directives.register("luatex.stacksize",function(v) set("stack_size",v) end) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["trac-log"] = package.loaded["trac-log"] or true + +-- original size: 29359, stripped down to: 20483 + +if not modules then modules={} end modules ['trac-log']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local next,type,select,print=next,type,select,print +local write_nl,write=texio and texio.write_nl or print,texio and texio.write or io.write +local format,gmatch,find=string.format,string.gmatch,string.find +local concat,insert,remove=table.concat,table.insert,table.remove +local topattern=string.topattern +local utfchar=utf.char +local datetime=os.date +local openfile=io.open +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +local texgetcount=tex and tex.getcount +local variant="default" +logs=logs or {} +local logs=logs +local moreinfo=[[ +More information about ConTeXt and the tools that come with it can be found at: +]].."\n"..[[ +maillist : ntg-context@ntg.nl / http://www.ntg.nl/mailman/listinfo/ntg-context +webpage : http://www.pragma-ade.nl / http://tex.aanhet.net +wiki : http://contextgarden.net +]] +formatters.add ( + formatters,"unichr", + [["U+" .. format("%%05X",%s) .. " (" .. utfchar(%s) .. ")"]] +) +formatters.add ( + formatters,"chruni", + [[utfchar(%s) .. " (U+" .. format("%%05X",%s) .. ")"]] +) +local function ignore() end +setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) +local report,subreport,status,settarget,setformats,settranslations +local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline +if tex and (tex.jobname or tex.formatname) then + local function useluawrites() + local texio_write_nl=texio.write_nl + local texio_write=texio.write + local io_write=io.write + write_nl=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write_nl("log",...) + texio_write_nl("term","") + io_write(...) + elseif target=="log" then + texio_write_nl("log",...) + elseif target=="term" then + texio_write_nl("term","") + io_write(...) + elseif target~="none" then + texio_write_nl("log",target,...) + texio_write_nl("term","") + io_write(target,...) + end + end + write=function(target,...) + if not io_write then + io_write=io.write + end + if target=="term and log" then + texio_write("log",...) + io_write(...) + elseif target=="log" then + texio_write("log",...) + elseif target=="term" then + io_write(...) + elseif target~="none" then + texio_write("log",target,...) + io_write(target,...) + end + end + texio.write=write + texio.write_nl=write_nl + useluawrites=ignore + end + local whereto="both" + local target=nil + local targets=nil + local formats=table.setmetatableindex("self") + local translations=table.setmetatableindex("self") + local report_yes,subreport_yes,direct_yes,subdirect_yes,status_yes + local report_nop,subreport_nop,direct_nop,subdirect_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s > %s\n"], + report_nop=formatters["%-15s >\n"], + direct_yes=formatters["%-15s > %s"], + direct_nop=formatters["%-15s >"], + subreport_yes=formatters["%-15s > %s > %s\n"], + subreport_nop=formatters["%-15s > %s >\n"], + subdirect_yes=formatters["%-15s > %s > %s"], + subdirect_nop=formatters["%-15s > %s >"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + targets={ + logfile="log", + log="log", + file="log", + console="term", + terminal="term", + both="term and log", + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;33m%-15s [0;1m>[0m %s\n"], + report_nop=formatters["[0;33m%-15s [0;1m>[0m\n"], + direct_yes=formatters["[0;33m%-15s [0;1m>[0m %s"], + direct_nop=formatters["[0;33m%-15s [0;1m>[0m"], + subreport_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s\n"], + subreport_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m\n"], + subdirect_yes=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m %s"], + subdirect_nop=formatters["[0;33m%-15s [0;1m>[0;35m %s [0;1m>[0m"], + status_yes=formatters["[0;33m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;33m%-15s [0;1m:[0m\n"], + }, + targets={ + logfile="none", + log="none", + file="none", + console="term", + terminal="term", + both="term", + }, + } + } + logs.flush=io.flush + writer=function(...) + write_nl(target,...) + end + newline=function() + write_nl(target,"\n") + end + report=function(a,b,c,...) + if c then + write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) + elseif b then + write_nl(target,report_yes(translations[a],formats[b])) + elseif a then + write_nl(target,report_nop(translations[a])) + else + write_nl(target,"\n") + end + end + direct=function(a,b,c,...) + if c then + return direct_yes(translations[a],formatters[formats[b]](c,...)) + elseif b then + return direct_yes(translations[a],formats[b]) + elseif a then + return direct_nop(translations[a]) + else + return "" + end + end + subreport=function(a,s,b,c,...) + if c then + write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) + elseif b then + write_nl(target,subreport_yes(translations[a],translations[s],formats[b])) + elseif a then + write_nl(target,subreport_nop(translations[a],translations[s])) + else + write_nl(target,"\n") + end + end + subdirect=function(a,s,b,c,...) + if c then + return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) + elseif b then + return subdirect_yes(translations[a],translations[s],formats[b]) + elseif a then + return subdirect_nop(translations[a],translations[s]) + else + return "" + end + end + status=function(a,b,c,...) + if c then + write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) + elseif b then + write_nl(target,status_yes(translations[a],formats[b])) + elseif a then + write_nl(target,status_nop(translations[a])) + else + write_nl(target,"\n") + end + end + settarget=function(askedwhereto) + whereto=askedwhereto or whereto or "both" + target=targets[whereto] + if not target then + whereto="both" + target=targets[whereto] + end + if target=="term" or target=="term and log" then + logs.flush=io.flush + else + logs.flush=ignore + end + end + local stack={} + pushtarget=function(newtarget) + insert(stack,target) + settarget(newtarget) + end + poptarget=function() + if #stack>0 then + settarget(remove(stack)) + end + end + setformats=function(f) + formats=f + end + settranslations=function(t) + translations=t + end + setprocessor=function(f) + local writeline=write_nl + write_nl=function(target,...) + writeline(target,f(...)) + end + end + setformatters=function(specification) + local t=nil + local f=nil + local d=variants.default + if not specification then + elseif type(specification)=="table" then + t=specification.targets + f=specification.formats or specification + else + local v=variants[specification] + if v then + t=v.targets + f=v.formats + variant=specification + end + end + targets=t or d.targets + target=targets[whereto] or target + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + direct_yes=f.direct_yes + direct_nop=f.direct_nop + subdirect_yes=f.subdirect_yes + subdirect_nop=f.subdirect_nop + status_yes=f.status_yes + status_nop=f.status_nop + if variant=="ansi" then + useluawrites() + end + settarget(whereto) + end + setformatters(variant) + setlogfile=ignore + settimedlog=ignore +else + local report_yes,subreport_yes,status_yes + local report_nop,subreport_nop,status_nop + local variants={ + default={ + formats={ + report_yes=formatters["%-15s | %s"], + report_nop=formatters["%-15s |"], + subreport_yes=formatters["%-15s | %s | %s"], + subreport_nop=formatters["%-15s | %s |"], + status_yes=formatters["%-15s : %s\n"], + status_nop=formatters["%-15s :\n"], + }, + }, + ansi={ + formats={ + report_yes=formatters["[0;32m%-15s [0;1m|[0m %s"], + report_nop=formatters["[0;32m%-15s [0;1m|[0m"], + subreport_yes=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m %s"], + subreport_nop=formatters["[0;32m%-15s [0;1m|[0;31m %s [0;1m|[0m"], + status_yes=formatters["[0;32m%-15s [0;1m:[0m %s\n"], + status_nop=formatters["[0;32m%-15s [0;1m:[0m\n"], + }, + }, + } + logs.flush=ignore + writer=function(s) + write_nl(s) + end + newline=function() + write_nl("\n") + end + report=function(a,b,c,...) + if c then + write_nl(report_yes(a,formatters[b](c,...))) + elseif b then + write_nl(report_yes(a,b)) + elseif a then + write_nl(report_nop(a)) + else + write_nl("") + end + end + subreport=function(a,sub,b,c,...) + if c then + write_nl(subreport_yes(a,sub,formatters[b](c,...))) + elseif b then + write_nl(subreport_yes(a,sub,b)) + elseif a then + write_nl(subreport_nop(a,sub)) + else + write_nl("") + end + end + status=function(a,b,c,...) + if c then + write_nl(status_yes(a,formatters[b](c,...))) + elseif b then + write_nl(status_yes(a,b)) + elseif a then + write_nl(status_nop(a)) + else + write_nl("\n") + end + end + direct=ignore + subdirect=ignore + settarget=ignore + pushtarget=ignore + poptarget=ignore + setformats=ignore + settranslations=ignore + setprocessor=function(f) + local writeline=write_nl + write_nl=function(s) + writeline(f(s)) + end + end + setformatters=function(specification) + local f=nil + local d=variants.default + if specification then + if type(specification)=="table" then + f=specification.formats or specification + else + local v=variants[specification] + if v then + f=v.formats + end + end + end + if f then + d=d.formats + else + f=d.formats + d=f + end + setmetatableindex(f,d) + report_yes=f.report_yes + report_nop=f.report_nop + subreport_yes=f.subreport_yes + subreport_nop=f.subreport_nop + status_yes=f.status_yes + status_nop=f.status_nop + end + setformatters(variant) + setlogfile=function(name,keepopen) + if name and name~="" then + local localtime=os.localtime + local writeline=write_nl + if keepopen then + local f=io.open(name,"ab") + write_nl=function(s) + writeline(s) + f:write(localtime()," | ",s,"\n") + end + else + write_nl=function(s) + writeline(s) + local f=io.open(name,"ab") + f:write(localtime()," | ",s,"\n") + f:close() + end + end + end + setlogfile=ignore + end + settimedlog=function() + local localtime=os.localtime + local writeline=write_nl + write_nl=function(s) + writeline(localtime().." | "..s) + end + settimedlog=ignore + end +end +logs.report=report +logs.subreport=subreport +logs.status=status +logs.settarget=settarget +logs.pushtarget=pushtarget +logs.poptarget=poptarget +logs.setformats=setformats +logs.settranslations=settranslations +logs.setlogfile=setlogfile +logs.settimedlog=settimedlog +logs.setprocessor=setprocessor +logs.setformatters=setformatters +logs.direct=direct +logs.subdirect=subdirect +logs.writer=writer +logs.newline=newline +local data,states={},nil +function logs.reporter(category,subcategory) + local logger=data[category] + if not logger then + local state=false + if states==true then + state=true + elseif type(states)=="table" then + for c,_ in next,states do + if find(category,c) then + state=true + break + end + end + end + logger={ + reporters={}, + state=state, + } + data[category]=logger + end + local reporter=logger.reporters[subcategory or "default"] + if not reporter then + if subcategory then + reporter=function(...) + if not logger.state then + subreport(category,subcategory,...) + end + end + logger.reporters[subcategory]=reporter + else + local tag=category + reporter=function(...) + if not logger.state then + report(category,...) + end + end + logger.reporters.default=reporter + end + end + return reporter +end +logs.new=logs.reporter +local ctxreport=logs.writer +function logs.setmessenger(m) + ctxreport=m +end +function logs.messenger(category,subcategory) + if subcategory then + return function(...) + ctxreport(subdirect(category,subcategory,...)) + end + else + return function(...) + ctxreport(direct(category,...)) + end + end +end +local function setblocked(category,value) + if category==true then + category,value="*",true + elseif category==false then + category,value="*",false + elseif value==nil then + value=true + end + if category=="*" then + states=value + for k,v in next,data do + v.state=value + end + else + states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) + for c,_ in next,states do + local v=data[c] + if v then + v.state=value + else + c=topattern(c,true,true) + for k,v in next,data do + if find(k,c) then + v.state=value + end + end + end + end + end +end +function logs.disable(category,value) + setblocked(category,value==nil and true or value) +end +function logs.enable(category) + setblocked(category,false) +end +function logs.categories() + return table.sortedkeys(data) +end +function logs.show() + local n,c,s,max=0,0,0,0 + for category,v in table.sortedpairs(data) do + n=n+1 + local state=v.state + local reporters=v.reporters + local nc=#category + if nc>c then + c=nc + end + for subcategory,_ in next,reporters do + local ns=#subcategory + if ns>c then + s=ns + end + local m=nc+ns + if m>max then + max=m + end + end + local subcategories=concat(table.sortedkeys(reporters),", ") + if state==true then + state="disabled" + elseif state==false then + state="enabled" + else + state="unknown" + end + report("logging","category %a, subcategories %a, state %a",category,subcategories,state) + end + report("logging","categories: %s, max category: %s, max subcategory: %s, max combined: %s",n,c,s,max) +end +local delayed_reporters={} +setmetatableindex(delayed_reporters,function(t,k) + local v=logs.reporter(k.name) + t[k]=v + return v +end) +function utilities.setters.report(setter,...) + delayed_reporters[setter](...) +end +directives.register("logs.blocked",function(v) + setblocked(v,true) +end) +directives.register("logs.target",function(v) + settarget(v) +end) +local report_pages=logs.reporter("pages") +local real,user,sub +function logs.start_page_number() + real=texgetcount("realpageno") + user=texgetcount("userpageno") + sub=texgetcount("subpageno") +end +local timing=false +local starttime=nil +local lasttime=nil +trackers.register("pages.timing",function(v) + starttime=os.clock() + timing=true +end) +function logs.stop_page_number() + if timing then + local elapsed,average + local stoptime=os.clock() + if not lasttime or real<2 then + elapsed=stoptime + average=stoptime + starttime=stoptime + else + elapsed=stoptime-lasttime + average=(stoptime-starttime)/(real-1) + end + lasttime=stoptime + if real<=0 then + report_pages("flushing page, time %0.04f / %0.04f",elapsed,average) + elseif user<=0 then + report_pages("flushing realpage %s, time %0.04f / %0.04f",real,elapsed,average) + elseif sub<=0 then + report_pages("flushing realpage %s, userpage %s, time %0.04f / %0.04f",real,user,elapsed,average) + else + report_pages("flushing realpage %s, userpage %s, subpage %s, time %0.04f / %0.04f",real,user,sub,elapsed,average) + end + else + if real<=0 then + report_pages("flushing page") + elseif user<=0 then + report_pages("flushing realpage %s",real) + elseif sub<=0 then + report_pages("flushing realpage %s, userpage %s",real,user) + else + report_pages("flushing realpage %s, userpage %s, subpage %s",real,user,sub) + end + end + logs.flush() +end +local report_files=logs.reporter("files") +local nesting=0 +local verbose=false +local hasscheme=url.hasscheme +function logs.show_open(name) +end +function logs.show_close(name) +end +function logs.show_load(name) +end +local simple=logs.reporter("comment") +logs.simple=simple +logs.simpleline=simple +logs.setprogram=ignore +logs.extendbanner=ignore +logs.reportlines=ignore +logs.reportbanner=ignore +logs.reportline=ignore +logs.simplelines=ignore +logs.help=ignore +local Carg,C,lpegmatch=lpeg.Carg,lpeg.C,lpeg.match +local p_newline=lpeg.patterns.newline +local linewise=( + Carg(1)*C((1-p_newline)^1)/function(t,s) t.report(s) end+Carg(1)*p_newline^2/function(t) t.report() end+p_newline +)^1 +local function reportlines(t,str) + if str then + lpegmatch(linewise,str,1,t) + end +end +local function reportbanner(t) + local banner=t.banner + if banner then + t.report(banner) + t.report() + end +end +local function reportversion(t) + local banner=t.banner + if banner then + t.report(banner) + end +end +local function reporthelp(t,...) + local helpinfo=t.helpinfo + if type(helpinfo)=="string" then + reportlines(t,helpinfo) + elseif type(helpinfo)=="table" then + for i=1,select("#",...) do + reportlines(t,t.helpinfo[select(i,...)]) + if i<n then + t.report() + end + end + end +end +local function reportinfo(t) + t.report() + reportlines(t,t.moreinfo) +end +local function reportexport(t,method) + report(t.helpinfo) +end +local reporters={ + lines=reportlines, + banner=reportbanner, + version=reportversion, + help=reporthelp, + info=reportinfo, + export=reportexport, +} +local exporters={ +} +logs.reporters=reporters +logs.exporters=exporters +function logs.application(t) + t.name=t.name or "unknown" + t.banner=t.banner + t.moreinfo=moreinfo + t.report=logs.reporter(t.name) + t.help=function(...) + reporters.banner(t) + reporters.help(t,...) + reporters.info(t) + end + t.export=function(...) + reporters.export(t,...) + end + t.identify=function() + reporters.banner(t) + end + t.version=function() + reporters.version(t) + end + return t +end +local f_syslog=formatters["%s %s => %s => %s => %s\r"] +function logs.system(whereto,process,jobname,category,fmt,arg,...) + local message=f_syslog(datetime("%d/%m/%y %H:%m:%S"),process,jobname,category,arg==nil and fmt or format(fmt,arg,...)) + for i=1,10 do + local f=openfile(whereto,"a") + if f then + f:write(message) + f:close() + break + else + sleep(0.1) + end + end +end +local report_system=logs.reporter("system","logs") +function logs.obsolete(old,new) + local o=loadstring("return "..new)() + if type(o)=="function" then + return function(...) + report_system("function %a is obsolete, use %a",old,new) + loadstring(old.."="..new.." return "..old)()(...) + end + elseif type(o)=="table" then + local t,m={},{} + m.__index=function(t,k) + report_system("table %a is obsolete, use %a",old,new) + m.__index,m.__newindex=o,o + return o[k] + end + m.__newindex=function(t,k,v) + report_system("table %a is obsolete, use %a",old,new) + m.__index,m.__newindex=o,o + o[k]=v + end + if libraries then + libraries.obsolete[old]=t + end + setmetatable(t,m) + return t + end +end +if utilities then + utilities.report=report_system +end +if tex and tex.error then + function logs.texerrormessage(...) + tex.error(format(...),{}) + end +else + function logs.texerrormessage(...) + print(format(...)) + end +end +io.stdout:setvbuf('no') +io.stderr:setvbuf('no') +if package.helpers.report then + package.helpers.report=logs.reporter("package loader") +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["trac-inf"] = package.loaded["trac-inf"] or true + +-- original size: 6704, stripped down to: 5343 + +if not modules then modules={} end modules ['trac-inf']={ + version=1.001, + comment="companion to trac-inf.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local type,tonumber,select=type,tonumber,select +local format,lower,find=string.format,string.lower,string.find +local concat=table.concat +local clock=os.gettimeofday or os.clock +local setmetatableindex=table.setmetatableindex +local serialize=table.serialize +local formatters=string.formatters +statistics=statistics or {} +local statistics=statistics +statistics.enable=true +statistics.threshold=0.01 +local statusinfo,n,registered,timers={},0,{},{} +setmetatableindex(timers,function(t,k) + local v={ timing=0,loadtime=0 } + t[k]=v + return v +end) +local function hastiming(instance) + return instance and timers[instance] +end +local function resettiming(instance) + timers[instance or "notimer"]={ timing=0,loadtime=0 } +end +local function starttiming(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing or 0 + if it==0 then + timer.starttime=clock() + if not timer.loadtime then + timer.loadtime=0 + end + end + timer.timing=it+1 +end +local function stoptiming(instance) + local timer=timers[instance or "notimer"] + local it=timer.timing + if it>1 then + timer.timing=it-1 + else + local starttime=timer.starttime + if starttime then + local stoptime=clock() + local loadtime=stoptime-starttime + timer.stoptime=stoptime + timer.loadtime=timer.loadtime+loadtime + timer.timing=0 + return loadtime + end + end + return 0 +end +local function elapsed(instance) + if type(instance)=="number" then + return instance or 0 + else + local timer=timers[instance or "notimer"] + return timer and timer.loadtime or 0 + end +end +local function elapsedtime(instance) + return format("%0.3f",elapsed(instance)) +end +local function elapsedindeed(instance) + return elapsed(instance)>statistics.threshold +end +local function elapsedseconds(instance,rest) + if elapsedindeed(instance) then + return format("%0.3f seconds %s",elapsed(instance),rest or "") + end +end +statistics.hastiming=hastiming +statistics.resettiming=resettiming +statistics.starttiming=starttiming +statistics.stoptiming=stoptiming +statistics.elapsed=elapsed +statistics.elapsedtime=elapsedtime +statistics.elapsedindeed=elapsedindeed +statistics.elapsedseconds=elapsedseconds +function statistics.register(tag,fnc) + if statistics.enable and type(fnc)=="function" then + local rt=registered[tag] or (#statusinfo+1) + statusinfo[rt]={ tag,fnc } + registered[tag]=rt + if #tag>n then n=#tag end + end +end +local report=logs.reporter("mkiv lua stats") +function statistics.show() + if statistics.enable then + local register=statistics.register + register("used platform",function() + return format("%s, type: %s, binary subtree: %s", + os.platform or "unknown",os.type or "unknown",environment.texos or "unknown") + end) + register("luatex banner",function() + return lower(status.banner) + end) + register("control sequences",function() + return format("%s of %s + %s",status.cs_count,status.hash_size,status.hash_extra) + end) + register("callbacks",function() + local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 + return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) + end) + if jit then + local jitstatus={ jit.status() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end + end + register("lua properties",function() + local list=status.list() + local hashchar=tonumber(list.luatex_hashchars) + local mask=lua.mask or "ascii" + return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", + jit and "luajit" or "lua", + statistics.memused(), + list.luatex_hashtype or "default", + hashchar and 2^hashchar or "unknown", + mask, + mask=="utf" and "τεχ" or "tex") + end) + register("runtime",statistics.runtime) + logs.newline() + for i=1,#statusinfo do + local s=statusinfo[i] + local r=s[2]() + if r then + report("%s: %s",s[1],r) + end + end + statistics.enable=false + end +end +function statistics.memused() + local round=math.round or math.floor + return format("%s MB (ctx: %s MB)",round(collectgarbage("count")/1000),round(status.luastate_bytes/1000000)) +end +starttiming(statistics) +function statistics.formatruntime(runtime) + return format("%s seconds",runtime) +end +function statistics.runtime() + stoptiming(statistics) + return statistics.formatruntime(elapsedtime(statistics)) +end +local report=logs.reporter("system") +function statistics.timed(action) + starttiming("run") + action() + stoptiming("run") + report("total runtime: %s seconds",elapsedtime("run")) +end +function statistics.tracefunction(base,tag,...) + for i=1,select("#",...) do + local name=select(i,...) + local stat={} + local func=base[name] + setmetatableindex(stat,function(t,k) t[k]=0 return 0 end) + base[name]=function(n,k,v) stat[k]=stat[k]+1 return func(n,k,v) end + statistics.register(formatters["%s.%s"](tag,name),function() return serialize(stat,"calls") end) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["trac-pro"] = package.loaded["trac-pro"] or true + +-- original size: 5829, stripped down to: 3501 + +if not modules then modules={} end modules ['trac-pro']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local getmetatable,setmetatable,rawset,type=getmetatable,setmetatable,rawset,type +local trace_namespaces=false trackers.register("system.namespaces",function(v) trace_namespaces=v end) +local report_system=logs.reporter("system","protection") +namespaces=namespaces or {} +local namespaces=namespaces +local registered={} +local function report_index(k,name) + if trace_namespaces then + report_system("reference to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) + else + report_system("reference to %a in protected namespace %a",k,name) + end +end +local function report_newindex(k,name) + if trace_namespaces then + report_system("assignment to %a in protected namespace %a: %s",k,name) + debugger.showtraceback(report_system) + else + report_system("assignment to %a in protected namespace %a",k,name) + end +end +local function register(name) + local data=name=="global" and _G or _G[name] + if not data then + return + end + registered[name]=data + local m=getmetatable(data) + if not m then + m={} + setmetatable(data,m) + end + local index,newindex={},{} + m.__saved__index=m.__index + m.__no__index=function(t,k) + if not index[k] then + index[k]=true + report_index(k,name) + end + return nil + end + m.__saved__newindex=m.__newindex + m.__no__newindex=function(t,k,v) + if not newindex[k] then + newindex[k]=true + report_newindex(k,name) + end + rawset(t,k,v) + end + m.__protection__depth=0 +end +local function private(name) + local data=registered[name] + if not data then + data=_G[name] + if not data then + data={} + _G[name]=data + end + register(name) + end + return data +end +local function protect(name) + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>0 then + m.__protection__depth=pd+1 + else + m.__save_d_index,m.__saved__newindex=m.__index,m.__newindex + m.__index,m.__newindex=m.__no__index,m.__no__newindex + m.__protection__depth=1 + end +end +local function unprotect(name) + local data=registered[name] + if not data then + return + end + local m=getmetatable(data) + local pd=m.__protection__depth + if pd>1 then + m.__protection__depth=pd-1 + else + m.__index,m.__newindex=m.__saved__index,m.__saved__newindex + m.__protection__depth=0 + end +end +local function protectall() + for name,_ in next,registered do + if name~="global" then + protect(name) + end + end +end +local function unprotectall() + for name,_ in next,registered do + if name~="global" then + unprotect(name) + end + end +end +namespaces.register=register +namespaces.private=private +namespaces.protect=protect +namespaces.unprotect=unprotect +namespaces.protectall=protectall +namespaces.unprotectall=unprotectall +namespaces.private("namespaces") registered={} register("global") +directives.register("system.protect",function(v) + if v then + protectall() + else + unprotectall() + end +end) +directives.register("system.checkglobals",function(v) + if v then + report_system("enabling global namespace guard") + protect("global") + else + report_system("disabling global namespace guard") + unprotect("global") + end +end) + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-lua"] = package.loaded["util-lua"] or true + +-- original size: 4982, stripped down to: 3511 + +if not modules then modules={} end modules ['util-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + comment="the strip code is written by Peter Cawley", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rep,sub,byte,dump,format=string.rep,string.sub,string.byte,string.dump,string.format +local load,loadfile,type=load,loadfile,type +utilities=utilities or {} +utilities.lua=utilities.lua or {} +local luautilities=utilities.lua +local report_lua=logs.reporter("system","lua") +local tracestripping=false +local forcestupidcompile=true +luautilities.stripcode=true +luautilities.alwaysstripcode=false +luautilities.nofstrippedchunks=0 +luautilities.nofstrippedbytes=0 +local strippedchunks={} +luautilities.strippedchunks=strippedchunks +luautilities.suffixes={ + tma="tma", + tmc=jit and "tmb" or "tmc", + lua="lua", + luc=jit and "lub" or "luc", + lui="lui", + luv="luv", + luj="luj", + tua="tua", + tuc="tuc", +} +local function register(name) + if tracestripping then + report_lua("stripped bytecode from %a",name or "unknown") + end + strippedchunks[#strippedchunks+1]=name + luautilities.nofstrippedchunks=luautilities.nofstrippedchunks+1 +end +local function stupidcompile(luafile,lucfile,strip) + local code=io.loaddata(luafile) + if code and code~="" then + code=load(code) + if code then + code=dump(code,strip and luautilities.stripcode or luautilities.alwaysstripcode) + if code and code~="" then + register(name) + io.savedata(lucfile,code) + return true,0 + end + else + report_lua("fatal error %a in file %a",1,luafile) + end + else + report_lua("fatal error %a in file %a",2,luafile) + end + return false,0 +end +function luautilities.loadedluacode(fullname,forcestrip,name) + name=name or fullname + local code,message + if environment.loadpreprocessedfile then + code,message=environment.loadpreprocessedfile(fullname) + else + code,message=loadfile(fullname) + end + if code then + code() + else + report_lua("loading of file %a failed:\n\t%s",fullname,message or "no message") + end + if forcestrip and luautilities.stripcode then + if type(forcestrip)=="function" then + forcestrip=forcestrip(fullname) + end + if forcestrip or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end + elseif luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end +end +function luautilities.strippedloadstring(code,forcestrip,name) + local code,message=load(code) + if not code then + report_lua("loading of file %a failed:\n\t%s",name,message or "no message") + end + if forcestrip and luautilities.stripcode or luautilities.alwaysstripcode then + register(name) + return load(dump(code,true)),0 + else + return code,0 + end +end +function luautilities.compile(luafile,lucfile,cleanup,strip,fallback) + report_lua("compiling %a into %a",luafile,lucfile) + os.remove(lucfile) + local done=stupidcompile(luafile,lucfile,strip~=false) + if done then + report_lua("dumping %a into %a stripped",luafile,lucfile) + if cleanup==true and lfs.isfile(lucfile) and lfs.isfile(luafile) then + report_lua("removing %a",luafile) + os.remove(luafile) + end + end + return done +end +function luautilities.loadstripped(...) + local l=load(...) + if l then + return load(dump(l,true)) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-deb"] = package.loaded["util-deb"] or true + +-- original size: 3898, stripped down to: 2644 + +if not modules then modules={} end modules ['util-deb']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local debug=require "debug" +local getinfo=debug.getinfo +local type,next,tostring=type,next,tostring +local format,find=string.format,string.find +local is_boolean=string.is_boolean +utilities=utilities or {} +local debugger=utilities.debugger or {} +utilities.debugger=debugger +local counters={} +local names={} +local report=logs.reporter("debugger") +local function hook() + local f=getinfo(2) + if f then + local n="unknown" + if f.what=="C" then + n=f.name or '<anonymous>' + if not names[n] then + names[n]=format("%42s",n) + end + else + n=f.name or f.namewhat or f.what + if not n or n=="" then + n="?" + end + if not names[n] then + names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + end + counters[n]=(counters[n] or 0)+1 + end +end +function debugger.showstats(printer,threshold) + printer=printer or report + threshold=threshold or 0 + local total,grandtotal,functions=0,0,0 + local dataset={} + for name,count in next,counters do + dataset[#dataset+1]={ name,count } + end + table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + for i=1,#dataset do + local d=dataset[i] + local name=d[1] + local count=d[2] + if count>threshold and not find(name,"for generator") then + printer(format("%8i %s\n",count,names[name])) + total=total+count + end + grandtotal=grandtotal+count + functions=functions+1 + end + printer("\n") + printer(format("functions : % 10i\n",functions)) + printer(format("total : % 10i\n",total)) + printer(format("grand total: % 10i\n",grandtotal)) + printer(format("threshold : % 10i\n",threshold)) +end +function debugger.savestats(filename,threshold) + local f=io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end +function debugger.enable() + debug.sethook(hook,"c") +end +function debugger.disable() + debug.sethook() +end +local function showtraceback(rep) + local level=2 + local reporter=rep or report + while true do + local info=getinfo(level,"Sl") + if not info then + break + elseif info.what=="C" then + reporter("%2i : %s",level-1,"C function") + else + reporter("%2i : %s : %s",level-1,info.short_src,info.currentline) + end + level=level+1 + end +end +debugger.showtraceback=showtraceback + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-mrg"] = package.loaded["util-mrg"] or true + +-- original size: 7757, stripped down to: 6015 + +if not modules then modules={} end modules ['util-mrg']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gsub,format=string.gsub,string.format +local concat=table.concat +local type,next=type,next +local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt,Cb,Cg=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt,lpeg.Cb,lpeg.Cg +local lpegmatch,patterns=lpeg.match,lpeg.patterns +utilities=utilities or {} +local merger=utilities.merger or {} +utilities.merger=merger +merger.strip_comment=true +local report=logs.reporter("system","merge") +utilities.report=report +local m_begin_merge="begin library merge" +local m_end_merge="end library merge" +local m_begin_closure="do -- create closure to overcome 200 locals limit" +local m_end_closure="end -- of closure" +local m_pattern="%c+".."%-%-%s+"..m_begin_merge.."%c+(.-)%c+".."%-%-%s+"..m_end_merge.."%c+" +local m_format="\n\n-- "..m_begin_merge.."\n%s\n".."-- "..m_end_merge.."\n\n" +local m_faked="-- ".."created merged file".."\n\n".."-- "..m_begin_merge.."\n\n".."-- "..m_end_merge.."\n\n" +local m_report=[[ +-- used libraries : %s +-- skipped libraries : %s +-- original bytes : %s +-- stripped bytes : %s +]] +local m_preloaded=[[package.loaded[%q] = package.loaded[%q] or true]] +local function self_fake() + return m_faked +end +local function self_nothing() + return "" +end +local function self_load(name) + local data=io.loaddata(name) or "" + if data=="" then + report("unknown file %a",name) + else + report("inserting file %a",name) + end + return data or "" +end +local space=patterns.space +local eol=patterns.newline +local equals=P("=")^0 +local open=P("[")*Cg(equals,"init")*P("[")*P("\n")^-1 +local close=P("]")*C(equals)*P("]") +local closeeq=Cmt(close*Cb("init"),function(s,i,a,b) return a==b end) +local longstring=open*(1-closeeq)^0*close +local quoted=patterns.quoted +local digit=patterns.digit +local emptyline=space^0*eol +local operator1=P("<=")+P(">=")+P("~=")+P("..")+S("/^<>=*+%%") +local operator2=S("*+/") +local operator3=S("-") +local operator4=P("..") +local separator=S(",;") +local ignore=(P("]")*space^1*P("=")*space^1*P("]"))/"]=["+(P("=")*space^1*P("{"))/"={"+(P("(")*space^1)/"("+(P("{")*(space+eol)^1*P("}"))/"{}" +local strings=quoted +local longcmt=(emptyline^0*P("--")*longstring*emptyline^0)/"" +local longstr=longstring +local comment=emptyline^0*P("--")*P("-")^0*(1-eol)^0*emptyline^1/"\n" +local optionalspaces=space^0/"" +local mandatespaces=space^1/"" +local optionalspacing=(eol+space)^0/"" +local mandatespacing=(eol+space)^1/"" +local pack=digit*space^1*operator4*optionalspacing+optionalspacing*operator1*optionalspacing+optionalspacing*operator2*optionalspaces+mandatespacing*operator3*mandatespaces+optionalspaces*separator*optionalspaces +local lines=emptyline^2/"\n" +local spaces=(space*space)/" " +local compact=Cs (( + ignore+strings+longcmt+longstr+comment+pack+lines+spaces+1 +)^1 ) +local strip=Cs((emptyline^2/"\n"+1)^0) +local stripreturn=Cs((1-P("return")*space^1*P(1-space-eol)^1*(space+eol)^0*P(-1))^1) +function merger.compact(data) + return lpegmatch(strip,lpegmatch(compact,data)) +end +local function self_compact(data) + local delta=0 + if merger.strip_comment then + local before=#data + data=lpegmatch(compact,data) + data=lpegmatch(strip,data) + local after=#data + delta=before-after + report("original size %s, compacted to %s, stripped %s",before,after,delta) + data=format("-- original size: %s, stripped down to: %s\n\n%s",before,after,data) + end + return lpegmatch(stripreturn,data) or data,delta +end +local function self_save(name,data) + if data~="" then + io.savedata(name,data) + report("saving %s with size %s",name,#data) + end +end +local function self_swap(data,code) + return data~="" and (gsub(data,m_pattern,function() return format(m_format,code) end,1)) or "" +end +local function self_libs(libs,list) + local result,f,frozen,foundpath={},nil,false,nil + result[#result+1]="\n" + if type(libs)=='string' then libs={ libs } end + if type(list)=='string' then list={ list } end + for i=1,#libs do + local lib=libs[i] + for j=1,#list do + local pth=gsub(list[j],"\\","/") + report("checking library path %a",pth) + local name=pth.."/"..lib + if lfs.isfile(name) then + foundpath=pth + end + end + if foundpath then break end + end + if foundpath then + report("using library path %a",foundpath) + local right,wrong,original,stripped={},{},0,0 + for i=1,#libs do + local lib=libs[i] + local fullname=foundpath.."/"..lib + if lfs.isfile(fullname) then + report("using library %a",fullname) + local preloaded=file.nameonly(lib) + local data=io.loaddata(fullname,true) + original=original+#data + local data,delta=self_compact(data) + right[#right+1]=lib + result[#result+1]=m_begin_closure + result[#result+1]=format(m_preloaded,preloaded,preloaded) + result[#result+1]=data + result[#result+1]=m_end_closure + stripped=stripped+delta + else + report("skipping library %a",fullname) + wrong[#wrong+1]=lib + end + end + right=#right>0 and concat(right," ") or "-" + wrong=#wrong>0 and concat(wrong," ") or "-" + report("used libraries: %a",right) + report("skipped libraries: %a",wrong) + report("original bytes: %a",original) + report("stripped bytes: %a",stripped) + result[#result+1]=format(m_report,right,wrong,original,stripped) + else + report("no valid library path found") + end + return concat(result,"\n\n") +end +function merger.selfcreate(libs,list,target) + if target then + self_save(target,self_swap(self_fake(),self_libs(libs,list))) + end +end +function merger.selfmerge(name,libs,list,target) + self_save(target or name,self_swap(self_load(name),self_libs(libs,list))) +end +function merger.selfclean(name) + self_save(name,self_swap(self_load(name),self_nothing())) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true + +-- original size: 7100, stripped down to: 3978 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-env"] = package.loaded["util-env"] or true + +-- original size: 8022, stripped down to: 5038 + +if not modules then modules={} end modules ['util-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate,mark=utilities.storage.allocate,utilities.storage.mark +local format,sub,match,gsub,find=string.format,string.sub,string.match,string.gsub,string.find +local unquoted,quoted,optionalquoted=string.unquoted,string.quoted,string.optionalquoted +local concat,insert,remove=table.concat,table.insert,table.remove +environment=environment or {} +local environment=environment +os.setlocale(nil,nil) +function os.setlocale() +end +local validengines=allocate { + ["luatex"]=true, + ["luajittex"]=true, +} +local basicengines=allocate { + ["luatex"]="luatex", + ["texlua"]="luatex", + ["texluac"]="luatex", + ["luajittex"]="luajittex", + ["texluajit"]="luajittex", +} +local luaengines=allocate { + ["lua"]=true, + ["luajit"]=true, +} +environment.validengines=validengines +environment.basicengines=basicengines +if not arg then + environment.used_as_library=true +elseif luaengines[file.removesuffix(arg[-1])] then +elseif validengines[file.removesuffix(arg[0])] then + if arg[1]=="--luaonly" then + arg[-1]=arg[0] + arg[ 0]=arg[2] + for k=3,#arg do + arg[k-2]=arg[k] + end + remove(arg) + remove(arg) + else + end + local originalzero=file.basename(arg[0]) + local specialmapping={ luatools=="base" } + if originalzero~="mtxrun" and originalzero~="mtxrun.lua" then + arg[0]=specialmapping[originalzero] or originalzero + insert(arg,0,"--script") + insert(arg,0,"mtxrun") + end +end +environment.arguments=allocate() +environment.files=allocate() +environment.sortedflags=nil +function environment.initializearguments(arg) + local arguments,files={},{} + environment.arguments,environment.files,environment.sortedflags=arguments,files,nil + for index=1,#arg do + local argument=arg[index] + if index>0 then + local flag,value=match(argument,"^%-+(.-)=(.-)$") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=unquoted(value or "") + else + flag=match(argument,"^%-+(.+)") + if flag then + flag=gsub(flag,"^c:","") + arguments[flag]=true + else + files[#files+1]=argument + end + end + end + end + environment.ownname=file.reslash(environment.ownname or arg[0] or 'unknown.lua') +end +function environment.setargument(name,value) + environment.arguments[name]=value +end +function environment.getargument(name,partial) + local arguments,sortedflags=environment.arguments,environment.sortedflags + if arguments[name] then + return arguments[name] + elseif partial then + if not sortedflags then + sortedflags=allocate(table.sortedkeys(arguments)) + for k=1,#sortedflags do + sortedflags[k]="^"..sortedflags[k] + end + environment.sortedflags=sortedflags + end + for k=1,#sortedflags do + local v=sortedflags[k] + if find(name,v) then + return arguments[sub(v,2,#v)] + end + end + end + return nil +end +environment.argument=environment.getargument +function environment.splitarguments(separator) + local done,before,after=false,{},{} + local originalarguments=environment.originalarguments + for k=1,#originalarguments do + local v=originalarguments[k] + if not done and v==separator then + done=true + elseif done then + after[#after+1]=v + else + before[#before+1]=v + end + end + return before,after +end +function environment.reconstructcommandline(arg,noquote) + local resolveprefix=resolvers.resolve + arg=arg or environment.originalarguments + if noquote and #arg==1 then + return unquoted(resolveprefix and resolveprefix(arg[1]) or arg[1]) + elseif #arg>0 then + local result={} + for i=1,#arg do + result[i]=optionalquoted(resolveprefix and resolveprefix(arg[i]) or resolveprefix) + end + return concat(result," ") + else + return "" + end +end +function environment.relativepath(path,root) + if not path then + path="" + end + if not file.is_rootbased_path(path) then + if not root then + root=file.pathpart(environment.ownscript or environment.ownname or ".") + end + if root=="" then + root="." + end + path=root.."/"..path + end + return file.collapsepath(path,true) +end +if arg then + local newarg,instring={},false + for index=1,#arg do + local argument=arg[index] + if find(argument,"^\"") then + newarg[#newarg+1]=gsub(argument,"^\"","") + if not find(argument,"\"$") then + instring=true + end + elseif find(argument,"\"$") then + newarg[#newarg]=newarg[#newarg].." "..gsub(argument,"\"$","") + instring=false + elseif instring then + newarg[#newarg]=newarg[#newarg].." "..argument + else + newarg[#newarg+1]=argument + end + end + for i=1,-5,-1 do + newarg[i]=arg[i] + end + environment.initializearguments(newarg) + environment.originalarguments=mark(newarg) + environment.rawarguments=mark(arg) + arg={} +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["luat-env"] = package.loaded["luat-env"] or true + +-- original size: 6174, stripped down to: 4141 + + if not modules then modules={} end modules ['luat-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rawset,rawget,loadfile,assert=rawset,rawget,loadfile,assert +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_lua=logs.reporter("resolvers","lua") +local luautilities=utilities.lua +local luasuffixes=luautilities.suffixes +local texgettoks=tex and tex.gettoks +environment=environment or {} +local environment=environment +local mt={ + __index=function(_,k) + if k=="version" then + local version=texgettoks and texgettoks("contextversiontoks") + if version and version~="" then + rawset(environment,"version",version) + return version + else + return "unknown" + end + elseif k=="kind" then + local kind=texgettoks and texgettoks("contextkindtoks") + if kind and kind~="" then + rawset(environment,"kind",kind) + return kind + else + return "unknown" + end + elseif k=="jobname" or k=="formatname" then + local name=tex and tex[k] + if name or name=="" then + rawset(environment,k,name) + return name + else + return "unknown" + end + elseif k=="outputfilename" then + local name=environment.jobname + rawset(environment,k,name) + return name + end + end +} +setmetatable(environment,mt) +function environment.texfile(filename) + return resolvers.findfile(filename,'tex') +end +function environment.luafile(filename) + local resolved=resolvers.findfile(filename,'tex') or "" + if resolved~="" then + return resolved + end + resolved=resolvers.findfile(filename,'texmfscripts') or "" + if resolved~="" then + return resolved + end + return resolvers.findfile(filename,'luatexlibs') or "" +end +local stripindeed=false directives.register("system.compile.strip",function(v) stripindeed=v end) +local function strippable(filename) + if stripindeed then + local modu=modules[file.nameonly(filename)] + return modu and modu.dataonly + else + return false + end +end +function environment.luafilechunk(filename,silent) + filename=file.replacesuffix(filename,"lua") + local fullname=environment.luafile(filename) + if fullname and fullname~="" then + local data=luautilities.loadedluacode(fullname,strippable,filename) + if not silent then + report_lua("loading file %a %s",fullname,not data and "failed" or "succeeded") + end + return data + else + if not silent then + report_lua("unknown file %a",filename) + end + return nil + end +end +function environment.loadluafile(filename,version) + local lucname,luaname,chunk + local basename=file.removesuffix(filename) + if basename==filename then + luaname=file.addsuffix(basename,luasuffixes.lua) + lucname=file.addsuffix(basename,luasuffixes.luc) + else + luaname=basename + lucname=nil + end + local fullname=(lucname and environment.luafile(lucname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %a",fullname) + end + chunk=loadfile(fullname) + end + if chunk then + assert(chunk)() + if version then + local v=version + if modules and modules[filename] then + v=modules[filename].version + elseif versions and versions[filename] then + v=versions[filename] + end + if v==version then + return true + else + if trace_locating then + report_lua("version mismatch for %a, lua version %a, luc version %a",filename,v,version) + end + environment.loadluafile(filename) + end + else + return true + end + end + fullname=(luaname and environment.luafile(luaname)) or "" + if fullname~="" then + if trace_locating then + report_lua("loading %a",fullname) + end + chunk=loadfile(fullname) + if not chunk then + if trace_locating then + report_lua("unknown file %a",filename) + end + else + assert(chunk)() + return true + end + end + return false +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true + +-- original size: 45683, stripped down to: 27866 + +if not modules then modules={} end modules ['lxml-tab']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local report_xml=logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end +if lpeg.setmaxstack then lpeg.setmaxstack(1000) end +xml=xml or {} +local xml=xml +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset +local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub +local utfchar=utf.char +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs +local formatters=string.formatters +xml.xmlns=xml.xmlns or {} +local check=P(false) +local parse=check +function xml.registerns(namespace,pattern) + check=check+C(P(lower(pattern)))/namespace + parse=P { P(check)+1*V(1) } +end +function xml.checkns(namespace,url) + local ns=lpegmatch(parse,lower(url)) + if ns and namespace~=ns then + xml.xmlns[namespace]=ns + end +end +function xml.resolvens(url) + return lpegmatch(parse,lower(url)) or "" +end +local nsremap,resolvens=xml.xmlns,xml.resolvens +local stack={} +local top={} +local dt={} +local at={} +local xmlns={} +local errorstr=nil +local entities={} +local strip=false +local cleanup=false +local utfize=false +local resolve_predefined=false +local unify_predefined=false +local dcache={} +local hcache={} +local acache={} +local mt={} +local function initialize_mt(root) + mt={ __index=root } +end +function xml.setproperty(root,k,v) + getmetatable(root).__index[k]=v +end +function xml.checkerror(top,toclose) + return "" +end +local function add_attribute(namespace,tag,value) + if cleanup and #value>0 then + value=cleanup(value) + end + if tag=="xmlns" then + xmlns[#xmlns+1]=resolvens(value) + at[tag]=value + elseif namespace=="" then + at[tag]=value + elseif namespace=="xmlns" then + xml.checkns(tag,value) + at["xmlns:"..tag]=value + else + at[namespace..":"..tag]=value + end +end +local function add_empty(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace + top=stack[#stack] + dt=top.dt + local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } + dt[#dt+1]=t + setmetatable(t,mt) + if at.xmlns then + remove(xmlns) + end + at={} +end +local function add_begin(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace + top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + setmetatable(top,mt) + dt=top.dt + stack[#stack+1]=top + at={} +end +local function add_end(spacing,namespace,tag) + if #spacing>0 then + dt[#dt+1]=spacing + end + local toclose=remove(stack) + top=stack[#stack] + if #stack<1 then + errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) + elseif toclose.tg~=tag then + errorstr=formatters["unable to close %s with %s %s"](toclose.tg,tag,xml.checkerror(top,toclose) or "") + report_xml(errorstr) + end + dt=top.dt + dt[#dt+1]=toclose + if toclose.at.xmlns then + remove(xmlns) + end +end +local function add_text(text) + local n=#dt + if cleanup and #text>0 then + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..cleanup(text) + else + dt[n+1]=cleanup(text) + end + else + dt[1]=cleanup(text) + end + else + if n>0 then + local s=dt[n] + if type(s)=="string" then + dt[n]=s..text + else + dt[n+1]=text + end + else + dt[1]=text + end + end +end +local function add_special(what,spacing,text) + if #spacing>0 then + dt[#dt+1]=spacing + end + if strip and (what=="@cm@" or what=="@dt@") then + else + dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + end +end +local function set_message(txt) + errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") +end +local reported_attribute_errors={} +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + report_xml("invalid attribute value %a",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + report_xml("invalid attribute specification %a",str) + reported_attribute_errors[str]=true + at._error_=str + end + return str +end +local badentity="&error;" +local badentity="&" +xml.placeholders={ + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, + unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, + unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, +} +local placeholders=xml.placeholders +local function fromhex(s) + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return formatters["h:%s"](s),true + end +end +local function fromdec(s) + local n=tonumber(s) + if n then + return utfchar(n) + else + return formatters["d:%s"](s),true + end +end +local p_rest=(1-P(";"))^0 +local p_many=P(1)^0 +local p_char=lpegpatterns.utf8character +local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) +local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", +} +local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", +} +local nofprivates=0xF0000 +local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", +} +local privates_p={} +local privates_n={ +} +local escaped=utf.remapper(privates_u,"dynamic") +local unprivatized=utf.remapper(privates_p,"dynamic") +xml.unprivatized=unprivatized +local function unescaped(s) + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + end + return p +end +xml.privatetoken=unescaped +xml.privatecodes=privates_n +local function handle_hex_entity(str) + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" + end + hcache[str]=h + end + return h +end +local function handle_dec_entity(str) + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" + end + dcache[str]=d + end + return d +end +xml.parsedentitylpeg=parsedentity +local function handle_any_entity(str) + if resolve then + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + if trace_entities then + report_xml("resolving entity &%s; to predefined %a",str,a) + end + else + if type(resolve)=="function" then + a=resolve(str) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then + if trace_entities then + report_xml("expanding entity &%s; to function call",str) + end + a=a(str) or "" + end + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then + if trace_entities then + report_xml("resolving entity &%s; to external %s",str,a) + end + else + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str=="" then + a=badentity + else + a="&"..str..";" + end + end + end + end + acache[str]=a + elseif trace_entities then + if not acache[str] then + report_xml("converting entity &%s; to %a",str,a) + acache[str]=a + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end + end + return a + end +end +local function handle_end_entity(str) + report_xml("error in entity, %a found without ending %a",str,";") + return str +end +local function handle_crap_error(chr) + report_xml("error in parsing, unexpected %a found ",chr) + add_text(chr) + return chr +end +local space=S(' \r\n\t') +local open=P('<') +local close=P('>') +local squote=S("'") +local dquote=S('"') +local equal=P('=') +local slash=P('/') +local colon=P(':') +local semicolon=P(';') +local ampersand=P('&') +local valid=R('az','AZ','09')+S('_-.') +local name_yes=C(valid^1)*colon*C(valid^1) +local name_nop=C(P(true))*C(valid^1) +local name=name_yes+name_nop +local utfbom=lpegpatterns.utfbom +local spacing=C(space^0) +local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 +local hexentitycontent=R("AF","af","09")^0 +local decentitycontent=R("09")^0 +local parsedentity=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity) +local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) +local text_unparsed=C((1-open)^1) +local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local somespace=space^1 +local optionalspace=space^0 +local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) +local endofattributes=slash*close+close +local whatever=space*name*optionalspace*equal +local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error +local attributevalue=value+wrongvalue +local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute +local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 +local parsedtext=text_parsed/add_text +local unparsedtext=text_unparsed/add_text +local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } +local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty +local beginelement=(spacing*open*name*attributes*optionalspace*close)/add_begin +local endelement=(spacing*open*slash*name*optionalspace*close)/add_end +local begincomment=open*P("!--") +local endcomment=P("--")*close +local begininstruction=open*P("?") +local endinstruction=P("?")*close +local begincdata=open*P("![CDATA[") +local endcdata=P("]]")*close +local someinstruction=C((1-endinstruction)^0) +local somecomment=C((1-endcomment )^0) +local somecdata=C((1-endcdata )^0) +local function normalentity(k,v ) entities[k]=v end +local function systementity(k,v,n) entities[k]=v end +local function publicentity(k,v,n) entities[k]=v end +local begindoctype=open*P("!DOCTYPE") +local enddoctype=close +local beginset=P("[") +local endset=P("]") +local doctypename=C((1-somespace-close)^0) +local elementdoctype=optionalspace*P("<!ELEMENT")*(1-close)^0*close +local basiccomment=begincomment*((1-endcomment)^0)*endcomment +local normalentitytype=(doctypename*somespace*value)/normalentity +local publicentitytype=(doctypename*somespace*P("PUBLIC")*somespace*value)/publicentity +local systementitytype=(doctypename*somespace*P("SYSTEM")*somespace*value*somespace*P("NDATA")*somespace*doctypename)/systementity +local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype)*optionalspace*close +local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+basiccomment+space)^0*optionalspace*endset +local definitiondoctype=doctypename*somespace*doctypeset +local publicdoctype=doctypename*somespace*P("PUBLIC")*somespace*value*somespace*value*somespace*doctypeset +local systemdoctype=doctypename*somespace*P("SYSTEM")*somespace*value*somespace*doctypeset +local simpledoctype=(1-close)^1 +local somedoctype=C((somespace*(publicdoctype+systemdoctype+definitiondoctype+simpledoctype)*optionalspace)^0) +local instruction=(spacing*begininstruction*someinstruction*endinstruction)/function(...) add_special("@pi@",...) end +local comment=(spacing*begincomment*somecomment*endcomment )/function(...) add_special("@cm@",...) end +local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special("@cd@",...) end +local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end +local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand +local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata +local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error +local trailer=space^0*(text_unparsed/set_message)^0 +local grammar_parsed_text=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, + parent=beginelement*V("children")^0*endelement, + children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, +} +local grammar_unparsed_text=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, + parent=beginelement*V("children")^0*endelement, + children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, +} +local function _xmlconvert_(data,settings) + settings=settings or {} + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + stack,top,at,xmlns,errorstr={},{},{},{},nil + acache,hcache,dcache={},{},{} + reported_attribute_errors={} + if settings.parent_root then + mt=getmetatable(settings.parent_root) + else + initialize_mt(top) + end + stack[#stack+1]=top + top.dt={} + dt=top.dt + if not data or data=="" then + errorstr="empty xml file" + elseif utfize or resolve then + if lpegmatch(grammar_parsed_text,data) then + else + errorstr="invalid xml file - parsed text" + end + elseif type(data)=="string" then + if lpegmatch(grammar_unparsed_text,data) then + errorstr="" + else + errorstr="invalid xml file - unparsed text" + end + else + errorstr="invalid xml file - no text at all" + end + local result + if errorstr and errorstr~="" then + result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } +setmetatable(result,mt) +setmetatable(result.dt[1],mt) + setmetatable(stack,mt) + local errorhandler=settings.error_handler + if errorhandler==false then + else + errorhandler=errorhandler or xml.errorhandler + if errorhandler then + local currentresource=settings.currentresource + if currentresource and currentresource~="" then + xml.errorhandler(formatters["load error in [%s]: %s"](currentresource,errorstr)) + else + xml.errorhandler(formatters["load error: %s"](errorstr)) + end + end + end + else + result=stack[1] + end + if not settings.no_root then + result={ special=true,ns="",tg='@rt@',dt=result.dt,at={},entities=entities,settings=settings } + setmetatable(result,mt) + local rdt=result.dt + for k=1,#rdt do + local v=rdt[k] + if type(v)=="table" and not v.special then + result.ri=k + v.__p__=result + break + end + end + end + if errorstr and errorstr~="" then + result.error=true + else + errorstr=nil + end + result.statistics={ + errormessage=errorstr, + entities={ + decimals=dcache, + hexadecimals=hcache, + names=acache, + } + } + strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil + unify_predefined,cleanup,entities=nil,nil,nil + stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil + acache,hcache,dcache=nil,nil,nil + reported_attribute_errors,mt,errorhandler=nil,nil,nil + return result +end +local function xmlconvert(data,settings) + local ok,result=pcall(function() return _xmlconvert_(data,settings) end) + if ok then + return result + else + return _xmlconvert_("",settings) + end +end +xml.convert=xmlconvert +function xml.inheritedconvert(data,xmldata) + local settings=xmldata.settings + if settings then + settings.parent_root=xmldata + end + local xc=xmlconvert(data,settings) + return xc +end +function xml.is_valid(root) + return root and root.dt and root.dt[1] and type(root.dt[1])=="table" and not root.dt[1].er +end +function xml.package(tag,attributes,data) + local ns,tg=match(tag,"^(.-):?([^:]+)$") + local t={ ns=ns,tg=tg,dt=data or "",at=attributes or {} } + setmetatable(t,mt) + return t +end +function xml.is_valid(root) + return root and not root.error +end +xml.errorhandler=report_xml +function xml.load(filename,settings) + local data="" + if type(filename)=="string" then + local f=io.open(filename,'r') + if f then + data=f:read("*all") + f:close() + end + elseif filename then + data=filename:read("*all") + end + if settings then + settings.currentresource=filename + local result=xmlconvert(data,settings) + settings.currentresource=nil + return result + else + return xmlconvert(data,{ currentresource=filename }) + end +end +local no_root={ no_root=true } +function xml.toxml(data) + if type(data)=="string" then + local root={ xmlconvert(data,no_root) } + return (#root>1 and root) or root[1] + else + return data + end +end +local function copy(old,tables) + if old then + tables=tables or {} + local new={} + if not tables[old] then + tables[old]=new + end + for k,v in next,old do + new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v + end + local mt=getmetatable(old) + if mt then + setmetatable(new,mt) + end + return new + else + return {} + end +end +xml.copy=copy +function xml.checkbom(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" and find(v.dt[1],"xml.*version=") then + return + end + end + insert(dt,1,{ special=true,ns="",tg="@pi@",dt={ "xml version='1.0' standalone='yes'" } } ) + insert(dt,2,"\n" ) + end +end +local f_attribute=formatters['%s=%q'] +local function verbose_element(e,handlers,escape) + local handle=handlers.handle + local serialize=handlers.serialize + local ens,etg,eat,edt,ern=e.ns,e.tg,e.at,e.dt,e.rn + local ats=eat and next(eat) and {} + if ats then + local n=0 + for k,v in next,eat do + n=n+1 + ats[n]=f_attribute(k,escaped(v)) + end + end + if ern and trace_entities and ern~=ens then + ens=ern + end + if ens~="" then + if edt and #edt>0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) + else + serialize(e,handlers) + end + end + handle("</",ens,":",etg,">") + else + if ats then + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") + end + end + else + if edt and #edt>0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") + end + for i=1,#edt do + local e=edt[i] + if type(e)=="string" then + handle(escaped(e)) + else + serialize(e,handlers) + end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") + else + handle("<",etg,"/>") + end + end + end +end +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[",e.dt[1],"]]>") +end +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end +local function verbose_text(e,handlers) + handlers.handle(escaped(e)) +end +local function verbose_document(e,handlers) + local serialize=handlers.serialize + local functions=handlers.functions + for i=1,#e do + local ei=e[i] + if type(ei)=="string" then + functions["@tx@"](ei,handlers) + else + serialize(ei,handlers) + end + end +end +local function serialize(e,handlers,...) + if e then + local initialize=handlers.initialize + local finalize=handlers.finalize + local functions=handlers.functions + if initialize then + local state=initialize(...) + if not state==true then + return state + end + end + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end + end +end +local function xserialize(e,handlers) + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end +end +local handlers={} +local function newhandlers(settings) + local t=table.copy(handlers[settings and settings.parent or "verbose"] or {}) + if settings then + for k,v in next,settings do + if type(v)=="table" then + local tk=t[k] if not tk then tk={} t[k]=tk end + for kk,vv in next,v do + tk[kk]=vv + end + else + t[k]=v + end + end + if settings.name then + handlers[settings.name]=t + end + end + utilities.storage.mark(t) + return t +end +local nofunction=function() end +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name]=fnc or nofunction +end +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end +function xml.gethandlers(name) + return handlers[name] +end +newhandlers { + name="verbose", + initialize=false, + finalize=false, + serialize=xserialize, + handle=print, + functions={ + ["@dc@"]=verbose_document, + ["@dt@"]=verbose_doctype, + ["@rt@"]=verbose_root, + ["@el@"]=verbose_element, + ["@pi@"]=verbose_pi, + ["@cm@"]=verbose_comment, + ["@cd@"]=verbose_cdata, + ["@tx@"]=verbose_text, + } +} +local result +local xmlfilehandler=newhandlers { + name="file", + initialize=function(name) + result=io.open(name,"wb") + return result + end, + finalize=function() + result:close() + return true + end, + handle=function(...) + result:write(...) + end, +} +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end +local result +local xmlstringhandler=newhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, +} +local function xmltostring(root) + if not root then + return "" + elseif type(root)=="string" then + return root + else + return serialize(root,xmlstringhandler) or "" + end +end +local function __tostring(root) + return (root and xmltostring(root)) or "" +end +initialize_mt=function(root) + mt={ __tostring=__tostring,__index=root } +end +xml.defaulthandlers=handlers +xml.newhandlers=newhandlers +xml.serialize=serialize +xml.tostring=xmltostring +local function xmlstring(e,handle) + if not handle or (e.special and e.tg~="@rt@") then + elseif e.tg then + local edt=e.dt + if edt then + for i=1,#edt do + xmlstring(edt[i],handle) + end + end + else + handle(e) + end +end +xml.string=xmlstring +function xml.settings(e) + while e do + local s=e.settings + if s then + return s + else + e=e.__p__ + end + end + return nil +end +function xml.root(e) + local r=e + while e do + e=e.__p__ + if e then + r=e + end + end + return r +end +function xml.parent(root) + return root.__p__ +end +function xml.body(root) + return root.ri and root.dt[root.ri] or root +end +function xml.name(root) + if not root then + return "" + end + local ns=root.ns + local tg=root.tg + if ns=="" then + return tg + else + return ns..":"..tg + end +end +function xml.erase(dt,k) + if dt then + if k then + dt[k]="" + else for k=1,#dt do + dt[1]={ "" } + end end + end +end +function xml.assign(dt,k,root) + if dt and k then + dt[k]=type(root)=="table" and xml.body(root) or root + return dt[k] + else + return xml.body(root) + end +end +function xml.tocdata(e,wrapper) + local whatever=type(e)=="table" and xmltostring(e.dt) or e or "" + if wrapper then + whatever=formatters["<%s>%s</%s>"](wrapper,whatever,wrapper) + end + local t={ special=true,ns="",tg="@cd@",at={},rn="",dt={ whatever },__p__=e } + setmetatable(t,getmetatable(e)) + e.dt={ t } +end +function xml.makestandalone(root) + if root.ri then + local dt=root.dt + for k=1,#dt do + local v=dt[k] + if type(v)=="table" and v.special and v.tg=="@pi@" then + local txt=v.dt[1] + if find(txt,"xml.*version=") then + v.dt[1]=txt.." standalone='yes'" + break + end + end + end + end + return root +end +function xml.kind(e) + local dt=e and e.dt + if dt then + local n=#dt + if n==1 then + local d=dt[1] + if d.special then + local tg=d.tg + if tg=="@cd@" then + return "cdata" + elseif tg=="@cm" then + return "comment" + elseif tg=="@pi@" then + return "instruction" + elseif tg=="@dt@" then + return "declaration" + end + elseif type(d)=="string" then + return "text" + end + return "element" + elseif n>0 then + return "mixed" + end + end + return "empty" +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true + +-- original size: 48229, stripped down to: 30684 + +if not modules then modules={} end modules ['lxml-lpt']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat,remove,insert=table.concat,table.remove,table.insert +local type,next,tonumber,tostring,setmetatable,load,select=type,next,tonumber,tostring,setmetatable,load,select +local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string.lower,string.gmatch,string.gsub,string.find,string.rep +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local setmetatableindex=table.setmetatableindex +local formatters=string.formatters +local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end +local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end +local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local report_lpath=logs.reporter("xml","lpath") +local xml=xml +local lpathcalls=0 function xml.lpathcalls () return lpathcalls end +local lpathcached=0 function xml.lpathcached() return lpathcached end +xml.functions=xml.functions or {} +local functions=xml.functions +xml.expressions=xml.expressions or {} +local expressions=xml.expressions +xml.finalizers=xml.finalizers or {} +local finalizers=xml.finalizers +xml.specialhandler=xml.specialhandler or {} +local specialhandler=xml.specialhandler +lpegpatterns.xml=lpegpatterns.xml or {} +local xmlpatterns=lpegpatterns.xml +finalizers.xml=finalizers.xml or {} +finalizers.tex=finalizers.tex or {} +local function fallback (t,name) + local fn=finalizers[name] + if fn then + t[name]=fn + else + report_lpath("unknown sub finalizer %a",name) + fn=function() end + end + return fn +end +setmetatableindex(finalizers.xml,fallback) +setmetatableindex(finalizers.tex,fallback) +xml.defaultprotocol="xml" +local apply_axis={} +apply_axis['root']=function(list) + local collected={} + for l=1,#list do + local ll=list[l] + local rt=ll + while ll do + ll=ll.__p__ + if ll then + rt=ll + end + end + collected[l]=rt + end + return collected +end +apply_axis['self']=function(list) + return list +end +apply_axis['child']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local dt=ll.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + end + end + ll.en=en + end + end + return collected +end +local function collect(list,collected,c) + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end + end + list.en=en + end + return c +end +apply_axis['descendant']=function(list) + local collected,c={},0 + for l=1,#list do + c=collect(list[l],collected,c) + end + return collected +end +local function collect(list,collected,c) + local dt=list.dt + if dt then + local en=0 + for k=1,#dt do + local dk=dt[k] + if dk.tg then + c=c+1 + collected[c]=dk + dk.ni=k + en=en+1 + dk.ei=en + c=collect(dk,collected,c) + end + end + list.en=en + end + return c +end +apply_axis['descendant-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + if ll.special~=true then + c=c+1 + collected[c]=ll + end + c=collect(ll,collected,c) + end + return collected +end +apply_axis['ancestor']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end + end + end + return collected +end +apply_axis['ancestor-or-self']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + c=c+1 + collected[c]=ll + while ll do + ll=ll.__p__ + if ll then + c=c+1 + collected[c]=ll + end + end + end + return collected +end +apply_axis['parent']=function(list) + local collected,c={},0 + for l=1,#list do + local pl=list[l].__p__ + if pl then + c=c+1 + collected[c]=pl + end + end + return collected +end +apply_axis['attribute']=function(list) + return {} +end +apply_axis['namespace']=function(list) + return {} +end +apply_axis['following']=function(list) + return {} +end +apply_axis['preceding']=function(list) + return {} +end +apply_axis['following-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni+1,#d do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end + end + end + return collected +end +apply_axis['preceding-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=1,ll.ni-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end + end + end + return collected +end +apply_axis['reverse-sibling']=function(list) + local collected,c={},0 + for l=1,#list do + local ll=list[l] + local p=ll.__p__ + local d=p.dt + for i=ll.ni-1,1,-1 do + local di=d[i] + if type(di)=="table" then + c=c+1 + collected[c]=di + end + end + end + return collected +end +apply_axis['auto-descendant-or-self']=apply_axis['descendant-or-self'] +apply_axis['auto-descendant']=apply_axis['descendant'] +apply_axis['auto-child']=apply_axis['child'] +apply_axis['auto-self']=apply_axis['self'] +apply_axis['initial-child']=apply_axis['child'] +local function apply_nodes(list,directive,nodes) + local maxn=#nodes + if maxn==3 then + local nns,ntg=nodes[2],nodes[3] + if not nns and not ntg then + if directive then + return list + else + return {} + end + else + local collected,c,m,p={},0,0,nil + if not nns then + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then + if directive then + if ntg==ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif ntg~=ltg then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + elseif not ntg then + for l=1,#list do + local ll=list[l] + local lns=ll.rn or ll.ns + if lns then + if directive then + if lns==nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif lns~=nns then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + else + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=ltg==ntg and lns==nns + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + end + return collected + end + else + local collected,c,m,p={},0,0,nil + for l=1,#list do + local ll=list[l] + local ltg=ll.tg + if ltg then + local lns=ll.rn or ll.ns + local ok=false + for n=1,maxn,3 do + local nns,ntg=nodes[n+1],nodes[n+2] + ok=(not ntg or ltg==ntg) and (not nns or lns==nns) + if ok then + break + end + end + if directive then + if ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + elseif not ok then + local llp=ll.__p__;if llp~=p then p,m=llp,1 else m=m+1 end + c=c+1 + collected[c],ll.mi=ll,m + end + end + end + return collected + end +end +local quit_expression=false +local function apply_expression(list,expression,order) + local collected,c={},0 + quit_expression=false + for l=1,#list do + local ll=list[l] + if expression(list,ll,l,order) then + c=c+1 + collected[c]=ll + end + if quit_expression then + break + end + end + return collected +end +local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb +local spaces=S(" \n\r\t\f")^0 +local lp_space=S(" \n\r\t\f") +local lp_any=P(1) +local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") +local lp_doequal=P("=")/"==" +local lp_or=P("|")/" or " +local lp_and=P("&")/" and " +local lp_builtin=P ( + P("text")/"(ll.dt[1] or '')"+ + P("content")/"ll.dt"+ + P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ + P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ + P("ns")/"ll.ns" + )*((spaces*P("(")*spaces*P(")"))/"") +local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") +local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" +local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" +local lp_fastpos=lp_fastpos_n+lp_fastpos_p +local lp_reserved=C("and")+C("or")+C("not")+C("div")+C("mod")+C("true")+C("false") +local lp_lua_function=Cs((R("az","AZ","__")^1*(P(".")*R("az","AZ","__")^1)^1)*("("))/"%0" +local lp_function=C(R("az","AZ","__")^1)*P("(")/function(t) + if expressions[t] then + return "expr."..t.."(" + else + return "expr.error(" + end +end +local lparent=P("(") +local rparent=P(")") +local noparent=1-(lparent+rparent) +local nested=P{lparent*(noparent+V(1))^0*rparent} +local value=P(lparent*C((noparent+nested)^0)*rparent) +local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_number=S("+-")*R("09")^1 +local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") +local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) +local cleaner +local lp_special=(C(P("name")+P("text")+P("tag")+P("count")+P("child")))*value/function(t,s) + if expressions[t] then + s=s and s~="" and lpegmatch(cleaner,s) + if s and s~="" then + return "expr."..t.."(ll,"..s..")" + else + return "expr."..t.."(ll)" + end + else + return "expr.error("..t..")" + end +end +local content=lp_builtin+lp_attribute+lp_special+lp_noequal+lp_doequal+lp_or+lp_and+lp_reserved+lp_lua_function+lp_function+lp_content+ + lp_child+lp_any +local converter=Cs ( + lp_fastpos+(P { lparent*(V(1))^0*rparent+content } )^0 +) +cleaner=Cs (( + lp_reserved+lp_number+lp_string+1 )^1 ) +local template_e=[[ + local expr = xml.expressions + return function(list,ll,l,order) + return %s + end +]] +local template_f_y=[[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] +local template_f_n=[[ + return xml.finalizers['%s']['%s'] +]] +local register_self={ kind="axis",axis="self" } +local register_parent={ kind="axis",axis="parent" } +local register_descendant={ kind="axis",axis="descendant" } +local register_child={ kind="axis",axis="child" } +local register_descendant_or_self={ kind="axis",axis="descendant-or-self" } +local register_root={ kind="axis",axis="root" } +local register_ancestor={ kind="axis",axis="ancestor" } +local register_ancestor_or_self={ kind="axis",axis="ancestor-or-self" } +local register_attribute={ kind="axis",axis="attribute" } +local register_namespace={ kind="axis",axis="namespace" } +local register_following={ kind="axis",axis="following" } +local register_following_sibling={ kind="axis",axis="following-sibling" } +local register_preceding={ kind="axis",axis="preceding" } +local register_preceding_sibling={ kind="axis",axis="preceding-sibling" } +local register_reverse_sibling={ kind="axis",axis="reverse-sibling" } +local register_auto_descendant_or_self={ kind="axis",axis="auto-descendant-or-self" } +local register_auto_descendant={ kind="axis",axis="auto-descendant" } +local register_auto_self={ kind="axis",axis="auto-self" } +local register_auto_child={ kind="axis",axis="auto-child" } +local register_initial_child={ kind="axis",axis="initial-child" } +local register_all_nodes={ kind="nodes",nodetest=true,nodes={ true,false,false } } +local skip={} +local function errorrunner_e(str,cnv) + if not skip[str] then + report_lpath("error in expression: %s => %s",str,cnv) + skip[str]=cnv or str + end + return false +end +local function errorrunner_f(str,arg) + report_lpath("error in finalizer: %s(%s)",str,arg or "") + return false +end +local function register_nodes(nodetest,nodes) + return { kind="nodes",nodetest=nodetest,nodes=nodes } +end +local function register_expression(expression) + local converted=lpegmatch(converter,expression) + local runner=load(format(template_e,converted)) + runner=(runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind="expression",expression=expression,converted=converted,evaluator=runner } +end +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments~="" then + runner=load(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) + else + runner=load(format(template_f_n,protocol or xml.defaultprotocol,name)) + end + runner=(runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind="finalizer",name=name,arguments=arguments,finalizer=runner } +end +local expression=P { "ex", + ex="["*C((V("sq")+V("dq")+(1-S("[]"))+V("ex"))^0)*"]", + sq="'"*(1-S("'"))^0*"'", + dq='"'*(1-S('"'))^0*'"', +} +local arguments=P { "ar", + ar="("*Cs((V("sq")+V("dq")+V("nq")+P(1-P(")")))^0)*")", + nq=((1-S("),'\""))^1)/function(s) return format("%q",s) end, + sq=P("'")*(1-P("'"))^0*P("'"), + dq=P('"')*(1-P('"'))^0*P('"'), +} +local function register_error(str) + return { kind="error",error=format("unparsed: %s",str) } +end +local special_1=P("*")*Cc(register_auto_descendant)*Cc(register_all_nodes) +local special_2=P("/")*Cc(register_auto_self) +local special_3=P("")*Cc(register_auto_self) +local no_nextcolon=P(-1)+#(1-P(":")) +local no_nextlparent=P(-1)+#(1-P("(")) +local pathparser=Ct { "patterns", + patterns=spaces*V("protocol")*spaces*( + (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) + ), + protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), + step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + special=special_1+special_2+special_3, + initial=(P("/")*spaces*Cc(register_initial_child))^-1, + error=(P(1)^1)/register_error, + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, + s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), + s_descendant=P("**")*Cc(register_descendant), + s_child=P("*")*no_nextcolon*Cc(register_child ), + s_parent=P("..")*Cc(register_parent ), + s_self=P("." )*Cc(register_self ), + s_root=P("^^")*Cc(register_root ), + s_ancestor=P("^")*Cc(register_ancestor ), + descendant=P("descendant::")*Cc(register_descendant ), + child=P("child::")*Cc(register_child ), + parent=P("parent::")*Cc(register_parent ), + self=P("self::")*Cc(register_self ), + root=P('root::')*Cc(register_root ), + ancestor=P('ancestor::')*Cc(register_ancestor ), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), + following=P('following::')*Cc(register_following ), + following_sibling=P('following-sibling::')*Cc(register_following_sibling ), + preceding=P('preceding::')*Cc(register_preceding ), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, + expressions=expression/register_expression, + letters=R("az")^1, + name=(1-S("/[]()|:*!"))^1, + negate=P("!")*Cc(false), + nodefunction=V("negate")+P("not")*Cc(false)+Cc(true), + nodetest=V("negate")+Cc(true), + nodename=(V("negate")+Cc(true))*spaces*((V("wildnodename")*P(":")*V("wildnodename"))+(Cc(false)*V("wildnodename"))), + wildnodename=(C(V("name"))+P("*")*Cc(false))*no_nextlparent, + nodeset=spaces*Ct(V("nodename")*(spaces*P("|")*spaces*V("nodename"))^0)*spaces, + finalizer=(Cb("protocol")*P("/")^-1*C(V("name"))*arguments*P(-1))/register_finalizer, +} +xmlpatterns.pathparser=pathparser +local cache={} +local function nodesettostring(set,nodetest) + local t={} + for i=1,#set,3 do + local directive,ns,tg=set[i],set[i+1],set[i+2] + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + tg=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1]=(directive and tg) or format("not(%s)",tg) + end + if nodetest==false then + return format("not(%s)",concat(t,"|")) + else + return concat(t,"|") + end +end +local function tagstostring(list) + if #list==0 then + return "no elements" + else + local t={} + for i=1,#list do + local li=list[i] + local ns,tg=li.ns,li.tg + if not ns or ns=="" then ns="*" end + if not tg or tg=="" then tg="*" end + t[i]=(tg=="@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end +end +xml.nodesettostring=nodesettostring +local lpath +local lshowoptions={ functions=false } +local function lshow(parsed) + if type(parsed)=="string" then + parsed=lpath(parsed) + end + report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, + table.serialize(parsed,false,lshowoptions)) +end +xml.lshow=lshow +local function add_comment(p,str) + local pc=p.comment + if not pc then + p.comment={ str } + else + pc[#pc+1]=str + end +end +lpath=function (pattern) + lpathcalls=lpathcalls+1 + if type(pattern)=="table" then + return pattern + else + local parsed=cache[pattern] + if parsed then + lpathcached=lpathcached+1 + else + parsed=lpegmatch(pathparser,pattern) + if parsed then + parsed.pattern=pattern + local np=#parsed + if np==0 then + parsed={ pattern=pattern,register_self,state="parsing error" } + report_lpath("parsing error in pattern: %s",pattern) + lshow(parsed) + else + local pi=parsed[1] + if pi.axis=="auto-child" then + if false then + add_comment(parsed,"auto-child replaced by auto-descendant-or-self") + parsed[1]=register_auto_descendant_or_self + else + add_comment(parsed,"auto-child replaced by auto-descendant") + parsed[1]=register_auto_descendant + end + elseif pi.axis=="initial-child" and np>1 and parsed[2].axis then + add_comment(parsed,"initial-child removed") + remove(parsed,1) + end + local np=#parsed + if np>1 then + local pnp=parsed[np] + if pnp.kind=="nodes" and pnp.nodetest==true then + local nodes=pnp.nodes + if nodes[1]==true and nodes[2]==false and nodes[3]==false then + add_comment(parsed,"redundant final wildcard filter removed") + remove(parsed,np) + end + end + end + end + else + parsed={ pattern=pattern } + end + cache[pattern]=parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) + end + end + return parsed + end +end +xml.lpath=lpath +local profiled={} xml.profiled=profiled +local function profiled_apply(list,parsed,nofparsed,order) + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 + p.finalized=p.finalized+1 + return collected + end + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + p.finalized=p.finalized+1 + return collected + end + return nil + end + end + if collected then + p.matched=p.matched+1 + end + return collected +end +local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil + end + end + return collected +end +local function normal_apply(list,parsed,nofparsed,order) + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + local axis=pi.axis + if axis~="self" then + collected=apply_axis[axis](collected) + end + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected==0 then + local pf=i<nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) + end + return nil + end + end + return collected +end +local function applylpath(list,pattern) + if not list then + return + end + local parsed=cache[pattern] + if parsed then + lpathcalls=lpathcalls+1 + lpathcached=lpathcached+1 + elseif type(pattern)=="table" then + lpathcalls=lpathcalls+1 + parsed=pattern + else + parsed=lpath(pattern) or pattern + end + if not parsed then + return + end + local nofparsed=#parsed + if nofparsed==0 then + return + end + if not trace_lpath then + return normal_apply ({ list },parsed,nofparsed,list.mi) + elseif trace_lprofile then + return profiled_apply({ list },parsed,nofparsed,list.mi) + else + return traced_apply ({ list },parsed,nofparsed,list.mi) + end +end +xml.applylpath=applylpath +function xml.filter(root,pattern) + return applylpath(root,pattern) +end +expressions.child=function(e,pattern) + return applylpath(e,pattern) +end +expressions.count=function(e,pattern) + local collected=applylpath(e,pattern) + return pattern and (collected and #collected) or 0 +end +expressions.oneof=function(s,...) + for i=1,select("#",...) do + if s==select(i,...) then + return true + end + end + return false +end +expressions.error=function(str) + xml.errorhandler(format("unknown function in lpath expression: %s",tostring(str or "?"))) + return false +end +expressions.undefined=function(s) + return s==nil +end +expressions.quit=function(s) + if s or s==nil then + quit_expression=true + end + return true +end +expressions.print=function(...) + print(...) + return true +end +expressions.find=find +expressions.upper=upper +expressions.lower=lower +expressions.number=tonumber +expressions.boolean=toboolean +function expressions.contains(str,pattern) + local t=type(str) + if t=="string" then + if find(str,pattern) then + return true + end + elseif t=="table" then + for i=1,#str do + local d=str[i] + if type(d)=="string" and find(d,pattern) then + return true + end + end + end + return false +end +function xml.expressions.idstring(str) + return type(str)=="string" and gsub(str,"^#","") or "" +end +local function traverse(root,pattern,handle) + local collected=applylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local r=e.__p__ + handle(r,r.dt,e.ni) + end + end +end +local function selection(root,pattern,handle) + local collected=applylpath(root,pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) + end + else + return collected + end + end +end +xml.traverse=traverse +xml.selection=selection +local function dofunction(collected,fnc,...) + if collected then + local f=functions[fnc] + if f then + for c=1,#collected do + f(collected[c],...) + end + else + report_lpath("unknown function %a",fnc) + end + end +end +finalizers.xml["function"]=dofunction +finalizers.tex["function"]=dofunction +expressions.text=function(e,n) + local rdt=e.__p__.dt + return rdt and rdt[n] or "" +end +expressions.name=function(e,n) + local found=false + n=tonumber(n) or 0 + if n==0 then + found=type(e)=="table" and e + elseif n<0 then + local d,k=e.__p__.dt,e.ni + for i=k-1,1,-1 do + local di=d[i] + if type(di)=="table" then + if n==-1 then + found=di + break + else + n=n+1 + end + end + end + else + local d,k=e.__p__.dt,e.ni + for i=k+1,#d,1 do + local di=d[i] + if type(di)=="table" then + if n==1 then + found=di + break + else + n=n-1 + end + end + end + end + if found then + local ns,tg=found.rn or found.ns or "",found.tg + if ns~="" then + return ns..":"..tg + else + return tg + end + else + return "" + end +end +expressions.tag=function(e,n) + if not e then + return "" + else + local found=false + n=tonumber(n) or 0 + if n==0 then + found=(type(e)=="table") and e + elseif n<0 then + local d,k=e.__p__.dt,e.ni + for i=k-1,1,-1 do + local di=d[i] + if type(di)=="table" then + if n==-1 then + found=di + break + else + n=n+1 + end + end + end + else + local d,k=e.__p__.dt,e.ni + for i=k+1,#d,1 do + local di=d[i] + if type(di)=="table" then + if n==1 then + found=di + break + else + n=n-1 + end + end + end + end + return (found and found.tg) or "" + end +end +local dummy=function() end +function xml.elements(root,pattern,reverse) + local collected=applylpath(root,pattern) + if not collected then + return dummy + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 + return function() + if c>1 then + c=c-1 + local e=collected[c] + local r=e.__p__ + return r,r.dt,e.ni + end + end + else + local c=0 + return function() + if c<n then + c=c+1 + local e=collected[c] + local r=e.__p__ + return r,r.dt,e.ni + end + end + end +end +function xml.collected(root,pattern,reverse) + local collected=applylpath(root,pattern) + if not collected then + return dummy + end + local n=#collected + if n==0 then + return dummy + end + if reverse then + local c=n+1 + return function() + if c>1 then + c=c-1 + return collected[c] + end + end + else + local c=0 + return function() + if c<n then + c=c+1 + return collected[c] + end + end + end +end +function xml.inspect(collection,pattern) + pattern=pattern or "." + for e in xml.collected(collection,pattern or ".") do + report_lpath("pattern: %s\n\n%s\n",pattern,xml.tostring(e)) + end +end +local function split(e) + local dt=e.dt + if dt then + for i=1,#dt do + local dti=dt[i] + if type(dti)=="string" then + dti=gsub(dti,"^[\n\r]*(.-)[\n\r]*","%1") + dti=gsub(dti,"[\n\r]+","\n\n") + dt[i]=dti + else + split(dti) + end + end + end + return e +end +function xml.finalizers.paragraphs(c) + for i=1,#c do + split(c[i]) + end + return c +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true + +-- original size: 3684, stripped down to: 1957 + +if not modules then modules={} end modules ['lxml-mis']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local xml,lpeg,string=xml,lpeg,string +local concat=table.concat +local type,next,tonumber,tostring,setmetatable,loadstring=type,next,tonumber,tostring,setmetatable,loadstring +local format,gsub,match=string.format,string.gsub,string.match +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local P,S,R,C,V,Cc,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.Cc,lpeg.Cs +lpegpatterns.xml=lpegpatterns.xml or {} +local xmlpatterns=lpegpatterns.xml +local function xmlgsub(t,old,new) + local dt=t.dt + if dt then + for k=1,#dt do + local v=dt[k] + if type(v)=="string" then + dt[k]=gsub(v,old,new) + else + xmlgsub(v,old,new) + end + end + end +end +function xml.stripleadingspaces(dk,d,k) + if d and k then + local dkm=d[k-1] + if dkm and type(dkm)=="string" then + local s=match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") + end + end +end +local normal=(1-S("<&>"))^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local escaped=Cs(normal*(special*normal)^0) +local normal=(1-S"&")^0 +local special=P("<")/"<"+P(">")/">"+P("&")/"&" +local unescaped=Cs(normal*(special*normal)^0) +local cleansed=Cs(((P("<")*(1-P(">"))^0*P(">"))/""+1)^0) +xmlpatterns.escaped=escaped +xmlpatterns.unescaped=unescaped +xmlpatterns.cleansed=cleansed +function xml.escaped (str) return lpegmatch(escaped,str) end +function xml.unescaped(str) return lpegmatch(unescaped,str) end +function xml.cleansed (str) return lpegmatch(cleansed,str) end +function xml.fillin(root,pattern,str,check) + local e=xml.first(root,pattern) + if e then + local n=#e.dt + if not check or n==0 or (n==1 and e.dt[1]=="") then + e.dt={ str } + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true + +-- original size: 28786, stripped down to: 20578 + +if not modules then modules={} end modules ['lxml-aux']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_manipulations=false trackers.register("lxml.manipulations",function(v) trace_manipulations=v end) +local trace_inclusions=false trackers.register("lxml.inclusions",function(v) trace_inclusions=v end) +local report_xml=logs.reporter("xml") +local xml=xml +local xmlcopy,xmlname=xml.copy,xml.name +local xmlinheritedconvert=xml.inheritedconvert +local xmlapplylpath=xml.applylpath +local xmlfilter=xml.filter +local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable +local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat +local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip +local utfbyte=utf.byte +local lpegmatch=lpeg.match +local striplinepatterns=utilities.strings.striplinepatterns +local function report(what,pattern,c,e) + report_xml("%s element %a, root %a, position %a, index %a, pattern %a",what,xmlname(e),xmlname(e.__p__),c,e.ni,pattern) +end +local function withelements(e,handle,depth) + if e and handle then + local edt=e.dt + if edt then + depth=depth or 0 + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + handle(e,depth) + withelements(e,handle,depth+1) + end + end + end + end +end +xml.withelements=withelements +function xml.withelement(e,n,handle) + if e and n~=0 and handle then + local edt=e.dt + if edt then + if n>0 then + for i=1,#edt do + local ei=edt[i] + if type(ei)=="table" then + if n==1 then + handle(ei) + return + else + n=n-1 + end + end + end + elseif n<0 then + for i=#edt,1,-1 do + local ei=edt[i] + if type(ei)=="table" then + if n==-1 then + handle(ei) + return + else + n=n+1 + end + end + end + end + end + end +end +function xml.each(root,pattern,handle,reverse) + local collected=xmlapplylpath(root,pattern) + if collected then + if handle then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end + end + end + return collected + end +end +function xml.processattributes(root,pattern,handle) + local collected=xmlapplylpath(root,pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end + end + return collected +end +function xml.collect(root,pattern) + return xmlapplylpath(root,pattern) +end +function xml.collecttexts(root,pattern,flatten) + local collected=xmlapplylpath(root,pattern) + if collected and flatten then + local xmltostring=xml.tostring + for c=1,#collected do + collected[c]=xmltostring(collected[c].dt) + end + end + return collected or {} +end +function xml.collect_tags(root,pattern,nonamespace) + local collected=xmlapplylpath(root,pattern) + if collected then + local t,n={},0 + for c=1,#collected do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace then + t[n]=tg + elseif ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end + end + return t + end +end +local no_root={ no_root=true } +local function redo_ni(d) + for k=1,#d do + local dk=d[k] + if type(dk)=="table" then + dk.ni=k + end + end +end +xml.reindex=redo_ni +local function xmltoelement(whatever,root) + if not whatever then + return nil + end + local element + if type(whatever)=="string" then + element=xmlinheritedconvert(whatever,root) + else + element=whatever + end + if element.error then + return whatever + end + if element then + end + return element +end +xml.toelement=xmltoelement +local function copiedelement(element,newparent) + if type(element)=="string" then + return element + else + element=xmlcopy(element).dt + if newparent and type(element)=="table" then + element.__p__=newparent + end + return element + end +end +function xml.delete(root,pattern) + if not pattern or pattern=="" then + local p=root.__p__ + if p then + if trace_manipulations then + report('deleting',"--",c,root) + end + local d=p.dt + remove(d,root.ni) + redo_ni(d) + end + else + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('deleting',pattern,c,e) + end + local d=p.dt + local ni=e.ni + if ni<=#d then + if false then + p.dt[ni]="" + else + remove(d,ni) + redo_ni(d) + end + else + end + end + end + end + end +end +function xml.replace(root,pattern,whatever) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local p=e.__p__ + if p then + if trace_manipulations then + report('replacing',pattern,c,e) + end + local d=p.dt + d[e.ni]=copiedelement(element,p) + redo_ni(d) + end + end + end +end +local function wrap(e,wrapper) + local t={ + rn=e.rn, + tg=e.tg, + ns=e.ns, + at=e.at, + dt=e.dt, + __p__=e, + } + setmetatable(t,getmetatable(e)) + e.rn=wrapper.rn or e.rn or "" + e.tg=wrapper.tg or e.tg or "" + e.ns=wrapper.ns or e.ns or "" + e.at=fastcopy(wrapper.at) + e.dt={ t } +end +function xml.wrap(root,pattern,whatever) + if whatever then + local wrapper=xmltoelement(whatever,root) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if trace_manipulations then + report('wrapping',pattern,c,e) + end + wrap(e,wrapper) + end + end + else + wrap(root,xmltoelement(pattern)) + end +end +local function inject_element(root,pattern,whatever,prepend) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function inject_e(e) + local r=e.__p__ + local d,k,rri=r.dt,e.ni,r.ri + local edt=(rri and d[rri].dt) or (d and d[k] and d[k].dt) + if edt then + local be,af + local cp=copiedelement(element,e) + if prepend then + be,af=cp,edt + else + be,af=edt,cp + end + local bn=#be + for i=1,#af do + bn=bn+1 + be[bn]=af[i] + end + if rri then + r.dt[rri].dt=be + else + d[k].dt=be + end + redo_ni(d) + end + end + if not collected then + elseif collected.tg then + inject_e(collected) + else + for c=1,#collected do + inject_e(collected[c]) + end + end +end +local function insert_element(root,pattern,whatever,before) + local element=root and xmltoelement(whatever,root) + local collected=element and xmlapplylpath(root,pattern) + local function insert_e(e) + local r=e.__p__ + local d,k=r.dt,e.ni + if not before then + k=k+1 + end + insert(d,k,copiedelement(element,r)) + redo_ni(d) + end + if not collected then + elseif collected.tg then + insert_e(collected) + else + for c=1,#collected do + insert_e(collected[c]) + end + end +end +xml.insert_element=insert_element +xml.insertafter=insert_element +xml.insertbefore=function(r,p,e) insert_element(r,p,e,true) end +xml.injectafter=inject_element +xml.injectbefore=function(r,p,e) inject_element(r,p,e,true) end +local function include(xmldata,pattern,attribute,recursive,loaddata,level) + pattern=pattern or 'include' + loaddata=loaddata or io.loaddata + local collected=xmlapplylpath(xmldata,pattern) + if collected then + if not level then + level=1 + end + for c=1,#collected do + local ek=collected[c] + local name=nil + local ekdt=ek.dt + local ekat=ek.at + local ekrt=ek.__p__ + local epdt=ekrt.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt + end + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then + break + end + end + end + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") + end + end + if not data or data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local inclusions=xmldata.settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + else + xmldata.settings.inclusions={ name } + end + if child.er then + local badinclusions=xmldata.settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + xmldata.settings.badinclusions={ name } + end + end + end + end + end + end +end +xml.include=include +function xml.inclusion(e,default) + while e do + local f=e.__f__ + if f then + return f + else + e=e.__p__ + end + end + return default +end +local function getinclusions(key,e,sorted) + while e do + local settings=e.settings + if settings then + local inclusions=settings[key] + if inclusions then + inclusions=table.unique(inclusions) + if sorted then + table.sort(inclusions) + end + return inclusions + else + e=e.__p__ + end + else + e=e.__p__ + end + end +end +function xml.inclusions(e,sorted) + return getinclusions("inclusions",e,sorted) +end +function xml.badinclusions(e,sorted) + return getinclusions("badinclusions",e,sorted) +end +local b_collapser=lpeg.patterns.b_collapser +local m_collapser=lpeg.patterns.m_collapser +local e_collapser=lpeg.patterns.e_collapser +local b_stripper=lpeg.patterns.b_stripper +local m_stripper=lpeg.patterns.m_stripper +local e_stripper=lpeg.patterns.e_stripper +local lpegmatch=lpeg.match +local function stripelement(e,nolines,anywhere) + local edt=e.dt + if edt then + local n=#edt + if n==0 then + return e + elseif anywhere then + local t={} + local m=0 + for e=1,n do + local str=edt[e] + if type(str)~="string" then + m=m+1 + t[m]=str + elseif str~="" then + if nolines then + str=lpegmatch((n==1 and b_collapser) or (n==m and e_collapser) or m_collapser,str) + else + str=lpegmatch((n==1 and b_stripper) or (n==m and e_stripper) or m_stripper,str) + end + if str~="" then + m=m+1 + t[m]=str + end + end + end + e.dt=t + else + local str=edt[1] + if type(str)=="string" then + if str~="" then + str=lpegmatch(nolines and b_collapser or b_stripper,str) + end + if str=="" then + remove(edt,1) + n=n-1 + else + edt[1]=str + end + end + if n>0 then + str=edt[n] + if type(str)=="string" then + if str=="" then + remove(edt) + else + str=lpegmatch(nolines and e_collapser or e_stripper,str) + if str=="" then + remove(edt) + else + edt[n]=str + end + end + end + end + end + end + return e +end +xml.stripelement=stripelement +function xml.strip(root,pattern,nolines,anywhere) + local collected=xmlapplylpath(root,pattern) + if collected then + for i=1,#collected do + stripelement(collected[i],nolines,anywhere) + end + end +end +local function renamespace(root,oldspace,newspace) + local ndt=#root.dt + for i=1,ndt or 0 do + local e=root[i] + if type(e)=="table" then + if e.ns==oldspace then + e.ns=newspace + if e.rn then + e.rn=newspace + end + end + local edt=e.dt + if edt then + renamespace(edt,oldspace,newspace) + end + end + end +end +xml.renamespace=renamespace +function xml.remaptag(root,pattern,newtg) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].tg=newtg + end + end +end +function xml.remapnamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + collected[c].ns=newns + end + end +end +function xml.checknamespace(root,pattern,newns) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + if (not e.rn or e.rn=="") and e.ns=="" then + e.rn=newns + end + end + end +end +function xml.remapname(root,pattern,newtg,newns,newrn) + local collected=xmlapplylpath(root,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + e.tg,e.ns,e.rn=newtg,newns,newrn + end + end +end +function xml.cdatatotext(e) + local dt=e.dt + if #dt==1 then + local first=dt[1] + if first.tg=="@cd@" then + e.dt=first.dt + end + else + end +end +function xml.texttocdata(e) + local dt=e.dt + local s=xml.tostring(dt) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +function xml.elementtocdata(e) + local dt=e.dt + local s=xml.tostring(e) + e.tg="@cd@" + e.special=true + e.ns="" + e.rn="" + e.dt={ s } + e.at=nil +end +xml.builtinentities=table.tohash { "amp","quot","apos","lt","gt" } +local entities=characters and characters.entities or nil +local builtinentities=xml.builtinentities +function xml.addentitiesdoctype(root,option) + if not entities then + require("char-ent") + entities=characters.entities + end + if entities and root and root.tg=="@rt@" and root.statistics then + local list={} + local hexify=option=="hexadecimal" + for k,v in table.sortedhash(root.statistics.entities.names) do + if not builtinentities[k] then + local e=entities[k] + if not e then + e=format("[%s]",k) + elseif hexify then + e=format("&#%05X;",utfbyte(k)) + end + list[#list+1]=format(" <!ENTITY %s %q >",k,e) + end + end + local dt=root.dt + local n=dt[1].tg=="@pi@" and 2 or 1 + if #list>0 then + insert(dt,n,{ "\n" }) + insert(dt,n,{ + tg="@dt@", + dt={ format("Something [\n%s\n] ",concat(list)) }, + ns="", + special=true, + }) + insert(dt,n,{ "\n\n" }) + else + end + end +end +xml.all=xml.each +xml.insert=xml.insertafter +xml.inject=xml.injectafter +xml.after=xml.insertafter +xml.before=xml.insertbefore +xml.process=xml.each +xml.obsolete=xml.obsolete or {} +local obsolete=xml.obsolete +xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip +xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect +xml.delete_element=xml.delete obsolete.delete_element=xml.delete +xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.each_element=xml.each obsolete.each_element=xml.each +xml.process_elements=xml.process obsolete.process_elements=xml.process +xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter +xml.insert_element_before=xml.insertbefore obsolete.insert_element_before=xml.insertbefore +xml.inject_element_after=xml.injectafter obsolete.inject_element_after=xml.injectafter +xml.inject_element_before=xml.injectbefore obsolete.inject_element_before=xml.injectbefore +xml.process_attributes=xml.processattributes obsolete.process_attributes=xml.processattributes +xml.collect_texts=xml.collecttexts obsolete.collect_texts=xml.collecttexts +xml.inject_element=xml.inject obsolete.inject_element=xml.inject +xml.remap_tag=xml.remaptag obsolete.remap_tag=xml.remaptag +xml.remap_name=xml.remapname obsolete.remap_name=xml.remapname +xml.remap_namespace=xml.remapnamespace obsolete.remap_namespace=xml.remapnamespace +function xml.cdata(e) + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end + end + return "" +end +function xml.finalizers.xml.cdata(collected) + if collected then + local e=collected[1] + if e then + local dt=e.dt + if dt and #dt==1 then + local first=dt[1] + return first.tg=="@cd@" and first.dt[1] or "" + end + end + end + return "" +end +function xml.insertcomment(e,str,n) + insert(e.dt,n or 1,{ + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.insertcdata(e,str,n) + insert(e.dt,n or 1,{ + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + }) +end +function xml.setcomment(e,str,n) + e.dt={ { + tg="@cm@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.setcdata(e,str) + e.dt={ { + tg="@cd@", + ns="", + special=true, + at={}, + dt={ str }, + } } +end +function xml.separate(x,pattern) + local collected=xmlapplylpath(x,pattern) + if collected then + for c=1,#collected do + local e=collected[c] + local d=e.dt + if d==x then + report_xml("warning: xml.separate changes root") + x=d + end + local t,n={ "\n" },1 + local i,nd=1,#d + while i<=nd do + while i<=nd do + local di=d[i] + if type(di)=="string" then + if di=="\n" or find(di,"^%s+$") then + i=i+1 + else + d[i]=strip(di) + break + end + else + break + end + end + if i>nd then + break + end + t[n+1]="\n" + t[n+2]=d[i] + t[n+3]="\n" + n=n+3 + i=i+1 + end + t[n+1]="\n" + setmetatable(t,getmetatable(d)) + e.dt=t + end + end + return x +end +local helpers=xml.helpers or {} +xml.helpers=helpers +local function normal(e,action) + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)=="string" and str~="" then + edt[i]=action(str) + end + end + end +end +local function recurse(e,action) + local edt=e.dt + if edt then + for i=1,#edt do + local str=edt[i] + if type(str)~="string" then + recurse(str,action) + elseif str~="" then + edt[i]=action(str) + end + end + end +end +function helpers.recursetext(collected,action,recursive) + if recursive then + for i=1,#collected do + recurse(collected[i],action) + end + else + for i=1,#collected do + normal(collected[i],action) + end + end +end +local specials={ + ["@rt@"]="root", + ["@pi@"]="instruction", + ["@cm@"]="comment", + ["@dt@"]="declaration", + ["@cd@"]="cdata", +} +local function convert(x,strip,flat) + local ns=x.ns + local tg=x.tg + local at=x.at + local dt=x.dt + local node=flat and { + [0]=(not x.special and (ns~="" and ns..":"..tg or tg)) or nil, + } or { + _namespace=ns~="" and ns or nil, + _tag=not x.special and tg or nil, + _type=specials[tg] or "_element", + } + if at then + for k,v in next,at do + node[k]=v + end + end + local n=0 + for i=1,#dt do + local di=dt[i] + if type(di)=="table" then + if flat and di.special then + else + di=convert(di,strip,flat) + if di then + n=n+1 + node[n]=di + end + end + elseif strip then + di=lpegmatch(strip,di) + if di~="" then + n=n+1 + node[n]=di + end + else + n=n+1 + node[n]=di + end + end + if next(node) then + return node + end +end +function xml.totable(x,strip,flat) + if type(x)=="table" then + if strip then + strip=striplinepatterns[strip] + end + return convert(x,strip,flat) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true + +-- original size: 10274, stripped down to: 7538 + +if not modules then modules={} end modules ['lxml-xml']={ + version=1.001, + comment="this module is the basis for the lxml-* ones", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local concat=table.concat +local find,lower,upper=string.find,string.lower,string.upper +local xml=xml +local finalizers=xml.finalizers.xml +local xmlfilter=xml.filter +local xmltostring=xml.tostring +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmlnewhandlers=xml.newhandlers +local function first(collected) + return collected and collected[1] +end +local function last(collected) + return collected and collected[#collected] +end +local function all(collected) + return collected +end +local reverse=table.reversed +local function attribute(collected,name) + if collected and #collected>0 then + local at=collected[1].at + return at and at[name] + end +end +local function att(id,name) + local at=id.at + return at and at[name] +end +local function count(collected) + return collected and #collected or 0 +end +local function position(collected,n) + if not collected then + return 0 + end + local nc=#collected + if nc==0 then + return 0 + end + n=tonumber(n) or 0 + if n<0 then + return collected[nc+n+1] + elseif n>0 then + return collected[n] + else + return collected[1].mi or 0 + end +end +local function match(collected) + return collected and #collected>0 and collected[1].mi or 0 +end +local function index(collected) + return collected and #collected>0 and collected[1].ni or 0 +end +local function attributes(collected,arguments) + if collected and #collected>0 then + local at=collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at + end + end +end +local function chainattribute(collected,arguments) + if collected and #collected>0 then + local e=collected[1] + while e do + local at=e.at + if at then + local a=at[arguments] + if a then + return a + end + else + break + end + e=e.__p__ + end + end + return "" +end +local function raw(collected) + if collected and #collected>0 then + local e=collected[1] or collected + return e and xmltostring(e) or "" + else + return "" + end +end +local xmltexthandler=xmlnewhandlers { + name="string", + initialize=function() + result={} + return result + end, + finalize=function() + return concat(result) + end, + handle=function(...) + result[#result+1]=concat {... } + end, + escape=false, +} +local function xmltotext(root) + local dt=root.dt + if not dt then + return "" + end + local nt=#dt + if nt==0 then + return "" + elseif nt==1 and type(dt[1])=="string" then + return dt[1] + else + return xmlserialize(root,xmltexthandler) or "" + end +end +local function text(collected) + if collected then + local e=collected[1] or collected + return e and xmltotext(e) or "" + else + return "" + end +end +local function texts(collected) + if not collected then + return {} + end + local nc=#collected + if nc==0 then + return {} + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + if e and e.dt then + n=n+1 + t[n]=e.dt + end + end + return t +end +local function tag(collected,n) + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + return c and c.tg +end +local function name(collected,n) + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local c + if n==0 or not n then + c=collected[1] + elseif n>1 then + c=collected[n] + else + c=collected[nc-n+1] + end + if not c then + elseif c.ns=="" then + return c.tg + else + return c.ns..":"..c.tg + end +end +local function tags(collected,nonamespace) + if not collected then + return + end + local nc=#collected + if nc==0 then + return + end + local t,n={},0 + for c=1,nc do + local e=collected[c] + local ns,tg=e.ns,e.tg + n=n+1 + if nonamespace or ns=="" then + t[n]=tg + else + t[n]=ns..":"..tg + end + end + return t +end +local function empty(collected,spacesonly) + if not collected then + return true + end + local nc=#collected + if nc==0 then + return true + end + for c=1,nc do + local e=collected[c] + if e then + local edt=e.dt + if edt then + local n=#edt + if n==1 then + local edk=edt[1] + local typ=type(edk) + if typ=="table" then + return false + elseif edk~="" then + return false + elseif spacesonly and not find(edk,"%S") then + return false + end + elseif n>1 then + return false + end + end + end + end + return true +end +finalizers.first=first +finalizers.last=last +finalizers.all=all +finalizers.reverse=reverse +finalizers.elements=all +finalizers.default=all +finalizers.attribute=attribute +finalizers.att=att +finalizers.count=count +finalizers.position=position +finalizers.match=match +finalizers.index=index +finalizers.attributes=attributes +finalizers.chainattribute=chainattribute +finalizers.text=text +finalizers.texts=texts +finalizers.tag=tag +finalizers.name=name +finalizers.tags=tags +finalizers.empty=empty +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) +end +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) +end +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) +end +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) +end +function xml.raw(id,pattern) + if pattern then + return raw(xmlfilter(id,pattern)) + else + return raw(id) + end +end +function xml.text(id,pattern) + if pattern then + local collected=xmlfilter(id,pattern) + return collected and #collected>0 and xmltotext(collected[1]) or "" + elseif id then + return xmltotext(id) or "" + else + return "" + end +end +xml.content=text +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) +end +function xml.match(id,pattern) + return match(xmlfilter(id,pattern)) +end +function xml.empty(id,pattern,spacesonly) + return empty(xmlfilter(id,pattern),spacesonly) +end +xml.all=xml.filter +xml.index=xml.position +xml.found=xml.filter +local function totable(x) + local t={} + for e in xmlcollected(x[1] or x,"/*") do + t[e.tg]=xmltostring(e.dt) or "" + end + return next(t) and t or nil +end +xml.table=totable +finalizers.table=totable +local function textonly(e,t) + if e then + local edt=e.dt + if edt then + for i=1,#edt do + local e=edt[i] + if type(e)=="table" then + textonly(e,t) + else + t[#t+1]=e + end + end + end + end + return t +end +function xml.textonly(e) + return concat(textonly(e,{})) +end +function finalizers.lowerall(collected) + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=lower(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[lower(k)]=v + end + e.at=t + end + end + end +end +function finalizers.upperall(collected) + for c=1,#collected do + local e=collected[c] + if not e.special then + e.tg=upper(e.tg) + local eat=e.at + if eat then + local t={} + for k,v in next,eat do + t[upper(k)]=v + end + e.at=t + end + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["trac-xml"] = package.loaded["trac-xml"] or true + +-- original size: 6351, stripped down to: 4919 + +if not modules then modules={} end modules ['trac-xml']={ + version=1.001, + comment="companion to trac-log.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local formatters=string.formatters +local reporters=logs.reporters +local xmlserialize=xml.serialize +local xmlcollected=xml.collected +local xmltext=xml.text +local xmlfirst=xml.first +local function showhelp(specification,...) + local root=xml.convert(specification.helpinfo or "") + if not root then + return + end + local xs=xml.gethandlers("string") + xml.sethandlersfunction(xs,"short",function(e,handler) xmlserialize(e.dt,handler) end) + xml.sethandlersfunction(xs,"ref",function(e,handler) handler.handle("--"..e.at.name) end) + local wantedcategories=select("#",...)==0 and true or table.tohash {... } + local nofcategories=xml.count(root,"/application/flags/category") + local report=specification.report + for category in xmlcollected(root,"/application/flags/category") do + local categoryname=category.at.name or "" + if wantedcategories==true or wantedcategories[categoryname] then + if nofcategories>1 then + report("%s options:",categoryname) + report() + end + for subcategory in xmlcollected(category,"/subcategory") do + for flag in xmlcollected(subcategory,"/flag") do + local name=flag.at.name + local value=flag.at.value + local short=xmltext(xmlfirst(flag,"/short")) + if value then + report("--%-20s %s",formatters["%s=%s"](name,value),short) + else + report("--%-20s %s",name,short) + end + end + report() + end + end + end + for category in xmlcollected(root,"/application/examples/category") do + local title=xmltext(xmlfirst(category,"/title")) + if title and title~="" then + report() + report(title) + report() + end + for subcategory in xmlcollected(category,"/subcategory") do + for example in xmlcollected(subcategory,"/example") do + local command=xmltext(xmlfirst(example,"/command")) + local comment=xmltext(xmlfirst(example,"/comment")) + report(command) + end + report() + end + end + for comment in xmlcollected(root,"/application/comments/comment") do + local comment=xmltext(comment) + report() + report(comment) + report() + end +end +local reporthelp=reporters.help +local exporthelp=reporters.export +local function xmlfound(t) + local helpinfo=t.helpinfo + if type(helpinfo)=="table" then + return false + end + if type(helpinfo)~="string" then + helpinfo="Warning: no helpinfo found." + t.helpinfo=helpinfo + return false + end + if string.find(helpinfo,".xml$") then + local ownscript=environment.ownscript + local helpdata=false + if ownscript then + local helpfile=file.join(file.pathpart(ownscript),helpinfo) + helpdata=io.loaddata(helpfile) + if helpdata=="" then + helpdata=false + end + end + if not helpdata then + local helpfile=resolvers.findfile(helpinfo,"tex") + helpdata=helpfile and io.loaddata(helpfile) + end + if helpdata and helpdata~="" then + helpinfo=helpdata + else + helpinfo=formatters["Warning: help file %a is not found."](helpinfo) + end + end + t.helpinfo=helpinfo + return string.find(t.helpinfo,"^<%?xml") and true or false +end +function reporters.help(t,...) + if xmlfound(t) then + showhelp(t,...) + else + reporthelp(t,...) + end +end +function reporters.export(t,methods,filename) + if not xmlfound(t) then + return exporthelp(t) + end + if not methods or methods=="" then + methods=environment.arguments["exporthelp"] + end + if not filename or filename=="" then + filename=environment.files[1] + end + dofile(resolvers.findfile("trac-exp.lua","tex")) + local exporters=logs.exporters + if not exporters or not methods then + return exporthelp(t) + end + if methods=="all" then + methods=table.keys(exporters) + elseif type(methods)=="string" then + methods=utilities.parsers.settings_to_array(methods) + else + return exporthelp(t) + end + if type(filename)~="string" or filename=="" then + filename=false + elseif file.pathpart(filename)=="" then + t.report("export file %a will not be saved on the current path (safeguard)",filename) + return + end + for i=1,#methods do + local method=methods[i] + local exporter=exporters[method] + if exporter then + local result=exporter(t,method) + if result and result~="" then + if filename then + local fullname=file.replacesuffix(filename,method) + t.report("saving export in %a",fullname) + io.savedata(fullname,result) + else + reporters.lines(t,result) + end + else + t.report("no output from exporter %a",method) + end + else + t.report("unknown exporter %a",method) + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-ini"] = package.loaded["data-ini"] or true + +-- original size: 11085, stripped down to: 7662 + +if not modules then modules={} end modules ['data-ini']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local next,type,getmetatable,rawset=next,type,getmetatable,rawset +local gsub,find,gmatch,char=string.gsub,string.find,string.gmatch,string.char +local filedirname,filebasename,filejoin=file.dirname,file.basename,file.join +local ostype,osname,osuname,ossetenv,osgetenv=os.type,os.name,os.uname,os.setenv,os.getenv +local P,S,R,C,Cs,Cc,lpegmatch=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers.register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local report_initialization=logs.reporter("resolvers","initialization") +resolvers=resolvers or {} +local resolvers=resolvers +texconfig.kpse_init=false +texconfig.shell_escape='t' +if not (environment and environment.default_texmfcnf) and kpse and kpse.default_texmfcnf then + local default_texmfcnf=kpse.default_texmfcnf() + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOLOC","selfautoloc:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTODIR","selfautodir:") + default_texmfcnf=gsub(default_texmfcnf,"$SELFAUTOPARENT","selfautoparent:") + default_texmfcnf=gsub(default_texmfcnf,"$HOME","home:") + environment.default_texmfcnf=default_texmfcnf +end +kpse={ original=kpse } +setmetatable(kpse,{ + __index=function(kp,name) + report_initialization("fatal error: kpse library is accessed (key: %s)",name) + os.exit() + end +} ) +do + local osfontdir=osgetenv("OSFONTDIR") + if osfontdir and osfontdir~="" then + elseif osname=="windows" then + ossetenv("OSFONTDIR","c:/windows/fonts//") + elseif osname=="macosx" then + ossetenv("OSFONTDIR","$HOME/Library/Fonts//;/Library/Fonts//;/System/Library/Fonts//") + end +end +do + local homedir=osgetenv(ostype=="windows" and 'USERPROFILE' or 'HOME') or '' + if not homedir or homedir=="" then + homedir=char(127) + end + homedir=file.collapsepath(homedir) + ossetenv("HOME",homedir) + ossetenv("USERPROFILE",homedir) + environment.homedir=homedir +end +do + local args=environment.originalarguments or arg + if not environment.ownmain then + environment.ownmain=status and string.match(string.lower(status.banner),"this is ([%a]+)") or "luatex" + end + local ownbin=environment.ownbin or args[-2] or arg[-2] or args[-1] or arg[-1] or arg[0] or "luatex" + local ownpath=environment.ownpath or os.selfdir + ownbin=file.collapsepath(ownbin) + ownpath=file.collapsepath(ownpath) + if not ownpath or ownpath=="" or ownpath=="unset" then + ownpath=args[-1] or arg[-1] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + if not ownpath or ownpath=="" then + ownpath=args[-0] or arg[-0] + ownpath=ownpath and filedirname(gsub(ownpath,"\\","/")) + end + local binary=ownbin + if not ownpath or ownpath=="" then + ownpath=ownpath and filedirname(binary) + end + if not ownpath or ownpath=="" then + if os.binsuffix~="" then + binary=file.replacesuffix(binary,os.binsuffix) + end + local path=osgetenv("PATH") + if path then + for p in gmatch(path,"[^"..io.pathseparator.."]+") do + local b=filejoin(p,binary) + if lfs.isfile(b) then + local olddir=lfs.currentdir() + if lfs.chdir(p) then + local pp=lfs.currentdir() + if trace_locating and p~=pp then + report_initialization("following symlink %a to %a",p,pp) + end + ownpath=pp + lfs.chdir(olddir) + else + if trace_locating then + report_initialization("unable to check path %a",p) + end + ownpath=p + end + break + end + end + end + end + if not ownpath or ownpath=="" then + ownpath="." + report_initialization("forcing fallback to ownpath %a",ownpath) + elseif trace_locating then + report_initialization("using ownpath %a",ownpath) + end + end + environment.ownbin=ownbin + environment.ownpath=ownpath +end +resolvers.ownpath=environment.ownpath +function resolvers.getownpath() + return environment.ownpath +end +do + local ownpath=environment.ownpath or dir.current() + if ownpath then + ossetenv('SELFAUTOLOC',file.collapsepath(ownpath)) + ossetenv('SELFAUTODIR',file.collapsepath(ownpath.."/..")) + ossetenv('SELFAUTOPARENT',file.collapsepath(ownpath.."/../..")) + else + report_initialization("error: unable to locate ownpath") + os.exit() + end +end +local texos=environment.texos or osgetenv("TEXOS") +local texmfos=environment.texmfos or osgetenv('SELFAUTODIR') +if not texos or texos=="" then + texos=file.basename(texmfos) +end +ossetenv('TEXMFOS',texmfos) +ossetenv('TEXOS',texos) +ossetenv('SELFAUTOSYSTEM',os.platform) +environment.texos=texos +environment.texmfos=texmfos +local texroot=environment.texroot or osgetenv("TEXROOT") +if not texroot or texroot=="" then + texroot=osgetenv('SELFAUTOPARENT') + ossetenv('TEXROOT',texroot) +end +environment.texroot=file.collapsepath(texroot) +if type(profiler)=="table" and not jit then + directives.register("system.profile",function() + profiler.start("luatex-profile.log") + end) +end +local prefixes=utilities.storage.allocate() +resolvers.prefixes=prefixes +local resolved={} +local abstract={} +local dynamic={} +function resolvers.resetresolve(str) + resolved,abstract={},{} +end +function resolvers.allprefixes(separator) + local all=table.sortedkeys(prefixes) + if separator then + for i=1,#all do + all[i]=all[i]..":" + end + end + return all +end +local function _resolve_(method,target) + local action=prefixes[method] + if action then + return action(target) + else + return method..":"..target + end +end +function resolvers.unresolve(str) + return abstract[str] or str +end +function resolvers.setdynamic(str) + dynamic[str]=true +end +local pattern=Cs((C(R("az")^2)*P(":")*C((1-S(" \"\';,"))^1)/_resolve_+P(1))^0) +local prefix=C(R("az")^2)*P(":") +local target=C((1-S(" \"\';,"))^1) +local notarget=(#S(";,")+P(-1))*Cc("") +local p_resolve=Cs(((prefix*(target+notarget))/_resolve_+P(1))^0) +local p_simple=prefix*P(-1) +local function resolve(str) + if type(str)=="table" then + local res={} + for i=1,#str do + res[i]=resolve(str[i]) + end + return res + end + local res=resolved[str] + if res then + return res + end + local simple=lpegmatch(p_simple,str) + local action=prefixes[simple] + if action then + local res=action(res) + if not dynamic[simple] then + resolved[simple]=res + abstract[res]=simple + end + return res + end + res=lpegmatch(p_resolve,str) + resolved[str]=res + abstract[res]=str + return res +end +resolvers.resolve=resolve +if type(osuname)=="function" then + for k,v in next,osuname() do + if not prefixes[k] then + prefixes[k]=function() return v end + end + end +end +if ostype=="unix" then + local pattern + local function makepattern(t,k,v) + if t then + rawset(t,k,v) + end + local colon=P(":") + for k,v in table.sortedpairs(prefixes) do + if p then + p=P(k)+p + else + p=P(k) + end + end + pattern=Cs((p*colon+colon/";"+P(1))^0) + end + makepattern() + table.setmetatablenewindex(prefixes,makepattern) + function resolvers.repath(str) + return lpegmatch(pattern,str) + end +else + function resolvers.repath(str) + return str + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-exp"] = package.loaded["data-exp"] or true + +-- original size: 17216, stripped down to: 10657 + +if not modules then modules={} end modules ['data-exp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub +local concat,sort=table.concat,table.sort +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S +local type,next=type,next +local isdir=lfs.isdir +local ostype=os.type +local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_globbing=true trackers.register("resolvers.globbing",function(v) trace_globbing=v end) +local report_expansions=logs.reporter("resolvers","expansions") +local report_globbing=logs.reporter("resolvers","globbing") +local resolvers=resolvers +local resolveprefix=resolvers.resolve +local function f_both(a,b) + local t,n={},0 + for sb in gmatch(b,"[^,]+") do + for sa in gmatch(a,"[^,]+") do + n=n+1;t[n]=sa..sb + end + end + return concat(t,",") +end +local comma=P(",") +local nocomma=(1-comma)^1 +local docomma=comma^1/"," +local before=Cs((nocomma*Carg(1)+docomma)^0) +local after=Cs((Carg(1)*nocomma+docomma)^0) +local both=Cs(((C(nocomma)*Carg(1))/function(a,b) return lpegmatch(before,b,1,a) end+docomma)^0) +local function f_first (a,b) return lpegmatch(after,b,1,a) end +local function f_second(a,b) return lpegmatch(before,a,1,b) end +local function f_both (a,b) return lpegmatch(both,b,1,a) end +local left=P("{") +local right=P("}") +local var=P((1-S("{}" ))^0) +local set=P((1-S("{},"))^0) +local other=P(1) +local l_first=Cs((Cc("{")*(C(set)*left*C(var)*right/f_first)*Cc("}")+other )^0 ) +local l_second=Cs((Cc("{")*(left*C(var)*right*C(set)/f_second)*Cc("}")+other )^0 ) +local l_both=Cs((Cc("{")*(left*C(var)*right*left*C(var)*right/f_both)*Cc("}")+other )^0 ) +local l_rest=Cs((left*var*(left/"")*var*(right/"")*var*right+other )^0 ) +local stripper_1=lpeg.stripper ("{}@") +local replacer_1=lpeg.replacer { { ",}",",@}" },{ "{,","{@," },} +local function splitpathexpr(str,newlist,validate) + if trace_expansions then + report_expansions("expanding variable %a",str) + end + local t,ok,done=newlist or {},false,false + local n=#t + str=lpegmatch(replacer_1,str) + repeat + local old=str + repeat + local old=str + str=lpegmatch(l_first,str) + until old==str + repeat + local old=str + str=lpegmatch(l_second,str) + until old==str + repeat + local old=str + str=lpegmatch(l_both,str) + until old==str + repeat + local old=str + str=lpegmatch(l_rest,str) + until old==str + until old==str + str=lpegmatch(stripper_1,str) + if validate then + for s in gmatch(str,"[^,]+") do + s=validate(s) + if s then + n=n+1 + t[n]=s + end + end + else + for s in gmatch(str,"[^,]+") do + n=n+1 + t[n]=s + end + end + if trace_expansions then + for k=1,#t do + report_expansions("% 4i: %s",k,t[k]) + end + end + return t +end +local function validate(s) + s=collapsepath(s) + return s~="" and not find(s,"^!*unset/*$") and s +end +resolvers.validatedpath=validate +function resolvers.expandedpathfromlist(pathlist) + local newlist={} + for k=1,#pathlist do + splitpathexpr(pathlist[k],newlist,validate) + end + return newlist +end +local usedhomedir=nil +local donegation=(P("!")/"" )^0 +local doslashes=(P("\\")/"/"+1)^0 +local function expandedhome() + if not usedhomedir then + usedhomedir=lpegmatch(Cs(donegation*doslashes),environment.homedir or "") + if usedhomedir=="~" or usedhomedir=="" or not isdir(usedhomedir) then + if trace_expansions then + report_expansions("no home dir set, ignoring dependent path using current path") + end + usedhomedir="." + end + end + return usedhomedir +end +local dohome=((P("~")+P("$HOME")+P("%HOME%"))/expandedhome)^0 +local cleanup=Cs(donegation*dohome*doslashes) +resolvers.cleanpath=function(str) + return str and lpegmatch(cleanup,str) or "" +end +local expandhome=P("~")/"$HOME" +local dodouble=P('"')/""*(expandhome+(1-P('"')))^0*P('"')/"" +local dosingle=P("'")/""*(expandhome+(1-P("'")))^0*P("'")/"" +local dostring=(expandhome+1 )^0 +local stripper=Cs( + lpegpatterns.unspacer*(dosingle+dodouble+dostring)*lpegpatterns.unspacer +) +function resolvers.checkedvariable(str) + return type(str)=="string" and lpegmatch(stripper,str) or str +end +local cache={} +local splitter=lpeg.tsplitat(";") +local backslashswapper=lpeg.replacer("\\","/") +local function splitconfigurationpath(str) + if str then + local found=cache[str] + if not found then + if str=="" then + found={} + else + local split=lpegmatch(splitter,lpegmatch(backslashswapper,str)) + found={} + local noffound=0 + for i=1,#split do + local s=split[i] + if not find(s,"^{*unset}*") then + noffound=noffound+1 + found[noffound]=s + end + end + if trace_expansions then + report_expansions("splitting path specification %a",str) + for k=1,noffound do + report_expansions("% 4i: %s",k,found[k]) + end + end + cache[str]=found + end + end + return found + end +end +resolvers.splitconfigurationpath=splitconfigurationpath +function resolvers.splitpath(str) + if type(str)=='table' then + return str + else + return splitconfigurationpath(str) + end +end +function resolvers.joinpath(str) + if type(str)=='table' then + return joinpath(str) + else + return str + end +end +local attributes,directory=lfs.attributes,lfs.dir +local weird=P(".")^1+lpeg.anywhere(S("~`!#$%^&*()={}[]:;\"\'||<>,?\n\r\t")) +local lessweird=P(".")^1+lpeg.anywhere(S("~`#$%^&*:;\"\'||<>,?\n\r\t")) +local timer={} +local scanned={} +local nofscans=0 +local scancache={} +local fullcache={} +local nofsharedscans=0 +local function scan(files,remap,spec,path,n,m,r,onlyone,tolerant) + local full=path=="" and spec or (spec..path..'/') + local dirs={} + local nofdirs=0 + local pattern=tolerant and lessweird or weird + for name in directory(full) do + if not lpegmatch(pattern,name) then + local mode=attributes(full..name,"mode") + if mode=="file" then + n=n+1 + local lower=lower(name) + local paths=files[lower] + if paths then + if onlyone then + else + if type(paths)=="string" then + files[lower]={ paths,path } + else + paths[#paths+1]=path + end + if name~=lower then + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end + end + end + else + files[lower]=path + if name~=lower then + local rl=remap[lower] + if not rl then + remap[lower]=name + r=r+1 + elseif trace_globbing and rl~=name then + report_globbing("confusing filename, name: %a, lower: %a, already: %a",name,lower,rl) + end + end + end + elseif mode=="directory" then + m=m+1 + nofdirs=nofdirs+1 + if path~="" then + dirs[nofdirs]=path.."/"..name + else + dirs[nofdirs]=name + end + end + end + end + if nofdirs>0 then + sort(dirs) + for i=1,nofdirs do + files,remap,n,m,r=scan(files,remap,spec,dirs[i],n,m,r,onlyonce,tolerant) + end + end + scancache[sub(full,1,-2)]=files + return files,remap,n,m,r +end +function resolvers.scanfiles(path,branch,usecache,onlyonce,tolerant) + local realpath=resolveprefix(path) + if usecache then + local content=fullcache[realpath] + if content then + if trace_locating then + report_expansions("using cached scan of path %a, branch %a",path,branch or path) + end + nofsharedscans=nofsharedscans+1 + return content + end + end + statistics.starttiming(timer) + if trace_locating then + report_expansions("scanning path %a, branch %a",path,branch or path) + end + local content + if isdir(realpath) then + local files,remap,n,m,r=scan({},{},realpath..'/',"",0,0,0,onlyonce,tolerant) + content={ + metadata={ + path=path, + files=n, + directories=m, + remappings=r, + }, + files=files, + remap=remap, + } + if trace_locating then + report_expansions("%s files found on %s directories with %s uppercase remappings",n,m,r) + end + else + content={ + metadata={ + path=path, + files=0, + directories=0, + remappings=0, + }, + files={}, + remap={}, + } + if trace_locating then + report_expansions("invalid path %a",realpath) + end + end + if usecache then + scanned[#scanned+1]=realpath + fullcache[realpath]=content + end + nofscans=nofscans+1 + statistics.stoptiming(timer) + return content +end +function resolvers.simplescanfiles(path,branch,usecache) + return resolvers.scanfiles(path,branch,usecache,true,true) +end +function resolvers.scandata() + table.sort(scanned) + return { + n=nofscans, + shared=nofsharedscans, + time=statistics.elapsedtime(timer), + paths=scanned, + } +end +function resolvers.get_from_content(content,path,name) + if not content then + return + end + local files=content.files + if not files then + return + end + local remap=content.remap + if not remap then + return + end + if name then + local used=lower(name) + return path,remap[used] or used + else + local name=path + local used=lower(name) + local path=files[used] + if path then + return path,remap[used] or used + end + end +end +local nothing=function() end +function resolvers.filtered_from_content(content,pattern) + if content and type(pattern)=="string" then + local pattern=lower(pattern) + local files=content.files + local remap=content.remap + if files and remap then + local n=next(files) + local function iterator() + while n do + local k=n + n=next(files,k) + if find(k,pattern) then + return files[k],remap and remap[k] or k + end + end + end + return iterator + end + end + return nothing +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-env"] = package.loaded["data-env"] or true + +-- original size: 9216, stripped down to: 6798 + +if not modules then modules={} end modules ['data-env']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local lower,gsub=string.lower,string.gsub +local resolvers=resolvers +local allocate=utilities.storage.allocate +local setmetatableindex=table.setmetatableindex +local suffixonly=file.suffixonly +local formats=allocate() +local suffixes=allocate() +local dangerous=allocate() +local suffixmap=allocate() +local usertypes=allocate() +resolvers.formats=formats +resolvers.suffixes=suffixes +resolvers.dangerous=dangerous +resolvers.suffixmap=suffixmap +resolvers.usertypes=usertypes +local luasuffixes=utilities.lua.suffixes +local relations=allocate { + core={ + ofm={ + names={ "ofm","omega font metric","omega font metrics" }, + variable='OFMFONTS', + suffixes={ 'ofm','tfm' }, + }, + ovf={ + names={ "ovf","omega virtual font","omega virtual fonts" }, + variable='OVFFONTS', + suffixes={ 'ovf','vf' }, + }, + tfm={ + names={ "tfm","tex font metric","tex font metrics" }, + variable='TFMFONTS', + suffixes={ 'tfm' }, + }, + vf={ + names={ "vf","virtual font","virtual fonts" }, + variable='VFFONTS', + suffixes={ 'vf' }, + }, + otf={ + names={ "otf","opentype","opentype font","opentype fonts"}, + variable='OPENTYPEFONTS', + suffixes={ 'otf' }, + }, + ttf={ + names={ "ttf","truetype","truetype font","truetype fonts","truetype collection","truetype collections","truetype dictionary","truetype dictionaries" }, + variable='TTFONTS', + suffixes={ 'ttf','ttc','dfont' }, + }, + afm={ + names={ "afm","adobe font metric","adobe font metrics" }, + variable="AFMFONTS", + suffixes={ "afm" }, + }, + pfb={ + names={ "pfb","type1","type 1","type1 font","type 1 font","type1 fonts","type 1 fonts" }, + variable='T1FONTS', + suffixes={ 'pfb','pfa' }, + }, + fea={ + names={ "fea","font feature","font features","font feature file","font feature files" }, + variable='FONTFEATURES', + suffixes={ 'fea' }, + }, + cid={ + names={ "cid","cid map","cid maps","cid file","cid files" }, + variable='FONTCIDMAPS', + suffixes={ 'cid','cidmap' }, + }, + fmt={ + names={ "fmt","format","tex format" }, + variable='TEXFORMATS', + suffixes={ 'fmt' }, + }, + mem={ + names={ 'mem',"metapost format" }, + variable='MPMEMS', + suffixes={ 'mem' }, + }, + mp={ + names={ "mp" }, + variable='MPINPUTS', + suffixes={ 'mp','mpvi','mpiv','mpii' }, + usertype=true, + }, + tex={ + names={ "tex" }, + variable='TEXINPUTS', + suffixes={ "tex","mkvi","mkiv","mkii","cld","lfg","xml" }, + usertype=true, + }, + icc={ + names={ "icc","icc profile","icc profiles" }, + variable='ICCPROFILES', + suffixes={ 'icc' }, + }, + texmfscripts={ + names={ "texmfscript","texmfscripts","script","scripts" }, + variable='TEXMFSCRIPTS', + suffixes={ 'lua','rb','pl','py' }, + }, + lua={ + names={ "lua" }, + variable='LUAINPUTS', + suffixes={ luasuffixes.lua,luasuffixes.luc,luasuffixes.tma,luasuffixes.tmc }, + usertype=true, + }, + lib={ + names={ "lib" }, + variable='CLUAINPUTS', + suffixes=os.libsuffix and { os.libsuffix } or { 'dll','so' }, + }, + bib={ + names={ 'bib' }, + variable='BIBINPUTS', + suffixes={ 'bib' }, + usertype=true, + }, + bst={ + names={ 'bst' }, + variable='BSTINPUTS', + suffixes={ 'bst' }, + usertype=true, + }, + fontconfig={ + names={ 'fontconfig','fontconfig file','fontconfig files' }, + variable='FONTCONFIG_PATH', + }, + }, + obsolete={ + enc={ + names={ "enc","enc files","enc file","encoding files","encoding file" }, + variable='ENCFONTS', + suffixes={ 'enc' }, + }, + map={ + names={ "map","map files","map file" }, + variable='TEXFONTMAPS', + suffixes={ 'map' }, + }, + lig={ + names={ "lig files","lig file","ligature file","ligature files" }, + variable='LIGFONTS', + suffixes={ 'lig' }, + }, + opl={ + names={ "opl" }, + variable='OPLFONTS', + suffixes={ 'opl' }, + }, + ovp={ + names={ "ovp" }, + variable='OVPFONTS', + suffixes={ 'ovp' }, + }, + }, + kpse={ + base={ + names={ 'base',"metafont format" }, + variable='MFBASES', + suffixes={ 'base','bas' }, + }, + cmap={ + names={ 'cmap','cmap files','cmap file' }, + variable='CMAPFONTS', + suffixes={ 'cmap' }, + }, + cnf={ + names={ 'cnf' }, + suffixes={ 'cnf' }, + }, + web={ + names={ 'web' }, + suffixes={ 'web','ch' } + }, + cweb={ + names={ 'cweb' }, + suffixes={ 'w','web','ch' }, + }, + gf={ + names={ 'gf' }, + suffixes={ '<resolution>gf' }, + }, + mf={ + names={ 'mf' }, + variable='MFINPUTS', + suffixes={ 'mf' }, + }, + mft={ + names={ 'mft' }, + suffixes={ 'mft' }, + }, + pk={ + names={ 'pk' }, + suffixes={ '<resolution>pk' }, + }, + }, +} +resolvers.relations=relations +function resolvers.updaterelations() + for category,categories in next,relations do + for name,relation in next,categories do + local rn=relation.names + local rv=relation.variable + if rn and rv then + local rs=relation.suffixes + local ru=relation.usertype + for i=1,#rn do + local rni=lower(gsub(rn[i]," ","")) + formats[rni]=rv + if rs then + suffixes[rni]=rs + for i=1,#rs do + local rsi=rs[i] + suffixmap[rsi]=rni + end + end + end + if ru then + usertypes[name]=true + end + end + end + end +end +resolvers.updaterelations() +local function simplified(t,k) + return k and rawget(t,lower(gsub(k," ",""))) or nil +end +setmetatableindex(formats,simplified) +setmetatableindex(suffixes,simplified) +setmetatableindex(suffixmap,simplified) +function resolvers.suffixofformat(str) + local s=suffixes[str] + return s and s[1] or "" +end +function resolvers.suffixofformat(str) + return suffixes[str] or {} +end +for name,format in next,formats do + dangerous[name]=true +end +dangerous.tex=nil +function resolvers.formatofvariable(str) + return formats[str] or '' +end +function resolvers.formatofsuffix(str) + return suffixmap[suffixonly(str)] or 'tex' +end +function resolvers.variableofformat(str) + return formats[str] or '' +end +function resolvers.variableofformatorsuffix(str) + local v=formats[str] + if v then + return v + end + v=suffixmap[suffixonly(str)] + if v then + return formats[v] + end + return '' +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-tmp"] = package.loaded["data-tmp"] or true + +-- original size: 15618, stripped down to: 11629 + +if not modules then modules={} end modules ['data-tmp']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,concat=string.format,string.lower,string.gsub,table.concat +local concat=table.concat +local mkdirs,isdir,isfile=dir.mkdirs,lfs.isdir,lfs.isfile +local addsuffix,is_writable,is_readable=file.addsuffix,file.is_writable,file.is_readable +local formatters=string.formatters +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local report_caches=logs.reporter("resolvers","caches") +local report_resolvers=logs.reporter("resolvers","caching") +local resolvers=resolvers +local cleanpath=resolvers.cleanpath +local directive_cleanup=false directives.register("system.compile.cleanup",function(v) directive_cleanup=v end) +local directive_strip=false directives.register("system.compile.strip",function(v) directive_strip=v end) +local compile=utilities.lua.compile +function utilities.lua.compile(luafile,lucfile,cleanup,strip) + if cleanup==nil then cleanup=directive_cleanup end + if strip==nil then strip=directive_strip end + return compile(luafile,lucfile,cleanup,strip) +end +caches=caches or {} +local caches=caches +local luasuffixes=utilities.lua.suffixes +caches.base=caches.base or "luatex-cache" +caches.more=caches.more or "context" +caches.direct=false +caches.tree=false +caches.force=true +caches.ask=false +caches.relocate=false +caches.defaults={ "TMPDIR","TEMPDIR","TMP","TEMP","HOME","HOMEPATH" } +local writable,readables,usedreadables=nil,{},{} +local function identify() + local texmfcaches=resolvers.cleanpathlist("TEXMFCACHE") + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + cachepath=file.collapsepath(cachepath) + local valid=isdir(cachepath) + if valid then + if is_readable(cachepath) then + readables[#readables+1]=cachepath + if not writable and is_writable(cachepath) then + writable=cachepath + end + end + elseif not writable and caches.force then + local cacheparent=file.dirname(cachepath) + if is_writable(cacheparent) and true then + if not caches.ask or io.ask(format("\nShould I create the cache path %s?",cachepath),"no",{ "yes","no" })=="yes" then + mkdirs(cachepath) + if isdir(cachepath) and is_writable(cachepath) then + report_caches("path %a created",cachepath) + writable=cachepath + readables[#readables+1]=cachepath + end + end + end + end + end + end + end + local texmfcaches=caches.defaults + if texmfcaches then + for k=1,#texmfcaches do + local cachepath=texmfcaches[k] + cachepath=resolvers.expansion(cachepath) + if cachepath~="" then + cachepath=resolvers.resolve(cachepath) + cachepath=resolvers.cleanpath(cachepath) + local valid=isdir(cachepath) + if valid and is_readable(cachepath) then + if not writable and is_writable(cachepath) then + readables[#readables+1]=cachepath + writable=cachepath + break + end + end + end + end + end + if not writable then + report_caches("fatal error: there is no valid writable cache path defined") + os.exit() + elseif #readables==0 then + report_caches("fatal error: there is no valid readable cache path defined") + os.exit() + end + writable=dir.expandname(resolvers.cleanpath(writable)) + local base,more,tree=caches.base,caches.more,caches.tree or caches.treehash() + if tree then + caches.tree=tree + writable=mkdirs(writable,base,more,tree) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more,tree) + end + else + writable=mkdirs(writable,base,more) + for i=1,#readables do + readables[i]=file.join(readables[i],base,more) + end + end + if trace_cache then + for i=1,#readables do + report_caches("using readable path %a (order %s)",readables[i],i) + end + report_caches("using writable path %a",writable) + end + identify=function() + return writable,readables + end + return writable,readables +end +function caches.usedpaths(separator) + local writable,readables=identify() + if #readables>1 then + local result={} + local done={} + for i=1,#readables do + local readable=readables[i] + if readable==writable then + done[readable]=true + result[#result+1]=formatters["readable+writable: %a"](readable) + elseif usedreadables[i] then + done[readable]=true + result[#result+1]=formatters["readable: %a"](readable) + end + end + if not done[writable] then + result[#result+1]=formatters["writable: %a"](writable) + end + return concat(result,separator or " | ") + else + return writable or "?" + end +end +function caches.configfiles() + return concat(resolvers.instance.specification,";") +end +function caches.hashed(tree) + tree=gsub(tree,"[\\/]+$","") + tree=lower(tree) + local hash=md5.hex(tree) + if trace_cache or trace_locating then + report_caches("hashing tree %a, hash %a",tree,hash) + end + return hash +end +function caches.treehash() + local tree=caches.configfiles() + if not tree or tree=="" then + return false + else + return caches.hashed(tree) + end +end +local r_cache,w_cache={},{} +local function getreadablepaths(...) + local tags={... } + local hash=concat(tags,"/") + local done=r_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done={} + for i=1,#readables do + done[i]=file.join(readables[i],...) + end + else + done=readables + end + r_cache[hash]=done + end + return done +end +local function getwritablepath(...) + local tags={... } + local hash=concat(tags,"/") + local done=w_cache[hash] + if not done then + local writable,readables=identify() + if #tags>0 then + done=mkdirs(writable,...) + else + done=writable + end + w_cache[hash]=done + end + return done +end +caches.getreadablepaths=getreadablepaths +caches.getwritablepath=getwritablepath +function caches.getfirstreadablefile(filename,...) + local rd=getreadablepaths(...) + for i=1,#rd do + local path=rd[i] + local fullname=file.join(path,filename) + if is_readable(fullname) then + usedreadables[i]=true + return fullname,path + end + end + return caches.setfirstwritablefile(filename,...) +end +function caches.getfirstreadablefile_TEST_ME_FIRST(filename,...) + local fullname,path=caches.setfirstwritablefile(filename,...) + if is_readable(fullname) then + return fullname,path + end + local rd=getreadablepaths(...) + for i=1,#rd do + local path=rd[i] + local fullname=file.join(path,filename) + if is_readable(fullname) then + usedreadables[i]=true + return fullname,path + end + end + return fullname,path +end +function caches.setfirstwritablefile(filename,...) + local wr=getwritablepath(...) + local fullname=file.join(wr,filename) + return fullname,wr +end +function caches.define(category,subcategory) + return function() + return getwritablepath(category,subcategory) + end +end +function caches.setluanames(path,name) + return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) +end +function caches.loaddata(readables,name) + if type(readables)=="string" then + readables={ readables } + end + for i=1,#readables do + local path=readables[i] + local tmaname,tmcname=caches.setluanames(path,name) + local loader=false + if isfile(tmcname) then + loader=loadfile(tmcname) + end + if not loader and isfile(tmaname) then + utilities.lua.compile(tmaname,tmcname) + if isfile(tmcname) then + loader=loadfile(tmcname) + end + if not loader then + loader=loadfile(tmaname) + end + end + if loader then + loader=loader() + collectgarbage("step") + return loader + end + end + return false +end +function caches.is_writable(filepath,filename) + local tmaname,tmcname=caches.setluanames(filepath,filename) + return is_writable(tmaname) +end +local saveoptions={ compact=true } +function caches.savedata(filepath,filename,data,raw) + local tmaname,tmcname=caches.setluanames(filepath,filename) + data.cache_uuid=os.uuid() + if caches.direct then + file.savedata(tmaname,table.serialize(data,true,saveoptions)) + else + table.tofile(tmaname,data,true,saveoptions) + end + utilities.lua.compile(tmaname,tmcname) +end +local content_state={} +function caches.contentstate() + return content_state or {} +end +function caches.loadcontent(cachename,dataname,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.getfirstreadablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end + local blob=loadfile(addsuffix(filename,luasuffixes.luc)) or loadfile(addsuffix(filename,luasuffixes.lua)) + if blob then + local data=blob() + if data and data.content then + if data.type==dataname then + if data.version==resolvers.cacheversion then + content_state[#content_state+1]=data.uuid + if trace_locating then + report_resolvers("loading %a for %a from %a",dataname,cachename,filename) + end + return data.content + else + report_resolvers("skipping %a for %a from %a (version mismatch)",dataname,cachename,filename) + end + else + report_resolvers("skipping %a for %a from %a (datatype mismatch)",dataname,cachename,filename) + end + elseif trace_locating then + report_resolvers("skipping %a for %a from %a (no content)",dataname,cachename,filename) + end + elseif trace_locating then + report_resolvers("skipping %a for %a from %a (invalid file)",dataname,cachename,filename) + end +end +function caches.collapsecontent(content) + for k,v in next,content do + if type(v)=="table" and #v==1 then + content[k]=v[1] + end + end +end +function caches.savecontent(cachename,dataname,content,filename) + if not filename then + local name=caches.hashed(cachename) + local full,path=caches.setfirstwritablefile(addsuffix(name,luasuffixes.lua),"trees") + filename=file.join(path,name) + end + local luaname=addsuffix(filename,luasuffixes.lua) + local lucname=addsuffix(filename,luasuffixes.luc) + if trace_locating then + report_resolvers("preparing %a for %a",dataname,cachename) + end + local data={ + type=dataname, + root=cachename, + version=resolvers.cacheversion, + date=os.date("%Y-%m-%d"), + time=os.date("%H:%M:%S"), + content=content, + uuid=os.uuid(), + } + local ok=io.savedata(luaname,table.serialize(data,true)) + if ok then + if trace_locating then + report_resolvers("category %a, cachename %a saved in %a",dataname,cachename,luaname) + end + if utilities.lua.compile(luaname,lucname) then + if trace_locating then + report_resolvers("%a compiled to %a",dataname,lucname) + end + return true + else + if trace_locating then + report_resolvers("compiling failed for %a, deleting file %a",dataname,lucname) + end + os.remove(lucname) + end + elseif trace_locating then + report_resolvers("unable to save %a in %a (access error)",dataname,luaname) + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-met"] = package.loaded["data-met"] or true + +-- original size: 5347, stripped down to: 4015 + +if not modules then modules={} end modules ['data-met']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,format=string.find,string.format +local sequenced=table.sequenced +local addurlscheme,urlhashed=url.addscheme,url.hashed +local getcurrentdir=lfs.currentdir +local trace_locating=false +local trace_methods=false +trackers.register("resolvers.locating",function(v) trace_methods=v end) +trackers.register("resolvers.methods",function(v) trace_methods=v end) +local report_methods=logs.reporter("resolvers","methods") +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registered={} +local function splitmethod(filename) + if not filename then + return { scheme="unknown",original=filename } + end + if type(filename)=="table" then + return filename + end + filename=file.collapsepath(filename,".") + if not find(filename,"://",1,true) then + return { scheme="file",path=filename,original=filename,filename=filename } + end + local specification=url.hashed(filename) + if not specification.scheme or specification.scheme=="" then + return { scheme="file",path=filename,original=filename,filename=filename } + else + return specification + end +end +resolvers.splitmethod=splitmethod +local function methodhandler(what,first,...) + local method=registered[what] + if method then + local how,namespace=method.how,method.namespace + if how=="uri" or how=="url" then + local specification=splitmethod(first) + local scheme=specification.scheme + local resolver=namespace and namespace[scheme] + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,scheme,first) + end + return resolver(specification,...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,"default",first) + end + return resolver(specification,...) + elseif trace_methods then + report_methods("resolving, method %a, how %a, handler %a, argument %a",what,how,"unset") + end + end + elseif how=="tag" then + local resolver=namespace and namespace[first] + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, tag %a",what,how,first) + end + return resolver(...) + else + resolver=namespace.default or namespace.file + if resolver then + if trace_methods then + report_methods("resolving, method %a, how %a, tag %a",what,how,"default") + end + return resolver(...) + elseif trace_methods then + report_methods("resolving, method %a, how %a, tag %a",what,how,"unset") + end + end + end + else + report_methods("resolving, invalid method %a") + end +end +resolvers.methodhandler=methodhandler +function resolvers.registermethod(name,namespace,how) + registered[name]={ how=how or "tag",namespace=namespace } + namespace["byscheme"]=function(scheme,filename,...) + if scheme=="file" then + return methodhandler(name,filename,...) + else + return methodhandler(name,addurlscheme(filename,scheme),...) + end + end +end +local concatinators=allocate { notfound=file.join } +local locators=allocate { notfound=function() end } +local hashers=allocate { notfound=function() end } +local generators=allocate { notfound=function() end } +resolvers.concatinators=concatinators +resolvers.locators=locators +resolvers.hashers=hashers +resolvers.generators=generators +local registermethod=resolvers.registermethod +registermethod("concatinators",concatinators,"tag") +registermethod("locators",locators,"uri") +registermethod("hashers",hashers,"uri") +registermethod("generators",generators,"uri") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-res"] = package.loaded["data-res"] or true + +-- original size: 67003, stripped down to: 46291 + +if not modules then modules={} end modules ['data-res']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find,lower,upper,match,gmatch=string.gsub,string.find,string.lower,string.upper,string.match,string.gmatch +local concat,insert,remove,sortedkeys,sortedhash=table.concat,table.insert,table.remove,table.sortedkeys,table.sortedhash +local next,type,rawget=next,type,rawget +local os=os +local P,S,R,C,Cc,Cs,Ct,Carg=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.Cc,lpeg.Cs,lpeg.Ct,lpeg.Carg +local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns +local formatters=string.formatters +local filedirname=file.dirname +local filebasename=file.basename +local suffixonly=file.suffixonly +local addsuffix=file.addsuffix +local removesuffix=file.removesuffix +local filejoin=file.join +local collapsepath=file.collapsepath +local joinpath=file.joinpath +local is_qualified_path=file.is_qualified_path +local allocate=utilities.storage.allocate +local settings_to_array=utilities.parsers.settings_to_array +local getcurrentdir=lfs.currentdir +local isfile=lfs.isfile +local isdir=lfs.isdir +local setmetatableindex=table.setmetatableindex +local luasuffixes=utilities.lua.suffixes +local trace_locating=false trackers .register("resolvers.locating",function(v) trace_locating=v end) +local trace_detail=false trackers .register("resolvers.details",function(v) trace_detail=v end) +local trace_expansions=false trackers .register("resolvers.expansions",function(v) trace_expansions=v end) +local trace_paths=false trackers .register("resolvers.paths",function(v) trace_paths=v end) +local resolve_otherwise=true directives.register("resolvers.otherwise",function(v) resolve_otherwise=v end) +local report_resolving=logs.reporter("resolvers","resolving") +local resolvers=resolvers +local expandedpathfromlist=resolvers.expandedpathfromlist +local checkedvariable=resolvers.checkedvariable +local splitconfigurationpath=resolvers.splitconfigurationpath +local methodhandler=resolvers.methodhandler +local filtered=resolvers.filtered_from_content +local lookup=resolvers.get_from_content +local cleanpath=resolvers.cleanpath +local resolveprefix=resolvers.resolve +local initializesetter=utilities.setters.initialize +local ostype,osname,osenv,ossetenv,osgetenv=os.type,os.name,os.env,os.setenv,os.getenv +resolvers.cacheversion="1.100" +resolvers.configbanner="" +resolvers.homedir=environment.homedir +resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } +resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnfstate="unknown" +if environment.default_texmfcnf then + resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf +else + resolvers.luacnfspec=concat ({ + "home:texmf/web2c", + "selfautoparent:/texmf-local/web2c", + "selfautoparent:/texmf-context/web2c", + "selfautoparent:/texmf-dist/web2c", + "selfautoparent:/texmf/web2c", + },";") +end +local unset_variable="unset" +local formats=resolvers.formats +local suffixes=resolvers.suffixes +local usertypes=resolvers.usertypes +local dangerous=resolvers.dangerous +local suffixmap=resolvers.suffixmap +resolvers.defaultsuffixes={ "tex" } +resolvers.instance=resolvers.instance or nil +local instance=resolvers.instance or nil +function resolvers.setenv(key,value,raw) + if instance then + instance.environment[key]=value + ossetenv(key,raw and value or resolveprefix(value)) + end +end +local function getenv(key) + local value=rawget(instance.environment,key) + if value and value~="" then + return value + else + local e=osgetenv(key) + return e~=nil and e~="" and checkedvariable(e) or "" + end +end +resolvers.getenv=getenv +resolvers.env=getenv +local function resolvevariable(k) + return instance.expansions[k] +end +local dollarstripper=lpeg.stripper("$") +local inhibitstripper=P("!")^0*Cs(P(1)^0) +local backslashswapper=lpeg.replacer("\\","/") +local somevariable=P("$")/"" +local somekey=C(R("az","AZ","09","__","--")^1) +local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) +local variableexpander=Cs((somevariable*(somekey/resolvevariable)+somethingelse)^1 ) +local cleaner=P("\\")/"/"+P(";")*S("!{}/\\")^0*P(";")^1/";" +local variablecleaner=Cs((cleaner+P(1))^0) +local somevariable=R("az","AZ","09","__","--")^1/resolvevariable +local variable=(P("$")/"")*(somevariable+(P("{")/"")*somevariable*(P("}")/"")) +local variableresolver=Cs((variable+P(1))^0) +local function expandedvariable(var) + return lpegmatch(variableexpander,var) or var +end +function resolvers.newinstance() + if trace_locating then + report_resolving("creating instance") + end + local environment,variables,expansions,order=allocate(),allocate(),allocate(),allocate() + local newinstance={ + environment=environment, + variables=variables, + expansions=expansions, + order=order, + files=allocate(), + setups=allocate(), + found=allocate(), + foundintrees=allocate(), + hashes=allocate(), + hashed=allocate(), + pathlists=false, + specification=allocate(), + lists=allocate(), + data=allocate(), + fakepaths=allocate(), + remember=true, + diskcache=true, + renewcache=false, + renewtree=false, + loaderror=false, + savelists=true, + pattern=nil, + force_suffixes=true, + pathstack={}, + } + setmetatableindex(variables,function(t,k) + local v + for i=1,#order do + v=order[i][k] + if v~=nil then + t[k]=v + return v + end + end + if v==nil then + v="" + end + t[k]=v + return v + end) + setmetatableindex(environment,function(t,k) + local v=osgetenv(k) + if v==nil then + v=variables[k] + end + if v~=nil then + v=checkedvariable(v) or "" + end + v=resolvers.repath(v) + t[k]=v + return v + end) + setmetatableindex(expansions,function(t,k) + local v=environment[k] + if type(v)=="string" then + v=lpegmatch(variableresolver,v) + v=lpegmatch(variablecleaner,v) + end + t[k]=v + return v + end) + return newinstance +end +function resolvers.setinstance(someinstance) + instance=someinstance + resolvers.instance=someinstance + return someinstance +end +function resolvers.reset() + return resolvers.setinstance(resolvers.newinstance()) +end +local function reset_hashes() + instance.lists={} + instance.pathlists=false + instance.found={} +end +local function reset_caches() + instance.lists={} + instance.pathlists=false +end +local slash=P("/") +local pathexpressionpattern=Cs ( + Cc("^")*( + Cc("%")*S(".-")+slash^2*P(-1)/"/.*" ++slash^2/"/"+(1-slash)*P(-1)*Cc("/")+P(1) + )^1*Cc("$") +) +local cache={} +local function makepathexpression(str) + if str=="." then + return "^%./$" + else + local c=cache[str] + if not c then + c=lpegmatch(pathexpressionpattern,str) + cache[str]=c + end + return c + end +end +local function reportcriticalvariables(cnfspec) + if trace_locating then + for i=1,#resolvers.criticalvars do + local k=resolvers.criticalvars[i] + local v=resolvers.getenv(k) or "unknown" + report_resolving("variable %a set to %a",k,v) + end + report_resolving() + if cnfspec then + report_resolving("using configuration specification %a",type(cnfspec)=="table" and concat(cnfspec,",") or cnfspec) + end + report_resolving() + end + reportcriticalvariables=function() end +end +local function identify_configuration_files() + local specification=instance.specification + if #specification==0 then + local cnfspec=getenv("TEXMFCNF") + if cnfspec=="" then + cnfspec=resolvers.luacnfspec + resolvers.luacnfstate="default" + else + resolvers.luacnfstate="environment" + end + reportcriticalvariables(cnfspec) + local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) + local luacnfname=resolvers.luacnfname + for i=1,#cnfpaths do + local filepath=cnfpaths[i] + local filename=collapsepath(filejoin(filepath,luacnfname)) + local realname=resolveprefix(filename) + if trace_locating then + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) + report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) + end + if isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found configuration file %a",realname) + end + end + end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("configuration files already identified") + end +end +local function load_configuration_files() + local specification=instance.specification + if #specification>0 then + local luacnfname=resolvers.luacnfname + for i=1,#specification do + local filename=specification[i] + local pathname=filedirname(filename) + local filename=filejoin(pathname,luacnfname) + local realname=resolveprefix(filename) + local blob=loadfile(realname) + if blob then + local setups=instance.setups + local data=blob() + local parent=data and data.parent + if parent then + local filename=filejoin(pathname,parent) + local realname=resolveprefix(filename) + local blob=loadfile(realname) + if blob then + local parentdata=blob() + if parentdata then + report_resolving("loading configuration file %a",filename) + data=table.merged(parentdata,data) + end + end + end + data=data and data.content + if data then + if trace_locating then + report_resolving("loading configuration file %a",filename) + report_resolving() + end + local variables=data.variables or {} + local warning=false + for k,v in next,data do + local variant=type(v) + if variant=="table" then + initializesetter(filename,k,v) + elseif variables[k]==nil then + if trace_locating and not warning then + report_resolving("variables like %a in configuration file %a should move to the 'variables' subtable", + k,resolveprefix(filename)) + warning=true + end + variables[k]=v + end + end + setups[pathname]=variables + if resolvers.luacnfstate=="default" then + local cnfspec=variables["TEXMFCNF"] + if cnfspec then + if trace_locating then + report_resolving("reloading configuration due to TEXMF redefinition") + end + resolvers.setenv("TEXMFCNF",cnfspec) + instance.specification={} + identify_configuration_files() + load_configuration_files() + resolvers.luacnfstate="configuration" + break + end + end + else + if trace_locating then + report_resolving("skipping configuration file %a (no content)",filename) + end + setups[pathname]={} + instance.loaderror=true + end + elseif trace_locating then + report_resolving("skipping configuration file %a (no valid format)",filename) + end + instance.order[#instance.order+1]=instance.setups[pathname] + if instance.loaderror then + break + end + end + elseif trace_locating then + report_resolving("warning: no lua configuration files found") + end +end +local function load_file_databases() + instance.loaderror,instance.files=false,allocate() + if not instance.renewcache then + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + resolvers.hashers.byscheme(hash.type,hash.name) + if instance.loaderror then break end + end + end +end +local function locate_file_databases() + local texmfpaths=resolvers.expandedpathlist("TEXMF") + if #texmfpaths>0 then + for i=1,#texmfpaths do + local path=collapsepath(texmfpaths[i]) + path=gsub(path,"/+$","") + local stripped=lpegmatch(inhibitstripper,path) + if stripped~="" then + local runtime=stripped==path + path=cleanpath(path) + local spec=resolvers.splitmethod(stripped) + if runtime and (spec.noscheme or spec.scheme=="file") then + stripped="tree:///"..stripped + elseif spec.scheme=="cache" or spec.scheme=="file" then + stripped=spec.path + end + if trace_locating then + if runtime then + report_resolving("locating list of %a (runtime) (%s)",path,stripped) + else + report_resolving("locating list of %a (cached)",path) + end + end + methodhandler('locators',stripped) + end + end + if trace_locating then + report_resolving() + end + elseif trace_locating then + report_resolving("no texmf paths are defined (using TEXMF)") + end +end +local function generate_file_databases() + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + methodhandler('generators',hash.name) + end + if trace_locating then + report_resolving() + end +end +local function save_file_databases() + for i=1,#instance.hashes do + local hash=instance.hashes[i] + local cachename=hash.name + if hash.cache then + local content=instance.files[cachename] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree %a",cachename) + end + caches.savecontent(cachename,"files",content) + elseif trace_locating then + report_resolving("not saving runtime tree %a",cachename) + end + end +end +function resolvers.renew(hashname) + if hashname and hashname~="" then + local expanded=resolvers.expansion(hashname) or "" + if expanded~="" then + if trace_locating then + report_resolving("identifying tree %a from %a",expanded,hashname) + end + hashname=expanded + else + if trace_locating then + report_resolving("identifying tree %a",hashname) + end + end + local realpath=resolveprefix(hashname) + if isdir(realpath) then + if trace_locating then + report_resolving("using path %a",realpath) + end + methodhandler('generators',hashname) + local content=instance.files[hashname] + caches.collapsecontent(content) + if trace_locating then + report_resolving("saving tree %a",hashname) + end + caches.savecontent(hashname,"files",content) + else + report_resolving("invalid path %a",realpath) + end + end +end +local function load_databases() + locate_file_databases() + if instance.diskcache and not instance.renewcache then + load_file_databases() + if instance.loaderror then + generate_file_databases() + save_file_databases() + end + else + generate_file_databases() + if instance.renewcache then + save_file_databases() + end + end +end +function resolvers.appendhash(type,name,cache) + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash %a appended",name) + end + insert(instance.hashes,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end +end +function resolvers.prependhash(type,name,cache) + if not instance.hashed[name] then + if trace_locating then + report_resolving("hash %a prepended",name) + end + insert(instance.hashes,1,{ type=type,name=name,cache=cache } ) + instance.hashed[name]=cache + end +end +function resolvers.extendtexmfvariable(specification) + local t=resolvers.splitpath(getenv("TEXMF")) + insert(t,1,specification) + local newspec=concat(t,",") + if instance.environment["TEXMF"] then + instance.environment["TEXMF"]=newspec + elseif instance.variables["TEXMF"] then + instance.variables["TEXMF"]=newspec + else + end + reset_hashes() +end +function resolvers.splitexpansions() + local ie=instance.expansions + for k,v in next,ie do + local t,tn,h,p={},0,{},splitconfigurationpath(v) + for kk=1,#p do + local vv=p[kk] + if vv~="" and not h[vv] then + tn=tn+1 + t[tn]=vv + h[vv]=true + end + end + if #t>1 then + ie[k]=t + else + ie[k]=t[1] + end + end +end +function resolvers.datastate() + return caches.contentstate() +end +function resolvers.variable(name) + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.variables[name] + return result~=nil and result or "" +end +function resolvers.expansion(name) + local name=name and lpegmatch(dollarstripper,name) + local result=name and instance.expansions[name] + return result~=nil and result or "" +end +function resolvers.unexpandedpathlist(str) + local pth=resolvers.variable(str) + local lst=resolvers.splitpath(pth) + return expandedpathfromlist(lst) +end +function resolvers.unexpandedpath(str) + return joinpath(resolvers.unexpandedpathlist(str)) +end +function resolvers.pushpath(name) + local pathstack=instance.pathstack + local lastpath=pathstack[#pathstack] + local pluspath=filedirname(name) + if lastpath then + lastpath=collapsepath(filejoin(lastpath,pluspath)) + else + lastpath=collapsepath(pluspath) + end + insert(pathstack,lastpath) + if trace_paths then + report_resolving("pushing path %a",lastpath) + end +end +function resolvers.poppath() + local pathstack=instance.pathstack + if trace_paths and #pathstack>0 then + report_resolving("popping path %a",pathstack[#pathstack]) + end + remove(pathstack) +end +function resolvers.stackpath() + local pathstack=instance.pathstack + local currentpath=pathstack[#pathstack] + return currentpath~="" and currentpath or nil +end +local done={} +function resolvers.resetextrapath() + local ep=instance.extra_paths + if not ep then + done={} + instance.extra_paths={} + elseif #ep>0 then + done={} + reset_caches() + end +end +function resolvers.registerextrapath(paths,subpaths) + if not subpaths or subpaths=="" then + if not paths or path=="" then + return + elseif done[paths] then + return + end + end + local paths=settings_to_array(paths) + local subpaths=settings_to_array(subpaths) + local ep=instance.extra_paths or {} + local oldn=#ep + local newn=oldn + local nofpaths=#paths + local nofsubpaths=#subpaths + if nofpaths>0 then + if nofsubpaths>0 then + for i=1,nofpaths do + local p=paths[i] + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=p.."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=cleanpath(ps) + done[ps]=true + end + end + end + else + for i=1,nofpaths do + local p=paths[i] + if not done[p] then + newn=newn+1 + ep[newn]=cleanpath(p) + done[p]=true + end + end + end + elseif nofsubpaths>0 then + for i=1,oldn do + for j=1,nofsubpaths do + local s=subpaths[j] + local ps=ep[i].."/"..s + if not done[ps] then + newn=newn+1 + ep[newn]=cleanpath(ps) + done[ps]=true + end + end + end + end + if newn>0 then + instance.extra_paths=ep + end + if newn~=oldn then + reset_caches() + end +end +function resolvers.pushextrapath(path) + local paths=settings_to_array(path) + if instance.extra_stack then + insert(instance.extra_stack,1,paths) + else + instance.extra_stack={ paths } + end + reset_caches() +end +function resolvers.popextrapath() + if instance.extra_stack then + reset_caches() + return remove(instance.extra_stack,1) + end +end +local function made_list(instance,list,extra_too) + local done={} + local new={} + local newn=0 + local function add(p) + for k=1,#p do + local v=p[k] + if not done[v] then + done[v]=true + newn=newn+1 + new[newn]=v + end + end + end + for k=1,#list do + local v=list[k] + if done[v] then + elseif find(v,"^[%.%/]$") then + done[v]=true + newn=newn+1 + new[newn]=v + else + break + end + end + if extra_too then + local es=instance.extra_stack + if es and #es>0 then + for k=1,#es do + add(es[k]) + end + end + local ep=instance.extra_paths + if ep and #ep>0 then + add(ep) + end + end + add(list) + return new +end +function resolvers.cleanpathlist(str) + local t=resolvers.expandedpathlist(str) + if t then + for i=1,#t do + t[i]=collapsepath(cleanpath(t[i])) + end + end + return t +end +function resolvers.expandpath(str) + return joinpath(resolvers.expandedpathlist(str)) +end +function resolvers.expandedpathlist(str,extra_too) + if not str then + return {} + elseif instance.savelists then + str=lpegmatch(dollarstripper,str) + local lists=instance.lists + local lst=lists[str] + if not lst then + local l=made_list(instance,resolvers.splitpath(resolvers.expansion(str)),extra_too) + lst=expandedpathfromlist(l) + lists[str]=lst + end + return lst + else + local lst=resolvers.splitpath(resolvers.expansion(str)) + return made_list(instance,expandedpathfromlist(lst),extra_too) + end +end +function resolvers.expandedpathlistfromvariable(str) + str=lpegmatch(dollarstripper,str) + local tmp=resolvers.variableofformatorsuffix(str) + return resolvers.expandedpathlist(tmp~="" and tmp or str) +end +function resolvers.expandpathfromvariable(str) + return joinpath(resolvers.expandedpathlistfromvariable(str)) +end +function resolvers.cleanedpathlist(v) + local t=resolvers.expandedpathlist(v) + for i=1,#t do + t[i]=resolvers.resolve(resolvers.cleanpath(t[i])) + end + return t +end +function resolvers.expandbraces(str) + local ori=str + local pth=expandedpathfromlist(resolvers.splitpath(ori)) + return joinpath(pth) +end +function resolvers.registerfilehash(name,content,someerror) + if content then + instance.files[name]=content + else + instance.files[name]={} + if somerror==true then + instance.loaderror=someerror + end + end +end +local function isreadable(name) + local readable=isfile(name) + if trace_detail then + if readable then + report_resolving("file %a is readable",name) + else + report_resolving("file %a is not readable",name) + end + end + return readable +end +local function collect_files(names) + local filelist={} + local noffiles=0 + local function check(hash,root,pathname,path,name) + if not pathname or find(path,pathname) then + local variant=hash.type + local search=filejoin(root,path,name) + local result=methodhandler('concatinators',variant,root,path,name) + if trace_detail then + report_resolving("match: variant %a, search %a, result %a",variant,search,result) + end + noffiles=noffiles+1 + filelist[noffiles]={ variant,search,result } + end + end + for k=1,#names do + local filename=names[k] + if trace_detail then + report_resolving("checking name %a",filename) + end + local basename=filebasename(filename) + local pathname=filedirname(filename) + if pathname=="" or find(pathname,"^%.") then + pathname=false + else + pathname=gsub(pathname,"%*",".*") + pathname="/"..pathname.."$" + end + local hashes=instance.hashes + for h=1,#hashes do + local hash=hashes[h] + local hashname=hash.name + local content=hashname and instance.files[hashname] + if content then + if trace_detail then + report_resolving("deep checking %a, base %a, pattern %a",hashname,basename,pathname) + end + local path,name=lookup(content,basename) + if path then + local metadata=content.metadata + local realroot=metadata and metadata.path or hashname + if type(path)=="string" then + check(hash,realroot,pathname,path,name) + else + for i=1,#path do + check(hash,realroot,pathname,path[i],name) + end + end + end + elseif trace_locating then + report_resolving("no match in %a (%s)",hashname,basename) + end + end + end + return noffiles>0 and filelist or nil +end +local fit={} +function resolvers.registerintrees(filename,format,filetype,usedmethod,foundname) + local foundintrees=instance.foundintrees + if usedmethod=="direct" and filename==foundname and fit[foundname] then + else + local t={ + filename=filename, + format=format~="" and format or nil, + filetype=filetype~="" and filetype or nil, + usedmethod=usedmethod, + foundname=foundname, + } + fit[foundname]=t + foundintrees[#foundintrees+1]=t + end +end +local function can_be_dir(name) + local fakepaths=instance.fakepaths + if not fakepaths[name] then + if isdir(name) then + fakepaths[name]=1 + else + fakepaths[name]=2 + end + end + return fakepaths[name]==1 +end +local preparetreepattern=Cs((P(".")/"%%."+P("-")/"%%-"+P(1))^0*Cc("$")) +local collect_instance_files +local function find_analyze(filename,askedformat,allresults) + local filetype,wantedfiles,ext='',{},suffixonly(filename) + wantedfiles[#wantedfiles+1]=filename + if askedformat=="" then + if ext=="" or not suffixmap[ext] then + local defaultsuffixes=resolvers.defaultsuffixes + local formatofsuffix=resolvers.formatofsuffix + for i=1,#defaultsuffixes do + local forcedname=filename..'.'..defaultsuffixes[i] + wantedfiles[#wantedfiles+1]=forcedname + filetype=formatofsuffix(forcedname) + if trace_locating then + report_resolving("forcing filetype %a",filetype) + end + end + else + filetype=resolvers.formatofsuffix(filename) + if trace_locating then + report_resolving("using suffix based filetype %a",filetype) + end + end + else + if ext=="" or not suffixmap[ext] then + local format_suffixes=suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + wantedfiles[#wantedfiles+1]=filename.."."..format_suffixes[i] + end + end + end + filetype=askedformat + if trace_locating then + report_resolving("using given filetype %a",filetype) + end + end + return filetype,wantedfiles +end +local function find_direct(filename,allresults) + if not dangerous[askedformat] and isreadable(filename) then + if trace_detail then + report_resolving("file %a found directly",filename) + end + return "direct",{ filename } + end +end +local function find_wildcard(filename,allresults) + if find(filename,'*',1,true) then + if trace_locating then + report_resolving("checking wildcard %a",filename) + end + local result=resolvers.findwildcardfiles(filename) + if result then + return "wildcard",result + end + end +end +local function find_qualified(filename,allresults,askedformat,alsostripped) + if not is_qualified_path(filename) then + return + end + if trace_locating then + report_resolving("checking qualified name %a",filename) + end + if isreadable(filename) then + if trace_detail then + report_resolving("qualified file %a found",filename) + end + return "qualified",{ filename } + end + if trace_detail then + report_resolving("locating qualified file %a",filename) + end + local forcedname,suffix="",suffixonly(filename) + if suffix=="" then + local format_suffixes=askedformat=="" and resolvers.defaultsuffixes or suffixes[askedformat] + if format_suffixes then + for i=1,#format_suffixes do + local s=format_suffixes[i] + forcedname=filename.."."..s + if isreadable(forcedname) then + if trace_locating then + report_resolving("no suffix, forcing format filetype %a",s) + end + return "qualified",{ forcedname } + end + end + end + end + if alsostripped and suffix and suffix~="" then + local basename=filebasename(filename) + local pattern=lpegmatch(preparetreepattern,filename) + local savedformat=askedformat + local format=savedformat or "" + if format=="" then + askedformat=resolvers.formatofsuffix(suffix) + end + if not format then + askedformat="othertextfiles" + end + if basename~=filename then + local resolved=collect_instance_files(basename,askedformat,allresults) + if #resolved==0 then + local lowered=lower(basename) + if filename~=lowered then + resolved=collect_instance_files(lowered,askedformat,allresults) + end + end + resolvers.format=savedformat + if #resolved>0 then + local result={} + for r=1,#resolved do + local rr=resolved[r] + if find(rr,pattern) then + result[#result+1]=rr + end + end + if #result>0 then + return "qualified",result + end + end + end + end +end +local function check_subpath(fname) + if isreadable(fname) then + if trace_detail then + report_resolving("found %a by deep scanning",fname) + end + return fname + end +end +local function makepathlist(list,filetype) + local typespec=resolvers.variableofformat(filetype) + local pathlist=resolvers.expandedpathlist(typespec,filetype and usertypes[filetype]) + local entry={} + if pathlist and #pathlist>0 then + for k=1,#pathlist do + local path=pathlist[k] + local prescanned=find(path,'^!!') + local resursive=find(path,'//$') + local pathname=lpegmatch(inhibitstripper,path) + local expression=makepathexpression(pathname) + local barename=gsub(pathname,"/+$","") + barename=resolveprefix(barename) + local scheme=url.hasscheme(barename) + local schemename=gsub(barename,"%.%*$",'') + entry[k]={ + path=path, + pathname=pathname, + prescanned=prescanned, + recursive=recursive, + expression=expression, + barename=barename, + scheme=scheme, + schemename=schemename, + } + end + entry.typespec=typespec + list[filetype]=entry + else + list[filetype]=false + end + return entry +end +local function find_intree(filename,filetype,wantedfiles,allresults) + local pathlists=instance.pathlists + if not pathlists then + pathlists=setmetatableindex(allocate(),makepathlist) + instance.pathlists=pathlists + end + local pathlist=pathlists[filetype] + if pathlist then + local method="intree" + local filelist=collect_files(wantedfiles) + local dirlist={} + local result={} + if filelist then + for i=1,#filelist do + dirlist[i]=filedirname(filelist[i][3]).."/" + end + end + if trace_detail then + report_resolving("checking filename %a in tree",filename) + end + for k=1,#pathlist do + local entry=pathlist[k] + local path=entry.path + local pathname=entry.pathname + local done=false + if filelist then + local expression=entry.expression + if trace_detail then + report_resolving("using pattern %a for path %a",expression,pathname) + end + for k=1,#filelist do + local fl=filelist[k] + local f=fl[2] + local d=dirlist[k] + if find(d,expression) or find(resolveprefix(d),expression) then + result[#result+1]=resolveprefix(fl[3]) + done=true + if allresults then + if trace_detail then + report_resolving("match to %a in hash for file %a and path %a, continue scanning",expression,f,d) + end + else + if trace_detail then + report_resolving("match to %a in hash for file %a and path %a, quit scanning",expression,f,d) + end + break + end + elseif trace_detail then + report_resolving("no match to %a in hash for file %a and path %a",expression,f,d) + end + end + end + if done then + method="database" + else + method="filesystem" + local scheme=entry.scheme + if not scheme or scheme=="file" then + local pname=entry.schemename + if not find(pname,"*",1,true) then + if can_be_dir(pname) then + if not done and not entry.prescanned then + if trace_detail then + report_resolving("quick root scan for %a",pname) + end + for k=1,#wantedfiles do + local w=wantedfiles[k] + local fname=check_subpath(filejoin(pname,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + if not done and entry.recursive then + if trace_detail then + report_resolving("scanning filesystem for %a",pname) + end + local files=resolvers.simplescanfiles(pname,false,true) + for k=1,#wantedfiles do + local w=wantedfiles[k] + local subpath=files[w] + if not subpath or subpath=="" then + elseif type(subpath)=="string" then + local fname=check_subpath(filejoin(pname,subpath,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + else + for i=1,#subpath do + local sp=subpath[i] + if sp=="" then + else + local fname=check_subpath(filejoin(pname,sp,w)) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + end + if done and not allresults then + break + end + end + end + end + end + end + else + end + else + for k=1,#wantedfiles do + local pname=entry.barename + local fname=methodhandler('finders',pname.."/"..wantedfiles[k]) + if fname then + result[#result+1]=fname + done=true + if not allresults then + break + end + end + end + end + end + if done and not allresults then + break + end + end + if #result>0 then + return method,result + end + end +end +local function find_onpath(filename,filetype,wantedfiles,allresults) + if trace_detail then + report_resolving("checking filename %a, filetype %a, wanted files %a",filename,filetype,concat(wantedfiles," | ")) + end + local result={} + for k=1,#wantedfiles do + local fname=wantedfiles[k] + if fname and isreadable(fname) then + filename=fname + result[#result+1]=filejoin('.',fname) + if not allresults then + break + end + end + end + if #result>0 then + return "onpath",result + end +end +local function find_otherwise(filename,filetype,wantedfiles,allresults) + local filelist=collect_files(wantedfiles) + local fl=filelist and filelist[1] + if fl then + return "otherwise",{ resolveprefix(fl[3]) } + end +end +collect_instance_files=function(filename,askedformat,allresults) + if not filename or filename=="" then + return {} + end + askedformat=askedformat or "" + filename=collapsepath(filename,".") + filename=gsub(filename,"^%./",getcurrentdir().."/") + if allresults then + local filetype,wantedfiles=find_analyze(filename,askedformat) + local results={ + { find_direct (filename,true) }, + { find_wildcard (filename,true) }, + { find_qualified(filename,true,askedformat) }, + { find_intree (filename,filetype,wantedfiles,true) }, + { find_onpath (filename,filetype,wantedfiles,true) }, + { find_otherwise(filename,filetype,wantedfiles,true) }, + } + local result,status,done={},{},{} + for k,r in next,results do + local method,list=r[1],r[2] + if method and list then + for i=1,#list do + local c=collapsepath(list[i]) + if not done[c] then + result[#result+1]=c + done[c]=true + end + status[#status+1]=formatters["%-10s: %s"](method,c) + end + end + end + if trace_detail then + report_resolving("lookup status: %s",table.serialize(status,filename)) + end + return result,status + else + local method,result,stamp,filetype,wantedfiles + if instance.remember then + if askedformat=="" then + stamp=formatters["%s::%s"](suffixonly(filename),filename) + else + stamp=formatters["%s::%s"](askedformat,filename) + end + result=stamp and instance.found[stamp] + if result then + if trace_locating then + report_resolving("remembered file %a",filename) + end + return result + end + end + method,result=find_direct(filename) + if not result then + method,result=find_wildcard(filename) + if not result then + method,result=find_qualified(filename,false,askedformat) + if not result then + filetype,wantedfiles=find_analyze(filename,askedformat) + method,result=find_intree(filename,filetype,wantedfiles) + if not result then + method,result=find_onpath(filename,filetype,wantedfiles) + if resolve_otherwise and not result then + method,result=find_otherwise(filename,filetype,wantedfiles) + end + end + end + end + end + if result and #result>0 then + local foundname=collapsepath(result[1]) + resolvers.registerintrees(filename,askedformat,filetype,method,foundname) + result={ foundname } + else + result={} + end + if stamp then + if trace_locating then + report_resolving("remembering file %a using hash %a",filename,stamp) + end + instance.found[stamp]=result + end + return result + end +end +local function findfiles(filename,filetype,allresults) + if not filename or filename=="" then + return {} + end + local result,status=collect_instance_files(filename,filetype or "",allresults) + if not result or #result==0 then + local lowered=lower(filename) + if filename~=lowered then + result,status=collect_instance_files(lowered,filetype or "",allresults) + end + end + return result or {},status +end +function resolvers.findfiles(filename,filetype) + return findfiles(filename,filetype,true) +end +function resolvers.findfile(filename,filetype) + return findfiles(filename,filetype,false)[1] or "" +end +function resolvers.findpath(filename,filetype) + return filedirname(findfiles(filename,filetype,false)[1] or "") +end +local function findgivenfiles(filename,allresults) + local base=filebasename(filename) + local result={} + local hashes=instance.hashes + local function okay(hash,path,name) + local found=methodhandler('concatinators',hash.type,hash.name,path,name) + if found and found~="" then + result[#result+1]=resolveprefix(found) + return not allresults + end + end + for k=1,#hashes do + local hash=hashes[k] + local content=instance.files[hash.name] + if content then + local path,name=lookup(content,base) + if not path then + elseif type(path)=="string" then + if okay(hash,path,name) then + return result + end + else + for i=1,#path do + if okay(hash,path[i],name) then + return result + end + end + end + end + end + return result +end +function resolvers.findgivenfiles(filename) + return findgivenfiles(filename,true) +end +function resolvers.findgivenfile(filename) + return findgivenfiles(filename,false)[1] or "" +end +local makewildcard=Cs( + (P("^")^0*P("/")*P(-1)+P(-1))/".*"+(P("^")^0*P("/")/"")^0*(P("*")/".*"+P("-")/"%%-"+P(".")/"%%."+P("?")/"."+P("\\")/"/"+P(1))^0 +) +function resolvers.wildcardpattern(pattern) + return lpegmatch(makewildcard,pattern) or pattern +end +local function findwildcardfiles(filename,allresults,result) + local result=result or {} + local base=filebasename(filename) + local dirn=filedirname(filename) + local path=lower(lpegmatch(makewildcard,dirn) or dirn) + local name=lower(lpegmatch(makewildcard,base) or base) + local files=instance.files + if find(name,"*",1,true) then + local hashes=instance.hashes + local function okay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end + for k=1,#hashes do + local hash=hashes[k] + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + for found,base in filtered(files[hashname],name) do + if type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end + end + end + end + end + else + local function okayokay(found,path,base,hashname,hashtype) + if find(found,path) then + local full=methodhandler('concatinators',hashtype,hashname,found,base) + if full and full~="" then + result[#result+1]=resolveprefix(full) + return not allresults + end + end + end + local hashes=instance.hashes + for k=1,#hashes do + local hash=hashes[k] + local hashname=hash.name + local hashtype=hash.type + if hashname and hashtype then + local found,base=lookup(content,base) + if not found then + elseif type(found)=='string' then + if okay(found,path,base,hashname,hashtype) then + break + end + else + for i=1,#found do + if okay(found[i],path,base,hashname,hashtype) then + break + end + end + end + end + end + end + return result +end +function resolvers.findwildcardfiles(filename,result) + return findwildcardfiles(filename,true,result) +end +function resolvers.findwildcardfile(filename) + return findwildcardfiles(filename,false)[1] or "" +end +function resolvers.automount() +end +function resolvers.load(option) + statistics.starttiming(instance) + identify_configuration_files() + load_configuration_files() + if option~="nofiles" then + load_databases() + resolvers.automount() + end + statistics.stoptiming(instance) + local files=instance.files + return files and next(files) and true +end +function resolvers.loadtime() + return statistics.elapsedtime(instance) +end +local function report(str) + if trace_locating then + report_resolving(str) + else + print(str) + end +end +function resolvers.dowithfilesandreport(command,files,...) + if files and #files>0 then + if trace_locating then + report('') + end + if type(files)=="string" then + files={ files } + end + for f=1,#files do + local file=files[f] + local result=command(file,...) + if type(result)=='string' then + report(result) + else + for i=1,#result do + report(result[i]) + end + end + end + end +end +function resolvers.showpath(str) + return joinpath(resolvers.expandedpathlist(resolvers.formatofvariable(str))) +end +function resolvers.registerfile(files,name,path) + if files[name] then + if type(files[name])=='string' then + files[name]={ files[name],path } + else + files[name]=path + end + else + files[name]=path + end +end +function resolvers.dowithpath(name,func) + local pathlist=resolvers.expandedpathlist(name) + for i=1,#pathlist do + func("^"..cleanpath(pathlist[i])) + end +end +function resolvers.dowithvariable(name,func) + func(expandedvariable(name)) +end +function resolvers.locateformat(name) + local engine=environment.ownmain or "luatex" + local barename=removesuffix(name) + local fullname=addsuffix(barename,"fmt") + local fmtname=caches.getfirstreadablefile(fullname,"formats",engine) or "" + if fmtname=="" then + fmtname=resolvers.findfile(fullname) + fmtname=cleanpath(fmtname) + end + if fmtname~="" then + local barename=removesuffix(fmtname) + local luaname=addsuffix(barename,luasuffixes.lua) + local lucname=addsuffix(barename,luasuffixes.luc) + local luiname=addsuffix(barename,luasuffixes.lui) + if isfile(luiname) then + return barename,luiname + elseif isfile(lucname) then + return barename,lucname + elseif isfile(luaname) then + return barename,luaname + end + end + return nil,nil +end +function resolvers.booleanvariable(str,default) + local b=resolvers.expansion(str) + if b=="" then + return default + else + b=toboolean(b) + return (b==nil and default) or b + end +end +function resolvers.dowithfilesintree(pattern,handle,before,after) + local instance=resolvers.instance + local hashes=instance.hashes + for i=1,#hashes do + local hash=hashes[i] + local blobtype=hash.type + local blobpath=hash.name + if blobtype and blobpath then + local total=0 + local checked=0 + local done=0 + if before then + before(blobtype,blobpath,pattern) + end + for path,name in filtered(instance.files[blobpath],pattern) do + if type(path)=="string" then + checked=checked+1 + if handle(blobtype,blobpath,path,name) then + done=done+1 + end + else + checked=checked+#path + for i=1,#path do + if handle(blobtype,blobpath,path[i],name) then + done=done+1 + end + end + end + end + if after then + after(blobtype,blobpath,pattern,total,checked,done) + end + end + end +end +local obsolete=resolvers.obsolete or {} +resolvers.obsolete=obsolete +resolvers.find_file=resolvers.findfile obsolete.find_file=resolvers.findfile +resolvers.find_files=resolvers.findfiles obsolete.find_files=resolvers.findfiles + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-pre"] = package.loaded["data-pre"] or true + +-- original size: 3950, stripped down to: 2935 + +if not modules then modules={} end modules ['data-pre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local prefixes=resolvers.prefixes +local cleanpath=resolvers.cleanpath +local findgivenfile=resolvers.findgivenfile +local expansion=resolvers.expansion +local getenv=resolvers.getenv +local basename=file.basename +local dirname=file.dirname +local joinpath=file.join +local isfile=lfs.isfile +prefixes.environment=function(str) + return cleanpath(expansion(str)) +end +local function relative(str,n) + if not isfile(str) then + local pstr="./"..str + if isfile(pstr) then + str=pstr + else + local p="../" + for i=1,n or 2 do + local pstr=p..str + if isfile(pstr) then + str=pstr + break + else + p=p.."../" + end + end + end + end + return cleanpath(str) +end +local function locate(str) + local fullname=findgivenfile(str) or "" + return cleanpath(fullname~="" and fullname or str) +end +prefixes.relative=relative +prefixes.locate=locate +prefixes.auto=function(str) + local fullname=relative(str) + if not isfile(fullname) then + fullname=locate(str) + end + return fullname +end +prefixes.filename=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(basename((fullname~="" and fullname) or str)) +end +prefixes.pathname=function(str) + local fullname=findgivenfile(str) or "" + return cleanpath(dirname((fullname~="" and fullname) or str)) +end +prefixes.selfautoloc=function(str) + return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) +end +prefixes.selfautoparent=function(str) + return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) +end +prefixes.selfautodir=function(str) + return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) +end +prefixes.home=function(str) + return cleanpath(joinpath(getenv('HOME'),str)) +end +prefixes.env=prefixes.environment +prefixes.rel=prefixes.relative +prefixes.loc=prefixes.locate +prefixes.kpse=prefixes.locate +prefixes.full=prefixes.locate +prefixes.file=prefixes.filename +prefixes.path=prefixes.pathname +local function toppath() + local inputstack=resolvers.inputstack + if not inputstack then + return "." + end + local pathname=dirname(inputstack[#inputstack] or "") + if pathname=="" then + return "." + else + return pathname + end +end +local function jobpath() + local path=resolvers.stackpath() + if not path or path=="" then + return "." + else + return path + end +end +resolvers.toppath=toppath +resolvers.jobpath=jobpath +prefixes.toppath=function(str) return cleanpath(joinpath(toppath(),str)) end +prefixes.jobpath=function(str) return cleanpath(joinpath(jobpath(),str)) end +resolvers.setdynamic("toppath") +resolvers.setdynamic("jobpath") +prefixes.jobfile=prefixes.jobpath +resolvers.setdynamic("jobfile") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-inp"] = package.loaded["data-inp"] or true + +-- original size: 910, stripped down to: 823 + +if not modules then modules={} end modules ['data-inp']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local methodhandler=resolvers.methodhandler +local registermethod=resolvers.registermethod +local finders=allocate { helpers={},notfound=function() end } +local openers=allocate { helpers={},notfound=function() end } +local loaders=allocate { helpers={},notfound=function() return false,nil,0 end } +registermethod("finders",finders,"uri") +registermethod("openers",openers,"uri") +registermethod("loaders",loaders,"uri") +resolvers.finders=finders +resolvers.openers=openers +resolvers.loaders=loaders + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-out"] = package.loaded["data-out"] or true + +-- original size: 530, stripped down to: 475 + +if not modules then modules={} end modules ['data-out']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local allocate=utilities.storage.allocate +local resolvers=resolvers +local registermethod=resolvers.registermethod +local savers=allocate { helpers={} } +resolvers.savers=savers +registermethod("savers",savers,"uri") + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-fil"] = package.loaded["data-fil"] or true + +-- original size: 3863, stripped down to: 3310 + +if not modules then modules={} end modules ['data-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_files=logs.reporter("resolvers","files") +local resolvers=resolvers +local resolveprefix=resolvers.resolve +local finders,openers,loaders,savers=resolvers.finders,resolvers.openers,resolvers.loaders,resolvers.savers +local locators,hashers,generators,concatinators=resolvers.locators,resolvers.hashers,resolvers.generators,resolvers.concatinators +local checkgarbage=utilities.garbagecollector and utilities.garbagecollector.check +function locators.file(specification) + local filename=specification.filename + local realname=resolveprefix(filename) + if realname and realname~='' and lfs.isdir(realname) then + if trace_locating then + report_files("file locator %a found as %a",filename,realname) + end + resolvers.appendhash('file',filename,true) + elseif trace_locating then + report_files("file locator %a not found",filename) + end +end +function hashers.file(specification) + local pathname=specification.filename + local content=caches.loadcontent(pathname,'files') + resolvers.registerfilehash(pathname,content,content==nil) +end +function generators.file(specification) + local pathname=specification.filename + local content=resolvers.scanfiles(pathname,false,true) + resolvers.registerfilehash(pathname,content,true) +end +concatinators.file=file.join +function finders.file(specification,filetype) + local filename=specification.filename + local foundname=resolvers.findfile(filename,filetype) + if foundname and foundname~="" then + if trace_locating then + report_files("file finder: %a found",filename) + end + return foundname + else + if trace_locating then + report_files("file finder: %a not found",filename) + end + return finders.notfound() + end +end +function openers.helpers.textopener(tag,filename,f) + return { + reader=function() return f:read () end, + close=function() logs.show_close(filename) return f:close() end, + } +end +function openers.file(specification,filetype) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"r") + if f then + if trace_locating then + report_files("file opener: %a opened",filename) + end + return openers.helpers.textopener("file",filename,f) + end + end + if trace_locating then + report_files("file opener: %a not found",filename) + end + return openers.notfound() +end +function loaders.file(specification,filetype) + local filename=specification.filename + if filename and filename~="" then + local f=io.open(filename,"rb") + if f then + logs.show_load(filename) + if trace_locating then + report_files("file loader: %a loaded",filename) + end + local s=f:read("*a") + if checkgarbage then + checkgarbage(#s) + end + f:close() + if s then + return true,s,#s + end + end + end + if trace_locating then + report_files("file loader: %a not found",filename) + end + return loaders.notfound() +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-con"] = package.loaded["data-con"] or true + +-- original size: 5010, stripped down to: 3588 + +if not modules then modules={} end modules ['data-con']={ + version=1.100, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub=string.format,string.lower,string.gsub +local trace_cache=false trackers.register("resolvers.cache",function(v) trace_cache=v end) +local trace_containers=false trackers.register("resolvers.containers",function(v) trace_containers=v end) +local trace_storage=false trackers.register("resolvers.storage",function(v) trace_storage=v end) +containers=containers or {} +local containers=containers +containers.usecache=true +local report_containers=logs.reporter("resolvers","containers") +local allocated={} +local mt={ + __index=function(t,k) + if k=="writable" then + local writable=caches.getwritablepath(t.category,t.subcategory) or { "." } + t.writable=writable + return writable + elseif k=="readables" then + local readables=caches.getreadablepaths(t.category,t.subcategory) or { "." } + t.readables=readables + return readables + end + end, + __storage__=true +} +function containers.define(category,subcategory,version,enabled) + if category and subcategory then + local c=allocated[category] + if not c then + c={} + allocated[category]=c + end + local s=c[subcategory] + if not s then + s={ + category=category, + subcategory=subcategory, + storage={}, + enabled=enabled, + version=version or math.pi, + trace=false, + } + setmetatable(s,mt) + c[subcategory]=s + end + return s + end +end +function containers.is_usable(container,name) + return container.enabled and caches and caches.is_writable(container.writable,name) +end +function containers.is_valid(container,name) + if name and name~="" then + local storage=container.storage[name] + return storage and storage.cache_version==container.version + else + return false + end +end +function containers.read(container,name) + local storage=container.storage + local stored=storage[name] + if not stored and container.enabled and caches and containers.usecache then + stored=caches.loaddata(container.readables,name) + if stored and stored.cache_version==container.version then + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","load",container.subcategory,name) + end + else + stored=nil + end + storage[name]=stored + elseif stored then + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","reuse",container.subcategory,name) + end + end + return stored +end +function containers.write(container,name,data) + if data then + data.cache_version=container.version + if container.enabled and caches then + local unique,shared=data.unique,data.shared + data.unique,data.shared=nil,nil + caches.savedata(container.writable,name,data) + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","save",container.subcategory,name) + end + data.unique,data.shared=unique,shared + end + if trace_cache or trace_containers then + report_containers("action %a, category %a, name %a","store",container.subcategory,name) + end + container.storage[name]=data + end + return data +end +function containers.content(container,name) + return container.storage[name] +end +function containers.cleanname(name) + return (gsub(lower(name),"[^%w\128-\255]+","-")) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-use"] = package.loaded["data-use"] or true + +-- original size: 3899, stripped down to: 2984 + +if not modules then modules={} end modules ['data-use']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,lower,gsub,find=string.format,string.lower,string.gsub,string.find +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_mounts=logs.reporter("resolvers","mounts") +local resolvers=resolvers +resolvers.automounted=resolvers.automounted or {} +function resolvers.automount(usecache) + local mountpaths=resolvers.cleanpathlist(resolvers.expansion('TEXMFMOUNT')) + if (not mountpaths or #mountpaths==0) and usecache then + mountpaths=caches.getreadablepaths("mount") + end + if mountpaths and #mountpaths>0 then + statistics.starttiming(resolvers.instance) + for k=1,#mountpaths do + local root=mountpaths[k] + local f=io.open(root.."/url.tmi") + if f then + for line in f:lines() do + if line then + if find(line,"^[%%#%-]") then + elseif find(line,"^zip://") then + if trace_locating then + report_mounts("mounting %a",line) + end + table.insert(resolvers.automounted,line) + resolvers.usezipfile(line) + end + end + end + f:close() + end + end + statistics.stoptiming(resolvers.instance) + end +end +statistics.register("used config file",function() return caches.configfiles() end) +statistics.register("used cache path",function() return caches.usedpaths() end) +function statistics.savefmtstatus(texname,formatbanner,sourcefile) + local enginebanner=status.banner + if formatbanner and enginebanner and sourcefile then + local luvname=file.replacesuffix(texname,"luv") + local luvdata={ + enginebanner=enginebanner, + formatbanner=formatbanner, + sourcehash=md5.hex(io.loaddata(resolvers.findfile(sourcefile)) or "unknown"), + sourcefile=sourcefile, + } + io.savedata(luvname,table.serialize(luvdata,true)) + end +end +function statistics.checkfmtstatus(texname) + local enginebanner=status.banner + if enginebanner and texname then + local luvname=file.replacesuffix(texname,"luv") + if lfs.isfile(luvname) then + local luv=dofile(luvname) + if luv and luv.sourcefile then + local sourcehash=md5.hex(io.loaddata(resolvers.findfile(luv.sourcefile)) or "unknown") + local luvbanner=luv.enginebanner or "?" + if luvbanner~=enginebanner then + return format("engine mismatch (luv: %s <> bin: %s)",luvbanner,enginebanner) + end + local luvhash=luv.sourcehash or "?" + if luvhash~=sourcehash then + return format("source mismatch (luv: %s <> bin: %s)",luvhash,sourcehash) + end + else + return "invalid status file" + end + else + return "missing status file" + end + end + return true +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-zip"] = package.loaded["data-zip"] or true + +-- original size: 8772, stripped down to: 6841 + +if not modules then modules={} end modules ['data-zip']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format,find,match=string.format,string.find,string.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_zip=logs.reporter("resolvers","zip") +local resolvers=resolvers +zip=zip or {} +local zip=zip +zip.archives=zip.archives or {} +local archives=zip.archives +zip.registeredfiles=zip.registeredfiles or {} +local registeredfiles=zip.registeredfiles +local function validzip(str) + if not find(str,"^zip://") then + return "zip:///"..str + else + return str + end +end +function zip.openarchive(name) + if not name or name=="" then + return nil + else + local arch=archives[name] + if not arch then + local full=resolvers.findfile(name) or "" + arch=full~="" and zip.open(full) or false + archives[name]=arch + end + return arch + end +end +function zip.closearchive(name) + if not name or (name=="" and archives[name]) then + zip.close(archives[name]) + archives[name]=nil + end +end +function resolvers.locators.zip(specification) + local archive=specification.filename + local zipfile=archive and archive~="" and zip.openarchive(archive) + if trace_locating then + if zipfile then + report_zip("locator: archive %a found",archive) + else + report_zip("locator: archive %a not found",archive) + end + end +end +function resolvers.hashers.zip(specification) + local archive=specification.filename + if trace_locating then + report_zip("loading file %a",archive) + end + resolvers.usezipfile(specification.original) +end +function resolvers.concatinators.zip(zipfile,path,name) + if not path or path=="" then + return format('%s?name=%s',zipfile,name) + else + return format('%s?name=%s/%s',zipfile,path,name) + end +end +function resolvers.finders.zip(specification) + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("finder: archive %a found",archive) + end + local dfile=zfile:open(queryname) + if dfile then + dfile=zfile:close() + if trace_locating then + report_zip("finder: file %a found",queryname) + end + return specification.original + elseif trace_locating then + report_zip("finder: file %a not found",queryname) + end + elseif trace_locating then + report_zip("finder: unknown archive %a",archive) + end + end + end + if trace_locating then + report_zip("finder: %a not found",original) + end + return resolvers.finders.notfound() +end +function resolvers.openers.zip(specification) + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("opener; archive %a opened",archive) + end + local dfile=zfile:open(queryname) + if dfile then + if trace_locating then + report_zip("opener: file %a found",queryname) + end + return resolvers.openers.helpers.textopener('zip',original,dfile) + elseif trace_locating then + report_zip("opener: file %a not found",queryname) + end + elseif trace_locating then + report_zip("opener: unknown archive %a",archive) + end + end + end + if trace_locating then + report_zip("opener: %a not found",original) + end + return resolvers.openers.notfound() +end +function resolvers.loaders.zip(specification) + local original=specification.original + local archive=specification.filename + if archive then + local query=url.query(specification.query) + local queryname=query.name + if queryname then + local zfile=zip.openarchive(archive) + if zfile then + if trace_locating then + report_zip("loader: archive %a opened",archive) + end + local dfile=zfile:open(queryname) + if dfile then + logs.show_load(original) + if trace_locating then + report_zip("loader; file %a loaded",original) + end + local s=dfile:read("*all") + dfile:close() + return true,s,#s + elseif trace_locating then + report_zip("loader: file %a not found",queryname) + end + elseif trace_locating then + report_zip("loader; unknown archive %a",archive) + end + end + end + if trace_locating then + report_zip("loader: %a not found",original) + end + return resolvers.openers.notfound() +end +function resolvers.usezipfile(archive) + local specification=resolvers.splitmethod(archive) + local archive=specification.filename + if archive and not registeredfiles[archive] then + local z=zip.openarchive(archive) + if z then + local instance=resolvers.instance + local tree=url.query(specification.query).tree or "" + if trace_locating then + report_zip("registering: archive %a",archive) + end + statistics.starttiming(instance) + resolvers.prependhash('zip',archive) + resolvers.extendtexmfvariable(archive) + registeredfiles[archive]=z + instance.files[archive]=resolvers.registerzipfile(z,tree) + statistics.stoptiming(instance) + elseif trace_locating then + report_zip("registering: unknown archive %a",archive) + end + elseif trace_locating then + report_zip("registering: archive %a not found",archive) + end +end +function resolvers.registerzipfile(z,tree) + local names={} + local files={} + local remap={} + local n=0 + local filter=tree=="" and "^(.+)/(.-)$" or format("^%s/(.+)/(.-)$",tree) + local register=resolvers.registerfile + if trace_locating then + report_zip("registering: using filter %a",filter) + end + for i in z:files() do + local filename=i.filename + local path,name=match(filename,filter) + if not path then + n=n+1 + register(names,filename,"") + local usedname=lower(filename) + files[usedname]="" + if usedname~=filename then + remap[usedname]=filename + end + elseif name and name~="" then + n=n+1 + register(names,name,path) + local usedname=lower(name) + files[usedname]=path + if usedname~=name then + remap[usedname]=name + end + else + end + end + report_zip("registering: %s files registered",n) + return { + files=files, + remap=remap, + } +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-tre"] = package.loaded["data-tre"] or true + +-- original size: 8479, stripped down to: 5580 + +if not modules then modules={} end modules ['data-tre']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find,gsub,lower=string.find,string.gsub,string.lower +local basename,dirname,joinname=file.basename,file.dirname,file .join +local globdir,isdir,isfile=dir.glob,lfs.isdir,lfs.isfile +local P,lpegmatch=lpeg.P,lpeg.match +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local report_trees=logs.reporter("resolvers","trees") +local resolvers=resolvers +local resolveprefix=resolvers.resolve +local notfound=resolvers.finders.notfound +local lookup=resolvers.get_from_content +local collectors={} +local found={} +function resolvers.finders.tree(specification) + local spec=specification.filename + local okay=found[spec] + if okay==nil then + if spec~="" then + local path=dirname(spec) + local name=basename(spec) + if path=="" then + path="." + end + local names=collectors[path] + if not names then + local pattern=find(path,"/%*+$") and path or (path.."/*") + names=globdir(pattern) + collectors[path]=names + end + local pattern="/"..gsub(name,"([%.%-%+])","%%%1").."$" + for i=1,#names do + local fullname=names[i] + if find(fullname,pattern) then + found[spec]=fullname + return fullname + end + end + local pattern=lower(pattern) + for i=1,#names do + local fullname=lower(names[i]) + if find(fullname,pattern) then + if isfile(fullname) then + found[spec]=fullname + return fullname + else + break + end + end + end + end + okay=notfound() + found[spec]=okay + end + return okay +end +function resolvers.locators.tree(specification) + local name=specification.filename + local realname=resolveprefix(name) + if realname and realname~='' and isdir(realname) then + if trace_locating then + report_trees("locator %a found",realname) + end + resolvers.appendhash('tree',name,false) + elseif trace_locating then + report_trees("locator %a not found",name) + end +end +function resolvers.hashers.tree(specification) + local name=specification.filename + report_trees("analyzing %a",name) + resolvers.methodhandler("hashers",name) + resolvers.generators.file(specification) +end +local collectors={} +local splitter=lpeg.splitat("/**/") +local stripper=lpeg.replacer { [P("/")*P("*")^1*P(-1)]="" } +table.setmetatableindex(collectors,function(t,k) + local rootname=lpegmatch(stripper,k) + local dataname=joinname(rootname,"dirlist") + local content=caches.loadcontent(dataname,"files",dataname) + if not content then + content=resolvers.scanfiles(rootname,nil,nil,false,true) + caches.savecontent(dataname,"files",content,dataname) + end + t[k]=content + return content +end) +local function checked(root,p,n) + if p then + if type(p)=="table" then + for i=1,#p do + local fullname=joinname(root,p[i],n) + if isfile(fullname) then + return fullname + end + end + else + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + end + return notfound() +end +local function resolve(specification) + local filename=specification.filename + if filename~="" then + local root,rest=lpegmatch(splitter,filename) + if root and rest then + local path,name=dirname(rest),basename(rest) + if name~=rest then + local content=collectors[root] + local p,n=lookup(content,name) + if not p then + return notfound() + end + local pattern=".*/"..path.."$" + local istable=type(p)=="table" + if istable then + for i=1,#p do + local pi=p[i] + if pi==path or find(pi,pattern) then + local fullname=joinname(root,pi,n) + if isfile(fullname) then + return fullname + end + end + end + elseif p==path or find(p,pattern) then + local fullname=joinname(root,p,n) + if isfile(fullname) then + return fullname + end + end + local queries=specification.queries + if queries and queries.option=="fileonly" then + return checked(root,p,n) + else + return notfound() + end + end + end + local path,name=dirname(filename),basename(filename) + local root=lpegmatch(stripper,path) + local content=collectors[path] + local p,n=lookup(content,name) + if p then + return checked(root,p,n) + end + end + return notfound() +end +resolvers.finders .dirlist=resolve +resolvers.locators .dirlist=resolvers.locators .tree +resolvers.hashers .dirlist=resolvers.hashers .tree +resolvers.generators.dirlist=resolvers.generators.file +resolvers.openers .dirlist=resolvers.openers .file +resolvers.loaders .dirlist=resolvers.loaders .file +function resolvers.finders.dirfile(specification) + local queries=specification.queries + if queries then + queries.option="fileonly" + else + specification.queries={ option="fileonly" } + end + return resolve(specification) +end +resolvers.locators .dirfile=resolvers.locators .dirlist +resolvers.hashers .dirfile=resolvers.hashers .dirlist +resolvers.generators.dirfile=resolvers.generators.dirlist +resolvers.openers .dirfile=resolvers.openers .dirlist +resolvers.loaders .dirfile=resolvers.loaders .dirlist + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-sch"] = package.loaded["data-sch"] or true + +-- original size: 6569, stripped down to: 5304 + +if not modules then modules={} end modules ['data-sch']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local load=load +local gsub,concat,format=string.gsub,table.concat,string.format +local finders,openers,loaders=resolvers.finders,resolvers.openers,resolvers.loaders +local trace_schemes=false trackers.register("resolvers.schemes",function(v) trace_schemes=v end) +local report_schemes=logs.reporter("resolvers","schemes") +local http=require("socket.http") +local ltn12=require("ltn12") +local resolvers=resolvers +local schemes=resolvers.schemes or {} +resolvers.schemes=schemes +local cleaners={} +schemes.cleaners=cleaners +local threshold=24*60*60 +directives.register("schemes.threshold",function(v) threshold=tonumber(v) or threshold end) +function cleaners.none(specification) + return specification.original +end +function cleaners.strip(specification) + local path,name=file.splitbase(specification.original) + if path=="" then + return (gsub(name,"[^%a%d%.]+","-")) + else + return (gsub((gsub(path,"%.","-").."-"..name),"[^%a%d%.]+","-")) + end +end +function cleaners.md5(specification) + return file.addsuffix(md5.hex(specification.original),file.suffix(specification.path)) +end +local cleaner=cleaners.strip +directives.register("schemes.cleanmethod",function(v) cleaner=cleaners[v] or cleaners.strip end) +function resolvers.schemes.cleanname(specification) + local hash=cleaner(specification) + if trace_schemes then + report_schemes("hashing %a to %a",specification.original,hash) + end + return hash +end +local cached,loaded,reused,thresholds,handlers={},{},{},{},{} +local function runcurl(name,cachename) + local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name + os.execute(command) +end +local function fetch(specification) + local original=specification.original + local scheme=specification.scheme + local cleanname=schemes.cleanname(specification) + local cachename=caches.setfirstwritablefile(cleanname,"schemes") + if not cached[original] then + statistics.starttiming(schemes) + if not io.exists(cachename) or (os.difftime(os.time(),lfs.attributes(cachename).modification)>(thresholds[protocol] or threshold)) then + cached[original]=cachename + local handler=handlers[scheme] + if handler then + if trace_schemes then + report_schemes("fetching %a, protocol %a, method %a",original,scheme,"built-in") + end + logs.flush() + handler(specification,cachename) + else + if trace_schemes then + report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") + end + logs.flush() + runcurl(original,cachename) + end + end + if io.exists(cachename) then + cached[original]=cachename + if trace_schemes then + report_schemes("using cached %a, protocol %a, cachename %a",original,scheme,cachename) + end + else + cached[original]="" + if trace_schemes then + report_schemes("using missing %a, protocol %a",original,scheme) + end + end + loaded[scheme]=loaded[scheme]+1 + statistics.stoptiming(schemes) + else + if trace_schemes then + report_schemes("reusing %a, protocol %a",original,scheme) + end + reused[scheme]=reused[scheme]+1 + end + return cached[original] +end +local function finder(specification,filetype) + return resolvers.methodhandler("finders",fetch(specification),filetype) +end +local opener=openers.file +local loader=loaders.file +local function install(scheme,handler,newthreshold) + handlers [scheme]=handler + loaded [scheme]=0 + reused [scheme]=0 + finders [scheme]=finder + openers [scheme]=opener + loaders [scheme]=loader + thresholds[scheme]=newthreshold or threshold +end +schemes.install=install +local function http_handler(specification,cachename) + local tempname=cachename..".tmp" + local f=io.open(tempname,"wb") + local status,message=http.request { + url=specification.original, + sink=ltn12.sink.file(f) + } + if not status then + os.remove(tempname) + else + os.remove(cachename) + os.rename(tempname,cachename) + end + return cachename +end +install('http',http_handler) +install('https') +install('ftp') +statistics.register("scheme handling time",function() + local l,r,nl,nr={},{},0,0 + for k,v in table.sortedhash(loaded) do + if v>0 then + nl=nl+1 + l[nl]=k..":"..v + end + end + for k,v in table.sortedhash(reused) do + if v>0 then + nr=nr+1 + r[nr]=k..":"..v + end + end + local n=nl+nr + if n>0 then + l=nl>0 and concat(l) or "none" + r=nr>0 and concat(r) or "none" + return format("%s seconds, %s processed, threshold %s seconds, loaded: %s, reused: %s", + statistics.elapsedtime(schemes),n,threshold,l,r) + else + return nil + end +end) +local httprequest=http.request +local toquery=url.toquery +local function fetchstring(url,data) + local q=data and toquery(data) + if q then + url=url.."?"..q + end + local reply=httprequest(url) + return reply +end +schemes.fetchstring=fetchstring +function schemes.fetchtable(url,data) + local reply=fetchstring(url,data) + if reply then + local s=load("return "..reply) + if s then + return s() + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-lua"] = package.loaded["data-lua"] or true + +-- original size: 4313, stripped down to: 3227 + +if not modules then modules={} end modules ['data-lua']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local package,lpeg=package,lpeg +local gsub=string.gsub +local concat=table.concat +local addsuffix=file.addsuffix +local P,S,Cs,lpegmatch=lpeg.P,lpeg.S,lpeg.Cs,lpeg.match +local luasuffixes={ 'tex','lua' } +local libsuffixes={ 'lib' } +local luaformats={ 'TEXINPUTS','LUAINPUTS' } +local libformats={ 'CLUAINPUTS' } +local helpers=package.helpers or {} +local methods=helpers.methods or {} +local resolvers=resolvers +local resolveprefix=resolvers.resolve +helpers.report=logs.reporter("resolvers","libraries") +trackers.register("resolvers.libraries",function(v) helpers.trace=v end) +trackers.register("resolvers.locating",function(v) helpers.trace=v end) +helpers.sequence={ + "already loaded", + "preload table", + "lua variable format", + "lib variable format", + "lua extra list", + "lib extra list", + "path specification", + "cpath specification", + "all in one fallback", + "not loaded", +} +local pattern=Cs(P("!")^0/""*(P("/")*P(-1)/"/"+P("/")^1/"/"+1)^0) +function helpers.cleanpath(path) + return resolveprefix(lpegmatch(pattern,path)) +end +local loadedaslib=helpers.loadedaslib +local getextraluapaths=package.extraluapaths +local getextralibpaths=package.extralibpaths +local registerpath=helpers.registerpath +local lualibfile=helpers.lualibfile +local luaformatpaths +local libformatpaths +local function getluaformatpaths() + if not luaformatpaths then + luaformatpaths={} + for i=1,#luaformats do + registerpath("lua format","lua",luaformatpaths,resolvers.expandedpathlistfromvariable(luaformats[i])) + end + end + return luaformatpaths +end +local function getlibformatpaths() + if not libformatpaths then + libformatpaths={} + for i=1,#libformats do + registerpath("lib format","lib",libformatpaths,resolvers.expandedpathlistfromvariable(libformats[i])) + end + end + return libformatpaths +end +local function loadedbyformat(name,rawname,suffixes,islib,what) + local trace=helpers.trace + local report=helpers.report + for i=1,#suffixes do + local format=suffixes[i] + local resolved=resolvers.findfile(name,format) or "" + if trace then + report("%s format, identifying %a using format %a",what,name,format) + end + if resolved~="" then + if trace then + report("%s format, %a found on %a",what,name,resolved) + end + if islib then + return loadedaslib(resolved,rawname) + else + return loadfile(resolved) + end + end + end +end +helpers.loadedbyformat=loadedbyformat +methods["lua variable format"]=function(name) + if helpers.trace then + helpers.report("%s format, checking %s paths","lua",#getluaformatpaths()) + end + return loadedbyformat(addsuffix(lualibfile(name),"lua"),name,luasuffixes,false,"lua") +end +methods["lib variable format"]=function(name) + if helpers.trace then + helpers.report("%s format, checking %s paths","lib",#getlibformatpaths()) + end + return loadedbyformat(addsuffix(lualibfile(name),os.libsuffix),name,libsuffixes,true,"lib") +end +resolvers.loadlualib=require + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-aux"] = package.loaded["data-aux"] or true + +-- original size: 2431, stripped down to: 1996 + +if not modules then modules={} end modules ['data-aux']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local find=string.find +local type,next=type,next +local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) +local resolvers=resolvers +local report_scripts=logs.reporter("resolvers","scripts") +function resolvers.updatescript(oldname,newname) + local scriptpath="context/lua" + newname=file.addsuffix(newname,"lua") + local oldscript=resolvers.cleanpath(oldname) + if trace_locating then + report_scripts("to be replaced old script %a",oldscript) + end + local newscripts=resolvers.findfiles(newname) or {} + if #newscripts==0 then + if trace_locating then + report_scripts("unable to locate new script") + end + else + for i=1,#newscripts do + local newscript=resolvers.cleanpath(newscripts[i]) + if trace_locating then + report_scripts("checking new script %a",newscript) + end + if oldscript==newscript then + if trace_locating then + report_scripts("old and new script are the same") + end + elseif not find(newscript,scriptpath) then + if trace_locating then + report_scripts("new script should come from %a",scriptpath) + end + elseif not (find(oldscript,file.removesuffix(newname).."$") or find(oldscript,newname.."$")) then + if trace_locating then + report_scripts("invalid new script name") + end + else + local newdata=io.loaddata(newscript) + if newdata then + if trace_locating then + report_scripts("old script content replaced by new content") + end + io.savedata(oldscript,newdata) + break + elseif trace_locating then + report_scripts("unable to load new script") + end + end + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-tmf"] = package.loaded["data-tmf"] or true + +-- original size: 2601, stripped down to: 1627 + +if not modules then modules={} end modules ['data-tmf']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local resolvers=resolvers +local report_tds=logs.reporter("resolvers","tds") +function resolvers.load_tree(tree,resolve) + if type(tree)=="string" and tree~="" then + local getenv,setenv=resolvers.getenv,resolvers.setenv + local texos="texmf-"..os.platform + local oldroot=environment.texroot + local newroot=file.collapsepath(tree) + local newtree=file.join(newroot,texos) + local newpath=file.join(newtree,"bin") + if not lfs.isdir(newtree) then + report_tds("no %a under tree %a",texos,tree) + os.exit() + end + if not lfs.isdir(newpath) then + report_tds("no '%s/bin' under tree %a",texos,tree) + os.exit() + end + local texmfos=newtree + environment.texroot=newroot + environment.texos=texos + environment.texmfos=texmfos + if resolve then + resolvers.luacnfspec=resolvers.resolve(resolvers.luacnfspec) + end + setenv('SELFAUTOPARENT',newroot) + setenv('SELFAUTODIR',newtree) + setenv('SELFAUTOLOC',newpath) + setenv('TEXROOT',newroot) + setenv('TEXOS',texos) + setenv('TEXMFOS',texmfos) + setenv('TEXMFCNF',resolvers.luacnfspec,true) + setenv('PATH',newpath..io.pathseparator..getenv('PATH')) + report_tds("changing from root %a to %a",oldroot,newroot) + report_tds("prepending %a to PATH",newpath) + report_tds("setting TEXMFCNF to %a",resolvers.luacnfspec) + report_tds() + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["data-lst"] = package.loaded["data-lst"] or true + +-- original size: 2734, stripped down to: 2354 + +if not modules then modules={} end modules ['data-lst']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local rawget,type,next=rawget,type,next +local find,concat,upper=string.find,table.concat,string.upper +local fastcopy,sortedpairs=table.fastcopy,table.sortedpairs +local resolvers=resolvers +local listers=resolvers.listers or {} +resolvers.listers=listers +local resolveprefix=resolvers.resolve +local report_lists=logs.reporter("resolvers","lists") +local function tabstr(str) + if type(str)=='table' then + return concat(str," | ") + else + return str + end +end +function listers.variables(pattern) + local instance=resolvers.instance + local environment=instance.environment + local variables=instance.variables + local expansions=instance.expansions + local pattern=upper(pattern or "") + local configured={} + local order=instance.order + for i=1,#order do + for k,v in next,order[i] do + if v~=nil and configured[k]==nil then + configured[k]=v + end + end + end + local env=fastcopy(environment) + local var=fastcopy(variables) + local exp=fastcopy(expansions) + for key,value in sortedpairs(configured) do + if key~="" and (pattern=="" or find(upper(key),pattern)) then + report_lists(key) + report_lists(" env: %s",tabstr(rawget(environment,key)) or "unset") + report_lists(" var: %s",tabstr(configured[key]) or "unset") + report_lists(" exp: %s",tabstr(expansions[key]) or "unset") + report_lists(" res: %s",tabstr(resolveprefix(expansions[key])) or "unset") + end + end + instance.environment=fastcopy(env) + instance.variables=fastcopy(var) + instance.expansions=fastcopy(exp) +end +local report_resolved=logs.reporter("system","resolved") +function listers.configurations() + local configurations=resolvers.instance.specification + for i=1,#configurations do + report_resolved("file : %s",resolveprefix(configurations[i])) + end + report_resolved("") + local list=resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.luacnfspec)) + for i=1,#list do + local li=resolveprefix(list[i]) + if lfs.isdir(li) then + report_resolved("path - %s",li) + else + report_resolved("path + %s",li) + end + end +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-lib"] = package.loaded["util-lib"] or true + +-- original size: 11549, stripped down to: 5905 + +if not modules then modules={} end modules ['util-lib']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files", +} +local gsub,find=string.gsub,string.find +local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join +local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles +local loaded=package.loaded +local report_swiglib=logs.reporter("swiglib") +local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local done=false +local function requireswiglib(required,version) + local trace_swiglib=trace_swiglib or package.helpers.trace + local library=loaded[required] + if library==nil then + if trace_swiglib then + report_swiglib("requiring library %a with version %a",required,version or "any") + end + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + local required_name=required_base.."."..os.libsuffix + local version=type(version)=="string" and version~="" and version or false + local engine=environment.ownmain or false + if trace_swiglib and not done then + local list=resolvers.expandedpathlistfromvariable("lib") + for i=1,#list do + report_swiglib("tds path %i: %s",i,list[i]) + end + end + local function found(locate,asked_library,how,...) + if trace_swiglib then + report_swiglib("checking %s: %a",how,asked_library) + end + return locate(asked_library,...) + end + local function check(locate,...) + local found=nil + if version then + local asked_library=joinfile(required_path,version,required_name) + if trace_swiglib then + report_swiglib("checking %s: %a","with version",asked_library) + end + found=locate(asked_library,...) + end + if not found or found=="" then + local asked_library=joinfile(required_path,required_name) + if trace_swiglib then + report_swiglib("checking %s: %a","with version",asked_library) + end + found=locate(asked_library,...) + end + return found and found~="" and found or false + end + local function attempt(checkpattern) + if trace_swiglib then + report_swiglib("checking tds lib paths strictly") + end + local found=findfile and check(findfile,"lib") + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + if trace_swiglib then + report_swiglib("checking tds lib paths with wildcard") + end + local asked_library=joinfile(required_path,".*",required_name) + if trace_swiglib then + report_swiglib("checking %s: %a","latest version",asked_library) + end + local list=findfiles(asked_library,"lib",true) + if list and #list>0 then + table.sort(list) + local found=list[#list] + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + if trace_swiglib then + report_swiglib("checking lib paths") + end + package.extralibpath(environment.ownpath) + local paths=package.libpaths() + for i=1,#paths do + local found=check(lfs.isfile) + if found and (not checkpattern or find(found,checkpattern)) then + return found + end + end + return false + end + local found_library=nil + if engine then + if trace_swiglib then + report_swiglib("attemp 1, engine %a",engine) + end + found_library=attempt("/"..engine.."/") + if not found_library then + if trace_swiglib then + report_swiglib("attemp 2, no engine",asked_library) + end + found_library=attempt() + end + else + found_library=attempt() + end + if not found_library then + if trace_swiglib then + report_swiglib("not found: %a",required) + end + library=false + else + local path=pathpart(found_library) + local base=nameonly(found_library) + dir.push(path) + if trace_swiglib then + report_swiglib("found: %a",found_library) + end + local message=nil + local opener="luaopen_"..required_base + library,message=package.loadlib(found_library,opener) + local libtype=type(library) + if libtype=="function" then + library=library() + else + report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") + library=false + end + dir.pop() + end + if not library then + report_swiglib("unknown: %a",required) + elseif trace_swiglib then + report_swiglib("stored: %a",required) + end + loaded[required]=library + else + report_swiglib("reused: %a",required) + end + return library +end +local savedrequire=require +function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) + else + return savedrequire(name) + end +end +local swiglibs={} +local initializer="core" +function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) + if trace_swiglib then + report_swiglib("loading %a",name) + end + if not find(name,"%."..initializer.."$") then + fullname="swiglib."..name.."."..initializer + else + fullname="swiglib."..name + end + library=requireswiglib(fullname,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) + end + return library +end +statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end +end) + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["luat-sta"] = package.loaded["luat-sta"] or true + +-- original size: 5703, stripped down to: 2507 + +if not modules then modules={} end modules ['luat-sta']={ + version=1.001, + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local gmatch,match=string.gmatch,string.match +local type=type +states=states or {} +local states=states +states.data=states.data or {} +local data=states.data +states.hash=states.hash or {} +local hash=states.hash +states.tag=states.tag or "" +states.filename=states.filename or "" +function states.save(filename,tag) + tag=tag or states.tag + filename=file.addsuffix(filename or states.filename,'lus') + io.savedata(filename, + "-- generator : luat-sta.lua\n".."-- state tag : "..tag.."\n\n"..table.serialize(data[tag or states.tag] or {},true) + ) +end +function states.load(filename,tag) + states.filename=filename + states.tag=tag or "whatever" + states.filename=file.addsuffix(states.filename,'lus') + data[states.tag],hash[states.tag]=(io.exists(filename) and dofile(filename)) or {},{} +end +local function set_by_tag(tag,key,value,default,persistent) + local d,h=data[tag],hash[tag] + if d then + if type(d)=="table" then + local dkey,hkey=key,key + local pre,post=match(key,"(.+)%.([^%.]+)$") + if pre and post then + for k in gmatch(pre,"[^%.]+") do + local dk=d[k] + if not dk then + dk={} + d[k]=dk + elseif type(dk)=="string" then + break + end + d=dk + end + dkey,hkey=post,key + end + if value==nil then + value=default + elseif value==false then + elseif persistent then + value=value or d[dkey] or default + else + value=value or default + end + d[dkey],h[hkey]=value,value + elseif type(d)=="string" then + data[tag],hash[tag]=value,value + end + end +end +local function get_by_tag(tag,key,default) + local h=hash[tag] + if h and h[key] then + return h[key] + else + local d=data[tag] + if d then + for k in gmatch(key,"[^%.]+") do + local dk=d[k] + if dk~=nil then + d=dk + else + return default + end + end + if d==false then + return false + else + return d or default + end + end + end +end +states.set_by_tag=set_by_tag +states.get_by_tag=get_by_tag +function states.set(key,value,default,persistent) + set_by_tag(states.tag,key,value,default,persistent) +end +function states.get(key,default) + return get_by_tag(states.tag,key,default) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true + +-- original size: 5955, stripped down to: 4926 + +if not modules then modules={} end modules ['luat-fmt']={ + version=1.001, + comment="companion to mtxrun", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local format=string.format +local concat=table.concat +local quoted=string.quoted +local luasuffixes=utilities.lua.suffixes +local report_format=logs.reporter("resolvers","formats") +local function primaryflags() + local trackers=environment.argument("trackers") + local directives=environment.argument("directives") + local flags={} + if trackers and trackers~="" then + flags={ "--trackers="..quoted(trackers) } + end + if directives and directives~="" then + flags={ "--directives="..quoted(directives) } + end + if environment.argument("jit") then + flags={ "--jiton" } + end + return concat(flags," ") +end +function environment.make_format(name) + local engine=environment.ownmain or "luatex" + local olddir=dir.current() + local path=caches.getwritablepath("formats",engine) or "" + if path~="" then + lfs.chdir(path) + end + report_format("using format path %a",dir.current()) + local texsourcename=file.addsuffix(name,"mkiv") + local fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + if fulltexsourcename=="" then + texsourcename=file.addsuffix(name,"tex") + fulltexsourcename=resolvers.findfile(texsourcename,"tex") or "" + end + if fulltexsourcename=="" then + report_format("no tex source file with name %a (mkiv or tex)",name) + lfs.chdir(olddir) + return + else + report_format("using tex source file %a",fulltexsourcename) + end + local texsourcepath=dir.expandname(file.dirname(fulltexsourcename)) + local specificationname=file.replacesuffix(fulltexsourcename,"lus") + local fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + if fullspecificationname=="" then + specificationname=file.join(texsourcepath,"context.lus") + fullspecificationname=resolvers.findfile(specificationname,"tex") or "" + end + if fullspecificationname=="" then + report_format("unknown stub specification %a",specificationname) + lfs.chdir(olddir) + return + end + local specificationpath=file.dirname(fullspecificationname) + local usedluastub=nil + local usedlualibs=dofile(fullspecificationname) + if type(usedlualibs)=="string" then + usedluastub=file.join(file.dirname(fullspecificationname),usedlualibs) + elseif type(usedlualibs)=="table" then + report_format("using stub specification %a",fullspecificationname) + local texbasename=file.basename(name) + local luastubname=file.addsuffix(texbasename,luasuffixes.lua) + local lucstubname=file.addsuffix(texbasename,luasuffixes.luc) + report_format("creating initialization file %a",luastubname) + utilities.merger.selfcreate(usedlualibs,specificationpath,luastubname) + if utilities.lua.compile(luastubname,lucstubname) and lfs.isfile(lucstubname) then + report_format("using compiled initialization file %a",lucstubname) + usedluastub=lucstubname + else + report_format("using uncompiled initialization file %a",luastubname) + usedluastub=luastubname + end + else + report_format("invalid stub specification %a",fullspecificationname) + lfs.chdir(olddir) + return + end + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") + report_format("running command: %s\n",command) + os.execute(command) + local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" + local mp=dir.glob(pattern) + if mp then + for i=1,#mp do + local name=mp[i] + report_format("removing related mplib format %a",file.basename(name)) + os.remove(name) + end + end + lfs.chdir(olddir) +end +function environment.run_format(name,data,more) + if name and name~="" then + local engine=environment.ownmain or "luatex" + local barename=file.removesuffix(name) + local fmtname=caches.getfirstreadablefile(file.addsuffix(barename,"fmt"),"formats",engine) + if fmtname=="" then + fmtname=resolvers.findfile(file.addsuffix(barename,"fmt")) or "" + end + fmtname=resolvers.cleanpath(fmtname) + if fmtname=="" then + report_format("no format with name %a",name) + else + local barename=file.removesuffix(name) + local luaname=file.addsuffix(barename,"luc") + if not lfs.isfile(luaname) then + luaname=file.addsuffix(barename,"lua") + end + if not lfs.isfile(luaname) then + report_format("using format name %a",fmtname) + report_format("no luc/lua file with name %a",barename) + else + local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") + report_format("running command: %s",command) + os.execute(command) + end + end + end +end + + +end -- of closure + +-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- skipped libraries : - +-- original bytes : 745618 +-- stripped bytes : 269191 + +-- end library merge + +-- We need this hack till luatex is fixed. +-- +-- for k,v in pairs(arg) do print(k,v) end + +if arg and (arg[0] == 'luatex' or arg[0] == 'luatex.exe') and arg[1] == "--luaonly" then + arg[-1]=arg[0] arg[0]=arg[2] for k=3,#arg do arg[k-2]=arg[k] end arg[#arg]=nil arg[#arg]=nil +end + +-- End of hack. + +local format, gsub, gmatch, match, find = string.format, string.gsub, string.gmatch, string.match, string.find +local concat = table.concat + +local ownname = environment and environment.ownname or arg[0] or 'mtxrun.lua' +local ownpath = gsub(match(ownname,"^(.+)[\\/].-$") or ".","\\","/") +local owntree = environment and environment.ownpath or ownpath + +local ownlibs = { -- order can be made better + + 'l-lua.lua', + 'l-package.lua', + 'l-lpeg.lua', + 'l-function.lua', + 'l-string.lua', + 'l-table.lua', + 'l-io.lua', + 'l-number.lua', + 'l-set.lua', + 'l-os.lua', + 'l-file.lua', + 'l-gzip.lua', + 'l-md5.lua', + 'l-url.lua', + 'l-dir.lua', + 'l-boolean.lua', + 'l-unicode.lua', + 'l-math.lua', + + 'util-str.lua', -- code might move to l-string + 'util-tab.lua', + 'util-sto.lua', + 'util-prs.lua', + 'util-fmt.lua', + + 'trac-set.lua', + 'trac-log.lua', + 'trac-inf.lua', -- was before trac-set + 'trac-pro.lua', -- not really needed + 'util-lua.lua', -- indeed here? + 'util-deb.lua', + + 'util-mrg.lua', + 'util-tpl.lua', + + 'util-env.lua', + 'luat-env.lua', -- can come before inf (as in mkiv) + + 'lxml-tab.lua', + 'lxml-lpt.lua', + -- 'lxml-ent.lua', + 'lxml-mis.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', + + 'trac-xml.lua', + + 'data-ini.lua', + 'data-exp.lua', + 'data-env.lua', + 'data-tmp.lua', + 'data-met.lua', + 'data-res.lua', + 'data-pre.lua', + 'data-inp.lua', + 'data-out.lua', + 'data-fil.lua', + 'data-con.lua', + 'data-use.lua', +-- 'data-tex.lua', +-- 'data-bin.lua', + 'data-zip.lua', + 'data-tre.lua', + 'data-sch.lua', + 'data-lua.lua', + 'data-aux.lua', -- updater + 'data-tmf.lua', + 'data-lst.lua', + + 'util-lib.lua', -- swiglib + + 'luat-sta.lua', + 'luat-fmt.lua', + +} + +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/context/sources/data-tmf.lua + +local ownlist = { + -- '.', + -- ownpath , + owntree .. "/../../../../context/sources", -- HH's development path + owntree .. "/../../texmf-local/tex/context/base", + owntree .. "/../../texmf-context/tex/context/base", + owntree .. "/../../texmf/tex/context/base", + owntree .. "/../../../texmf-local/tex/context/base", + owntree .. "/../../../texmf-context/tex/context/base", + owntree .. "/../../../texmf/tex/context/base", +} + +if ownpath == "." then table.remove(ownlist,1) end + +own = { + name = ownname, + path = ownpath, + tree = owntree, + list = ownlist, + libs = ownlibs, +} + +local function locate_libs() + for l=1,#ownlibs do + local lib = ownlibs[l] + for p =1,#ownlist do + local pth = ownlist[p] + local filename = pth .. "/" .. lib + local found = lfs.isfile(filename) + if found then + package.path = package.path .. ";" .. pth .. "/?.lua" -- in case l-* does a require + return pth + end + end + end +end + +local function load_libs() + local found = locate_libs() + if found then + for l=1,#ownlibs do + local filename = found .. "/" .. ownlibs[l] + local codeblob = loadfile(filename) + if codeblob then + codeblob() + end + end + else + resolvers = nil + end +end + +if not resolvers then + load_libs() +end + +if not resolvers then + print("") + print("Mtxrun is unable to start up due to lack of libraries. You may") + print("try to run 'lua mtxrun.lua --selfmerge' in the path where this") + print("script is located (normally under ..../scripts/context/lua) which") + print("will make this script library independent.") + os.exit() +end + +-- verbosity + +----- e_verbose = environment.arguments["verbose"] + +local e_verbose = false + +-- some common flags (also passed through environment) + +local e_silent = environment.argument("silent") +local e_noconsole = environment.argument("noconsole") + +local e_trackers = environment.argument("trackers") +local e_directives = environment.argument("directives") +local e_experiments = environment.argument("experiments") + +if e_silent == true then + e_silent = "*" +end + +if type(e_silent) == "string" then + if type(e_directives) == "string" then + e_directives = format("%s,logs.blocked={%s}",e_directives,e_silent) + else + e_directives = format("logs.blocked={%s}",e_silent) + end +end + +if e_noconsole then + if type(e_directives) == "string" then + e_directives = format("%s,logs.target=file",e_directives) + else + e_directives = format("logs.target=file") + end +end + +if e_trackers then trackers .enable(e_trackers) end +if e_directives then directives .enable(e_directives) end +if e_experiments then experiments.enable(e_experiments) end + +if not environment.trackers then environment.trackers = e_trackers end +if not environment.directives then environment.directives = e_directives end +if not environment.experiments then environment.experiments = e_experiments end + +-- + +local instance = resolvers.reset() + +local helpinfo = [[ +<?xml version="1.0" ?> +<application> + <metadata> + <entry name="name">mtxrun</entry> + <entry name="detail">ConTeXt TDS Runner Tool</entry> + <entry name="version">1.31</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="script"><short>run an mtx script (lua prefered method) (<ref name="noquotes"/>), no script gives list</short></flag> + <flag name="evaluate"><short>run code passed on the commandline (between quotes)</short></flag> + <flag name="execute"><short>run a script or program (texmfstart method) (<ref name="noquotes"/>)</short></flag> + <flag name="resolve"><short>resolve prefixed arguments</short></flag> + <flag name="ctxlua"><short>run internally (using preloaded libs)</short></flag> + <flag name="internal"><short>run script using built in libraries (same as <ref name="ctxlua"/>)</short></flag> + <flag name="locate"><short>locate given filename in database (default) or system (<ref name="first"/> <ref name="all"/> <ref name="detail"/>)</short></flag> + </subcategory> + <subcategory> + <flag name="autotree"><short>use texmf tree cf. env texmfstart_tree or texmfstarttree</short></flag> + <flag name="tree" value="pathtotree"><short>use given texmf tree (default file: setuptex.tmf)</short></flag> + <flag name="environment" value="name"><short>use given (tmf) environment file</short></flag> + <flag name="path" value="runpath"><short>go to given path before execution</short></flag> + <flag name="ifchanged" value="filename"><short>only execute when given file has changed (md checksum)</short></flag> + <flag name="iftouched" value="old,new"><short>only execute when given file has changed (time stamp)</short></flag> + </subcategory> + <subcategory> + <flag name="makestubs"><short>create stubs for (context related) scripts</short></flag> + <flag name="removestubs"><short>remove stubs (context related) scripts</short></flag> + <flag name="stubpath" value="binpath"><short>paths where stubs wil be written</short></flag> + <flag name="windows"><short>create windows (mswin) stubs</short></flag> + <flag name="unix"><short>create unix (linux) stubs</short></flag> + </subcategory> + <subcategory> + <flag name="verbose"><short>give a bit more info</short></flag> + <flag name="trackers" value="list"><short>enable given trackers</short></flag> + <flag name="progname" value="str"><short>format or backend</short></flag> + <flag name="systeminfo" value="str"><short>show current operating system, processor, etc</short></flag> + </subcategory> + <subcategory> + <flag name="edit"><short>launch editor with found file</short></flag> + <flag name="launch"><short>launch files like manuals, assumes os support (<ref name="all"/>)</short></flag> + </subcategory> + <subcategory> + <flag name="timedrun"><short>run a script and time its run</short></flag> + <flag name="autogenerate"><short>regenerate databases if needed (handy when used to run context in an editor)</short></flag> + </subcategory> + <subcategory> + <flag name="usekpse"><short>use kpse as fallback (when no mkiv and cache installed, often slower)</short></flag> + <flag name="forcekpse"><short>force using kpse (handy when no mkiv and cache installed but less functionality)</short></flag> + </subcategory> + <subcategory> + <flag name="prefixes"><short>show supported prefixes</short></flag> + </subcategory> + <subcategory> + <flag name="generate"><short>generate file database</short></flag> + </subcategory> + <subcategory> + <flag name="variables"><short>show configuration variables</short></flag> + <flag name="configurations"><short>show configuration order</short></flag> + </subcategory> + <subcategory> + <flag name="directives"><short>show (known) directives</short></flag> + <flag name="trackers"><short>show (known) trackers</short></flag> + <flag name="experiments"><short>show (known) experiments</short></flag> + </subcategory> + <subcategory> + <flag name="expand-braces"><short>expand complex variable</short></flag> + <flag name="expand-path"><short>expand variable (resolve paths)</short></flag> + <flag name="expand-var"><short>expand variable (resolve references)</short></flag> + <flag name="show-path"><short>show path expansion of ...</short></flag> + <flag name="var-value"><short>report value of variable</short></flag> + <flag name="find-file"><short>report file location</short></flag> + <flag name="find-path"><short>report path of file</short></flag> + </subcategory> + <subcategory> + <flag name="pattern" value="string"><short>filter variables</short></flag> + </subcategory> + </category> + </flags> +</application> +]] + +local application = logs.application { + name = "mtxrun", + banner = "ConTeXt TDS Runner Tool 1.31", + helpinfo = helpinfo, +} + +local report = application.report + +messages = messages or { } -- for the moment + +runners = runners or { } -- global (might become local) + +runners.applications = { + ["lua"] = "luatex --luaonly", + ["luc"] = "luatex --luaonly", + ["pl"] = "perl", + ["py"] = "python", + ["rb"] = "ruby", +} + +runners.suffixes = { + 'rb', 'lua', 'py', 'pl' +} + +runners.registered = { + texexec = { 'texexec.rb', false }, -- context mkii runner (only tool not to be luafied) + texutil = { 'texutil.rb', true }, -- old perl based index sorter for mkii (old versions need it) + texfont = { 'texfont.pl', true }, -- perl script that makes mkii font metric files + texfind = { 'texfind.pl', false }, -- perltk based tex searching tool, mostly used at pragma + texshow = { 'texshow.pl', false }, -- perltk based context help system, will be luafied + -- texwork = { 'texwork.pl', false }, -- perltk based editing environment, only used at pragma + makempy = { 'makempy.pl', true }, + mptopdf = { 'mptopdf.pl', true }, + pstopdf = { 'pstopdf.rb', true }, -- converts ps (and some more) images, does some cleaning (replaced) + -- examplex = { 'examplex.rb', false }, + concheck = { 'concheck.rb', false }, + runtools = { 'runtools.rb', true }, + textools = { 'textools.rb', true }, + tmftools = { 'tmftools.rb', true }, + ctxtools = { 'ctxtools.rb', true }, + rlxtools = { 'rlxtools.rb', true }, + pdftools = { 'pdftools.rb', true }, + mpstools = { 'mpstools.rb', true }, + -- exatools = { 'exatools.rb', true }, + xmltools = { 'xmltools.rb', true }, + -- luatools = { 'luatools.lua', true }, + mtxtools = { 'mtxtools.rb', true }, + pdftrimwhite = { 'pdftrimwhite.pl', false }, +} + +runners.launchers = { + windows = { }, + unix = { }, +} + +-- like runners.libpath("framework"): looks on script's subpath + +function runners.libpath(...) + package.prepend_libpath(file.dirname(environment.ownscript),...) + package.prepend_libpath(file.dirname(environment.ownname) ,...) +end + +function runners.prepare() + local checkname = environment.argument("ifchanged") + if type(checkname) == "string" and checkname ~= "" then + local oldchecksum = file.loadchecksum(checkname) + local newchecksum = file.checksum(checkname) + if oldchecksum == newchecksum then + if e_verbose then + report("file '%s' is unchanged",checkname) + end + return "skip" + elseif e_verbose then + report("file '%s' is changed, processing started",checkname) + end + file.savechecksum(checkname) + end + local touchname = environment.argument("iftouched") + if type(touchname) == "string" and touchname ~= "" then + local oldname, newname = string.splitup(touchname, ",") + if oldname and newname and oldname ~= "" and newname ~= "" then + if not file.needs_updating(oldname,newname) then + if e_verbose then + report("file '%s' and '%s' have same age",oldname,newname) + end + return "skip" + elseif e_verbose then + report("file '%s' is older than '%s'",oldname,newname) + end + end + end + local runpath = environment.argument("path") + if type(runpath) == "string" and not lfs.chdir(runpath) then + report("unable to change to path '%s'",runpath) + return "error" + end + runners.prepare = function() end + return "run" +end + +function runners.execute_script(fullname,internal,nosplit) + local noquote = environment.argument("noquotes") + if fullname and fullname ~= "" then + local state = runners.prepare() + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + local path, name, suffix = file.splitname(fullname) + local result = "" + if path ~= "" then + result = fullname + elseif name then + name = gsub(name,"^int[%a]*:",function() + internal = true + return "" + end ) + name = gsub(name,"^script:","") + if suffix == "" and runners.registered[name] and runners.registered[name][1] then + name = runners.registered[name][1] + suffix = file.suffix(name) + end + if suffix == "" then + -- loop over known suffixes + for _,s in pairs(runners.suffixes) do + result = resolvers.findfile(name .. "." .. s, 'texmfscripts') + if result ~= "" then + break + end + end + elseif runners.applications[suffix] then + result = resolvers.findfile(name, 'texmfscripts') + else + -- maybe look on path + result = resolvers.findfile(name, 'other text files') + end + end + if result and result ~= "" then + if not no_split then + local before, after = environment.splitarguments(fullname) -- already done + environment.arguments_before, environment.arguments_after = before, after + end + if internal then + arg = { } for _,v in pairs(environment.arguments_after) do arg[#arg+1] = v end + environment.ownscript = result + dofile(result) + else +local texmfcnf = resolvers.getenv("TEXMFCNF") +if not texmfcnf or texmfcnf == "" then + texmfcnf = resolvers.expandedpathfromlist(resolvers.splitpath(resolvers.resolve(resolvers.luacnfspec))) + resolvers.setenv("TEXMFCNF",table.concat(texmfcnf,";")) -- for running texexec etc (after tl change to texmf-dist) +end + local binary = runners.applications[file.suffix(result)] + result = string.quoted(string.unquoted(result)) + -- if string.match(result,' ') and not string.match(result,"^\".*\"$") then + -- result = '"' .. result .. '"' + -- end + if binary and binary ~= "" then + result = binary .. " " .. result + end + local command = result .. " " .. environment.reconstructcommandline(environment.arguments_after,noquote) + if e_verbose then + report() + report("executing: %s",command) + report() + report() + io.flush() + end + -- no os.exec because otherwise we get the wrong return value + local code = os.execute(command) -- maybe spawn + if code == 0 then + return true + else + if binary then + binary = file.addsuffix(binary,os.binsuffix) + for p in gmatch(os.getenv("PATH"),"[^"..io.pathseparator.."]+") do + if lfs.isfile(file.join(p,binary)) then + return false + end + end + report() + report("This script needs '%s' which seems not to be installed.",binary) + report() + end + return false + end + end + end + end + end + return false +end + +function runners.execute_program(fullname) + local noquote = environment.argument("noquotes") + if fullname and fullname ~= "" then + local state = runners.prepare() + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + local before, after = environment.splitarguments(fullname) + for k=1,#after do after[k] = resolvers.resolve(after[k]) end + environment.initializearguments(after) + fullname = gsub(fullname,"^bin:","") + local command = fullname .. " " .. (environment.reconstructcommandline(after or "",noquote) or "") + report() + report("executing: %s",command) + report() + report() + io.flush() + local code = os.exec(command) -- (fullname,unpack(after)) does not work / maybe spawn + return code == 0 + end + end + return false +end + +-- the --usekpse flag will fallback (not default) on kpse (hm, we can better update mtx-stubs) + +local windows_stub = '@echo off\013\010setlocal\013\010set ownpath=%%~dp0%%\013\010texlua "%%ownpath%%mtxrun.lua" --usekpse --execute %s %%*\013\010endlocal\013\010' +local unix_stub = '#!/bin/sh\010mtxrun --usekpse --execute %s \"$@\"\010' + +function runners.handle_stubs(create) + local stubpath = environment.argument('stubpath') or '.' -- 'auto' no longer subpathssupported + local windows = environment.argument('windows') or environment.argument('mswin') or false + local unix = environment.argument('unix') or environment.argument('linux') or false + if not windows and not unix then + if os.platform == "unix" then + unix = true + else + windows = true + end + end + for _,v in pairs(runners.registered) do + local name, doit = v[1], v[2] + if doit then + local base = gsub(file.basename(name), "%.(.-)$", "") + if create then + if windows then + io.savedata(file.join(stubpath,base..".bat"),format(windows_stub,name)) + report("windows stub for '%s' created",base) + end + if unix then + io.savedata(file.join(stubpath,base),format(unix_stub,name)) + report("unix stub for '%s' created",base) + end + else + if windows and (os.remove(file.join(stubpath,base..'.bat')) or os.remove(file.join(stubpath,base..'.cmd'))) then + report("windows stub for '%s' removed", base) + end + if unix and (os.remove(file.join(stubpath,base)) or os.remove(file.join(stubpath,base..'.sh'))) then + report("unix stub for '%s' removed",base) + end + end + end + end +end + +function runners.resolve_string(filename) + if filename and filename ~= "" then + runners.report_location(resolvers.resolve(filename)) + end +end + +-- differs from texmfstart where locate appends .com .exe .bat ... todo + +function runners.locate_file(filename) -- was given file but only searches in tree + if filename and filename ~= "" then + if environment.argument("first") then + runners.report_location(resolvers.findfile(filename)) + -- resolvers.dowithfilesandreport(resolvers.findfile,filename) + elseif environment.argument("all") then + local result, status = resolvers.findfiles(filename) + if status and environment.argument("detail") then + runners.report_location(status) + else + runners.report_location(result) + end + else + runners.report_location(resolvers.findgivenfile(filename)) + -- resolvers.dowithfilesandreport(resolvers.findgivenfile,filename) + end + end +end + +function runners.locate_platform() + runners.report_location(os.platform) +end + +function runners.report_location(result) + if type(result) == "table" then + for i=1,#result do + if i > 1 then + io.write("\n") + end + io.write(result[i]) + end + else + io.write(result) + end +end + +function runners.edit_script(filename) -- we assume that gvim is present on most systems (todo: also in cnf file) + local editor = os.getenv("MTXRUN_EDITOR") or os.getenv("TEXMFSTART_EDITOR") or os.getenv("EDITOR") or 'gvim' + local rest = resolvers.resolve(filename) + if rest ~= "" then + local command = editor .. " " .. rest + if e_verbose then + report() + report("starting editor: %s",command) + report() + report() + end + os.launch(command) + end +end + +function runners.save_script_session(filename, list) + local t = { } + for i=1,#list do + local key = list[i] + t[key] = environment.arguments[key] + end + io.savedata(filename,table.serialize(t,true)) +end + +function runners.load_script_session(filename) + if lfs.isfile(filename) then + local t = io.loaddata(filename) + if t then + t = loadstring(t) + if t then t = t() end + for key, value in pairs(t) do + environment.arguments[key] = value + end + end + end +end + +function resolvers.launch(str) + -- maybe we also need to test on mtxrun.launcher.suffix environment + -- variable or on windows consult the assoc and ftype vars and such + local launchers = runners.launchers[os.platform] if launchers then + local suffix = file.suffix(str) if suffix then + local runner = launchers[suffix] if runner then + str = runner .. " " .. str + end + end + end + os.launch(str) +end + +function runners.launch_file(filename) + trackers.enable("resolvers.locating") + local allresults = environment.arguments["all"] + local pattern = environment.arguments["pattern"] + if not pattern or pattern == "" then + pattern = filename + end + if not pattern or pattern == "" then + report("provide name or --pattern=") + else + local t = resolvers.findfiles(pattern,nil,allresults) + if not t or #t == 0 then + t = resolvers.findfiles("*/" .. pattern,nil,allresults) + end + if not t or #t == 0 then + t = resolvers.findfiles("*/" .. pattern .. "*",nil,allresults) + end + if t and #t > 0 then + if allresults then + for _, v in pairs(t) do + report("launching %s", v) + resolvers.launch(v) + end + else + report("launching %s", t[1]) + resolvers.launch(t[1]) + end + else + report("no match for %s", pattern) + end + end +end + +local mtxprefixes = { + { "^mtx%-", "mtx-" }, + { "^mtx%-t%-", "mtx-t-" }, +} + +function runners.find_mtx_script(filename) + local function found(name) + local path = file.dirname(name) + if path and path ~= "" then + return false + else + local fullname = own and own.path and file.join(own.path,name) + return io.exists(fullname) and fullname + end + end + filename = file.addsuffix(filename,"lua") + local basename = file.removesuffix(file.basename(filename)) + local suffix = file.suffix(filename) + -- qualified path, raw name + local fullname = file.is_qualified_path(filename) and io.exists(filename) and filename + if fullname and fullname ~= "" then + return fullname + end + -- current path, raw name + fullname = "./" .. filename + fullname = io.exists(fullname) and fullname + if fullname and fullname ~= "" then + return fullname + end + -- mtx- prefix checking + for i=1,#mtxprefixes do + local mtxprefix = mtxprefixes[i] + mtxprefix = find(filename,mtxprefix[1]) and "" or mtxprefix[2] + -- context namespace, mtx-<filename> + fullname = mtxprefix .. filename + fullname = found(fullname) or resolvers.findfile(fullname) + if fullname and fullname ~= "" then + return fullname + end + -- context namespace, mtx-<filename>s + fullname = mtxprefix .. basename .. "s" .. "." .. suffix + fullname = found(fullname) or resolvers.findfile(fullname) + if fullname and fullname ~= "" then + return fullname + end + -- context namespace, mtx-<filename minus trailing s> + fullname = mtxprefix .. gsub(basename,"s$","") .. "." .. suffix + fullname = found(fullname) or resolvers.findfile(fullname) + if fullname and fullname ~= "" then + return fullname + end + end + -- context namespace, just <filename> + fullname = resolvers.findfile(filename) + return fullname +end + +function runners.register_arguments(...) + local arguments = environment.arguments_after + local passedon = { ... } + for i=#passedon,1,-1 do + local pi = passedon[i] + if pi then + table.insert(arguments,1,pi) + end + end +end + +function runners.execute_ctx_script(filename,...) + runners.register_arguments(...) + local arguments = environment.arguments_after + local fullname = runners.find_mtx_script(filename) or "" + if file.suffix(fullname) == "cld" then + -- handy in editors where we force --autopdf + report("running cld script: %s",filename) + table.insert(arguments,1,fullname) + table.insert(arguments,"--autopdf") + fullname = runners.find_mtx_script("context") or "" + end + -- retry after generate but only if --autogenerate + if fullname == "" and environment.argument("autogenerate") then -- might become the default + instance.renewcache = true + trackers.enable("resolvers.locating") + resolvers.load() + -- + fullname = runners.find_mtx_script(filename) or "" + end + -- that should do it + if fullname ~= "" then + local state = runners.prepare() + if state == 'error' then + return false + elseif state == 'skip' then + return true + elseif state == "run" then + -- load and save ... kind of undocumented + arg = { } for _,v in pairs(arguments) do arg[#arg+1] = resolvers.resolve(v) end + environment.initializearguments(arg) + local loadname = environment.arguments['load'] + if loadname then + if type(loadname) ~= "string" then loadname = file.basename(fullname) end + loadname = file.replacesuffix(loadname,"cfg") + runners.load_script_session(loadname) + end + filename = environment.files[1] + if e_verbose then + report("using script: %s\n",fullname) + end + environment.ownscript = fullname + dofile(fullname) + local savename = environment.arguments['save'] + if savename then + local save_list = runners.save_list + if save_list and next(save_list) then + if type(savename) ~= "string" then savename = file.basename(fullname) end + savename = file.replacesuffix(savename,"cfg") + runners.save_script_session(savename,save_list) + end + end + return true + end + else + if filename == "" or filename == "help" then + local context = resolvers.findfile("mtx-context.lua") + trackers.enable("resolvers.locating") + if context ~= "" then + local result = dir.glob((gsub(context,"mtx%-context","mtx-*"))) -- () needed + local valid = { } + table.sort(result) + for i=1,#result do + local scriptname = result[i] + local scriptbase = match(scriptname,".*mtx%-([^%-]-)%.lua") + if scriptbase then + local data = io.loaddata(scriptname) +local application = match(data,"local application.-=.-(%{.-%})") +if application then + application = loadstring("return " .. application) + if application then + application = application() + local banner = application.banner + if banner then + local description, version = match(banner,"^(.-) ([%d.]+)$") + if description then + valid[#valid+1] = { scriptbase, version, description } + else + valid[#valid+1] = { scriptbase, "", banner } + end + end + end +end + end + end + if #valid > 0 then + application.identify() + report("no script name given, known scripts:") + report() + for k=1,#valid do + local v = valid[k] + report("%-12s %4s %s",v[1],v[2],v[3]) + end + end + else + report("no script name given") + end + else + filename = file.addsuffix(filename,"lua") + if file.is_qualified_path(filename) then + report("unknown script '%s'",filename) + else + report("unknown script '%s' or 'mtx-%s'",filename,filename) + end + end + return false + end +end + +function runners.prefixes() + application.identify() + report() + report(concat(resolvers.allprefixes(true)," ")) +end + +function runners.timedrun(filename) -- just for me + if filename and filename ~= "" then + runners.timed(function() os.execute(filename) end) + end +end + +function runners.timed(action) + statistics.timed(action) +end + +function runners.associate(filename) + os.launch(filename) +end + +function runners.evaluate(code,filename) -- for Luigi + if code == "loop" then + while true do + io.write("> ") + local code = io.read() + if code ~= "" then + local temp = string.match(code,"^= (.*)$") + if temp then + code = "print("..temp..")" + end + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("! " .. (message or code).."\n") + else + io.write(compiled()) + end + end + end + else + if type(code) ~= "string" or code == "" then + code = filename + end + if code ~= "" then + local compiled, message = loadstring(code) + if type(compiled) ~= "function" then + io.write("invalid lua code: " .. (message or code)) + return + end + io.write(compiled()) + end + end +end + +function runners.gethelp(filename) + local url = environment.argument("url") + if url and url ~= "" then + local command = string.gsub(environment.argument("command") or "unknown","^%s*\\*(.-)%s*$","%1") + url = utilities.templates.replace(url,{ command = command }) + os.launch(url) + else + report("no --url given") + end +end + +function runners.systeminfo() + report("architecture : %s",os.platform or "<unset>") + report("operating system : %s",os.name or "<unset>") + report("file architecture : %s",os.type or "<unset>") + report("binary path : %s",os.selfdir or "<unset>") + report("binary suffix : %s",os.binsuffix or "<unset>") + report("library suffix : %s",os.libsuffix or "<unset>") +end + +-- this is a bit dirty ... first we store the first filename and next we +-- split the arguments so that we only see the ones meant for this script +-- ... later we will use the second half + +local filename = environment.files[1] or "" +local ok = true + +local before, after = environment.splitarguments(filename) +environment.arguments_before, environment.arguments_after = before, after +environment.initializearguments(before) + +instance.lsrmode = environment.argument("lsr") or false + +e_verbose = environment.arguments["verbose"] -- delayed till here (we need the ones before script) + +if e_verbose then + trackers.enable("resolvers.locating") +end + +-- maybe the unset has to go to this level + +local is_mkii_stub = runners.registered[file.removesuffix(file.basename(filename))] + +local e_argument = environment.argument + +if e_argument("timedlog") then + logs.settimedlog() +end + +if e_argument("usekpse") or e_argument("forcekpse") or is_mkii_stub then + + resolvers.load_tree(e_argument('tree'),true) -- force resolve of TEXMFCNF + + os.setenv("engine","") + os.setenv("progname","") + + local remapper = { + otf = "opentype fonts", + ttf = "truetype fonts", + ttc = "truetype fonts", + pfb = "type1 fonts", + other = "other text files", + } + + local progname = e_argument("progname") or 'context' + + local function kpse_initialized() + texconfig.kpse_init = true + local t = os.clock() + local k = kpse.original.new("luatex",progname) + local dummy = k:find_file("mtxrun.lua") -- so that we're initialized + report("kpse fallback with progname '%s' initialized in %s seconds",progname,os.clock()-t) + kpse_initialized = function() return k end + return k + end + + local findfile = resolvers.findfile + local showpath = resolvers.showpath + + if e_argument("forcekpse") then + + function resolvers.findfile(name,kind) + return (kpse_initialized():find_file(resolvers.cleanpath(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + end + function resolvers.showpath(name) + return (kpse_initialized():show_path(name)) or "" + end + + elseif e_argument("usekpse") or is_mkii_stub then + + resolvers.load() + + function resolvers.findfile(name,kind) + local found = findfile(name,kind) or "" + if found ~= "" then + return found + else + return (kpse_initialized():find_file(resolvers.cleanpath(name),(kind ~= "" and (remapper[kind] or kind)) or "tex") or "") or "" + end + end + function resolvers.showpath(name) + local found = showpath(name) or "" + if found ~= "" then + return found + else + return (kpse_initialized():show_path(name)) or "" + end + end + + end + + function runners.loadbase() + end + +else + + function runners.loadbase(...) + if not resolvers.load(...) then + report("forcing cache reload") + instance.renewcache = true + trackers.enable("resolvers.locating") + if not resolvers.load(...) then + report("the resolver databases are not present or outdated") + end + end + end + + resolvers.load_tree(e_argument('tree'),e_argument("resolve")) + +end + +-- joke .. reminds me of messing with gigi terminals + +if e_argument("ansi") then + + logs.setformatters("ansi") + + local script = e_argument("script") or e_argument("scripts") + + if type(script) == "string" then + logs.writer("]0;"..script.."") -- for Alan to test + end + +end + +if e_argument("script") or e_argument("scripts") then + + -- run a script by loading it (using libs), pass args + + if e_argument("nofiledatabase") then + -- handy for mtx-update + else + runners.loadbase() + end + if is_mkii_stub then + ok = runners.execute_script(filename,false,true) + else + ok = runners.execute_ctx_script(filename) + end + +elseif e_argument("evaluate") then + + runners.evaluate(e_argument("evaluate"),filename) + +elseif e_argument("selfmerge") then + + -- embed used libraries + + runners.loadbase() + local found = locate_libs() + + if found then + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfmerge(mtxrun,own.libs,{ found }) + application.report("runner updated on resolved path: %s",mtxrun) + else + utilities.merger.selfmerge(own.name,own.libs,{ found }) + application.report("runner updated on relative path: %s",own.name) + end + end + +elseif e_argument("selfclean") then + + -- remove embedded libraries + + runners.loadbase() + + local mtxrun = resolvers.findfile("mtxrun.lua") -- includes local name + if lfs.isfile(mtxrun) then + utilities.merger.selfclean(mtxrun) + application.report("runner cleaned on resolved path: %s",mtxrun) + else + utilities.merger.selfclean(own.name) + application.report("runner cleaned on relative path: %s",own.name) + end + +elseif e_argument("selfupdate") then + + runners.loadbase() + trackers.enable("resolvers.locating") + resolvers.updatescript(own.name,"mtxrun") + +elseif e_argument("ctxlua") or e_argument("internal") then + + -- run a script by loading it (using libs) + + runners.loadbase() + ok = runners.execute_script(filename,true) + +elseif e_argument("execute") then + + -- execute script + + runners.loadbase() + ok = runners.execute_script(filename) + +elseif e_argument("direct") then + + -- equals bin: + + runners.loadbase() + ok = runners.execute_program(filename) + +elseif e_argument("edit") then + + -- edit file + + runners.loadbase() + runners.edit_script(filename) + +elseif e_argument("launch") then + + runners.loadbase() + runners.launch_file(filename) + +elseif e_argument("associate") then + + runners.associate(filename) + +elseif e_argument("gethelp") then + + runners.gethelp() + +elseif e_argument("makestubs") then + + -- make stubs (depricated) + + runners.handle_stubs(true) + +elseif e_argument("removestubs") then + + -- remove stub (depricated) + + runners.loadbase() + runners.handle_stubs(false) + +elseif e_argument("resolve") then + + -- resolve string + + runners.loadbase() + runners.resolve_string(filename) + +elseif e_argument("locate") then + + -- locate file (only database) + + runners.loadbase() + runners.locate_file(filename) + +elseif e_argument("platform") or e_argument("show-platform") then + + -- locate platform + + runners.loadbase() + runners.locate_platform() + +elseif e_argument("prefixes") then + + runners.loadbase() + runners.prefixes() + +elseif e_argument("timedrun") then + + -- locate platform + + runners.loadbase() + runners.timedrun(filename) + +elseif e_argument("variables") or e_argument("show-variables") or e_argument("expansions") or e_argument("show-expansions") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expansions",filename) + + resolvers.load("nofiles") + resolvers.listers.variables(e_argument("pattern")) + +elseif e_argument("configurations") or e_argument("show-configurations") then + + -- luatools: runners.execute_ctx_script("mtx-base","--configurations",filename) + + resolvers.load("nofiles") + resolvers.listers.configurations() + +elseif e_argument("find-file") then + + -- luatools: runners.execute_ctx_script("mtx-base","--find-file",filename) + + resolvers.load() + local e_all = e_argument("all") + local e_pattern = e_argument("pattern") + local e_format = e_argument("format") + local finder = e_all and resolvers.findfiles or resolvers.findfile + if not e_pattern then + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(finder,environment.files,e_format) + elseif type(e_pattern) == "string" then + resolvers.dowithfilesandreport(finder,{ e_pattern },e_format) + end + +elseif e_argument("find-path") then + + -- luatools: runners.execute_ctx_script("mtx-base","--find-path",filename) + + resolvers.load() + local path = resolvers.findpath(filename, instance.my_format) + if e_verbose then + report(path) + else + print(path) + end + +elseif e_argument("expand-braces") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expand-braces",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.expandbraces, environment.files) + +elseif e_argument("expand-path") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expand-path",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) + +elseif e_argument("expand-var") or e_argument("expand-variable") then + + -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.expansion, environment.files) + +elseif e_argument("show-path") or e_argument("path-value") then + + -- luatools: runners.execute_ctx_script("mtx-base","--show-path",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.showpath, environment.files) + +elseif e_argument("var-value") or e_argument("show-value") then + + -- luatools: runners.execute_ctx_script("mtx-base","--show-value",filename) + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.variable,environment.files) + +elseif e_argument("format-path") then + + -- luatools: runners.execute_ctx_script("mtx-base","--format-path",filename) + + resolvers.load() + report(caches.getwritablepath("format")) + +elseif e_argument("pattern") then + + -- luatools + + runners.execute_ctx_script("mtx-base","--pattern='" .. e_argument("pattern") .. "'",filename) + +elseif e_argument("generate") then + + -- luatools + + if filename and filename ~= "" then + resolvers.load("nofiles") + trackers.enable("resolvers.locating") + resolvers.renew(filename) + else + instance.renewcache = true + trackers.enable("resolvers.locating") + resolvers.load() + end + + e_verbose = true + +elseif e_argument("make") or e_argument("ini") or e_argument("compile") then + + -- luatools: runners.execute_ctx_script("mtx-base","--make",filename) + + resolvers.load() + trackers.enable("resolvers.locating") + environment.make_format(filename) + +elseif e_argument("run") then + + -- luatools + + runners.execute_ctx_script("mtx-base","--run",filename) + +elseif e_argument("fmt") then + + -- luatools + + runners.execute_ctx_script("mtx-base","--fmt",filename) + +elseif e_argument("help") and filename=='base' then + + -- luatools + + runners.execute_ctx_script("mtx-base","--help") + +elseif e_argument("version") then + + application.version() + + application.report("source path",environment.ownbin) + +elseif e_argument("directives") then + + directives.show() + +elseif e_argument("trackers") then + + trackers.show() + +elseif e_argument("experiments") then + + experiments.show() + +elseif e_argument("exporthelp") then + + runners.loadbase() + application.export(e_argument("exporthelp"),filename) + +elseif e_argument("systeminfo") then + + runners.systeminfo() + +elseif e_argument("help") or filename=='help' or filename == "" then + + application.help() + +elseif find(filename,"^bin:") then + + runners.loadbase() + ok = runners.execute_program(filename) + +elseif is_mkii_stub then + + -- execute mkii script + + runners.loadbase() + ok = runners.execute_script(filename,false,true) + +elseif false then + + runners.loadbase() + ok = runners.execute_ctx_script(filename) + if not ok then + ok = runners.execute_script(filename) + end + +elseif environment.files[1] == 'texmfcnf.lua' then -- so that we don't need to load mtx-base + + resolvers.load("nofiles") + resolvers.listers.configurations() + +else + runners.loadbase() + runners.execute_ctx_script("mtx-base",filename) + +end + +if e_verbose then + report() + report("elapsed lua time: %0.3f seconds",os.runtime()) +end + +if os.type ~= "windows" then + texio.write("\n") -- is this still valid? +end + +if ok == false then ok = 1 elseif ok == true or ok == nil then ok = 0 end + +-- os.exit(ok,true) -- true forces a cleanup in 5.2+ + +os.exit(ok) -- true forces a cleanup in 5.2+ but reports a wrong number then diff --git a/scripts/context/stubs/win64/mtxrunjit.exe b/scripts/context/stubs/win64/mtxrunjit.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/mtxrunjit.exe diff --git a/scripts/context/stubs/win64/mtxworks.exe b/scripts/context/stubs/win64/mtxworks.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/mtxworks.exe diff --git a/scripts/context/stubs/win64/pstopdf.exe b/scripts/context/stubs/win64/pstopdf.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/pstopdf.exe diff --git a/scripts/context/stubs/win64/texexec.exe b/scripts/context/stubs/win64/texexec.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/texexec.exe diff --git a/scripts/context/stubs/win64/texmfstart.exe b/scripts/context/stubs/win64/texmfstart.exe Binary files differnew file mode 100644 index 000000000..93290a6e0 --- /dev/null +++ b/scripts/context/stubs/win64/texmfstart.exe |