diff options
author | Hans Hagen <pragma@wxs.nl> | 2017-05-14 19:58:50 +0200 |
---|---|---|
committer | Context Git Mirror Bot <phg42.2a@gmail.com> | 2017-05-14 19:58:50 +0200 |
commit | fd0c4577a4b6e85ca2db664906e1a03807ce133f (patch) | |
tree | fa23fcc04248d03ff82e34634b8ef1bb9cf28acb /scripts | |
parent | db581096187dc2d3cbdbe4cdc39d247c168b1607 (diff) | |
download | context-fd0c4577a4b6e85ca2db664906e1a03807ce133f.tar.gz |
2017-05-14 19:15:00
Diffstat (limited to 'scripts')
22 files changed, 10824 insertions, 3243 deletions
diff --git a/scripts/context/lua/mtx-cache.lua b/scripts/context/lua/mtx-cache.lua index bff1cb496..56d3df188 100644 --- a/scripts/context/lua/mtx-cache.lua +++ b/scripts/context/lua/mtx-cache.lua @@ -11,7 +11,7 @@ local helpinfo = [[ <application> <metadata> <entry name="name">mtx-cache</entry> - <entry name="detail">ConTeXt & MetaTeX Cache Management</entry> + <entry name="detail">ConTeXt & MetaTeX Cache Management</entry> <entry name="version">0.10</entry> </metadata> <flags> @@ -22,13 +22,19 @@ local helpinfo = [[ <flag name="list"><short>show cache</short></flag> </subcategory> <subcategory> - <flag name="all"><short>all (not yet implemented)</short></flag> + <flag name="fonts"><short>only wipe fonts</short></flag> </subcategory> </category> </flags> </application> ]] + +local find = string.find +local filesuffix, replacesuffix = file.suffix, file.replacesuffix +local isfile = lfs.isfile +local remove = os.remove + local application = logs.application { name = "mtx-cache", banner = "ConTeXt & MetaTeX Cache Management 0.10", @@ -45,7 +51,7 @@ local function collect(path) local tmas, tmcs, rest = { }, { }, { } for i=1,#all do local name = all[i] - local suffix = file.suffix(name) + local suffix = filesuffix(name) if suffix == "tma" then tmas[#tmas+1] = name elseif suffix == "tmc" then @@ -70,19 +76,22 @@ end local function purge(banner,path,list,all) report("%s: %s",banner,path) report() + local fonts = environment.argument("fonts") local n = 0 for i=1,#list do local filename = list[i] - if string.find(filename,"luatex%-cache") then -- safeguard - if all then - os.remove(filename) + if find(filename,"luatex%-cache") then -- safeguard + if fonts and not find(filename,"fonts") then + -- skip + elseif all then + remove(filename) n = n + 1 - else - local suffix = file.suffix(filename) + elseif not fonts or find(filename,"fonts") then + local suffix = filesuffix(filename) if suffix == "tma" then - local checkname = file.replacesuffix(filename,"tma","tmc") - if lfs.isfile(checkname) then - os.remove(filename) + local checkname = replacesuffix(filename,"tma","tmc") + if isfile(checkname) then + remove(filename) n = n + 1 end end diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 84254d570..6863cffb2 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -1,4 +1,4 @@ -if not modules then modules = { } end modules ['mtx-context'] = { +if not modules then modules = { } end modules['mtx-context'] = { version = 1.001, comment = "companion to mtxrun.lua", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -12,7 +12,7 @@ if not modules then modules = { } end modules ['mtx-context'] = { local type, next, tostring, tonumber = type, next, tostring, tonumber local format, gmatch, match, gsub, find = string.format, string.gmatch, string.match, string.gsub, string.find local quote, validstring = string.quote, string.valid -local concat = table.concat +local sort, concat, insert, sortedhash = table.sort, table.concat, table.insert, table.sortedhash local settings_to_array = utilities.parsers.settings_to_array local appendtable = table.append local lpegpatterns, lpegmatch, Cs, P = lpeg.patterns, lpeg.match, lpeg.Cs, lpeg.P @@ -34,7 +34,7 @@ local formatters = string.formatters local application = logs.application { name = "mtx-context", - banner = "ConTeXt Process Management 0.63", + banner = "ConTeXt Process Management 1.01", -- helpinfo = helpinfo, -- table with { category_a = text_1, category_b = text_2 } or helpstring or xml_blob helpinfo = "mtx-context.xml", } @@ -159,6 +159,67 @@ local defaultformats = { "cont-nl", } +-- purging files (we should have an mkii and mkiv variants) + +local generic_files = { + "texexec.tex", "texexec.tui", "texexec.tuo", + "texexec.tuc", "texexec.tua", + "texexec.ps", "texexec.pdf", "texexec.dvi", + "cont-opt.tex", "cont-opt.bak" +} + +local obsolete_results = { + "dvi", +} + +local temporary_runfiles = { + "tui", -- mkii two pass file + "tua", -- mkiv obsolete + "tup", "ted", "tes", -- texexec + "top", -- mkii options file + "log", -- tex log file + "tmp", -- mkii buffer file + "run", -- mkii stub + "bck", -- backup (obsolete) + "rlg", -- resource log + "ctl", -- + "mpt", "mpx", "mpd", "mpo", "mpb", -- metafun + "prep", -- context preprocessed + "pgf", -- tikz + "aux", "blg", -- bibtex +} + +local temporary_suffixes = { + "prep", -- context preprocessed +} +local synctex_runfiles = { + "synctex", "synctex.gz", "syncctx" -- synctex +} + +local persistent_runfiles = { + "tuo", -- mkii two pass file + "tub", -- mkii buffer file + "top", -- mkii options file + "tuc", -- mkiv two pass file + "bbl", -- bibtex +} + +local special_runfiles = { + "%-mpgraph", "%-mprun", "%-temp%-" +} + +local function purge_file(dfile,cfile) + if cfile and validfile(cfile) then + if removefile(dfile) then + return filebasename(dfile) + end + elseif dfile then + if removefile(dfile) then + return filebasename(dfile) + end + end +end + -- process information local ctxrunner = { } -- namespace will go @@ -520,10 +581,15 @@ local function run_texexec(filename,a_purge,a_purgeall) end end --- +-- context mode will become the only method some day -local function check_synctex(a_synctex) - return a_synctex and (tonumber(a_synctex) or (toboolean(a_synctex,true) and 1) or (a_synctex == "zipped" and 1) or (a_synctex == "unzipped" and -1)) or nil +local function check_synctex(a_synctex) -- context is intercepted elsewhere + return a_synctex and ( + tonumber(a_synctex) or + (toboolean(a_synctex,true) and 1) or + (a_synctex == "zipped" and 1) or + (a_synctex == "unzipped" and -1) + ) or nil end function scripts.context.run(ctxdata,filename) @@ -583,7 +649,7 @@ function scripts.context.run(ctxdata,filename) local a_nonstopmode = getargument("nonstopmode") local a_scollmode = getargument("scrollmode") local a_once = getargument("once") - local a_synctex = getargument("synctex") + local a_synctex = getargument("syncctx") and "context" or getargument("synctex") local a_backend = getargument("backend") local a_arrange = getargument("arrange") local a_noarrange = getargument("noarrange") @@ -609,11 +675,17 @@ function scripts.context.run(ctxdata,filename) -- a_batchmode = (a_batchmode and "batchmode") or (a_nonstopmode and "nonstopmode") or (a_scrollmode and "scrollmode") or nil - a_synctex = check_synctex(a_synctex) + -- a_synctex = check_synctex(a_synctex) -- for i=1,#filelist do -- local filename = filelist[i] + + if filename == "" then + report("warning: bad filename") + break + end + local basename = filebasename(filename) -- use splitter local pathname = filepathpart(filename) -- @@ -630,7 +702,7 @@ function scripts.context.run(ctxdata,filename) -- local analysis = preamble_analyze(filename) -- - a_synctex = a_synctex or check_synctex(analysis.synctex) + a_synctex = a_synctex or analysis.synctex -- if a_mkii or analysis.engine == 'pdftex' or analysis.engine == 'xetex' then run_texexec(filename,a_purge,a_purgeall) @@ -738,9 +810,9 @@ function scripts.context.run(ctxdata,filename) -- local l_flags = { ["interaction"] = a_batchmode, - ["synctex"] = a_synctex, - ["no-parse-first-line"] = true, -- obsolete - ["safer"] = a_safer, + ["synctex"] = check_synctex(a_synctex), -- otherwise not working + ["no-parse-first-line"] = true, -- obsolete + ["safer"] = a_safer, -- better use --sandbox -- ["no-mktex"] = true, -- ["file-line-error-style"] = true, ["fmt"] = formatfile, @@ -769,13 +841,16 @@ function scripts.context.run(ctxdata,filename) end -- if a_profile then - directives[#directives+1] = "system.profile" + directives[#directives+1] = format("system.profile=%s",tonumber(a_profile) or 0) end -- + for i=1,#synctex_runfiles do + removefile(fileaddsuffix(jobname,synctex_runfiles[i])) + end if a_synctex then - report("warning: synctex is enabled") -- can add upto 5% runtime directives[#directives+1] = format("system.synctex=%s",a_synctex) end + -- if #directives > 0 then c_flags.directives = concat(directives,",") end @@ -790,6 +865,7 @@ function scripts.context.run(ctxdata,filename) c_flags.forcedruns = multipass_forcedruns and multipass_forcedruns > 0 and multipass_forcedruns or nil c_flags.currentrun = currentrun c_flags.noarrange = a_noarrange or a_arrange or nil + c_flags.profile = a_profile and (tonumber(a_profile) or 0) or nil -- local command = luatex_command(l_flags,c_flags,mainfile,a_engine) -- @@ -829,6 +905,9 @@ function scripts.context.run(ctxdata,filename) -- end -- + if a_synctex == "context" then + renamefile(fileaddsuffix(jobname,"syncctx"),fileaddsuffix(jobname,"synctex")) + end if a_arrange then -- c_flags.final = true @@ -975,7 +1054,7 @@ function scripts.context.pipe() -- still used? end local function make_mkiv_format(name,engine) - environment.make_format(name,environment.arguments.silent) -- jit is picked up later + environment.make_format(name) -- jit is picked up later end local make_mkii_format @@ -990,7 +1069,7 @@ do -- more or less copied from mtx-plain.lua: else print("mktexlsr silent run") -- we use a basic print end - os.remove("temp.log") + removefile("temp.log") else report("running mktexlsr") os.execute("mktexlsr") @@ -998,7 +1077,7 @@ do -- more or less copied from mtx-plain.lua: end local function engine(texengine,texformat) - local command = string.format('%s --ini --etex --8bit %s \\dump',texengine,file.addsuffix(texformat,"mkii")) + local command = string.format('%s --ini --etex --8bit %s \\dump',texengine,fileaddsuffix(texformat,"mkii")) if environment.arguments.silent then statistics.starttiming() local command = format("%s > temp.log",command) @@ -1009,7 +1088,7 @@ do -- more or less copied from mtx-plain.lua: else print(format("%s silent make > format %q made in %.3f seconds",texengine,texformat,runtime)) -- we use a basic print end - os.remove("temp.log") + removefile("temp.log") else report("running command: %s",command) os.execute(command) @@ -1073,7 +1152,7 @@ do -- more or less copied from mtx-plain.lua: for k, v in next, environment.arguments do t[#t+1] = string.format("--mtx:%s=%s",k,v) end - execute('%s --fmt=%s %s "%s"',texengine,file.removesuffix(texformat),table.concat(t," "),filename) + execute('%s --fmt=%s %s "%s"',texengine,removesuffix(texformat),table.concat(t," "),filename) end make_mkii_format = function(name,engine) @@ -1218,67 +1297,6 @@ function scripts.context.version() end end --- purging files (we should have an mkii and mkiv variants) - -local generic_files = { - "texexec.tex", "texexec.tui", "texexec.tuo", - "texexec.tuc", "texexec.tua", - "texexec.ps", "texexec.pdf", "texexec.dvi", - "cont-opt.tex", "cont-opt.bak" -} - -local obsolete_results = { - "dvi", -} - -local temporary_runfiles = { - "tui", -- mkii two pass file - "tua", -- mkiv obsolete - "tup", "ted", "tes", -- texexec - "top", -- mkii options file - "log", -- tex log file - "tmp", -- mkii buffer file - "run", -- mkii stub - "bck", -- backup (obsolete) - "rlg", -- resource log - "ctl", -- - "mpt", "mpx", "mpd", "mpo", "mpb", -- metafun - "prep", -- context preprocessed - "pgf", -- tikz - "aux", "blg", -- bibtex -} - -local temporary_suffixes = { - "prep", -- context preprocessed -} -local synctex_runfiles = { - "synctex", "synctex.gz", -- synctex -} - -local persistent_runfiles = { - "tuo", -- mkii two pass file - "tub", -- mkii buffer file - "top", -- mkii options file - "tuc", -- mkiv two pass file - "bbl", -- bibtex -} - -local special_runfiles = { - "-mpgraph", "-mprun", "-temp-" -} - -local function purge_file(dfile,cfile) - if cfile and validfile(cfile) then - if removefile(dfile) then - return filebasename(dfile) - end - elseif dfile then - if removefile(dfile) then - return filebasename(dfile) - end - end -end - function scripts.context.purge_job(jobname,all,mkiitoo,fulljobname) if jobname and jobname ~= "" then jobname = filebasename(jobname) @@ -1438,6 +1456,7 @@ function scripts.context.modules(pattern) dir.glob(filejoinname(filepathpart(found,pattern)),list) end local done = { } -- todo : sort + local none = { x = { }, m = { }, s = { }, t = { } } for i=1,#list do local v = list[i] local base = filebasename(v) @@ -1445,7 +1464,7 @@ function scripts.context.modules(pattern) done[base] = true local suffix = filesuffix(base) if suffix == "tex" or suffix == "mkiv" or suffix == "mkvi" or suffix == "mkix" or suffix == "mkxi" then - local prefix = match(base,"^([xmst])%-") + local prefix, rest = match(base,"^([xmst])%-(.*)") if prefix then v = resolvers.findfile(base) -- so that files on my dev path are seen local data = io.loaddata(v) or "" @@ -1465,11 +1484,26 @@ function scripts.context.modules(pattern) end end report() + else + insert(none[prefix],rest) end end end end end + + local function show(k,v) + sort(v) + if #v > 0 then + report() + for i=1,#v do + report("%s : %s",k,v[i]) + end + end + end + for k, v in sortedhash(none) do + show(k,v) + end end -- extras @@ -1693,6 +1727,10 @@ end -- getting it done +if getargument("timedlog") then + logs.settimedlog() +end + if getargument("nostats") then setargument("nostatistics",true) setargument("nostat",nil) @@ -1720,10 +1758,10 @@ do end if getargument("once") then - multipass_nofruns = 1 + multipass_nofruns = 1 else if getargument("runs") then - multipass_nofruns = tonumber(getargument("runs")) or nil + multipass_nofruns = tonumber(getargument("runs")) or nil end multipass_forcedruns = tonumber(getargument("forcedruns")) or nil end @@ -1747,9 +1785,9 @@ elseif getargument("update") then scripts.context.update() elseif getargument("expert") then application.help("expert", "special") -elseif getargument("modules") then +elseif getargument("showmodules") or getargument("modules") then scripts.context.modules() -elseif getargument("extras") then +elseif getargument("showextras") or getargument("extras") then scripts.context.extras(environment.files[1] or getargument("extras")) elseif getargument("extra") then scripts.context.extra() diff --git a/scripts/context/lua/mtx-context.xml b/scripts/context/lua/mtx-context.xml index 2ba7ee59e..25e92d7a6 100644 --- a/scripts/context/lua/mtx-context.xml +++ b/scripts/context/lua/mtx-context.xml @@ -4,7 +4,7 @@ <metadata> <entry name="name">mtx-context</entry> <entry name="detail">ConTeXt Process Management</entry> - <entry name="version">0.62</entry> + <entry name="version">1.01</entry> <entry name="comment">external helpinfo file</entry> </metadata> <flags> @@ -126,7 +126,10 @@ <short>run without stopping</short> </flag> <flag name="synctex"> - <short>run with synctex enabled (optional value: zipped, unzipped, 1, -1)</short> + <short>run with synctex enabled (optional value: zipped, unzipped, 1, -1, context)</short> + </flag> + <flag name="syncttx"> + <short>a shortcut for --synctex=context</short> </flag> <flag name="nodates"> <short>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</short> diff --git a/scripts/context/lua/mtx-dvi.lua b/scripts/context/lua/mtx-dvi.lua new file mode 100644 index 000000000..57f146ee6 --- /dev/null +++ b/scripts/context/lua/mtx-dvi.lua @@ -0,0 +1,411 @@ +if not modules then modules = { } end modules ['mtx-dvi'] = { + version = 1.001, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This is just a tool that I use for checking dvi issues in LuaTeX and it has +-- no real use otherwise. When needed (or on request) I can extend this script. +-- Speed is hardly an issue and I didn't spend much time on generalizing the +-- code either. + +local formatters = string.formatters +local byte = string.byte + +local P, R, S, C, Cc, Ct, Cmt = lpeg.P, lpeg.R, lpeg.S, lpeg.C, lpeg.Cc, lpeg.Ct, lpeg.Cmt +local lpegmatch = lpeg.match + +local readbyte = utilities.files.readbyte +local readcardinal1 = utilities.files.readcardinal1 +local readcardinal2 = utilities.files.readcardinal2 +local readcardinal3 = utilities.files.readcardinal3 +local readcardinal4 = utilities.files.readcardinal4 +local readinteger1 = utilities.files.readinteger1 +local readinteger2 = utilities.files.readinteger2 +local readinteger3 = utilities.files.readinteger3 +local readinteger4 = utilities.files.readinteger4 +local readstring = utilities.files.readstring + +local helpinfo = [[ +<?xml version="1.0"?> +<application> + <metadata> + <entry name="name">mtx-dvi</entry> + <entry name="detail">ConTeXt DVI Helpers</entry> + <entry name="version">0.01</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="list"><short>list dvi commands</short></flag> + </subcategory> + </category> + </flags> +</application> +]] + +local application = logs.application { + name = "mtx-dvi", + banner = "ConTeXt DVI Helpers 0.10", + helpinfo = helpinfo, +} + +local report = application.report + +local f_set_char_fast = formatters["set char fast %C"] +local f_set_char = formatters["set char %C"] +local f_set_rule = formatters["set rule height=%p width=%p (%s %s)"] +local f_put_char = formatters["put char %C"] +local f_put_rule = formatters["put rule height=%p width=%p (%s %s)"] +local f_set_font = formatters["set font %i"] +local f_set_font_fast = formatters["set font fast %i"] +local f_right = formatters["right %p (%s)"] +local f_right_w = formatters["right w"] +local f_right_x = formatters["right x"] +local f_right_w_set = formatters["right set w %p (%s)"] +local f_right_x_set = formatters["right set x %p (%s)"] +local f_down = formatters["down %p (%s)"] +local f_down_y = formatters["down y"] +local f_down_z = formatters["down z"] +local f_down_y_set = formatters["down set y %p (%s)"] +local f_down_z_set = formatters["down set z %p (%s)"] +local f_page_begin = formatters["page begin (% t) %i"] +local f_page_end = formatters["page end"] +local f_nop = formatters["nop"] +local f_push = formatters["push %i"] +local f_pop = formatters["pop %i"] +local f_special = formatters["special %s"] +local f_preamble = formatters["preamble version=%s numerator=%s denominator=%s mag=%s comment=%s"] +local f_postamble_begin = formatters["postamble"] +local f_postamble_end = formatters["postamble end offset=%s version=%s"] +local f_define_font = formatters["define font k=%i checksum=%i scale=%p designsize=%p area=%s name=%s"] + +local currentdepth = 0 +local usedprinter = texio and texio.write_ln or print + +local handler = { } for i=0,255 do handler[i] = false end + +local function define_font(f,size) + local k = size == 1 and readcardinal1(f) + or size == 2 and readcardinal2(f) + or size == 3 and readcardinal3(f) + or readcardinal4(f) + local c = readcardinal4(f) + local s = readcardinal4(f) + local d = readcardinal4(f) + local a = readcardinal1(f) + local l = readcardinal1(f) + local a = readstring(f,a) + local l = readstring(f,l) + usedprinter(f_define_font(k,c,s,d,area,name)) +end + +handler[000] = function(f,b) + usedprinter(f_set_char_fast(b)) +end + +handler[128] = function(f) + usedprinter(f_set_char(readcardinal1(f))) +end +handler[129] = function(f) + usedprinter(f_set_char(readcardinal2(f))) +end +handler[130] = function(f) + usedprinter(f_set_char(readcardinal3(f))) +end +handler[131] = function(f) + usedprinter(f_set_char(readcardinal4(f))) +end + +handler[132] = function(f) + usedprinter(f_set_rule(readinteger4(f),readinteger4(f))) +end + +handler[133] = function(f) + usedprinter(f_put_char(readcardinal1(f))) +end +handler[134] = function(f) + usedprinter(f_put_char(readcardinal2(f))) +end +handler[135] = function(f) + usedprinter(f_put_char(readcardinal3(f))) +end +handler[136] = function(f) + usedprinter(f_put_char(readcardinal4(f))) +end + +handler[137] = function(f) + usedprinter(f_put_rule(readinteger4(f),readinteger4(f))) +end + +handler[138] = function(f) + usedprinter(f_nop()) +end + +handler[139] = function(f) + local pages = { } + for i=0,9 do + pages[i] = readinteger4(f) + end + usedprinter(f_page_begin(pages,readinteger4(f))) +end +handler[140] = function() + usedprinter(f_page_end()) +end + +handler[141] = function() + currentdepth = currentdepth + 1 + usedprinter(f_push(currentdepth)) +end +handler[142] = function() + usedprinter(f_pop(currentdepth)) + currentdepth = currentdepth - 1 +end + +handler[143] = function(f) + local d = readinteger1(f) + usedprinter(f_right(d,d)) +end +handler[144] = function(f) + local d = readinteger2(f) + usedprinter(f_right(d,d)) +end +handler[145] = function(f) + local d = readinteger3(f) + usedprinter(f_right(d,d)) +end +handler[146] = function(f) + local d = readinteger4(f) + usedprinter(f_right(d,d)) +end + +handler[147] = function() + usedprinter(f_right_w()) +end + +handler[148] = function(f) + local d = readinteger1(f) + usedprinter(f_right_w_set(d,d)) +end +handler[149] = function(f) + local d = readinteger2(f) + usedprinter(f_right_w_set(d,d)) +end +handler[150] = function(f) + local d = readinteger3(f) + usedprinter(f_right_w_set(d,d)) +end +handler[151] = function(f) + local d = readinteger4(f) + usedprinter(f_right_w_set(d,d)) +end + +handler[152] = function() + handlers.right_x() +end + +handler[153] = function(f) + local d = readinteger1(f) + usedprinter(f_right_x_set(d,d)) +end +handler[154] = function(f) + local d = readinteger2(f) + usedprinter(f_right_x_set(d,d)) +end +handler[155] = function(f) + local d = readinteger3(f) + usedprinter(f_right_x_set(d,d)) +end +handler[156] = function(f) + local d = readinteger4(f) + usedprinter(f_right_x_set(d,d)) +end + +handler[157] = function(f) + local d = readinteger1(f) + usedprinter(f_down(d,d)) +end + +handler[158] = function(f) + local d = readinteger2(f) + usedprinter(f_down(d,d)) +end +handler[159] = function(f) + local d = readinteger3(f) + usedprinter(f_down(d,d)) +end +handler[160] = function(f) + local d = readinteger4(f) + usedprinter(f_down(d,d)) +end +handler[161] = function() + usedprinter(f_down_y()) +end + +handler[162] = function(f) + local d = readinteger1(f) + usedprinter(f_down_y_set(d,d)) +end +handler[163] = function(f) + local d = readinteger2(f) + usedprinter(f_down_y_set(d,d)) +end +handler[164] = function(f) + local d = readinteger3(f) + usedprinter(f_down_y_set(d,d)) +end +handler[165] = function(f) + local d = readinteger4(f) + usedprinter(f_down_y_set(d,d)) +end + +handler[166] = function() + handlers.down_z() +end + +handler[167] = function(f) + local d = readinteger1(f) + usedprinter(f_down_z_set(d,d)) +end +handler[168] = function(f) + local d = readinteger2(f) + usedprinter(f_down_z_set(d,d)) +end +handler[169] = function(f) + local d = readinteger3(f) + usedprinter(f_down_z_set(d,d)) +end +handler[170] = function(f) + local d = readinteger4(f) + usedprinter(f_down_z_set(d,d)) +end + +handler[171] = function(f,b) + usedprinter(f_set_font_fast(b)) +end + +handler[235] = function(f) + usedprinter(f_set_font(readcardinal1(f))) +end +handler[236] = function(f) + usedprinter(f_set_font(readcardinal2(f))) +end +handler[237] = function(f) + usedprinter(f_set_font(readcardinal3(f))) +end +handler[238] = function(f) + usedprinter(f_set_font(readcardinal4(f))) +end + +handler[239] = function(f) + usedprinter(f_special(readstring(readcardinal1(f)))) +end +handler[240] = function(f) + usedprinter(f_special(readstring(readcardinal2(f)))) +end +handler[241] = function(f) + usedprinter(f_special(readstring(readcardinal3(f)))) +end +handler[242] = function(f) + usedprinter(f_special(readstring(readcardinal4(f)))) +end + +handler[243] = function(f) + define_font(f,1) +end +handler[244] = function(f) + define_font(f,2) +end +handler[245] = function(f) + define_font(f,3) +end +handler[246] = function(f) + define_font(f,4) +end + +handler[247] = function(f) + usedprinter(f_preamble( + readcardinal1(f), + readcardinal4(f), + readcardinal4(f), + readcardinal4(f), + readstring(f,readcardinal1(f)) + )) +end + +handler[248] = function(f) + usedprinter(f_postamble_begin( + readcardinal4(f), -- p + readcardinal4(f), -- num + readcardinal4(f), -- den + readcardinal4(f), -- mag + readcardinal4(f), -- l + readcardinal4(f), -- u + readcardinal2(f), -- s + readcardinal2(f) -- t + )) + while true do + local b = readbyte(f) + if b == 249 then + break + else + handler[b](f,b) + end + end + usedprinter(f_postamble_end( + readcardinal4(f), + readcardinal1(f) + )) + -- now 223's follow +end + +handler[250] = function() +end + +for i= 1,127 do handler[i] = handler[ 0] end +for i= 172,234 do handler[i] = handler[171] end +for i= 251,255 do handler[i] = handler[250] end + +scripts = scripts or { } +scripts.dvi = scripts.dvi or { } + +function scripts.dvi.list(filename,printer) + currentdepth = 0 + local f = io.open(filename) + if f then + local filesize = f:seek("end") + local position = 0 + f:seek("set",position) + local format = formatters["%0" .. #tostring(filesize) .. "i : %s"] + local flush = printer or (texio and texio.write_ln) or print + usedprinter = function(str) + flush(format(position,str)) + position = f:seek() + end + while true do + local b = readbyte(f) + if b == 223 then + return + else + handler[b](f,b) + end + end + f:close() + else + report("invalid filename %a",filename) + end +end + +local filename = environment.files[1] or "" + +if filename == "" then + application.help() +elseif environment.argument("list") then + scripts.dvi.list(filename) +elseif environment.argument("exporthelp") then + application.export(environment.argument("exporthelp"),filename) +else + application.help() +end diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua index 0c3224fe0..068d38a84 100644 --- a/scripts/context/lua/mtx-fonts.lua +++ b/scripts/context/lua/mtx-fonts.lua @@ -13,9 +13,11 @@ local givenfiles = environment.files local suffix, addsuffix, removesuffix, replacesuffix = file.suffix, file.addsuffix, file.removesuffix, file.replacesuffix local nameonly, basename, joinpath, collapsepath = file.nameonly, file.basename, file.join, file.collapsepath local lower = string.lower +local concat = table.concat +local write_nl = texio.write_nl -local otfversion = 2.825 -local otlversion = 3.020 +local otfversion = 2.826 +local otlversion = 3.029 local helpinfo = [[ <?xml version="1.0"?> @@ -118,8 +120,8 @@ loadmodule("font-otr.lua") loadmodule("font-cff.lua") loadmodule("font-ttf.lua") loadmodule("font-tmp.lua") -loadmodule("font-dsp.lua") -- not yet in distribution -loadmodule("font-oup.lua") -- not yet in distribution +loadmodule("font-dsp.lua") +loadmodule("font-oup.lua") loadmodule("font-onr.lua") @@ -228,14 +230,6 @@ function scripts.fonts.reload() end end -local function subfont(sf) - if sf then - return string.format("index: % 2s", sf) - else - return "" - end -end - local function fontweight(fw) if fw then return string.format("conflict: %s", fw) @@ -262,9 +256,14 @@ local function showfeatures(tag,specification) indeed("style : %s",specification.style or "<nostyle>") indeed("width : %s",specification.width or "<nowidth>") indeed("variant : %s",specification.variant or "<novariant>") - indeed("subfont : %s",subfont(specification.subfont)) + indeed("subfont : %s",specification.subfont or "") indeed("fweight : %s",fontweight(specification.fontweight)) -- maybe more + local instancenames = specification.instancenames + if instancenames then + report() + indeed("instances : % t",instancenames) + end local features = fonts.helpers.getfeatures(specification.filename,not getargument("nosave")) if features then for what, v in table.sortedhash(features) do @@ -285,7 +284,7 @@ local function showfeatures(tag,specification) else done = true end - report("% -8s % -8s % -8s",f,s,table.concat(table.sortedkeys(ss), " ")) -- todo: padd 4 + report("% -8s % -8s % -8s",f,s,concat(table.sortedkeys(ss), " ")) -- todo: padd 4 end end end @@ -325,13 +324,15 @@ local function list_specifications(t,info) entry.variant or "<novariant>", entry.fontname, entry.filename, - subfont(entry.subfont), + entry.subfont or "", fontweight(entry.fontweight), } end + table.insert(s,1,{"familyname","weight","style","width","variant","fontname","filename","subfont","fontweight"}) + table.insert(s,2,{"","","","","","","","",""}) utilities.formatters.formatcolumns(s) for k=1,#s do - texio.write_nl(s[k]) + write_nl(s[k]) end end end @@ -339,7 +340,7 @@ end local function list_matches(t,info) if t then - local s, w = table.sortedkeys(t), { 0, 0, 0 } + local s, w = table.sortedkeys(t), { 0, 0, 0, 0 } if info then for k=1,#s do local v = s[k] @@ -351,14 +352,18 @@ local function list_matches(t,info) local entry = t[v] s[k] = { v, + entry.familyname, entry.fontname, entry.filename, - subfont(entry.subfont) + tostring(entry.subfont or ""), + concat(entry.instancenames or { }, " "), } end + table.insert(s,1,{"identifier","familyname","fontname","filename","subfont","instances"}) + table.insert(s,2,{"","","","","","",""}) utilities.formatters.formatcolumns(s) for k=1,#s do - texio.write_nl(s[k]) + write_nl(s[k]) end end end @@ -452,7 +457,7 @@ function scripts.fonts.unpack() if data.creator == "context mkiv" then fonts.handlers.otf.readers.unpack(data) else - fonts.handlers.otf.enhancers.unpack(data) + fonts.handlers.otf.unpackdata(data) end io.savedata(savename,table.serialize(data,true)) else diff --git a/scripts/context/lua/mtx-grep.lua b/scripts/context/lua/mtx-grep.lua index d0f33cb21..0c4364630 100644 --- a/scripts/context/lua/mtx-grep.lua +++ b/scripts/context/lua/mtx-grep.lua @@ -6,6 +6,9 @@ if not modules then modules = { } end modules ['mtx-babel'] = { license = "see context related readme files" } +-- If needed this one can be optimized for speed as well as use some existing +-- helpers. We can quit faster on max, and probably use lpeg instead of find. + local helpinfo = [[ <?xml version="1.0"?> <application> @@ -20,10 +23,26 @@ local helpinfo = [[ <flag name="pattern"><short>search for pattern (optional)</short></flag> <flag name="count"><short>count matches only</short></flag> <flag name="nocomment"><short>skip lines that start with %% or #</short></flag> + <flag name="n"><short>show at most n matches</short></flag> + <flag name="first"><short>only show first match</short></flag> + <flag name="match"><short>return the match (if it is one)</short></flag> <flag name="xml"><short>pattern is lpath expression</short></flag> </subcategory> </category> </flags> + <examples> + <category> + <title>Examples</title> + <subcategory> + <example><command>mtxrun --script grep --pattern=module *.mkiv</command></example> + <example><command>mtxrun --script grep --pattern="modules.-%['(.-)'%]" char-*.lua --first</command></example> + <example><command>mtxrun --script grep --pattern=module --count *.mkiv</command></example> + <example><command>mtxrun --script grep --pattern=module --first *.mkiv</command></example> + <example><command>mtxrun --script grep --pattern=module --nocomment *.mkiv</command></example> + <example><command>mtxrun --script grep --pattern=module --n=10 *.mkiv</command></example> + </subcategory> + </category> + </examples> <comments> <comment>patterns are lua patterns and need to be escaped accordingly</comment> </comments> @@ -41,7 +60,8 @@ local report = application.report scripts = scripts or { } scripts.grep = scripts.grep or { } -local find, format = string.find, string.format +local find, match, format = string.find, string.match, string.format +local lpegmatch = lpeg.match local cr = lpeg.P("\r") local lf = lpeg.P("\n") @@ -59,14 +79,17 @@ function scripts.grep.find(pattern, files, offset) if pattern and pattern ~= "" then statistics.starttiming(scripts.grep) local nofmatches, noffiles, nofmatchedfiles = 0, 0, 0 - local n, m, name, check = 0, 0, "", nil - local count, nocomment = environment.argument("count"), environment.argument("nocomment") + local n, m, check = 0, 0, nil + local name = "" + local count = environment.argument("count") + local nocomment = environment.argument("nocomment") + local max = tonumber(environment.argument("n")) or (environment.argument("first") and 1) or false + local domatch = environment.argument("match") if environment.argument("xml") then for i=offset or 1, #files do local globbed = dir.glob(files[i]) for i=1,#globbed do - local nam = globbed[i] - name = nam + name = globbed[i] local data = xml.load(name) if data and not data.error then n, m, noffiles = 0, 0, noffiles + 1 @@ -83,7 +106,9 @@ function scripts.grep.find(pattern, files, offset) else for c in xml.collected(data,pattern) do m = m + 1 - write_nl(format("%s: %s",name,xml.tostring(c))) + if not max or m <= max then + write_nl(format("%s: %s",name,xml.tostring(c))) + end end end end @@ -107,8 +132,14 @@ function scripts.grep.find(pattern, files, offset) -- skip elseif find(line,pattern) then m = m + 1 - write_nl(format("%s %6i: %s",name,n,line)) - io.flush() + if not max or m <= max then + if domatch then + write_nl(match(line,pattern)) + else + write_nl(format("%s %6i: %s",name,n,line)) + end + io.flush() + end end end end @@ -125,22 +156,27 @@ function scripts.grep.find(pattern, files, offset) n = n + 1 if find(line,pattern) then m = m + 1 - write_nl(format("%s %6i: %s",name,n,line)) - io.flush() + if not max or m <= max then + if domatch then + write_nl(match(line,pattern)) + else + write_nl(format("%s %6i: %s",name,n,line)) + end + io.flush() + end end end end end - local capture = (content/check)^0 + local capture = (content/check)^0 -- todo: break out when max for i=offset or 1, #files do local globbed = dir.glob(files[i]) for i=1,#globbed do - local nam = globbed[i] - name = nam + name = globbed[i] local data = io.loaddata(name) if data then n, m, noffiles = 0, 0, noffiles + 1 - capture:match(data) + lpegmatch(capture,data) if count and m > 0 then nofmatches = nofmatches + m nofmatchedfiles = nofmatchedfiles + 1 diff --git a/scripts/context/lua/mtx-interface.lua b/scripts/context/lua/mtx-interface.lua index b82dd5abf..876a073e8 100644 --- a/scripts/context/lua/mtx-interface.lua +++ b/scripts/context/lua/mtx-interface.lua @@ -9,6 +9,7 @@ if not modules then modules = { } end modules ['mtx-cache'] = { local concat, sort, insert = table.concat, table.sort, table.insert local gsub, format, gmatch, find = string.gsub, string.format, string.gmatch, string.find local utfchar, utfgsub = utf.char, utf.gsub +local sortedkeys = table.sortedkeys local helpinfo = [[ <?xml version="1.0"?> @@ -226,167 +227,16 @@ function flushers.textpad(collected) end end --- function scripts.interface.editor(editor,split,forcedinterfaces) --- local interfaces= forcedinterfaces or environment.files --- if #interfaces == 0 then --- interfaces= userinterfaces --- end --- local xmlfile = resolvers.findfile("cont-en.xml") or "" --- if xmlfile == "" then --- report("unable to locate cont-en.xml") --- end --- local collected = { } --- for i=1,#interfaces do --- local interface = interfaces[i] --- local keyfile = resolvers.findfile(format("keys-%s.xml",interface)) or "" --- if keyfile == "" then --- report("unable to locate keys-*.xml") --- else --- local commands = { } --- local mappings = { } --- local environments = { } --- local x = xml.load(keyfile) --- for e, d, k in xml.elements(x,"/cd:interface/cd:commands/cd:command") do -- somehow this was variable --- local at = d[k].at --- local name, value = at.name, at.value --- if name and value then --- mappings[name] = value --- end --- end --- local x = xml.load(xmlfile) --- for e, d, k in xml.elements(x,"/cd:interface/cd:command") do --- local at = d[k].at --- local name, type = at.name, at["type"] --- if name and name ~= "" then --- local remapped = mappings[name] or name --- if type == "environment" then --- if split then --- environments[#environments+1] = remapped --- else --- commands[#commands+1] = "start" .. remapped --- commands[#commands+1] = "stop" .. remapped --- end --- else --- commands[#commands+1] = remapped --- end --- end --- end --- if #commands > 0 then --- sort(commands) --- sort(environments) --- collected[interface] = { --- commands = commands, --- environments = environments, --- } --- end --- end --- end --- -- awaiting completion of the xml file --- local definitions = dofile(resolvers.findfile("mult-def.lua")) --- if definitions then --- local commands = { en = { } } --- for command, languages in next, definitions.commands do --- commands.en[languages.en or command] = true --- for language, command in next, languages do --- local c = commands[language] --- if c then --- c[command] = true --- else --- commands[language] = { [command] = true } --- end --- end --- end --- for language, data in next, commands do --- local fromlua = data --- local fromxml = collected[language].commands --- for i=1,#fromxml do --- local c = fromxml[i] --- if not fromlua[c] then --- -- print(language,c) --- fromlua[c] = true --- end --- end --- collected[language].commands = table.sortedkeys(fromlua) --- end --- end --- -- --- flushers[editor](collected) --- end - --- function scripts.interface.editor(editor,split,forcedinterfaces) --- local interfaces= forcedinterfaces or environment.files --- if #interfaces == 0 then --- interfaces= userinterfaces --- end --- -- --- local filename = "context-en.xml" --- local xmlfile = resolvers.findfile(filename) or "" --- if xmlfile == "" then --- report("unable to locate %a",filename) --- return --- end --- local x = xml.load(xmlfile) --- -- --- local filename = "mult-def.lua" --- local deffile = resolvers.findfile(filename) or "" --- if deffile == "" then --- report("unable to locate %a",filename) --- return --- end --- local interface = dofile(filename) --- if not interface or not next(interface) then --- report("invalid file %a",filename) --- return --- end --- local variables = interface.variables --- local constants = interface.constants --- local commands = interface.commands --- local elements = interface.elements --- local collected = { } --- for i=1,#interfaces do --- local interface = interfaces[i] --- local i_commands = { } --- local i_environments = { } --- local start = elements.start[interface] or elements.start.en --- local stop = elements.stop [interface] or elements.stop .en --- for e, d, k in xml.elements(x,"cd:interface/cd:command") do --- local at = d[k].at --- local name = at["name"] or "" --- local type = at["type"] --- if name ~= "" then --- local c = commands[name] --- local n = c and (c[interface] or c.en) or name --- if type ~= "environment" then --- i_commands[#i_commands+1] = n --- elseif split then --- i_environments[#i_environments+1] = n --- else --- -- variables ? --- i_commands[#i_commands+1] = start .. n --- i_commands[#i_commands+1] = stop .. n --- end --- end --- end --- if #i_commands > 0 then --- sort(i_commands) --- sort(i_environments) --- collected[interface] = { --- commands = i_commands, --- environments = i_environments, --- } --- end --- end --- -- --- flushers[editor](collected) --- end +-- we could instead load context-en.xml function scripts.interface.editor(editor,split,forcedinterfaces) - local interfaces= forcedinterfaces or environment.files + local interfaces = forcedinterfaces or environment.files if #interfaces == 0 then interfaces= userinterfaces end -- - local filename = "i-context.xml" +-- local filename = "i-context.xml" + local filename = "context-en.xml" local xmlfile = resolvers.findfile(filename) or "" if xmlfile == "" then report("unable to locate %a",filename) @@ -414,14 +264,33 @@ function scripts.interface.editor(editor,split,forcedinterfaces) report("generating files for %a",editor) report("loading %a",xmlfile) local xmlroot = xml.load(xmlfile) - xml.include(xmlroot,"cd:interfacefile","filename",true,function(s) - local fullname = resolvers.findfile(s) - if fullname and fullname ~= "" then - report("including %a",fullname) - return io.loaddata(fullname) - end - end) +-- xml.include(xmlroot,"cd:interfacefile","filename",true,function(s) +-- local fullname = resolvers.findfile(s) +-- if fullname and fullname ~= "" then +-- report("including %a",fullname) +-- return io.loaddata(fullname) +-- end +-- end) +-- local definitions = { } +-- for e in xml.collected(xmlroot,"cd:interface/cd:define") do +-- definitions[e.at.name] = e.dt +-- end +-- local function resolve(root) +-- for e in xml.collected(root,"*") do +-- if e.tg == "resolve" then +-- local resolved = definitions[e.at.name or ""] +-- if resolved then +-- -- use proper replace helper +-- e.__p__.dt[e.ni] = resolved +-- resolved.__p__ = e.__p__ +-- resolve(resolved) +-- end +-- end +-- end +-- end +-- resolve(xmlroot) -- + for i=1,#interfaces do local interface = interfaces[i] local i_commands = { } @@ -436,24 +305,37 @@ function scripts.interface.editor(editor,split,forcedinterfaces) local c = commands[name] local n = c and (c[interface] or c.en) or name if at.generated == "yes" then + for e in xml.collected(e,"/cd:instances/cd:constant") do + local name = e.at.value + if name then + local c = variables[name] + local n = c and (c[interface] or c.en) or name + if type == "environment" then + i_commands[n] = true + elseif split then + i_environments[n] = true + else + i_commands[start..n] = true + i_commands[stop ..n] = true + end + end + end -- skip (for now) elseif type ~= "environment" then - i_commands[#i_commands+1] = n + i_commands[n] = true elseif split then - i_environments[#i_environments+1] = n + i_environments[n] = true else -- variables ? - i_commands[#i_commands+1] = start .. n - i_commands[#i_commands+1] = stop .. n + i_commands[start..n] = true + i_commands[stop ..n] = true end end end - if #i_commands > 0 then - sort(i_commands) - sort(i_environments) + if next(i_commands) then collected[interface] = { - commands = i_commands, - environments = i_environments, + commands = sortedkeys(i_commands), + environments = sortedkeys(i_environments), } end end @@ -500,7 +382,7 @@ function scripts.interface.mkii() texresult[#texresult+1] = format("%% definitions for interface %s for language %s\n%%",what,language) xmlresult[#xmlresult+1] = format("\t<!-- definitions for interface %s for language %s -->\n",what,language) xmlresult[#xmlresult+1] = format("\t<cd:%s>",what) - local sorted = table.sortedkeys(t) + local sorted = sortedkeys(t) for i=1,#sorted do local key = sorted[i] local v = t[key] diff --git a/scripts/context/lua/mtx-patterns.lua b/scripts/context/lua/mtx-patterns.lua index e8d4d3e65..d16912e4f 100644 --- a/scripts/context/lua/mtx-patterns.lua +++ b/scripts/context/lua/mtx-patterns.lua @@ -694,16 +694,37 @@ end -- -- use this call: -- --- mtxrun --script pattern --convert --path=c:/data/develop/svn-hyphen/trunk/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=e:/tmp/patterns - --- copy /Y *.hyp e:\tex-context\tex\texmf-context\tex\context\patterns --- copy /Y *.pat e:\tex-context\tex\texmf-context\tex\context\patterns --- copy /Y *.rme e:\tex-context\tex\texmf-context\tex\context\patterns --- copy /Y *.lua e:\tex-context\tex\texmf-context\tex\context\patterns - --- move /Y *.hyp e:\tex-context\tex\texmf-mine\tex\context\patterns --- move /Y *.pat e:\tex-context\tex\texmf-mine\tex\context\patterns --- move /Y *.rme e:\tex-context\tex\texmf-mine\tex\context\patterns --- move /Y *.lua e:\tex-context\tex\texmf-mine\tex\context\patterns - +-- setlocal +-- +-- rem tugsvn checkout: +-- +-- set patternsroot=c:/data/develop/svn-hyphen/trunk +-- +-- rem github checkout: +-- +-- set patternsroot=c:/data/repositories/tex-hyphen +-- +-- del /q c:\data\develop\tex-context\tex\texmf-local\tex\context\patterns\* +-- del /q c:\data\develop\tex-context\tex\texmf-mine\tex\context\patterns\* +-- del /q c:\data\develop\tex-context\tex\texmf-context\tex\context\patterns\* +-- +-- mtxrun --script pattern --convert --path=%patternsroot%/hyph-utf8/tex/generic/hyph-utf8/patterns/txt --destination=c:/data/develop/tmp/patterns +-- +-- copy /Y lang*.hyp c:\data\develop\tex-context\tex\texmf-context\tex\context\patterns +-- copy /Y lang*.pat c:\data\develop\tex-context\tex\texmf-context\tex\context\patterns +-- copy /Y lang*.rme c:\data\develop\tex-context\tex\texmf-context\tex\context\patterns +-- copy /Y lang*.lua c:\data\develop\tex-context\tex\texmf-context\tex\context\patterns +-- +-- move /Y lang*.hyp c:\data\develop\tex-context\tex\texmf-mine\tex\context\patterns +-- move /Y lang*.pat c:\data\develop\tex-context\tex\texmf-mine\tex\context\patterns +-- move /Y lang*.rme c:\data\develop\tex-context\tex\texmf-mine\tex\context\patterns +-- move /Y lang*.lua c:\data\develop\tex-context\tex\texmf-mine\tex\context\patterns +-- -- mtxrun --script pattern --words --update word-th.lua --compress +-- +-- copy /Y word*.lua c:\data\develop\tex-context\tex\texmf-context\tex\context\patterns +-- move /Y word*.lua c:\data\develop\tex-context\tex\texmf-mine\tex\context\patterns +-- +-- mtxrun --generate +-- +-- endlocal diff --git a/scripts/context/lua/mtx-pdf.lua b/scripts/context/lua/mtx-pdf.lua index 551aa5b37..0d3f37ad5 100644 --- a/scripts/context/lua/mtx-pdf.lua +++ b/scripts/context/lua/mtx-pdf.lua @@ -112,8 +112,12 @@ local function getfonts(pdffile) for i=1,pdffile.pages.n do local page = pdffile.pages[i] local fontlist = page.Resources.Font - for k, v in next, lpdf.epdf.expand(fontlist) do - usedfonts[k] = lpdf.epdf.expand(v) + if fontlist then + for k, v in next, lpdf.epdf.expand(fontlist) do + usedfonts[k] = lpdf.epdf.expand(v) + end + else + report("no fonts") end end return usedfonts diff --git a/scripts/context/lua/mtx-server-ctx-fonttest.lua b/scripts/context/lua/mtx-server-ctx-fonttest.lua index da87fe9e8..ab324d3cb 100644 --- a/scripts/context/lua/mtx-server-ctx-fonttest.lua +++ b/scripts/context/lua/mtx-server-ctx-fonttest.lua @@ -11,10 +11,15 @@ if not modules then modules = { } end modules ['mtx-server-ctx-fonttest'] = { dofile(resolvers.findfile("trac-lmx.lua","tex")) dofile(resolvers.findfile("font-ini.lua","tex")) dofile(resolvers.findfile("font-con.lua","tex")) -dofile(resolvers.findfile("font-oti.lua","tex")) -dofile(resolvers.findfile("font-otf.lua","tex")) -dofile(resolvers.findfile("font-otp.lua","tex")) -dofile(resolvers.findfile("font-ott.lua","tex")) +-- dofile(resolvers.findfile("font-oti.lua","tex")) +-- dofile(resolvers.findfile("font-otf.lua","tex")) +-- dofile(resolvers.findfile("font-otp.lua","tex")) +dofile(resolvers.findfile("font-otr.lua","tex")) +dofile(resolvers.findfile("font-cff.lua","tex")) +-- dofile(resolvers.findfile("font-ttf.lua","tex")) +-- dofile(resolvers.findfile("font-ott.lua","tex")) +dofile(resolvers.findfile("font-dsp.lua","tex")) +dofile(resolvers.findfile("font-onr.lua","tex")) dofile(resolvers.findfile("font-syn.lua","tex")) dofile(resolvers.findfile("font-mis.lua","tex")) diff --git a/scripts/context/lua/mtx-server-ctx-help.lua b/scripts/context/lua/mtx-server-ctx-help.lua index 04217dab9..2e8b87f32 100644 --- a/scripts/context/lua/mtx-server-ctx-help.lua +++ b/scripts/context/lua/mtx-server-ctx-help.lua @@ -245,6 +245,7 @@ local function getnames(root,interface) local found = { } local names = { } local groups = { } + local extra = { } for e in xmlcollected(root,'cd:interface/cd:interface') do local category = match(e.at.file or "","^i%-(.*)%.xml$") local list = { } @@ -256,6 +257,7 @@ local function getnames(root,interface) names[#names+1] = t list[#list+1] = t found[idname] = e + extra[csname] = e else -- variant end @@ -268,11 +270,11 @@ local function getnames(root,interface) end sort(names, function(a,b) return lower(a[2]) < lower(b[2]) end) sort(groups, function(a,b) return lower(a[1]) < lower(b[1]) end) - return names, groups, found + return names, groups, found, extra end local loaded = setmetatableindex(function(loaded,interface) - local names, groups, found = getnames(usedsetuproot,interface) + local names, groups, found, extra = getnames(usedsetuproot,interface) local current = { interface = interface, root = usedsetuproot, @@ -280,13 +282,14 @@ local loaded = setmetatableindex(function(loaded,interface) names = names, groups = groups, found = found, + extra = extra, } loaded[interface] = current return current end) local function collect(current,name,interface,lastmode) - local command = current.found[name] + local command = current.found[name] or current.extra[name] if command then local definitions = current.definitions local attributes = command.at or { } diff --git a/scripts/context/lua/mtx-unicode.lua b/scripts/context/lua/mtx-unicode.lua index 673febc65..557e70b79 100644 --- a/scripts/context/lua/mtx-unicode.lua +++ b/scripts/context/lua/mtx-unicode.lua @@ -46,13 +46,15 @@ local application = logs.application { helpinfo = helpinfo, } -local gmatch, match, gsub, find, lower, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.format -local concat = table.concat -local split = string.split +local gmatch, match, gsub, find, lower, upper, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.upper, string.format +local concat, sort = table.concat, table.sort +local split, splitlines, strip = string.split, string.splitlines, string.strip local are_equal = table.are_equal -local tonumber = tonumber +local tonumber, tostring, rawget = tonumber, tostring, rawget local lpegmatch = lpeg.match +local P, C, S, R, Cs, Ct, Cg, Cf, Cc = lpeg.P, lpeg.C, lpeg.S, lpeg.R, lpeg.Cs, lpeg.Ct, lpeg.Cg, lpeg.Cf, lpeg.Cc local formatters = string.formatters +local utfchar = utf.char local report = application.report @@ -73,8 +75,7 @@ local sparse = false local split_space_table = lpeg.tsplitat(" ") local split_space_two = lpeg.splitat (" ") local split_range_two = lpeg.splitat ("..") -local split_colon_table = lpeg.tsplitat(lpeg.P(" ")^0 * lpeg.P(";") * lpeg.P(" ")^0) - +local split_colon_table = lpeg.tsplitat(P(" ")^0 * P(";") * P(" ")^0) local skipped = { [0x002C6] = true, -- MODIFIER LETTER CIRCUMFLEX ACCENT @@ -91,6 +92,7 @@ function scripts.unicode.update() local eastasianwidth = texttables.eastasianwidth local standardizedvariants = texttables.standardizedvariants local arabicshaping = texttables.arabicshaping + local index = texttables.index local characterdata = characters.data -- for unicode, ud in table.sortedpairs(unicodedata) do @@ -331,7 +333,7 @@ function scripts.unicode.update() end for i=1,#standardizedvariants do local si = standardizedvariants[i] - local pair, addendum = si[1], string.strip(si[2]) + local pair, addendum = si[1], strip(si[2]) local first, second = lpegmatch(split_space_two,pair) -- string.splitup(pair," ") first = tonumber(first,16) second = tonumber(second,16) @@ -362,7 +364,7 @@ end local preamble local function splitdefinition(str,index) - local l = string.splitlines(str) + local l = splitlines(str) local t = { } if index then for i=1,#l do @@ -401,6 +403,22 @@ local function splitdefinition(str,index) return t end +local function splitindex(str) + -- ok, quick and dirty ... could be a nice lpeg instead + local l = splitlines(str) + local n = { } + for i=1,#l do + local a, b, c = match(l[i],"([^%,]+)%,?(.-)\t(.*)") + if a and b and c then + local name = b .. " " .. a + name = strip(name) + name = gsub(name,"%s+"," ") + n[name] = tonumber(c,16) + end + end + return n +end + function scripts.unicode.load() local fullname = resolvers.findfile("char-def.lua") report("using: %s",fullname) @@ -420,7 +438,7 @@ function scripts.unicode.load() report("using: %s",fullname) dofile(fullname) -- - preamble = data:gsub("characters%.data%s*=%s*%{.*","") + preamble = gsub(data,"characters%.data%s*=%s*%{.*","") -- textfiles = { unicodedata = resolvers.findfile("unicodedata.txt") or "", @@ -429,6 +447,7 @@ function scripts.unicode.load() eastasianwidth = resolvers.findfile("eastasianwidth.txt") or "", standardizedvariants = resolvers.findfile("standardizedvariants.txt") or "", arabicshaping = resolvers.findfile("arabicshaping.txt") or "", + index = resolvers.findfile("index.txt") or "", } -- textdata = { @@ -438,6 +457,7 @@ function scripts.unicode.load() eastasianwidth = textfiles.eastasianwidth ~= "" and io.loaddata(textfiles.eastasianwidth) or "", standardizedvariants = textfiles.standardizedvariants ~= "" and io.loaddata(textfiles.standardizedvariants) or "", arabicshaping = textfiles.arabicshaping ~= "" and io.loaddata(textfiles.arabicshaping) or "", + index = textfiles.index ~= "" and io.loaddata(textfiles.index) or "", } texttables = { unicodedata = splitdefinition(textdata.unicodedata,true), @@ -446,6 +466,7 @@ function scripts.unicode.load() eastasianwidth = splitdefinition(textdata.eastasianwidth,true), standardizedvariants = splitdefinition(textdata.standardizedvariants,false), arabicshaping = splitdefinition(textdata.arabicshaping,true), + index = splitindex(textdata.index), } return true else @@ -456,7 +477,9 @@ end function scripts.unicode.save(filename) if preamble then - io.savedata(filename,preamble .. table.serialize(characters.data,"characters.data", { hexify = true, noquotes = true } )) + local data = table.serialize(characters.data,"characters.data", { hexify = true, noquotes = true }) + data = gsub(data,"%{%s+%[0xFE0E%]=\"text style\",%s+%[0xFE0F%]=\"emoji style\",%s+%}","variants_emoji") + io.savedata(filename,preamble .. data) end end @@ -469,7 +492,7 @@ function scripts.unicode.extras() -- old code local fullname = resolvers.findfile("blocks.txt") or "" if fullname ~= "" then local data = io.loaddata(fullname) - local lines = string.splitlines(data) + local lines = splitlines(data) local map = { } local blocks = characters.blocks local result = { } @@ -499,11 +522,11 @@ function scripts.unicode.extras() -- old code end end end - table.sort(result) + sort(result) for i=1,#result do report(result[i]) end - table.sort(map) + sort(map) for i=1,#map do local m = map[i] if not blocks[m] then @@ -511,6 +534,128 @@ function scripts.unicode.extras() -- old code end end end + -- + local index = texttables.index + local blocks = characters.blocks + local data = characters.data + for k, v in next, index do + if k ~= lower(k) then + index[k] = nil + end + end + -- for k, v in next, data do + -- v.synonym = nil + -- v.synonyms = nil + -- end + for k, v in table.sortedhash(index) do + local d = data[v] + if d and d.description ~= upper(k) then + local synonyms = d.synonyms + if synonyms then + local n = #synonyms + local f = false + for i=1,n do + if synonyms[i] == k then + f = true + break + end + end + if not f then + synonyms[n+1] = k + end + -- synonyms = table.unique(synonyms) + -- d.synonyms = synonyms + sort(synonyms) + else + d.synonyms = { k } + end + end + end +end + +do + + local space = P(" ") + local spaces = space^0 + local semicolon = P(";") + local hash = P("#") + local newline = S("\n\r") + + local unicode = Cs(R("09","AF")^1)/function(n) return tonumber(n,16) end + * spaces + local components = Ct (unicode^1) + + -- local rubish_a = semicolon + -- * spaces + -- * P("Emoji_ZWJ_Sequence") + -- * spaces + -- * semicolon + -- * spaces + -- local description = C((1 - (spaces * (hash+newline)))^1) + -- local rubish_b = (1-newline)^0 + -- * newline^1 + -- + -- local pattern_1 = Ct ( ( + -- Cf ( Ct("") * + -- Cg (Cc("components") * components) + -- * rubish_a + -- * Cg (Cc("description") * description ) + -- * rubish_b + -- , rawset) + -- + P(1) )^1 ) + + local rubish_a = semicolon + * spaces + * P("non-")^0 * P("fully-qualified") + * spaces + * hash + * spaces + local textstring = C((1 - space)^1) + * spaces + local description = ((1 - (spaces * newline))^1) / string.lower + local rubish_b = (1-newline)^0 + * newline^1 + + local pattern_2 = Ct ( ( + Cf ( Ct("") * + Cg (Cc("components") * components) + * rubish_a + * Cg (Cc("textstring") * textstring) + * Cg (Cc("description") * description ) + * rubish_b + , rawset) + + P(1) )^1 ) + + function scripts.unicode.emoji(filename) + + local name = resolvers.findfile("emoji-test.txt") or "" + if name == "" then + return + end + local l = io.loaddata(name) + local t = lpegmatch(pattern_2,l) + + local hash = { } + + local replace = lpeg.replacer { + ["#"] = "hash", + ["*"] = "asterisk" + } + + for i=1,#t do + local v = t[i] + local d = v.description + local k = lpegmatch(replace,d) or d + hash[k] = v.components + end + local new = table.serialize(hash,"return", { hexify = true }) + local old = io.loaddata(resolvers.findfile("char-emj.lua")) + if old and old ~= "" then + new = gsub(old,"^(.-)return .*$","%1" .. new) + end + io.savedata(filename,new) + end + end -- the action @@ -525,6 +670,7 @@ else scripts.unicode.update() scripts.unicode.extras() scripts.unicode.save("char-def-new.lua") + scripts.unicode.emoji("char-emj-new.lua") else report("nothing to do") end diff --git a/scripts/context/lua/mtxlibs.lua b/scripts/context/lua/mtxlibs.lua index 6eee507ae..5e547cdee 100644 --- a/scripts/context/lua/mtxlibs.lua +++ b/scripts/context/lua/mtxlibs.lua @@ -68,6 +68,7 @@ local owntree = ownpath local ownlibs = { "l-lua.lua", + "l-sandbox.lua", "l-package.lua", "l-lpeg.lua", "l-function.lua", @@ -103,6 +104,7 @@ local ownlibs = { "util-mrg.lua", "util-tpl.lua", + "util-sbx.lua", "util-env.lua", -- "luat-env.lua", -- not relevant outside context diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 7b711a88d..873770cac 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 4734, stripped down to: 2626 +-- original size: 5125, stripped down to: 2881 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -162,6 +162,283 @@ if flush then local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end end +FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +if not FFISUPPORTED then + local okay;okay,ffi=pcall(require,"ffi") + FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +end +if not FFISUPPORTED then + ffi=nil +elseif not ffi.number then + ffi.number=tonumber +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 9667, stripped down to: 6678 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b==false then + return nil + elseif b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] or false +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end end -- of closure @@ -170,7 +447,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10949, stripped down to: 8037 +-- original size: 10587, stripped down to: 7815 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -460,7 +737,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38185, stripped down to: 20990 +-- original size: 37748, stripped down to: 20111 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -552,6 +829,7 @@ patterns.nonwhitespace=nonwhitespace local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local nospacer=Cs((whitespace^1/""+nonwhitespace^1)^0) local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) @@ -561,6 +839,7 @@ local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.nospacer=nospacer patterns.b_collapser=b_collapser patterns.m_collapser=m_collapser patterns.e_collapser=e_collapser @@ -1027,27 +1306,7 @@ function lpeg.append(list,pp,delayed,checked) end local p_false=P(false) local p_true=P(true) -local function make(t) - local function making(t) - local p=p_false - local keys=sortedkeys(t) - for i=1,#keys do - local k=keys[i] - if k~="" then - local v=t[k] - if v==true then - p=p+P(k)*p_true - elseif v==false then - else - p=p+P(k)*making(v) - end - end - end - if t[""] then - p=p+p_true - end - return p - end +local function make(t,rest) local p=p_false local keys=sortedkeys(t) for i=1,#keys do @@ -1058,10 +1317,13 @@ local function make(t) p=p+P(k)*p_true elseif v==false then else - p=p+P(k)*making(v) + p=p+P(k)*make(v,v[""]) end end end + if rest then + p=p+p_true + end return p end local function collapse(t,x) @@ -1264,7 +1526,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 372, stripped down to: 329 +-- original size: 361, stripped down to: 322 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1283,7 +1545,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5983, stripped down to: 2959 +-- original size: 6419, stripped down to: 3339 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1321,22 +1583,26 @@ end local stripper=patterns.stripper local fullstripper=patterns.fullstripper local collapser=patterns.collapser +local nospacer=patterns.nospacer local longtostring=patterns.longtostring function string.strip(str) - return lpegmatch(stripper,str) or "" + return str and lpegmatch(stripper,str) or "" end function string.fullstrip(str) - return lpegmatch(fullstripper,str) or "" + return str and lpegmatch(fullstripper,str) or "" end function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" + return str and lpegmatch(collapser,str) or "" +end +function string.nospaces(str) + return str and lpegmatch(nospacer,str) or "" end function string.longtostring(str) - return lpegmatch(longtostring,str) or "" + return str and lpegmatch(longtostring,str) or "" end local pattern=P(" ")^0*P(-1) function string.is_empty(str) - if str=="" then + if not str or str=="" then return true else return lpegmatch(pattern,str) and true or false @@ -1381,6 +1647,21 @@ function string.tformat(fmt,...) end string.quote=string.quoted string.unquote=string.unquoted +if not string.bytetable then + local limit=5000 + function string.bytetable(str) + local n=#str + if n>limit then + local t={ byte(str,1,limit) } + for i=limit+1,n do + t[i]=byte(str,i) + end + return t + else + return { byte(str,1,n) } + end + end +end end -- of closure @@ -1389,7 +1670,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 36997, stripped down to: 22376 +-- original size: 39608, stripped down to: 23165 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1716,19 +1997,23 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions,metacheck local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } -local function simple_table(t) +local function is_simple_table(t,hexify) local nt=#t if nt>0 then local n=0 for _,v in next,t do n=n+1 + if type(v)=="table" then + return nil + end end + local haszero=rawget(t,0) if n==nt then local tt={} for i=1,nt do @@ -1738,10 +2023,10 @@ local function simple_table(t) if hexify then tt[i]=format("0x%X",v) else - tt[i]=tostring(v) + tt[i]=v end elseif tv=="string" then - tt[i]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then tt[i]=v and "true" or "false" else @@ -1749,10 +2034,32 @@ local function simple_table(t) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + if hexify then + tt[i+1]=format("0x%X",v) + else + tt[i+1]=v + end + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil end +table.is_simple_table=is_simple_table local propername=patterns.propername local function dummy() end local function do_serialize(root,name,depth,level,indexed) @@ -1786,7 +2093,7 @@ local function do_serialize(root,name,depth,level,indexed) if compact then last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -1814,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -1851,6 +2158,7 @@ local function do_serialize(root,name,depth,level,indexed) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) @@ -1873,6 +2181,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,v)) else @@ -1888,13 +2197,14 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={},",depth,k and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then if tk=="number" then if hexify then @@ -1904,6 +2214,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={ %s },",depth,k and "true" or "false",concat(st,", "))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else @@ -1924,6 +2235,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%s,",depth,tostring(k),v and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%s,",depth,k,v and "true" or "false")) else @@ -1940,6 +2252,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=load(%q),",depth,k and "true" or "false",f)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=load(%q),",depth,k,f)) else @@ -1955,6 +2268,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",tostring(v))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,tostring(v))) else @@ -1976,6 +2290,7 @@ local function serialize(_handle,root,name,specification) functions=specification.functions compact=specification.compact inline=specification.inline and compact + metacheck=specification.metacheck if functions==nil then functions=true end @@ -1985,6 +2300,9 @@ local function serialize(_handle,root,name,specification) if inline==nil then inline=compact end + if metacheck==nil then + metacheck=true + end else noquotes=false hexify=false @@ -1992,6 +2310,7 @@ local function serialize(_handle,root,name,specification) compact=true inline=true functions=true + metacheck=true end if tname=="string" then if name=="return" then @@ -2015,7 +2334,7 @@ local function serialize(_handle,root,name,specification) handle("t={") end if root then - if getmetatable(root) then + if metacheck and getmetatable(root) then local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end @@ -2091,6 +2410,38 @@ local function flattened(t,f,depth) return f end table.flattened=flattened +local function collapsed(t,f,h) + if f==nil then + f={} + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsed(v,f,h) + elseif not h[v] then + f[#f+1]=v + h[v]=true + end + end + return f +end +local function collapsedhash(t,h) + if h==nil then + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsedhash(v,h) + else + h[v]=true + end + end + return h +end +table.collapsed=collapsed +table.collapsedhash=collapsedhash local function unnest(t,f) if not f then f={} @@ -2197,6 +2548,12 @@ function table.swapped(t,s) end return n end +function table.hashed(t) + for i=1,#t do + t[t[i]]=i + end + return t +end function table.mirrored(t) local n={} for k,v in next,t do @@ -2365,7 +2722,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 9001, stripped down to: 6512 +-- original size: 11790, stripped down to: 6961 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2375,6 +2732,7 @@ if not modules then modules={} end modules ['l-io']={ license="see context related readme files" } local io=io +local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor @@ -2384,50 +2742,56 @@ if string.find(os.getenv("PATH"),";",1,true) then else io.fileseparator,io.pathseparator="/",":" end -local function readall(f) - return f:read("*all") -end +local large=2^24 +local medium=large/16 +local small=medium/8 local function readall(f) local size=f:seek("end") - if size==0 then - return "" - elseif size<1024*1024 then + if size>0 then f:seek("set",0) - return f:read('*all') - else - local done=f:seek("set",0) - local step - if size<1024*1024 then - step=1024*1024 - elseif size>16*1024*1024 then - step=16*1024*1024 - else - step=floor(size/(1024*1024))*1024*1024/8 - end - local data={} - while true do - local r=f:read(step) - if not r then - return concat(data) - else - data[#data+1]=r - end - end + return f:read(size) + else + return "" end end io.readall=readall function io.loaddata(filename,textmode) - local f=io.open(filename,(textmode and 'r') or 'rb') + local f=open(filename,(textmode and 'r') or 'rb') if f then - local data=readall(f) + local size=f:seek("end") + local data=nil + if size>0 then + f:seek("set",0) + data=f:read(size) + end f:close() - if #data>0 then - return data + return data + end +end +function io.copydata(source,target,action) + local f=open(source,"rb") + if f then + local g=open(target,"wb") + if g then + local size=f:seek("end") + if size>0 then + f:seek("set",0) + local data=f:read(size) + if action then + data=action(data) + end + if data then + g:write(data) + end + end + g:close() end + f:close() + flush() end end function io.savedata(filename,data,joiner) - local f=io.open(filename,"wb") + local f=open(filename,"wb") if f then if type(data)=="table" then f:write(concat(data,joiner or "")) @@ -2437,40 +2801,70 @@ function io.savedata(filename,data,joiner) f:write(data or "") end f:close() - io.flush() + flush() return true else return false end end -function io.loadlines(filename,n) - local f=io.open(filename,'r') - if not f then - elseif n then - local lines={} - for i=1,n do - local line=f:read("*lines") - if line then - lines[#lines+1]=line - else - break +if fio and fio.readline then + local readline=fio.readline + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=readline(f) + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=readline(f) + f:close() + if line and #line>0 then + return line end end - f:close() - lines=concat(lines,"\n") - if #lines>0 then - return lines - end - else - local line=f:read("*line") or "" - f:close() - if #line>0 then - return line + end +else + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end end end end function io.loadchunk(filename,n) - local f=io.open(filename,'rb') + local f=open(filename,'rb') if f then local data=f:read(n or 1024) f:close() @@ -2480,7 +2874,7 @@ function io.loadchunk(filename,n) end end function io.exists(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return false else @@ -2489,7 +2883,7 @@ function io.exists(filename) end end function io.size(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return 0 else @@ -2498,11 +2892,11 @@ function io.size(filename) return s end end -function io.noflines(f) +local function noflines(f) if type(f)=="string" then - local f=io.open(filename) + local f=open(filename) if f then - local n=f and io.noflines(f) or 0 + local n=f and noflines(f) or 0 f:close() return n else @@ -2517,6 +2911,7 @@ function io.noflines(f) return n end end +io.noflines=noflines local nextchar={ [ 4]=function(f) return f:read(1,1,1,1) @@ -2594,16 +2989,16 @@ function io.bytes(f,n) end function io.ask(question,default,options) while true do - io.write(question) + write(question) if options then - io.write(format(" [%s]",concat(options,"|"))) + write(format(" [%s]",concat(options,"|"))) end if default then - io.write(format(" [%s]",default)) + write(format(" [%s]",default)) end - io.write(format(" ")) - io.flush() - local answer=io.read() + write(format(" ")) + flush() + local answer=read() answer=gsub(answer,"^%s*(.*)%s*$","%1") if answer=="" and default then return default @@ -2625,7 +3020,7 @@ function io.ask(question,default,options) end end end -local function readnumber(f,n,m) +local function readnumber(f,n,m) if m then f:seek("set",n) n=m @@ -2634,31 +3029,31 @@ local function readnumber(f,n,m) return byte(f:read(1)) elseif n==2 then local a,b=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==3 then local a,b,c=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==4 then local a,b,c,d=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==8 then local a,b=readnumber(f,4),readnumber(f,4) - return 256*a+b + return 0x100*a+b elseif n==12 then local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-2 then local b,a=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==-3 then local c,b,a=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-4 then local d,c,b,a=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==-8 then local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) - return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + return 0x100000000000000*a+0x1000000000000*b+0x10000000000*c+0x100000000*d+0x1000000*e+0x10000*f+0x100*g+h else return 0 end @@ -2680,7 +3075,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 5146, stripped down to: 2933 +-- original size: 5358, stripped down to: 3177 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2693,6 +3088,7 @@ local tostring,tonumber=tostring,tonumber local format,floor,match,rep=string.format,math.floor,string.match,string.rep local concat,insert=table.concat,table.insert local lpegmatch=lpeg.match +local floor=math.floor number=number or {} local number=number if bit32 then @@ -2817,6 +3213,26 @@ end function number.bits(n) return { bits(n,1) } end +function number.bytetodecimal(b) + local d=floor(b*100/255+0.5) + if d>100 then + return 100 + elseif d<-100 then + return -100 + else + return d + end +end +function number.decimaltobyte(d) + local b=floor(d*255/100+0.5) + if b>255 then + return 255 + elseif b<-255 then + return -255 + else + return b + end +end end -- of closure @@ -2825,7 +3241,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 2010, stripped down to: 1186 +-- original size: 1923, stripped down to: 1133 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2898,7 +3314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16390, stripped down to: 9734 +-- original size: 16268, stripped down to: 9246 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2974,7 +3390,7 @@ if not os.__getenv__ then end local execute=os.execute local iopopen=io.popen -function os.resultof(command) +local function resultof(command) local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" @@ -2984,9 +3400,13 @@ function os.resultof(command) return "" end end +os.resultof=resultof +function os.pipeto(command) + return iopopen(command,"w") +end if not io.fileseparator then if find(os.getenv("PATH"),";",1,true) then - io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "windows" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" end @@ -3029,17 +3449,6 @@ setmetatable(os,{ __index=function(t,k) return r and r(t,k) or nil end }) local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" -local function guess() - local architecture=os.resultof("uname -m") or "" - if architecture~="" then - return architecture - end - architecture=os.getenv("HOSTTYPE") or "" - if architecture~="" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end if platform~="" then os.platform=platform elseif os.type=="windows" then @@ -3056,7 +3465,7 @@ elseif os.type=="windows" then end elseif name=="linux" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="linux-64" elseif find(architecture,"ppc",1,true) then @@ -3070,7 +3479,7 @@ elseif name=="linux" then end elseif name=="macosx" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + local platform,architecture="",resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" elseif find(architecture,"i386",1,true) then @@ -3086,7 +3495,7 @@ elseif name=="macosx" then end elseif name=="sunos" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"sparc",1,true) then platform="solaris-sparc" else @@ -3098,7 +3507,7 @@ elseif name=="sunos" then end elseif name=="freebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else @@ -3110,7 +3519,7 @@ elseif name=="freebsd" then end elseif name=="kfreebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else @@ -3241,7 +3650,7 @@ if not os.sleep then end end local function isleapyear(year) - return (year%400==0) or ((year%100~=0) and (year%4==0)) + return (year%4==0) and (year%100~=0 or year%400==0) end os.isleapyear=isleapyear local days={ 31,28,31,30,31,30,31,31,30,31,30,31 } @@ -3280,7 +3689,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21648, stripped down to: 10238 +-- original size: 20997, stripped down to: 9986 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3617,13 +4026,15 @@ function file.robustname(str,strict) end end end -file.readdata=io.loaddata -file.savedata=io.savedata +local loaddata=io.loaddata +local savedata=io.savedata +file.readdata=loaddata +file.savedata=savedata function file.copy(oldname,newname) if oldname and newname then - local data=io.loaddata(oldname) + local data=loaddata(oldname) if data and data~="" then - file.savedata(newname,data) + savedata(newname,data) end end end @@ -3660,7 +4071,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1265, stripped down to: 1038 +-- original size: 1211, stripped down to: 1002 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3714,7 +4125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3355, stripped down to: 2321 +-- original size: 3309, stripped down to: 2314 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3744,6 +4155,8 @@ do if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + md5.sumhexa=md5.hex + md5.sumHEXA=md5.HEX end end function file.needsupdating(oldname,newname,threshold) @@ -3802,7 +4215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12897, stripped down to: 5882 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4019,7 +4432,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 17358, stripped down to: 11378 +-- original size: 17703, stripped down to: 11691 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4283,6 +4696,31 @@ local function globfiles(path,recurse,func,files) return files end dir.globfiles=globfiles +local function globdirs(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + if recurse then + globdirs(path.."/"..name,recurse,func,files) + end + end + end + end + end + return files +end +dir.globdirs=globdirs function dir.ls(pattern) return concat(glob(pattern),"\n") end @@ -4447,9 +4885,13 @@ end file.expandname=dir.expandname local stack={} function dir.push(newdir) - insert(stack,currentdir()) + local curdir=currentdir() + insert(stack,curdir) if newdir and newdir~="" then chdir(newdir) + return newdir + else + return curdir end end function dir.pop() @@ -4484,7 +4926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1919, stripped down to: 1621 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4556,7 +4998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 38699, stripped down to: 16321 +-- original size: 38263, stripped down to: 16330 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -5167,6 +5609,23 @@ function utf.chrlen(u) (u<0xFC and 5) or (u<0xFE and 6) or 0 end +local extract=bit32.extract +local char=string.char +function unicode.toutf32string(n) + if n<=0xFF then + return + char(n).."\000\000\000" + elseif n<=0xFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8)).."\000\000" + elseif n<=0xFFFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8)).."\000" + else + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8))..char(extract(n,24,8)) + end +end end -- of closure @@ -5175,7 +5634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 1012, stripped down to: 912 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5215,7 +5674,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 36053, stripped down to: 19685 +-- original size: 36148, stripped down to: 20179 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5227,7 +5686,7 @@ if not modules then modules={} end modules ['util-str']={ utilities=utilities or {} utilities.strings=utilities.strings or {} local strings=utilities.strings -local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local format,gsub,rep,sub,find=string.format,string.gsub,string.rep,string.sub,string.find local load,dump=load,string.dump local tonumber,type,tostring=tonumber,type,tostring local unpack,concat=table.unpack,table.concat @@ -5410,6 +5869,25 @@ function number.signed(i) return "-",-i end end +local digit=patterns.digit +local period=patterns.period +local three=digit*digit*digit +local splitter=Cs ( + (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) +) +patterns.formattednumber=splitter +function number.formatted(n,sep1,sep2) + local s=type(s)=="string" and n or format("%0.2f",n) + if sep1==true then + return lpegmatch(splitter,s,1,".",",") + elseif sep1=="." then + return lpegmatch(splitter,s,1,sep1,sep2 or ",") + elseif sep1=="," then + return lpegmatch(splitter,s,1,sep1,sep2 or ".") + else + return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") + end +end local zero=P("0")^1/"" local plus=P("+")/"" local minus=P("-") @@ -5435,6 +5913,27 @@ function number.sparseexponent(f,n) end return tostring(n) end +local hf={} +local hs={} +setmetatable(hf,{ __index=function(t,k) + local v="%."..k.."f" + t[k]=v + return v +end } ) +setmetatable(hs,{ __index=function(t,k) + local v="%"..k.."s" + t[k]=v + return v +end } ) +function number.formattedfloat(n,b,a) + local s=format(hf[a],n) + local l=(b or 0)+(a or 0)+1 + if #s<l then + return format(hs[l],s) + else + return s + end +end local template=[[ %s %s @@ -5462,6 +5961,7 @@ local autodouble=string.autodouble local sequenced=table.sequenced local formattednumber=number.formatted local sparseexponent=number.sparseexponent +local formattedfloat=number.formattedfloat ]] else environment={ @@ -5485,6 +5985,7 @@ else sequenced=table.sequenced, formattednumber=number.formatted, sparseexponent=number.sparseexponent, + formattedfloat=number.formattedfloat } end local arguments={ "a1" } @@ -5495,6 +5996,7 @@ setmetatable(arguments,{ __index=function(t,k) end }) local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_sub=(C((S("+-")+R("09"))^0)+Cc(0))*P(".")*(C((S("+-")+R("09"))^0)+Cc(0)) local prefix_tab=P("{")*C((1-P("}"))^0)*P("}")+C((1-R("az","AZ","09","%%"))^0) local format_s=function(f) n=n+1 @@ -5545,6 +6047,10 @@ local format_F=function(f) return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) end end +local format_k=function(b,a) + n=n+1 + return format("formattedfloat(a%s,%i,%i)",n,b or 0,a or 0) +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5693,25 +6199,6 @@ end local format_W=function(f) return format("nspaces[%s]",tonumber(f) or 0) end -local digit=patterns.digit -local period=patterns.period -local three=digit*digit*digit -local splitter=Cs ( - (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) -) -patterns.formattednumber=splitter -function number.formatted(n,sep1,sep2) - local s=type(s)=="string" and n or format("%0.2f",n) - if sep1==true then - return lpegmatch(splitter,s,1,".",",") - elseif sep1=="." then - return lpegmatch(splitter,s,1,sep1,sep2 or ",") - elseif sep1=="," then - return lpegmatch(splitter,s,1,sep1,sep2 or ".") - else - return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") - end -end local format_m=function(f) n=n+1 if not f or f=="" then @@ -5736,9 +6223,16 @@ end local format_extension=function(extensions,f,name) local extension=extensions[name] or "tostring(%s)" local f=tonumber(f) or 1 + local w=find(extension,"%.%.%.") if f==0 then + if w then + extension=gsub(extension,"%.%.%.","") + end return extension elseif f==1 then + if w then + extension=gsub(extension,"%.%.%.","%%s") + end n=n+1 local a="a"..n return format(extension,a,a) @@ -5746,10 +6240,13 @@ local format_extension=function(extensions,f,name) local a="a"..(n+f+1) return format(extension,a,a) else + if w then + extension=gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s") + end local t={} for i=1,f do n=n+1 - t[#t+1]="a"..n + t[i]="a"..n end return format(extension,unpack(t)) end @@ -5762,7 +6259,8 @@ local builder=Cs { "start", +V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") -+V("N") ++V("N") ++V("k") +V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") +V("W") +V("a") @@ -5789,6 +6287,7 @@ local builder=Cs { "start", ["S"]=(prefix_any*P("S"))/format_S, ["Q"]=(prefix_any*P("Q"))/format_S, ["N"]=(prefix_any*P("N"))/format_N, + ["k"]=(prefix_sub*P("k"))/format_k, ["c"]=(prefix_any*P("c"))/format_c, ["C"]=(prefix_any*P("C"))/format_C, ["r"]=(prefix_any*P("r"))/format_r, @@ -5909,7 +6408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28680, stripped down to: 18636 +-- original size: 27407, stripped down to: 17116 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5923,7 +6422,7 @@ utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort -local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local setmetatable,getmetatable,tonumber,tostring,rawget=setmetatable,getmetatable,tonumber,tostring,rawget local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs @@ -6063,7 +6562,7 @@ function table.tocsv(t,specification) r[f]=tostring(field) end end - result[#result+1]=concat(r,separator) + result[i+1]=concat(r,separator) end return concat(result,"\n") else @@ -6295,11 +6794,12 @@ function table.autokey(t,k) return v end local selfmapper={ __index=function(t,k) t[k]=k return k end } -function table.twowaymapper(t) - if not t then - t={} - else - for i=0,#t do +function table.twowaymapper(t) + if not t then + t={} + else + local zero=rawget(t,0) + for i=zero and 0 or 1,#t do local ti=t[i] if ti then local i=tostring(i) @@ -6307,7 +6807,6 @@ function table.twowaymapper(t) t[ti]=i end end - t[""]=t[0] or "" end setmetatable(t,selfmapper) return t @@ -6346,6 +6845,7 @@ local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize +local is_simple_table=table.is_simple_table local function serialize(root,name,specification) if type(specification)=="table" then return original_serialize(root,name,specification) @@ -6353,54 +6853,6 @@ local function serialize(root,name,specification) local t local n=1 local unknown=false - local function simple_table(t) - local nt=#t - if nt>0 then - local n=0 - for _,v in next,t do - n=n+1 - if type(v)=="table" then - return nil - end - end - local haszero=t[0] - if n==nt then - local tt={} - for i=1,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i]=v - elseif tv=="string" then - tt[i]=format("%q",v) - elseif tv=="boolean" then - tt[i]=v and "true" or "false" - else - return nil - end - end - return tt - elseif haszero and (n==nt+1) then - local tt={} - for i=0,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i+1]=v - elseif tv=="string" then - tt[i+1]=format("%q",v) - elseif tv=="boolean" then - tt[i+1]=v and "true" or "false" - else - return nil - end - end - tt[1]="[0] = "..tt[1] - return tt - end - end - return nil - end local function do_serialize(root,name,depth,level,indexed) if level>0 then n=n+1 @@ -6425,7 +6877,7 @@ local function serialize(root,name,specification) local last=0 last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -6448,7 +6900,7 @@ local function serialize(root,name,specification) if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else - local st=simple_table(v) + local st=is_simple_table(v) if st then n=n+1 t[n]=f_val_seq(depth,st) else @@ -6492,7 +6944,7 @@ local function serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else - local st=simple_table(v) + local st=is_simple_table(v) if not st then do_serialize(v,k,depth,level+1) elseif tk=="number" then @@ -6552,11 +7004,11 @@ local function serialize(root,name,specification) end if root then if getmetatable(root) then - local dummy=root._w_h_a_t_e_v_e_r_ + local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - local st=simple_table(root) + local st=is_simple_table(root) if st then return t[1]..f_fin_seq(st) else @@ -6570,7 +7022,12 @@ local function serialize(root,name,specification) end table.serialize=serialize if setinspector then - setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) + setinspector("table",function(v) + if type(v)=="table" then + print(serialize(v,"table",{ metacheck=false })) + return true + end + end) end @@ -6580,7 +7037,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fil"] = package.loaded["util-fil"] or true --- original size: 3577, stripped down to: 2870 +-- original size: 7567, stripped down to: 5575 if not modules then modules={} end modules ['util-fil']={ version=1.001, @@ -6590,7 +7047,9 @@ if not modules then modules={} end modules ['util-fil']={ license="see context related readme files" } local byte=string.byte -local extract=bit32.extract +local char=string.char +local extract=bit32 and bit32.extract +local floor=math.floor utilities=utilities or {} local files={} utilities.files=files @@ -6609,6 +7068,7 @@ end function files.size(f) return f:seek("end") end +files.getsize=files.size function files.setposition(f,n) if zerobased[f] then f:seek("set",n) @@ -6646,6 +7106,10 @@ end function files.readbytes(f,n) return byte(f:read(n),1,n) end +function files.readbytetable(f,n) + local s=f:read(n or 1) + return { byte(s,1,#s) } +end function files.readchar(f) return f:read(1) end @@ -6655,7 +7119,7 @@ end function files.readinteger1(f) local n=byte(f:read(1)) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6663,55 +7127,107 @@ end files.readcardinal1=files.readbyte files.readcardinal=files.readcardinal1 files.readinteger=files.readinteger1 +files.readsignedbyte=files.readinteger1 function files.readcardinal2(f) local a,b=byte(f:read(2),1,2) return 0x100*a+b end +function files.readcardinal2le(f) + local b,a=byte(f:read(2),1,2) + return 0x100*a+b +end function files.readinteger2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function files.readinteger2le(f) + local b,a=byte(f:read(2),1,2) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function files.readcardinal3(f) local a,b,c=byte(f:read(3),1,3) return 0x10000*a+0x100*b+c end +function files.readcardinal3le(f) + local c,b,a=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readinteger3(f) + local a,b,c=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function files.readinteger3le(f) + local c,b,a=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function files.readcardinal4(f) local a,b,c,d=byte(f:read(4),1,4) return 0x1000000*a+0x10000*b+0x100*c+d end +function files.readcardinal4le(f) + local d,c,b,a=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end function files.readinteger4(f) local a,b,c,d=byte(f:read(4),1,4) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.readfixed4(f) - local a,b,c,d=byte(f:read(4),1,4) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF +function files.readinteger4le(f) + local d,c,b,a=byte(f:read(4),1,4) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n+(0x100*c+d)/0xFFFF + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.read2dot14(f) +function files.readfixed2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - n=extract(n,30,2) - return n+m/0x4000 + return (0x100*a+b )+(0x100*c+d)/0x10000 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function files.skipshort(f,n) @@ -6720,6 +7236,55 @@ end function files.skiplong(f,n) f:read(4*(n or 1)) end +function files.writecardinal2(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + f:write(b,a) +end +function files.writecardinal4(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + n=floor(n/256) + local c=char(n%256) + n=floor(n/256) + local d=char(n%256) + f:write(d,c,b,a) +end +function files.writestring(f,s) + f:write(char(byte(s,1,#s))) +end +function files.writebyte(f,b) + f:write(char(b)) +end +if fio and fio.readcardinal1 then + files.readcardinal1=fio.readcardinal1 + files.readcardinal2=fio.readcardinal2 + files.readcardinal3=fio.readcardinal3 + files.readcardinal4=fio.readcardinal4 + files.readinteger1=fio.readinteger1 + files.readinteger2=fio.readinteger2 + files.readinteger3=fio.readinteger3 + files.readinteger4=fio.readinteger4 + files.read2dot14=fio.read2dot14 + files.setposition=fio.setposition + files.getposition=fio.getposition + files.readbyte=files.readcardinal1 + files.readsignedbyte=files.readinteger1 + files.readcardinal=files.readcardinal1 + files.readinteger=files.readinteger1 + local skipposition=fio.skipposition + files.skipposition=skipposition + files.readbytes=fio.readbytes + files.readbytetable=fio.readbytetable + function files.skipshort(f,n) + skipposition(f,2*(n or 1)) + end + function files.skiplong(f,n) + skipposition(f,4*(n or 1)) + end +end end -- of closure @@ -6728,7 +7293,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 4264, stripped down to: 3349 +-- original size: 8716, stripped down to: 6754 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -6738,7 +7303,7 @@ if not modules then modules={} end modules ['util-sac']={ license="see context related readme files" } local byte,sub=string.byte,string.sub -local extract=bit32.extract +local extract=bit32 and bit32.extract utilities=utilities or {} local streams={} utilities.streams=streams @@ -6796,6 +7361,12 @@ function streams.readbytes(f,n) f[2]=j return byte(f[1],i,j-1) end +function streams.readbytetable(f,n) + local i=f[2] + local j=i+n + f[2]=j + return { byte(f[1],i,j-1) } +end function streams.skipbytes(f,n) f[2]=f[2]+n end @@ -6815,7 +7386,7 @@ function streams.readinteger1(f) f[2]=i+1 local n=byte(f[1],i) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6830,16 +7401,33 @@ function streams.readcardinal2(f) local a,b=byte(f[1],i,j) return 0x100*a+b end +function streams.readcardinal2LE(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + return 0x100*a+b +end function streams.readinteger2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function streams.readinteger2le(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function streams.readcardinal3(f) @@ -6849,6 +7437,35 @@ function streams.readcardinal3(f) local a,b,c=byte(f[1],i,j) return 0x10000*a+0x100*b+c end +function streams.readcardinal3le(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readinteger3(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function streams.readinteger3le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function streams.readcardinal4(f) local i=f[2] local j=i+3 @@ -6861,11 +7478,21 @@ function streams.readinteger4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d + end +end +function streams.readinteger4le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local d,c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 + else + return 0x1000000*a+0x10000*b+0x100*c+d end end function streams.readfixed4(f) @@ -6873,26 +7500,38 @@ function streams.readfixed4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - return n+(0x100*c+d)/0xFFFF + return (0x100*a+b )+(0x100*c+d)/0x10000 end end -function streams.read2dot14(f) +function streams.readfixed2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 - else - n=extract(n,30,2) - return n+m/0x4000 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function streams.skipshort(f,n) @@ -6901,6 +7540,92 @@ end function streams.skiplong(f,n) f[2]=f[2]+4*(n or 1) end +if sio and sio.readcardinal2 then + local readcardinal1=sio.readcardinal1 + local readcardinal2=sio.readcardinal2 + local readcardinal3=sio.readcardinal3 + local readcardinal4=sio.readcardinal4 + local readinteger1=sio.readinteger1 + local readinteger2=sio.readinteger2 + local readinteger3=sio.readinteger3 + local readinteger4=sio.readinteger4 + local readfixed2=sio.readfixed2 + local readfixed4=sio.readfixed4 + local read2dot14=sio.read2dot14 + local readbytes=sio.readbytes + local readbytetable=sio.readbytetable + function streams.readcardinal1(f) + local i=f[2] + f[2]=i+1 + return readcardinal1(f[1],i) + end + function streams.readcardinal2(f) + local i=f[2] + f[2]=i+2 + return readcardinal2(f[1],i) + end + function streams.readcardinal3(f) + local i=f[2] + f[2]=i+3 + return readcardinal3(f[1],i) + end + function streams.readcardinal4(f) + local i=f[2] + f[2]=i+4 + return readcardinal4(f[1],i) + end + function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + return readinteger1(f[1],i) + end + function streams.readinteger2(f) + local i=f[2] + f[2]=i+2 + return readinteger2(f[1],i) + end + function streams.readinteger3(f) + local i=f[2] + f[2]=i+3 + return readinteger3(f[1],i) + end + function streams.readinteger4(f) + local i=f[2] + f[2]=i+4 + return readinteger4(f[1],i) + end + function streams.read2dot4(f) + local i=f[2] + f[2]=i+2 + return read2dot4(f[1],i) + end + function streams.readbytes(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytes(f[1],i,n) + end + function streams.readbytetable(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytetable(f[1],i,n) + end + streams.readbyte=streams.readcardinal1 + streams.readsignedbyte=streams.readinteger1 + streams.readcardinal=streams.readcardinal1 + streams.readinteger=streams.readinteger1 +end end -- of closure @@ -6909,7 +7634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4100, stripped down to: 2852 +-- original size: 3926, stripped down to: 2742 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -7049,7 +7774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 23411, stripped down to: 16177 +-- original size: 22883, stripped down to: 16045 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -7211,6 +7936,21 @@ function parsers.settings_to_array(str,strict) return { str } end end +function parsers.settings_to_numbers(str) + if not str or str=="" then + return {} + end + if type(str)=="table" then + elseif find(str,",",1,true) then + str=lpegmatch(pattern,str) + else + return { tonumber(str) } + end + for i=1,#str do + str[i]=tonumber(str[i]) + end + return str +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) local pattern=spaces*Ct(value*(separator*value)^0) function parsers.settings_to_array_obey_fences(str) @@ -7587,7 +8327,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2350, stripped down to: 1847 +-- original size: 2274, stripped down to: 1781 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7668,7 +8408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12862, stripped down to: 9104 +-- original size: 12454, stripped down to: 8840 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7854,7 +8594,6 @@ function setters.list(t) return user,system end function setters.show(t) - local category=t.name local list=setters.list(t) t.report() for k=1,#list do @@ -7981,7 +8720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 30767, stripped down to: 21355 +-- original size: 30007, stripped down to: 20818 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -8027,6 +8766,14 @@ if tex and (tex.jobname or tex.formatname) then if texio.setescape then texio.setescape(0) end + if arg then + for k,v in next,arg do + if v=="--ansi" or v=="--c:ansi" then + variant="ansi" + break + end + end + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -8612,7 +9359,6 @@ function logs.stop_page_number() end logs.flush() end -local report_files=logs.reporter("files") local nesting=0 local verbose=false local hasscheme=url.hasscheme @@ -8774,7 +9520,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6917, stripped down to: 5484 +-- original size: 8036, stripped down to: 5567 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8806,11 +9552,13 @@ end local function resettiming(instance) timers[instance or "notimer"]={ timing=0,loadtime=0 } end +local ticks=clock +local seconds=function(n) return n or 0 end local function starttiming(instance) local timer=timers[instance or "notimer"] local it=timer.timing or 0 if it==0 then - timer.starttime=clock() + timer.starttime=ticks() if not timer.loadtime then timer.loadtime=0 end @@ -8824,12 +9572,13 @@ local function stoptiming(instance) timer.timing=it-1 else local starttime=timer.starttime - if starttime then - local stoptime=clock() + if starttime and starttime>0 then + local stoptime=ticks() local loadtime=stoptime-starttime timer.stoptime=stoptime timer.loadtime=timer.loadtime+loadtime timer.timing=0 + timer.starttime=0 return loadtime end end @@ -8840,7 +9589,7 @@ local function elapsed(instance) return instance or 0 else local timer=timers[instance or "notimer"] - return timer and timer.loadtime or 0 + return timer and seconds(timer.loadtime) or 0 end end local function elapsedtime(instance) @@ -8888,10 +9637,13 @@ function statistics.show() local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) - if jit then - local jitstatus={ jit.status() } - if jitstatus[1] then - register("luajit options",concat(jitstatus," ",2)) + if TEXENGINE=="luajittex" and JITSUPPORTED then + local jitstatus=jit.status + if jitstatus then + local jitstatus={ jitstatus() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end end end register("lua properties",function() @@ -8955,7 +9707,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 6039, stripped down to: 3616 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -9102,7 +9854,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 5142, stripped down to: 3611 +-- original size: 5396, stripped down to: 3708 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -9224,6 +9976,17 @@ function luautilities.loadstripped(...) return load(dump(l,true)) end end +local finalizers={} +setmetatable(finalizers,{ + __gc=function(t) + for i=1,#t do + pcall(t[i]) + end + end +} ) +function luautilities.registerfinalizer(f) + finalizers[#finalizers+1]=f +end end -- of closure @@ -9232,7 +9995,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 4030, stripped down to: 2718 +-- original size: 8911, stripped down to: 6504 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -9242,75 +10005,230 @@ if not modules then modules={} end modules ['util-deb']={ license="see context related readme files" } local debug=require "debug" -local getinfo=debug.getinfo -local type,next,tostring=type,next,tostring -local format,find=string.format,string.find -local is_boolean=string.is_boolean +local getinfo,sethook=debug.getinfo,debug.sethook +local type,next,tostring,tonumber=type,next,tostring,tonumber +local format,find,sub,gsub=string.format,string.find,string.sub,string.gsub +local insert,remove,sort=table.insert,table.remove,table.sort +local setmetatableindex=table.setmetatableindex utilities=utilities or {} local debugger=utilities.debugger or {} utilities.debugger=debugger -local counters={} -local names={} local report=logs.reporter("debugger") -local function hook() - local f=getinfo(2) - if f then - local n="unknown" - if f.what=="C" then - n=f.name or '<anonymous>' - if not names[n] then - names[n]=format("%42s",n) +local ticks=os.gettimeofday or os.clock +local seconds=function(n) return n or 0 end +local overhead=0 +local dummycalls=10*1000 +local nesting=0 +local names={} +local initialize=false +if not (FFISUPPORTED and ffi) then +elseif os.type=="windows" then + initialize=function() + local kernel=ffilib("kernel32","system") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end end - else - n=f.name or f.namewhat or f.what - if not n or n=="" then - n="?" + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end + initialize=false + end +elseif os.type=="unix" then + initialize=function() + local C=ffi.C + local tonumber=ffi.number or tonumber + ffi.cdef [[ + /* what a mess */ + typedef int clk_id_t; + typedef enum { CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_PROCESS_CPUTIME_ID } clk_id; + typedef struct timespec { long sec; long nsec; } ctx_timespec; + int clock_gettime(clk_id_t timerid, struct timespec *t); + ]] + local target=ffi.new("ctx_timespec[?]",1) + local clock=C.CLOCK_PROCESS_CPUTIME_ID + ticks=function () + C.clock_gettime(clock,target) + return tonumber(target[0].sec*1000000000+target[0].nsec) + end + seconds=function(ticks) + return ticks/1000000000 + end + initialize=false + end +end +setmetatableindex(names,function(t,name) + local v=setmetatableindex(function(t,source) + local v=setmetatableindex(function(t,line) + local v={ total=0,count=0 } + t[line]=v + return v + end) + t[source]=v + return v + end) + t[name]=v + return v +end) +local function hook(where) + local f=getinfo(2,"nSl") + if f then + local source=f.short_src + if not source then + return + end + local line=f.linedefined or 0 + local name=f.name + if not name then + local what=f.what + if what=="C" then + name="<anonymous>" + else + name=f.namewhat or what or "<unknown>" end - if not names[n] then - names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + local data=names[name][source][line] + if where=="call" then + data.count=data.count+1 + insert(data,ticks()) + elseif where=="return" then + local t=remove(data) + if t then + data.total=data.total+ticks()-t end end - counters[n]=(counters[n] or 0)+1 end end -function debugger.showstats(printer,threshold) - printer=printer or report - threshold=threshold or 0 - local total,grandtotal,functions=0,0,0 +function debugger.showstats(printer,threshold) + local printer=printer or report + local calls=0 + local functions=0 local dataset={} - for name,count in next,counters do - dataset[#dataset+1]={ name,count } + local length=0 + local wholetime=0 + local threshold=threshold or 0 + for name,sources in next,names do + for source,lines in next,sources do + for line,data in next,lines do + local count=data.count + if count>threshold then + if #name>length then + length=#name + end + local total=data.total + local real=total + if real>0 then + real=total-(count*overhead/dummycalls) + if real<0 then + real=0 + end + wholetime=wholetime+real + end + if line<0 then + line=0 + end + dataset[#dataset+1]={ real,total,count,name,source,line } + end + end + end end - table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + sort(dataset,function(a,b) + if a[1]==b[1] then + if a[2]==b[2] then + if a[3]==b[3] then + if a[4]==b[4] then + if a[5]==b[5] then + return a[6]<b[6] + else + return a[5]<b[5] + end + else + return a[4]<b[4] + end + else + return b[3]<a[3] + end + else + return b[2]<a[2] + end + else + return b[1]<a[1] + end + end) + if length>50 then + length=50 + end + local fmt=string.formatters["%4.9k %4.9k %3.3k %8i %-"..length.."s %4i %s"] for i=1,#dataset do - local d=dataset[i] - local name=d[1] - local count=d[2] - if count>threshold and not find(name,"for generator") then - printer(format("%8i %s\n",count,names[name])) - total=total+count - end - grandtotal=grandtotal+count + local data=dataset[i] + local real=data[1] + local total=data[2] + local count=data[3] + local name=data[4] + local source=data[5] + local line=data[6] + local percent=real/wholetime + calls=calls+count functions=functions+1 + name=gsub(name,"%s+"," ") + if #name>length then + name=sub(name,1,length) + end + printer(fmt(seconds(total),seconds(real),percent,count,name,line,source)) end - printer("\n") - printer(format("functions : % 10i\n",functions)) - printer(format("total : % 10i\n",total)) - printer(format("grand total: % 10i\n",grandtotal)) - printer(format("threshold : % 10i\n",threshold)) + printer("") + printer(format("functions : %i",functions)) + printer(format("calls : %i",calls)) + printer(format("overhead : %f",seconds(overhead/1000))) end function debugger.savestats(filename,threshold) local f=io.open(filename,'w') if f then - debugger.showstats(function(str) f:write(str) end,threshold) + debugger.showstats(function(str) f:write(str,"\n") end,threshold) f:close() end end function debugger.enable() - debug.sethook(hook,"c") + if nesting==0 then + running=true + if initialize then + initialize() + end + sethook(hook,"cr") + local function dummy() end + local t=ticks() + for i=1,dummycalls do + dummy() + end + overhead=ticks()-t + end + if nesting>0 then + nesting=nesting+1 + end end function debugger.disable() - debug.sethook() + if nesting>0 then + nesting=nesting-1 + end + if nesting==0 then + sethook() + end end local function showtraceback(rep) local level=2 @@ -9334,9 +10252,661 @@ end -- of closure do -- create closure to overcome 200 locals limit +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true + +-- original size: 7100, stripped down to: 3978 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20309, stripped down to: 13848 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + if type(root)=="table" then + root,what=root[1],root[2] + end + if type(root)=="string" and root~="" then + root=collapsepath(expandname(root)) + if what=="r" or what=="ro" or what=="readable" then + what="read" + elseif what=="w" or what=="wo" or what=="writable" then + what="write" + end + validroots[root]=what=="write" or false + end + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + elseif filenamelogger then + filenamelogger(name,"*",name,false) + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + if trace then + report("resultof: %s",command) + end + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + if trace then + report("execute: %s",command) + end + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + if trace then + report("pipeto: %s",command) + end + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +function sandbox.getrunner(name) + return name and validrunners[name] +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if FFISUPPORTED and ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7757, stripped down to: 6015 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9511,154 +11081,9 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8284, stripped down to: 5176 +-- original size: 9246, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9845,7 +11270,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6358, stripped down to: 4257 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9998,7 +11423,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 56973, stripped down to: 35872 +-- original size: 57003, stripped down to: 35696 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -10013,7 +11438,7 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset +local type,next,setmetatable,getmetatable,tonumber,rawset,select=type,next,setmetatable,getmetatable,tonumber,rawset,select local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local sort=table.sort local utfchar=utf.char @@ -10140,6 +11565,7 @@ local function add_empty(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nt, __p__=top } dt[nt]=t @@ -10161,6 +11587,7 @@ local function add_begin(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nil, __p__=stack[level] } setmetatable(top,mt) @@ -10188,6 +11615,7 @@ local function add_end(spacing,namespace,tag) dt=top.dt nt=#dt+1 dt[nt]=toclose + toclose.ni=nt if toclose.at.xmlns then remove(xmlns) end @@ -10232,7 +11660,13 @@ local function add_special(what,spacing,text) if strip and (what=="@cm@" or what=="@dt@") then else nt=nt+1 - dt[nt]={ special=true,ns="",tg=what,dt={ text } } + dt[nt]={ + special=true, + ns="", + tg=what, + ni=nil, + dt={ text }, + } end end local function set_message(txt) @@ -10285,7 +11719,6 @@ do end local p_rest=(1-P(";"))^0 local p_many=P(1)^0 - local p_char=lpegpatterns.utf8character local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) xml.parsedentitylpeg=parsedentity local predefined_unified={ @@ -10327,13 +11760,27 @@ do [ [[}]] ]="&U+7D;", [ [[~]] ]="&U+7E;", } + local privates_x={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[']] ]="&U+27;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } local privates_n={ } local escaped=utf.remapper(privates_u,"dynamic") local unprivatized=utf.remapper(privates_p,"dynamic") local unspecialized=utf.remapper(privates_s,"dynamic") + local despecialized=utf.remapper(privates_x,"dynamic") xml.unprivatized=unprivatized xml.unspecialized=unspecialized + xml.despecialized=despecialized xml.escaped=escaped local function unescaped(s) local p=privates_n[s] @@ -10778,6 +12225,10 @@ local grammar_unparsed_text=P { "preamble", local function _xmlconvert_(data,settings) settings=settings or {} preparexmlstate(settings) + local preprocessor=settings.preprocessor + if data and data~="" and type(preprocessor)=="function" then + data=preprocessor(data,settings) or data + end if settings.parent_root then mt=getmetatable(settings.parent_root) else @@ -10919,14 +12370,24 @@ function xml.toxml(data) return data end end -local function copy(old) +local function copy(old,p) if old then local new={} for k,v in next,old do - if type(v)=="table" then - new[k]=table.copy(v) - else + local t=type(v)=="table" + if k=="at" then + local t={} + for k,v in next,v do + t[k]=v + end + new[k]=t + elseif k=="dt" then + v.__p__=nil + v=copy(v,new) new[k]=v + v.__p__=p + else + new[k]=v end end local mt=getmetatable(old) @@ -11157,18 +12618,26 @@ local xmlfilehandler=newhandlers { function xml.save(root,name) serialize(root,xmlfilehandler,name) end -local result +local result,r,threshold={},0,512 local xmlstringhandler=newhandlers { name="string", initialize=function() - result={} + r=0 return result end, finalize=function() - return concat(result) + local done=concat(result,"",1,r) + r=0 + if r>threshold then + result={} + end + return done end, handle=function(...) - result[#result+1]=concat {... } + for i=1,select("#",...) do + r=r+1 + result[r]=select(i,...) + end end, } local function xmltostring(root) @@ -11320,7 +12789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 53892, stripped down to: 32508 +-- original size: 53301, stripped down to: 32477 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -11702,6 +13171,14 @@ local function apply_expression(list,expression,order) end return collected end +local function apply_selector(list,specification) + if xml.applyselector then + apply_selector=xml.applyselector + return apply_selector(list,specification) + else + return list + end +end local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb local spaces=S(" \n\r\t\f")^0 local lp_space=S(" \n\r\t\f") @@ -11825,6 +13302,9 @@ end local function register_nodes(nodetest,nodes) return { kind="nodes",nodetest=nodetest,nodes=nodes } end +local function register_selector(specification) + return { kind="selector",specification=specification } +end local function register_expression(expression) local converted=lpegmatch(converter,expression) local runner=load(format(template_e,converted)) @@ -11865,34 +13345,36 @@ local pathparser=Ct { "patterns", (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), - step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + step=((V("shortcuts")+V("selector")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, - shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor")+V("s_lastmatch"), shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), s_descendant=P("**")*Cc(register_descendant), - s_child=P("*")*no_nextcolon*Cc(register_child ), - s_parent=P("..")*Cc(register_parent ), - s_self=P("." )*Cc(register_self ), - s_root=P("^^")*Cc(register_root ), - s_ancestor=P("^")*Cc(register_ancestor ), - descendant=P("descendant::")*Cc(register_descendant ), - child=P("child::")*Cc(register_child ), - parent=P("parent::")*Cc(register_parent ), - self=P("self::")*Cc(register_self ), - root=P('root::')*Cc(register_root ), - ancestor=P('ancestor::')*Cc(register_ancestor ), - descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), - ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), - following=P('following::')*Cc(register_following ), - following_sibling=P('following-sibling::')*Cc(register_following_sibling ), - preceding=P('preceding::')*Cc(register_preceding ), - preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), - reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), - last_match=P('last-match::')*Cc(register_last_match ), + s_child=P("*")*no_nextcolon*Cc(register_child), + s_parent=P("..")*Cc(register_parent), + s_self=P("." )*Cc(register_self), + s_root=P("^^")*Cc(register_root), + s_ancestor=P("^")*Cc(register_ancestor), + s_lastmatch=P("=")*Cc(register_last_match), + descendant=P("descendant::")*Cc(register_descendant), + child=P("child::")*Cc(register_child), + parent=P("parent::")*Cc(register_parent), + self=P("self::")*Cc(register_self), + root=P('root::')*Cc(register_root), + ancestor=P('ancestor::')*Cc(register_ancestor), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self), + following=P('following::')*Cc(register_following), + following_sibling=P('following-sibling::')*Cc(register_following_sibling), + preceding=P('preceding::')*Cc(register_preceding), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling), + last_match=P('last-match::')*Cc(register_last_match), + selector=P("{")*C((1-P("}"))^1)*P("}")/register_selector, nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -12042,6 +13524,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) p.matched=p.matched+1 @@ -12083,6 +13567,9 @@ do elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) + report_lpath("% 10i : se : %s ",(collected and #collected) or 0,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") @@ -12114,6 +13601,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then return pi.finalizer(collected) end @@ -12170,6 +13659,13 @@ do function xml.lastmatch() return lastmatch end + local stack={} + function xml.pushmatch() + insert(stack,lastmatch) + end + function xml.popmatch() + lastmatch=remove(stack) + end end local applylpath=xml.applylpath function xml.filter(root,pattern) @@ -12449,7 +13945,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3787, stripped down to: 2003 +-- original size: 3684, stripped down to: 1957 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -12518,7 +14014,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 30566, stripped down to: 21741 +-- original size: 29835, stripped down to: 21174 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12534,7 +14030,6 @@ local xml=xml local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath -local xmlfilter=xml.filter local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip @@ -12738,7 +14233,17 @@ function xml.replace(root,pattern,whatever) report('replacing',pattern,c,e) end local d=p.dt - d[e.ni]=copiedelement(element,p) + local n=e.ni + local t=copiedelement(element,p) + if type(t)=="table" then + d[n]=t[1] + for i=2,#t do + n=n+1 + insert(d,n,t[i]) + end + else + d[n]=t + end redo_ni(d) end end @@ -13161,7 +14666,7 @@ local obsolete=xml.obsolete xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect xml.delete_element=xml.delete obsolete.delete_element=xml.delete -xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.replace_element=xml.replace obsolete.replace_element=xml.replace xml.each_element=xml.each obsolete.each_element=xml.each xml.process_elements=xml.process obsolete.process_elements=xml.process xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter @@ -13379,7 +14884,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10719, stripped down to: 7841 +-- original size: 10274, stripped down to: 7538 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -13757,7 +15262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6534, stripped down to: 5072 +-- original size: 6407, stripped down to: 4965 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13907,6 +15412,7 @@ function reporters.export(t,methods,filename) if filename then local fullname=file.replacesuffix(filename,method) t.report("saving export in %a",fullname) + dir.mkdirs(file.pathpart(fullname)) io.savedata(fullname,result) else reporters.lines(t,result) @@ -13927,7 +15433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11444, stripped down to: 7830 +-- original size: 11099, stripped down to: 7516 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -14072,11 +15578,6 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if type(profiler)=="table" and not jit then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) -end local prefixes=utilities.storage.allocate() resolvers.prefixes=prefixes local resolved={} @@ -14183,7 +15684,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18619, stripped down to: 11042 +-- original size: 17958, stripped down to: 10705 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -14199,7 +15700,6 @@ local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next local isdir=lfs.isdir -local ostype=os.type local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -14568,7 +16068,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9649, stripped down to: 7131 +-- original size: 9342, stripped down to: 6887 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -14852,7 +16352,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16066, stripped down to: 11938 +-- original size: 16088, stripped down to: 11435 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -15056,18 +16556,6 @@ end caches.getreadablepaths=getreadablepaths caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd=getreadablepaths(...) - for i=1,#rd do - local path=rd[i] - local fullname=file.join(path,filename) - if is_readable(fullname) then - usedreadables[i]=true - return fullname,path - end - end - return caches.setfirstwritablefile(filename,...) -end -function caches.getfirstreadablefile_TEST_ME_FIRST(filename,...) local fullname,path=caches.setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path @@ -15096,18 +16584,22 @@ end function caches.setluanames(path,name) return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end -function caches.loaddata(readables,name) +function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } end for i=1,#readables do local path=readables[i] - local tmaname,tmcname=caches.setluanames(path,name) local loader=false + local tmaname,tmcname=caches.setluanames(path,name) if isfile(tmcname) then loader=loadfile(tmcname) end if not loader and isfile(tmaname) then + local tmacrap,tmcname=caches.setluanames(writable,name) + if isfile(tmcname) then + loader=loadfile(tmcname) + end utilities.lua.compile(tmaname,tmcname) if isfile(tmcname) then loader=loadfile(tmcname) @@ -15228,7 +16720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5488, stripped down to: 4101 +-- original size: 5310, stripped down to: 3980 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -15240,7 +16732,6 @@ if not modules then modules={} end modules ['data-met']={ local find,format=string.find,string.format local sequenced=table.sequenced local addurlscheme,urlhashed=url.addscheme,url.hashed -local getcurrentdir=lfs.currentdir local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) @@ -15347,7 +16838,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67241, stripped down to: 46427 +-- original size: 67524, stripped down to: 46632 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15401,6 +16892,7 @@ resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf @@ -15444,7 +16936,6 @@ local function resolvevariable(k) end local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) @@ -15590,23 +17081,29 @@ local function identify_configuration_files() end reportcriticalvariables(cnfspec) local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname=resolvers.luacnfname - for i=1,#cnfpaths do - local filepath=cnfpaths[i] - local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolveprefix(filename) - if trace_locating then - local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) - report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) - end - if isfile(realname) then - specification[#specification+1]=filename + local function locatecnf(luacnfname,kind) + for i=1,#cnfpaths do + local filepath=cnfpaths[i] + local filename=collapsepath(filejoin(filepath,luacnfname)) + local realname=resolveprefix(filename) if trace_locating then - report_resolving("found configuration file %a",realname) + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) + report_resolving("looking for %s %a on %s path %a from specification %a", + kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath) + end + if isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found %s configuration file %a",kind,realname) + end end end end + locatecnf(resolvers.luacnfname,"regular") + if #specification==0 then + locatecnf(resolvers.luacnffallback,"fallback") + end if trace_locating then report_resolving() end @@ -16903,7 +18400,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4236, stripped down to: 3144 +-- original size: 4090, stripped down to: 3059 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -17025,7 +18522,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 935, stripped down to: 838 +-- original size: 910, stripped down to: 823 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -17055,7 +18552,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 548, stripped down to: 483 +-- original size: 530, stripped down to: 475 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -17078,7 +18575,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3976, stripped down to: 3391 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -17186,7 +18683,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5148, stripped down to: 3680 +-- original size: 5029, stripped down to: 3607 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -17256,7 +18753,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name) + stored=caches.loaddata(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -17305,7 +18802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4000, stripped down to: 3052 +-- original size: 4045, stripped down to: 3110 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -17350,7 +18847,7 @@ function resolvers.automount(usecache) end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) -function statistics.savefmtstatus(texname,formatbanner,sourcefile) +function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") @@ -17361,6 +18858,10 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) sourcefile=sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) + lua.registerfinalizer(function() + logs.report("format banner","%s",banner) + logs.newline() + end) end end function statistics.checkfmtstatus(texname) @@ -17396,7 +18897,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 9036, stripped down to: 7041 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -17633,7 +19134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8712, stripped down to: 5726 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17822,7 +19323,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6779, stripped down to: 5444 +-- original size: 6653, stripped down to: 5467 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17868,11 +19369,21 @@ function resolvers.schemes.cleanname(specification) end return hash end -local cached,loaded,reused,thresholds,handlers={},{},{},{},{} -local function runcurl(name,cachename) - local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name - os.execute(command) -end +local cached={} +local loaded={} +local reused={} +local thresholds={} +local handlers={} +local runner=sandbox.registerrunner { + name="curl resolver", + method="execute", + program="curl", + template="--silent -- insecure --create-dirs --output %cachename% %original%", + checkers={ + cachename="cache", + original="url", + } +} local function fetch(specification) local original=specification.original local scheme=specification.scheme @@ -17894,7 +19405,10 @@ local function fetch(specification) report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") end logs.flush() - runcurl(original,cachename) + runner { + original=original, + cachename=cachename, + } end end if io.exists(cachename) then @@ -18003,7 +19517,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4447, stripped down to: 3302 +-- original size: 4207, stripped down to: 3137 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -18045,8 +19559,6 @@ function helpers.cleanpath(path) return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib -local getextraluapaths=package.extraluapaths -local getextralibpaths=package.extralibpaths local registerpath=helpers.registerpath local lualibfile=helpers.lualibfile local luaformatpaths @@ -18112,7 +19624,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2494, stripped down to: 2047 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -18179,7 +19691,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2674, stripped down to: 1658 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -18235,7 +19747,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2815, stripped down to: 2415 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -18315,7 +19827,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11846, stripped down to: 6059 +-- original size: 13595, stripped down to: 7500 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -18324,35 +19836,51 @@ if not modules then modules={} end modules ['util-lib']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } -local gsub,find=string.gsub,string.find -local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join -local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles -local loaded=package.loaded -local report_swiglib=logs.reporter("swiglib") -local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local type=type +local next=next +local pcall=pcall +local gsub=string.gsub +local find=string.find +local sort=table.sort +local pathpart=file.pathpart +local nameonly=file.nameonly +local joinfile=file.join +local removesuffix=file.removesuffix +local findfile=resolvers.findfile +local findfiles=resolvers.findfiles +local expandpaths=resolvers.expandedpathlistfromvariable +local qualifiedpath=file.is_qualified_path +local isfile=lfs.isfile local done=false -local function requireswiglib(required,version) - local trace_swiglib=trace_swiglib or package.helpers.trace - local library=loaded[required] - if library==nil then - if trace_swiglib then - report_swiglib("requiring library %a with version %a",required,version or "any") - end - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) +local function locate(required,version,trace,report,action) + if type(required)~="string" then + report("provide a proper library name") + return + end + if trace then + report("requiring library %a with version %a",required,version or "any") + end + local found_library=nil + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + if qualifiedpath(required) then + if isfile(required) then + found_library=required + end + else local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false local engine=environment.ownmain or false - if trace_swiglib and not done then - local list=resolvers.expandedpathlistfromvariable("lib") + if trace and not done then + local list=expandpaths("lib") for i=1,#list do - report_swiglib("tds path %i: %s",i,list[i]) + report("tds path %i: %s",i,list[i]) end end local function found(locate,asked_library,how,...) - if trace_swiglib then - report_swiglib("checking %s: %a",how,asked_library) + if trace then + report("checking %s: %a",how,asked_library) end return locate(asked_library,...) end @@ -18360,45 +19888,45 @@ local function requireswiglib(required,version) local found=nil if version then local asked_library=joinfile(required_path,version,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end if not found or found=="" then local asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end return found and found~="" and found or false end local function attempt(checkpattern) - if trace_swiglib then - report_swiglib("checking tds lib paths strictly") + if trace then + report("checking tds lib paths strictly") end local found=findfile and check(findfile,"lib") if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace_swiglib then - report_swiglib("checking tds lib paths with wildcard") + if trace then + report("checking tds lib paths with wildcard") end local asked_library=joinfile(required_path,".*",required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","latest version",asked_library) + if trace then + report("checking %s: %a","latest version",asked_library) end local list=findfiles(asked_library,"lib",true) if list and #list>0 then - table.sort(list) + sort(list) local found=list[#list] if found and (not checkpattern or find(found,checkpattern)) then return found end end - if trace_swiglib then - report_swiglib("checking lib paths") + if trace then + report("checking lib paths") end package.extralibpath(environment.ownpath) local paths=package.libpaths() @@ -18410,89 +19938,143 @@ local function requireswiglib(required,version) end return false end - local found_library=nil if engine then - if trace_swiglib then - report_swiglib("attemp 1, engine %a",engine) + if trace then + report("attemp 1, engine %a",engine) end found_library=attempt("/"..engine.."/") if not found_library then - if trace_swiglib then - report_swiglib("attemp 2, no engine",asked_library) + if trace then + report("attemp 2, no engine",asked_library) end found_library=attempt() end else found_library=attempt() end - if not found_library then - if trace_swiglib then - report_swiglib("not found: %a",required) - end + end + if not found_library then + if trace then + report("not found: %a",required) + end + library=false + else + if trace then + report("found: %a",found_library) + end + local message,result=action(found_library,required_base) + if result then + library=result + else library=false + report("load error: message %a, library %a",tostring(message),found_library or "no library") + end + end + if not library then + report("unknown: %a",required) + elseif trace then + report("stored: %a",required) + end + return library +end +do + local report_swiglib=logs.reporter("swiglib") + local trace_swiglib=false + local savedrequire=require + local loadedlibs={} + local loadlib=package.loadlib + local pushdir=dir.push + local popdir=dir.pop + trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) + function requireswiglib(required,version) + local library=loadedlibs[library] + if library==nil then + local trace_swiglib=trace_swiglib or package.helpers.trace + library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) + pushdir(pathpart(name)) + local opener="luaopen_"..base + if trace_swiglib then + report_swiglib("opening: %a with %a",name,opener) + end + local library,message=loadlib(name,opener) + local libtype=type(library) + if libtype=="function" then + library=library() + message=true + else + report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") + library=false + end + popdir() + return message,library + end) + loadedlibs[required]=library or false + end + return library + end + function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) else - local path=pathpart(found_library) - local base=nameonly(found_library) - dir.push(path) + return savedrequire(name) + end + end + local swiglibs={} + local initializer="core" + function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) if trace_swiglib then - report_swiglib("found: %a",found_library) - end - local message=nil - local opener="luaopen_"..required_base - library,message=package.loadlib(found_library,opener) - local libtype=type(library) - if libtype=="function" then - library=library() + report_swiglib("loading %a",name) + end + if not find(name,"%."..initializer.."$") then + fullname="swiglib."..name.."."..initializer else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + fullname="swiglib."..name end - dir.pop() + library=requireswiglib(fullname,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) end - if not library then - report_swiglib("unknown: %a",required) - elseif trace_swiglib then - report_swiglib("stored: %a",required) - end - loaded[required]=library - else - report_swiglib("reused: %a",required) + return library end - return library + statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end + end) end -local savedrequire=require -function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) +if FFISUPPORTED and ffi and ffi.load then + local report_ffilib=logs.reporter("ffilib") + local trace_ffilib=false + local savedffiload=ffi.load + trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) + local function locateindeed(name) + local message,library=pcall(savedffiload,removesuffix(name)) + if type(library)=="userdata" then + return library + else + return false + end end -end -local swiglibs={} -local initializer="core" -function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) + function ffilib(required,version) + if version=="system" then + return locateindeed(name) + else + return locate(required,version,trace_ffilib,report_ffilib,locateindeed) end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer + end + function ffi.load(name) + local library=ffilib(name) + if type(library)=="userdata" then + return library else - fullname="swiglib."..name + report_ffilib("trying to load %a using normal loader",name) + return savedffiload(name) end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end -statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end -end) end -- of closure @@ -18501,7 +20083,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5914, stripped down to: 2584 +-- original size: 5703, stripped down to: 2507 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -18604,7 +20186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 6967, stripped down to: 5631 +-- original size: 9144, stripped down to: 7291 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -18618,23 +20200,67 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local trackers=environment.argument("trackers") - local directives=environment.argument("directives") +local function primaryflags() + local arguments=environment.arguments + local flags={} + if arguments.silent then + flags[#flags+1]="--interaction=batchmode" + end + if arguments.jit then + flags[#flags+1]="--jiton" + end + return concat(flags," ") +end +local function secondaryflags() + local arguments=environment.arguments + local trackers=arguments.trackers + local directives=arguments.directives local flags={} if trackers and trackers~="" then - flags={ "--trackers="..quoted(trackers) } + flags[#flags+1]="--c:trackers="..quoted(trackers) end if directives and directives~="" then - flags={ "--directives="..quoted(directives) } + flags[#flags+1]="--c:directives="..quoted(directives) + end + if arguments.silent then + flags[#flags+1]="--c:silent" end - if environment.argument("jit") then - flags={ "--jiton" } + if arguments.jit then + flags[#flags+1]="--c:jiton" + end + if arguments.ansi then + flags[#flags+1]="--c:ansi" end return concat(flags," ") end -function environment.make_format(name,silent) +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] +local checkers={ + primaryflags="string", + secondaryflags="string", + luafile="readable", + texfile="readable", + redirect="string", + dump="string", +} +local runners={ + luatex=sandbox.registerrunner { + name="make luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="make luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} +function environment.make_format(name,arguments) local engine=environment.ownmain or "luatex" + local silent=environment.arguments.silent local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" if path~="" then @@ -18690,11 +20316,20 @@ function environment.make_format(name,silent) lfs.chdir(olddir) return end - local dump=os.platform=="unix" and "\\\\dump" or "\\dump" - if silent then + local specification={ + primaryflags=primaryflags(), + secondaryflags=secondaryflags(), + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), + dump=os.platform=="unix" and "\\\\dump" or "\\dump", + } + local runner=runners[engine] + if not runner then + report_format("format %a cannot be generated, no runner available for engine %a",name,engine) + elseif silent then statistics.starttiming() - local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - local result=os.execute(command) + specification.redirect="> temp.log" + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -18703,9 +20338,7 @@ function environment.make_format(name,silent) end os.remove("temp.log") else - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - report_format("running command: %s\n",command) - os.execute(command) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -18718,6 +20351,30 @@ function environment.make_format(name,silent) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -18739,9 +20396,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -18750,10 +20416,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 797557 --- stripped bytes : 289197 +-- original bytes : 842443 +-- stripped bytes : 306317 -- end library merge @@ -18777,6 +20443,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -18810,8 +20477,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) @@ -18983,7 +20651,7 @@ local helpinfo = [[ <metadata> <entry name="name">mtxrun</entry> <entry name="detail">ConTeXt TDS Runner Tool</entry> - <entry name="version">1.31</entry> + <entry name="version">1.32</entry> </metadata> <flags> <category name="basic"> @@ -19046,6 +20714,7 @@ local helpinfo = [[ </subcategory> <subcategory> <flag name="expand-braces"><short>expand complex variable</short></flag> + <flag name="resolve-path"><short>expand variable (completely resolve paths)</short></flag> <flag name="expand-path"><short>expand variable (resolve paths)</short></flag> <flag name="expand-var"><short>expand variable (resolve references)</short></flag> <flag name="show-path"><short>show path expansion of ...</short></flag> @@ -19063,7 +20732,7 @@ local helpinfo = [[ local application = logs.application { name = "mtxrun", - banner = "ConTeXt TDS Runner Tool 1.31", + banner = "ConTeXt TDS Runner Tool 1.32", helpinfo = helpinfo, } @@ -20029,6 +21698,13 @@ elseif e_argument("expand-path") then environment.initializearguments(environment.arguments_after) resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) +elseif e_argument("resolve-path") then + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.cleanedpathlist, environment.files) + elseif e_argument("expand-var") or e_argument("expand-variable") then -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) diff --git a/scripts/context/ruby/rslb/base.rb b/scripts/context/ruby/rslb/base.rb new file mode 100644 index 000000000..66ce3a19e --- /dev/null +++ b/scripts/context/ruby/rslb/base.rb @@ -0,0 +1,270 @@ +# hans hagen, pragma-ade, hasselt nl +# experimental code, don't touch it + +require 'rexml/document.rb' + +class Array + + def downcase + self.collect do |l| + l.to_s.downcase + end + end + +end + +class Resource + + @@rslburl = 'http://www.pragma-ade.com/rng/rslb.rng' + @@rslbns = 'rl' + @@rslbtmp = 'rslbtool-tmp.xml' + + def initialize (filename='',namespace=@@rslbns) + @ns = if namespace then @@rslbns + ':' else '' end + set_filename(filename) + @library = REXML::Document.new(skeleton, + {:ignore_whitespace_nodes => :all, + :compress_whitespace => :all}) + @lastindex = 0 + @texexec = 'texexec' + @downcaselabels = true + @downcasefilenames = true + end + + def keeplabelcase + @downcaselabels = false + end + + def keepfilenamecase + @downcasefilenames = false + end + + def outer_skeleton (str) + tmp = if @ns.empty? then '' else " xmlns:#{@ns.sub(':','')}='#{@@rslburl}'" end + "<?xml version='1.0'?>\n" + "<#{@ns}library#{tmp}>\n" + str + "\n</#{@ns}library>" + end + + def skeleton + outer_skeleton("<#{@ns}description>" + + "<#{@ns}organization>unknown</#{@ns}organization>" + + "<#{@ns}project>unknown</#{@ns}project>" + + "<#{@ns}product>unknown</#{@ns}product>" + + "<#{@ns}comment>unknown</#{@ns}comment>" + + "</#{@ns}description>") + end + + def set_filename (filename) + @filename = if filename.empty? then 'unknown' else filename end + @fullname = @filename + @filename = File.basename(@filename).sub(/\..*$/,'') + end + + def set_texexec (filename) + print "setting texexec binary to: #{filename}\n" + @texexec = filename + end + + def load (filename='') + set_filename(filename) + if not filename.empty? and FileTest.file?(filename) # todo: test op valide fig base + @library = REXML::Document.new(File.new(filename), + {:ignore_whitespace_nodes => :all, + :compress_whitespace => :all}) + unless @library.root.prefix.empty? + @ns = @library.root.prefix + ':' + end + else + initialize(filename,!@ns.empty?) + end + end + + def save (filename) + filename += '.xml' unless filename =~ /\..*?$/ + if not filename.empty? and f = open(filename,'w') + @library.write(f,0) + f.close + end + end + + def figure_labels + REXML::XPath.match(@library.root,"/#{@ns}library/#{@ns}figure/#{@ns}label/text()") + end + + def figure_records + @library.elements.to_a("/#{@ns}library/#{@ns}figure") + end + + def figure_files + REXML::XPath.match(@library.root,"/#{@ns}library/#{@ns}figure/#{@ns}file/text()") + end + + def delete_figure (label='') + return if label.empty? + labels = figure_labels + labels.each_index do |i| + if labels[i].to_s.downcase == label.downcase + @library.elements.delete_all("/#{@ns}library/#{@ns}figure[#{i+1}]") + end + end + end + + def add_figure (file='',label='',prefix='') + return if file.empty? or file.match(/^#{@filename}\..*$/i) + labels = figure_labels + prefix = @filename if prefix.empty? + if label.empty? + i = @lastindex + loop do + i += 1 + label = prefix + ' ' + i.to_s + break unless labels.include?(label) + end + else + delete_figure(label) unless label.empty? + end + e = REXML::Element.new("#{@ns}figure") + l = REXML::Element.new("#{@ns}label") + f = REXML::Element.new("#{@ns}file") + l.text, f.text = label, file + e.add_element(l) + e.add_element(f) + @library.root.add_element(e) + end + + def add_figures (list='',prefix='') + if @downcasefilenames then + files = figure_files.downcase + [list].flatten.downcase.each do |f| + next unless FileTest.file?(f) + add_figure(f,'',prefix) unless files.include?(f) + end + else + files = figure_files + [list].flatten.each do |f| + next unless FileTest.file?(f) + add_figure(f,'',prefix) unless files.include?(f) + end + end + end + + def delete_figures (list='') + [list].flatten.downcase.each do |l| + delete_figure(l) + end + end + + def sort_figures + if @downcaselabels then + labels = figure_labels.downcase + else + labels = figure_labels + end + return unless labels + figures = figure_records + @library.elements.delete_all("/#{@ns}library/#{@ns}figure") + labels = labels.collect do |l| # prepare numbers + l.gsub(/(\d+)/) do |d| sprintf('%05d', d) end + end + labels.sort.each do |s| + @library.root.add_element(figures[labels.index(s)]) + end + end + + def purge_figures + REXML::XPath.each(@library.root,"/#{@ns}library/#{@ns}figure") do |e| + filename = REXML::XPath.match(e,"#{@ns}file/text()").to_s + e.parent.delete(e) unless FileTest.file?(filename) + end + end + + def run_command(command) + print "calling #{command}\n" + print "\n" + begin + system(command) + rescue + # sorry again + end + print "\n" + end + + def create_dummies(process=false,result='',zipalso='') + result = @filename if result.empty? + list = REXML::XPath.match(@library.root,"/#{@ns}library/#{@ns}usage") + begin + File.delete(result+'.pdf') + rescue + # no way + end + return unless list && list.length>0 + done = Array.new + list.each do |e| + t = REXML::XPath.match(e,"#{@ns}type/text()") + s = REXML::XPath.match(e,"#{@ns}state/text()") + if t && (t.to_s == 'figure') && s && (s.to_s == 'missing') + begin + f = REXML::XPath.match(e,"#{@ns}file/text()").to_s + if done.index(f) + print "skipping dummy figure: " + f + "\n" + elsif f =~ /\s/o + print "skipping crappy fname: " + f + "\n" + elsif f == 'dummy' + print "skipping dummy figure: " + f + "\n" + else + print "creating dummy figure: " + f + "\n" + if process && (x = open(@@rslbtmp,'w')) + x.puts(outer_skeleton(e.to_s)) + x.close + run_command ("#{@texexec} --pdf --once --batch --silent --random --use=res-10 --xml --result=#{f} #{@@rslbtmp}") + end + done.push(f+'.pdf') + begin + File.delete(@@rslbtmp) + rescue + # sorry once more + end + end + rescue + # sorry, skip 'm + end + end + end + if process && (done.length>0) + begin + File.delete(result + '.zip') + rescue + # ok + end + run_command("zip #{result+'.zip'} #{@fullname}") + unless zipalso.empty? + begin + zipalso.split(',').each do |name| + run_command("zip #{result+'.zip'} #{name}") + end + end + end + done.each do |name| + run_command("zip #{result+'.zip'} #{name}") + end + run_command("#{@texexec} --pdf --batch --silent --use=res-11 --xml --result=#{result} #{@fullname}") + done.each do |name| + begin + File.delete(name) + rescue + # sorry + end + end + end + end + +end + +# reslib = Resource.new +# reslib.load('f.xml') # reslib.load('figbase.xml') +# reslib.delete_figure('figbase 5') +# reslib.delete_figure('figbase 5') +# reslib.add_figure('a.pdf') +# reslib.add_figure('b.pdf','something') +# reslib.add_figure('c.pdf') +# reslib.add_files('x.pdf') +# reslib.save('figbase.tmp') diff --git a/scripts/context/stubs/install/first-setup.sh b/scripts/context/stubs/install/first-setup.sh index 9249fd2e0..ea72d55d4 100644 --- a/scripts/context/stubs/install/first-setup.sh +++ b/scripts/context/stubs/install/first-setup.sh @@ -5,14 +5,11 @@ # you may change this if you want ... CONTEXTROOT="$PWD/tex" -# suggested by Tobias Florek to check for ruby & rsync +# suggested by Tobias Florek to check for rsync if [ ! -x "`which rsync`" ]; then echo "You need to install rsync first." exit 1 fi -if [ ! -x "`which ruby`" ]; then - echo "You might want to install Ruby first if you want to use pdfTeX or XeTeX." -fi system=`uname -s` cpu=`uname -m` @@ -23,11 +20,32 @@ case "$system" in case "$cpu" in i*86) platform="linux" ;; x86_64|ia64) platform="linux-64" ;; + # a little bit of cheating with ppc64 (won't work on Gentoo) ppc|ppc64) platform="linux-ppc" ;; + # we currently support just mipsel, but Debian is lying (reports mips64) # we need more hacks to fix the situation, this is just a temporary solution mips|mips64|mipsel|mips64el) platform="linux-mipsel" ;; + + armv7l) platform="linux-armhf" + # machine id output by uname(1) is insufficent to determine whether this + # is a soft or hard float system so we check ourselves. + # a) binutils, this should work almost everywhere + if $(which readelf >/dev/null 2>&1); then + readelf -A /proc/self/exe | grep -q '^ \+Tag_ABI_VFP_args' + if [ ! $? ]; then + platform="linux-armel" + fi + # b) debian-specific fallback + elif $(which dpkg >/dev/null 2>&1); then + if [ "$(dpkg --print-architecture)" = armel ]; then + platform="linux-armel" + fi + fi + # else go with hard fp + ;; + *) platform="unknown" ;; esac ;; # Mac OS X @@ -49,15 +67,29 @@ case "$system" in # kFreeBSD (debian) GNU/kFreeBSD) case "$cpu" in - i*86) platform="kfreebsd-i386" ;; - x86_64|amd64) platform="kfreebsd-amd64" ;; + #i*86) platform="kfreebsd-i386" ;; + #x86_64|amd64) platform="kfreebsd-amd64" ;; *) platform="unknown" ;; esac ;; # cygwin + # OpenBSD + OpenBSD|freebsd) + case "$cpu" in + i*86) platform="openbsd" ;; + x86_64) platform="openbsd" ;; # no special binaries are available yet + amd64) platform="openbsd-amd64" ;; + *) platform="unknown" ;; + esac ;; CYGWIN*) case "$cpu" in i*86) platform="cygwin" ;; - x86_64|ia64) platform="cygwin-64" ;; + x86_64|ia64) platform="cygwin" ;; + *) platform="unknown" ;; + esac ;; + # UWIN + UWIN*) + case "$cpu" in + i*86) platform="mswin" ;; *) platform="unknown" ;; esac ;; # SunOS/Solaris @@ -87,6 +119,10 @@ if test "$platform" = "unknown" ; then echo "Error: your system \"$system $cpu\" is not supported yet." echo "Please report to the ConTeXt mailing-list (ntg-context@ntg.nl)" exit +elif test "$platform" = "linux-ppc" ; then + echo "Error: support for your system \"$platform\" has been dropped." + echo "Please ask on to the ConTeXt mailing-list if you still need it (ntg-context@ntg.nl)" + exit fi # if you want to enforce some specific platform @@ -96,11 +132,16 @@ fi # download or rsync the latest scripts first rsync -rlptv rsync://contextgarden.net/minimals/setup/$platform/bin . +# use native windows binaries on cygwin +if test "$platform" = "cygwin" ; then + platform=mswin +fi + # download or update the distribution # you may remove the --context=beta switch if you want to use "current" # you can use --engine=luatex if you want just mkiv env PATH="$PWD/bin:$CONTEXTROOT/texmf-$platform/bin:$PATH" \ -mtxrun --script ./bin/mtx-update.lua --force --update --make --context=beta --platform=$platform --texroot="$CONTEXTROOT" $@ +./bin/mtxrun --script ./bin/mtx-update.lua --force --update --make --context=beta --platform=$platform --texroot="$CONTEXTROOT" $@ echo echo "When you want to use context, you need to initialize the tree by typing:" @@ -113,8 +154,3 @@ echo "to PATH variable if you want to set it permanently." echo "This can usually be done in .bashrc, .bash_profile" echo "(or whatever file is used to initialize your shell)." echo - -if [ ! -x "`which ruby`" ]; then - echo "You might want to install Ruby first if you want to use pdfTeX or XeTeX." - echo -fi diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 7b711a88d..873770cac 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 4734, stripped down to: 2626 +-- original size: 5125, stripped down to: 2881 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -162,6 +162,283 @@ if flush then local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end end +FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +if not FFISUPPORTED then + local okay;okay,ffi=pcall(require,"ffi") + FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +end +if not FFISUPPORTED then + ffi=nil +elseif not ffi.number then + ffi.number=tonumber +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 9667, stripped down to: 6678 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b==false then + return nil + elseif b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] or false +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end end -- of closure @@ -170,7 +447,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10949, stripped down to: 8037 +-- original size: 10587, stripped down to: 7815 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -460,7 +737,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38185, stripped down to: 20990 +-- original size: 37748, stripped down to: 20111 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -552,6 +829,7 @@ patterns.nonwhitespace=nonwhitespace local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local nospacer=Cs((whitespace^1/""+nonwhitespace^1)^0) local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) @@ -561,6 +839,7 @@ local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.nospacer=nospacer patterns.b_collapser=b_collapser patterns.m_collapser=m_collapser patterns.e_collapser=e_collapser @@ -1027,27 +1306,7 @@ function lpeg.append(list,pp,delayed,checked) end local p_false=P(false) local p_true=P(true) -local function make(t) - local function making(t) - local p=p_false - local keys=sortedkeys(t) - for i=1,#keys do - local k=keys[i] - if k~="" then - local v=t[k] - if v==true then - p=p+P(k)*p_true - elseif v==false then - else - p=p+P(k)*making(v) - end - end - end - if t[""] then - p=p+p_true - end - return p - end +local function make(t,rest) local p=p_false local keys=sortedkeys(t) for i=1,#keys do @@ -1058,10 +1317,13 @@ local function make(t) p=p+P(k)*p_true elseif v==false then else - p=p+P(k)*making(v) + p=p+P(k)*make(v,v[""]) end end end + if rest then + p=p+p_true + end return p end local function collapse(t,x) @@ -1264,7 +1526,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 372, stripped down to: 329 +-- original size: 361, stripped down to: 322 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1283,7 +1545,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5983, stripped down to: 2959 +-- original size: 6419, stripped down to: 3339 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1321,22 +1583,26 @@ end local stripper=patterns.stripper local fullstripper=patterns.fullstripper local collapser=patterns.collapser +local nospacer=patterns.nospacer local longtostring=patterns.longtostring function string.strip(str) - return lpegmatch(stripper,str) or "" + return str and lpegmatch(stripper,str) or "" end function string.fullstrip(str) - return lpegmatch(fullstripper,str) or "" + return str and lpegmatch(fullstripper,str) or "" end function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" + return str and lpegmatch(collapser,str) or "" +end +function string.nospaces(str) + return str and lpegmatch(nospacer,str) or "" end function string.longtostring(str) - return lpegmatch(longtostring,str) or "" + return str and lpegmatch(longtostring,str) or "" end local pattern=P(" ")^0*P(-1) function string.is_empty(str) - if str=="" then + if not str or str=="" then return true else return lpegmatch(pattern,str) and true or false @@ -1381,6 +1647,21 @@ function string.tformat(fmt,...) end string.quote=string.quoted string.unquote=string.unquoted +if not string.bytetable then + local limit=5000 + function string.bytetable(str) + local n=#str + if n>limit then + local t={ byte(str,1,limit) } + for i=limit+1,n do + t[i]=byte(str,i) + end + return t + else + return { byte(str,1,n) } + end + end +end end -- of closure @@ -1389,7 +1670,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 36997, stripped down to: 22376 +-- original size: 39608, stripped down to: 23165 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1716,19 +1997,23 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions,metacheck local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } -local function simple_table(t) +local function is_simple_table(t,hexify) local nt=#t if nt>0 then local n=0 for _,v in next,t do n=n+1 + if type(v)=="table" then + return nil + end end + local haszero=rawget(t,0) if n==nt then local tt={} for i=1,nt do @@ -1738,10 +2023,10 @@ local function simple_table(t) if hexify then tt[i]=format("0x%X",v) else - tt[i]=tostring(v) + tt[i]=v end elseif tv=="string" then - tt[i]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then tt[i]=v and "true" or "false" else @@ -1749,10 +2034,32 @@ local function simple_table(t) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + if hexify then + tt[i+1]=format("0x%X",v) + else + tt[i+1]=v + end + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil end +table.is_simple_table=is_simple_table local propername=patterns.propername local function dummy() end local function do_serialize(root,name,depth,level,indexed) @@ -1786,7 +2093,7 @@ local function do_serialize(root,name,depth,level,indexed) if compact then last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -1814,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -1851,6 +2158,7 @@ local function do_serialize(root,name,depth,level,indexed) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) @@ -1873,6 +2181,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,v)) else @@ -1888,13 +2197,14 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={},",depth,k and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then if tk=="number" then if hexify then @@ -1904,6 +2214,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={ %s },",depth,k and "true" or "false",concat(st,", "))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else @@ -1924,6 +2235,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%s,",depth,tostring(k),v and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%s,",depth,k,v and "true" or "false")) else @@ -1940,6 +2252,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=load(%q),",depth,k and "true" or "false",f)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=load(%q),",depth,k,f)) else @@ -1955,6 +2268,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",tostring(v))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,tostring(v))) else @@ -1976,6 +2290,7 @@ local function serialize(_handle,root,name,specification) functions=specification.functions compact=specification.compact inline=specification.inline and compact + metacheck=specification.metacheck if functions==nil then functions=true end @@ -1985,6 +2300,9 @@ local function serialize(_handle,root,name,specification) if inline==nil then inline=compact end + if metacheck==nil then + metacheck=true + end else noquotes=false hexify=false @@ -1992,6 +2310,7 @@ local function serialize(_handle,root,name,specification) compact=true inline=true functions=true + metacheck=true end if tname=="string" then if name=="return" then @@ -2015,7 +2334,7 @@ local function serialize(_handle,root,name,specification) handle("t={") end if root then - if getmetatable(root) then + if metacheck and getmetatable(root) then local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end @@ -2091,6 +2410,38 @@ local function flattened(t,f,depth) return f end table.flattened=flattened +local function collapsed(t,f,h) + if f==nil then + f={} + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsed(v,f,h) + elseif not h[v] then + f[#f+1]=v + h[v]=true + end + end + return f +end +local function collapsedhash(t,h) + if h==nil then + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsedhash(v,h) + else + h[v]=true + end + end + return h +end +table.collapsed=collapsed +table.collapsedhash=collapsedhash local function unnest(t,f) if not f then f={} @@ -2197,6 +2548,12 @@ function table.swapped(t,s) end return n end +function table.hashed(t) + for i=1,#t do + t[t[i]]=i + end + return t +end function table.mirrored(t) local n={} for k,v in next,t do @@ -2365,7 +2722,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 9001, stripped down to: 6512 +-- original size: 11790, stripped down to: 6961 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2375,6 +2732,7 @@ if not modules then modules={} end modules ['l-io']={ license="see context related readme files" } local io=io +local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor @@ -2384,50 +2742,56 @@ if string.find(os.getenv("PATH"),";",1,true) then else io.fileseparator,io.pathseparator="/",":" end -local function readall(f) - return f:read("*all") -end +local large=2^24 +local medium=large/16 +local small=medium/8 local function readall(f) local size=f:seek("end") - if size==0 then - return "" - elseif size<1024*1024 then + if size>0 then f:seek("set",0) - return f:read('*all') - else - local done=f:seek("set",0) - local step - if size<1024*1024 then - step=1024*1024 - elseif size>16*1024*1024 then - step=16*1024*1024 - else - step=floor(size/(1024*1024))*1024*1024/8 - end - local data={} - while true do - local r=f:read(step) - if not r then - return concat(data) - else - data[#data+1]=r - end - end + return f:read(size) + else + return "" end end io.readall=readall function io.loaddata(filename,textmode) - local f=io.open(filename,(textmode and 'r') or 'rb') + local f=open(filename,(textmode and 'r') or 'rb') if f then - local data=readall(f) + local size=f:seek("end") + local data=nil + if size>0 then + f:seek("set",0) + data=f:read(size) + end f:close() - if #data>0 then - return data + return data + end +end +function io.copydata(source,target,action) + local f=open(source,"rb") + if f then + local g=open(target,"wb") + if g then + local size=f:seek("end") + if size>0 then + f:seek("set",0) + local data=f:read(size) + if action then + data=action(data) + end + if data then + g:write(data) + end + end + g:close() end + f:close() + flush() end end function io.savedata(filename,data,joiner) - local f=io.open(filename,"wb") + local f=open(filename,"wb") if f then if type(data)=="table" then f:write(concat(data,joiner or "")) @@ -2437,40 +2801,70 @@ function io.savedata(filename,data,joiner) f:write(data or "") end f:close() - io.flush() + flush() return true else return false end end -function io.loadlines(filename,n) - local f=io.open(filename,'r') - if not f then - elseif n then - local lines={} - for i=1,n do - local line=f:read("*lines") - if line then - lines[#lines+1]=line - else - break +if fio and fio.readline then + local readline=fio.readline + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=readline(f) + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=readline(f) + f:close() + if line and #line>0 then + return line end end - f:close() - lines=concat(lines,"\n") - if #lines>0 then - return lines - end - else - local line=f:read("*line") or "" - f:close() - if #line>0 then - return line + end +else + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end end end end function io.loadchunk(filename,n) - local f=io.open(filename,'rb') + local f=open(filename,'rb') if f then local data=f:read(n or 1024) f:close() @@ -2480,7 +2874,7 @@ function io.loadchunk(filename,n) end end function io.exists(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return false else @@ -2489,7 +2883,7 @@ function io.exists(filename) end end function io.size(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return 0 else @@ -2498,11 +2892,11 @@ function io.size(filename) return s end end -function io.noflines(f) +local function noflines(f) if type(f)=="string" then - local f=io.open(filename) + local f=open(filename) if f then - local n=f and io.noflines(f) or 0 + local n=f and noflines(f) or 0 f:close() return n else @@ -2517,6 +2911,7 @@ function io.noflines(f) return n end end +io.noflines=noflines local nextchar={ [ 4]=function(f) return f:read(1,1,1,1) @@ -2594,16 +2989,16 @@ function io.bytes(f,n) end function io.ask(question,default,options) while true do - io.write(question) + write(question) if options then - io.write(format(" [%s]",concat(options,"|"))) + write(format(" [%s]",concat(options,"|"))) end if default then - io.write(format(" [%s]",default)) + write(format(" [%s]",default)) end - io.write(format(" ")) - io.flush() - local answer=io.read() + write(format(" ")) + flush() + local answer=read() answer=gsub(answer,"^%s*(.*)%s*$","%1") if answer=="" and default then return default @@ -2625,7 +3020,7 @@ function io.ask(question,default,options) end end end -local function readnumber(f,n,m) +local function readnumber(f,n,m) if m then f:seek("set",n) n=m @@ -2634,31 +3029,31 @@ local function readnumber(f,n,m) return byte(f:read(1)) elseif n==2 then local a,b=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==3 then local a,b,c=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==4 then local a,b,c,d=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==8 then local a,b=readnumber(f,4),readnumber(f,4) - return 256*a+b + return 0x100*a+b elseif n==12 then local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-2 then local b,a=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==-3 then local c,b,a=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-4 then local d,c,b,a=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==-8 then local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) - return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + return 0x100000000000000*a+0x1000000000000*b+0x10000000000*c+0x100000000*d+0x1000000*e+0x10000*f+0x100*g+h else return 0 end @@ -2680,7 +3075,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 5146, stripped down to: 2933 +-- original size: 5358, stripped down to: 3177 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2693,6 +3088,7 @@ local tostring,tonumber=tostring,tonumber local format,floor,match,rep=string.format,math.floor,string.match,string.rep local concat,insert=table.concat,table.insert local lpegmatch=lpeg.match +local floor=math.floor number=number or {} local number=number if bit32 then @@ -2817,6 +3213,26 @@ end function number.bits(n) return { bits(n,1) } end +function number.bytetodecimal(b) + local d=floor(b*100/255+0.5) + if d>100 then + return 100 + elseif d<-100 then + return -100 + else + return d + end +end +function number.decimaltobyte(d) + local b=floor(d*255/100+0.5) + if b>255 then + return 255 + elseif b<-255 then + return -255 + else + return b + end +end end -- of closure @@ -2825,7 +3241,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 2010, stripped down to: 1186 +-- original size: 1923, stripped down to: 1133 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2898,7 +3314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16390, stripped down to: 9734 +-- original size: 16268, stripped down to: 9246 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2974,7 +3390,7 @@ if not os.__getenv__ then end local execute=os.execute local iopopen=io.popen -function os.resultof(command) +local function resultof(command) local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" @@ -2984,9 +3400,13 @@ function os.resultof(command) return "" end end +os.resultof=resultof +function os.pipeto(command) + return iopopen(command,"w") +end if not io.fileseparator then if find(os.getenv("PATH"),";",1,true) then - io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "windows" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" end @@ -3029,17 +3449,6 @@ setmetatable(os,{ __index=function(t,k) return r and r(t,k) or nil end }) local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" -local function guess() - local architecture=os.resultof("uname -m") or "" - if architecture~="" then - return architecture - end - architecture=os.getenv("HOSTTYPE") or "" - if architecture~="" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end if platform~="" then os.platform=platform elseif os.type=="windows" then @@ -3056,7 +3465,7 @@ elseif os.type=="windows" then end elseif name=="linux" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="linux-64" elseif find(architecture,"ppc",1,true) then @@ -3070,7 +3479,7 @@ elseif name=="linux" then end elseif name=="macosx" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + local platform,architecture="",resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" elseif find(architecture,"i386",1,true) then @@ -3086,7 +3495,7 @@ elseif name=="macosx" then end elseif name=="sunos" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"sparc",1,true) then platform="solaris-sparc" else @@ -3098,7 +3507,7 @@ elseif name=="sunos" then end elseif name=="freebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else @@ -3110,7 +3519,7 @@ elseif name=="freebsd" then end elseif name=="kfreebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else @@ -3241,7 +3650,7 @@ if not os.sleep then end end local function isleapyear(year) - return (year%400==0) or ((year%100~=0) and (year%4==0)) + return (year%4==0) and (year%100~=0 or year%400==0) end os.isleapyear=isleapyear local days={ 31,28,31,30,31,30,31,31,30,31,30,31 } @@ -3280,7 +3689,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21648, stripped down to: 10238 +-- original size: 20997, stripped down to: 9986 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3617,13 +4026,15 @@ function file.robustname(str,strict) end end end -file.readdata=io.loaddata -file.savedata=io.savedata +local loaddata=io.loaddata +local savedata=io.savedata +file.readdata=loaddata +file.savedata=savedata function file.copy(oldname,newname) if oldname and newname then - local data=io.loaddata(oldname) + local data=loaddata(oldname) if data and data~="" then - file.savedata(newname,data) + savedata(newname,data) end end end @@ -3660,7 +4071,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1265, stripped down to: 1038 +-- original size: 1211, stripped down to: 1002 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3714,7 +4125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3355, stripped down to: 2321 +-- original size: 3309, stripped down to: 2314 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3744,6 +4155,8 @@ do if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + md5.sumhexa=md5.hex + md5.sumHEXA=md5.HEX end end function file.needsupdating(oldname,newname,threshold) @@ -3802,7 +4215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12897, stripped down to: 5882 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4019,7 +4432,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 17358, stripped down to: 11378 +-- original size: 17703, stripped down to: 11691 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4283,6 +4696,31 @@ local function globfiles(path,recurse,func,files) return files end dir.globfiles=globfiles +local function globdirs(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + if recurse then + globdirs(path.."/"..name,recurse,func,files) + end + end + end + end + end + return files +end +dir.globdirs=globdirs function dir.ls(pattern) return concat(glob(pattern),"\n") end @@ -4447,9 +4885,13 @@ end file.expandname=dir.expandname local stack={} function dir.push(newdir) - insert(stack,currentdir()) + local curdir=currentdir() + insert(stack,curdir) if newdir and newdir~="" then chdir(newdir) + return newdir + else + return curdir end end function dir.pop() @@ -4484,7 +4926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1919, stripped down to: 1621 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4556,7 +4998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 38699, stripped down to: 16321 +-- original size: 38263, stripped down to: 16330 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -5167,6 +5609,23 @@ function utf.chrlen(u) (u<0xFC and 5) or (u<0xFE and 6) or 0 end +local extract=bit32.extract +local char=string.char +function unicode.toutf32string(n) + if n<=0xFF then + return + char(n).."\000\000\000" + elseif n<=0xFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8)).."\000\000" + elseif n<=0xFFFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8)).."\000" + else + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8))..char(extract(n,24,8)) + end +end end -- of closure @@ -5175,7 +5634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 1012, stripped down to: 912 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5215,7 +5674,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 36053, stripped down to: 19685 +-- original size: 36148, stripped down to: 20179 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5227,7 +5686,7 @@ if not modules then modules={} end modules ['util-str']={ utilities=utilities or {} utilities.strings=utilities.strings or {} local strings=utilities.strings -local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local format,gsub,rep,sub,find=string.format,string.gsub,string.rep,string.sub,string.find local load,dump=load,string.dump local tonumber,type,tostring=tonumber,type,tostring local unpack,concat=table.unpack,table.concat @@ -5410,6 +5869,25 @@ function number.signed(i) return "-",-i end end +local digit=patterns.digit +local period=patterns.period +local three=digit*digit*digit +local splitter=Cs ( + (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) +) +patterns.formattednumber=splitter +function number.formatted(n,sep1,sep2) + local s=type(s)=="string" and n or format("%0.2f",n) + if sep1==true then + return lpegmatch(splitter,s,1,".",",") + elseif sep1=="." then + return lpegmatch(splitter,s,1,sep1,sep2 or ",") + elseif sep1=="," then + return lpegmatch(splitter,s,1,sep1,sep2 or ".") + else + return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") + end +end local zero=P("0")^1/"" local plus=P("+")/"" local minus=P("-") @@ -5435,6 +5913,27 @@ function number.sparseexponent(f,n) end return tostring(n) end +local hf={} +local hs={} +setmetatable(hf,{ __index=function(t,k) + local v="%."..k.."f" + t[k]=v + return v +end } ) +setmetatable(hs,{ __index=function(t,k) + local v="%"..k.."s" + t[k]=v + return v +end } ) +function number.formattedfloat(n,b,a) + local s=format(hf[a],n) + local l=(b or 0)+(a or 0)+1 + if #s<l then + return format(hs[l],s) + else + return s + end +end local template=[[ %s %s @@ -5462,6 +5961,7 @@ local autodouble=string.autodouble local sequenced=table.sequenced local formattednumber=number.formatted local sparseexponent=number.sparseexponent +local formattedfloat=number.formattedfloat ]] else environment={ @@ -5485,6 +5985,7 @@ else sequenced=table.sequenced, formattednumber=number.formatted, sparseexponent=number.sparseexponent, + formattedfloat=number.formattedfloat } end local arguments={ "a1" } @@ -5495,6 +5996,7 @@ setmetatable(arguments,{ __index=function(t,k) end }) local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_sub=(C((S("+-")+R("09"))^0)+Cc(0))*P(".")*(C((S("+-")+R("09"))^0)+Cc(0)) local prefix_tab=P("{")*C((1-P("}"))^0)*P("}")+C((1-R("az","AZ","09","%%"))^0) local format_s=function(f) n=n+1 @@ -5545,6 +6047,10 @@ local format_F=function(f) return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) end end +local format_k=function(b,a) + n=n+1 + return format("formattedfloat(a%s,%i,%i)",n,b or 0,a or 0) +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5693,25 +6199,6 @@ end local format_W=function(f) return format("nspaces[%s]",tonumber(f) or 0) end -local digit=patterns.digit -local period=patterns.period -local three=digit*digit*digit -local splitter=Cs ( - (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) -) -patterns.formattednumber=splitter -function number.formatted(n,sep1,sep2) - local s=type(s)=="string" and n or format("%0.2f",n) - if sep1==true then - return lpegmatch(splitter,s,1,".",",") - elseif sep1=="." then - return lpegmatch(splitter,s,1,sep1,sep2 or ",") - elseif sep1=="," then - return lpegmatch(splitter,s,1,sep1,sep2 or ".") - else - return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") - end -end local format_m=function(f) n=n+1 if not f or f=="" then @@ -5736,9 +6223,16 @@ end local format_extension=function(extensions,f,name) local extension=extensions[name] or "tostring(%s)" local f=tonumber(f) or 1 + local w=find(extension,"%.%.%.") if f==0 then + if w then + extension=gsub(extension,"%.%.%.","") + end return extension elseif f==1 then + if w then + extension=gsub(extension,"%.%.%.","%%s") + end n=n+1 local a="a"..n return format(extension,a,a) @@ -5746,10 +6240,13 @@ local format_extension=function(extensions,f,name) local a="a"..(n+f+1) return format(extension,a,a) else + if w then + extension=gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s") + end local t={} for i=1,f do n=n+1 - t[#t+1]="a"..n + t[i]="a"..n end return format(extension,unpack(t)) end @@ -5762,7 +6259,8 @@ local builder=Cs { "start", +V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") -+V("N") ++V("N") ++V("k") +V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") +V("W") +V("a") @@ -5789,6 +6287,7 @@ local builder=Cs { "start", ["S"]=(prefix_any*P("S"))/format_S, ["Q"]=(prefix_any*P("Q"))/format_S, ["N"]=(prefix_any*P("N"))/format_N, + ["k"]=(prefix_sub*P("k"))/format_k, ["c"]=(prefix_any*P("c"))/format_c, ["C"]=(prefix_any*P("C"))/format_C, ["r"]=(prefix_any*P("r"))/format_r, @@ -5909,7 +6408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28680, stripped down to: 18636 +-- original size: 27407, stripped down to: 17116 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5923,7 +6422,7 @@ utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort -local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local setmetatable,getmetatable,tonumber,tostring,rawget=setmetatable,getmetatable,tonumber,tostring,rawget local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs @@ -6063,7 +6562,7 @@ function table.tocsv(t,specification) r[f]=tostring(field) end end - result[#result+1]=concat(r,separator) + result[i+1]=concat(r,separator) end return concat(result,"\n") else @@ -6295,11 +6794,12 @@ function table.autokey(t,k) return v end local selfmapper={ __index=function(t,k) t[k]=k return k end } -function table.twowaymapper(t) - if not t then - t={} - else - for i=0,#t do +function table.twowaymapper(t) + if not t then + t={} + else + local zero=rawget(t,0) + for i=zero and 0 or 1,#t do local ti=t[i] if ti then local i=tostring(i) @@ -6307,7 +6807,6 @@ function table.twowaymapper(t) t[ti]=i end end - t[""]=t[0] or "" end setmetatable(t,selfmapper) return t @@ -6346,6 +6845,7 @@ local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize +local is_simple_table=table.is_simple_table local function serialize(root,name,specification) if type(specification)=="table" then return original_serialize(root,name,specification) @@ -6353,54 +6853,6 @@ local function serialize(root,name,specification) local t local n=1 local unknown=false - local function simple_table(t) - local nt=#t - if nt>0 then - local n=0 - for _,v in next,t do - n=n+1 - if type(v)=="table" then - return nil - end - end - local haszero=t[0] - if n==nt then - local tt={} - for i=1,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i]=v - elseif tv=="string" then - tt[i]=format("%q",v) - elseif tv=="boolean" then - tt[i]=v and "true" or "false" - else - return nil - end - end - return tt - elseif haszero and (n==nt+1) then - local tt={} - for i=0,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i+1]=v - elseif tv=="string" then - tt[i+1]=format("%q",v) - elseif tv=="boolean" then - tt[i+1]=v and "true" or "false" - else - return nil - end - end - tt[1]="[0] = "..tt[1] - return tt - end - end - return nil - end local function do_serialize(root,name,depth,level,indexed) if level>0 then n=n+1 @@ -6425,7 +6877,7 @@ local function serialize(root,name,specification) local last=0 last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -6448,7 +6900,7 @@ local function serialize(root,name,specification) if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else - local st=simple_table(v) + local st=is_simple_table(v) if st then n=n+1 t[n]=f_val_seq(depth,st) else @@ -6492,7 +6944,7 @@ local function serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else - local st=simple_table(v) + local st=is_simple_table(v) if not st then do_serialize(v,k,depth,level+1) elseif tk=="number" then @@ -6552,11 +7004,11 @@ local function serialize(root,name,specification) end if root then if getmetatable(root) then - local dummy=root._w_h_a_t_e_v_e_r_ + local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - local st=simple_table(root) + local st=is_simple_table(root) if st then return t[1]..f_fin_seq(st) else @@ -6570,7 +7022,12 @@ local function serialize(root,name,specification) end table.serialize=serialize if setinspector then - setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) + setinspector("table",function(v) + if type(v)=="table" then + print(serialize(v,"table",{ metacheck=false })) + return true + end + end) end @@ -6580,7 +7037,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fil"] = package.loaded["util-fil"] or true --- original size: 3577, stripped down to: 2870 +-- original size: 7567, stripped down to: 5575 if not modules then modules={} end modules ['util-fil']={ version=1.001, @@ -6590,7 +7047,9 @@ if not modules then modules={} end modules ['util-fil']={ license="see context related readme files" } local byte=string.byte -local extract=bit32.extract +local char=string.char +local extract=bit32 and bit32.extract +local floor=math.floor utilities=utilities or {} local files={} utilities.files=files @@ -6609,6 +7068,7 @@ end function files.size(f) return f:seek("end") end +files.getsize=files.size function files.setposition(f,n) if zerobased[f] then f:seek("set",n) @@ -6646,6 +7106,10 @@ end function files.readbytes(f,n) return byte(f:read(n),1,n) end +function files.readbytetable(f,n) + local s=f:read(n or 1) + return { byte(s,1,#s) } +end function files.readchar(f) return f:read(1) end @@ -6655,7 +7119,7 @@ end function files.readinteger1(f) local n=byte(f:read(1)) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6663,55 +7127,107 @@ end files.readcardinal1=files.readbyte files.readcardinal=files.readcardinal1 files.readinteger=files.readinteger1 +files.readsignedbyte=files.readinteger1 function files.readcardinal2(f) local a,b=byte(f:read(2),1,2) return 0x100*a+b end +function files.readcardinal2le(f) + local b,a=byte(f:read(2),1,2) + return 0x100*a+b +end function files.readinteger2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function files.readinteger2le(f) + local b,a=byte(f:read(2),1,2) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function files.readcardinal3(f) local a,b,c=byte(f:read(3),1,3) return 0x10000*a+0x100*b+c end +function files.readcardinal3le(f) + local c,b,a=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readinteger3(f) + local a,b,c=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function files.readinteger3le(f) + local c,b,a=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function files.readcardinal4(f) local a,b,c,d=byte(f:read(4),1,4) return 0x1000000*a+0x10000*b+0x100*c+d end +function files.readcardinal4le(f) + local d,c,b,a=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end function files.readinteger4(f) local a,b,c,d=byte(f:read(4),1,4) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.readfixed4(f) - local a,b,c,d=byte(f:read(4),1,4) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF +function files.readinteger4le(f) + local d,c,b,a=byte(f:read(4),1,4) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n+(0x100*c+d)/0xFFFF + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.read2dot14(f) +function files.readfixed2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - n=extract(n,30,2) - return n+m/0x4000 + return (0x100*a+b )+(0x100*c+d)/0x10000 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function files.skipshort(f,n) @@ -6720,6 +7236,55 @@ end function files.skiplong(f,n) f:read(4*(n or 1)) end +function files.writecardinal2(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + f:write(b,a) +end +function files.writecardinal4(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + n=floor(n/256) + local c=char(n%256) + n=floor(n/256) + local d=char(n%256) + f:write(d,c,b,a) +end +function files.writestring(f,s) + f:write(char(byte(s,1,#s))) +end +function files.writebyte(f,b) + f:write(char(b)) +end +if fio and fio.readcardinal1 then + files.readcardinal1=fio.readcardinal1 + files.readcardinal2=fio.readcardinal2 + files.readcardinal3=fio.readcardinal3 + files.readcardinal4=fio.readcardinal4 + files.readinteger1=fio.readinteger1 + files.readinteger2=fio.readinteger2 + files.readinteger3=fio.readinteger3 + files.readinteger4=fio.readinteger4 + files.read2dot14=fio.read2dot14 + files.setposition=fio.setposition + files.getposition=fio.getposition + files.readbyte=files.readcardinal1 + files.readsignedbyte=files.readinteger1 + files.readcardinal=files.readcardinal1 + files.readinteger=files.readinteger1 + local skipposition=fio.skipposition + files.skipposition=skipposition + files.readbytes=fio.readbytes + files.readbytetable=fio.readbytetable + function files.skipshort(f,n) + skipposition(f,2*(n or 1)) + end + function files.skiplong(f,n) + skipposition(f,4*(n or 1)) + end +end end -- of closure @@ -6728,7 +7293,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 4264, stripped down to: 3349 +-- original size: 8716, stripped down to: 6754 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -6738,7 +7303,7 @@ if not modules then modules={} end modules ['util-sac']={ license="see context related readme files" } local byte,sub=string.byte,string.sub -local extract=bit32.extract +local extract=bit32 and bit32.extract utilities=utilities or {} local streams={} utilities.streams=streams @@ -6796,6 +7361,12 @@ function streams.readbytes(f,n) f[2]=j return byte(f[1],i,j-1) end +function streams.readbytetable(f,n) + local i=f[2] + local j=i+n + f[2]=j + return { byte(f[1],i,j-1) } +end function streams.skipbytes(f,n) f[2]=f[2]+n end @@ -6815,7 +7386,7 @@ function streams.readinteger1(f) f[2]=i+1 local n=byte(f[1],i) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6830,16 +7401,33 @@ function streams.readcardinal2(f) local a,b=byte(f[1],i,j) return 0x100*a+b end +function streams.readcardinal2LE(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + return 0x100*a+b +end function streams.readinteger2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function streams.readinteger2le(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function streams.readcardinal3(f) @@ -6849,6 +7437,35 @@ function streams.readcardinal3(f) local a,b,c=byte(f[1],i,j) return 0x10000*a+0x100*b+c end +function streams.readcardinal3le(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readinteger3(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function streams.readinteger3le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function streams.readcardinal4(f) local i=f[2] local j=i+3 @@ -6861,11 +7478,21 @@ function streams.readinteger4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d + end +end +function streams.readinteger4le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local d,c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 + else + return 0x1000000*a+0x10000*b+0x100*c+d end end function streams.readfixed4(f) @@ -6873,26 +7500,38 @@ function streams.readfixed4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - return n+(0x100*c+d)/0xFFFF + return (0x100*a+b )+(0x100*c+d)/0x10000 end end -function streams.read2dot14(f) +function streams.readfixed2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 - else - n=extract(n,30,2) - return n+m/0x4000 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function streams.skipshort(f,n) @@ -6901,6 +7540,92 @@ end function streams.skiplong(f,n) f[2]=f[2]+4*(n or 1) end +if sio and sio.readcardinal2 then + local readcardinal1=sio.readcardinal1 + local readcardinal2=sio.readcardinal2 + local readcardinal3=sio.readcardinal3 + local readcardinal4=sio.readcardinal4 + local readinteger1=sio.readinteger1 + local readinteger2=sio.readinteger2 + local readinteger3=sio.readinteger3 + local readinteger4=sio.readinteger4 + local readfixed2=sio.readfixed2 + local readfixed4=sio.readfixed4 + local read2dot14=sio.read2dot14 + local readbytes=sio.readbytes + local readbytetable=sio.readbytetable + function streams.readcardinal1(f) + local i=f[2] + f[2]=i+1 + return readcardinal1(f[1],i) + end + function streams.readcardinal2(f) + local i=f[2] + f[2]=i+2 + return readcardinal2(f[1],i) + end + function streams.readcardinal3(f) + local i=f[2] + f[2]=i+3 + return readcardinal3(f[1],i) + end + function streams.readcardinal4(f) + local i=f[2] + f[2]=i+4 + return readcardinal4(f[1],i) + end + function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + return readinteger1(f[1],i) + end + function streams.readinteger2(f) + local i=f[2] + f[2]=i+2 + return readinteger2(f[1],i) + end + function streams.readinteger3(f) + local i=f[2] + f[2]=i+3 + return readinteger3(f[1],i) + end + function streams.readinteger4(f) + local i=f[2] + f[2]=i+4 + return readinteger4(f[1],i) + end + function streams.read2dot4(f) + local i=f[2] + f[2]=i+2 + return read2dot4(f[1],i) + end + function streams.readbytes(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytes(f[1],i,n) + end + function streams.readbytetable(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytetable(f[1],i,n) + end + streams.readbyte=streams.readcardinal1 + streams.readsignedbyte=streams.readinteger1 + streams.readcardinal=streams.readcardinal1 + streams.readinteger=streams.readinteger1 +end end -- of closure @@ -6909,7 +7634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4100, stripped down to: 2852 +-- original size: 3926, stripped down to: 2742 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -7049,7 +7774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 23411, stripped down to: 16177 +-- original size: 22883, stripped down to: 16045 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -7211,6 +7936,21 @@ function parsers.settings_to_array(str,strict) return { str } end end +function parsers.settings_to_numbers(str) + if not str or str=="" then + return {} + end + if type(str)=="table" then + elseif find(str,",",1,true) then + str=lpegmatch(pattern,str) + else + return { tonumber(str) } + end + for i=1,#str do + str[i]=tonumber(str[i]) + end + return str +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) local pattern=spaces*Ct(value*(separator*value)^0) function parsers.settings_to_array_obey_fences(str) @@ -7587,7 +8327,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2350, stripped down to: 1847 +-- original size: 2274, stripped down to: 1781 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7668,7 +8408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12862, stripped down to: 9104 +-- original size: 12454, stripped down to: 8840 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7854,7 +8594,6 @@ function setters.list(t) return user,system end function setters.show(t) - local category=t.name local list=setters.list(t) t.report() for k=1,#list do @@ -7981,7 +8720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 30767, stripped down to: 21355 +-- original size: 30007, stripped down to: 20818 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -8027,6 +8766,14 @@ if tex and (tex.jobname or tex.formatname) then if texio.setescape then texio.setescape(0) end + if arg then + for k,v in next,arg do + if v=="--ansi" or v=="--c:ansi" then + variant="ansi" + break + end + end + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -8612,7 +9359,6 @@ function logs.stop_page_number() end logs.flush() end -local report_files=logs.reporter("files") local nesting=0 local verbose=false local hasscheme=url.hasscheme @@ -8774,7 +9520,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6917, stripped down to: 5484 +-- original size: 8036, stripped down to: 5567 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8806,11 +9552,13 @@ end local function resettiming(instance) timers[instance or "notimer"]={ timing=0,loadtime=0 } end +local ticks=clock +local seconds=function(n) return n or 0 end local function starttiming(instance) local timer=timers[instance or "notimer"] local it=timer.timing or 0 if it==0 then - timer.starttime=clock() + timer.starttime=ticks() if not timer.loadtime then timer.loadtime=0 end @@ -8824,12 +9572,13 @@ local function stoptiming(instance) timer.timing=it-1 else local starttime=timer.starttime - if starttime then - local stoptime=clock() + if starttime and starttime>0 then + local stoptime=ticks() local loadtime=stoptime-starttime timer.stoptime=stoptime timer.loadtime=timer.loadtime+loadtime timer.timing=0 + timer.starttime=0 return loadtime end end @@ -8840,7 +9589,7 @@ local function elapsed(instance) return instance or 0 else local timer=timers[instance or "notimer"] - return timer and timer.loadtime or 0 + return timer and seconds(timer.loadtime) or 0 end end local function elapsedtime(instance) @@ -8888,10 +9637,13 @@ function statistics.show() local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) - if jit then - local jitstatus={ jit.status() } - if jitstatus[1] then - register("luajit options",concat(jitstatus," ",2)) + if TEXENGINE=="luajittex" and JITSUPPORTED then + local jitstatus=jit.status + if jitstatus then + local jitstatus={ jitstatus() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end end end register("lua properties",function() @@ -8955,7 +9707,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 6039, stripped down to: 3616 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -9102,7 +9854,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 5142, stripped down to: 3611 +-- original size: 5396, stripped down to: 3708 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -9224,6 +9976,17 @@ function luautilities.loadstripped(...) return load(dump(l,true)) end end +local finalizers={} +setmetatable(finalizers,{ + __gc=function(t) + for i=1,#t do + pcall(t[i]) + end + end +} ) +function luautilities.registerfinalizer(f) + finalizers[#finalizers+1]=f +end end -- of closure @@ -9232,7 +9995,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 4030, stripped down to: 2718 +-- original size: 8911, stripped down to: 6504 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -9242,75 +10005,230 @@ if not modules then modules={} end modules ['util-deb']={ license="see context related readme files" } local debug=require "debug" -local getinfo=debug.getinfo -local type,next,tostring=type,next,tostring -local format,find=string.format,string.find -local is_boolean=string.is_boolean +local getinfo,sethook=debug.getinfo,debug.sethook +local type,next,tostring,tonumber=type,next,tostring,tonumber +local format,find,sub,gsub=string.format,string.find,string.sub,string.gsub +local insert,remove,sort=table.insert,table.remove,table.sort +local setmetatableindex=table.setmetatableindex utilities=utilities or {} local debugger=utilities.debugger or {} utilities.debugger=debugger -local counters={} -local names={} local report=logs.reporter("debugger") -local function hook() - local f=getinfo(2) - if f then - local n="unknown" - if f.what=="C" then - n=f.name or '<anonymous>' - if not names[n] then - names[n]=format("%42s",n) +local ticks=os.gettimeofday or os.clock +local seconds=function(n) return n or 0 end +local overhead=0 +local dummycalls=10*1000 +local nesting=0 +local names={} +local initialize=false +if not (FFISUPPORTED and ffi) then +elseif os.type=="windows" then + initialize=function() + local kernel=ffilib("kernel32","system") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end end - else - n=f.name or f.namewhat or f.what - if not n or n=="" then - n="?" + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end + initialize=false + end +elseif os.type=="unix" then + initialize=function() + local C=ffi.C + local tonumber=ffi.number or tonumber + ffi.cdef [[ + /* what a mess */ + typedef int clk_id_t; + typedef enum { CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_PROCESS_CPUTIME_ID } clk_id; + typedef struct timespec { long sec; long nsec; } ctx_timespec; + int clock_gettime(clk_id_t timerid, struct timespec *t); + ]] + local target=ffi.new("ctx_timespec[?]",1) + local clock=C.CLOCK_PROCESS_CPUTIME_ID + ticks=function () + C.clock_gettime(clock,target) + return tonumber(target[0].sec*1000000000+target[0].nsec) + end + seconds=function(ticks) + return ticks/1000000000 + end + initialize=false + end +end +setmetatableindex(names,function(t,name) + local v=setmetatableindex(function(t,source) + local v=setmetatableindex(function(t,line) + local v={ total=0,count=0 } + t[line]=v + return v + end) + t[source]=v + return v + end) + t[name]=v + return v +end) +local function hook(where) + local f=getinfo(2,"nSl") + if f then + local source=f.short_src + if not source then + return + end + local line=f.linedefined or 0 + local name=f.name + if not name then + local what=f.what + if what=="C" then + name="<anonymous>" + else + name=f.namewhat or what or "<unknown>" end - if not names[n] then - names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + local data=names[name][source][line] + if where=="call" then + data.count=data.count+1 + insert(data,ticks()) + elseif where=="return" then + local t=remove(data) + if t then + data.total=data.total+ticks()-t end end - counters[n]=(counters[n] or 0)+1 end end -function debugger.showstats(printer,threshold) - printer=printer or report - threshold=threshold or 0 - local total,grandtotal,functions=0,0,0 +function debugger.showstats(printer,threshold) + local printer=printer or report + local calls=0 + local functions=0 local dataset={} - for name,count in next,counters do - dataset[#dataset+1]={ name,count } + local length=0 + local wholetime=0 + local threshold=threshold or 0 + for name,sources in next,names do + for source,lines in next,sources do + for line,data in next,lines do + local count=data.count + if count>threshold then + if #name>length then + length=#name + end + local total=data.total + local real=total + if real>0 then + real=total-(count*overhead/dummycalls) + if real<0 then + real=0 + end + wholetime=wholetime+real + end + if line<0 then + line=0 + end + dataset[#dataset+1]={ real,total,count,name,source,line } + end + end + end end - table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + sort(dataset,function(a,b) + if a[1]==b[1] then + if a[2]==b[2] then + if a[3]==b[3] then + if a[4]==b[4] then + if a[5]==b[5] then + return a[6]<b[6] + else + return a[5]<b[5] + end + else + return a[4]<b[4] + end + else + return b[3]<a[3] + end + else + return b[2]<a[2] + end + else + return b[1]<a[1] + end + end) + if length>50 then + length=50 + end + local fmt=string.formatters["%4.9k %4.9k %3.3k %8i %-"..length.."s %4i %s"] for i=1,#dataset do - local d=dataset[i] - local name=d[1] - local count=d[2] - if count>threshold and not find(name,"for generator") then - printer(format("%8i %s\n",count,names[name])) - total=total+count - end - grandtotal=grandtotal+count + local data=dataset[i] + local real=data[1] + local total=data[2] + local count=data[3] + local name=data[4] + local source=data[5] + local line=data[6] + local percent=real/wholetime + calls=calls+count functions=functions+1 + name=gsub(name,"%s+"," ") + if #name>length then + name=sub(name,1,length) + end + printer(fmt(seconds(total),seconds(real),percent,count,name,line,source)) end - printer("\n") - printer(format("functions : % 10i\n",functions)) - printer(format("total : % 10i\n",total)) - printer(format("grand total: % 10i\n",grandtotal)) - printer(format("threshold : % 10i\n",threshold)) + printer("") + printer(format("functions : %i",functions)) + printer(format("calls : %i",calls)) + printer(format("overhead : %f",seconds(overhead/1000))) end function debugger.savestats(filename,threshold) local f=io.open(filename,'w') if f then - debugger.showstats(function(str) f:write(str) end,threshold) + debugger.showstats(function(str) f:write(str,"\n") end,threshold) f:close() end end function debugger.enable() - debug.sethook(hook,"c") + if nesting==0 then + running=true + if initialize then + initialize() + end + sethook(hook,"cr") + local function dummy() end + local t=ticks() + for i=1,dummycalls do + dummy() + end + overhead=ticks()-t + end + if nesting>0 then + nesting=nesting+1 + end end function debugger.disable() - debug.sethook() + if nesting>0 then + nesting=nesting-1 + end + if nesting==0 then + sethook() + end end local function showtraceback(rep) local level=2 @@ -9334,9 +10252,661 @@ end -- of closure do -- create closure to overcome 200 locals limit +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true + +-- original size: 7100, stripped down to: 3978 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20309, stripped down to: 13848 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + if type(root)=="table" then + root,what=root[1],root[2] + end + if type(root)=="string" and root~="" then + root=collapsepath(expandname(root)) + if what=="r" or what=="ro" or what=="readable" then + what="read" + elseif what=="w" or what=="wo" or what=="writable" then + what="write" + end + validroots[root]=what=="write" or false + end + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + elseif filenamelogger then + filenamelogger(name,"*",name,false) + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + if trace then + report("resultof: %s",command) + end + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + if trace then + report("execute: %s",command) + end + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + if trace then + report("pipeto: %s",command) + end + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +function sandbox.getrunner(name) + return name and validrunners[name] +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if FFISUPPORTED and ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7757, stripped down to: 6015 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9511,154 +11081,9 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8284, stripped down to: 5176 +-- original size: 9246, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9845,7 +11270,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6358, stripped down to: 4257 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9998,7 +11423,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 56973, stripped down to: 35872 +-- original size: 57003, stripped down to: 35696 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -10013,7 +11438,7 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset +local type,next,setmetatable,getmetatable,tonumber,rawset,select=type,next,setmetatable,getmetatable,tonumber,rawset,select local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local sort=table.sort local utfchar=utf.char @@ -10140,6 +11565,7 @@ local function add_empty(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nt, __p__=top } dt[nt]=t @@ -10161,6 +11587,7 @@ local function add_begin(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nil, __p__=stack[level] } setmetatable(top,mt) @@ -10188,6 +11615,7 @@ local function add_end(spacing,namespace,tag) dt=top.dt nt=#dt+1 dt[nt]=toclose + toclose.ni=nt if toclose.at.xmlns then remove(xmlns) end @@ -10232,7 +11660,13 @@ local function add_special(what,spacing,text) if strip and (what=="@cm@" or what=="@dt@") then else nt=nt+1 - dt[nt]={ special=true,ns="",tg=what,dt={ text } } + dt[nt]={ + special=true, + ns="", + tg=what, + ni=nil, + dt={ text }, + } end end local function set_message(txt) @@ -10285,7 +11719,6 @@ do end local p_rest=(1-P(";"))^0 local p_many=P(1)^0 - local p_char=lpegpatterns.utf8character local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) xml.parsedentitylpeg=parsedentity local predefined_unified={ @@ -10327,13 +11760,27 @@ do [ [[}]] ]="&U+7D;", [ [[~]] ]="&U+7E;", } + local privates_x={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[']] ]="&U+27;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } local privates_n={ } local escaped=utf.remapper(privates_u,"dynamic") local unprivatized=utf.remapper(privates_p,"dynamic") local unspecialized=utf.remapper(privates_s,"dynamic") + local despecialized=utf.remapper(privates_x,"dynamic") xml.unprivatized=unprivatized xml.unspecialized=unspecialized + xml.despecialized=despecialized xml.escaped=escaped local function unescaped(s) local p=privates_n[s] @@ -10778,6 +12225,10 @@ local grammar_unparsed_text=P { "preamble", local function _xmlconvert_(data,settings) settings=settings or {} preparexmlstate(settings) + local preprocessor=settings.preprocessor + if data and data~="" and type(preprocessor)=="function" then + data=preprocessor(data,settings) or data + end if settings.parent_root then mt=getmetatable(settings.parent_root) else @@ -10919,14 +12370,24 @@ function xml.toxml(data) return data end end -local function copy(old) +local function copy(old,p) if old then local new={} for k,v in next,old do - if type(v)=="table" then - new[k]=table.copy(v) - else + local t=type(v)=="table" + if k=="at" then + local t={} + for k,v in next,v do + t[k]=v + end + new[k]=t + elseif k=="dt" then + v.__p__=nil + v=copy(v,new) new[k]=v + v.__p__=p + else + new[k]=v end end local mt=getmetatable(old) @@ -11157,18 +12618,26 @@ local xmlfilehandler=newhandlers { function xml.save(root,name) serialize(root,xmlfilehandler,name) end -local result +local result,r,threshold={},0,512 local xmlstringhandler=newhandlers { name="string", initialize=function() - result={} + r=0 return result end, finalize=function() - return concat(result) + local done=concat(result,"",1,r) + r=0 + if r>threshold then + result={} + end + return done end, handle=function(...) - result[#result+1]=concat {... } + for i=1,select("#",...) do + r=r+1 + result[r]=select(i,...) + end end, } local function xmltostring(root) @@ -11320,7 +12789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 53892, stripped down to: 32508 +-- original size: 53301, stripped down to: 32477 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -11702,6 +13171,14 @@ local function apply_expression(list,expression,order) end return collected end +local function apply_selector(list,specification) + if xml.applyselector then + apply_selector=xml.applyselector + return apply_selector(list,specification) + else + return list + end +end local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb local spaces=S(" \n\r\t\f")^0 local lp_space=S(" \n\r\t\f") @@ -11825,6 +13302,9 @@ end local function register_nodes(nodetest,nodes) return { kind="nodes",nodetest=nodetest,nodes=nodes } end +local function register_selector(specification) + return { kind="selector",specification=specification } +end local function register_expression(expression) local converted=lpegmatch(converter,expression) local runner=load(format(template_e,converted)) @@ -11865,34 +13345,36 @@ local pathparser=Ct { "patterns", (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), - step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + step=((V("shortcuts")+V("selector")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, - shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor")+V("s_lastmatch"), shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), s_descendant=P("**")*Cc(register_descendant), - s_child=P("*")*no_nextcolon*Cc(register_child ), - s_parent=P("..")*Cc(register_parent ), - s_self=P("." )*Cc(register_self ), - s_root=P("^^")*Cc(register_root ), - s_ancestor=P("^")*Cc(register_ancestor ), - descendant=P("descendant::")*Cc(register_descendant ), - child=P("child::")*Cc(register_child ), - parent=P("parent::")*Cc(register_parent ), - self=P("self::")*Cc(register_self ), - root=P('root::')*Cc(register_root ), - ancestor=P('ancestor::')*Cc(register_ancestor ), - descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), - ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), - following=P('following::')*Cc(register_following ), - following_sibling=P('following-sibling::')*Cc(register_following_sibling ), - preceding=P('preceding::')*Cc(register_preceding ), - preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), - reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), - last_match=P('last-match::')*Cc(register_last_match ), + s_child=P("*")*no_nextcolon*Cc(register_child), + s_parent=P("..")*Cc(register_parent), + s_self=P("." )*Cc(register_self), + s_root=P("^^")*Cc(register_root), + s_ancestor=P("^")*Cc(register_ancestor), + s_lastmatch=P("=")*Cc(register_last_match), + descendant=P("descendant::")*Cc(register_descendant), + child=P("child::")*Cc(register_child), + parent=P("parent::")*Cc(register_parent), + self=P("self::")*Cc(register_self), + root=P('root::')*Cc(register_root), + ancestor=P('ancestor::')*Cc(register_ancestor), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self), + following=P('following::')*Cc(register_following), + following_sibling=P('following-sibling::')*Cc(register_following_sibling), + preceding=P('preceding::')*Cc(register_preceding), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling), + last_match=P('last-match::')*Cc(register_last_match), + selector=P("{")*C((1-P("}"))^1)*P("}")/register_selector, nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -12042,6 +13524,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) p.matched=p.matched+1 @@ -12083,6 +13567,9 @@ do elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) + report_lpath("% 10i : se : %s ",(collected and #collected) or 0,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") @@ -12114,6 +13601,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then return pi.finalizer(collected) end @@ -12170,6 +13659,13 @@ do function xml.lastmatch() return lastmatch end + local stack={} + function xml.pushmatch() + insert(stack,lastmatch) + end + function xml.popmatch() + lastmatch=remove(stack) + end end local applylpath=xml.applylpath function xml.filter(root,pattern) @@ -12449,7 +13945,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3787, stripped down to: 2003 +-- original size: 3684, stripped down to: 1957 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -12518,7 +14014,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 30566, stripped down to: 21741 +-- original size: 29835, stripped down to: 21174 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12534,7 +14030,6 @@ local xml=xml local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath -local xmlfilter=xml.filter local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip @@ -12738,7 +14233,17 @@ function xml.replace(root,pattern,whatever) report('replacing',pattern,c,e) end local d=p.dt - d[e.ni]=copiedelement(element,p) + local n=e.ni + local t=copiedelement(element,p) + if type(t)=="table" then + d[n]=t[1] + for i=2,#t do + n=n+1 + insert(d,n,t[i]) + end + else + d[n]=t + end redo_ni(d) end end @@ -13161,7 +14666,7 @@ local obsolete=xml.obsolete xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect xml.delete_element=xml.delete obsolete.delete_element=xml.delete -xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.replace_element=xml.replace obsolete.replace_element=xml.replace xml.each_element=xml.each obsolete.each_element=xml.each xml.process_elements=xml.process obsolete.process_elements=xml.process xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter @@ -13379,7 +14884,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10719, stripped down to: 7841 +-- original size: 10274, stripped down to: 7538 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -13757,7 +15262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6534, stripped down to: 5072 +-- original size: 6407, stripped down to: 4965 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13907,6 +15412,7 @@ function reporters.export(t,methods,filename) if filename then local fullname=file.replacesuffix(filename,method) t.report("saving export in %a",fullname) + dir.mkdirs(file.pathpart(fullname)) io.savedata(fullname,result) else reporters.lines(t,result) @@ -13927,7 +15433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11444, stripped down to: 7830 +-- original size: 11099, stripped down to: 7516 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -14072,11 +15578,6 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if type(profiler)=="table" and not jit then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) -end local prefixes=utilities.storage.allocate() resolvers.prefixes=prefixes local resolved={} @@ -14183,7 +15684,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18619, stripped down to: 11042 +-- original size: 17958, stripped down to: 10705 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -14199,7 +15700,6 @@ local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next local isdir=lfs.isdir -local ostype=os.type local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -14568,7 +16068,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9649, stripped down to: 7131 +-- original size: 9342, stripped down to: 6887 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -14852,7 +16352,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16066, stripped down to: 11938 +-- original size: 16088, stripped down to: 11435 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -15056,18 +16556,6 @@ end caches.getreadablepaths=getreadablepaths caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd=getreadablepaths(...) - for i=1,#rd do - local path=rd[i] - local fullname=file.join(path,filename) - if is_readable(fullname) then - usedreadables[i]=true - return fullname,path - end - end - return caches.setfirstwritablefile(filename,...) -end -function caches.getfirstreadablefile_TEST_ME_FIRST(filename,...) local fullname,path=caches.setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path @@ -15096,18 +16584,22 @@ end function caches.setluanames(path,name) return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end -function caches.loaddata(readables,name) +function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } end for i=1,#readables do local path=readables[i] - local tmaname,tmcname=caches.setluanames(path,name) local loader=false + local tmaname,tmcname=caches.setluanames(path,name) if isfile(tmcname) then loader=loadfile(tmcname) end if not loader and isfile(tmaname) then + local tmacrap,tmcname=caches.setluanames(writable,name) + if isfile(tmcname) then + loader=loadfile(tmcname) + end utilities.lua.compile(tmaname,tmcname) if isfile(tmcname) then loader=loadfile(tmcname) @@ -15228,7 +16720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5488, stripped down to: 4101 +-- original size: 5310, stripped down to: 3980 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -15240,7 +16732,6 @@ if not modules then modules={} end modules ['data-met']={ local find,format=string.find,string.format local sequenced=table.sequenced local addurlscheme,urlhashed=url.addscheme,url.hashed -local getcurrentdir=lfs.currentdir local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) @@ -15347,7 +16838,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67241, stripped down to: 46427 +-- original size: 67524, stripped down to: 46632 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15401,6 +16892,7 @@ resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf @@ -15444,7 +16936,6 @@ local function resolvevariable(k) end local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) @@ -15590,23 +17081,29 @@ local function identify_configuration_files() end reportcriticalvariables(cnfspec) local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname=resolvers.luacnfname - for i=1,#cnfpaths do - local filepath=cnfpaths[i] - local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolveprefix(filename) - if trace_locating then - local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) - report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) - end - if isfile(realname) then - specification[#specification+1]=filename + local function locatecnf(luacnfname,kind) + for i=1,#cnfpaths do + local filepath=cnfpaths[i] + local filename=collapsepath(filejoin(filepath,luacnfname)) + local realname=resolveprefix(filename) if trace_locating then - report_resolving("found configuration file %a",realname) + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) + report_resolving("looking for %s %a on %s path %a from specification %a", + kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath) + end + if isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found %s configuration file %a",kind,realname) + end end end end + locatecnf(resolvers.luacnfname,"regular") + if #specification==0 then + locatecnf(resolvers.luacnffallback,"fallback") + end if trace_locating then report_resolving() end @@ -16903,7 +18400,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4236, stripped down to: 3144 +-- original size: 4090, stripped down to: 3059 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -17025,7 +18522,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 935, stripped down to: 838 +-- original size: 910, stripped down to: 823 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -17055,7 +18552,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 548, stripped down to: 483 +-- original size: 530, stripped down to: 475 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -17078,7 +18575,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3976, stripped down to: 3391 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -17186,7 +18683,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5148, stripped down to: 3680 +-- original size: 5029, stripped down to: 3607 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -17256,7 +18753,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name) + stored=caches.loaddata(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -17305,7 +18802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4000, stripped down to: 3052 +-- original size: 4045, stripped down to: 3110 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -17350,7 +18847,7 @@ function resolvers.automount(usecache) end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) -function statistics.savefmtstatus(texname,formatbanner,sourcefile) +function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") @@ -17361,6 +18858,10 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) sourcefile=sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) + lua.registerfinalizer(function() + logs.report("format banner","%s",banner) + logs.newline() + end) end end function statistics.checkfmtstatus(texname) @@ -17396,7 +18897,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 9036, stripped down to: 7041 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -17633,7 +19134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8712, stripped down to: 5726 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17822,7 +19323,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6779, stripped down to: 5444 +-- original size: 6653, stripped down to: 5467 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17868,11 +19369,21 @@ function resolvers.schemes.cleanname(specification) end return hash end -local cached,loaded,reused,thresholds,handlers={},{},{},{},{} -local function runcurl(name,cachename) - local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name - os.execute(command) -end +local cached={} +local loaded={} +local reused={} +local thresholds={} +local handlers={} +local runner=sandbox.registerrunner { + name="curl resolver", + method="execute", + program="curl", + template="--silent -- insecure --create-dirs --output %cachename% %original%", + checkers={ + cachename="cache", + original="url", + } +} local function fetch(specification) local original=specification.original local scheme=specification.scheme @@ -17894,7 +19405,10 @@ local function fetch(specification) report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") end logs.flush() - runcurl(original,cachename) + runner { + original=original, + cachename=cachename, + } end end if io.exists(cachename) then @@ -18003,7 +19517,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4447, stripped down to: 3302 +-- original size: 4207, stripped down to: 3137 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -18045,8 +19559,6 @@ function helpers.cleanpath(path) return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib -local getextraluapaths=package.extraluapaths -local getextralibpaths=package.extralibpaths local registerpath=helpers.registerpath local lualibfile=helpers.lualibfile local luaformatpaths @@ -18112,7 +19624,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2494, stripped down to: 2047 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -18179,7 +19691,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2674, stripped down to: 1658 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -18235,7 +19747,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2815, stripped down to: 2415 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -18315,7 +19827,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11846, stripped down to: 6059 +-- original size: 13595, stripped down to: 7500 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -18324,35 +19836,51 @@ if not modules then modules={} end modules ['util-lib']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } -local gsub,find=string.gsub,string.find -local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join -local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles -local loaded=package.loaded -local report_swiglib=logs.reporter("swiglib") -local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local type=type +local next=next +local pcall=pcall +local gsub=string.gsub +local find=string.find +local sort=table.sort +local pathpart=file.pathpart +local nameonly=file.nameonly +local joinfile=file.join +local removesuffix=file.removesuffix +local findfile=resolvers.findfile +local findfiles=resolvers.findfiles +local expandpaths=resolvers.expandedpathlistfromvariable +local qualifiedpath=file.is_qualified_path +local isfile=lfs.isfile local done=false -local function requireswiglib(required,version) - local trace_swiglib=trace_swiglib or package.helpers.trace - local library=loaded[required] - if library==nil then - if trace_swiglib then - report_swiglib("requiring library %a with version %a",required,version or "any") - end - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) +local function locate(required,version,trace,report,action) + if type(required)~="string" then + report("provide a proper library name") + return + end + if trace then + report("requiring library %a with version %a",required,version or "any") + end + local found_library=nil + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + if qualifiedpath(required) then + if isfile(required) then + found_library=required + end + else local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false local engine=environment.ownmain or false - if trace_swiglib and not done then - local list=resolvers.expandedpathlistfromvariable("lib") + if trace and not done then + local list=expandpaths("lib") for i=1,#list do - report_swiglib("tds path %i: %s",i,list[i]) + report("tds path %i: %s",i,list[i]) end end local function found(locate,asked_library,how,...) - if trace_swiglib then - report_swiglib("checking %s: %a",how,asked_library) + if trace then + report("checking %s: %a",how,asked_library) end return locate(asked_library,...) end @@ -18360,45 +19888,45 @@ local function requireswiglib(required,version) local found=nil if version then local asked_library=joinfile(required_path,version,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end if not found or found=="" then local asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end return found and found~="" and found or false end local function attempt(checkpattern) - if trace_swiglib then - report_swiglib("checking tds lib paths strictly") + if trace then + report("checking tds lib paths strictly") end local found=findfile and check(findfile,"lib") if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace_swiglib then - report_swiglib("checking tds lib paths with wildcard") + if trace then + report("checking tds lib paths with wildcard") end local asked_library=joinfile(required_path,".*",required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","latest version",asked_library) + if trace then + report("checking %s: %a","latest version",asked_library) end local list=findfiles(asked_library,"lib",true) if list and #list>0 then - table.sort(list) + sort(list) local found=list[#list] if found and (not checkpattern or find(found,checkpattern)) then return found end end - if trace_swiglib then - report_swiglib("checking lib paths") + if trace then + report("checking lib paths") end package.extralibpath(environment.ownpath) local paths=package.libpaths() @@ -18410,89 +19938,143 @@ local function requireswiglib(required,version) end return false end - local found_library=nil if engine then - if trace_swiglib then - report_swiglib("attemp 1, engine %a",engine) + if trace then + report("attemp 1, engine %a",engine) end found_library=attempt("/"..engine.."/") if not found_library then - if trace_swiglib then - report_swiglib("attemp 2, no engine",asked_library) + if trace then + report("attemp 2, no engine",asked_library) end found_library=attempt() end else found_library=attempt() end - if not found_library then - if trace_swiglib then - report_swiglib("not found: %a",required) - end + end + if not found_library then + if trace then + report("not found: %a",required) + end + library=false + else + if trace then + report("found: %a",found_library) + end + local message,result=action(found_library,required_base) + if result then + library=result + else library=false + report("load error: message %a, library %a",tostring(message),found_library or "no library") + end + end + if not library then + report("unknown: %a",required) + elseif trace then + report("stored: %a",required) + end + return library +end +do + local report_swiglib=logs.reporter("swiglib") + local trace_swiglib=false + local savedrequire=require + local loadedlibs={} + local loadlib=package.loadlib + local pushdir=dir.push + local popdir=dir.pop + trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) + function requireswiglib(required,version) + local library=loadedlibs[library] + if library==nil then + local trace_swiglib=trace_swiglib or package.helpers.trace + library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) + pushdir(pathpart(name)) + local opener="luaopen_"..base + if trace_swiglib then + report_swiglib("opening: %a with %a",name,opener) + end + local library,message=loadlib(name,opener) + local libtype=type(library) + if libtype=="function" then + library=library() + message=true + else + report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") + library=false + end + popdir() + return message,library + end) + loadedlibs[required]=library or false + end + return library + end + function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) else - local path=pathpart(found_library) - local base=nameonly(found_library) - dir.push(path) + return savedrequire(name) + end + end + local swiglibs={} + local initializer="core" + function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) if trace_swiglib then - report_swiglib("found: %a",found_library) - end - local message=nil - local opener="luaopen_"..required_base - library,message=package.loadlib(found_library,opener) - local libtype=type(library) - if libtype=="function" then - library=library() + report_swiglib("loading %a",name) + end + if not find(name,"%."..initializer.."$") then + fullname="swiglib."..name.."."..initializer else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + fullname="swiglib."..name end - dir.pop() + library=requireswiglib(fullname,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) end - if not library then - report_swiglib("unknown: %a",required) - elseif trace_swiglib then - report_swiglib("stored: %a",required) - end - loaded[required]=library - else - report_swiglib("reused: %a",required) + return library end - return library + statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end + end) end -local savedrequire=require -function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) +if FFISUPPORTED and ffi and ffi.load then + local report_ffilib=logs.reporter("ffilib") + local trace_ffilib=false + local savedffiload=ffi.load + trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) + local function locateindeed(name) + local message,library=pcall(savedffiload,removesuffix(name)) + if type(library)=="userdata" then + return library + else + return false + end end -end -local swiglibs={} -local initializer="core" -function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) + function ffilib(required,version) + if version=="system" then + return locateindeed(name) + else + return locate(required,version,trace_ffilib,report_ffilib,locateindeed) end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer + end + function ffi.load(name) + local library=ffilib(name) + if type(library)=="userdata" then + return library else - fullname="swiglib."..name + report_ffilib("trying to load %a using normal loader",name) + return savedffiload(name) end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end -statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end -end) end -- of closure @@ -18501,7 +20083,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5914, stripped down to: 2584 +-- original size: 5703, stripped down to: 2507 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -18604,7 +20186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 6967, stripped down to: 5631 +-- original size: 9144, stripped down to: 7291 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -18618,23 +20200,67 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local trackers=environment.argument("trackers") - local directives=environment.argument("directives") +local function primaryflags() + local arguments=environment.arguments + local flags={} + if arguments.silent then + flags[#flags+1]="--interaction=batchmode" + end + if arguments.jit then + flags[#flags+1]="--jiton" + end + return concat(flags," ") +end +local function secondaryflags() + local arguments=environment.arguments + local trackers=arguments.trackers + local directives=arguments.directives local flags={} if trackers and trackers~="" then - flags={ "--trackers="..quoted(trackers) } + flags[#flags+1]="--c:trackers="..quoted(trackers) end if directives and directives~="" then - flags={ "--directives="..quoted(directives) } + flags[#flags+1]="--c:directives="..quoted(directives) + end + if arguments.silent then + flags[#flags+1]="--c:silent" end - if environment.argument("jit") then - flags={ "--jiton" } + if arguments.jit then + flags[#flags+1]="--c:jiton" + end + if arguments.ansi then + flags[#flags+1]="--c:ansi" end return concat(flags," ") end -function environment.make_format(name,silent) +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] +local checkers={ + primaryflags="string", + secondaryflags="string", + luafile="readable", + texfile="readable", + redirect="string", + dump="string", +} +local runners={ + luatex=sandbox.registerrunner { + name="make luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="make luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} +function environment.make_format(name,arguments) local engine=environment.ownmain or "luatex" + local silent=environment.arguments.silent local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" if path~="" then @@ -18690,11 +20316,20 @@ function environment.make_format(name,silent) lfs.chdir(olddir) return end - local dump=os.platform=="unix" and "\\\\dump" or "\\dump" - if silent then + local specification={ + primaryflags=primaryflags(), + secondaryflags=secondaryflags(), + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), + dump=os.platform=="unix" and "\\\\dump" or "\\dump", + } + local runner=runners[engine] + if not runner then + report_format("format %a cannot be generated, no runner available for engine %a",name,engine) + elseif silent then statistics.starttiming() - local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - local result=os.execute(command) + specification.redirect="> temp.log" + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -18703,9 +20338,7 @@ function environment.make_format(name,silent) end os.remove("temp.log") else - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - report_format("running command: %s\n",command) - os.execute(command) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -18718,6 +20351,30 @@ function environment.make_format(name,silent) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -18739,9 +20396,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -18750,10 +20416,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 797557 --- stripped bytes : 289197 +-- original bytes : 842443 +-- stripped bytes : 306317 -- end library merge @@ -18777,6 +20443,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -18810,8 +20477,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) @@ -18983,7 +20651,7 @@ local helpinfo = [[ <metadata> <entry name="name">mtxrun</entry> <entry name="detail">ConTeXt TDS Runner Tool</entry> - <entry name="version">1.31</entry> + <entry name="version">1.32</entry> </metadata> <flags> <category name="basic"> @@ -19046,6 +20714,7 @@ local helpinfo = [[ </subcategory> <subcategory> <flag name="expand-braces"><short>expand complex variable</short></flag> + <flag name="resolve-path"><short>expand variable (completely resolve paths)</short></flag> <flag name="expand-path"><short>expand variable (resolve paths)</short></flag> <flag name="expand-var"><short>expand variable (resolve references)</short></flag> <flag name="show-path"><short>show path expansion of ...</short></flag> @@ -19063,7 +20732,7 @@ local helpinfo = [[ local application = logs.application { name = "mtxrun", - banner = "ConTeXt TDS Runner Tool 1.31", + banner = "ConTeXt TDS Runner Tool 1.32", helpinfo = helpinfo, } @@ -20029,6 +21698,13 @@ elseif e_argument("expand-path") then environment.initializearguments(environment.arguments_after) resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) +elseif e_argument("resolve-path") then + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.cleanedpathlist, environment.files) + elseif e_argument("expand-var") or e_argument("expand-variable") then -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) diff --git a/scripts/context/stubs/unix/ctxtools b/scripts/context/stubs/unix/ctxtools new file mode 100644 index 000000000..2e6bd4afa --- /dev/null +++ b/scripts/context/stubs/unix/ctxtools @@ -0,0 +1,2 @@ +#!/bin/sh +mtxrun --script ctxtools "$@" diff --git a/scripts/context/stubs/unix/mptopdf b/scripts/context/stubs/unix/mptopdf new file mode 100644 index 000000000..147333740 --- /dev/null +++ b/scripts/context/stubs/unix/mptopdf @@ -0,0 +1,2 @@ +#!/bin/sh +mtxrun --script mptopdf "$@" diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 7b711a88d..873770cac 100644 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 4734, stripped down to: 2626 +-- original size: 5125, stripped down to: 2881 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -162,6 +162,283 @@ if flush then local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end end +FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +if not FFISUPPORTED then + local okay;okay,ffi=pcall(require,"ffi") + FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +end +if not FFISUPPORTED then + ffi=nil +elseif not ffi.number then + ffi.number=tonumber +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 9667, stripped down to: 6678 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b==false then + return nil + elseif b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] or false +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end end -- of closure @@ -170,7 +447,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10949, stripped down to: 8037 +-- original size: 10587, stripped down to: 7815 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -460,7 +737,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38185, stripped down to: 20990 +-- original size: 37748, stripped down to: 20111 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -552,6 +829,7 @@ patterns.nonwhitespace=nonwhitespace local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local nospacer=Cs((whitespace^1/""+nonwhitespace^1)^0) local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) @@ -561,6 +839,7 @@ local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.nospacer=nospacer patterns.b_collapser=b_collapser patterns.m_collapser=m_collapser patterns.e_collapser=e_collapser @@ -1027,27 +1306,7 @@ function lpeg.append(list,pp,delayed,checked) end local p_false=P(false) local p_true=P(true) -local function make(t) - local function making(t) - local p=p_false - local keys=sortedkeys(t) - for i=1,#keys do - local k=keys[i] - if k~="" then - local v=t[k] - if v==true then - p=p+P(k)*p_true - elseif v==false then - else - p=p+P(k)*making(v) - end - end - end - if t[""] then - p=p+p_true - end - return p - end +local function make(t,rest) local p=p_false local keys=sortedkeys(t) for i=1,#keys do @@ -1058,10 +1317,13 @@ local function make(t) p=p+P(k)*p_true elseif v==false then else - p=p+P(k)*making(v) + p=p+P(k)*make(v,v[""]) end end end + if rest then + p=p+p_true + end return p end local function collapse(t,x) @@ -1264,7 +1526,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 372, stripped down to: 329 +-- original size: 361, stripped down to: 322 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1283,7 +1545,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5983, stripped down to: 2959 +-- original size: 6419, stripped down to: 3339 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1321,22 +1583,26 @@ end local stripper=patterns.stripper local fullstripper=patterns.fullstripper local collapser=patterns.collapser +local nospacer=patterns.nospacer local longtostring=patterns.longtostring function string.strip(str) - return lpegmatch(stripper,str) or "" + return str and lpegmatch(stripper,str) or "" end function string.fullstrip(str) - return lpegmatch(fullstripper,str) or "" + return str and lpegmatch(fullstripper,str) or "" end function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" + return str and lpegmatch(collapser,str) or "" +end +function string.nospaces(str) + return str and lpegmatch(nospacer,str) or "" end function string.longtostring(str) - return lpegmatch(longtostring,str) or "" + return str and lpegmatch(longtostring,str) or "" end local pattern=P(" ")^0*P(-1) function string.is_empty(str) - if str=="" then + if not str or str=="" then return true else return lpegmatch(pattern,str) and true or false @@ -1381,6 +1647,21 @@ function string.tformat(fmt,...) end string.quote=string.quoted string.unquote=string.unquoted +if not string.bytetable then + local limit=5000 + function string.bytetable(str) + local n=#str + if n>limit then + local t={ byte(str,1,limit) } + for i=limit+1,n do + t[i]=byte(str,i) + end + return t + else + return { byte(str,1,n) } + end + end +end end -- of closure @@ -1389,7 +1670,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 36997, stripped down to: 22376 +-- original size: 39608, stripped down to: 23165 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1716,19 +1997,23 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions,metacheck local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } -local function simple_table(t) +local function is_simple_table(t,hexify) local nt=#t if nt>0 then local n=0 for _,v in next,t do n=n+1 + if type(v)=="table" then + return nil + end end + local haszero=rawget(t,0) if n==nt then local tt={} for i=1,nt do @@ -1738,10 +2023,10 @@ local function simple_table(t) if hexify then tt[i]=format("0x%X",v) else - tt[i]=tostring(v) + tt[i]=v end elseif tv=="string" then - tt[i]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then tt[i]=v and "true" or "false" else @@ -1749,10 +2034,32 @@ local function simple_table(t) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + if hexify then + tt[i+1]=format("0x%X",v) + else + tt[i+1]=v + end + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil end +table.is_simple_table=is_simple_table local propername=patterns.propername local function dummy() end local function do_serialize(root,name,depth,level,indexed) @@ -1786,7 +2093,7 @@ local function do_serialize(root,name,depth,level,indexed) if compact then last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -1814,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -1851,6 +2158,7 @@ local function do_serialize(root,name,depth,level,indexed) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) @@ -1873,6 +2181,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,v)) else @@ -1888,13 +2197,14 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={},",depth,k and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then if tk=="number" then if hexify then @@ -1904,6 +2214,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={ %s },",depth,k and "true" or "false",concat(st,", "))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else @@ -1924,6 +2235,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%s,",depth,tostring(k),v and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%s,",depth,k,v and "true" or "false")) else @@ -1940,6 +2252,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=load(%q),",depth,k and "true" or "false",f)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=load(%q),",depth,k,f)) else @@ -1955,6 +2268,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",tostring(v))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,tostring(v))) else @@ -1976,6 +2290,7 @@ local function serialize(_handle,root,name,specification) functions=specification.functions compact=specification.compact inline=specification.inline and compact + metacheck=specification.metacheck if functions==nil then functions=true end @@ -1985,6 +2300,9 @@ local function serialize(_handle,root,name,specification) if inline==nil then inline=compact end + if metacheck==nil then + metacheck=true + end else noquotes=false hexify=false @@ -1992,6 +2310,7 @@ local function serialize(_handle,root,name,specification) compact=true inline=true functions=true + metacheck=true end if tname=="string" then if name=="return" then @@ -2015,7 +2334,7 @@ local function serialize(_handle,root,name,specification) handle("t={") end if root then - if getmetatable(root) then + if metacheck and getmetatable(root) then local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end @@ -2091,6 +2410,38 @@ local function flattened(t,f,depth) return f end table.flattened=flattened +local function collapsed(t,f,h) + if f==nil then + f={} + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsed(v,f,h) + elseif not h[v] then + f[#f+1]=v + h[v]=true + end + end + return f +end +local function collapsedhash(t,h) + if h==nil then + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsedhash(v,h) + else + h[v]=true + end + end + return h +end +table.collapsed=collapsed +table.collapsedhash=collapsedhash local function unnest(t,f) if not f then f={} @@ -2197,6 +2548,12 @@ function table.swapped(t,s) end return n end +function table.hashed(t) + for i=1,#t do + t[t[i]]=i + end + return t +end function table.mirrored(t) local n={} for k,v in next,t do @@ -2365,7 +2722,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 9001, stripped down to: 6512 +-- original size: 11790, stripped down to: 6961 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2375,6 +2732,7 @@ if not modules then modules={} end modules ['l-io']={ license="see context related readme files" } local io=io +local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor @@ -2384,50 +2742,56 @@ if string.find(os.getenv("PATH"),";",1,true) then else io.fileseparator,io.pathseparator="/",":" end -local function readall(f) - return f:read("*all") -end +local large=2^24 +local medium=large/16 +local small=medium/8 local function readall(f) local size=f:seek("end") - if size==0 then - return "" - elseif size<1024*1024 then + if size>0 then f:seek("set",0) - return f:read('*all') - else - local done=f:seek("set",0) - local step - if size<1024*1024 then - step=1024*1024 - elseif size>16*1024*1024 then - step=16*1024*1024 - else - step=floor(size/(1024*1024))*1024*1024/8 - end - local data={} - while true do - local r=f:read(step) - if not r then - return concat(data) - else - data[#data+1]=r - end - end + return f:read(size) + else + return "" end end io.readall=readall function io.loaddata(filename,textmode) - local f=io.open(filename,(textmode and 'r') or 'rb') + local f=open(filename,(textmode and 'r') or 'rb') if f then - local data=readall(f) + local size=f:seek("end") + local data=nil + if size>0 then + f:seek("set",0) + data=f:read(size) + end f:close() - if #data>0 then - return data + return data + end +end +function io.copydata(source,target,action) + local f=open(source,"rb") + if f then + local g=open(target,"wb") + if g then + local size=f:seek("end") + if size>0 then + f:seek("set",0) + local data=f:read(size) + if action then + data=action(data) + end + if data then + g:write(data) + end + end + g:close() end + f:close() + flush() end end function io.savedata(filename,data,joiner) - local f=io.open(filename,"wb") + local f=open(filename,"wb") if f then if type(data)=="table" then f:write(concat(data,joiner or "")) @@ -2437,40 +2801,70 @@ function io.savedata(filename,data,joiner) f:write(data or "") end f:close() - io.flush() + flush() return true else return false end end -function io.loadlines(filename,n) - local f=io.open(filename,'r') - if not f then - elseif n then - local lines={} - for i=1,n do - local line=f:read("*lines") - if line then - lines[#lines+1]=line - else - break +if fio and fio.readline then + local readline=fio.readline + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=readline(f) + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=readline(f) + f:close() + if line and #line>0 then + return line end end - f:close() - lines=concat(lines,"\n") - if #lines>0 then - return lines - end - else - local line=f:read("*line") or "" - f:close() - if #line>0 then - return line + end +else + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end end end end function io.loadchunk(filename,n) - local f=io.open(filename,'rb') + local f=open(filename,'rb') if f then local data=f:read(n or 1024) f:close() @@ -2480,7 +2874,7 @@ function io.loadchunk(filename,n) end end function io.exists(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return false else @@ -2489,7 +2883,7 @@ function io.exists(filename) end end function io.size(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return 0 else @@ -2498,11 +2892,11 @@ function io.size(filename) return s end end -function io.noflines(f) +local function noflines(f) if type(f)=="string" then - local f=io.open(filename) + local f=open(filename) if f then - local n=f and io.noflines(f) or 0 + local n=f and noflines(f) or 0 f:close() return n else @@ -2517,6 +2911,7 @@ function io.noflines(f) return n end end +io.noflines=noflines local nextchar={ [ 4]=function(f) return f:read(1,1,1,1) @@ -2594,16 +2989,16 @@ function io.bytes(f,n) end function io.ask(question,default,options) while true do - io.write(question) + write(question) if options then - io.write(format(" [%s]",concat(options,"|"))) + write(format(" [%s]",concat(options,"|"))) end if default then - io.write(format(" [%s]",default)) + write(format(" [%s]",default)) end - io.write(format(" ")) - io.flush() - local answer=io.read() + write(format(" ")) + flush() + local answer=read() answer=gsub(answer,"^%s*(.*)%s*$","%1") if answer=="" and default then return default @@ -2625,7 +3020,7 @@ function io.ask(question,default,options) end end end -local function readnumber(f,n,m) +local function readnumber(f,n,m) if m then f:seek("set",n) n=m @@ -2634,31 +3029,31 @@ local function readnumber(f,n,m) return byte(f:read(1)) elseif n==2 then local a,b=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==3 then local a,b,c=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==4 then local a,b,c,d=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==8 then local a,b=readnumber(f,4),readnumber(f,4) - return 256*a+b + return 0x100*a+b elseif n==12 then local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-2 then local b,a=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==-3 then local c,b,a=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-4 then local d,c,b,a=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==-8 then local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) - return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + return 0x100000000000000*a+0x1000000000000*b+0x10000000000*c+0x100000000*d+0x1000000*e+0x10000*f+0x100*g+h else return 0 end @@ -2680,7 +3075,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 5146, stripped down to: 2933 +-- original size: 5358, stripped down to: 3177 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2693,6 +3088,7 @@ local tostring,tonumber=tostring,tonumber local format,floor,match,rep=string.format,math.floor,string.match,string.rep local concat,insert=table.concat,table.insert local lpegmatch=lpeg.match +local floor=math.floor number=number or {} local number=number if bit32 then @@ -2817,6 +3213,26 @@ end function number.bits(n) return { bits(n,1) } end +function number.bytetodecimal(b) + local d=floor(b*100/255+0.5) + if d>100 then + return 100 + elseif d<-100 then + return -100 + else + return d + end +end +function number.decimaltobyte(d) + local b=floor(d*255/100+0.5) + if b>255 then + return 255 + elseif b<-255 then + return -255 + else + return b + end +end end -- of closure @@ -2825,7 +3241,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 2010, stripped down to: 1186 +-- original size: 1923, stripped down to: 1133 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2898,7 +3314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16390, stripped down to: 9734 +-- original size: 16268, stripped down to: 9246 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2974,7 +3390,7 @@ if not os.__getenv__ then end local execute=os.execute local iopopen=io.popen -function os.resultof(command) +local function resultof(command) local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" @@ -2984,9 +3400,13 @@ function os.resultof(command) return "" end end +os.resultof=resultof +function os.pipeto(command) + return iopopen(command,"w") +end if not io.fileseparator then if find(os.getenv("PATH"),";",1,true) then - io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "windows" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" end @@ -3029,17 +3449,6 @@ setmetatable(os,{ __index=function(t,k) return r and r(t,k) or nil end }) local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" -local function guess() - local architecture=os.resultof("uname -m") or "" - if architecture~="" then - return architecture - end - architecture=os.getenv("HOSTTYPE") or "" - if architecture~="" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end if platform~="" then os.platform=platform elseif os.type=="windows" then @@ -3056,7 +3465,7 @@ elseif os.type=="windows" then end elseif name=="linux" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="linux-64" elseif find(architecture,"ppc",1,true) then @@ -3070,7 +3479,7 @@ elseif name=="linux" then end elseif name=="macosx" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + local platform,architecture="",resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" elseif find(architecture,"i386",1,true) then @@ -3086,7 +3495,7 @@ elseif name=="macosx" then end elseif name=="sunos" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"sparc",1,true) then platform="solaris-sparc" else @@ -3098,7 +3507,7 @@ elseif name=="sunos" then end elseif name=="freebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else @@ -3110,7 +3519,7 @@ elseif name=="freebsd" then end elseif name=="kfreebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else @@ -3241,7 +3650,7 @@ if not os.sleep then end end local function isleapyear(year) - return (year%400==0) or ((year%100~=0) and (year%4==0)) + return (year%4==0) and (year%100~=0 or year%400==0) end os.isleapyear=isleapyear local days={ 31,28,31,30,31,30,31,31,30,31,30,31 } @@ -3280,7 +3689,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21648, stripped down to: 10238 +-- original size: 20997, stripped down to: 9986 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3617,13 +4026,15 @@ function file.robustname(str,strict) end end end -file.readdata=io.loaddata -file.savedata=io.savedata +local loaddata=io.loaddata +local savedata=io.savedata +file.readdata=loaddata +file.savedata=savedata function file.copy(oldname,newname) if oldname and newname then - local data=io.loaddata(oldname) + local data=loaddata(oldname) if data and data~="" then - file.savedata(newname,data) + savedata(newname,data) end end end @@ -3660,7 +4071,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1265, stripped down to: 1038 +-- original size: 1211, stripped down to: 1002 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3714,7 +4125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3355, stripped down to: 2321 +-- original size: 3309, stripped down to: 2314 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3744,6 +4155,8 @@ do if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + md5.sumhexa=md5.hex + md5.sumHEXA=md5.HEX end end function file.needsupdating(oldname,newname,threshold) @@ -3802,7 +4215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12897, stripped down to: 5882 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4019,7 +4432,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 17358, stripped down to: 11378 +-- original size: 17703, stripped down to: 11691 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4283,6 +4696,31 @@ local function globfiles(path,recurse,func,files) return files end dir.globfiles=globfiles +local function globdirs(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + if recurse then + globdirs(path.."/"..name,recurse,func,files) + end + end + end + end + end + return files +end +dir.globdirs=globdirs function dir.ls(pattern) return concat(glob(pattern),"\n") end @@ -4447,9 +4885,13 @@ end file.expandname=dir.expandname local stack={} function dir.push(newdir) - insert(stack,currentdir()) + local curdir=currentdir() + insert(stack,curdir) if newdir and newdir~="" then chdir(newdir) + return newdir + else + return curdir end end function dir.pop() @@ -4484,7 +4926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1919, stripped down to: 1621 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4556,7 +4998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 38699, stripped down to: 16321 +-- original size: 38263, stripped down to: 16330 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -5167,6 +5609,23 @@ function utf.chrlen(u) (u<0xFC and 5) or (u<0xFE and 6) or 0 end +local extract=bit32.extract +local char=string.char +function unicode.toutf32string(n) + if n<=0xFF then + return + char(n).."\000\000\000" + elseif n<=0xFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8)).."\000\000" + elseif n<=0xFFFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8)).."\000" + else + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8))..char(extract(n,24,8)) + end +end end -- of closure @@ -5175,7 +5634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 1012, stripped down to: 912 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5215,7 +5674,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 36053, stripped down to: 19685 +-- original size: 36148, stripped down to: 20179 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5227,7 +5686,7 @@ if not modules then modules={} end modules ['util-str']={ utilities=utilities or {} utilities.strings=utilities.strings or {} local strings=utilities.strings -local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local format,gsub,rep,sub,find=string.format,string.gsub,string.rep,string.sub,string.find local load,dump=load,string.dump local tonumber,type,tostring=tonumber,type,tostring local unpack,concat=table.unpack,table.concat @@ -5410,6 +5869,25 @@ function number.signed(i) return "-",-i end end +local digit=patterns.digit +local period=patterns.period +local three=digit*digit*digit +local splitter=Cs ( + (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) +) +patterns.formattednumber=splitter +function number.formatted(n,sep1,sep2) + local s=type(s)=="string" and n or format("%0.2f",n) + if sep1==true then + return lpegmatch(splitter,s,1,".",",") + elseif sep1=="." then + return lpegmatch(splitter,s,1,sep1,sep2 or ",") + elseif sep1=="," then + return lpegmatch(splitter,s,1,sep1,sep2 or ".") + else + return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") + end +end local zero=P("0")^1/"" local plus=P("+")/"" local minus=P("-") @@ -5435,6 +5913,27 @@ function number.sparseexponent(f,n) end return tostring(n) end +local hf={} +local hs={} +setmetatable(hf,{ __index=function(t,k) + local v="%."..k.."f" + t[k]=v + return v +end } ) +setmetatable(hs,{ __index=function(t,k) + local v="%"..k.."s" + t[k]=v + return v +end } ) +function number.formattedfloat(n,b,a) + local s=format(hf[a],n) + local l=(b or 0)+(a or 0)+1 + if #s<l then + return format(hs[l],s) + else + return s + end +end local template=[[ %s %s @@ -5462,6 +5961,7 @@ local autodouble=string.autodouble local sequenced=table.sequenced local formattednumber=number.formatted local sparseexponent=number.sparseexponent +local formattedfloat=number.formattedfloat ]] else environment={ @@ -5485,6 +5985,7 @@ else sequenced=table.sequenced, formattednumber=number.formatted, sparseexponent=number.sparseexponent, + formattedfloat=number.formattedfloat } end local arguments={ "a1" } @@ -5495,6 +5996,7 @@ setmetatable(arguments,{ __index=function(t,k) end }) local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_sub=(C((S("+-")+R("09"))^0)+Cc(0))*P(".")*(C((S("+-")+R("09"))^0)+Cc(0)) local prefix_tab=P("{")*C((1-P("}"))^0)*P("}")+C((1-R("az","AZ","09","%%"))^0) local format_s=function(f) n=n+1 @@ -5545,6 +6047,10 @@ local format_F=function(f) return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) end end +local format_k=function(b,a) + n=n+1 + return format("formattedfloat(a%s,%i,%i)",n,b or 0,a or 0) +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5693,25 +6199,6 @@ end local format_W=function(f) return format("nspaces[%s]",tonumber(f) or 0) end -local digit=patterns.digit -local period=patterns.period -local three=digit*digit*digit -local splitter=Cs ( - (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) -) -patterns.formattednumber=splitter -function number.formatted(n,sep1,sep2) - local s=type(s)=="string" and n or format("%0.2f",n) - if sep1==true then - return lpegmatch(splitter,s,1,".",",") - elseif sep1=="." then - return lpegmatch(splitter,s,1,sep1,sep2 or ",") - elseif sep1=="," then - return lpegmatch(splitter,s,1,sep1,sep2 or ".") - else - return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") - end -end local format_m=function(f) n=n+1 if not f or f=="" then @@ -5736,9 +6223,16 @@ end local format_extension=function(extensions,f,name) local extension=extensions[name] or "tostring(%s)" local f=tonumber(f) or 1 + local w=find(extension,"%.%.%.") if f==0 then + if w then + extension=gsub(extension,"%.%.%.","") + end return extension elseif f==1 then + if w then + extension=gsub(extension,"%.%.%.","%%s") + end n=n+1 local a="a"..n return format(extension,a,a) @@ -5746,10 +6240,13 @@ local format_extension=function(extensions,f,name) local a="a"..(n+f+1) return format(extension,a,a) else + if w then + extension=gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s") + end local t={} for i=1,f do n=n+1 - t[#t+1]="a"..n + t[i]="a"..n end return format(extension,unpack(t)) end @@ -5762,7 +6259,8 @@ local builder=Cs { "start", +V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") -+V("N") ++V("N") ++V("k") +V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") +V("W") +V("a") @@ -5789,6 +6287,7 @@ local builder=Cs { "start", ["S"]=(prefix_any*P("S"))/format_S, ["Q"]=(prefix_any*P("Q"))/format_S, ["N"]=(prefix_any*P("N"))/format_N, + ["k"]=(prefix_sub*P("k"))/format_k, ["c"]=(prefix_any*P("c"))/format_c, ["C"]=(prefix_any*P("C"))/format_C, ["r"]=(prefix_any*P("r"))/format_r, @@ -5909,7 +6408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28680, stripped down to: 18636 +-- original size: 27407, stripped down to: 17116 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5923,7 +6422,7 @@ utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort -local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local setmetatable,getmetatable,tonumber,tostring,rawget=setmetatable,getmetatable,tonumber,tostring,rawget local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs @@ -6063,7 +6562,7 @@ function table.tocsv(t,specification) r[f]=tostring(field) end end - result[#result+1]=concat(r,separator) + result[i+1]=concat(r,separator) end return concat(result,"\n") else @@ -6295,11 +6794,12 @@ function table.autokey(t,k) return v end local selfmapper={ __index=function(t,k) t[k]=k return k end } -function table.twowaymapper(t) - if not t then - t={} - else - for i=0,#t do +function table.twowaymapper(t) + if not t then + t={} + else + local zero=rawget(t,0) + for i=zero and 0 or 1,#t do local ti=t[i] if ti then local i=tostring(i) @@ -6307,7 +6807,6 @@ function table.twowaymapper(t) t[ti]=i end end - t[""]=t[0] or "" end setmetatable(t,selfmapper) return t @@ -6346,6 +6845,7 @@ local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize +local is_simple_table=table.is_simple_table local function serialize(root,name,specification) if type(specification)=="table" then return original_serialize(root,name,specification) @@ -6353,54 +6853,6 @@ local function serialize(root,name,specification) local t local n=1 local unknown=false - local function simple_table(t) - local nt=#t - if nt>0 then - local n=0 - for _,v in next,t do - n=n+1 - if type(v)=="table" then - return nil - end - end - local haszero=t[0] - if n==nt then - local tt={} - for i=1,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i]=v - elseif tv=="string" then - tt[i]=format("%q",v) - elseif tv=="boolean" then - tt[i]=v and "true" or "false" - else - return nil - end - end - return tt - elseif haszero and (n==nt+1) then - local tt={} - for i=0,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i+1]=v - elseif tv=="string" then - tt[i+1]=format("%q",v) - elseif tv=="boolean" then - tt[i+1]=v and "true" or "false" - else - return nil - end - end - tt[1]="[0] = "..tt[1] - return tt - end - end - return nil - end local function do_serialize(root,name,depth,level,indexed) if level>0 then n=n+1 @@ -6425,7 +6877,7 @@ local function serialize(root,name,specification) local last=0 last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -6448,7 +6900,7 @@ local function serialize(root,name,specification) if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else - local st=simple_table(v) + local st=is_simple_table(v) if st then n=n+1 t[n]=f_val_seq(depth,st) else @@ -6492,7 +6944,7 @@ local function serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else - local st=simple_table(v) + local st=is_simple_table(v) if not st then do_serialize(v,k,depth,level+1) elseif tk=="number" then @@ -6552,11 +7004,11 @@ local function serialize(root,name,specification) end if root then if getmetatable(root) then - local dummy=root._w_h_a_t_e_v_e_r_ + local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - local st=simple_table(root) + local st=is_simple_table(root) if st then return t[1]..f_fin_seq(st) else @@ -6570,7 +7022,12 @@ local function serialize(root,name,specification) end table.serialize=serialize if setinspector then - setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) + setinspector("table",function(v) + if type(v)=="table" then + print(serialize(v,"table",{ metacheck=false })) + return true + end + end) end @@ -6580,7 +7037,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fil"] = package.loaded["util-fil"] or true --- original size: 3577, stripped down to: 2870 +-- original size: 7567, stripped down to: 5575 if not modules then modules={} end modules ['util-fil']={ version=1.001, @@ -6590,7 +7047,9 @@ if not modules then modules={} end modules ['util-fil']={ license="see context related readme files" } local byte=string.byte -local extract=bit32.extract +local char=string.char +local extract=bit32 and bit32.extract +local floor=math.floor utilities=utilities or {} local files={} utilities.files=files @@ -6609,6 +7068,7 @@ end function files.size(f) return f:seek("end") end +files.getsize=files.size function files.setposition(f,n) if zerobased[f] then f:seek("set",n) @@ -6646,6 +7106,10 @@ end function files.readbytes(f,n) return byte(f:read(n),1,n) end +function files.readbytetable(f,n) + local s=f:read(n or 1) + return { byte(s,1,#s) } +end function files.readchar(f) return f:read(1) end @@ -6655,7 +7119,7 @@ end function files.readinteger1(f) local n=byte(f:read(1)) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6663,55 +7127,107 @@ end files.readcardinal1=files.readbyte files.readcardinal=files.readcardinal1 files.readinteger=files.readinteger1 +files.readsignedbyte=files.readinteger1 function files.readcardinal2(f) local a,b=byte(f:read(2),1,2) return 0x100*a+b end +function files.readcardinal2le(f) + local b,a=byte(f:read(2),1,2) + return 0x100*a+b +end function files.readinteger2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function files.readinteger2le(f) + local b,a=byte(f:read(2),1,2) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function files.readcardinal3(f) local a,b,c=byte(f:read(3),1,3) return 0x10000*a+0x100*b+c end +function files.readcardinal3le(f) + local c,b,a=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readinteger3(f) + local a,b,c=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function files.readinteger3le(f) + local c,b,a=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function files.readcardinal4(f) local a,b,c,d=byte(f:read(4),1,4) return 0x1000000*a+0x10000*b+0x100*c+d end +function files.readcardinal4le(f) + local d,c,b,a=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end function files.readinteger4(f) local a,b,c,d=byte(f:read(4),1,4) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.readfixed4(f) - local a,b,c,d=byte(f:read(4),1,4) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF +function files.readinteger4le(f) + local d,c,b,a=byte(f:read(4),1,4) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n+(0x100*c+d)/0xFFFF + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.read2dot14(f) +function files.readfixed2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - n=extract(n,30,2) - return n+m/0x4000 + return (0x100*a+b )+(0x100*c+d)/0x10000 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function files.skipshort(f,n) @@ -6720,6 +7236,55 @@ end function files.skiplong(f,n) f:read(4*(n or 1)) end +function files.writecardinal2(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + f:write(b,a) +end +function files.writecardinal4(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + n=floor(n/256) + local c=char(n%256) + n=floor(n/256) + local d=char(n%256) + f:write(d,c,b,a) +end +function files.writestring(f,s) + f:write(char(byte(s,1,#s))) +end +function files.writebyte(f,b) + f:write(char(b)) +end +if fio and fio.readcardinal1 then + files.readcardinal1=fio.readcardinal1 + files.readcardinal2=fio.readcardinal2 + files.readcardinal3=fio.readcardinal3 + files.readcardinal4=fio.readcardinal4 + files.readinteger1=fio.readinteger1 + files.readinteger2=fio.readinteger2 + files.readinteger3=fio.readinteger3 + files.readinteger4=fio.readinteger4 + files.read2dot14=fio.read2dot14 + files.setposition=fio.setposition + files.getposition=fio.getposition + files.readbyte=files.readcardinal1 + files.readsignedbyte=files.readinteger1 + files.readcardinal=files.readcardinal1 + files.readinteger=files.readinteger1 + local skipposition=fio.skipposition + files.skipposition=skipposition + files.readbytes=fio.readbytes + files.readbytetable=fio.readbytetable + function files.skipshort(f,n) + skipposition(f,2*(n or 1)) + end + function files.skiplong(f,n) + skipposition(f,4*(n or 1)) + end +end end -- of closure @@ -6728,7 +7293,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 4264, stripped down to: 3349 +-- original size: 8716, stripped down to: 6754 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -6738,7 +7303,7 @@ if not modules then modules={} end modules ['util-sac']={ license="see context related readme files" } local byte,sub=string.byte,string.sub -local extract=bit32.extract +local extract=bit32 and bit32.extract utilities=utilities or {} local streams={} utilities.streams=streams @@ -6796,6 +7361,12 @@ function streams.readbytes(f,n) f[2]=j return byte(f[1],i,j-1) end +function streams.readbytetable(f,n) + local i=f[2] + local j=i+n + f[2]=j + return { byte(f[1],i,j-1) } +end function streams.skipbytes(f,n) f[2]=f[2]+n end @@ -6815,7 +7386,7 @@ function streams.readinteger1(f) f[2]=i+1 local n=byte(f[1],i) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6830,16 +7401,33 @@ function streams.readcardinal2(f) local a,b=byte(f[1],i,j) return 0x100*a+b end +function streams.readcardinal2LE(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + return 0x100*a+b +end function streams.readinteger2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function streams.readinteger2le(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function streams.readcardinal3(f) @@ -6849,6 +7437,35 @@ function streams.readcardinal3(f) local a,b,c=byte(f[1],i,j) return 0x10000*a+0x100*b+c end +function streams.readcardinal3le(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readinteger3(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function streams.readinteger3le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function streams.readcardinal4(f) local i=f[2] local j=i+3 @@ -6861,11 +7478,21 @@ function streams.readinteger4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d + end +end +function streams.readinteger4le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local d,c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 + else + return 0x1000000*a+0x10000*b+0x100*c+d end end function streams.readfixed4(f) @@ -6873,26 +7500,38 @@ function streams.readfixed4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - return n+(0x100*c+d)/0xFFFF + return (0x100*a+b )+(0x100*c+d)/0x10000 end end -function streams.read2dot14(f) +function streams.readfixed2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 - else - n=extract(n,30,2) - return n+m/0x4000 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function streams.skipshort(f,n) @@ -6901,6 +7540,92 @@ end function streams.skiplong(f,n) f[2]=f[2]+4*(n or 1) end +if sio and sio.readcardinal2 then + local readcardinal1=sio.readcardinal1 + local readcardinal2=sio.readcardinal2 + local readcardinal3=sio.readcardinal3 + local readcardinal4=sio.readcardinal4 + local readinteger1=sio.readinteger1 + local readinteger2=sio.readinteger2 + local readinteger3=sio.readinteger3 + local readinteger4=sio.readinteger4 + local readfixed2=sio.readfixed2 + local readfixed4=sio.readfixed4 + local read2dot14=sio.read2dot14 + local readbytes=sio.readbytes + local readbytetable=sio.readbytetable + function streams.readcardinal1(f) + local i=f[2] + f[2]=i+1 + return readcardinal1(f[1],i) + end + function streams.readcardinal2(f) + local i=f[2] + f[2]=i+2 + return readcardinal2(f[1],i) + end + function streams.readcardinal3(f) + local i=f[2] + f[2]=i+3 + return readcardinal3(f[1],i) + end + function streams.readcardinal4(f) + local i=f[2] + f[2]=i+4 + return readcardinal4(f[1],i) + end + function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + return readinteger1(f[1],i) + end + function streams.readinteger2(f) + local i=f[2] + f[2]=i+2 + return readinteger2(f[1],i) + end + function streams.readinteger3(f) + local i=f[2] + f[2]=i+3 + return readinteger3(f[1],i) + end + function streams.readinteger4(f) + local i=f[2] + f[2]=i+4 + return readinteger4(f[1],i) + end + function streams.read2dot4(f) + local i=f[2] + f[2]=i+2 + return read2dot4(f[1],i) + end + function streams.readbytes(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytes(f[1],i,n) + end + function streams.readbytetable(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytetable(f[1],i,n) + end + streams.readbyte=streams.readcardinal1 + streams.readsignedbyte=streams.readinteger1 + streams.readcardinal=streams.readcardinal1 + streams.readinteger=streams.readinteger1 +end end -- of closure @@ -6909,7 +7634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4100, stripped down to: 2852 +-- original size: 3926, stripped down to: 2742 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -7049,7 +7774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 23411, stripped down to: 16177 +-- original size: 22883, stripped down to: 16045 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -7211,6 +7936,21 @@ function parsers.settings_to_array(str,strict) return { str } end end +function parsers.settings_to_numbers(str) + if not str or str=="" then + return {} + end + if type(str)=="table" then + elseif find(str,",",1,true) then + str=lpegmatch(pattern,str) + else + return { tonumber(str) } + end + for i=1,#str do + str[i]=tonumber(str[i]) + end + return str +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) local pattern=spaces*Ct(value*(separator*value)^0) function parsers.settings_to_array_obey_fences(str) @@ -7587,7 +8327,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2350, stripped down to: 1847 +-- original size: 2274, stripped down to: 1781 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7668,7 +8408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12862, stripped down to: 9104 +-- original size: 12454, stripped down to: 8840 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7854,7 +8594,6 @@ function setters.list(t) return user,system end function setters.show(t) - local category=t.name local list=setters.list(t) t.report() for k=1,#list do @@ -7981,7 +8720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 30767, stripped down to: 21355 +-- original size: 30007, stripped down to: 20818 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -8027,6 +8766,14 @@ if tex and (tex.jobname or tex.formatname) then if texio.setescape then texio.setescape(0) end + if arg then + for k,v in next,arg do + if v=="--ansi" or v=="--c:ansi" then + variant="ansi" + break + end + end + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -8612,7 +9359,6 @@ function logs.stop_page_number() end logs.flush() end -local report_files=logs.reporter("files") local nesting=0 local verbose=false local hasscheme=url.hasscheme @@ -8774,7 +9520,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6917, stripped down to: 5484 +-- original size: 8036, stripped down to: 5567 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8806,11 +9552,13 @@ end local function resettiming(instance) timers[instance or "notimer"]={ timing=0,loadtime=0 } end +local ticks=clock +local seconds=function(n) return n or 0 end local function starttiming(instance) local timer=timers[instance or "notimer"] local it=timer.timing or 0 if it==0 then - timer.starttime=clock() + timer.starttime=ticks() if not timer.loadtime then timer.loadtime=0 end @@ -8824,12 +9572,13 @@ local function stoptiming(instance) timer.timing=it-1 else local starttime=timer.starttime - if starttime then - local stoptime=clock() + if starttime and starttime>0 then + local stoptime=ticks() local loadtime=stoptime-starttime timer.stoptime=stoptime timer.loadtime=timer.loadtime+loadtime timer.timing=0 + timer.starttime=0 return loadtime end end @@ -8840,7 +9589,7 @@ local function elapsed(instance) return instance or 0 else local timer=timers[instance or "notimer"] - return timer and timer.loadtime or 0 + return timer and seconds(timer.loadtime) or 0 end end local function elapsedtime(instance) @@ -8888,10 +9637,13 @@ function statistics.show() local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) - if jit then - local jitstatus={ jit.status() } - if jitstatus[1] then - register("luajit options",concat(jitstatus," ",2)) + if TEXENGINE=="luajittex" and JITSUPPORTED then + local jitstatus=jit.status + if jitstatus then + local jitstatus={ jitstatus() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end end end register("lua properties",function() @@ -8955,7 +9707,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 6039, stripped down to: 3616 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -9102,7 +9854,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 5142, stripped down to: 3611 +-- original size: 5396, stripped down to: 3708 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -9224,6 +9976,17 @@ function luautilities.loadstripped(...) return load(dump(l,true)) end end +local finalizers={} +setmetatable(finalizers,{ + __gc=function(t) + for i=1,#t do + pcall(t[i]) + end + end +} ) +function luautilities.registerfinalizer(f) + finalizers[#finalizers+1]=f +end end -- of closure @@ -9232,7 +9995,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 4030, stripped down to: 2718 +-- original size: 8911, stripped down to: 6504 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -9242,75 +10005,230 @@ if not modules then modules={} end modules ['util-deb']={ license="see context related readme files" } local debug=require "debug" -local getinfo=debug.getinfo -local type,next,tostring=type,next,tostring -local format,find=string.format,string.find -local is_boolean=string.is_boolean +local getinfo,sethook=debug.getinfo,debug.sethook +local type,next,tostring,tonumber=type,next,tostring,tonumber +local format,find,sub,gsub=string.format,string.find,string.sub,string.gsub +local insert,remove,sort=table.insert,table.remove,table.sort +local setmetatableindex=table.setmetatableindex utilities=utilities or {} local debugger=utilities.debugger or {} utilities.debugger=debugger -local counters={} -local names={} local report=logs.reporter("debugger") -local function hook() - local f=getinfo(2) - if f then - local n="unknown" - if f.what=="C" then - n=f.name or '<anonymous>' - if not names[n] then - names[n]=format("%42s",n) +local ticks=os.gettimeofday or os.clock +local seconds=function(n) return n or 0 end +local overhead=0 +local dummycalls=10*1000 +local nesting=0 +local names={} +local initialize=false +if not (FFISUPPORTED and ffi) then +elseif os.type=="windows" then + initialize=function() + local kernel=ffilib("kernel32","system") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end end - else - n=f.name or f.namewhat or f.what - if not n or n=="" then - n="?" + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end + initialize=false + end +elseif os.type=="unix" then + initialize=function() + local C=ffi.C + local tonumber=ffi.number or tonumber + ffi.cdef [[ + /* what a mess */ + typedef int clk_id_t; + typedef enum { CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_PROCESS_CPUTIME_ID } clk_id; + typedef struct timespec { long sec; long nsec; } ctx_timespec; + int clock_gettime(clk_id_t timerid, struct timespec *t); + ]] + local target=ffi.new("ctx_timespec[?]",1) + local clock=C.CLOCK_PROCESS_CPUTIME_ID + ticks=function () + C.clock_gettime(clock,target) + return tonumber(target[0].sec*1000000000+target[0].nsec) + end + seconds=function(ticks) + return ticks/1000000000 + end + initialize=false + end +end +setmetatableindex(names,function(t,name) + local v=setmetatableindex(function(t,source) + local v=setmetatableindex(function(t,line) + local v={ total=0,count=0 } + t[line]=v + return v + end) + t[source]=v + return v + end) + t[name]=v + return v +end) +local function hook(where) + local f=getinfo(2,"nSl") + if f then + local source=f.short_src + if not source then + return + end + local line=f.linedefined or 0 + local name=f.name + if not name then + local what=f.what + if what=="C" then + name="<anonymous>" + else + name=f.namewhat or what or "<unknown>" end - if not names[n] then - names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + local data=names[name][source][line] + if where=="call" then + data.count=data.count+1 + insert(data,ticks()) + elseif where=="return" then + local t=remove(data) + if t then + data.total=data.total+ticks()-t end end - counters[n]=(counters[n] or 0)+1 end end -function debugger.showstats(printer,threshold) - printer=printer or report - threshold=threshold or 0 - local total,grandtotal,functions=0,0,0 +function debugger.showstats(printer,threshold) + local printer=printer or report + local calls=0 + local functions=0 local dataset={} - for name,count in next,counters do - dataset[#dataset+1]={ name,count } + local length=0 + local wholetime=0 + local threshold=threshold or 0 + for name,sources in next,names do + for source,lines in next,sources do + for line,data in next,lines do + local count=data.count + if count>threshold then + if #name>length then + length=#name + end + local total=data.total + local real=total + if real>0 then + real=total-(count*overhead/dummycalls) + if real<0 then + real=0 + end + wholetime=wholetime+real + end + if line<0 then + line=0 + end + dataset[#dataset+1]={ real,total,count,name,source,line } + end + end + end end - table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + sort(dataset,function(a,b) + if a[1]==b[1] then + if a[2]==b[2] then + if a[3]==b[3] then + if a[4]==b[4] then + if a[5]==b[5] then + return a[6]<b[6] + else + return a[5]<b[5] + end + else + return a[4]<b[4] + end + else + return b[3]<a[3] + end + else + return b[2]<a[2] + end + else + return b[1]<a[1] + end + end) + if length>50 then + length=50 + end + local fmt=string.formatters["%4.9k %4.9k %3.3k %8i %-"..length.."s %4i %s"] for i=1,#dataset do - local d=dataset[i] - local name=d[1] - local count=d[2] - if count>threshold and not find(name,"for generator") then - printer(format("%8i %s\n",count,names[name])) - total=total+count - end - grandtotal=grandtotal+count + local data=dataset[i] + local real=data[1] + local total=data[2] + local count=data[3] + local name=data[4] + local source=data[5] + local line=data[6] + local percent=real/wholetime + calls=calls+count functions=functions+1 + name=gsub(name,"%s+"," ") + if #name>length then + name=sub(name,1,length) + end + printer(fmt(seconds(total),seconds(real),percent,count,name,line,source)) end - printer("\n") - printer(format("functions : % 10i\n",functions)) - printer(format("total : % 10i\n",total)) - printer(format("grand total: % 10i\n",grandtotal)) - printer(format("threshold : % 10i\n",threshold)) + printer("") + printer(format("functions : %i",functions)) + printer(format("calls : %i",calls)) + printer(format("overhead : %f",seconds(overhead/1000))) end function debugger.savestats(filename,threshold) local f=io.open(filename,'w') if f then - debugger.showstats(function(str) f:write(str) end,threshold) + debugger.showstats(function(str) f:write(str,"\n") end,threshold) f:close() end end function debugger.enable() - debug.sethook(hook,"c") + if nesting==0 then + running=true + if initialize then + initialize() + end + sethook(hook,"cr") + local function dummy() end + local t=ticks() + for i=1,dummycalls do + dummy() + end + overhead=ticks()-t + end + if nesting>0 then + nesting=nesting+1 + end end function debugger.disable() - debug.sethook() + if nesting>0 then + nesting=nesting-1 + end + if nesting==0 then + sethook() + end end local function showtraceback(rep) local level=2 @@ -9334,9 +10252,661 @@ end -- of closure do -- create closure to overcome 200 locals limit +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true + +-- original size: 7100, stripped down to: 3978 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20309, stripped down to: 13848 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + if type(root)=="table" then + root,what=root[1],root[2] + end + if type(root)=="string" and root~="" then + root=collapsepath(expandname(root)) + if what=="r" or what=="ro" or what=="readable" then + what="read" + elseif what=="w" or what=="wo" or what=="writable" then + what="write" + end + validroots[root]=what=="write" or false + end + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + elseif filenamelogger then + filenamelogger(name,"*",name,false) + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + if trace then + report("resultof: %s",command) + end + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + if trace then + report("execute: %s",command) + end + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + if trace then + report("pipeto: %s",command) + end + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +function sandbox.getrunner(name) + return name and validrunners[name] +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if FFISUPPORTED and ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7757, stripped down to: 6015 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9511,154 +11081,9 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8284, stripped down to: 5176 +-- original size: 9246, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9845,7 +11270,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6358, stripped down to: 4257 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9998,7 +11423,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 56973, stripped down to: 35872 +-- original size: 57003, stripped down to: 35696 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -10013,7 +11438,7 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset +local type,next,setmetatable,getmetatable,tonumber,rawset,select=type,next,setmetatable,getmetatable,tonumber,rawset,select local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local sort=table.sort local utfchar=utf.char @@ -10140,6 +11565,7 @@ local function add_empty(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nt, __p__=top } dt[nt]=t @@ -10161,6 +11587,7 @@ local function add_begin(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nil, __p__=stack[level] } setmetatable(top,mt) @@ -10188,6 +11615,7 @@ local function add_end(spacing,namespace,tag) dt=top.dt nt=#dt+1 dt[nt]=toclose + toclose.ni=nt if toclose.at.xmlns then remove(xmlns) end @@ -10232,7 +11660,13 @@ local function add_special(what,spacing,text) if strip and (what=="@cm@" or what=="@dt@") then else nt=nt+1 - dt[nt]={ special=true,ns="",tg=what,dt={ text } } + dt[nt]={ + special=true, + ns="", + tg=what, + ni=nil, + dt={ text }, + } end end local function set_message(txt) @@ -10285,7 +11719,6 @@ do end local p_rest=(1-P(";"))^0 local p_many=P(1)^0 - local p_char=lpegpatterns.utf8character local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) xml.parsedentitylpeg=parsedentity local predefined_unified={ @@ -10327,13 +11760,27 @@ do [ [[}]] ]="&U+7D;", [ [[~]] ]="&U+7E;", } + local privates_x={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[']] ]="&U+27;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } local privates_n={ } local escaped=utf.remapper(privates_u,"dynamic") local unprivatized=utf.remapper(privates_p,"dynamic") local unspecialized=utf.remapper(privates_s,"dynamic") + local despecialized=utf.remapper(privates_x,"dynamic") xml.unprivatized=unprivatized xml.unspecialized=unspecialized + xml.despecialized=despecialized xml.escaped=escaped local function unescaped(s) local p=privates_n[s] @@ -10778,6 +12225,10 @@ local grammar_unparsed_text=P { "preamble", local function _xmlconvert_(data,settings) settings=settings or {} preparexmlstate(settings) + local preprocessor=settings.preprocessor + if data and data~="" and type(preprocessor)=="function" then + data=preprocessor(data,settings) or data + end if settings.parent_root then mt=getmetatable(settings.parent_root) else @@ -10919,14 +12370,24 @@ function xml.toxml(data) return data end end -local function copy(old) +local function copy(old,p) if old then local new={} for k,v in next,old do - if type(v)=="table" then - new[k]=table.copy(v) - else + local t=type(v)=="table" + if k=="at" then + local t={} + for k,v in next,v do + t[k]=v + end + new[k]=t + elseif k=="dt" then + v.__p__=nil + v=copy(v,new) new[k]=v + v.__p__=p + else + new[k]=v end end local mt=getmetatable(old) @@ -11157,18 +12618,26 @@ local xmlfilehandler=newhandlers { function xml.save(root,name) serialize(root,xmlfilehandler,name) end -local result +local result,r,threshold={},0,512 local xmlstringhandler=newhandlers { name="string", initialize=function() - result={} + r=0 return result end, finalize=function() - return concat(result) + local done=concat(result,"",1,r) + r=0 + if r>threshold then + result={} + end + return done end, handle=function(...) - result[#result+1]=concat {... } + for i=1,select("#",...) do + r=r+1 + result[r]=select(i,...) + end end, } local function xmltostring(root) @@ -11320,7 +12789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 53892, stripped down to: 32508 +-- original size: 53301, stripped down to: 32477 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -11702,6 +13171,14 @@ local function apply_expression(list,expression,order) end return collected end +local function apply_selector(list,specification) + if xml.applyselector then + apply_selector=xml.applyselector + return apply_selector(list,specification) + else + return list + end +end local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb local spaces=S(" \n\r\t\f")^0 local lp_space=S(" \n\r\t\f") @@ -11825,6 +13302,9 @@ end local function register_nodes(nodetest,nodes) return { kind="nodes",nodetest=nodetest,nodes=nodes } end +local function register_selector(specification) + return { kind="selector",specification=specification } +end local function register_expression(expression) local converted=lpegmatch(converter,expression) local runner=load(format(template_e,converted)) @@ -11865,34 +13345,36 @@ local pathparser=Ct { "patterns", (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), - step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + step=((V("shortcuts")+V("selector")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, - shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor")+V("s_lastmatch"), shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), s_descendant=P("**")*Cc(register_descendant), - s_child=P("*")*no_nextcolon*Cc(register_child ), - s_parent=P("..")*Cc(register_parent ), - s_self=P("." )*Cc(register_self ), - s_root=P("^^")*Cc(register_root ), - s_ancestor=P("^")*Cc(register_ancestor ), - descendant=P("descendant::")*Cc(register_descendant ), - child=P("child::")*Cc(register_child ), - parent=P("parent::")*Cc(register_parent ), - self=P("self::")*Cc(register_self ), - root=P('root::')*Cc(register_root ), - ancestor=P('ancestor::')*Cc(register_ancestor ), - descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), - ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), - following=P('following::')*Cc(register_following ), - following_sibling=P('following-sibling::')*Cc(register_following_sibling ), - preceding=P('preceding::')*Cc(register_preceding ), - preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), - reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), - last_match=P('last-match::')*Cc(register_last_match ), + s_child=P("*")*no_nextcolon*Cc(register_child), + s_parent=P("..")*Cc(register_parent), + s_self=P("." )*Cc(register_self), + s_root=P("^^")*Cc(register_root), + s_ancestor=P("^")*Cc(register_ancestor), + s_lastmatch=P("=")*Cc(register_last_match), + descendant=P("descendant::")*Cc(register_descendant), + child=P("child::")*Cc(register_child), + parent=P("parent::")*Cc(register_parent), + self=P("self::")*Cc(register_self), + root=P('root::')*Cc(register_root), + ancestor=P('ancestor::')*Cc(register_ancestor), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self), + following=P('following::')*Cc(register_following), + following_sibling=P('following-sibling::')*Cc(register_following_sibling), + preceding=P('preceding::')*Cc(register_preceding), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling), + last_match=P('last-match::')*Cc(register_last_match), + selector=P("{")*C((1-P("}"))^1)*P("}")/register_selector, nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -12042,6 +13524,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) p.matched=p.matched+1 @@ -12083,6 +13567,9 @@ do elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) + report_lpath("% 10i : se : %s ",(collected and #collected) or 0,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") @@ -12114,6 +13601,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then return pi.finalizer(collected) end @@ -12170,6 +13659,13 @@ do function xml.lastmatch() return lastmatch end + local stack={} + function xml.pushmatch() + insert(stack,lastmatch) + end + function xml.popmatch() + lastmatch=remove(stack) + end end local applylpath=xml.applylpath function xml.filter(root,pattern) @@ -12449,7 +13945,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3787, stripped down to: 2003 +-- original size: 3684, stripped down to: 1957 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -12518,7 +14014,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 30566, stripped down to: 21741 +-- original size: 29835, stripped down to: 21174 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12534,7 +14030,6 @@ local xml=xml local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath -local xmlfilter=xml.filter local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip @@ -12738,7 +14233,17 @@ function xml.replace(root,pattern,whatever) report('replacing',pattern,c,e) end local d=p.dt - d[e.ni]=copiedelement(element,p) + local n=e.ni + local t=copiedelement(element,p) + if type(t)=="table" then + d[n]=t[1] + for i=2,#t do + n=n+1 + insert(d,n,t[i]) + end + else + d[n]=t + end redo_ni(d) end end @@ -13161,7 +14666,7 @@ local obsolete=xml.obsolete xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect xml.delete_element=xml.delete obsolete.delete_element=xml.delete -xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.replace_element=xml.replace obsolete.replace_element=xml.replace xml.each_element=xml.each obsolete.each_element=xml.each xml.process_elements=xml.process obsolete.process_elements=xml.process xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter @@ -13379,7 +14884,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10719, stripped down to: 7841 +-- original size: 10274, stripped down to: 7538 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -13757,7 +15262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6534, stripped down to: 5072 +-- original size: 6407, stripped down to: 4965 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13907,6 +15412,7 @@ function reporters.export(t,methods,filename) if filename then local fullname=file.replacesuffix(filename,method) t.report("saving export in %a",fullname) + dir.mkdirs(file.pathpart(fullname)) io.savedata(fullname,result) else reporters.lines(t,result) @@ -13927,7 +15433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11444, stripped down to: 7830 +-- original size: 11099, stripped down to: 7516 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -14072,11 +15578,6 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if type(profiler)=="table" and not jit then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) -end local prefixes=utilities.storage.allocate() resolvers.prefixes=prefixes local resolved={} @@ -14183,7 +15684,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18619, stripped down to: 11042 +-- original size: 17958, stripped down to: 10705 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -14199,7 +15700,6 @@ local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next local isdir=lfs.isdir -local ostype=os.type local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -14568,7 +16068,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9649, stripped down to: 7131 +-- original size: 9342, stripped down to: 6887 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -14852,7 +16352,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16066, stripped down to: 11938 +-- original size: 16088, stripped down to: 11435 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -15056,18 +16556,6 @@ end caches.getreadablepaths=getreadablepaths caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd=getreadablepaths(...) - for i=1,#rd do - local path=rd[i] - local fullname=file.join(path,filename) - if is_readable(fullname) then - usedreadables[i]=true - return fullname,path - end - end - return caches.setfirstwritablefile(filename,...) -end -function caches.getfirstreadablefile_TEST_ME_FIRST(filename,...) local fullname,path=caches.setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path @@ -15096,18 +16584,22 @@ end function caches.setluanames(path,name) return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end -function caches.loaddata(readables,name) +function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } end for i=1,#readables do local path=readables[i] - local tmaname,tmcname=caches.setluanames(path,name) local loader=false + local tmaname,tmcname=caches.setluanames(path,name) if isfile(tmcname) then loader=loadfile(tmcname) end if not loader and isfile(tmaname) then + local tmacrap,tmcname=caches.setluanames(writable,name) + if isfile(tmcname) then + loader=loadfile(tmcname) + end utilities.lua.compile(tmaname,tmcname) if isfile(tmcname) then loader=loadfile(tmcname) @@ -15228,7 +16720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5488, stripped down to: 4101 +-- original size: 5310, stripped down to: 3980 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -15240,7 +16732,6 @@ if not modules then modules={} end modules ['data-met']={ local find,format=string.find,string.format local sequenced=table.sequenced local addurlscheme,urlhashed=url.addscheme,url.hashed -local getcurrentdir=lfs.currentdir local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) @@ -15347,7 +16838,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67241, stripped down to: 46427 +-- original size: 67524, stripped down to: 46632 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15401,6 +16892,7 @@ resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf @@ -15444,7 +16936,6 @@ local function resolvevariable(k) end local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) @@ -15590,23 +17081,29 @@ local function identify_configuration_files() end reportcriticalvariables(cnfspec) local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname=resolvers.luacnfname - for i=1,#cnfpaths do - local filepath=cnfpaths[i] - local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolveprefix(filename) - if trace_locating then - local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) - report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) - end - if isfile(realname) then - specification[#specification+1]=filename + local function locatecnf(luacnfname,kind) + for i=1,#cnfpaths do + local filepath=cnfpaths[i] + local filename=collapsepath(filejoin(filepath,luacnfname)) + local realname=resolveprefix(filename) if trace_locating then - report_resolving("found configuration file %a",realname) + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) + report_resolving("looking for %s %a on %s path %a from specification %a", + kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath) + end + if isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found %s configuration file %a",kind,realname) + end end end end + locatecnf(resolvers.luacnfname,"regular") + if #specification==0 then + locatecnf(resolvers.luacnffallback,"fallback") + end if trace_locating then report_resolving() end @@ -16903,7 +18400,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4236, stripped down to: 3144 +-- original size: 4090, stripped down to: 3059 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -17025,7 +18522,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 935, stripped down to: 838 +-- original size: 910, stripped down to: 823 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -17055,7 +18552,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 548, stripped down to: 483 +-- original size: 530, stripped down to: 475 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -17078,7 +18575,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3976, stripped down to: 3391 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -17186,7 +18683,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5148, stripped down to: 3680 +-- original size: 5029, stripped down to: 3607 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -17256,7 +18753,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name) + stored=caches.loaddata(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -17305,7 +18802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4000, stripped down to: 3052 +-- original size: 4045, stripped down to: 3110 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -17350,7 +18847,7 @@ function resolvers.automount(usecache) end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) -function statistics.savefmtstatus(texname,formatbanner,sourcefile) +function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") @@ -17361,6 +18858,10 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) sourcefile=sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) + lua.registerfinalizer(function() + logs.report("format banner","%s",banner) + logs.newline() + end) end end function statistics.checkfmtstatus(texname) @@ -17396,7 +18897,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 9036, stripped down to: 7041 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -17633,7 +19134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8712, stripped down to: 5726 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17822,7 +19323,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6779, stripped down to: 5444 +-- original size: 6653, stripped down to: 5467 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17868,11 +19369,21 @@ function resolvers.schemes.cleanname(specification) end return hash end -local cached,loaded,reused,thresholds,handlers={},{},{},{},{} -local function runcurl(name,cachename) - local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name - os.execute(command) -end +local cached={} +local loaded={} +local reused={} +local thresholds={} +local handlers={} +local runner=sandbox.registerrunner { + name="curl resolver", + method="execute", + program="curl", + template="--silent -- insecure --create-dirs --output %cachename% %original%", + checkers={ + cachename="cache", + original="url", + } +} local function fetch(specification) local original=specification.original local scheme=specification.scheme @@ -17894,7 +19405,10 @@ local function fetch(specification) report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") end logs.flush() - runcurl(original,cachename) + runner { + original=original, + cachename=cachename, + } end end if io.exists(cachename) then @@ -18003,7 +19517,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4447, stripped down to: 3302 +-- original size: 4207, stripped down to: 3137 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -18045,8 +19559,6 @@ function helpers.cleanpath(path) return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib -local getextraluapaths=package.extraluapaths -local getextralibpaths=package.extralibpaths local registerpath=helpers.registerpath local lualibfile=helpers.lualibfile local luaformatpaths @@ -18112,7 +19624,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2494, stripped down to: 2047 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -18179,7 +19691,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2674, stripped down to: 1658 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -18235,7 +19747,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2815, stripped down to: 2415 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -18315,7 +19827,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11846, stripped down to: 6059 +-- original size: 13595, stripped down to: 7500 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -18324,35 +19836,51 @@ if not modules then modules={} end modules ['util-lib']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } -local gsub,find=string.gsub,string.find -local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join -local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles -local loaded=package.loaded -local report_swiglib=logs.reporter("swiglib") -local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local type=type +local next=next +local pcall=pcall +local gsub=string.gsub +local find=string.find +local sort=table.sort +local pathpart=file.pathpart +local nameonly=file.nameonly +local joinfile=file.join +local removesuffix=file.removesuffix +local findfile=resolvers.findfile +local findfiles=resolvers.findfiles +local expandpaths=resolvers.expandedpathlistfromvariable +local qualifiedpath=file.is_qualified_path +local isfile=lfs.isfile local done=false -local function requireswiglib(required,version) - local trace_swiglib=trace_swiglib or package.helpers.trace - local library=loaded[required] - if library==nil then - if trace_swiglib then - report_swiglib("requiring library %a with version %a",required,version or "any") - end - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) +local function locate(required,version,trace,report,action) + if type(required)~="string" then + report("provide a proper library name") + return + end + if trace then + report("requiring library %a with version %a",required,version or "any") + end + local found_library=nil + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + if qualifiedpath(required) then + if isfile(required) then + found_library=required + end + else local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false local engine=environment.ownmain or false - if trace_swiglib and not done then - local list=resolvers.expandedpathlistfromvariable("lib") + if trace and not done then + local list=expandpaths("lib") for i=1,#list do - report_swiglib("tds path %i: %s",i,list[i]) + report("tds path %i: %s",i,list[i]) end end local function found(locate,asked_library,how,...) - if trace_swiglib then - report_swiglib("checking %s: %a",how,asked_library) + if trace then + report("checking %s: %a",how,asked_library) end return locate(asked_library,...) end @@ -18360,45 +19888,45 @@ local function requireswiglib(required,version) local found=nil if version then local asked_library=joinfile(required_path,version,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end if not found or found=="" then local asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end return found and found~="" and found or false end local function attempt(checkpattern) - if trace_swiglib then - report_swiglib("checking tds lib paths strictly") + if trace then + report("checking tds lib paths strictly") end local found=findfile and check(findfile,"lib") if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace_swiglib then - report_swiglib("checking tds lib paths with wildcard") + if trace then + report("checking tds lib paths with wildcard") end local asked_library=joinfile(required_path,".*",required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","latest version",asked_library) + if trace then + report("checking %s: %a","latest version",asked_library) end local list=findfiles(asked_library,"lib",true) if list and #list>0 then - table.sort(list) + sort(list) local found=list[#list] if found and (not checkpattern or find(found,checkpattern)) then return found end end - if trace_swiglib then - report_swiglib("checking lib paths") + if trace then + report("checking lib paths") end package.extralibpath(environment.ownpath) local paths=package.libpaths() @@ -18410,89 +19938,143 @@ local function requireswiglib(required,version) end return false end - local found_library=nil if engine then - if trace_swiglib then - report_swiglib("attemp 1, engine %a",engine) + if trace then + report("attemp 1, engine %a",engine) end found_library=attempt("/"..engine.."/") if not found_library then - if trace_swiglib then - report_swiglib("attemp 2, no engine",asked_library) + if trace then + report("attemp 2, no engine",asked_library) end found_library=attempt() end else found_library=attempt() end - if not found_library then - if trace_swiglib then - report_swiglib("not found: %a",required) - end + end + if not found_library then + if trace then + report("not found: %a",required) + end + library=false + else + if trace then + report("found: %a",found_library) + end + local message,result=action(found_library,required_base) + if result then + library=result + else library=false + report("load error: message %a, library %a",tostring(message),found_library or "no library") + end + end + if not library then + report("unknown: %a",required) + elseif trace then + report("stored: %a",required) + end + return library +end +do + local report_swiglib=logs.reporter("swiglib") + local trace_swiglib=false + local savedrequire=require + local loadedlibs={} + local loadlib=package.loadlib + local pushdir=dir.push + local popdir=dir.pop + trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) + function requireswiglib(required,version) + local library=loadedlibs[library] + if library==nil then + local trace_swiglib=trace_swiglib or package.helpers.trace + library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) + pushdir(pathpart(name)) + local opener="luaopen_"..base + if trace_swiglib then + report_swiglib("opening: %a with %a",name,opener) + end + local library,message=loadlib(name,opener) + local libtype=type(library) + if libtype=="function" then + library=library() + message=true + else + report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") + library=false + end + popdir() + return message,library + end) + loadedlibs[required]=library or false + end + return library + end + function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) else - local path=pathpart(found_library) - local base=nameonly(found_library) - dir.push(path) + return savedrequire(name) + end + end + local swiglibs={} + local initializer="core" + function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) if trace_swiglib then - report_swiglib("found: %a",found_library) - end - local message=nil - local opener="luaopen_"..required_base - library,message=package.loadlib(found_library,opener) - local libtype=type(library) - if libtype=="function" then - library=library() + report_swiglib("loading %a",name) + end + if not find(name,"%."..initializer.."$") then + fullname="swiglib."..name.."."..initializer else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + fullname="swiglib."..name end - dir.pop() + library=requireswiglib(fullname,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) end - if not library then - report_swiglib("unknown: %a",required) - elseif trace_swiglib then - report_swiglib("stored: %a",required) - end - loaded[required]=library - else - report_swiglib("reused: %a",required) + return library end - return library + statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end + end) end -local savedrequire=require -function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) +if FFISUPPORTED and ffi and ffi.load then + local report_ffilib=logs.reporter("ffilib") + local trace_ffilib=false + local savedffiload=ffi.load + trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) + local function locateindeed(name) + local message,library=pcall(savedffiload,removesuffix(name)) + if type(library)=="userdata" then + return library + else + return false + end end -end -local swiglibs={} -local initializer="core" -function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) + function ffilib(required,version) + if version=="system" then + return locateindeed(name) + else + return locate(required,version,trace_ffilib,report_ffilib,locateindeed) end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer + end + function ffi.load(name) + local library=ffilib(name) + if type(library)=="userdata" then + return library else - fullname="swiglib."..name + report_ffilib("trying to load %a using normal loader",name) + return savedffiload(name) end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end -statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end -end) end -- of closure @@ -18501,7 +20083,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5914, stripped down to: 2584 +-- original size: 5703, stripped down to: 2507 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -18604,7 +20186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 6967, stripped down to: 5631 +-- original size: 9144, stripped down to: 7291 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -18618,23 +20200,67 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local trackers=environment.argument("trackers") - local directives=environment.argument("directives") +local function primaryflags() + local arguments=environment.arguments + local flags={} + if arguments.silent then + flags[#flags+1]="--interaction=batchmode" + end + if arguments.jit then + flags[#flags+1]="--jiton" + end + return concat(flags," ") +end +local function secondaryflags() + local arguments=environment.arguments + local trackers=arguments.trackers + local directives=arguments.directives local flags={} if trackers and trackers~="" then - flags={ "--trackers="..quoted(trackers) } + flags[#flags+1]="--c:trackers="..quoted(trackers) end if directives and directives~="" then - flags={ "--directives="..quoted(directives) } + flags[#flags+1]="--c:directives="..quoted(directives) + end + if arguments.silent then + flags[#flags+1]="--c:silent" end - if environment.argument("jit") then - flags={ "--jiton" } + if arguments.jit then + flags[#flags+1]="--c:jiton" + end + if arguments.ansi then + flags[#flags+1]="--c:ansi" end return concat(flags," ") end -function environment.make_format(name,silent) +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] +local checkers={ + primaryflags="string", + secondaryflags="string", + luafile="readable", + texfile="readable", + redirect="string", + dump="string", +} +local runners={ + luatex=sandbox.registerrunner { + name="make luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="make luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} +function environment.make_format(name,arguments) local engine=environment.ownmain or "luatex" + local silent=environment.arguments.silent local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" if path~="" then @@ -18690,11 +20316,20 @@ function environment.make_format(name,silent) lfs.chdir(olddir) return end - local dump=os.platform=="unix" and "\\\\dump" or "\\dump" - if silent then + local specification={ + primaryflags=primaryflags(), + secondaryflags=secondaryflags(), + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), + dump=os.platform=="unix" and "\\\\dump" or "\\dump", + } + local runner=runners[engine] + if not runner then + report_format("format %a cannot be generated, no runner available for engine %a",name,engine) + elseif silent then statistics.starttiming() - local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - local result=os.execute(command) + specification.redirect="> temp.log" + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -18703,9 +20338,7 @@ function environment.make_format(name,silent) end os.remove("temp.log") else - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - report_format("running command: %s\n",command) - os.execute(command) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -18718,6 +20351,30 @@ function environment.make_format(name,silent) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -18739,9 +20396,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -18750,10 +20416,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 797557 --- stripped bytes : 289197 +-- original bytes : 842443 +-- stripped bytes : 306317 -- end library merge @@ -18777,6 +20443,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -18810,8 +20477,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) @@ -18983,7 +20651,7 @@ local helpinfo = [[ <metadata> <entry name="name">mtxrun</entry> <entry name="detail">ConTeXt TDS Runner Tool</entry> - <entry name="version">1.31</entry> + <entry name="version">1.32</entry> </metadata> <flags> <category name="basic"> @@ -19046,6 +20714,7 @@ local helpinfo = [[ </subcategory> <subcategory> <flag name="expand-braces"><short>expand complex variable</short></flag> + <flag name="resolve-path"><short>expand variable (completely resolve paths)</short></flag> <flag name="expand-path"><short>expand variable (resolve paths)</short></flag> <flag name="expand-var"><short>expand variable (resolve references)</short></flag> <flag name="show-path"><short>show path expansion of ...</short></flag> @@ -19063,7 +20732,7 @@ local helpinfo = [[ local application = logs.application { name = "mtxrun", - banner = "ConTeXt TDS Runner Tool 1.31", + banner = "ConTeXt TDS Runner Tool 1.32", helpinfo = helpinfo, } @@ -20029,6 +21698,13 @@ elseif e_argument("expand-path") then environment.initializearguments(environment.arguments_after) resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) +elseif e_argument("resolve-path") then + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.cleanedpathlist, environment.files) + elseif e_argument("expand-var") or e_argument("expand-variable") then -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) diff --git a/scripts/context/stubs/unix/pstopdf b/scripts/context/stubs/unix/pstopdf new file mode 100644 index 000000000..116f5f4a3 --- /dev/null +++ b/scripts/context/stubs/unix/pstopdf @@ -0,0 +1,2 @@ +#!/bin/sh +mtxrun --script pstopdf "$@" diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua index 7b711a88d..873770cac 100644 --- a/scripts/context/stubs/win64/mtxrun.lua +++ b/scripts/context/stubs/win64/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 4734, stripped down to: 2626 +-- original size: 5125, stripped down to: 2881 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -162,6 +162,283 @@ if flush then local spawn=os.spawn if spawn then function os.spawn (...) flush() return spawn (...) end end local popen=io.popen if popen then function io.popen (...) flush() return popen (...) end end end +FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +if not FFISUPPORTED then + local okay;okay,ffi=pcall(require,"ffi") + FFISUPPORTED=type(ffi)=="table" and ffi.os~="" and ffi.arch~="" and ffi.load +end +if not FFISUPPORTED then + ffi=nil +elseif not ffi.number then + ffi.number=tonumber +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["l-sandbox"] = package.loaded["l-sandbox"] or true + +-- original size: 9667, stripped down to: 6678 + +if not modules then modules={} end modules ['l-sandbox']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local global=_G +local next=next +local unpack=unpack or table.unpack +local type=type +local tprint=texio.write_nl or print +local tostring=tostring +local format=string.format +local concat=table.concat +local sort=table.sort +local gmatch=string.gmatch +local gsub=string.gsub +local requiem=require +sandbox={} +local sandboxed=false +local overloads={} +local skiploads={} +local initializers={} +local finalizers={} +local originals={} +local comments={} +local trace=false +local logger=false +local blocked={} +local function report(...) + tprint("sandbox ! "..format(...)) +end +sandbox.report=report +function sandbox.setreporter(r) + report=r + sandbox.report=r +end +function sandbox.settrace(v) + trace=v +end +function sandbox.setlogger(l) + logger=type(l)=="function" and l or false +end +local function register(func,overload,comment) + if type(func)=="function" then + if type(overload)=="string" then + comment=overload + overload=nil + end + local function f(...) + if sandboxed then + local overload=overloads[f] + if overload then + if logger then + local result={ overload(func,...) } + logger { + comment=comments[f] or tostring(f), + arguments={... }, + result=result[1] and true or false, + } + return unpack(result) + else + return overload(func,...) + end + else + end + else + return func(...) + end + end + if comment then + comments[f]=comment + if trace then + report("registering function: %s",comment) + end + end + overloads[f]=overload or false + originals[f]=func + return f + end +end +local function redefine(func,comment) + if type(func)=="function" then + skiploads[func]=comment or comments[func] or "unknown" + if overloads[func]==false then + overloads[func]=nil + end + end +end +sandbox.register=register +sandbox.redefine=redefine +function sandbox.original(func) + return originals and originals[func] or func +end +function sandbox.overload(func,overload,comment) + comment=comment or comments[func] or "?" + if type(func)~="function" then + if trace then + report("overloading unknown function: %s",comment) + end + elseif type(overload)~="function" then + if trace then + report("overloading function with bad overload: %s",comment) + end + elseif overloads[func]==nil then + if trace then + report("function is not registered: %s",comment) + end + elseif skiploads[func] then + if trace then + report("function is not skipped: %s",comment) + end + else + if trace then + report("overloading function: %s",comment) + end + overloads[func]=overload + end + return func +end +local function whatever(specification,what,target) + if type(specification)~="table" then + report("%s needs a specification",what) + elseif type(specification.category)~="string" or type(specification.action)~="function" then + report("%s needs a category and action",what) + elseif not sandboxed then + target[#target+1]=specification + elseif trace then + report("already enabled, discarding %s",what) + end +end +function sandbox.initializer(specification) + whatever(specification,"initializer",initializers) +end +function sandbox.finalizer(specification) + whatever(specification,"finalizer",finalizers) +end +function require(name) + local n=gsub(name,"^.*[\\/]","") + local n=gsub(n,"[%.].*$","") + local b=blocked[n] + if b==false then + return nil + elseif b then + if trace then + report("using blocked: %s",n) + end + return b + else + if trace then + report("requiring: %s",name) + end + return requiem(name) + end +end +function blockrequire(name,lib) + if trace then + report("preventing reload of: %s",name) + end + blocked[name]=lib or _G[name] or false +end +function sandbox.enable() + if not sandboxed then + for i=1,#initializers do + initializers[i].action() + end + for i=1,#finalizers do + finalizers[i].action() + end + local nnot=0 + local nyes=0 + local cnot={} + local cyes={} + local skip={} + for k,v in next,overloads do + local c=comments[k] + if v then + if c then + cyes[#cyes+1]=c + else + nyes=nyes+1 + end + else + if c then + cnot[#cnot+1]=c + else + nnot=nnot+1 + end + end + end + for k,v in next,skiploads do + skip[#skip+1]=v + end + if #cyes>0 then + sort(cyes) + report("overloaded known: %s",concat(cyes," | ")) + end + if nyes>0 then + report("overloaded unknown: %s",nyes) + end + if #cnot>0 then + sort(cnot) + report("not overloaded known: %s",concat(cnot," | ")) + end + if nnot>0 then + report("not overloaded unknown: %s",nnot) + end + if #skip>0 then + sort(skip) + report("not overloaded redefined: %s",concat(skip," | ")) + end + initializers=nil + finalizers=nil + originals=nil + sandboxed=true + end +end +blockrequire("lfs",lfs) +blockrequire("io",io) +blockrequire("os",os) +blockrequire("ffi",ffi) +local function supported(library) + local l=_G[library] + return l +end +loadfile=register(loadfile,"loadfile") +if supported("io") then + io.open=register(io.open,"io.open") + io.popen=register(io.popen,"io.popen") + io.lines=register(io.lines,"io.lines") + io.output=register(io.output,"io.output") + io.input=register(io.input,"io.input") +end +if supported("os") then + os.execute=register(os.execute,"os.execute") + os.spawn=register(os.spawn,"os.spawn") + os.exec=register(os.exec,"os.exec") + os.rename=register(os.rename,"os.rename") + os.remove=register(os.remove,"os.remove") +end +if supported("lfs") then + lfs.chdir=register(lfs.chdir,"lfs.chdir") + lfs.mkdir=register(lfs.mkdir,"lfs.mkdir") + lfs.rmdir=register(lfs.rmdir,"lfs.rmdir") + lfs.isfile=register(lfs.isfile,"lfs.isfile") + lfs.isdir=register(lfs.isdir,"lfs.isdir") + lfs.attributes=register(lfs.attributes,"lfs.attributes") + lfs.dir=register(lfs.dir,"lfs.dir") + lfs.lock_dir=register(lfs.lock_dir,"lfs.lock_dir") + lfs.touch=register(lfs.touch,"lfs.touch") + lfs.link=register(lfs.link,"lfs.link") + lfs.setmode=register(lfs.setmode,"lfs.setmode") + lfs.readlink=register(lfs.readlink,"lfs.readlink") + lfs.shortname=register(lfs.shortname,"lfs.shortname") + lfs.symlinkattributes=register(lfs.symlinkattributes,"lfs.symlinkattributes") +end end -- of closure @@ -170,7 +447,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10949, stripped down to: 8037 +-- original size: 10587, stripped down to: 7815 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -460,7 +737,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 38185, stripped down to: 20990 +-- original size: 37748, stripped down to: 20111 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -552,6 +829,7 @@ patterns.nonwhitespace=nonwhitespace local stripper=spacer^0*C((spacer^0*nonspacer^1)^0) local fullstripper=whitespace^0*C((whitespace^0*nonwhitespace^1)^0) local collapser=Cs(spacer^0/""*nonspacer^0*((spacer^0/" "*nonspacer^1)^0)) +local nospacer=Cs((whitespace^1/""+nonwhitespace^1)^0) local b_collapser=Cs(whitespace^0/""*(nonwhitespace^1+whitespace^1/" ")^0) local e_collapser=Cs((whitespace^1*P(-1)/""+nonwhitespace^1+whitespace^1/" ")^0) local m_collapser=Cs((nonwhitespace^1+whitespace^1/" ")^0) @@ -561,6 +839,7 @@ local m_stripper=Cs((nonspacer^1+spacer^1/" ")^0) patterns.stripper=stripper patterns.fullstripper=fullstripper patterns.collapser=collapser +patterns.nospacer=nospacer patterns.b_collapser=b_collapser patterns.m_collapser=m_collapser patterns.e_collapser=e_collapser @@ -1027,27 +1306,7 @@ function lpeg.append(list,pp,delayed,checked) end local p_false=P(false) local p_true=P(true) -local function make(t) - local function making(t) - local p=p_false - local keys=sortedkeys(t) - for i=1,#keys do - local k=keys[i] - if k~="" then - local v=t[k] - if v==true then - p=p+P(k)*p_true - elseif v==false then - else - p=p+P(k)*making(v) - end - end - end - if t[""] then - p=p+p_true - end - return p - end +local function make(t,rest) local p=p_false local keys=sortedkeys(t) for i=1,#keys do @@ -1058,10 +1317,13 @@ local function make(t) p=p+P(k)*p_true elseif v==false then else - p=p+P(k)*making(v) + p=p+P(k)*make(v,v[""]) end end end + if rest then + p=p+p_true + end return p end local function collapse(t,x) @@ -1264,7 +1526,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 372, stripped down to: 329 +-- original size: 361, stripped down to: 322 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1283,7 +1545,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5983, stripped down to: 2959 +-- original size: 6419, stripped down to: 3339 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1321,22 +1583,26 @@ end local stripper=patterns.stripper local fullstripper=patterns.fullstripper local collapser=patterns.collapser +local nospacer=patterns.nospacer local longtostring=patterns.longtostring function string.strip(str) - return lpegmatch(stripper,str) or "" + return str and lpegmatch(stripper,str) or "" end function string.fullstrip(str) - return lpegmatch(fullstripper,str) or "" + return str and lpegmatch(fullstripper,str) or "" end function string.collapsespaces(str) - return lpegmatch(collapser,str) or "" + return str and lpegmatch(collapser,str) or "" +end +function string.nospaces(str) + return str and lpegmatch(nospacer,str) or "" end function string.longtostring(str) - return lpegmatch(longtostring,str) or "" + return str and lpegmatch(longtostring,str) or "" end local pattern=P(" ")^0*P(-1) function string.is_empty(str) - if str=="" then + if not str or str=="" then return true else return lpegmatch(pattern,str) and true or false @@ -1381,6 +1647,21 @@ function string.tformat(fmt,...) end string.quote=string.quoted string.unquote=string.unquoted +if not string.bytetable then + local limit=5000 + function string.bytetable(str) + local n=#str + if n>limit then + local t={ byte(str,1,limit) } + for i=limit+1,n do + t[i]=byte(str,i) + end + return t + else + return { byte(str,1,n) } + end + end +end end -- of closure @@ -1389,7 +1670,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 36997, stripped down to: 22376 +-- original size: 39608, stripped down to: 23165 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -1716,19 +1997,23 @@ function table.fromhash(t) end return hsh end -local noquotes,hexify,handle,compact,inline,functions +local noquotes,hexify,handle,compact,inline,functions,metacheck local reserved=table.tohash { 'and','break','do','else','elseif','end','false','for','function','if', 'in','local','nil','not','or','repeat','return','then','true','until','while', 'NaN','goto', } -local function simple_table(t) +local function is_simple_table(t,hexify) local nt=#t if nt>0 then local n=0 for _,v in next,t do n=n+1 + if type(v)=="table" then + return nil + end end + local haszero=rawget(t,0) if n==nt then local tt={} for i=1,nt do @@ -1738,10 +2023,10 @@ local function simple_table(t) if hexify then tt[i]=format("0x%X",v) else - tt[i]=tostring(v) + tt[i]=v end elseif tv=="string" then - tt[i]=format("%q",v) + tt[i]=format("%q",v) elseif tv=="boolean" then tt[i]=v and "true" or "false" else @@ -1749,10 +2034,32 @@ local function simple_table(t) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + if hexify then + tt[i+1]=format("0x%X",v) + else + tt[i+1]=v + end + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil end +table.is_simple_table=is_simple_table local propername=patterns.propername local function dummy() end local function do_serialize(root,name,depth,level,indexed) @@ -1786,7 +2093,7 @@ local function do_serialize(root,name,depth,level,indexed) if compact then last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -1814,7 +2121,7 @@ local function do_serialize(root,name,depth,level,indexed) if next(v)==nil then handle(format("%s {},",depth)) elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then handle(format("%s { %s },",depth,concat(st,", "))) else @@ -1851,6 +2158,7 @@ local function do_serialize(root,name,depth,level,indexed) else handle(format("%s [%s]=%s,",depth,k and "true" or "false",v)) end + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then if hexify then handle(format("%s %s=0x%X,",depth,k,v)) @@ -1873,6 +2181,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",v)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,v)) else @@ -1888,13 +2197,14 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={},",depth,k and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={},",depth,k)) else handle(format("%s [%q]={},",depth,k)) end elseif inline then - local st=simple_table(v) + local st=is_simple_table(v,hexify) if st then if tk=="number" then if hexify then @@ -1904,6 +2214,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]={ %s },",depth,k and "true" or "false",concat(st,", "))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s={ %s },",depth,k,concat(st,", "))) else @@ -1924,6 +2235,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%s,",depth,tostring(k),v and "true" or "false")) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%s,",depth,k,v and "true" or "false")) else @@ -1940,6 +2252,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=load(%q),",depth,k and "true" or "false",f)) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=load(%q),",depth,k,f)) else @@ -1955,6 +2268,7 @@ local function do_serialize(root,name,depth,level,indexed) end elseif tk=="boolean" then handle(format("%s [%s]=%q,",depth,k and "true" or "false",tostring(v))) + elseif tk~="string" then elseif noquotes and not reserved[k] and lpegmatch(propername,k) then handle(format("%s %s=%q,",depth,k,tostring(v))) else @@ -1976,6 +2290,7 @@ local function serialize(_handle,root,name,specification) functions=specification.functions compact=specification.compact inline=specification.inline and compact + metacheck=specification.metacheck if functions==nil then functions=true end @@ -1985,6 +2300,9 @@ local function serialize(_handle,root,name,specification) if inline==nil then inline=compact end + if metacheck==nil then + metacheck=true + end else noquotes=false hexify=false @@ -1992,6 +2310,7 @@ local function serialize(_handle,root,name,specification) compact=true inline=true functions=true + metacheck=true end if tname=="string" then if name=="return" then @@ -2015,7 +2334,7 @@ local function serialize(_handle,root,name,specification) handle("t={") end if root then - if getmetatable(root) then + if metacheck and getmetatable(root) then local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end @@ -2091,6 +2410,38 @@ local function flattened(t,f,depth) return f end table.flattened=flattened +local function collapsed(t,f,h) + if f==nil then + f={} + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsed(v,f,h) + elseif not h[v] then + f[#f+1]=v + h[v]=true + end + end + return f +end +local function collapsedhash(t,h) + if h==nil then + h={} + end + for k=1,#t do + local v=t[k] + if type(v)=="table" then + collapsedhash(v,h) + else + h[v]=true + end + end + return h +end +table.collapsed=collapsed +table.collapsedhash=collapsedhash local function unnest(t,f) if not f then f={} @@ -2197,6 +2548,12 @@ function table.swapped(t,s) end return n end +function table.hashed(t) + for i=1,#t do + t[t[i]]=i + end + return t +end function table.mirrored(t) local n={} for k,v in next,t do @@ -2365,7 +2722,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 9001, stripped down to: 6512 +-- original size: 11790, stripped down to: 6961 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2375,6 +2732,7 @@ if not modules then modules={} end modules ['l-io']={ license="see context related readme files" } local io=io +local open,flush,write,read=io.open,io.flush,io.write,io.read local byte,find,gsub,format=string.byte,string.find,string.gsub,string.format local concat=table.concat local floor=math.floor @@ -2384,50 +2742,56 @@ if string.find(os.getenv("PATH"),";",1,true) then else io.fileseparator,io.pathseparator="/",":" end -local function readall(f) - return f:read("*all") -end +local large=2^24 +local medium=large/16 +local small=medium/8 local function readall(f) local size=f:seek("end") - if size==0 then - return "" - elseif size<1024*1024 then + if size>0 then f:seek("set",0) - return f:read('*all') - else - local done=f:seek("set",0) - local step - if size<1024*1024 then - step=1024*1024 - elseif size>16*1024*1024 then - step=16*1024*1024 - else - step=floor(size/(1024*1024))*1024*1024/8 - end - local data={} - while true do - local r=f:read(step) - if not r then - return concat(data) - else - data[#data+1]=r - end - end + return f:read(size) + else + return "" end end io.readall=readall function io.loaddata(filename,textmode) - local f=io.open(filename,(textmode and 'r') or 'rb') + local f=open(filename,(textmode and 'r') or 'rb') if f then - local data=readall(f) + local size=f:seek("end") + local data=nil + if size>0 then + f:seek("set",0) + data=f:read(size) + end f:close() - if #data>0 then - return data + return data + end +end +function io.copydata(source,target,action) + local f=open(source,"rb") + if f then + local g=open(target,"wb") + if g then + local size=f:seek("end") + if size>0 then + f:seek("set",0) + local data=f:read(size) + if action then + data=action(data) + end + if data then + g:write(data) + end + end + g:close() end + f:close() + flush() end end function io.savedata(filename,data,joiner) - local f=io.open(filename,"wb") + local f=open(filename,"wb") if f then if type(data)=="table" then f:write(concat(data,joiner or "")) @@ -2437,40 +2801,70 @@ function io.savedata(filename,data,joiner) f:write(data or "") end f:close() - io.flush() + flush() return true else return false end end -function io.loadlines(filename,n) - local f=io.open(filename,'r') - if not f then - elseif n then - local lines={} - for i=1,n do - local line=f:read("*lines") - if line then - lines[#lines+1]=line - else - break +if fio and fio.readline then + local readline=fio.readline + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=readline(f) + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=readline(f) + f:close() + if line and #line>0 then + return line end end - f:close() - lines=concat(lines,"\n") - if #lines>0 then - return lines - end - else - local line=f:read("*line") or "" - f:close() - if #line>0 then - return line + end +else + function io.loadlines(filename,n) + local f=open(filename,'r') + if not f then + elseif n then + local lines={} + for i=1,n do + local line=f:read("*lines") + if line then + lines[i]=line + else + break + end + end + f:close() + lines=concat(lines,"\n") + if #lines>0 then + return lines + end + else + local line=f:read("*line") or "" + f:close() + if #line>0 then + return line + end end end end function io.loadchunk(filename,n) - local f=io.open(filename,'rb') + local f=open(filename,'rb') if f then local data=f:read(n or 1024) f:close() @@ -2480,7 +2874,7 @@ function io.loadchunk(filename,n) end end function io.exists(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return false else @@ -2489,7 +2883,7 @@ function io.exists(filename) end end function io.size(filename) - local f=io.open(filename) + local f=open(filename) if f==nil then return 0 else @@ -2498,11 +2892,11 @@ function io.size(filename) return s end end -function io.noflines(f) +local function noflines(f) if type(f)=="string" then - local f=io.open(filename) + local f=open(filename) if f then - local n=f and io.noflines(f) or 0 + local n=f and noflines(f) or 0 f:close() return n else @@ -2517,6 +2911,7 @@ function io.noflines(f) return n end end +io.noflines=noflines local nextchar={ [ 4]=function(f) return f:read(1,1,1,1) @@ -2594,16 +2989,16 @@ function io.bytes(f,n) end function io.ask(question,default,options) while true do - io.write(question) + write(question) if options then - io.write(format(" [%s]",concat(options,"|"))) + write(format(" [%s]",concat(options,"|"))) end if default then - io.write(format(" [%s]",default)) + write(format(" [%s]",default)) end - io.write(format(" ")) - io.flush() - local answer=io.read() + write(format(" ")) + flush() + local answer=read() answer=gsub(answer,"^%s*(.*)%s*$","%1") if answer=="" and default then return default @@ -2625,7 +3020,7 @@ function io.ask(question,default,options) end end end -local function readnumber(f,n,m) +local function readnumber(f,n,m) if m then f:seek("set",n) n=m @@ -2634,31 +3029,31 @@ local function readnumber(f,n,m) return byte(f:read(1)) elseif n==2 then local a,b=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==3 then local a,b,c=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==4 then local a,b,c,d=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==8 then local a,b=readnumber(f,4),readnumber(f,4) - return 256*a+b + return 0x100*a+b elseif n==12 then local a,b,c=readnumber(f,4),readnumber(f,4),readnumber(f,4) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-2 then local b,a=byte(f:read(2),1,2) - return 256*a+b + return 0x100*a+b elseif n==-3 then local c,b,a=byte(f:read(3),1,3) - return 256*256*a+256*b+c + return 0x10000*a+0x100*b+c elseif n==-4 then local d,c,b,a=byte(f:read(4),1,4) - return 256*256*256*a+256*256*b+256*c+d + return 0x1000000*a+0x10000*b+0x100*c+d elseif n==-8 then local h,g,f,e,d,c,b,a=byte(f:read(8),1,8) - return 256*256*256*256*256*256*256*a+256*256*256*256*256*256*b+256*256*256*256*256*c+256*256*256*256*d+256*256*256*e+256*256*f+256*g+h + return 0x100000000000000*a+0x1000000000000*b+0x10000000000*c+0x100000000*d+0x1000000*e+0x10000*f+0x100*g+h else return 0 end @@ -2680,7 +3075,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 5146, stripped down to: 2933 +-- original size: 5358, stripped down to: 3177 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2693,6 +3088,7 @@ local tostring,tonumber=tostring,tonumber local format,floor,match,rep=string.format,math.floor,string.match,string.rep local concat,insert=table.concat,table.insert local lpegmatch=lpeg.match +local floor=math.floor number=number or {} local number=number if bit32 then @@ -2817,6 +3213,26 @@ end function number.bits(n) return { bits(n,1) } end +function number.bytetodecimal(b) + local d=floor(b*100/255+0.5) + if d>100 then + return 100 + elseif d<-100 then + return -100 + else + return d + end +end +function number.decimaltobyte(d) + local b=floor(d*255/100+0.5) + if b>255 then + return 255 + elseif b<-255 then + return -255 + else + return b + end +end end -- of closure @@ -2825,7 +3241,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 2010, stripped down to: 1186 +-- original size: 1923, stripped down to: 1133 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2898,7 +3314,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 16390, stripped down to: 9734 +-- original size: 16268, stripped down to: 9246 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -2974,7 +3390,7 @@ if not os.__getenv__ then end local execute=os.execute local iopopen=io.popen -function os.resultof(command) +local function resultof(command) local handle=iopopen(command,"r") if handle then local result=handle:read("*all") or "" @@ -2984,9 +3400,13 @@ function os.resultof(command) return "" end end +os.resultof=resultof +function os.pipeto(command) + return iopopen(command,"w") +end if not io.fileseparator then if find(os.getenv("PATH"),";",1,true) then - io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "mswin" + io.fileseparator,io.pathseparator,os.type="\\",";",os.type or "windows" else io.fileseparator,io.pathseparator,os.type="/",":",os.type or "unix" end @@ -3029,17 +3449,6 @@ setmetatable(os,{ __index=function(t,k) return r and r(t,k) or nil end }) local name,platform=os.name or "linux",os.getenv("MTX_PLATFORM") or "" -local function guess() - local architecture=os.resultof("uname -m") or "" - if architecture~="" then - return architecture - end - architecture=os.getenv("HOSTTYPE") or "" - if architecture~="" then - return architecture - end - return os.resultof("echo $HOSTTYPE") or "" -end if platform~="" then os.platform=platform elseif os.type=="windows" then @@ -3056,7 +3465,7 @@ elseif os.type=="windows" then end elseif name=="linux" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="linux-64" elseif find(architecture,"ppc",1,true) then @@ -3070,7 +3479,7 @@ elseif name=="linux" then end elseif name=="macosx" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("echo $HOSTTYPE") or "" + local platform,architecture="",resultof("echo $HOSTTYPE") or "" if architecture=="" then platform="osx-intel" elseif find(architecture,"i386",1,true) then @@ -3086,7 +3495,7 @@ elseif name=="macosx" then end elseif name=="sunos" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"sparc",1,true) then platform="solaris-sparc" else @@ -3098,7 +3507,7 @@ elseif name=="sunos" then end elseif name=="freebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.resultof("uname -m") or "" + local platform,architecture="",resultof("uname -m") or "" if find(architecture,"amd64",1,true) then platform="freebsd-amd64" else @@ -3110,7 +3519,7 @@ elseif name=="freebsd" then end elseif name=="kfreebsd" then function resolvers.platform(t,k) - local platform,architecture="",os.getenv("HOSTTYPE") or os.resultof("uname -m") or "" + local platform,architecture="",os.getenv("HOSTTYPE") or resultof("uname -m") or "" if find(architecture,"x86_64",1,true) then platform="kfreebsd-amd64" else @@ -3241,7 +3650,7 @@ if not os.sleep then end end local function isleapyear(year) - return (year%400==0) or ((year%100~=0) and (year%4==0)) + return (year%4==0) and (year%100~=0 or year%400==0) end os.isleapyear=isleapyear local days={ 31,28,31,30,31,30,31,31,30,31,30,31 } @@ -3280,7 +3689,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 21648, stripped down to: 10238 +-- original size: 20997, stripped down to: 9986 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3617,13 +4026,15 @@ function file.robustname(str,strict) end end end -file.readdata=io.loaddata -file.savedata=io.savedata +local loaddata=io.loaddata +local savedata=io.savedata +file.readdata=loaddata +file.savedata=savedata function file.copy(oldname,newname) if oldname and newname then - local data=io.loaddata(oldname) + local data=loaddata(oldname) if data and data~="" then - file.savedata(newname,data) + savedata(newname,data) end end end @@ -3660,7 +4071,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1265, stripped down to: 1038 +-- original size: 1211, stripped down to: 1002 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3714,7 +4125,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3355, stripped down to: 2321 +-- original size: 3309, stripped down to: 2314 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3744,6 +4155,8 @@ do if not md5.HEX then function md5.HEX(str) if str then return lpegmatch(bytestoHEX,md5sum(str)) end end end if not md5.hex then function md5.hex(str) if str then return lpegmatch(bytestohex,md5sum(str)) end end end if not md5.dec then function md5.dec(str) if str then return lpegmatch(bytestodec,md5sum(str)) end end end + md5.sumhexa=md5.hex + md5.sumHEXA=md5.HEX end end function file.needsupdating(oldname,newname,threshold) @@ -3802,7 +4215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12897, stripped down to: 5882 +-- original size: 12531, stripped down to: 5721 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4019,7 +4432,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 17358, stripped down to: 11378 +-- original size: 17703, stripped down to: 11691 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4283,6 +4696,31 @@ local function globfiles(path,recurse,func,files) return files end dir.globfiles=globfiles +local function globdirs(path,recurse,func,files) + if type(func)=="string" then + local s=func + func=function(name) return find(name,s) end + end + files=files or {} + local noffiles=#files + for name in walkdir(path) do + if find(name,"^%.") then + else + local mode=attributes(name,'mode') + if mode=="directory" then + if not func or func(name) then + noffiles=noffiles+1 + files[noffiles]=path.."/"..name + if recurse then + globdirs(path.."/"..name,recurse,func,files) + end + end + end + end + end + return files +end +dir.globdirs=globdirs function dir.ls(pattern) return concat(glob(pattern),"\n") end @@ -4447,9 +4885,13 @@ end file.expandname=dir.expandname local stack={} function dir.push(newdir) - insert(stack,currentdir()) + local curdir=currentdir() + insert(stack,curdir) if newdir and newdir~="" then chdir(newdir) + return newdir + else + return curdir end end function dir.pop() @@ -4484,7 +4926,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1919, stripped down to: 1621 +-- original size: 1850, stripped down to: 1568 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4556,7 +4998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 38699, stripped down to: 16321 +-- original size: 38263, stripped down to: 16330 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -5167,6 +5609,23 @@ function utf.chrlen(u) (u<0xFC and 5) or (u<0xFE and 6) or 0 end +local extract=bit32.extract +local char=string.char +function unicode.toutf32string(n) + if n<=0xFF then + return + char(n).."\000\000\000" + elseif n<=0xFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8)).."\000\000" + elseif n<=0xFFFFFF then + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8)).."\000" + else + return + char(extract(n,0,8))..char(extract(n,8,8))..char(extract(n,16,8))..char(extract(n,24,8)) + end +end end -- of closure @@ -5175,7 +5634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 1012, stripped down to: 912 +-- original size: 974, stripped down to: 890 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5215,7 +5674,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 36053, stripped down to: 19685 +-- original size: 36148, stripped down to: 20179 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5227,7 +5686,7 @@ if not modules then modules={} end modules ['util-str']={ utilities=utilities or {} utilities.strings=utilities.strings or {} local strings=utilities.strings -local format,gsub,rep,sub=string.format,string.gsub,string.rep,string.sub +local format,gsub,rep,sub,find=string.format,string.gsub,string.rep,string.sub,string.find local load,dump=load,string.dump local tonumber,type,tostring=tonumber,type,tostring local unpack,concat=table.unpack,table.concat @@ -5410,6 +5869,25 @@ function number.signed(i) return "-",-i end end +local digit=patterns.digit +local period=patterns.period +local three=digit*digit*digit +local splitter=Cs ( + (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) +) +patterns.formattednumber=splitter +function number.formatted(n,sep1,sep2) + local s=type(s)=="string" and n or format("%0.2f",n) + if sep1==true then + return lpegmatch(splitter,s,1,".",",") + elseif sep1=="." then + return lpegmatch(splitter,s,1,sep1,sep2 or ",") + elseif sep1=="," then + return lpegmatch(splitter,s,1,sep1,sep2 or ".") + else + return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") + end +end local zero=P("0")^1/"" local plus=P("+")/"" local minus=P("-") @@ -5435,6 +5913,27 @@ function number.sparseexponent(f,n) end return tostring(n) end +local hf={} +local hs={} +setmetatable(hf,{ __index=function(t,k) + local v="%."..k.."f" + t[k]=v + return v +end } ) +setmetatable(hs,{ __index=function(t,k) + local v="%"..k.."s" + t[k]=v + return v +end } ) +function number.formattedfloat(n,b,a) + local s=format(hf[a],n) + local l=(b or 0)+(a or 0)+1 + if #s<l then + return format(hs[l],s) + else + return s + end +end local template=[[ %s %s @@ -5462,6 +5961,7 @@ local autodouble=string.autodouble local sequenced=table.sequenced local formattednumber=number.formatted local sparseexponent=number.sparseexponent +local formattedfloat=number.formattedfloat ]] else environment={ @@ -5485,6 +5985,7 @@ else sequenced=table.sequenced, formattednumber=number.formatted, sparseexponent=number.sparseexponent, + formattedfloat=number.formattedfloat } end local arguments={ "a1" } @@ -5495,6 +5996,7 @@ setmetatable(arguments,{ __index=function(t,k) end }) local prefix_any=C((S("+- .")+R("09"))^0) +local prefix_sub=(C((S("+-")+R("09"))^0)+Cc(0))*P(".")*(C((S("+-")+R("09"))^0)+Cc(0)) local prefix_tab=P("{")*C((1-P("}"))^0)*P("}")+C((1-R("az","AZ","09","%%"))^0) local format_s=function(f) n=n+1 @@ -5545,6 +6047,10 @@ local format_F=function(f) return format("format((a%s %% 1 == 0) and '%%i' or '%%%sf',a%s)",n,f,n) end end +local format_k=function(b,a) + n=n+1 + return format("formattedfloat(a%s,%i,%i)",n,b or 0,a or 0) +end local format_g=function(f) n=n+1 return format("format('%%%sg',a%s)",f,n) @@ -5693,25 +6199,6 @@ end local format_W=function(f) return format("nspaces[%s]",tonumber(f) or 0) end -local digit=patterns.digit -local period=patterns.period -local three=digit*digit*digit -local splitter=Cs ( - (((1-(three^1*period))^1+C(three))*(Carg(1)*three)^1+C((1-period)^1))*(P(1)/""*Carg(2))*C(2) -) -patterns.formattednumber=splitter -function number.formatted(n,sep1,sep2) - local s=type(s)=="string" and n or format("%0.2f",n) - if sep1==true then - return lpegmatch(splitter,s,1,".",",") - elseif sep1=="." then - return lpegmatch(splitter,s,1,sep1,sep2 or ",") - elseif sep1=="," then - return lpegmatch(splitter,s,1,sep1,sep2 or ".") - else - return lpegmatch(splitter,s,1,sep1 or ",",sep2 or ".") - end -end local format_m=function(f) n=n+1 if not f or f=="" then @@ -5736,9 +6223,16 @@ end local format_extension=function(extensions,f,name) local extension=extensions[name] or "tostring(%s)" local f=tonumber(f) or 1 + local w=find(extension,"%.%.%.") if f==0 then + if w then + extension=gsub(extension,"%.%.%.","") + end return extension elseif f==1 then + if w then + extension=gsub(extension,"%.%.%.","%%s") + end n=n+1 local a="a"..n return format(extension,a,a) @@ -5746,10 +6240,13 @@ local format_extension=function(extensions,f,name) local a="a"..(n+f+1) return format(extension,a,a) else + if w then + extension=gsub(extension,"%.%.%.",rep("%%s,",f-1).."%%s") + end local t={} for i=1,f do n=n+1 - t[#t+1]="a"..n + t[i]="a"..n end return format(extension,unpack(t)) end @@ -5762,7 +6259,8 @@ local builder=Cs { "start", +V("s")+V("q")+V("i")+V("d")+V("f")+V("F")+V("g")+V("G")+V("e")+V("E")+V("x")+V("X")+V("o") +V("c")+V("C")+V("S") +V("Q") -+V("N") ++V("N") ++V("k") +V("r")+V("h")+V("H")+V("u")+V("U")+V("p")+V("b")+V("t")+V("T")+V("l")+V("L")+V("I")+V("w") +V("W") +V("a") @@ -5789,6 +6287,7 @@ local builder=Cs { "start", ["S"]=(prefix_any*P("S"))/format_S, ["Q"]=(prefix_any*P("Q"))/format_S, ["N"]=(prefix_any*P("N"))/format_N, + ["k"]=(prefix_sub*P("k"))/format_k, ["c"]=(prefix_any*P("c"))/format_c, ["C"]=(prefix_any*P("C"))/format_C, ["r"]=(prefix_any*P("r"))/format_r, @@ -5909,7 +6408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 28680, stripped down to: 18636 +-- original size: 27407, stripped down to: 17116 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -5923,7 +6422,7 @@ utilities.tables=utilities.tables or {} local tables=utilities.tables local format,gmatch,gsub,sub=string.format,string.gmatch,string.gsub,string.sub local concat,insert,remove,sort=table.concat,table.insert,table.remove,table.sort -local setmetatable,getmetatable,tonumber,tostring=setmetatable,getmetatable,tonumber,tostring +local setmetatable,getmetatable,tonumber,tostring,rawget=setmetatable,getmetatable,tonumber,tostring,rawget local type,next,rawset,tonumber,tostring,load,select=type,next,rawset,tonumber,tostring,load,select local lpegmatch,P,Cs,Cc=lpeg.match,lpeg.P,lpeg.Cs,lpeg.Cc local sortedkeys,sortedpairs=table.sortedkeys,table.sortedpairs @@ -6063,7 +6562,7 @@ function table.tocsv(t,specification) r[f]=tostring(field) end end - result[#result+1]=concat(r,separator) + result[i+1]=concat(r,separator) end return concat(result,"\n") else @@ -6295,11 +6794,12 @@ function table.autokey(t,k) return v end local selfmapper={ __index=function(t,k) t[k]=k return k end } -function table.twowaymapper(t) - if not t then - t={} - else - for i=0,#t do +function table.twowaymapper(t) + if not t then + t={} + else + local zero=rawget(t,0) + for i=zero and 0 or 1,#t do local ti=t[i] if ti then local i=tostring(i) @@ -6307,7 +6807,6 @@ function table.twowaymapper(t) t[ti]=i end end - t[""]=t[0] or "" end setmetatable(t,selfmapper) return t @@ -6346,6 +6845,7 @@ local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") local original_serialize=table.serialize +local is_simple_table=table.is_simple_table local function serialize(root,name,specification) if type(specification)=="table" then return original_serialize(root,name,specification) @@ -6353,54 +6853,6 @@ local function serialize(root,name,specification) local t local n=1 local unknown=false - local function simple_table(t) - local nt=#t - if nt>0 then - local n=0 - for _,v in next,t do - n=n+1 - if type(v)=="table" then - return nil - end - end - local haszero=t[0] - if n==nt then - local tt={} - for i=1,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i]=v - elseif tv=="string" then - tt[i]=format("%q",v) - elseif tv=="boolean" then - tt[i]=v and "true" or "false" - else - return nil - end - end - return tt - elseif haszero and (n==nt+1) then - local tt={} - for i=0,nt do - local v=t[i] - local tv=type(v) - if tv=="number" then - tt[i+1]=v - elseif tv=="string" then - tt[i+1]=format("%q",v) - elseif tv=="boolean" then - tt[i+1]=v and "true" or "false" - else - return nil - end - end - tt[1]="[0] = "..tt[1] - return tt - end - end - return nil - end local function do_serialize(root,name,depth,level,indexed) if level>0 then n=n+1 @@ -6425,7 +6877,7 @@ local function serialize(root,name,specification) local last=0 last=#root for k=1,last do - if root[k]==nil then + if rawget(root,k)==nil then last=k-1 break end @@ -6448,7 +6900,7 @@ local function serialize(root,name,specification) if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else - local st=simple_table(v) + local st=is_simple_table(v) if st then n=n+1 t[n]=f_val_seq(depth,st) else @@ -6492,7 +6944,7 @@ local function serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else - local st=simple_table(v) + local st=is_simple_table(v) if not st then do_serialize(v,k,depth,level+1) elseif tk=="number" then @@ -6552,11 +7004,11 @@ local function serialize(root,name,specification) end if root then if getmetatable(root) then - local dummy=root._w_h_a_t_e_v_e_r_ + local dummy=root._w_h_a_t_e_v_e_r_ root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - local st=simple_table(root) + local st=is_simple_table(root) if st then return t[1]..f_fin_seq(st) else @@ -6570,7 +7022,12 @@ local function serialize(root,name,specification) end table.serialize=serialize if setinspector then - setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) + setinspector("table",function(v) + if type(v)=="table" then + print(serialize(v,"table",{ metacheck=false })) + return true + end + end) end @@ -6580,7 +7037,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fil"] = package.loaded["util-fil"] or true --- original size: 3577, stripped down to: 2870 +-- original size: 7567, stripped down to: 5575 if not modules then modules={} end modules ['util-fil']={ version=1.001, @@ -6590,7 +7047,9 @@ if not modules then modules={} end modules ['util-fil']={ license="see context related readme files" } local byte=string.byte -local extract=bit32.extract +local char=string.char +local extract=bit32 and bit32.extract +local floor=math.floor utilities=utilities or {} local files={} utilities.files=files @@ -6609,6 +7068,7 @@ end function files.size(f) return f:seek("end") end +files.getsize=files.size function files.setposition(f,n) if zerobased[f] then f:seek("set",n) @@ -6646,6 +7106,10 @@ end function files.readbytes(f,n) return byte(f:read(n),1,n) end +function files.readbytetable(f,n) + local s=f:read(n or 1) + return { byte(s,1,#s) } +end function files.readchar(f) return f:read(1) end @@ -6655,7 +7119,7 @@ end function files.readinteger1(f) local n=byte(f:read(1)) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6663,55 +7127,107 @@ end files.readcardinal1=files.readbyte files.readcardinal=files.readcardinal1 files.readinteger=files.readinteger1 +files.readsignedbyte=files.readinteger1 function files.readcardinal2(f) local a,b=byte(f:read(2),1,2) return 0x100*a+b end +function files.readcardinal2le(f) + local b,a=byte(f:read(2),1,2) + return 0x100*a+b +end function files.readinteger2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function files.readinteger2le(f) + local b,a=byte(f:read(2),1,2) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function files.readcardinal3(f) local a,b,c=byte(f:read(3),1,3) return 0x10000*a+0x100*b+c end +function files.readcardinal3le(f) + local c,b,a=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readinteger3(f) + local a,b,c=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function files.readinteger3le(f) + local c,b,a=byte(f:read(3),1,3) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function files.readcardinal4(f) local a,b,c,d=byte(f:read(4),1,4) return 0x1000000*a+0x10000*b+0x100*c+d end +function files.readcardinal4le(f) + local d,c,b,a=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end function files.readinteger4(f) local a,b,c,d=byte(f:read(4),1,4) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.readfixed4(f) - local a,b,c,d=byte(f:read(4),1,4) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF +function files.readinteger4le(f) + local d,c,b,a=byte(f:read(4),1,4) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n+(0x100*c+d)/0xFFFF + return 0x1000000*a+0x10000*b+0x100*c+d end end -function files.read2dot14(f) +function files.readfixed2(f) local a,b=byte(f:read(2),1,2) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - n=extract(n,30,2) - return n+m/0x4000 + return (0x100*a+b )+(0x100*c+d)/0x10000 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function files.skipshort(f,n) @@ -6720,6 +7236,55 @@ end function files.skiplong(f,n) f:read(4*(n or 1)) end +function files.writecardinal2(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + f:write(b,a) +end +function files.writecardinal4(f,n) + local a=char(n%256) + n=floor(n/256) + local b=char(n%256) + n=floor(n/256) + local c=char(n%256) + n=floor(n/256) + local d=char(n%256) + f:write(d,c,b,a) +end +function files.writestring(f,s) + f:write(char(byte(s,1,#s))) +end +function files.writebyte(f,b) + f:write(char(b)) +end +if fio and fio.readcardinal1 then + files.readcardinal1=fio.readcardinal1 + files.readcardinal2=fio.readcardinal2 + files.readcardinal3=fio.readcardinal3 + files.readcardinal4=fio.readcardinal4 + files.readinteger1=fio.readinteger1 + files.readinteger2=fio.readinteger2 + files.readinteger3=fio.readinteger3 + files.readinteger4=fio.readinteger4 + files.read2dot14=fio.read2dot14 + files.setposition=fio.setposition + files.getposition=fio.getposition + files.readbyte=files.readcardinal1 + files.readsignedbyte=files.readinteger1 + files.readcardinal=files.readcardinal1 + files.readinteger=files.readinteger1 + local skipposition=fio.skipposition + files.skipposition=skipposition + files.readbytes=fio.readbytes + files.readbytetable=fio.readbytetable + function files.skipshort(f,n) + skipposition(f,2*(n or 1)) + end + function files.skiplong(f,n) + skipposition(f,4*(n or 1)) + end +end end -- of closure @@ -6728,7 +7293,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sac"] = package.loaded["util-sac"] or true --- original size: 4264, stripped down to: 3349 +-- original size: 8716, stripped down to: 6754 if not modules then modules={} end modules ['util-sac']={ version=1.001, @@ -6738,7 +7303,7 @@ if not modules then modules={} end modules ['util-sac']={ license="see context related readme files" } local byte,sub=string.byte,string.sub -local extract=bit32.extract +local extract=bit32 and bit32.extract utilities=utilities or {} local streams={} utilities.streams=streams @@ -6796,6 +7361,12 @@ function streams.readbytes(f,n) f[2]=j return byte(f[1],i,j-1) end +function streams.readbytetable(f,n) + local i=f[2] + local j=i+n + f[2]=j + return { byte(f[1],i,j-1) } +end function streams.skipbytes(f,n) f[2]=f[2]+n end @@ -6815,7 +7386,7 @@ function streams.readinteger1(f) f[2]=i+1 local n=byte(f[1],i) if n>=0x80 then - return n-0xFF-1 + return n-0x100 else return n end @@ -6830,16 +7401,33 @@ function streams.readcardinal2(f) local a,b=byte(f[1],i,j) return 0x100*a+b end +function streams.readcardinal2LE(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + return 0x100*a+b +end function streams.readinteger2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1 + if a>=0x80 then + return 0x100*a+b-0x10000 else - return n + return 0x100*a+b + end +end +function streams.readinteger2le(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x100*a+b-0x10000 + else + return 0x100*a+b end end function streams.readcardinal3(f) @@ -6849,6 +7437,35 @@ function streams.readcardinal3(f) local a,b,c=byte(f[1],i,j) return 0x10000*a+0x100*b+c end +function streams.readcardinal3le(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readinteger3(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end +function streams.readinteger3le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x10000*a+0x100*b+c-0x1000000 + else + return 0x10000*a+0x100*b+c + end +end function streams.readcardinal4(f) local i=f[2] local j=i+3 @@ -6861,11 +7478,21 @@ function streams.readinteger4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x1000000*a+0x10000*b+0x100*c+d - if n>=0x8000000 then - return n-0xFFFFFFFF-1 + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 else - return n + return 0x1000000*a+0x10000*b+0x100*c+d + end +end +function streams.readinteger4le(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local d,c,b,a=byte(f[1],i,j) + if a>=0x80 then + return 0x1000000*a+0x10000*b+0x100*c+d-0x100000000 + else + return 0x1000000*a+0x10000*b+0x100*c+d end end function streams.readfixed4(f) @@ -6873,26 +7500,38 @@ function streams.readfixed4(f) local j=i+3 f[2]=j+1 local a,b,c,d=byte(f[1],i,j) - local n=0x100*a+b - if n>=0x8000 then - return n-0xFFFF-1+(0x100*c+d)/0xFFFF + if a>=0x80 then + return (0x100*a+b-0x10000)+(0x100*c+d)/0x10000 else - return n+(0x100*c+d)/0xFFFF + return (0x100*a+b )+(0x100*c+d)/0x10000 end end -function streams.read2dot14(f) +function streams.readfixed2(f) local i=f[2] local j=i+1 f[2]=j+1 local a,b=byte(f[1],i,j) - local n=0x100*a+b - local m=extract(n,0,30) - if n>0x7FFF then - n=extract(n,30,2) - return m/0x4000-4 - else - n=extract(n,30,2) - return n+m/0x4000 + if a>=0x80 then + return (a-0x100)+b/0x100 + else + return (a )+b/0x100 + end +end +if extract then + local extract=bit32.extract + local band=bit32.band + function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + if a>=0x80 then + local n=-(0x100*a+b) + return-(extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + else + local n=0x100*a+b + return (extract(n,14,2)+(band(n,0x3FFF)/16384.0)) + end end end function streams.skipshort(f,n) @@ -6901,6 +7540,92 @@ end function streams.skiplong(f,n) f[2]=f[2]+4*(n or 1) end +if sio and sio.readcardinal2 then + local readcardinal1=sio.readcardinal1 + local readcardinal2=sio.readcardinal2 + local readcardinal3=sio.readcardinal3 + local readcardinal4=sio.readcardinal4 + local readinteger1=sio.readinteger1 + local readinteger2=sio.readinteger2 + local readinteger3=sio.readinteger3 + local readinteger4=sio.readinteger4 + local readfixed2=sio.readfixed2 + local readfixed4=sio.readfixed4 + local read2dot14=sio.read2dot14 + local readbytes=sio.readbytes + local readbytetable=sio.readbytetable + function streams.readcardinal1(f) + local i=f[2] + f[2]=i+1 + return readcardinal1(f[1],i) + end + function streams.readcardinal2(f) + local i=f[2] + f[2]=i+2 + return readcardinal2(f[1],i) + end + function streams.readcardinal3(f) + local i=f[2] + f[2]=i+3 + return readcardinal3(f[1],i) + end + function streams.readcardinal4(f) + local i=f[2] + f[2]=i+4 + return readcardinal4(f[1],i) + end + function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + return readinteger1(f[1],i) + end + function streams.readinteger2(f) + local i=f[2] + f[2]=i+2 + return readinteger2(f[1],i) + end + function streams.readinteger3(f) + local i=f[2] + f[2]=i+3 + return readinteger3(f[1],i) + end + function streams.readinteger4(f) + local i=f[2] + f[2]=i+4 + return readinteger4(f[1],i) + end + function streams.read2dot4(f) + local i=f[2] + f[2]=i+2 + return read2dot4(f[1],i) + end + function streams.readbytes(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytes(f[1],i,n) + end + function streams.readbytetable(f,n) + local i=f[2] + local s=f[3] + local p=i+n + if p>s then + f[2]=s+1 + else + f[2]=p + end + return readbytetable(f[1],i,n) + end + streams.readbyte=streams.readcardinal1 + streams.readsignedbyte=streams.readinteger1 + streams.readcardinal=streams.readcardinal1 + streams.readinteger=streams.readinteger1 +end end -- of closure @@ -6909,7 +7634,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4100, stripped down to: 2852 +-- original size: 3926, stripped down to: 2742 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -7049,7 +7774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 23411, stripped down to: 16177 +-- original size: 22883, stripped down to: 16045 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -7211,6 +7936,21 @@ function parsers.settings_to_array(str,strict) return { str } end end +function parsers.settings_to_numbers(str) + if not str or str=="" then + return {} + end + if type(str)=="table" then + elseif find(str,",",1,true) then + str=lpegmatch(pattern,str) + else + return { tonumber(str) } + end + for i=1,#str do + str[i]=tonumber(str[i]) + end + return str +end local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) local pattern=spaces*Ct(value*(separator*value)^0) function parsers.settings_to_array_obey_fences(str) @@ -7587,7 +8327,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2350, stripped down to: 1847 +-- original size: 2274, stripped down to: 1781 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7668,7 +8408,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12862, stripped down to: 9104 +-- original size: 12454, stripped down to: 8840 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7854,7 +8594,6 @@ function setters.list(t) return user,system end function setters.show(t) - local category=t.name local list=setters.list(t) t.report() for k=1,#list do @@ -7981,7 +8720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 30767, stripped down to: 21355 +-- original size: 30007, stripped down to: 20818 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -8027,6 +8766,14 @@ if tex and (tex.jobname or tex.formatname) then if texio.setescape then texio.setescape(0) end + if arg then + for k,v in next,arg do + if v=="--ansi" or v=="--c:ansi" then + variant="ansi" + break + end + end + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -8612,7 +9359,6 @@ function logs.stop_page_number() end logs.flush() end -local report_files=logs.reporter("files") local nesting=0 local verbose=false local hasscheme=url.hasscheme @@ -8774,7 +9520,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6917, stripped down to: 5484 +-- original size: 8036, stripped down to: 5567 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8806,11 +9552,13 @@ end local function resettiming(instance) timers[instance or "notimer"]={ timing=0,loadtime=0 } end +local ticks=clock +local seconds=function(n) return n or 0 end local function starttiming(instance) local timer=timers[instance or "notimer"] local it=timer.timing or 0 if it==0 then - timer.starttime=clock() + timer.starttime=ticks() if not timer.loadtime then timer.loadtime=0 end @@ -8824,12 +9572,13 @@ local function stoptiming(instance) timer.timing=it-1 else local starttime=timer.starttime - if starttime then - local stoptime=clock() + if starttime and starttime>0 then + local stoptime=ticks() local loadtime=stoptime-starttime timer.stoptime=stoptime timer.loadtime=timer.loadtime+loadtime timer.timing=0 + timer.starttime=0 return loadtime end end @@ -8840,7 +9589,7 @@ local function elapsed(instance) return instance or 0 else local timer=timers[instance or "notimer"] - return timer and timer.loadtime or 0 + return timer and seconds(timer.loadtime) or 0 end end local function elapsedtime(instance) @@ -8888,10 +9637,13 @@ function statistics.show() local total,indirect=status.callbacks or 0,status.indirect_callbacks or 0 return format("%s direct, %s indirect, %s total",total-indirect,indirect,total) end) - if jit then - local jitstatus={ jit.status() } - if jitstatus[1] then - register("luajit options",concat(jitstatus," ",2)) + if TEXENGINE=="luajittex" and JITSUPPORTED then + local jitstatus=jit.status + if jitstatus then + local jitstatus={ jitstatus() } + if jitstatus[1] then + register("luajit options",concat(jitstatus," ",2)) + end end end register("lua properties",function() @@ -8955,7 +9707,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 6039, stripped down to: 3616 +-- original size: 5829, stripped down to: 3501 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -9102,7 +9854,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 5142, stripped down to: 3611 +-- original size: 5396, stripped down to: 3708 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -9224,6 +9976,17 @@ function luautilities.loadstripped(...) return load(dump(l,true)) end end +local finalizers={} +setmetatable(finalizers,{ + __gc=function(t) + for i=1,#t do + pcall(t[i]) + end + end +} ) +function luautilities.registerfinalizer(f) + finalizers[#finalizers+1]=f +end end -- of closure @@ -9232,7 +9995,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 4030, stripped down to: 2718 +-- original size: 8911, stripped down to: 6504 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -9242,75 +10005,230 @@ if not modules then modules={} end modules ['util-deb']={ license="see context related readme files" } local debug=require "debug" -local getinfo=debug.getinfo -local type,next,tostring=type,next,tostring -local format,find=string.format,string.find -local is_boolean=string.is_boolean +local getinfo,sethook=debug.getinfo,debug.sethook +local type,next,tostring,tonumber=type,next,tostring,tonumber +local format,find,sub,gsub=string.format,string.find,string.sub,string.gsub +local insert,remove,sort=table.insert,table.remove,table.sort +local setmetatableindex=table.setmetatableindex utilities=utilities or {} local debugger=utilities.debugger or {} utilities.debugger=debugger -local counters={} -local names={} local report=logs.reporter("debugger") -local function hook() - local f=getinfo(2) - if f then - local n="unknown" - if f.what=="C" then - n=f.name or '<anonymous>' - if not names[n] then - names[n]=format("%42s",n) +local ticks=os.gettimeofday or os.clock +local seconds=function(n) return n or 0 end +local overhead=0 +local dummycalls=10*1000 +local nesting=0 +local names={} +local initialize=false +if not (FFISUPPORTED and ffi) then +elseif os.type=="windows" then + initialize=function() + local kernel=ffilib("kernel32","system") + if kernel then + local tonumber=ffi.number or tonumber + ffi.cdef[[ + int QueryPerformanceFrequency(int64_t *lpFrequency); + int QueryPerformanceCounter(int64_t *lpPerformanceCount); + ]] + local target=ffi.new("__int64[1]") + ticks=function() + if kernel.QueryPerformanceCounter(target)==1 then + return tonumber(target[0]) + else + return 0 + end end - else - n=f.name or f.namewhat or f.what - if not n or n=="" then - n="?" + local target=ffi.new("__int64[1]") + seconds=function(ticks) + if kernel.QueryPerformanceFrequency(target)==1 then + return ticks/tonumber(target[0]) + else + return 0 + end + end + end + initialize=false + end +elseif os.type=="unix" then + initialize=function() + local C=ffi.C + local tonumber=ffi.number or tonumber + ffi.cdef [[ + /* what a mess */ + typedef int clk_id_t; + typedef enum { CLOCK_REALTIME, CLOCK_MONOTONIC, CLOCK_PROCESS_CPUTIME_ID } clk_id; + typedef struct timespec { long sec; long nsec; } ctx_timespec; + int clock_gettime(clk_id_t timerid, struct timespec *t); + ]] + local target=ffi.new("ctx_timespec[?]",1) + local clock=C.CLOCK_PROCESS_CPUTIME_ID + ticks=function () + C.clock_gettime(clock,target) + return tonumber(target[0].sec*1000000000+target[0].nsec) + end + seconds=function(ticks) + return ticks/1000000000 + end + initialize=false + end +end +setmetatableindex(names,function(t,name) + local v=setmetatableindex(function(t,source) + local v=setmetatableindex(function(t,line) + local v={ total=0,count=0 } + t[line]=v + return v + end) + t[source]=v + return v + end) + t[name]=v + return v +end) +local function hook(where) + local f=getinfo(2,"nSl") + if f then + local source=f.short_src + if not source then + return + end + local line=f.linedefined or 0 + local name=f.name + if not name then + local what=f.what + if what=="C" then + name="<anonymous>" + else + name=f.namewhat or what or "<unknown>" end - if not names[n] then - names[n]=format("%42s : % 5i : %s",n,f.linedefined or 0,f.short_src or "unknown source") + end + local data=names[name][source][line] + if where=="call" then + data.count=data.count+1 + insert(data,ticks()) + elseif where=="return" then + local t=remove(data) + if t then + data.total=data.total+ticks()-t end end - counters[n]=(counters[n] or 0)+1 end end -function debugger.showstats(printer,threshold) - printer=printer or report - threshold=threshold or 0 - local total,grandtotal,functions=0,0,0 +function debugger.showstats(printer,threshold) + local printer=printer or report + local calls=0 + local functions=0 local dataset={} - for name,count in next,counters do - dataset[#dataset+1]={ name,count } + local length=0 + local wholetime=0 + local threshold=threshold or 0 + for name,sources in next,names do + for source,lines in next,sources do + for line,data in next,lines do + local count=data.count + if count>threshold then + if #name>length then + length=#name + end + local total=data.total + local real=total + if real>0 then + real=total-(count*overhead/dummycalls) + if real<0 then + real=0 + end + wholetime=wholetime+real + end + if line<0 then + line=0 + end + dataset[#dataset+1]={ real,total,count,name,source,line } + end + end + end end - table.sort(dataset,function(a,b) return a[2]==b[2] and b[1]>a[1] or a[2]>b[2] end) + sort(dataset,function(a,b) + if a[1]==b[1] then + if a[2]==b[2] then + if a[3]==b[3] then + if a[4]==b[4] then + if a[5]==b[5] then + return a[6]<b[6] + else + return a[5]<b[5] + end + else + return a[4]<b[4] + end + else + return b[3]<a[3] + end + else + return b[2]<a[2] + end + else + return b[1]<a[1] + end + end) + if length>50 then + length=50 + end + local fmt=string.formatters["%4.9k %4.9k %3.3k %8i %-"..length.."s %4i %s"] for i=1,#dataset do - local d=dataset[i] - local name=d[1] - local count=d[2] - if count>threshold and not find(name,"for generator") then - printer(format("%8i %s\n",count,names[name])) - total=total+count - end - grandtotal=grandtotal+count + local data=dataset[i] + local real=data[1] + local total=data[2] + local count=data[3] + local name=data[4] + local source=data[5] + local line=data[6] + local percent=real/wholetime + calls=calls+count functions=functions+1 + name=gsub(name,"%s+"," ") + if #name>length then + name=sub(name,1,length) + end + printer(fmt(seconds(total),seconds(real),percent,count,name,line,source)) end - printer("\n") - printer(format("functions : % 10i\n",functions)) - printer(format("total : % 10i\n",total)) - printer(format("grand total: % 10i\n",grandtotal)) - printer(format("threshold : % 10i\n",threshold)) + printer("") + printer(format("functions : %i",functions)) + printer(format("calls : %i",calls)) + printer(format("overhead : %f",seconds(overhead/1000))) end function debugger.savestats(filename,threshold) local f=io.open(filename,'w') if f then - debugger.showstats(function(str) f:write(str) end,threshold) + debugger.showstats(function(str) f:write(str,"\n") end,threshold) f:close() end end function debugger.enable() - debug.sethook(hook,"c") + if nesting==0 then + running=true + if initialize then + initialize() + end + sethook(hook,"cr") + local function dummy() end + local t=ticks() + for i=1,dummycalls do + dummy() + end + overhead=ticks()-t + end + if nesting>0 then + nesting=nesting+1 + end end function debugger.disable() - debug.sethook() + if nesting>0 then + nesting=nesting-1 + end + if nesting==0 then + sethook() + end end local function showtraceback(rep) local level=2 @@ -9334,9 +10252,661 @@ end -- of closure do -- create closure to overcome 200 locals limit +package.loaded["util-tpl"] = package.loaded["util-tpl"] or true + +-- original size: 7100, stripped down to: 3978 + +if not modules then modules={} end modules ['util-tpl']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +utilities.templates=utilities.templates or {} +local templates=utilities.templates +local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) +local report_template=logs.reporter("template") +local tostring=tostring +local format,sub,byte=string.format,string.sub,string.byte +local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns +local replacer +local function replacekey(k,t,how,recursive) + local v=t[k] + if not v then + if trace_template then + report_template("unknown key %a",k) + end + return "" + else + v=tostring(v) + if trace_template then + report_template("setting key %a to value %a",k,v) + end + if recursive then + return lpegmatch(replacer,v,1,t,how,recursive) + else + return v + end + end +end +local sqlescape=lpeg.replacer { + { "'","''" }, + { "\\","\\\\" }, + { "\r\n","\\n" }, + { "\r","\\n" }, +} +local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) +lpegpatterns.sqlescape=sqlescape +lpegpatterns.sqlquoted=sqlquoted +local luaescape=lpegpatterns.luaescape +local escapers={ + lua=function(s) + return lpegmatch(luaescape,s) + end, + sql=function(s) + return lpegmatch(sqlescape,s) + end, +} +local quotedescapers={ + lua=function(s) + return format("%q",s) + end, + sql=function(s) + return lpegmatch(sqlquoted,s) + end, +} +local luaescaper=escapers.lua +local quotedluaescaper=quotedescapers.lua +local function replacekeyunquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and escapers[how] or luaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replacekeyquoted(s,t,how,recurse) + if how==false then + return replacekey(s,t,how,recurse) + else + local escaper=how and quotedescapers[how] or quotedluaescaper + return escaper(replacekey(s,t,how,recurse)) + end +end +local function replaceoptional(l,m,r,t,how,recurse) + local v=t[l] + return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" +end +local single=P("%") +local double=P("%%") +local lquoted=P("%[") +local rquoted=P("]%") +local lquotedq=P("%(") +local rquotedq=P(")%") +local escape=double/'%%' +local nosingle=single/'' +local nodouble=double/'' +local nolquoted=lquoted/'' +local norquoted=rquoted/'' +local nolquotedq=lquotedq/'' +local norquotedq=rquotedq/'' +local noloptional=P("%?")/'' +local noroptional=P("?%")/'' +local nomoptional=P(":")/'' +local args=Carg(1)*Carg(2)*Carg(3) +local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle +local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq +local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted +local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional +local any=P(1) + replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) +local function replace(str,mapping,how,recurse) + if mapping and str then + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + else + return str + end +end +templates.replace=replace +function templates.replacer(str,how,recurse) + return function(mapping) + return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str + end +end +function templates.load(filename,mapping,how,recurse) + local data=io.loaddata(filename) or "" + if mapping and next(mapping) then + return replace(data,mapping,how,recurse) + else + return data + end +end +function templates.resolve(t,mapping,how,recurse) + if not mapping then + mapping=t + end + for k,v in next,t do + t[k]=replace(v,mapping,how,recurse) + end + return t +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sbx"] = package.loaded["util-sbx"] or true + +-- original size: 20309, stripped down to: 13848 + +if not modules then modules={} end modules ['util-sbx']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +if not sandbox then require("l-sandbox") end +local next,type=next,type +local replace=utilities.templates.replace +local collapsepath=file.collapsepath +local expandname=dir.expandname +local sortedhash=table.sortedhash +local lpegmatch=lpeg.match +local platform=os.type +local P,S,C=lpeg.P,lpeg.S,lpeg.C +local gsub=string.gsub +local lower=string.lower +local find=string.find +local concat=string.concat +local unquoted=string.unquoted +local optionalquoted=string.optionalquoted +local basename=file.basename +local sandbox=sandbox +local validroots={} +local validrunners={} +local validbinaries=true +local validlibraries=true +local validators={} +local finalized=nil +local trace=false +local p_validroot=nil +local p_split=lpeg.firstofsplit(" ") +local report=logs.reporter("sandbox") +trackers.register("sandbox",function(v) trace=v end) +sandbox.setreporter(report) +sandbox.finalizer { + category="files", + action=function() + finalized=true + end +} +local function registerroot(root,what) + if finalized then + report("roots are already finalized") + else + if type(root)=="table" then + root,what=root[1],root[2] + end + if type(root)=="string" and root~="" then + root=collapsepath(expandname(root)) + if what=="r" or what=="ro" or what=="readable" then + what="read" + elseif what=="w" or what=="wo" or what=="writable" then + what="write" + end + validroots[root]=what=="write" or false + end + end +end +sandbox.finalizer { + category="files", + action=function() + if p_validroot then + report("roots are already initialized") + else + sandbox.registerroot(".","write") + for name in sortedhash(validroots) do + if p_validroot then + p_validroot=P(name)+p_validroot + else + p_validroot=P(name) + end + end + p_validroot=p_validroot/validroots + end + end +} +local function registerbinary(name) + if finalized then + report("binaries are already finalized") + elseif type(name)=="string" and name~="" then + if not validbinaries then + return + end + if validbinaries==true then + validbinaries={ [name]=true } + else + validbinaries[name]=true + end + elseif name==true then + validbinaries={} + end +end +local function registerlibrary(name) + if finalized then + report("libraries are already finalized") + elseif type(name)=="string" and name~="" then + if not validlibraries then + return + end + if validlibraries==true then + validlibraries={ [name]=true } + else + validlibraries[name]=true + end + elseif name==true then + validlibraries={} + end +end +local p_write=S("wa") p_write=(1-p_write)^0*p_write +local p_path=S("\\/~$%:") p_path=(1-p_path )^0*p_path +local function normalized(name) + if platform=="windows" then + name=gsub(name,"/","\\") + end + return name +end +function sandbox.possiblepath(name) + return lpegmatch(p_path,name) and true or false +end +local filenamelogger=false +function sandbox.setfilenamelogger(l) + filenamelogger=type(l)=="function" and l or false +end +local function validfilename(name,what) + if p_validroot and type(name)=="string" and lpegmatch(p_path,name) then + local asked=collapsepath(expandname(name)) + local okay=lpegmatch(p_validroot,asked) + if okay==true then + if filenamelogger then + filenamelogger(name,"w",asked,true) + end + return name + elseif okay==false then + if not what then + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + elseif lpegmatch(p_write,what) then + if filenamelogger then + filenamelogger(name,"w",asked,false) + end + return + else + if filenamelogger then + filenamelogger(name,"r",asked,true) + end + return name + end + elseif filenamelogger then + filenamelogger(name,"*",name,false) + end + else + return name + end +end +local function readable(name,finalized) + return validfilename(name,"r") +end +local function normalizedreadable(name,finalized) + local valid=validfilename(name,"r") + if valid then + return normalized(valid) + end +end +local function writeable(name,finalized) + return validfilename(name,"w") +end +local function normalizedwriteable(name,finalized) + local valid=validfilename(name,"w") + if valid then + return normalized(valid) + end +end +validators.readable=readable +validators.writeable=normalizedwriteable +validators.normalizedreadable=normalizedreadable +validators.normalizedwriteable=writeable +validators.filename=readable +table.setmetatableindex(validators,function(t,k) + if k then + t[k]=readable + end + return readable +end) +function validators.string(s,finalized) + if finalized and suspicious(s) then + return "" + else + return s + end +end +function validators.cache(s) + if finalized then + return basename(s) + else + return s + end +end +function validators.url(s) + if finalized and find("^file:") then + return "" + else + return s + end +end +local function filehandlerone(action,one,...) + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + else + end +end +local function filehandlertwo(action,one,two,...) + local checkedone=validfilename(one) + if checkedone then + local checkedtwo=validfilename(two) + if checkedtwo then + return action(one,two,...) + else + end + else + end +end +local function iohandler(action,one,...) + if type(one)=="string" then + local checkedone=validfilename(one) + if checkedone then + return action(one,...) + end + elseif one then + return action(one,...) + else + return action() + end +end +local osexecute=sandbox.original(os.execute) +local iopopen=sandbox.original(io.popen) +local reported={} +local function validcommand(name,program,template,checkers,defaults,variables,reporter,strict) + if validbinaries~=false and (validbinaries==true or validbinaries[program]) then + if variables then + for variable,value in next,variables do + local checker=validators[checkers[variable]] + if checker then + value=checker(unquoted(value),strict) + if value then + variables[variable]=optionalquoted(value) + else + report("variable %a with value %a fails the check",variable,value) + return + end + else + report("variable %a has no checker",variable) + return + end + end + for variable,default in next,defaults do + local value=variables[variable] + if not value or value=="" then + local checker=validators[checkers[variable]] + if checker then + default=checker(unquoted(default),strict) + if default then + variables[variable]=optionalquoted(default) + else + report("variable %a with default %a fails the check",variable,default) + return + end + end + end + end + end + local command=program.." "..replace(template,variables) + if reporter then + reporter("executing runner %a: %s",name,command) + elseif trace then + report("executing runner %a: %s",name,command) + end + return command + elseif not reported[name] then + report("executing program %a of runner %a is not permitted",program,name) + reported[name]=true + end +end +local runners={ + resultof=function(...) + local command=validcommand(...) + if command then + if trace then + report("resultof: %s",command) + end + local handle=iopopen(command,"r") + if handle then + local result=handle:read("*all") or "" + handle:close() + return result + end + end + end, + execute=function(...) + local command=validcommand(...) + if command then + if trace then + report("execute: %s",command) + end + return osexecute(command) + end + end, + pipeto=function(...) + local command=validcommand(...) + if command then + if trace then + report("pipeto: %s",command) + end + return iopopen(command,"w") + end + end, +} +function sandbox.registerrunner(specification) + if type(specification)=="string" then + local wrapped=validrunners[specification] + inspect(table.sortedkeys(validrunners)) + if wrapped then + return wrapped + else + report("unknown predefined runner %a",specification) + return + end + end + if type(specification)~="table" then + report("specification should be a table (or string)") + return + end + local name=specification.name + if type(name)~="string" then + report("invalid name, string expected",name) + return + end + if validrunners[name] then + report("invalid name, runner %a already defined") + return + end + local program=specification.program + if type(program)=="string" then + elseif type(program)=="table" then + program=program[platform] or program.default or program.unix + end + if type(program)~="string" or program=="" then + report("invalid runner %a specified for platform %a",name,platform) + return + end + local template=specification.template + if not template then + report("missing template for runner %a",name) + return + end + local method=specification.method or "execute" + local checkers=specification.checkers or {} + local defaults=specification.defaults or {} + local runner=runners[method] + if runner then + local finalized=finalized + local wrapped=function(variables) + return runner(name,program,template,checkers,defaults,variables,specification.reporter,finalized) + end + validrunners[name]=wrapped + return wrapped + else + validrunners[name]=nil + report("invalid method for runner %a",name) + end +end +function sandbox.getrunner(name) + return name and validrunners[name] +end +local function suspicious(str) + return (find(str,"[/\\]") or find(command,"%.%.")) and true or false +end +local function binaryrunner(action,command,...) + if validbinaries==false then + report("no binaries permitted, ignoring command: %s",command) + return + end + if type(command)~="string" then + report("command should be a string") + return + end + local program=lpegmatch(p_split,command) + if not program or program=="" then + report("unable to filter binary from command: %s",command) + return + end + if validbinaries==true then + elseif not validbinaries[program] then + report("binary not permitted, ignoring command: %s",command) + return + elseif suspicious(command) then + report("/ \\ or .. found, ignoring command (use sandbox.registerrunner): %s",command) + return + end + return action(command,...) +end +local function dummyrunner(action,command,...) + if type(command)=="table" then + command=concat(command," ",command[0] and 0 or 1) + end + report("ignoring command: %s",command) +end +sandbox.filehandlerone=filehandlerone +sandbox.filehandlertwo=filehandlertwo +sandbox.iohandler=iohandler +function sandbox.disablerunners() + validbinaries=false +end +function sandbox.disablelibraries() + validlibraries=false +end +if FFISUPPORTED and ffi then + function sandbox.disablelibraries() + validlibraries=false + for k,v in next,ffi do + if k~="gc" then + ffi[k]=nil + end + end + end + local load=ffi.load + if load then + local reported={} + function ffi.load(name,...) + if validlibraries==false then + elseif validlibraries==true then + return load(name,...) + elseif validlibraries[name] then + return load(name,...) + else + end + if not reported[name] then + report("using library %a is not permitted",name) + reported[name]=true + end + return nil + end + end +end +local overload=sandbox.overload +local register=sandbox.register + overload(loadfile,filehandlerone,"loadfile") +if io then + overload(io.open,filehandlerone,"io.open") + overload(io.popen,binaryrunner,"io.popen") + overload(io.input,iohandler,"io.input") + overload(io.output,iohandler,"io.output") + overload(io.lines,filehandlerone,"io.lines") +end +if os then + overload(os.execute,binaryrunner,"os.execute") + overload(os.spawn,dummyrunner,"os.spawn") + overload(os.exec,dummyrunner,"os.exec") + overload(os.resultof,binaryrunner,"os.resultof") + overload(os.pipeto,binaryrunner,"os.pipeto") + overload(os.rename,filehandlertwo,"os.rename") + overload(os.remove,filehandlerone,"os.remove") +end +if lfs then + overload(lfs.chdir,filehandlerone,"lfs.chdir") + overload(lfs.mkdir,filehandlerone,"lfs.mkdir") + overload(lfs.rmdir,filehandlerone,"lfs.rmdir") + overload(lfs.isfile,filehandlerone,"lfs.isfile") + overload(lfs.isdir,filehandlerone,"lfs.isdir") + overload(lfs.attributes,filehandlerone,"lfs.attributes") + overload(lfs.dir,filehandlerone,"lfs.dir") + overload(lfs.lock_dir,filehandlerone,"lfs.lock_dir") + overload(lfs.touch,filehandlerone,"lfs.touch") + overload(lfs.link,filehandlertwo,"lfs.link") + overload(lfs.setmode,filehandlerone,"lfs.setmode") + overload(lfs.readlink,filehandlerone,"lfs.readlink") + overload(lfs.shortname,filehandlerone,"lfs.shortname") + overload(lfs.symlinkattributes,filehandlerone,"lfs.symlinkattributes") +end +if zip then + zip.open=register(zip.open,filehandlerone,"zip.open") +end +if fontloader then + fontloader.open=register(fontloader.open,filehandlerone,"fontloader.open") + fontloader.info=register(fontloader.info,filehandlerone,"fontloader.info") +end +if epdf then + epdf.open=register(epdf.open,filehandlerone,"epdf.open") +end +sandbox.registerroot=registerroot +sandbox.registerbinary=registerbinary +sandbox.registerlibrary=registerlibrary +sandbox.validfilename=validfilename + + +end -- of closure + +do -- create closure to overcome 200 locals limit + package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7985, stripped down to: 6153 +-- original size: 7757, stripped down to: 6015 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9511,154 +11081,9 @@ end -- of closure do -- create closure to overcome 200 locals limit -package.loaded["util-tpl"] = package.loaded["util-tpl"] or true - --- original size: 7313, stripped down to: 4076 - -if not modules then modules={} end modules ['util-tpl']={ - version=1.001, - comment="companion to luat-lib.mkiv", - author="Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright="PRAGMA ADE / ConTeXt Development Team", - license="see context related readme files" -} -utilities.templates=utilities.templates or {} -local templates=utilities.templates -local trace_template=false trackers.register("templates.trace",function(v) trace_template=v end) -local report_template=logs.reporter("template") -local tostring=tostring -local format,sub,byte=string.format,string.sub,string.byte -local P,C,R,Cs,Cc,Carg,lpegmatch,lpegpatterns=lpeg.P,lpeg.C,lpeg.R,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.match,lpeg.patterns -local replacer -local function replacekey(k,t,how,recursive) - local v=t[k] - if not v then - if trace_template then - report_template("unknown key %a",k) - end - return "" - else - v=tostring(v) - if trace_template then - report_template("setting key %a to value %a",k,v) - end - if recursive then - return lpegmatch(replacer,v,1,t,how,recursive) - else - return v - end - end -end -local sqlescape=lpeg.replacer { - { "'","''" }, - { "\\","\\\\" }, - { "\r\n","\\n" }, - { "\r","\\n" }, -} -local sqlquoted=Cs(Cc("'")*sqlescape*Cc("'")) -lpegpatterns.sqlescape=sqlescape -lpegpatterns.sqlquoted=sqlquoted -local luaescape=lpegpatterns.luaescape -local escapers={ - lua=function(s) - return lpegmatch(luaescape,s) - end, - sql=function(s) - return lpegmatch(sqlescape,s) - end, -} -local quotedescapers={ - lua=function(s) - return format("%q",s) - end, - sql=function(s) - return lpegmatch(sqlquoted,s) - end, -} -local luaescaper=escapers.lua -local quotedluaescaper=quotedescapers.lua -local function replacekeyunquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and escapers[how] or luaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replacekeyquoted(s,t,how,recurse) - if how==false then - return replacekey(s,t,how,recurse) - else - local escaper=how and quotedescapers[how] or quotedluaescaper - return escaper(replacekey(s,t,how,recurse)) - end -end -local function replaceoptional(l,m,r,t,how,recurse) - local v=t[l] - return v and v~="" and lpegmatch(replacer,r,1,t,how or "lua",recurse or false) or "" -end -local single=P("%") -local double=P("%%") -local lquoted=P("%[") -local rquoted=P("]%") -local lquotedq=P("%(") -local rquotedq=P(")%") -local escape=double/'%%' -local nosingle=single/'' -local nodouble=double/'' -local nolquoted=lquoted/'' -local norquoted=rquoted/'' -local nolquotedq=lquotedq/'' -local norquotedq=rquotedq/'' -local noloptional=P("%?")/'' -local noroptional=P("?%")/'' -local nomoptional=P(":")/'' -local args=Carg(1)*Carg(2)*Carg(3) -local key=nosingle*((C((1-nosingle )^1)*args)/replacekey )*nosingle -local quoted=nolquotedq*((C((1-norquotedq )^1)*args)/replacekeyquoted )*norquotedq -local unquoted=nolquoted*((C((1-norquoted )^1)*args)/replacekeyunquoted)*norquoted -local optional=noloptional*((C((1-nomoptional)^1)*nomoptional*C((1-noroptional)^1)*args)/replaceoptional)*noroptional -local any=P(1) - replacer=Cs((unquoted+quoted+escape+optional+key+any)^0) -local function replace(str,mapping,how,recurse) - if mapping and str then - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - else - return str - end -end -templates.replace=replace -function templates.replacer(str,how,recurse) - return function(mapping) - return lpegmatch(replacer,str,1,mapping,how or "lua",recurse or false) or str - end -end -function templates.load(filename,mapping,how,recurse) - local data=io.loaddata(filename) or "" - if mapping and next(mapping) then - return replace(data,mapping,how,recurse) - else - return data - end -end -function templates.resolve(t,mapping,how,recurse) - if not mapping then - mapping=t - end - for k,v in next,t do - t[k]=replace(v,mapping,how,recurse) - end - return t -end - - -end -- of closure - -do -- create closure to overcome 200 locals limit - package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8284, stripped down to: 5176 +-- original size: 9246, stripped down to: 5038 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9845,7 +11270,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6358, stripped down to: 4257 +-- original size: 6174, stripped down to: 4141 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9998,7 +11423,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 56973, stripped down to: 35872 +-- original size: 57003, stripped down to: 35696 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -10013,7 +11438,7 @@ if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert -local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset +local type,next,setmetatable,getmetatable,tonumber,rawset,select=type,next,setmetatable,getmetatable,tonumber,rawset,select local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub local sort=table.sort local utfchar=utf.char @@ -10140,6 +11565,7 @@ local function add_empty(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nt, __p__=top } dt[nt]=t @@ -10161,6 +11587,7 @@ local function add_begin(spacing,namespace,tag) tg=tag, at=at, dt={}, + ni=nil, __p__=stack[level] } setmetatable(top,mt) @@ -10188,6 +11615,7 @@ local function add_end(spacing,namespace,tag) dt=top.dt nt=#dt+1 dt[nt]=toclose + toclose.ni=nt if toclose.at.xmlns then remove(xmlns) end @@ -10232,7 +11660,13 @@ local function add_special(what,spacing,text) if strip and (what=="@cm@" or what=="@dt@") then else nt=nt+1 - dt[nt]={ special=true,ns="",tg=what,dt={ text } } + dt[nt]={ + special=true, + ns="", + tg=what, + ni=nil, + dt={ text }, + } end end local function set_message(txt) @@ -10285,7 +11719,6 @@ do end local p_rest=(1-P(";"))^0 local p_many=P(1)^0 - local p_char=lpegpatterns.utf8character local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) xml.parsedentitylpeg=parsedentity local predefined_unified={ @@ -10327,13 +11760,27 @@ do [ [[}]] ]="&U+7D;", [ [[~]] ]="&U+7E;", } + local privates_x={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[']] ]="&U+27;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } local privates_n={ } local escaped=utf.remapper(privates_u,"dynamic") local unprivatized=utf.remapper(privates_p,"dynamic") local unspecialized=utf.remapper(privates_s,"dynamic") + local despecialized=utf.remapper(privates_x,"dynamic") xml.unprivatized=unprivatized xml.unspecialized=unspecialized + xml.despecialized=despecialized xml.escaped=escaped local function unescaped(s) local p=privates_n[s] @@ -10778,6 +12225,10 @@ local grammar_unparsed_text=P { "preamble", local function _xmlconvert_(data,settings) settings=settings or {} preparexmlstate(settings) + local preprocessor=settings.preprocessor + if data and data~="" and type(preprocessor)=="function" then + data=preprocessor(data,settings) or data + end if settings.parent_root then mt=getmetatable(settings.parent_root) else @@ -10919,14 +12370,24 @@ function xml.toxml(data) return data end end -local function copy(old) +local function copy(old,p) if old then local new={} for k,v in next,old do - if type(v)=="table" then - new[k]=table.copy(v) - else + local t=type(v)=="table" + if k=="at" then + local t={} + for k,v in next,v do + t[k]=v + end + new[k]=t + elseif k=="dt" then + v.__p__=nil + v=copy(v,new) new[k]=v + v.__p__=p + else + new[k]=v end end local mt=getmetatable(old) @@ -11157,18 +12618,26 @@ local xmlfilehandler=newhandlers { function xml.save(root,name) serialize(root,xmlfilehandler,name) end -local result +local result,r,threshold={},0,512 local xmlstringhandler=newhandlers { name="string", initialize=function() - result={} + r=0 return result end, finalize=function() - return concat(result) + local done=concat(result,"",1,r) + r=0 + if r>threshold then + result={} + end + return done end, handle=function(...) - result[#result+1]=concat {... } + for i=1,select("#",...) do + r=r+1 + result[r]=select(i,...) + end end, } local function xmltostring(root) @@ -11320,7 +12789,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 53892, stripped down to: 32508 +-- original size: 53301, stripped down to: 32477 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -11702,6 +13171,14 @@ local function apply_expression(list,expression,order) end return collected end +local function apply_selector(list,specification) + if xml.applyselector then + apply_selector=xml.applyselector + return apply_selector(list,specification) + else + return list + end +end local P,V,C,Cs,Cc,Ct,R,S,Cg,Cb=lpeg.P,lpeg.V,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Ct,lpeg.R,lpeg.S,lpeg.Cg,lpeg.Cb local spaces=S(" \n\r\t\f")^0 local lp_space=S(" \n\r\t\f") @@ -11825,6 +13302,9 @@ end local function register_nodes(nodetest,nodes) return { kind="nodes",nodetest=nodetest,nodes=nodes } end +local function register_selector(specification) + return { kind="selector",specification=specification } +end local function register_expression(expression) local converted=lpegmatch(converter,expression) local runner=load(format(template_e,converted)) @@ -11865,34 +13345,36 @@ local pathparser=Ct { "patterns", (V("special")*spaces*P(-1) )+(V("initial")*spaces*V("step")*spaces*(P("/")*spaces*V("step")*spaces)^0 ) ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), - step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, + step=((V("shortcuts")+V("selector")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, - shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor"), + shortcuts_a=V("s_descendant_or_self")+V("s_descendant")+V("s_child")+V("s_parent")+V("s_self")+V("s_root")+V("s_ancestor")+V("s_lastmatch"), shortcuts=V("shortcuts_a")*(spaces*"/"*spaces*V("shortcuts_a"))^0, s_descendant_or_self=(P("***/")+P("/"))*Cc(register_descendant_or_self), s_descendant=P("**")*Cc(register_descendant), - s_child=P("*")*no_nextcolon*Cc(register_child ), - s_parent=P("..")*Cc(register_parent ), - s_self=P("." )*Cc(register_self ), - s_root=P("^^")*Cc(register_root ), - s_ancestor=P("^")*Cc(register_ancestor ), - descendant=P("descendant::")*Cc(register_descendant ), - child=P("child::")*Cc(register_child ), - parent=P("parent::")*Cc(register_parent ), - self=P("self::")*Cc(register_self ), - root=P('root::')*Cc(register_root ), - ancestor=P('ancestor::')*Cc(register_ancestor ), - descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self ), - ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self ), - following=P('following::')*Cc(register_following ), - following_sibling=P('following-sibling::')*Cc(register_following_sibling ), - preceding=P('preceding::')*Cc(register_preceding ), - preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), - reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), - last_match=P('last-match::')*Cc(register_last_match ), + s_child=P("*")*no_nextcolon*Cc(register_child), + s_parent=P("..")*Cc(register_parent), + s_self=P("." )*Cc(register_self), + s_root=P("^^")*Cc(register_root), + s_ancestor=P("^")*Cc(register_ancestor), + s_lastmatch=P("=")*Cc(register_last_match), + descendant=P("descendant::")*Cc(register_descendant), + child=P("child::")*Cc(register_child), + parent=P("parent::")*Cc(register_parent), + self=P("self::")*Cc(register_self), + root=P('root::')*Cc(register_root), + ancestor=P('ancestor::')*Cc(register_ancestor), + descendant_or_self=P('descendant-or-self::')*Cc(register_descendant_or_self), + ancestor_or_self=P('ancestor-or-self::')*Cc(register_ancestor_or_self), + following=P('following::')*Cc(register_following), + following_sibling=P('following-sibling::')*Cc(register_following_sibling), + preceding=P('preceding::')*Cc(register_preceding), + preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling), + reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling), + last_match=P('last-match::')*Cc(register_last_match), + selector=P("{")*C((1-P("}"))^1)*P("}")/register_selector, nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -12042,6 +13524,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) p.matched=p.matched+1 @@ -12083,6 +13567,9 @@ do elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) + report_lpath("% 10i : se : %s ",(collected and #collected) or 0,pi.specification) elseif kind=="finalizer" then collected=pi.finalizer(collected) report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") @@ -12114,6 +13601,8 @@ do collected=apply_nodes(collected,pi.nodetest,pi.nodes) elseif kind=="expression" then collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="selector" then + collected=apply_selector(collected,pi.specification) elseif kind=="finalizer" then return pi.finalizer(collected) end @@ -12170,6 +13659,13 @@ do function xml.lastmatch() return lastmatch end + local stack={} + function xml.pushmatch() + insert(stack,lastmatch) + end + function xml.popmatch() + lastmatch=remove(stack) + end end local applylpath=xml.applylpath function xml.filter(root,pattern) @@ -12449,7 +13945,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3787, stripped down to: 2003 +-- original size: 3684, stripped down to: 1957 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -12518,7 +14014,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 30566, stripped down to: 21741 +-- original size: 29835, stripped down to: 21174 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12534,7 +14030,6 @@ local xml=xml local xmlcopy,xmlname=xml.copy,xml.name local xmlinheritedconvert=xml.inheritedconvert local xmlapplylpath=xml.applylpath -local xmlfilter=xml.filter local type,next,setmetatable,getmetatable=type,next,setmetatable,getmetatable local insert,remove,fastcopy,concat=table.insert,table.remove,table.fastcopy,table.concat local gmatch,gsub,format,find,strip=string.gmatch,string.gsub,string.format,string.find,string.strip @@ -12738,7 +14233,17 @@ function xml.replace(root,pattern,whatever) report('replacing',pattern,c,e) end local d=p.dt - d[e.ni]=copiedelement(element,p) + local n=e.ni + local t=copiedelement(element,p) + if type(t)=="table" then + d[n]=t[1] + for i=2,#t do + n=n+1 + insert(d,n,t[i]) + end + else + d[n]=t + end redo_ni(d) end end @@ -13161,7 +14666,7 @@ local obsolete=xml.obsolete xml.strip_whitespace=xml.strip obsolete.strip_whitespace=xml.strip xml.collect_elements=xml.collect obsolete.collect_elements=xml.collect xml.delete_element=xml.delete obsolete.delete_element=xml.delete -xml.replace_element=xml.replace obsolete.replace_element=xml.replacet +xml.replace_element=xml.replace obsolete.replace_element=xml.replace xml.each_element=xml.each obsolete.each_element=xml.each xml.process_elements=xml.process obsolete.process_elements=xml.process xml.insert_element_after=xml.insertafter obsolete.insert_element_after=xml.insertafter @@ -13379,7 +14884,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10719, stripped down to: 7841 +-- original size: 10274, stripped down to: 7538 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -13757,7 +15262,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6534, stripped down to: 5072 +-- original size: 6407, stripped down to: 4965 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13907,6 +15412,7 @@ function reporters.export(t,methods,filename) if filename then local fullname=file.replacesuffix(filename,method) t.report("saving export in %a",fullname) + dir.mkdirs(file.pathpart(fullname)) io.savedata(fullname,result) else reporters.lines(t,result) @@ -13927,7 +15433,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11444, stripped down to: 7830 +-- original size: 11099, stripped down to: 7516 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -14072,11 +15578,6 @@ if not texroot or texroot=="" then ossetenv('TEXROOT',texroot) end environment.texroot=file.collapsepath(texroot) -if type(profiler)=="table" and not jit then - directives.register("system.profile",function() - profiler.start("luatex-profile.log") - end) -end local prefixes=utilities.storage.allocate() resolvers.prefixes=prefixes local resolved={} @@ -14183,7 +15684,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 18619, stripped down to: 11042 +-- original size: 17958, stripped down to: 10705 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -14199,7 +15700,6 @@ local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next local isdir=lfs.isdir -local ostype=os.type local collapsepath,joinpath,basename=file.collapsepath,file.join,file.basename local trace_locating=false trackers.register("resolvers.locating",function(v) trace_locating=v end) local trace_expansions=false trackers.register("resolvers.expansions",function(v) trace_expansions=v end) @@ -14568,7 +16068,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9649, stripped down to: 7131 +-- original size: 9342, stripped down to: 6887 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -14852,7 +16352,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 16066, stripped down to: 11938 +-- original size: 16088, stripped down to: 11435 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -15056,18 +16556,6 @@ end caches.getreadablepaths=getreadablepaths caches.getwritablepath=getwritablepath function caches.getfirstreadablefile(filename,...) - local rd=getreadablepaths(...) - for i=1,#rd do - local path=rd[i] - local fullname=file.join(path,filename) - if is_readable(fullname) then - usedreadables[i]=true - return fullname,path - end - end - return caches.setfirstwritablefile(filename,...) -end -function caches.getfirstreadablefile_TEST_ME_FIRST(filename,...) local fullname,path=caches.setfirstwritablefile(filename,...) if is_readable(fullname) then return fullname,path @@ -15096,18 +16584,22 @@ end function caches.setluanames(path,name) return format("%s/%s.%s",path,name,luasuffixes.tma),format("%s/%s.%s",path,name,luasuffixes.tmc) end -function caches.loaddata(readables,name) +function caches.loaddata(readables,name,writable) if type(readables)=="string" then readables={ readables } end for i=1,#readables do local path=readables[i] - local tmaname,tmcname=caches.setluanames(path,name) local loader=false + local tmaname,tmcname=caches.setluanames(path,name) if isfile(tmcname) then loader=loadfile(tmcname) end if not loader and isfile(tmaname) then + local tmacrap,tmcname=caches.setluanames(writable,name) + if isfile(tmcname) then + loader=loadfile(tmcname) + end utilities.lua.compile(tmaname,tmcname) if isfile(tmcname) then loader=loadfile(tmcname) @@ -15228,7 +16720,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5488, stripped down to: 4101 +-- original size: 5310, stripped down to: 3980 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -15240,7 +16732,6 @@ if not modules then modules={} end modules ['data-met']={ local find,format=string.find,string.format local sequenced=table.sequenced local addurlscheme,urlhashed=url.addscheme,url.hashed -local getcurrentdir=lfs.currentdir local trace_locating=false local trace_methods=false trackers.register("resolvers.locating",function(v) trace_methods=v end) @@ -15347,7 +16838,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67241, stripped down to: 46427 +-- original size: 67524, stripped down to: 46632 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15401,6 +16892,7 @@ resolvers.configbanner="" resolvers.homedir=environment.homedir resolvers.criticalvars=allocate { "SELFAUTOLOC","SELFAUTODIR","SELFAUTOPARENT","TEXMFCNF","TEXMF","TEXOS" } resolvers.luacnfname="texmfcnf.lua" +resolvers.luacnffallback="contextcnf.lua" resolvers.luacnfstate="unknown" if environment.default_texmfcnf then resolvers.luacnfspec="home:texmf/web2c;"..environment.default_texmfcnf @@ -15444,7 +16936,6 @@ local function resolvevariable(k) end local dollarstripper=lpeg.stripper("$") local inhibitstripper=P("!")^0*Cs(P(1)^0) -local backslashswapper=lpeg.replacer("\\","/") local somevariable=P("$")/"" local somekey=C(R("az","AZ","09","__","--")^1) local somethingelse=P(";")*((1-S("!{}/\\"))^1*P(";")/"")+P(";")*(P(";")/"")+P(1) @@ -15590,23 +17081,29 @@ local function identify_configuration_files() end reportcriticalvariables(cnfspec) local cnfpaths=expandedpathfromlist(resolvers.splitpath(cnfspec)) - local luacnfname=resolvers.luacnfname - for i=1,#cnfpaths do - local filepath=cnfpaths[i] - local filename=collapsepath(filejoin(filepath,luacnfname)) - local realname=resolveprefix(filename) - if trace_locating then - local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") - local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) - report_resolving("looking for %a on %s path %a from specification %a",luacnfname,weirdpath and "weird" or "given",fullpath,filepath) - end - if isfile(realname) then - specification[#specification+1]=filename + local function locatecnf(luacnfname,kind) + for i=1,#cnfpaths do + local filepath=cnfpaths[i] + local filename=collapsepath(filejoin(filepath,luacnfname)) + local realname=resolveprefix(filename) if trace_locating then - report_resolving("found configuration file %a",realname) + local fullpath=gsub(resolveprefix(collapsepath(filepath)),"//","/") + local weirdpath=find(fullpath,"/texmf.+/texmf") or not find(fullpath,"/web2c",1,true) + report_resolving("looking for %s %a on %s path %a from specification %a", + kind,luacnfname,weirdpath and "weird" or "given",fullpath,filepath) + end + if isfile(realname) then + specification[#specification+1]=filename + if trace_locating then + report_resolving("found %s configuration file %a",kind,realname) + end end end end + locatecnf(resolvers.luacnfname,"regular") + if #specification==0 then + locatecnf(resolvers.luacnffallback,"fallback") + end if trace_locating then report_resolving() end @@ -16903,7 +18400,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 4236, stripped down to: 3144 +-- original size: 4090, stripped down to: 3059 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -17025,7 +18522,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 935, stripped down to: 838 +-- original size: 910, stripped down to: 823 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -17055,7 +18552,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 548, stripped down to: 483 +-- original size: 530, stripped down to: 475 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -17078,7 +18575,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3976, stripped down to: 3391 +-- original size: 3863, stripped down to: 3310 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -17186,7 +18683,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5148, stripped down to: 3680 +-- original size: 5029, stripped down to: 3607 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -17256,7 +18753,7 @@ function containers.read(container,name) local storage=container.storage local stored=storage[name] if not stored and container.enabled and caches and containers.usecache then - stored=caches.loaddata(container.readables,name) + stored=caches.loaddata(container.readables,name,container.writable) if stored and stored.cache_version==container.version then if trace_cache or trace_containers then report_containers("action %a, category %a, name %a","load",container.subcategory,name) @@ -17305,7 +18802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 4000, stripped down to: 3052 +-- original size: 4045, stripped down to: 3110 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -17350,7 +18847,7 @@ function resolvers.automount(usecache) end statistics.register("used config file",function() return caches.configfiles() end) statistics.register("used cache path",function() return caches.usedpaths() end) -function statistics.savefmtstatus(texname,formatbanner,sourcefile) +function statistics.savefmtstatus(texname,formatbanner,sourcefile,kind,banner) local enginebanner=status.banner if formatbanner and enginebanner and sourcefile then local luvname=file.replacesuffix(texname,"luv") @@ -17361,6 +18858,10 @@ function statistics.savefmtstatus(texname,formatbanner,sourcefile) sourcefile=sourcefile, } io.savedata(luvname,table.serialize(luvdata,true)) + lua.registerfinalizer(function() + logs.report("format banner","%s",banner) + logs.newline() + end) end end function statistics.checkfmtstatus(texname) @@ -17396,7 +18897,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 9036, stripped down to: 7041 +-- original size: 8772, stripped down to: 6841 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -17633,7 +19134,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8712, stripped down to: 5726 +-- original size: 8479, stripped down to: 5580 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17822,7 +19323,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6779, stripped down to: 5444 +-- original size: 6653, stripped down to: 5467 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17868,11 +19369,21 @@ function resolvers.schemes.cleanname(specification) end return hash end -local cached,loaded,reused,thresholds,handlers={},{},{},{},{} -local function runcurl(name,cachename) - local command="curl --silent --insecure --create-dirs --output "..cachename.." "..name - os.execute(command) -end +local cached={} +local loaded={} +local reused={} +local thresholds={} +local handlers={} +local runner=sandbox.registerrunner { + name="curl resolver", + method="execute", + program="curl", + template="--silent -- insecure --create-dirs --output %cachename% %original%", + checkers={ + cachename="cache", + original="url", + } +} local function fetch(specification) local original=specification.original local scheme=specification.scheme @@ -17894,7 +19405,10 @@ local function fetch(specification) report_schemes("fetching %a, protocol %a, method %a",original,scheme,"curl") end logs.flush() - runcurl(original,cachename) + runner { + original=original, + cachename=cachename, + } end end if io.exists(cachename) then @@ -18003,7 +19517,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4447, stripped down to: 3302 +-- original size: 4207, stripped down to: 3137 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -18045,8 +19559,6 @@ function helpers.cleanpath(path) return resolveprefix(lpegmatch(pattern,path)) end local loadedaslib=helpers.loadedaslib -local getextraluapaths=package.extraluapaths -local getextralibpaths=package.extralibpaths local registerpath=helpers.registerpath local lualibfile=helpers.lualibfile local luaformatpaths @@ -18112,7 +19624,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2494, stripped down to: 2047 +-- original size: 2431, stripped down to: 1996 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -18179,7 +19691,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2674, stripped down to: 1658 +-- original size: 2601, stripped down to: 1627 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -18235,7 +19747,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2815, stripped down to: 2415 +-- original size: 2734, stripped down to: 2354 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -18315,7 +19827,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11846, stripped down to: 6059 +-- original size: 13595, stripped down to: 7500 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -18324,35 +19836,51 @@ if not modules then modules={} end modules ['util-lib']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files", } -local gsub,find=string.gsub,string.find -local pathpart,nameonly,joinfile=file.pathpart,file.nameonly,file.join -local findfile,findfiles=resolvers and resolvers.findfile,resolvers and resolvers.findfiles -local loaded=package.loaded -local report_swiglib=logs.reporter("swiglib") -local trace_swiglib=false trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) +local type=type +local next=next +local pcall=pcall +local gsub=string.gsub +local find=string.find +local sort=table.sort +local pathpart=file.pathpart +local nameonly=file.nameonly +local joinfile=file.join +local removesuffix=file.removesuffix +local findfile=resolvers.findfile +local findfiles=resolvers.findfiles +local expandpaths=resolvers.expandedpathlistfromvariable +local qualifiedpath=file.is_qualified_path +local isfile=lfs.isfile local done=false -local function requireswiglib(required,version) - local trace_swiglib=trace_swiglib or package.helpers.trace - local library=loaded[required] - if library==nil then - if trace_swiglib then - report_swiglib("requiring library %a with version %a",required,version or "any") - end - local required_full=gsub(required,"%.","/") - local required_path=pathpart(required_full) - local required_base=nameonly(required_full) +local function locate(required,version,trace,report,action) + if type(required)~="string" then + report("provide a proper library name") + return + end + if trace then + report("requiring library %a with version %a",required,version or "any") + end + local found_library=nil + local required_full=gsub(required,"%.","/") + local required_path=pathpart(required_full) + local required_base=nameonly(required_full) + if qualifiedpath(required) then + if isfile(required) then + found_library=required + end + else local required_name=required_base.."."..os.libsuffix local version=type(version)=="string" and version~="" and version or false local engine=environment.ownmain or false - if trace_swiglib and not done then - local list=resolvers.expandedpathlistfromvariable("lib") + if trace and not done then + local list=expandpaths("lib") for i=1,#list do - report_swiglib("tds path %i: %s",i,list[i]) + report("tds path %i: %s",i,list[i]) end end local function found(locate,asked_library,how,...) - if trace_swiglib then - report_swiglib("checking %s: %a",how,asked_library) + if trace then + report("checking %s: %a",how,asked_library) end return locate(asked_library,...) end @@ -18360,45 +19888,45 @@ local function requireswiglib(required,version) local found=nil if version then local asked_library=joinfile(required_path,version,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end if not found or found=="" then local asked_library=joinfile(required_path,required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","with version",asked_library) + if trace then + report("checking %s: %a","with version",asked_library) end found=locate(asked_library,...) end return found and found~="" and found or false end local function attempt(checkpattern) - if trace_swiglib then - report_swiglib("checking tds lib paths strictly") + if trace then + report("checking tds lib paths strictly") end local found=findfile and check(findfile,"lib") if found and (not checkpattern or find(found,checkpattern)) then return found end - if trace_swiglib then - report_swiglib("checking tds lib paths with wildcard") + if trace then + report("checking tds lib paths with wildcard") end local asked_library=joinfile(required_path,".*",required_name) - if trace_swiglib then - report_swiglib("checking %s: %a","latest version",asked_library) + if trace then + report("checking %s: %a","latest version",asked_library) end local list=findfiles(asked_library,"lib",true) if list and #list>0 then - table.sort(list) + sort(list) local found=list[#list] if found and (not checkpattern or find(found,checkpattern)) then return found end end - if trace_swiglib then - report_swiglib("checking lib paths") + if trace then + report("checking lib paths") end package.extralibpath(environment.ownpath) local paths=package.libpaths() @@ -18410,89 +19938,143 @@ local function requireswiglib(required,version) end return false end - local found_library=nil if engine then - if trace_swiglib then - report_swiglib("attemp 1, engine %a",engine) + if trace then + report("attemp 1, engine %a",engine) end found_library=attempt("/"..engine.."/") if not found_library then - if trace_swiglib then - report_swiglib("attemp 2, no engine",asked_library) + if trace then + report("attemp 2, no engine",asked_library) end found_library=attempt() end else found_library=attempt() end - if not found_library then - if trace_swiglib then - report_swiglib("not found: %a",required) - end + end + if not found_library then + if trace then + report("not found: %a",required) + end + library=false + else + if trace then + report("found: %a",found_library) + end + local message,result=action(found_library,required_base) + if result then + library=result + else library=false + report("load error: message %a, library %a",tostring(message),found_library or "no library") + end + end + if not library then + report("unknown: %a",required) + elseif trace then + report("stored: %a",required) + end + return library +end +do + local report_swiglib=logs.reporter("swiglib") + local trace_swiglib=false + local savedrequire=require + local loadedlibs={} + local loadlib=package.loadlib + local pushdir=dir.push + local popdir=dir.pop + trackers.register("resolvers.swiglib",function(v) trace_swiglib=v end) + function requireswiglib(required,version) + local library=loadedlibs[library] + if library==nil then + local trace_swiglib=trace_swiglib or package.helpers.trace + library=locate(required,version,trace_swiglib,report_swiglib,function(name,base) + pushdir(pathpart(name)) + local opener="luaopen_"..base + if trace_swiglib then + report_swiglib("opening: %a with %a",name,opener) + end + local library,message=loadlib(name,opener) + local libtype=type(library) + if libtype=="function" then + library=library() + message=true + else + report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") + library=false + end + popdir() + return message,library + end) + loadedlibs[required]=library or false + end + return library + end + function require(name,version) + if find(name,"^swiglib%.") then + return requireswiglib(name,version) else - local path=pathpart(found_library) - local base=nameonly(found_library) - dir.push(path) + return savedrequire(name) + end + end + local swiglibs={} + local initializer="core" + function swiglib(name,version) + local library=swiglibs[name] + if not library then + statistics.starttiming(swiglibs) if trace_swiglib then - report_swiglib("found: %a",found_library) - end - local message=nil - local opener="luaopen_"..required_base - library,message=package.loadlib(found_library,opener) - local libtype=type(library) - if libtype=="function" then - library=library() + report_swiglib("loading %a",name) + end + if not find(name,"%."..initializer.."$") then + fullname="swiglib."..name.."."..initializer else - report_swiglib("load error: %a returns %a, message %a, library %a",opener,libtype,(string.gsub(message or "no message","[%s]+$","")),found_library or "no library") - library=false + fullname="swiglib."..name end - dir.pop() + library=requireswiglib(fullname,version) + swiglibs[name]=library + statistics.stoptiming(swiglibs) end - if not library then - report_swiglib("unknown: %a",required) - elseif trace_swiglib then - report_swiglib("stored: %a",required) - end - loaded[required]=library - else - report_swiglib("reused: %a",required) + return library end - return library + statistics.register("used swiglibs",function() + if next(swiglibs) then + return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) + end + end) end -local savedrequire=require -function require(name,version) - if find(name,"^swiglib%.") then - return requireswiglib(name,version) - else - return savedrequire(name) +if FFISUPPORTED and ffi and ffi.load then + local report_ffilib=logs.reporter("ffilib") + local trace_ffilib=false + local savedffiload=ffi.load + trackers.register("resolvers.ffilib",function(v) trace_ffilib=v end) + local function locateindeed(name) + local message,library=pcall(savedffiload,removesuffix(name)) + if type(library)=="userdata" then + return library + else + return false + end end -end -local swiglibs={} -local initializer="core" -function swiglib(name,version) - local library=swiglibs[name] - if not library then - statistics.starttiming(swiglibs) - if trace_swiglib then - report_swiglib("loading %a",name) + function ffilib(required,version) + if version=="system" then + return locateindeed(name) + else + return locate(required,version,trace_ffilib,report_ffilib,locateindeed) end - if not find(name,"%."..initializer.."$") then - fullname="swiglib."..name.."."..initializer + end + function ffi.load(name) + local library=ffilib(name) + if type(library)=="userdata" then + return library else - fullname="swiglib."..name + report_ffilib("trying to load %a using normal loader",name) + return savedffiload(name) end - library=requireswiglib(fullname,version) - swiglibs[name]=library - statistics.stoptiming(swiglibs) end - return library end -statistics.register("used swiglibs",function() - if next(swiglibs) then - return string.format("%s, initial load time %s seconds",table.concat(table.sortedkeys(swiglibs)," "),statistics.elapsedtime(swiglibs)) - end -end) end -- of closure @@ -18501,7 +20083,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5914, stripped down to: 2584 +-- original size: 5703, stripped down to: 2507 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -18604,7 +20186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 6967, stripped down to: 5631 +-- original size: 9144, stripped down to: 7291 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -18618,23 +20200,67 @@ local concat=table.concat local quoted=string.quoted local luasuffixes=utilities.lua.suffixes local report_format=logs.reporter("resolvers","formats") -local function primaryflags() - local trackers=environment.argument("trackers") - local directives=environment.argument("directives") +local function primaryflags() + local arguments=environment.arguments + local flags={} + if arguments.silent then + flags[#flags+1]="--interaction=batchmode" + end + if arguments.jit then + flags[#flags+1]="--jiton" + end + return concat(flags," ") +end +local function secondaryflags() + local arguments=environment.arguments + local trackers=arguments.trackers + local directives=arguments.directives local flags={} if trackers and trackers~="" then - flags={ "--trackers="..quoted(trackers) } + flags[#flags+1]="--c:trackers="..quoted(trackers) end if directives and directives~="" then - flags={ "--directives="..quoted(directives) } + flags[#flags+1]="--c:directives="..quoted(directives) + end + if arguments.silent then + flags[#flags+1]="--c:silent" end - if environment.argument("jit") then - flags={ "--jiton" } + if arguments.jit then + flags[#flags+1]="--c:jiton" + end + if arguments.ansi then + flags[#flags+1]="--c:ansi" end return concat(flags," ") end -function environment.make_format(name,silent) +local template=[[--ini %primaryflags% --lua=%luafile% %texfile% %secondaryflags% %dump% %redirect%]] +local checkers={ + primaryflags="string", + secondaryflags="string", + luafile="readable", + texfile="readable", + redirect="string", + dump="string", +} +local runners={ + luatex=sandbox.registerrunner { + name="make luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="make luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} +function environment.make_format(name,arguments) local engine=environment.ownmain or "luatex" + local silent=environment.arguments.silent local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" if path~="" then @@ -18690,11 +20316,20 @@ function environment.make_format(name,silent) lfs.chdir(olddir) return end - local dump=os.platform=="unix" and "\\\\dump" or "\\dump" - if silent then + local specification={ + primaryflags=primaryflags(), + secondaryflags=secondaryflags(), + luafile=quoted(usedluastub), + texfile=quoted(fulltexsourcename), + dump=os.platform=="unix" and "\\\\dump" or "\\dump", + } + local runner=runners[engine] + if not runner then + report_format("format %a cannot be generated, no runner available for engine %a",name,engine) + elseif silent then statistics.starttiming() - local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - local result=os.execute(command) + specification.redirect="> temp.log" + local result=runner(specification) local runtime=statistics.stoptiming() if result~=0 then print(format("%s silent make > fatal error when making format %q",engine,name)) @@ -18703,9 +20338,7 @@ function environment.make_format(name,silent) end os.remove("temp.log") else - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) - report_format("running command: %s\n",command) - os.execute(command) + runner(specification) end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) @@ -18718,6 +20351,30 @@ function environment.make_format(name,silent) end lfs.chdir(olddir) end +local template=[[%flags% --fmt=%fmtfile% --lua=%luafile% %texfile% %more%]] +local checkers={ + flags="string", + more="string", + fmtfile="readable", + luafile="readable", + texfile="readable", +} +local runners={ + luatex=sandbox.registerrunner { + name="run luatex format", + program="luatex", + template=template, + checkers=checkers, + reporter=report_format, + }, + luajittex=sandbox.registerrunner { + name="run luajittex format", + program="luajittex", + template=template, + checkers=checkers, + reporter=report_format, + }, +} function environment.run_format(name,data,more) if name and name~="" then local engine=environment.ownmain or "luatex" @@ -18739,9 +20396,18 @@ function environment.run_format(name,data,more) report_format("using format name %a",fmtname) report_format("no luc/lua file with name %a",barename) else - local command=format("%s %s --fmt=%s --lua=%s %s %s",engine,primaryflags(),quoted(barename),quoted(luaname),quoted(data),more~="" and quoted(more) or "") - report_format("running command: %s",command) - os.execute(command) + local runner=runners[engine] + if not runner then + report_format("format %a cannot be run, no runner available for engine %a",name,engine) + else + runner { + flags=primaryflags(), + fmtfile=quoted(barename), + luafile=quoted(luaname), + texfile=quoted(data), + more=more, + } + end end end end @@ -18750,10 +20416,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-sandbox.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-tpl.lua util-sbx.lua util-mrg.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 797557 --- stripped bytes : 289197 +-- original bytes : 842443 +-- stripped bytes : 306317 -- end library merge @@ -18777,6 +20443,7 @@ local owntree = environment and environment.ownpath or ownpath local ownlibs = { -- order can be made better 'l-lua.lua', + 'l-sandbox.lua', 'l-package.lua', 'l-lpeg.lua', 'l-function.lua', @@ -18810,8 +20477,9 @@ local ownlibs = { -- order can be made better 'util-lua.lua', -- indeed here? 'util-deb.lua', - 'util-mrg.lua', 'util-tpl.lua', + 'util-sbx.lua', + 'util-mrg.lua', 'util-env.lua', 'luat-env.lua', -- can come before inf (as in mkiv) @@ -18983,7 +20651,7 @@ local helpinfo = [[ <metadata> <entry name="name">mtxrun</entry> <entry name="detail">ConTeXt TDS Runner Tool</entry> - <entry name="version">1.31</entry> + <entry name="version">1.32</entry> </metadata> <flags> <category name="basic"> @@ -19046,6 +20714,7 @@ local helpinfo = [[ </subcategory> <subcategory> <flag name="expand-braces"><short>expand complex variable</short></flag> + <flag name="resolve-path"><short>expand variable (completely resolve paths)</short></flag> <flag name="expand-path"><short>expand variable (resolve paths)</short></flag> <flag name="expand-var"><short>expand variable (resolve references)</short></flag> <flag name="show-path"><short>show path expansion of ...</short></flag> @@ -19063,7 +20732,7 @@ local helpinfo = [[ local application = logs.application { name = "mtxrun", - banner = "ConTeXt TDS Runner Tool 1.31", + banner = "ConTeXt TDS Runner Tool 1.32", helpinfo = helpinfo, } @@ -20029,6 +21698,13 @@ elseif e_argument("expand-path") then environment.initializearguments(environment.arguments_after) resolvers.dowithfilesandreport(resolvers.expandpath, environment.files) +elseif e_argument("resolve-path") then + + resolvers.load("nofiles") + runners.register_arguments(filename) + environment.initializearguments(environment.arguments_after) + resolvers.dowithfilesandreport(resolvers.cleanedpathlist, environment.files) + elseif e_argument("expand-var") or e_argument("expand-variable") then -- luatools: runners.execute_ctx_script("mtx-base","--expand-var",filename) |