diff options
Diffstat (limited to 'scripts')
23 files changed, 7465 insertions, 3363 deletions
diff --git a/scripts/context/lua/mtx-chars.lua b/scripts/context/lua/mtx-chars.lua index 9f6852da2..d54804a1b 100644 --- a/scripts/context/lua/mtx-chars.lua +++ b/scripts/context/lua/mtx-chars.lua @@ -36,85 +36,15 @@ local application = logs.application { local report = application.report -local format, gmatch, upper, lower = string.format, string.gmatch, string.upper, string.lower -local tonumber = tonumber +local format, gmatch, upper, lower, find = string.format, string.gmatch, string.upper, string.lower, string.find +local formatters = string.formatters +local tonumber, type = tonumber, type local concat = table.concat local utfchar = utf.char scripts = scripts or { } scripts.chars = scripts.chars or { } ---~ local banner = [[ ---~ -- filename : char-mth.lua ---~ -- comment : companion to char-mth.tex (in ConTeXt) ---~ -- author : Hans Hagen, PRAGMA-ADE, Hasselt NL ---~ -- license : see context related readme files ---~ -- comment : generated from data file downloaded from STIX website ---~ ---~ if not versions then versions = { } end versions['char-mth'] = 1.001 ---~ if not characters then characters = { } end ---~ ]] ---~ ---~ function scripts.chars.stixtomkiv(inname,outname) ---~ if inname == "" then ---~ report("aquiring math data, invalid datafilename") ---~ end ---~ local f = io.open(inname) ---~ if not f then ---~ report("aquiring math data, invalid datafile") ---~ else ---~ report("aquiring math data, processing %s",inname) ---~ if not outname or outname == "" then ---~ outname = "char-mth.lua" ---~ end ---~ local classes = { ---~ N = "normal", ---~ A = "alphabetic", ---~ D = "diacritic", ---~ P = "punctuation", ---~ B = "binary", ---~ R = "relation", ---~ L = "large", ---~ O = "opening", ---~ C = "closing", ---~ F = "fence" ---~ } ---~ local valid, done = false, { } ---~ local g = io.open(outname,'w') ---~ g:write(banner) ---~ g:write(format("\ncharacters.math = {\n")) ---~ for l in f:lines() do ---~ if not valid then ---~ valid = l:find("AMS/TeX name") ---~ end ---~ if valid then ---~ local unicode = l:sub(2,6) ---~ if unicode:sub(1,1) ~= " " and unicode ~= "" and not done[unicode] then ---~ local mathclass, adobename, texname = l:sub(57,57) or "", l:sub(13,36) or "", l:sub(84,109) or "" ---~ texname, adobename = texname:gsub("[\\ ]",""), adobename:gsub("[\\ ]","") ---~ local t = { } ---~ if mathclass ~= "" then t[#t+1] = format("mathclass='%s'", classes[mathclass] or "unknown") end ---~ if adobename ~= "" then t[#t+1] = format("adobename='%s'", adobename ) end ---~ if texname ~= "" then t[#t+1] = format("texname='%s'" , texname ) end ---~ if #t > 0 then ---~ g:write(format("\t[0x%s] = { %s },\n",unicode, concat(t,", "))) ---~ end ---~ done[unicode] = true ---~ end ---~ end ---~ end ---~ if not valid then ---~ g:write("\t-- The data file is corrupt, invalid or maybe the format has changed.\n") ---~ report("aquiring math data, problems with data table") ---~ else ---~ report("aquiring math data, table saved in %s",outname) ---~ end ---~ g:write("}\n") ---~ g:close() ---~ f:close() ---~ end ---~ end - function scripts.chars.stixtomkiv(inname,outname) report("we no longer use this options but use our own tables instead") end @@ -133,6 +63,17 @@ local banner_pdf_2 = [[ \endinput ]] +local f_tounicode = formatters['\\pdfglyphtounicode{%s}{%04X}%%\n'] +local f_case = formatters['\\setXTXcharcodes "%05X "%05X "%05X %% %s\n'] +local f_range = formatters['\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharcodes\\recurselevel\\recurselevel\\recurselevel}\n'] +local f_classes = formatters['\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n'] +local f_charclass_a = formatters['\\defineXTXcharinjectionclass[lb:%s]\n'] +local f_charclass_b = formatters['\\dosetXTXcharacterclass{"%05X}{lb:%s}\n'] +local f_charclass_c = formatters['\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n'] +local f_hex = formatters['%s %05X"'] +local f_unicode = formatters['U+%05X'] +local f_entity = formatters[' ["%s"] = %q, -- %s'] + function scripts.chars.makepdfr() local chartable = resolvers.findfile("char-def.lua") or "" if chartable ~= "" then @@ -146,7 +87,7 @@ function scripts.chars.makepdfr() for i=1,#sd do local char = cd[sd[i]] if char.adobename then - f:write(format("\\pdfglyphtounicode{%s}{%04X}%%\n",char.adobename,char.unicodeslot)) + f:write(f_tounicode(char.adobename,char.unicodeslot)) end end f:write(banner_pdf_2) @@ -156,13 +97,13 @@ function scripts.chars.makepdfr() end end -local banner_utf_module = [[ +local banner_utf_module = formatters [ [[ %% filename : %s %% comment : generated by mtxrun --script chars --xtx %% author : Hans Hagen, PRAGMA-ADE, Hasselt NL %% copyright: PRAGMA ADE / ConTeXt Development Team %% license : see context related readme files -]] +]] ] local banner_utf_mappings = [[ @@ -219,7 +160,7 @@ function scripts.chars.makeencoutf() local f = io.open(name,'w') if f then report("writing '%s'",name) - f:write(format(banner_utf_module,name)) + f:write(banner_utf_module(name)) f:write(banner) f:write() return f @@ -232,7 +173,7 @@ function scripts.chars.makeencoutf() local data = characters and characters.data if data then local list = table.sortedkeys(characters.data) - local f = open("xetx-utf.tex",banner_utf_mappings) + local f = open("xetx-utf.mkii",banner_utf_mappings) if f then for i=1,#list do local code = list[i] @@ -240,9 +181,15 @@ function scripts.chars.makeencoutf() local chr = data[code] local cc = chr.category if cc == 'll' or cc == 'lu' or cc == 'lt' then - if not chr.lccode then chr.lccode = code end - if not chr.uccode then chr.uccode = code end - f:write(format('\\setXTXcharcodes "%05X "%05X "%05X %% %s\n',code,chr.lccode,chr.uccode,chr.description)) + local lccode = chr.lccode or code + local uccode = chr.uccode or code + if type(lccode) == "table" then + lccode = code + end + if type(uccode) == "table" then + uccode = code + end + f:write(f_case(code,lccode,uccode,chr.description)) end end end @@ -253,51 +200,57 @@ function scripts.chars.makeencoutf() if chr and chr.range then local cc = chr.category if cc == 'lo' then - f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharcodes\\recurselevel\\recurselevel\\recurselevel}\n',code,chr.range)) + f:write(f_range(code,chr.range)) end end end f:write(banner_utf_patch) close(f) end - local f = open("xetx-chr.tex",banner_utf_names) + local f = open("xetx-chr.mkii",banner_utf_names) if f then local length = 0 for i=1,#list do local code = list[i] if code > 0x5B and code <= 0xFFFF then local chr = data[code] - if chr and #(chr.contextname or "") > length then - length = #chr.contextname + if chr then + local l = #(chr.contextname or "") + if l > length then + length = l + end end end end - local template = "\\def\\%-".. length .. "s{\\char\"%05X } %% %s: %s\n" + local f_def = formatters["\\def\\%-".. length .. "s{\\char\"%05X } %% %s: %s\n"] for i=1,#list do local code = list[i] if code > 0x5B and code <= 0xFFFF then local chr = data[code] - if chr and chr.contextname then - local ch = utfchar(code) - f:write(format(template, chr.contextname, code, chr.description, ch)) + if chr then + local contextname = chr.contextname + if contextname and not find(contextname,"space$") then + local ch = utfchar(code) + f:write(f_def(contextname, code, chr.description, ch)) + end end end end close(f) end - local f = open("xetx-cls.tex",banner_utf_classes) + local f = open("xetx-cls.mkii",banner_utf_classes) if f then for k, v in next, xtxclasses do - f:write(format("\\defineXTXcharinjectionclass[lb:%s]\n",k)) + f:write(f_charclass_a(k)) end f:write("\n") local i_first, i_last, i_clb = nil, nil, nil local function flush() if i_first then if i_first == i_last then - f:write(format('\\dosetXTXcharacterclass{"%05X}{lb:%s}\n',i_first,i_clb)) + f:write(f_charclass_b(i_first,i_clb)) else - f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n',i_first,i_last,i_clb)) + f:write(f_charclass_c(i_first,i_last,i_clb)) end end i_first, i_last, i_clb = nil, nil, nil @@ -330,7 +283,7 @@ function scripts.chars.makeencoutf() if chr and chr.range then local lbc = chr.linebreak if xtxclasses[lbc] then - f:write(format('\\dofastrecurse{"%05X}{"%05X}{1}{\\dosetXTXcharacterclass\\fastrecursecounter{lb:%s}}\n',code,chr.range,lbc)) + f:write(f_classes(code,chr.range,lbc)) end end end @@ -362,11 +315,11 @@ function scripts.chars.xmlentities() if name == "newline" then -- let's forget about that one elseif name == "lt" then - str, hex = "<", format("%s %05X",hex,c) + str, hex = "<", f_hex(hex,c) elseif name == "gt" then - str, hex = ">", format("%s %05X",hex,c) + str, hex = ">", f_hex(hex,c) elseif name == "amp" then - str, hex = "&", format("%s %05X",hex,c) + str, hex = "&", f_hex(hex,c) else for t, c in gmatch(value,"&#([x]*)([^;]+);") do if t == "x" then @@ -375,14 +328,14 @@ function scripts.chars.xmlentities() c = tonumber(c) end if str then - str, hex = str .. utfchar(c), format("%s %05X",hex,c) + str, hex = str .. utfchar(c), f_hex(hex,c) else - str, hex = utfchar(c), format("U+%05X",c) + str, hex = utfchar(c), f_unicode(c) end end end if str and hex then - entities[#entities+1] = format(' ["%s"] = %q, -- %s',name,str,hex) + entities[#entities+1] = f_entity(name,str,hex) end end end @@ -406,28 +359,3 @@ elseif environment.argument("exporthelp") then else application.help() end - --- local http = require("socket.http") --- local ltn12 = require("ltn12") --- --- local t = { } --- local status, message = http.request { --- url = f, --- sink = ltn12.sink.table(t) --- } --- --- local template = [[ --- <?xml version='1.0' ?> --- --- <!DOCTYPE dummy [ --- --- %s --- --- ]> --- --- <dummy>This is just a placeholder.</dummy> --- ]] --- --- local e = string.format(template,io.loaddata(n)) --- local x = xml.convert(e, { utfize_entities = true } ) --- local entities = x.entities diff --git a/scripts/context/lua/mtx-check.lua b/scripts/context/lua/mtx-check.lua index c456b4414..16b9cb64a 100644 --- a/scripts/context/lua/mtx-check.lua +++ b/scripts/context/lua/mtx-check.lua @@ -114,7 +114,7 @@ local contextgrammar = P { "tokens", ["start"] = start, ["stop"] = stop, ["whatever"] = line + esc * 1 + C(P("%") * (1-line)^0), - ["grouped"] = l_g * (V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + line + (1 - l_g - r_g))^0 * r_g, + ["grouped"] = l_g * (V("start") + V("stop") + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + line + (1 - l_g - r_g))^0 * r_g, ["setup"] = l_s * (okay + V("whatever") + V("grouped") + V("setup") + V("display") + V("inline") + (1 - l_s - r_s))^0 * r_s, ["display"] = d_m * (V("whatever") + V("grouped") + (1 - d_m))^0 * d_m, ["inline"] = i_m * (V("whatever") + V("grouped") + (1 - i_m))^0 * i_m, diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index d624f6831..84254d570 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -34,7 +34,7 @@ local formatters = string.formatters local application = logs.application { name = "mtx-context", - banner = "ConTeXt Process Management 0.61", + banner = "ConTeXt Process Management 0.63", -- helpinfo = helpinfo, -- table with { category_a = text_1, category_b = text_2 } or helpstring or xml_blob helpinfo = "mtx-context.xml", } @@ -44,6 +44,8 @@ local application = logs.application { -- ["default-translate-file"] = true, -- ignored, input is assumed to be in UTF-8 encoding -- ["translate-file"] = true, -- ignored, input is assumed to be in UTF-8 encoding -- ["etex"] = true, -- ignored, the etex extensions are always active +-- ["parse-first-line"] = true, -- ignored, enable parsing of the first line of the input file +-- ["no-parse-first-line"] = true, -- ignored, disable parsing of the first line of the input file -- -- ["credits"] = true, -- display credits and exit -- ["debug-format"] = true, -- enable format debugging @@ -68,14 +70,14 @@ local application = logs.application { -- ["output-comment"] = true, -- use STRING for DVI file comment instead of date (no effect for PDF) -- ["output-directory"] = true, -- use existing DIR as the directory to write files in -- ["output-format"] = true, -- use FORMAT for job output; FORMAT is 'dvi' or 'pdf' --- ["parse-first-line"] = true, -- enable parsing of the first line of the input file --- ["no-parse-first-line"] = true, -- disable parsing of the first line of the input file -- ["progname"] = true, -- set the program name to STRING -- ["recorder"] = true, -- enable filename recorder -- ["safer"] = true, -- disable easily exploitable lua commands -- ["shell-escape"] = true, -- enable \write18{SHELL COMMAND} -- ["no-shell-escape"] = true, -- disable \write18{SHELL COMMAND} -- ["shell-restricted"] = true, -- restrict \write18 to a list of commands given in texmf.cnf +-- ["nodates"] = true, -- no production dates in pdf file +-- ["trailerid"] = true, -- alternative trailer id -- ["synctex"] = true, -- enable synctex -- ["version"] = true, -- display version and exit -- ["luaonly"] = true, -- run a lua file, then exit @@ -106,7 +108,7 @@ local function restart(engine_old,engine_new) local command = format("%s --luaonly %q %s --redirected",engine_new,environment.ownname,environment.reconstructcommandline()) report(format("redirect %s -> %s: %s",engine_old,engine_new,command)) local result = os.execute(command) - os.exit(result) + os.exit(result == 0 and 0 or 1) end if getargument("redirected") then @@ -173,7 +175,7 @@ end function ctxrunner.checkfile(ctxdata,ctxname,defaultname) - if not ctxdata.jobname or ctxdata.jobname == "" then + if not ctxdata.jobname or ctxdata.jobname == "" or getargument("noctx") then return end @@ -284,7 +286,7 @@ end local f_tempfile = formatters["%s-%s-%02d.tmp"] -local function backup(run,kind,filename) +local function backup(jobname,run,kind,filename) if run == 1 then for i=1,10 do local tmpname = f_tempfile(jobname,kind,i) @@ -307,7 +309,7 @@ local function multipass_copyluafile(jobname,run) local tuaname, tucname = jobname..".tua", jobname..".tuc" if validfile(tuaname) then if run then - backup(run,"tuc",tucname) + backup(jobname,run,"tuc",tucname) report("copying %a into %a",tuaname,tucname) report() end @@ -320,7 +322,7 @@ local function multipass_copylogfile(jobname,run) local logname = jobname..".log" if validfile(logname) then if run then - backup(run,"log",logname) + backup(jobname,run,"log",logname) report() end end @@ -570,26 +572,30 @@ function scripts.context.run(ctxdata,filename) return end -- - local a_mkii = getargument("mkii") or getargument("pdftex") or getargument("xetex") - local a_purge = getargument("purge") - local a_purgeall = getargument("purgeall") - local a_purgeresult = getargument("purgeresult") - local a_global = getargument("global") - local a_timing = getargument("timing") - local a_profile = getargument("profile") - local a_batchmode = getargument("batchmode") - local a_nonstopmode = getargument("nonstopmode") - local a_scollmode = getargument("scrollmode") - local a_once = getargument("once") - local a_synctex = getargument("synctex") - local a_backend = getargument("backend") - local a_arrange = getargument("arrange") - local a_noarrange = getargument("noarrange") - local a_jiton = getargument("jiton") - local a_jithash = getargument("jithash") - local a_texformat = getargument("texformat") - local a_keeptuc = getargument("keeptuc") - local a_keeplog = getargument("keeplog") + local a_mkii = getargument("mkii") or getargument("pdftex") or getargument("xetex") + local a_purge = getargument("purge") + local a_purgeall = getargument("purgeall") + local a_purgeresult = getargument("purgeresult") + local a_global = getargument("global") + local a_timing = getargument("timing") + local a_profile = getargument("profile") + local a_batchmode = getargument("batchmode") + local a_nonstopmode = getargument("nonstopmode") + local a_scollmode = getargument("scrollmode") + local a_once = getargument("once") + local a_synctex = getargument("synctex") + local a_backend = getargument("backend") + local a_arrange = getargument("arrange") + local a_noarrange = getargument("noarrange") + local a_jiton = getargument("jiton") + local a_jithash = getargument("jithash") + local a_texformat = getargument("texformat") + local a_keeptuc = getargument("keeptuc") + local a_keeplog = getargument("keeplog") + local a_export = getargument("export") + local a_nodates = getargument("nodates") + local a_trailerid = getargument("trailerid") + local a_nocompression = getargument("nocompression") -- the following flag is not officially supported because i cannot forsee -- side effects (so no bug reports please) .. we provide --sandbox that @@ -702,20 +708,23 @@ function scripts.context.run(ctxdata,filename) scripts.context.make(formatname) end -- - local oldhash = multipass_hashfiles(jobname) - local newhash = { } - local maxnofruns = once and 1 or multipass_nofruns + local oldhash = multipass_hashfiles(jobname) + local newhash = { } + local maxnofruns = once and 1 or multipass_nofruns + local fulljobname = validstring(filename) -- local c_flags = { - directives = directives, -- gets passed via mtxrun - trackers = trackers, -- gets passed via mtxrun - experiments = experiments, -- gets passed via mtxrun + directives = directives, -- gets passed via mtxrun + trackers = trackers, -- gets passed via mtxrun + experiments = experiments, -- gets passed via mtxrun -- - result = validstring(resultname), - input = validstring(getargument("input") or filename), -- alternative input - fulljobname = validstring(filename), - files = concat(files,","), - ctx = validstring(ctxname), + result = validstring(resultname), + input = validstring(getargument("input") or filename), -- alternative input + fulljobname = fulljobname, + files = concat(files,","), + ctx = validstring(ctxname), + export = a_export and true or nil, + nocompression = a_nocompression and true or nil, } -- for k, v in next, environment.arguments do @@ -725,11 +734,12 @@ function scripts.context.run(ctxdata,filename) end end -- + -- todo: --output-file=... in luatex -- local l_flags = { ["interaction"] = a_batchmode, ["synctex"] = a_synctex, - ["no-parse-first-line"] = true, + ["no-parse-first-line"] = true, -- obsolete ["safer"] = a_safer, -- ["no-mktex"] = true, -- ["file-line-error-style"] = true, @@ -748,21 +758,26 @@ function scripts.context.run(ctxdata,filename) c_flags.usemodule = "timing" end -- - if not a_profile then - -- okay - elseif c_flags.directives then - c_flags.directives = format("system.profile,%s",c_flags.directives) - else - c_flags.directives = "system.profile" + local directives = { } + -- + if a_nodates then + directives[#directives+1] = format("backend.date=%s",type(a_nodates) == "string" and a_nodates or " no") + end + -- + if a_trailerid then + directives[#directives+1] = format("backend.trailerid=%s",a_trailerid) + end + -- + if a_profile then + directives[#directives+1] = "system.profile" end -- if a_synctex then report("warning: synctex is enabled") -- can add upto 5% runtime - if c_flags.directives then - c_flags.directives = format("system.synctex=%s,%s",a_synctex,c_flags.directives) - else - c_flags.directives = format("system.synctex=%s",a_synctex) - end + directives[#directives+1] = format("system.synctex=%s",a_synctex) + end + if #directives > 0 then + c_flags.directives = concat(directives,",") end -- -- kindofrun: 1:first run, 2:successive run, 3:once, 4:last of maxruns @@ -836,9 +851,9 @@ function scripts.context.run(ctxdata,filename) end -- if a_purge then - scripts.context.purge_job(jobname) + scripts.context.purge_job(jobname,false,false,fulljobname) elseif a_purgeall then - scripts.context.purge_job(jobname,true) + scripts.context.purge_job(jobname,true,false,fulljobname) end -- if resultname then @@ -852,11 +867,13 @@ function scripts.context.run(ctxdata,filename) report("result renamed to: %s",newbase) end -- - if purge then - scripts.context.purge_job(resultname) - elseif purgeall then - scripts.context.purge_job(resultname,true) - end + -- -- needs checking + -- + -- if a_purge then + -- scripts.context.purge_job(resultname) + -- elseif a_purgeall then + -- scripts.context.purge_job(resultname,true) + -- end -- local pdfview = getargument("autopdf") if pdfview then @@ -958,13 +975,122 @@ function scripts.context.pipe() -- still used? end local function make_mkiv_format(name,engine) - environment.make_format(name) -- jit is picked up later + environment.make_format(name,environment.arguments.silent) -- jit is picked up later end -local function make_mkii_format(name,engine) - local command = format("mtxrun texexec.rb --make %s --%s",name,engine) - report("running command: %s",command) - os.spawn(command) +local make_mkii_format + +do -- more or less copied from mtx-plain.lua: + + local function mktexlsr() + if environment.arguments.silent then + local result = os.execute("mktexlsr --quiet > temp.log") + if result ~= 0 then + print("mktexlsr silent run > fatal error") -- we use a basic print + else + print("mktexlsr silent run") -- we use a basic print + end + os.remove("temp.log") + else + report("running mktexlsr") + os.execute("mktexlsr") + end + end + + local function engine(texengine,texformat) + local command = string.format('%s --ini --etex --8bit %s \\dump',texengine,file.addsuffix(texformat,"mkii")) + if environment.arguments.silent then + statistics.starttiming() + local command = format("%s > temp.log",command) + local result = os.execute(command) + local runtime = statistics.stoptiming() + if result ~= 0 then + print(format("%s silent make > fatal error when making format %q",texengine,texformat)) -- we use a basic print + else + print(format("%s silent make > format %q made in %.3f seconds",texengine,texformat,runtime)) -- we use a basic print + end + os.remove("temp.log") + else + report("running command: %s",command) + os.execute(command) + end + end + + local function resultof(...) + local command = string.format(...) + report("running command %a",command) + return string.strip(os.resultof(command) or "") + end + + local function make(texengine,texformat) + report("generating kpse file database") + mktexlsr() + local fmtpathspec = resultof("kpsewhich --var-value=TEXFORMATS --engine=%s",texengine) + if fmtpathspec ~= "" then + report("using path specification %a",fmtpathspec) + fmtpathspec = resultof('kpsewhich -expand-braces="%s"',fmtpathspec) + end + if fmtpathspec ~= "" then + report("using path expansion %a",fmtpathspec) + else + report("no valid path reported, trying alternative") + fmtpathspec = resultof("kpsewhich --show-path=fmt --engine=%s",texengine) + if fmtpathspec ~= "" then + report("using path expansion %a",fmtpathspec) + else + report("no valid path reported, falling back to current path") + fmtpathspec = "." + end + end + fmtpathspec = string.splitlines(fmtpathspec)[1] or fmtpathspec + fmtpathspec = fmtpathspec and file.splitpath(fmtpathspec) + local fmtpath = nil + if fmtpathspec then + for i=1,#fmtpathspec do + local path = fmtpathspec[i] + if path ~= "." then + dir.makedirs(path) + if lfs.isdir(path) and file.is_writable(path) then + fmtpath = path + break + end + end + end + end + if not fmtpath or fmtpath == "" then + fmtpath = "." + else + lfs.chdir(fmtpath) + end + engine(texengine,texformat) + report("generating kpse file database") + mktexlsr() + report("format %a saved on path %a",texformat,fmtpath) + end + + local function run(texengine,texformat,filename) + local t = { } + for k, v in next, environment.arguments do + t[#t+1] = string.format("--mtx:%s=%s",k,v) + end + execute('%s --fmt=%s %s "%s"',texengine,file.removesuffix(texformat),table.concat(t," "),filename) + end + + make_mkii_format = function(name,engine) + + -- let the binary sort it out + + os.setenv('SELFAUTOPARENT', "") + os.setenv('SELFAUTODIR', "") + os.setenv('SELFAUTOLOC', "") + os.setenv('TEXROOT', "") + os.setenv('TEXOS', "") + os.setenv('TEXMFOS', "") + os.setenv('TEXMFCNF', "") + + make(engine,name) + end + end function scripts.context.generate() @@ -1122,6 +1248,9 @@ local temporary_runfiles = { "aux", "blg", -- bibtex } +local temporary_suffixes = { + "prep", -- context preprocessed +} local synctex_runfiles = { "synctex", "synctex.gz", -- synctex } @@ -1150,7 +1279,7 @@ local function purge_file(dfile,cfile) end end -function scripts.context.purge_job(jobname,all,mkiitoo) +function scripts.context.purge_job(jobname,all,mkiitoo,fulljobname) if jobname and jobname ~= "" then jobname = filebasename(jobname) local filebase = removesuffix(jobname) @@ -1164,6 +1293,11 @@ function scripts.context.purge_job(jobname,all,mkiitoo) for i=1,#temporary_runfiles do deleted[#deleted+1] = purge_file(fileaddsuffix(filebase,temporary_runfiles[i])) end + if fulljobname and fulljobname ~= jobname then + for i=1,#temporary_suffixes do + deleted[#deleted+1] = purge_file(fileaddsuffix(fulljobname,temporary_suffixes[i],true)) + end + end if not environment.argument("synctex") then -- special case: not deleted when --synctex is given, but what if given in preamble for i=1,#synctex_runfiles do @@ -1445,7 +1579,7 @@ function scripts.context.update() report("quiting, no 'context.mkiv' found") return end - local basetree = basepath.match(basepath,"^(.-)tex/context/base/context.mkiv$") or "" + local basetree = basepath.match(basepath,"^(.-)tex/context/base/.*context.mkiv$") or "" if basetree == "" then report("quiting, no proper tds structure (%s)",basepath) return @@ -1487,7 +1621,7 @@ function scripts.context.update() report("quiting, unable to open '%s'",zipname) return end - local newfile = zip.loaddata(zipfile,"tex/context/base/context.mkiv") + local newfile = zip.loaddata(zipfile,"tex/context/base/mkiv/context.mkiv") if not newfile then report("quiting, unable to open '%s'","context.mkiv") return @@ -1600,7 +1734,7 @@ elseif getargument("make") then scripts.context.timed(function() scripts.context.make() end) elseif getargument("generate") then scripts.context.timed(function() scripts.context.generate() end) -elseif getargument("ctx") then +elseif getargument("ctx") and not getargument("noctx") then scripts.context.timed(scripts.context.ctx) -- elseif getargument("mp") or getargument("metapost") then -- scripts.context.timed(scripts.context.metapost) diff --git a/scripts/context/lua/mtx-context.xml b/scripts/context/lua/mtx-context.xml index c41093289..2ba7ee59e 100644 --- a/scripts/context/lua/mtx-context.xml +++ b/scripts/context/lua/mtx-context.xml @@ -4,7 +4,7 @@ <metadata> <entry name="name">mtx-context</entry> <entry name="detail">ConTeXt Process Management</entry> - <entry name="version">0.60</entry> + <entry name="version">0.62</entry> <entry name="comment">external helpinfo file</entry> </metadata> <flags> @@ -21,6 +21,9 @@ <flag name="ctx=name"> <short>use ctx file (process management specification)</short> </flag> + <flag name="noctx"> + <short>ignore ctx directives and flags</short> + </flag> <flag name="interface"> <short>use specified user interface (default: en)</short> </flag> @@ -125,6 +128,15 @@ <flag name="synctex"> <short>run with synctex enabled (optional value: zipped, unzipped, 1, -1)</short> </flag> + <flag name="nodates"> + <short>omit runtime dates in pdf file (optional value: a number (this 1970 offset time) or string "YYYY-MM-DD HH:MM")</short> + </flag> + <flag name="nocompression"> + <short>forcefully turns off compression in the backend</short> + </flag> + <flag name="trailerid"> + <short>alternative trailer id (or constant one)</short> + </flag> </subcategory> <subcategory> <flag name="generate"> diff --git a/scripts/context/lua/mtx-convert.lua b/scripts/context/lua/mtx-convert.lua index d5dba075a..b3e20ea87 100644 --- a/scripts/context/lua/mtx-convert.lua +++ b/scripts/context/lua/mtx-convert.lua @@ -49,6 +49,12 @@ convert.converters = convert.converters or { } local converters = convert.converters local gsprogram = (os.type == "windows" and (os.which("gswin64c.exe") or os.which("gswin32c.exe"))) or "gs" + +if string.find(gsprogram," ") then + -- c:/program files/...../gswinNNc.exe" + gsprogram = '"' .. gsprogram .. '"' +end + local gstemplate_eps = "%s -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -dEPSCrop -dNOPAUSE -dSAFER -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" local gstemplate_ps = "%s -q -sDEVICE=pdfwrite -dPDFSETTINGS=/prepress -dNOPAUSE -dSAFER -dNOCACHE -dBATCH -dAutoRotatePages=/None -dProcessColorModel=/DeviceCMYK -sOutputFile=%s %s -c quit" diff --git a/scripts/context/lua/mtx-fonts.lua b/scripts/context/lua/mtx-fonts.lua index 694e6a649..0c3224fe0 100644 --- a/scripts/context/lua/mtx-fonts.lua +++ b/scripts/context/lua/mtx-fonts.lua @@ -10,6 +10,13 @@ local getargument = environment.getargument local setargument = environment.setargument local givenfiles = environment.files +local suffix, addsuffix, removesuffix, replacesuffix = file.suffix, file.addsuffix, file.removesuffix, file.replacesuffix +local nameonly, basename, joinpath, collapsepath = file.nameonly, file.basename, file.join, file.collapsepath +local lower = string.lower + +local otfversion = 2.825 +local otlversion = 3.020 + local helpinfo = [[ <?xml version="1.0"?> <application> @@ -21,7 +28,8 @@ local helpinfo = [[ <flags> <category name="basic"> <subcategory> - <flag name="save"><short>save open type font in raw table</short></flag> + <flag name="save"><short>save open type font in raw table (ff format)</short></flag> + <flag name="convert"><short>save open type font in raw table (ctx format)</short></flag> <flag name="unpack"><short>save a tma file in a more readable format</short></flag> </subcategory> <subcategory> @@ -40,6 +48,8 @@ local helpinfo = [[ <flag name="info"><short>give more details</short></flag> <flag name="trackers" value="list"><short>enable trackers</short></flag> <flag name="statistics"><short>some info about the database</short></flag> + <flag name="names"><short>use name instead of unicodes</short></flag> + <flag name="cache" value="str"><short>use specific cache (otl or otf)</short></flag> </subcategory> </category> </flags> @@ -67,6 +77,11 @@ local helpinfo = [[ <example><command>mtxrun --script font --list --file --all somename</command></example> <example><command>mtxrun --script font --list --file --pattern=*somename*</command></example> </subcategory> + <subcategory> + <example><command>mtxrun --script font --save texgyrepagella-regular.otf</command></example> + <example><command>mtxrun --script font --convert texgyrepagella-regular.otf</command></example> + <example><command>mtxrun --script font --convert --names texgyrepagella-regular.otf</command></example> + </subcategory> </category> </examples> </application> @@ -84,10 +99,35 @@ local report = application.report if not fontloader then fontloader = fontforge end -dofile(resolvers.findfile("font-otp.lua","tex")) -- we need to unpack the font for analysis -dofile(resolvers.findfile("font-syn.lua","tex")) -dofile(resolvers.findfile("font-trt.lua","tex")) -dofile(resolvers.findfile("font-mis.lua","tex")) +local function loadmodule(filename) + local fullname = resolvers.findfile(filename,"tex") + if fullname and fullname ~= "" then + dofile(fullname) + end +end + +-- old loader code + +loadmodule("font-otp.lua") -- we need to unpack the font for analysis + +-- new loader code + +loadmodule("char-def.lua") + +loadmodule("font-otr.lua") +loadmodule("font-cff.lua") +loadmodule("font-ttf.lua") +loadmodule("font-tmp.lua") +loadmodule("font-dsp.lua") -- not yet in distribution +loadmodule("font-oup.lua") -- not yet in distribution + +loadmodule("font-onr.lua") + +-- extra code + +loadmodule("font-syn.lua") +loadmodule("font-trt.lua") +loadmodule("font-mis.lua") scripts = scripts or { } scripts.fonts = scripts.fonts or { } @@ -134,9 +174,9 @@ end function fonts.names.simple(alsotypeone) local simpleversion = 1.001 - local simplelist = { "ttf", "otf", "ttc", "dfont", alsotypeone and "afm" or nil } + local simplelist = { "ttf", "otf", "ttc", alsotypeone and "afm" or nil } local name = "luatex-fonts-names.lua" - local path = file.collapsepath(caches.getwritablepath("..","..","generic","fonts","data")) + local path = collapsepath(caches.getwritablepath("..","..","generic","fonts","data")) fonts.names.filters.list = simplelist fonts.names.version = simpleversion -- this number is the same as in font-dum.lua report("generating font database for 'luatex-fonts' version %s",fonts.names.version) @@ -154,7 +194,7 @@ function fonts.names.simple(alsotypeone) local format = simplelist[i] for tag, index in next, data.mappings[format] do local s = specifications[index] - simplemappings[tag] = { s.rawname, s.filename, s.subfont } + simplemappings[tag] = { s.rawname or nameonly(s.filename), s.filename, s.subfont } end end if environment.arguments.nocache then @@ -163,7 +203,7 @@ function fonts.names.simple(alsotypeone) dir.mkdirs(path) if lfs.isdir(path) then report("saving names on cache path %a",path) - name = file.join(path,name) + name = joinpath(path,name) else report("invalid cache path %a",path) end @@ -204,21 +244,28 @@ local function fontweight(fw) end end +local function indeed(f,s) + if s and s ~= "" then + report(f,s) + end +end + local function showfeatures(tag,specification) report() - report("mapping : %s",tag) - report("fontname: %s",specification.fontname) - report("fullname: %s",specification.fullname) - report("filename: %s",specification.filename) - report("family : %s",specification.familyname or "<nofamily>") - report("weight : %s",specification.weight or "<noweight>") - report("style : %s",specification.style or "<nostyle>") - report("width : %s",specification.width or "<nowidth>") - report("variant : %s",specification.variant or "<novariant>") - report("subfont : %s",subfont(specification.subfont)) - report("fweight : %s",fontweight(specification.fontweight)) + indeed("mapping : %s",tag) + indeed("fontname : %s",specification.fontname) + indeed("fullname : %s",specification.fullname) + indeed("filename : %s",specification.filename) + indeed("family : %s",specification.familyname or "<nofamily>") + -- indeed("subfamily : %s",specification.subfamilyname or "<nosubfamily>") + indeed("weight : %s",specification.weight or "<noweight>") + indeed("style : %s",specification.style or "<nostyle>") + indeed("width : %s",specification.width or "<nowidth>") + indeed("variant : %s",specification.variant or "<novariant>") + indeed("subfont : %s",subfont(specification.subfont)) + indeed("fweight : %s",fontweight(specification.fontweight)) -- maybe more - local features = fonts.helpers.getfeatures(specification.filename,specification.format) + local features = fonts.helpers.getfeatures(specification.filename,not getargument("nosave")) if features then for what, v in table.sortedhash(features) do local data = features[what] @@ -270,11 +317,12 @@ local function list_specifications(t,info) local v = s[k] local entry = t[v] s[k] = { - entry.familyname or "<nofamily>", - entry.weight or "<noweight>", - entry.style or "<nostyle>", - entry.width or "<nowidth>", - entry.variant or "<novariant>", + entry.familyname or "<nofamily>", + -- entry.subfamilyname or "<nosubfamily>", + entry.weight or "<noweight>", + entry.style or "<nostyle>", + entry.width or "<nowidth>", + entry.variant or "<novariant>", entry.fontname, entry.filename, subfont(entry.subfont), @@ -392,18 +440,23 @@ function scripts.fonts.justload() end function scripts.fonts.unpack() - local name = file.removesuffix(file.basename(givenfiles[1] or "")) + local name = removesuffix(basename(givenfiles[1] or "")) if name and name ~= "" then - local cache = containers.define("fonts", "otf", 2.742, true) + local cacheid = getargument("cache") or "otl" + local cache = containers.define("fonts", cacheid, otlversion, true) -- cache is temp local cleanname = containers.cleanname(name) local data = containers.read(cache,cleanname) if data then - local savename = file.addsuffix(cleanname .. "-unpacked","tma") + local savename = addsuffix(cleanname .. "-unpacked","tma") report("fontsave, saving data in %s",savename) - fonts.handlers.otf.enhancers.unpack(data) + if data.creator == "context mkiv" then + fonts.handlers.otf.readers.unpack(data) + else + fonts.handlers.otf.enhancers.unpack(data) + end io.savedata(savename,table.serialize(data,true)) else - report("unknown file %a",name) + report("unknown file %a in cache %a",name,cacheid) end end end @@ -415,9 +468,9 @@ function scripts.fonts.save() if fontblob then if fontblob.validation_state and table.contains(fontblob.validation_state,"bad_ps_fontname") then report("ignoring bad fontname for %a",name) - savename = file.nameonly(name) .. "-bad-ps-name" + savename = nameonly(name) .. "-bad-ps-name" end - savename = file.addsuffix(string.lower(savename),"lua") + savename = addsuffix(lower(savename),"lua") report("fontsave, saving data in %a",savename) table.tofile(savename,fontloader.to_table(fontblob),"return") fontloader.close(fontblob) @@ -426,7 +479,7 @@ function scripts.fonts.save() if name and name ~= "" then local filename = resolvers.findfile(name) -- maybe also search for opentype if filename and filename ~= "" then - local suffix = string.lower(file.suffix(filename)) + local suffix = lower(suffix(filename)) if suffix == 'ttf' or suffix == 'otf' or suffix == 'ttc' or suffix == "dfont" then local fontinfo = fontloader.info(filename) if fontinfo then @@ -453,6 +506,36 @@ function scripts.fonts.save() end end +function scripts.fonts.convert() -- new save + local name = givenfiles[1] or "" + local sub = givenfiles[2] or "" + if name and name ~= "" then + local filename = resolvers.findfile(name) -- maybe also search for opentype + if filename and filename ~= "" then + local suffix = lower(suffix(filename)) + if suffix == 'ttf' or suffix == 'otf' or suffix == 'ttc' then + local data = fonts.handlers.otf.readers.loadfont(filename,sub) + if data then + fonts.handlers.otf.readers.compact(data) + fonts.handlers.otf.readers.rehash(data,getargument("names") and "names" or "unicodes") + local savename = replacesuffix(lower(data.metadata.fullname or filename),"lua") + table.save(savename,data) + report("font: %a saved as %a",filename,savename) + else + report("font: %a not loaded",filename) + end + else + report("font: %a not saved",filename) + end + else + report("font: %a not found",name) + end + else + report("font: no name given") + end +end + + if getargument("names") then setargument("reload",true) setargument("simple",true) @@ -464,6 +547,8 @@ elseif getargument("reload") then scripts.fonts.reload() elseif getargument("save") then scripts.fonts.save() +elseif getargument("convert") then + scripts.fonts.convert() elseif getargument("justload") then scripts.fonts.justload() elseif getargument("unpack") then diff --git a/scripts/context/lua/mtx-interface.lua b/scripts/context/lua/mtx-interface.lua index 1640f0891..b82dd5abf 100644 --- a/scripts/context/lua/mtx-interface.lua +++ b/scripts/context/lua/mtx-interface.lua @@ -21,9 +21,7 @@ local helpinfo = [[ <flags> <category name="basic"> <subcategory> - <flag name="interfaces"><short>generate context interface files</short></flag> - <flag name="messages"><short>generate context message files</short></flag> - <flag name="labels"><short>generate context label files</short></flag> + <flag name="interfaces"><short>generate context mkii interface files</short></flag> </subcategory> <subcategory> <flag name="context"><short>equals <ref name="interfaces"/> <ref name="messages"/> <ref name="languages"/></short></flag> @@ -228,87 +226,235 @@ function flushers.textpad(collected) end end +-- function scripts.interface.editor(editor,split,forcedinterfaces) +-- local interfaces= forcedinterfaces or environment.files +-- if #interfaces == 0 then +-- interfaces= userinterfaces +-- end +-- local xmlfile = resolvers.findfile("cont-en.xml") or "" +-- if xmlfile == "" then +-- report("unable to locate cont-en.xml") +-- end +-- local collected = { } +-- for i=1,#interfaces do +-- local interface = interfaces[i] +-- local keyfile = resolvers.findfile(format("keys-%s.xml",interface)) or "" +-- if keyfile == "" then +-- report("unable to locate keys-*.xml") +-- else +-- local commands = { } +-- local mappings = { } +-- local environments = { } +-- local x = xml.load(keyfile) +-- for e, d, k in xml.elements(x,"/cd:interface/cd:commands/cd:command") do -- somehow this was variable +-- local at = d[k].at +-- local name, value = at.name, at.value +-- if name and value then +-- mappings[name] = value +-- end +-- end +-- local x = xml.load(xmlfile) +-- for e, d, k in xml.elements(x,"/cd:interface/cd:command") do +-- local at = d[k].at +-- local name, type = at.name, at["type"] +-- if name and name ~= "" then +-- local remapped = mappings[name] or name +-- if type == "environment" then +-- if split then +-- environments[#environments+1] = remapped +-- else +-- commands[#commands+1] = "start" .. remapped +-- commands[#commands+1] = "stop" .. remapped +-- end +-- else +-- commands[#commands+1] = remapped +-- end +-- end +-- end +-- if #commands > 0 then +-- sort(commands) +-- sort(environments) +-- collected[interface] = { +-- commands = commands, +-- environments = environments, +-- } +-- end +-- end +-- end +-- -- awaiting completion of the xml file +-- local definitions = dofile(resolvers.findfile("mult-def.lua")) +-- if definitions then +-- local commands = { en = { } } +-- for command, languages in next, definitions.commands do +-- commands.en[languages.en or command] = true +-- for language, command in next, languages do +-- local c = commands[language] +-- if c then +-- c[command] = true +-- else +-- commands[language] = { [command] = true } +-- end +-- end +-- end +-- for language, data in next, commands do +-- local fromlua = data +-- local fromxml = collected[language].commands +-- for i=1,#fromxml do +-- local c = fromxml[i] +-- if not fromlua[c] then +-- -- print(language,c) +-- fromlua[c] = true +-- end +-- end +-- collected[language].commands = table.sortedkeys(fromlua) +-- end +-- end +-- -- +-- flushers[editor](collected) +-- end + +-- function scripts.interface.editor(editor,split,forcedinterfaces) +-- local interfaces= forcedinterfaces or environment.files +-- if #interfaces == 0 then +-- interfaces= userinterfaces +-- end +-- -- +-- local filename = "context-en.xml" +-- local xmlfile = resolvers.findfile(filename) or "" +-- if xmlfile == "" then +-- report("unable to locate %a",filename) +-- return +-- end +-- local x = xml.load(xmlfile) +-- -- +-- local filename = "mult-def.lua" +-- local deffile = resolvers.findfile(filename) or "" +-- if deffile == "" then +-- report("unable to locate %a",filename) +-- return +-- end +-- local interface = dofile(filename) +-- if not interface or not next(interface) then +-- report("invalid file %a",filename) +-- return +-- end +-- local variables = interface.variables +-- local constants = interface.constants +-- local commands = interface.commands +-- local elements = interface.elements +-- local collected = { } +-- for i=1,#interfaces do +-- local interface = interfaces[i] +-- local i_commands = { } +-- local i_environments = { } +-- local start = elements.start[interface] or elements.start.en +-- local stop = elements.stop [interface] or elements.stop .en +-- for e, d, k in xml.elements(x,"cd:interface/cd:command") do +-- local at = d[k].at +-- local name = at["name"] or "" +-- local type = at["type"] +-- if name ~= "" then +-- local c = commands[name] +-- local n = c and (c[interface] or c.en) or name +-- if type ~= "environment" then +-- i_commands[#i_commands+1] = n +-- elseif split then +-- i_environments[#i_environments+1] = n +-- else +-- -- variables ? +-- i_commands[#i_commands+1] = start .. n +-- i_commands[#i_commands+1] = stop .. n +-- end +-- end +-- end +-- if #i_commands > 0 then +-- sort(i_commands) +-- sort(i_environments) +-- collected[interface] = { +-- commands = i_commands, +-- environments = i_environments, +-- } +-- end +-- end +-- -- +-- flushers[editor](collected) +-- end + function scripts.interface.editor(editor,split,forcedinterfaces) local interfaces= forcedinterfaces or environment.files if #interfaces == 0 then interfaces= userinterfaces end - local xmlfile = resolvers.findfile("cont-en.xml") or "" + -- + local filename = "i-context.xml" + local xmlfile = resolvers.findfile(filename) or "" if xmlfile == "" then - report("unable to locate cont-en.xml") + report("unable to locate %a",filename) + return + end + -- + local filename = "mult-def.lua" + local deffile = resolvers.findfile(filename) or "" + if deffile == "" then + report("unable to locate %a",filename) + return end + local interface = dofile(deffile) + if not interface or not next(interface) then + report("invalid file %a",filename) + return + end + local variables = interface.variables + local constants = interface.constants + local commands = interface.commands + local elements = interface.elements + -- local collected = { } - for i=1,#interfaces do - local interface = interfaces[i] - local keyfile = resolvers.findfile(format("keys-%s.xml",interface)) or "" - if keyfile == "" then - report("unable to locate keys-*.xml") - else - local commands = { } - local mappings = { } - local environments = { } - local x = xml.load(keyfile) - for e, d, k in xml.elements(x,"/cd:interface/cd:commands/cd:command") do -- somehow this was variable - local at = d[k].at - local name, value = at.name, at.value - if name and value then - mappings[name] = value - end - end - local x = xml.load(xmlfile) - for e, d, k in xml.elements(x,"/cd:interface/cd:command") do - local at = d[k].at - local name, type = at.name, at["type"] - if name and name ~= "" then - local remapped = mappings[name] or name - if type == "environment" then - if split then - environments[#environments+1] = remapped - else - commands[#commands+1] = "start" .. remapped - commands[#commands+1] = "stop" .. remapped - end - else - commands[#commands+1] = remapped - end - end - end - if #commands > 0 then - sort(commands) - sort(environments) - collected[interface] = { - commands = commands, - environments = environments, - } - end + -- + report("generating files for %a",editor) + report("loading %a",xmlfile) + local xmlroot = xml.load(xmlfile) + xml.include(xmlroot,"cd:interfacefile","filename",true,function(s) + local fullname = resolvers.findfile(s) + if fullname and fullname ~= "" then + report("including %a",fullname) + return io.loaddata(fullname) end - end - -- awaiting completion of the xml file - local definitions = dofile(resolvers.findfile("mult-def.lua")) - if definitions then - local commands = { en = { } } - for command, languages in next, definitions.commands do - commands.en[languages.en or command] = true - for language, command in next, languages do - local c = commands[language] - if c then - c[command] = true + end) + -- + for i=1,#interfaces do + local interface = interfaces[i] + local i_commands = { } + local i_environments = { } + local start = elements.start[interface] or elements.start.en + local stop = elements.stop [interface] or elements.stop .en + for e in xml.collected(xmlroot,"cd:interface/cd:command") do + local at = e.at + local name = at["name"] or "" + local type = at["type"] + if name ~= "" then + local c = commands[name] + local n = c and (c[interface] or c.en) or name + if at.generated == "yes" then + -- skip (for now) + elseif type ~= "environment" then + i_commands[#i_commands+1] = n + elseif split then + i_environments[#i_environments+1] = n else - commands[language] = { [command] = true } + -- variables ? + i_commands[#i_commands+1] = start .. n + i_commands[#i_commands+1] = stop .. n end end end - for language, data in next, commands do - local fromlua = data - local fromxml = collected[language].commands - for i=1,#fromxml do - local c = fromxml[i] - if not fromlua[c] then - -- print(language,c) - fromlua[c] = true - end - end - collected[language].commands = table.sortedkeys(fromlua) + if #i_commands > 0 then + sort(i_commands) + sort(i_environments) + collected[interface] = { + commands = i_commands, + environments = i_environments, + } end end -- @@ -341,7 +487,7 @@ function scripts.interface.check() end end -function scripts.interface.interfaces() +function scripts.interface.mkii() local filename = resolvers.findfile(environment.files[1] or "mult-def.lua") or "" if filename ~= "" then local interface = dofile(filename) @@ -367,7 +513,7 @@ function scripts.interface.interfaces() xmlresult[#xmlresult+1] = format("\t\t<cd:%s name='%s' value='%s'/>",tag,key,value) end end - xmlresult[#xmlresult+1] = format("\t</cd:%s>\n",tag) + xmlresult[#xmlresult+1] = format("\t</cd:%s>\n",what) end local function replace(str, element, attribute, category, othercategory, language) return str:gsub(format("(<%s[^>]-%s=)([\"\'])([^\"\']-)([\"\'])",element,attribute), function(a,b,c) @@ -386,6 +532,7 @@ function scripts.interface.interfaces() end -- we could just replace attributes for language, _ in next, commands.setuplayout do + -- keyword files local texresult, xmlresult = { }, { } texresult[#texresult+1] = format("%% this file is auto-generated, don't edit this file\n%%") xmlresult[#xmlresult+1] = format("<?xml version='1.0'?>\n",tag) @@ -402,9 +549,10 @@ function scripts.interface.interfaces() report("saving interface definitions '%s'",texfilename) io.savedata(xmlfilename,concat(xmlresult,"\n")) report("saving interface translations '%s'",xmlfilename) + -- mkii files if language ~= "en" and xmldata ~= "" then local newdata = xmldata:gsub("(<cd:interface.*language=.)en(.)","%1"..language.."%2",1) --- newdata = replace(newdata, 'cd:command', 'name', interface.commands, interface.elements, language) + -- newdata = replace(newdata, 'cd:command', 'name', interface.commands, interface.elements, language) newdata = replace(newdata, 'cd:string', 'value', interface.commands, interface.elements, language) newdata = replace(newdata, 'cd:variable' , 'value', interface.variables, nil, language) newdata = replace(newdata, 'cd:parameter', 'name', interface.constants, nil, language) @@ -415,6 +563,7 @@ function scripts.interface.interfaces() io.savedata(xmlfilename,newdata) report("saving interface specification '%s'",xmlfilename) end + -- mkiv is generated otherwise end end end @@ -439,33 +588,6 @@ function scripts.interface.preprocess() end end --- function scripts.interface.messages() --- local filename = resolvers.findfile(environment.files[1] or "mult-mes.lua") or "" --- if filename ~= "" then --- local messages = dofile(filename) --- report("messages for * loaded from '%s'",filename) --- report() --- for i=1,#messageinterfaces do --- local interface = messageinterfaces[i] --- local texresult = { } --- for category, data in next, messages do --- for tag, message in next, data do --- if tag ~= "files" then --- local msg = message[interface] or message["all"] or message["en"] --- if msg then --- texresult[#texresult+1] = format("\\setinterfacemessage{%s}{%s}{%s}",category,tag,msg) --- end --- end --- end --- end --- texresult[#texresult+1] = format("%%\n\\endinput") --- local interfacefile = format("mult-m%s.mkii",interface) --- io.savedata(interfacefile,concat(texresult,"\n")) --- report("messages for '%s' saved in '%s'",interface,interfacefile) --- end --- end --- end - function scripts.interface.toutf() local filename = environment.files[1] if filename then @@ -509,88 +631,10 @@ function scripts.interface.toutf() end end --- function scripts.interface.labels() --- require("char-def.lua") --- require("lang-txt.lua") --- local interfaces = require("mult-def.lua") --- local variables = interfaces.variables --- local contextnames = { } --- for unicode, data in next, characters.data do --- local contextname = data.contextname --- if contextname then --- contextnames[utfchar(unicode)] = "\\" .. contextname .. " " --- end --- end --- contextnames["i"] = nil --- contextnames["'"] = nil --- contextnames["\\"] = nil --- local function flush(f,kind,what,expand,namespace,prefix) --- local whatdata = languages.data.labels[what] --- f:write("\n") --- f:write(format("%% %s => %s\n",what,kind)) --- for tag, data in table.sortedpairs(whatdata) do --- if not data.hidden then --- f:write("\n") --- for language, text in table.sortedpairs(data.labels) do --- if text ~= "" then --- if expand then --- text = utfgsub(text,".",contextnames) --- text = gsub(text," ", "\ ") --- end --- if namespace and namespace[tag] then --- tag = prefix .. tag --- end --- if find(text,",") then --- text = "{" .. text .. "}" --- end --- if text == "" then --- -- skip --- else --- if type(text) == "table" then --- f:write(format("\\setup%stext[\\s!%s][%s={{%s},}]\n",kind,language,tag,text)) --- else --- f:write(format("\\setup%stext[\\s!%s][%s={{%s},{%s}}]\n",kind,language,tag,text[1],text[2])) --- end --- end --- end --- end --- end --- end --- end --- function flushall(txtname,expand) --- local f = io.open(txtname,"w") --- if f then --- report("saving '%s'",txtname) --- f:write("% this file is auto-generated, don't edit this file\n") --- flush(f,"head","titles",expand,variables,"\\v!") --- flush(f,"label","texts",expand,variables,"\\v!") --- flush(f,"mathlabel","functions",expand) --- flush(f,"taglabel","tags",expand) --- f:write("\n") --- f:write("\\endinput\n") --- f:close() --- end --- end --- flushall("lang-txt.mkii",true) --- flushall("lang-txt.mkiv",false) --- end - local ea = environment.argument -if ea("context") then - scripts.interface.interfaces() - -- scripts.interface.messages() - -- scripts.interface.labels() -elseif ea("interfaces") or ea("messages") or ea("labels") then - if ea("interfaces") then - scripts.interface.interfaces() - end - -- if ea("messages") then - -- scripts.interface.messages() - -- end - -- if ea("labels") then - -- scripts.interface.labels() - -- end +if ea("mkii") then + scripts.interface.mkii() elseif ea("preprocess") then scripts.interface.preprocess() elseif ea("toutf") then diff --git a/scripts/context/lua/mtx-modules.lua b/scripts/context/lua/mtx-modules.lua index f4003c1db..572e6a304 100644 --- a/scripts/context/lua/mtx-modules.lua +++ b/scripts/context/lua/mtx-modules.lua @@ -58,7 +58,7 @@ local report = application.report -- \stoptypen -- -- Macro definitions specific to the documentation are not surrounded by --- start-stop commands. The suffix specificaction can be overruled at runtime, +-- start-stop commands. The suffix specification can be overruled at runtime, -- but defaults to the file extension. This specification can be used for language -- depended verbatim typesetting. -- diff --git a/scripts/context/lua/mtx-package.lua b/scripts/context/lua/mtx-package.lua index 8c9e6b9fc..23da30bf2 100644 --- a/scripts/context/lua/mtx-package.lua +++ b/scripts/context/lua/mtx-package.lua @@ -55,7 +55,7 @@ function scripts.package.merge_luatex_files(name) collected[#collected+1] = format("-- parent file : %s\n",oldname) collected[#collected+1] = format("-- merge date : %s\n",os.date()) -- loadmodule can have extra arguments - for lib in gmatch(data,"loadmodule *%([\'\"](.-)[\'\"]") do + for lib in gmatch(data,"loadmodule *%([\'\"](.-)[\'\"]") do -- todo: not -- lines if file.basename(lib) ~= file.basename(newname) then local fullname = resolvers.findfile(lib) or "" if fullname == "" then diff --git a/scripts/context/lua/mtx-patterns.lua b/scripts/context/lua/mtx-patterns.lua index b7d41e2b2..e8d4d3e65 100644 --- a/scripts/context/lua/mtx-patterns.lua +++ b/scripts/context/lua/mtx-patterns.lua @@ -9,7 +9,8 @@ if not modules then modules = { } end modules ['mtx-patterns'] = { local format, find, concat, gsub, match, gmatch = string.format, string.find, table.concat, string.gsub, string.match, string.gmatch local byte, char = utf.byte, utf.char local addsuffix = file.addsuffix -local lpegmatch, validutf8 = lpeg.match, lpeg.patterns.validutf8 +local lpegmatch, lpegsplit, lpegpatterns, validutf8 = lpeg.match, lpeg.split, lpeg.patterns, lpeg.patterns.validutf8 +local P, V, Cs = lpeg.P, lpeg.V, lpeg.Cs local helpinfo = [[ <?xml version="1.0"?> @@ -105,7 +106,7 @@ scripts.patterns.list = { -- { "gr", "hyph-el-polyton", "greek" }, { "agr", "hyph-grc", "ancient greek", ignored_ancient_greek }, { "gb", "hyph-en-gb", "british english" }, - { "us", "hyph-en-us", "american english" }, + { "us", "hyph-en-us", "american english" }, -- { "eo", "hyph-eo", "esperanto" }, { "es", "hyph-es", "spanish" }, { "et", "hyph-et", "estonian" }, @@ -128,6 +129,7 @@ scripts.patterns.list = { -- { "??", "hyph-kmr", "kurmanji" }, -- { "kn", "hyph-kn", "kannada" }, { "la", "hyph-la", "latin" }, + { "ala", "hyph-la-x-classic", "ancient latin" }, -- { "lo", "hyph-lo", "lao" }, { "lt", "hyph-lt", "lithuanian" }, { "lv", "hyph-lv", "latvian" }, @@ -164,7 +166,7 @@ scripts.patterns.list = { local utf = unicode.utf8 function utf.check(str) - return lpeg.match(lpeg.patterns.validutf8,str) + return lpegmatch(lpegpatterns.validutf8,str) end -- *.tex @@ -213,9 +215,9 @@ function scripts.patterns.load(path,name,mnemonic,ignored) end if okay then -- split into lines - local how = lpeg.patterns.whitespace^1 - splitpatternsnew = lpeg.split(how,patterns) - splithyphenationsnew = lpeg.split(how,hyphenations) + local how = lpegpatterns.whitespace^1 + splitpatternsnew = lpegsplit(how,patterns) + splithyphenationsnew = lpegsplit(how,hyphenations) end if okay then -- remove comments @@ -315,23 +317,23 @@ function scripts.patterns.load(path,name,mnemonic,ignored) if ignored then for k, v in next, ignored do if p then - p = p + lpeg.P(char(k)) + p = p + P(char(k)) else - p = lpeg.P(char(k)) + p = P(char(k)) end end - p = lpeg.P{ p + 1 * lpeg.V(1) } -- anywhere + p = P{ p + 1 * V(1) } -- anywhere end -- replaced (all languages) local r = nil for k, v in next, replaced_whatever do if r then - r = r + lpeg.P(k)/v + r = r + P(k)/v else - r = lpeg.P(k)/v + r = P(k)/v end end - r = lpeg.Cs((r + 1)^0) + r = Cs((r + 1)^0) local result = { } for i=1,#what do local line = what[i] @@ -440,13 +442,13 @@ function scripts.patterns.save(destination,mnemonic,name,patternsnew,hyphenation if nofpatternsnew > 0 then local data = concat(patternsnew," ") patterndata = { - n = nofpatternsnew, - compression = compression, - length = #data, - data = compression and zlib.compress(data,9) or data, - characters = concat(table.sortedkeys(pusednew),""), - minhyphenmin = 1, -- determined by pattern author - minhyphenmax = 1, -- determined by pattern author + n = nofpatternsnew, + compression = compression, + length = #data, + data = compression and zlib.compress(data,9) or data, + characters = concat(table.sortedkeys(pusednew),""), + lefthyphenmin = 1, -- determined by pattern author + righthyphenmax = 1, -- determined by pattern author } else patterndata = { diff --git a/scripts/context/lua/mtx-plain.lua b/scripts/context/lua/mtx-plain.lua index 1076572fc..347f63f1d 100644 --- a/scripts/context/lua/mtx-plain.lua +++ b/scripts/context/lua/mtx-plain.lua @@ -12,6 +12,8 @@ if not modules then modules = { } end modules ['mtx-plain'] = { -- instead of kpse here, just like with the font database code (as that -- one also works with kpse runtime) +local format = string.format + local helpinfo = [[ <?xml version="1.0"?> <application> @@ -46,16 +48,22 @@ local report = application.report scripts = scripts or { } scripts.plain = scripts.plain or { } +local passed_options = table.tohash { + "utc" +} + local function execute(...) - local command = string.format(...) + local command = format(...) report("running command %a\n",command) - os.execute(command) + return os.execute(command) end local function resultof(...) - local command = string.format(...) + local command = format(...) report("running command %a",command) - return string.strip(os.resultof(command) or "") + local result = os.resultof(command) or "" + result = string.gsub(result,"[\n\r]+","") + return result end function scripts.plain.make(texengine,texformat) @@ -64,13 +72,13 @@ function scripts.plain.make(texengine,texformat) local fmtpathspec = resultof("kpsewhich --var-value=TEXFORMATS --engine=%s",texengine) if fmtpathspec ~= "" then report("using path specification %a",fmtpathspec) - fmtpathspec = resultof('kpsewhich -expand-braces="%s"',fmtpathspec) + fmtpathspec = resultof('kpsewhich --expand-braces="%s"',fmtpathspec) end if fmtpathspec ~= "" then report("using path expansion %a",fmtpathspec) else report("no valid path reported, trying alternative") - fmtpathspec = resultof("kpsewhich --show-path=fmt --engine=%s",texengine) + -- fmtpathspec = resultof("kpsewhich --show-path=fmt --engine=%s",texengine) if fmtpathspec ~= "" then report("using path expansion %a",fmtpathspec) else @@ -91,6 +99,7 @@ function scripts.plain.make(texengine,texformat) end end end +-- local fmtpath = resultof("kpsewhich --expand-path $safe-out-name=$TEXFORMATS") if not fmtpath or fmtpath == "" then fmtpath = "." else @@ -105,7 +114,13 @@ end function scripts.plain.run(texengine,texformat,filename) local t = { } for k, v in next, environment.arguments do - t[#t+1] = string.format("--mtx:%s=%s",k,v) + local m = passed_options[k] and "" or "mtx:" + if type(v) == "string" and v ~= "" then + v = format("--%s%s=%s",m,k,v) + elseif v then + v = format("--%s%s",m,k) + end + t[#t+1] = v end execute('%s --fmt=%s %s "%s"',texengine,file.removesuffix(texformat),table.concat(t," "),filename) end diff --git a/scripts/context/lua/mtx-profile.lua b/scripts/context/lua/mtx-profile.lua index 0d0c28084..355694e28 100644 --- a/scripts/context/lua/mtx-profile.lua +++ b/scripts/context/lua/mtx-profile.lua @@ -8,8 +8,9 @@ if not modules then modules = { } end modules ['mtx-profile'] = { -- todo: also line number -- todo: sort runtime as option +-- todo: make it more efficient .. real old code -local match, format, find = string.match, string.format, string.find +local match, format, find, gsub = string.match, string.format, string.find, string.gsub local helpinfo = [[ <?xml version="1.0"?> @@ -57,7 +58,7 @@ function scripts.profiler.analyze(filename) local totalruntime, totalcount, totalcalls = 0, 0, 0 for line in f:lines() do if not find(line,"__index") and not find(line,"__newindex") then - local stacklevel, filename, functionname, linenumber, currentline, localtime, totaltime = line:match("^(%d+)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)") + local stacklevel, filename, functionname, linenumber, currentline, localtime, totaltime = match(line,"^(%d+)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)\t(.-)") if not filename then -- next elseif filename == "=[C]" then @@ -65,7 +66,7 @@ function scripts.profiler.analyze(filename) calls[functionname] = (calls[functionname] or 0) + 1 end else - local filename = filename:match("^@(.*)$") + local filename = match(filename,"^@(.*)$") if filename then local fi = times[filename] if not fi then fi = { } times[filename] = fi end @@ -90,7 +91,7 @@ function scripts.profiler.analyze(filename) totalcount = totalcount + count if totaltime > timethreshold or count > countthreshold then totalruntime = totalruntime + totaltime - local functionfile, somenumber = functionname:match("^@(.+):(.-)$") + local functionfile, somenumber = match(functionname,"^@(.+):(.-)$") if functionfile then local number = tonumber(somenumber) if number then @@ -98,13 +99,13 @@ function scripts.profiler.analyze(filename) loaded[functionfile] = string.splitlines(io.loaddata(functionfile) or "") end functionname = loaded[functionfile][number] or functionname - functionname = functionname:gsub("^%s*","") - functionname = functionname:gsub("%s*%-%-.*$","") + functionname = gsub(functionname,"^%s*","") + functionname = gsub(functionname,"%s*%-%-.*$","") functionname = number .. ": " .. functionname end end filename = file.basename(filename) - print(functiontemplate:format(filename,totaltime,count,functionname)) + print(format(functiontemplate,filename,totaltime,count,functionname)) end end end @@ -149,7 +150,7 @@ function scripts.profiler.x_analyze(filename) end f:close() local noc = 0 -local criterium = 100 + local criterium = 100 for name, n in next, calls do if n > criterium then if find(name,"^@@[a-z][a-z]") then diff --git a/scripts/context/lua/mtx-server-ctx-help.lua b/scripts/context/lua/mtx-server-ctx-help.lua index d948c6e46..04217dab9 100644 --- a/scripts/context/lua/mtx-server-ctx-help.lua +++ b/scripts/context/lua/mtx-server-ctx-help.lua @@ -6,580 +6,561 @@ if not modules then modules = { } end modules ['mtx-server-ctx-help'] = { license = "see context related readme files" } --- todo in lua interface: noargument, oneargument, twoarguments, threearguments --- todo: pickup translations from mult file +local gsub, find, lower, match = string.gsub, string.find, string.lower, string.match +local concat, sort = table.concat, table.sort dofile(resolvers.findfile("trac-lmx.lua","tex")) +dofile(resolvers.findfile("util-sci.lua","tex")) +dofile(resolvers.findfile("char-def.lua","tex")) +dofile(resolvers.findfile("char-ini.lua","tex")) +dofile(resolvers.findfile("char-utf.lua","tex")) + +local scite = utilities.scite +local formatters = string.formatters +local sortedkeys = table.sortedkeys +local setmetatableindex = table.setmetatableindex +local lowercase = characters.lower +local uppercase = characters.upper +local interfaces = dofile(resolvers.findfile("mult-def.lua","tex")) +local i_setupstrings = interfaces.setupstrings +local i_commands = interfaces.commands +local i_variables = interfaces.variables +local i_constants = interfaces.constants +local i_elements = interfaces.elements +local report = logs.reporter("ctx-help") +local gettime = os.gettimeofday or os.clock + +local xmlcollected = xml.collected +local xmlfirst = xml.first +local xmltext = xml.text +local xmlload = xml.load + +document = document or { } +document.setups = document.setups or { } + +local usedsetupfile = resolvers.findfile("i-context.xml") or "" +local usedsetuproot = usedsetupfile ~= "" and xmlload(usedsetupfile) or false +local useddefinitions = { } + +if usedsetuproot then + report("main file loaded: %s",usedsetupfile) + xml.include(usedsetuproot,"cd:interfacefile","filename",true,function(s) + local fullname = resolvers.findfile(s) + if fullname and fullname ~= "" then + report("inclusion loaded: %s",fullname) + return io.loaddata(fullname) + end + end) +else + report("no main file") + return false, false +end --- problem ... serialize parent stack +local defaultinterface = "en" -local format, match, gsub, find = string.format, string.match, string.gsub, string.find -local concat = table.concat +-- todo: store mode|interface in field but then we need post -local report = logs.reporter("ctx-help") +for e in xmlcollected(usedsetuproot,"cd:define") do + useddefinitions[e.at.name] = e +end --- -- -- make this a module: cont-xx.lua +for e in xml.collected(usedsetuproot,"cd:interface/cd:interface") do + e.at.file = e.__f__ -- nicer +end -document = document or { } -document.setups = document.setups or { } +local f_divs_t = { + pe = formatters["<div dir='rtl' lang='arabic'>%s</div>"], +} -document.setups.div = { - pe = "<div dir='rtl' lang='arabic'>%s</div>" +local f_spans_t = { + pe = formatters["<span dir='rtl' lang='arabic'>%s</span>"] } -document.setups.span = { - pe = "<span dir='rtl' lang='arabic'>%s</span>" +local f_href_in_list_t = { + tex = formatters["<a class='setupmenuurl' href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s'>%s</a>"], + lua = formatters["<a class='setupmenuurl' href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s'>%s</a>"], } -document.setups.translations = document.setups.translations or { - - nl = { - ["title"] = "setup", - ["formula"] = "formule", - ["number"] = "getal", - ["list"] = "lijst", - ["dimension"] = "maat", - ["mark"] = "markering", - ["reference"] = "verwijzing", - ["command"] = "commando", - ["file"] = "file", - ["name"] = "naam", - ["identifier"] = "naam", - ["text"] = "tekst", - ["section"] = "sectie", - ["singular"] = "naam enkelvoud", - ["plural"] = "naam meervoud", - ["matrix"] = "n*m", - ["see"] = "zie", - ["inherits"] = "erft van", - ["optional"] = "optioneel", - ["displaymath"] = "formule", - ["index"] = "ingang", - ["math"] = "formule", - ["nothing"] = "leeg", - ["file"] = "file", - ["position"] = "positie", - ["reference"] = "verwijzing", - ["csname"] = "naam", - ["destination"] = "bestemming", - ["triplet"] = "triplet", - ["word"] = "woord", - ["content"] = "tekst", - }, - - en = { - ["title"] = "setup", - ["formula"] = "formula", - ["number"] = "number", - ["list"] = "list", - ["dimension"] = "dimension", - ["mark"] = "mark", - ["reference"] = "reference", - ["command"] = "command", - ["file"] = "file", - ["name"] = "name", - ["identifier"] = "identifier", - ["text"] = "text", - ["section"] = "section", - ["singular"] = "singular name", - ["plural"] = "plural name", - ["matrix"] = "n*m", - ["see"] = "see", - ["inherits"] = "inherits from", - ["optional"] = "optional", - ["displaymath"] = "formula", - ["index"] = "entry", - ["math"] = "formula", - ["nothing"] = "empty", - ["file"] = "file", - ["position"] = "position", - ["reference"] = "reference", - ["csname"] = "name", - ["destination"] = "destination", - ["triplet"] = "triplet", - ["word"] = "word", - ["content"] = "text", - - ["noargument"] = "\\cs", - ["oneargument"] = "\\cs#1{..}", - ["twoarguments"] = "\\cs#1#2{..}{..}", - ["threearguments"] = "\\cs#1#2#3{..}{..}{..}", - - }, - - de = { - ["title"] = "Setup", - ["formula"] = "Formel", - ["number"] = "Nummer", - ["list"] = "Liste", - ["dimension"] = "Dimension", - ["mark"] = "Beschriftung", - ["reference"] = "Referenz", - ["command"] = "Befehl", - ["file"] = "Datei", - ["name"] = "Name", - ["identifier"] = "Name", - ["text"] = "Text", - ["section"] = "Abschnitt", - ["singular"] = "singular", - ["plural"] = "plural", - ["matrix"] = "n*m", - ["see"] = "siehe", - ["inherits"] = "inherits from", - ["optional"] = "optioneel", - ["displaymath"] = "formula", - ["index"] = "entry", - ["math"] = "formula", - ["nothing"] = "empty", - ["file"] = "file", - ["position"] = "position", - ["reference"] = "reference", - ["csname"] = "name", - ["destination"] = "destination", - ["triplet"] = "triplet", - ["word"] = "word", - ["content"] = "text", - }, - - cz = { - ["title"] = "setup", - ["formula"] = "rovnice", - ["number"] = "cislo", - ["list"] = "seznam", - ["dimension"] = "dimenze", - ["mark"] = "znacka", - ["reference"] = "reference", - ["command"] = "prikaz", - ["file"] = "soubor", - ["name"] = "jmeno", - ["identifier"] = "jmeno", - ["text"] = "text", - ["section"] = "sekce", - ["singular"] = "jmeno v singularu", - ["plural"] = "jmeno v pluralu", - ["matrix"] = "n*m", - ["see"] = "viz", - ["inherits"] = "inherits from", - ["optional"] = "optioneel", - ["displaymath"] = "formula", - ["index"] = "entry", - ["math"] = "formula", - ["nothing"] = "empty", - ["file"] = "file", - ["position"] = "position", - ["reference"] = "reference", - ["csname"] = "name", - ["destination"] = "destination", - ["triplet"] = "triplet", - ["word"] = "word", - ["content"] = "text", - }, - - it = { - ["title"] = "setup", - ["formula"] = "formula", - ["number"] = "number", - ["list"] = "list", - ["dimension"] = "dimension", - ["mark"] = "mark", - ["reference"] = "reference", - ["command"] = "command", - ["file"] = "file", - ["name"] = "name", - ["identifier"] = "name", - ["text"] = "text", - ["section"] = "section", - ["singular"] = "singular name", - ["plural"] = "plural name", - ["matrix"] = "n*m", - ["see"] = "see", - ["inherits"] = "inherits from", - ["optional"] = "optioneel", - ["displaymath"] = "formula", - ["index"] = "entry", - ["math"] = "formula", - ["nothing"] = "empty", - ["file"] = "file", - ["position"] = "position", - ["reference"] = "reference", - ["csname"] = "name", - ["destination"] = "destination", - ["triplet"] = "triplet", - ["word"] = "word", - ["content"] = "text", - }, - - ro = { - ["title"] = "setari", - ["formula"] = "formula", - ["number"] = "numar", - ["list"] = "lista", - ["dimension"] = "dimensiune", - ["mark"] = "marcaj", - ["reference"] = "referinta", - ["command"] = "comanda", - ["file"] = "fisier", - ["name"] = "nume", - ["identifier"] = "nume", - ["text"] = "text", - ["section"] = "sectiune", - ["singular"] = "nume singular", - ["plural"] = "nume pluram", - ["matrix"] = "n*m", - ["see"] = "vezi", - ["inherits"] = "inherits from", - ["optional"] = "optioneel", - ["displaymath"] = "formula", - ["index"] = "entry", - ["math"] = "formula", - ["nothing"] = "empty", - ["file"] = "file", - ["position"] = "position", - ["reference"] = "reference", - ["csname"] = "name", - ["destination"] = "destination", - ["triplet"] = "triplet", - ["word"] = "word", - ["content"] = "text", - }, - - fr = { - ["title"] = "réglage", - ["formula"] = "formule", - ["number"] = "numéro", - ["list"] = "liste", - ["dimension"] = "dimension", - ["mark"] = "marquage", - ["reference"] = "reference", - ["command"] = "commande", - ["file"] = "fichier", - ["name"] = "nom", - ["identifier"] = "identificateur", - ["text"] = "texte", - ["section"] = "section", - ["singular"] = "nom singulier", - ["plural"] = "nom pluriel", - ["matrix"] = "n*m", - ["see"] = "vois", - ["inherits"] = "herite de", - ["optional"] = "optionel", - ["displaymath"] = "formule", - ["index"] = "entrée", - ["math"] = "formule", - ["nothing"] = "vide", - ["file"] = "fichier", - ["position"] = "position", - ["reference"] = "réference", - ["csname"] = "nom", - ["destination"] = "destination", - ["triplet"] = "triplet", - ["word"] = "mot", - ["content"] = "texte", - } +local f_href_in_list_i = { + tex = formatters["<a class='setupmenucmd' href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s' id='#current'>%s</a>"], + lua = formatters["<a class='setupmenucmd' href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s' id='#current'>%s</a>"], +} + +local f_href_as_command_t = { + tex = formatters["<a class='setuplisturl' href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s'>\\%s</a>"], + lua = formatters["<a class='setuplisturl' href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s'>context.%s</a>"], +} +local f_modes_t = { + tex = formatters["<a class='setupmodeurl' href='mtx-server-ctx-help.lua?interface=%s&mode=lua'>lua mode</a>"], + lua = formatters["<a class='setupmodeurl' href='mtx-server-ctx-help.lua?interface=%s&mode=tex'>tex mode</a>"], } -document.setups.formats = { - open_command = { - tex = [[\%s]], - lua = [[context.%s (]], - }, - close_command = { - tex = [[]], - lua = [[ )]], - }, - connector = { - tex = [[]], - lua = [[, ]], - }, - href_in_list = { - tex = [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>%s</a>]], - lua = [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>%s</a>]], - }, - href_as_command = { - tex = [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>\%s</a>]], - lua = [[<a href='mtx-server-ctx-help.lua?command=%s&mode=%s'>context.%s</a>]], - }, - modes = { - tex = [[<a href='mtx-server-ctx-help.lua?mode=lua'>lua mode</a>]], - lua = [[<a href='mtx-server-ctx-help.lua?mode=tex'>tex mode</a>]], - }, - optional_single = { - tex = "[optional string %s]", - lua = "{optional string %s}", - }, - optional_list = { - tex = "[optional list %s]", - lua = "{optional table %s}" , - } , - mandate_single = { - tex = "[mandate string %s]", - lua = "{mandate string %s}", - }, - mandate_list = { - tex = "[mandate list %s]", - lua = "{mandate list %s}", - }, - interface = [[<a href='mtx-server-ctx-help.lua?interface=%s&mode=%s'>%s</a>]], - source = [[<a href='mtx-server-ctx-help.lua?source=%s&mode=%s'>%s</a>]], - parameter = [[<tr><td width='15%%'>%s</td><td width='15%%'>%s</td><td width='70%%'>%s</td></tr>]], - parameters = [[<table width='100%%'>%s</table>]], - listing = [[<pre><t>%s</t></listing>]], - special = [[<i>%s</i>]], - default = [[<u>%s</u>]], +local f_views_t = { + groups = formatters["<a class='setupviewurl' href='mtx-server-ctx-help.lua?interface=%s&view=names'>names</a>"], + names = formatters["<a class='setupviewurl' href='mtx-server-ctx-help.lua?interface=%s&view=groups'>groups</a>"], } -local function translate(tag,int,noformat) - local t = document.setups.translations - local te = t["en"] - local ti = t[int] or te +local f_interface = formatters["<a href='mtx-server-ctx-help.lua?interface=%s&command=%s&mode=%s'>%s</a>"] +local f_source = formatters["<a href='mtx-server-ctx-help.lua?interface=%s&command=%s&source=%s&mode=%s'>%s</a>"] +local f_keyword = formatters[" <tr>\n <td width='15%%'>%s</td>\n <td width='85%%' colspan='2'>%s</td>\n </tr>\n"] +local f_parameter = formatters[" <tr>\n <td width='15%%'>%s</td>\n <td width='15%%'>%s</td>\n <td width='70%%'>%s</td>\n </tr>\n"] +local f_url = formatters[" <tr>\n <td width='15%%'>%s</td>\n <td width='85%%' colspan='2'><i>%s</i>: %s</td>\n </tr>\n"] +local f_parameters = formatters["\n<table width='100%%'>\n%s</table>\n"] +local f_instance = formatters["<tt>%s</tt>"] +local f_instances = formatters["\n<div class='setupinstances'><b>predefined instances</b>: %s</div>\n"] +local f_listing = formatters["<pre><t>%s</t></pre>"] +local f_special = formatters["<i>%s</i>"] +local f_default = formatters["<u>%s</u>"] +local f_group = formatters["<div class='setupmenugroup'>\n<div class='setupmenucategory'>%s</div>%s</div>"] + +-- replace('cd:string', 'value', i_commands, i_elements) +-- replace('cd:variable' , 'value', i_variables) +-- replace('cd:parameter', 'name', i_constants) +-- replace('cd:constant', 'type', i_variables) +-- replace('cd:constant', 'default', i_variables) +-- replace('cd:variable', 'type', i_variables) +-- replace('cd:inherit', 'name', i_commands, i_elements) + +local function translate(tag,interface,noformat) -- to be checked + local translation = i_setupstrings[tag] + local translated = translation and (translation[interface] or translation[interface]) or tag if noformat then - return ti[tag] or te[tag] or tag + return translated else - return format(document.setups.formats.special,ti[tag] or te[tag] or tag) + return f_special(translated) end end -local function translated(e,int) +local function translatedparameter(e,interface) local attributes = e.at local s = attributes.type or "?" - local tag = match(s,"^cd:(.*)$") - if attributes.default == "yes" then - return format(document.setups.formats.default,tag or "?") - elseif tag then - return translate(tag,int) + if find(s,"^cd:") then + local t = i_setupstrings[s] + local f = t and (t[interface] or t.en) or s + return f else - return s + local t = i_variables[s] + local f = t and (t[interface] or t.en) or s + return f end end -document.setups.loaded = document.setups.loaded or { } - -document.setups.current = { } -document.setups.showsources = true -document.setups.mode = "tex" - -function document.setups.load(filename) - filename = resolvers.findfile(filename) or "" - if filename ~= "" then - local current = document.setups.loaded[filename] - if not current then - local loaded = xml.load(filename) - if loaded then - -- xml.inject(document.setups.root,"/",loaded) - current = { - file = filename, - root = loaded, - names = { }, - used = { }, - } - document.setups.loaded[filename] = current - end +local function translatedkeyword(e,interface) + local attributes = e.at + local s = attributes.type or "?" + if find(s,"^cd:") then + local t = i_setupstrings[s] + local f = t and (t[interface] or t.en) or s + return f + else + local t = i_variables[s] + local f = t and (t[interface] or t.en) or s + if attributes.default == "yes" then + return f_default(f) + else + return f end - document.setups.current = current or { } end end -function document.setups.name(ek) - local at = ek.at +local function translatedvariable(s,interface) + local t = i_variables[s] + return t and (t[interface] or t.en) or s +end + +local function translatedconstant(s,interface) -- cache + local t = i_constants[s] + return t and (t[interface] or t.en) or s +end + +local function translatedelement(s,interface) -- cache + local t = i_elements[s] + return t and (t[interface] or t.en) or s +end + +local function translatedstring(s,interface) -- cache + local t = i_commands[s] + if t then + t = t[interface] or t.en + end + if t then + return t + end + t = i_elements[s] + return t and (t[interface] or t.en) or s +end + +local function translatedcommand(s,interface) -- cache + local t = i_commands[s] + return t and (t[interface] or t.en) or s +end + +local function makeidname(e) + local at = e.at local name = at.name if at.type == 'environment' then - name = "start" .. name + name = name .. ":environment" + end + if at.generated == "yes" then + name = name .. ":generated" end if at.variant then name = name .. ":" .. at.variant end - if at.generated == "yes" then - name = name .. "*" - end - return name:lower() + return lower(name) end -function document.setups.csname(ek,int) +local function makecsname(e,interface,prefix) -- stop ? local cs = "" - local at = ek.at or { } - if at.type == 'environment' then - cs = translate("start",int,true) .. cs + local at = e.at + local ok = false + local en = at.type == 'environment' + if prefix and en then + cs = translatedelement("start",interface) end - for e in xml.collected(ek,'cd:sequence/(cd:string|variable)') do - if e.tg == "string" then - cs = cs .. e.at.value + for f in xmlcollected(e,'cd:sequence/(cd:string|cd:variable)') do -- always at the start + local tag = f.tg + local val = f.at.value or "" + if tag == "string" then + cs = cs .. translatedstring(val,interface) + elseif tag == "variable" then + cs = cs .. f_special(translatedconstant("name",interface)) + else -- can't happen + cs = cs .. val + end + ok = true + end + if not ok then + if en then + cs = cs .. translatedstring(at.name,interface) else - cs = cs .. e.at.value -- to be translated + cs = cs .. translatedcommand(at.name,interface) end end return cs end -function document.setups.names() - local current = document.setups.current - local names = current.names - if not names or #names == 0 then - names = { } - local name = document.setups.name - local csname = document.setups.csname - for e in xml.collected(current.root,'cd:command') do - names[#names+1] = { e.at.name, csname(e,int) } +local function getnames(root,interface) + local found = { } + local names = { } + local groups = { } + for e in xmlcollected(root,'cd:interface/cd:interface') do + local category = match(e.at.file or "","^i%-(.*)%.xml$") + local list = { } + for e in xmlcollected(e,'cd:command') do + local idname = makeidname(e) + local csname = makecsname(e,interface,true) + if not found[idname] then + local t = { idname, csname } + names[#names+1] = t + list[#list+1] = t + found[idname] = e + else + -- variant + end + end + if #list > 0 then + sort(list, function(a,b) return lower(a[2]) < lower(b[2]) end) + groups[#groups+1] = { category, list } end - table.sort(names, function(a,b) return a[2]:lower() < b[2]:lower() end) - current.names = names - end - return names -end -function document.setups.show(name) - local current = document.setups.current - if current.root then - local name = gsub(name,"[<>]","") - local setup = xml.first(current.root,"cd:command[@name='" .. name .. "']") - current.used[#current.used+1] = setup - xml.sprint(setup) end + sort(names, function(a,b) return lower(a[2]) < lower(b[2]) end) + sort(groups, function(a,b) return lower(a[1]) < lower(b[1]) end) + return names, groups, found end -function document.setups.showused() - local current = document.setups.current - if current.root and next(current.used) then - local sorted = table.sortedkeys(current.used) - for i=1,#sorted do - xml.sprint(current.used[sorted[i]]) - end - end -end -function document.setups.showall() - local current = document.setups.current - if current.root then - local list = { } - for e in xml.collected(current.root,"cd:command") do - list[document.setups.name(e)] = e +local loaded = setmetatableindex(function(loaded,interface) + local names, groups, found = getnames(usedsetuproot,interface) + local current = { + interface = interface, + root = usedsetuproot, + definitions = useddefinitions, + names = names, + groups = groups, + found = found, + } + loaded[interface] = current + return current +end) + +local function collect(current,name,interface,lastmode) + local command = current.found[name] + if command then + local definitions = current.definitions + local attributes = command.at or { } + local generated = attributes.generated == "yes" + local environment = attributes.type == "environment" + local sequence = { } + local tags = { } + local arguments = { } + local parameters = { } + local instances = { } + local tag = "" + local category = attributes.category or "" + local source = attributes.file and f_source(lastinterface,lastcommand,attributes.file,lastmode,attributes.file) or "" + + -- first pass: construct the top line + + local start = environment and (attributes["begin"] or translatedelement("start",interface)) or "" + local stop = environment and (attributes["end"] or translatedelement("stop" ,interface)) or "" + local name = makecsname(command,interface) -- we can use the stored one + local valid = true + local texmode = lastmode == "tex" + + local function process(e) + for e in xmlcollected(e,"/*") do + if not e.special then + local tag = e.tg + local attributes = e.at + if tag == "resolve" then + local resolved = definitions[e.at.name or ""] + if resolved then + process(resolved) + end + else + -- we need a 'lua' tag i.e. we only support a subset of string/table + local delimiters = attributes.delimiters or "brackets" + local optional = attributes.optional == "yes" + local list = attributes.list == "yes" + if texmode then + local okay + if tag == "keywords" then + -- todo = optional + okay = i_setupstrings["cd:" .. delimiters .. (list and "-l" or "-s")] + elseif tag == "assignments" then + -- todo = optional + okay = i_setupstrings["cd:assignment" .. delimiters .. (list and "-l" or "-s")] + elseif tag == "delimiter" then + tag = "\\" .. attributes.name + elseif tag == "string" then + tag = translatedstring(attributes.value,interface) + else + -- todo = optional + okay = i_setupstrings["cd:" .. tag .. (list and "-l" or "-s")] + or i_setupstrings["cd:" .. tag] + end + if okay then + tag = okay.en or tag + end + else + local okay + if tag == "keywords" then + -- todo = optional + okay = i_setupstrings["cd:" .. delimiters .. (list and "-l" or "-s")] + elseif tag == "assignments" then + -- todo = optional + okay = i_setupstrings["cd:assignment" .. delimiters .. (list and "-l" or "-s")] + elseif tag == "delimiter" then + okay = false + elseif tag == "string" then + okay = false + else + -- todo = optional + okay = i_setupstrings["cd:" .. tag .. (list and "-l" or "-s")] + or i_setupstrings["cd:" .. tag] + end + if okay then + local luatag = okay.lua + if luatag then + tag = luatag + else + tag = "unsupported" + valid = false + end + else + tag = "unsupported" + valid = false + end + end + if tag then + sequence[#sequence+1] = tag + tags[#tags+1] = tag + end + end + end + end end - local sorted = table.sortedkeys(list) - for i=1,#sorted do - xml.sprint(list[sorted[i]]) + + if start and start ~= "" then + if texmode then + sequence[#sequence+1] = formatters["\\%s%s"](start,name) + else + sequence[#sequence+1] = formatters["context.%s%s("](start,name) + end + else + if texmode then + sequence[#sequence+1] = formatters["\\%s"](name) + else + sequence[#sequence+1] = formatters["context.%s("](name) + end end - end -end -function document.setups.resolve(name) - local current = document.setups.current - if current.root then - local e = xml.filter(current.root,format("cd:define[@name='%s']/text()",name)) - if e then - xml.sprint(e) + + for e in xmlcollected(command,"/cd:arguments") do + process(e) end - end -end -function document.setups.collect(name,int,lastmode) - local current = document.setups.current - local formats = document.setups.formats - local command = xml.filter(current.root,format("cd:command[@name='%s']/first()",name)) - if command then - local attributes = command.at or { } - local data = { - command = command, - category = attributes.category or "", - } - if document.setups.showsources then - data.source = (attributes.file and format(formats.source,attributes.file,lastmode,attributes.file)) or "" + if texmode then + if stop and stop ~= "" then + sequence[#sequence+1] = "\\" .. stop .. name + end else - data.source = attributes.file or "" - end - local n, sequence, tags = 0, { }, { } - sequence[#sequence+1] = format(formats.open_command[lastmode],document.setups.csname(command,int)) - local arguments, tag = { }, "" - for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do - n = n + 1 - local attributes = d[k].at - if #sequence > 1 then - local c = formats.connector[lastmode] - if c ~= "" then - sequence[#sequence+1] = c - end + for i=2,#sequence-1 do + sequence[i] = sequence[i] .. ", " end - if attributes.optional == 'yes' then - if attributes.list == 'yes' then - tag = format(formats.optional_list[lastmode],n) - else - tag = format(formats.optional_single[lastmode],n) - end + + if stop and stop ~= "" then + sequence[#sequence+1] = formatters[") context.%s%s()"](stop,name) else - if attributes.list == 'yes' then - tag = format(formats.mandate_list[lastmode],n) - else - tag = format(formats.mandate_single[lastmode],n) - end + sequence[#sequence+1] = ")" end - sequence[#sequence+1] = tag - tags[#tags+1] = tag end - sequence[#sequence+1] = formats.close_command[lastmode] - data.sequence = concat(sequence, " ") - local parameters, n = { }, 0 - for r, d, k in xml.elements(command,"(cd:keywords|cd:assignments)") do - n = n + 1 - if d[k].tg == "keywords" then - local left = tags[n] - local right = { } - for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do - local tag = d[k].tg + + if valid then + + sequence = concat(sequence," ") + + -- second pass: construct the descriptions + + local n = 0 + + local function process(e) + for e in xmlcollected(e,"/*") do + local tag = e.tg + if tag == "resolve" then - local name = d[k].at.name or "" - if name ~= "" then - local resolved = xml.filter(current.root,format("cd:define[@name='%s']",name)) - for r, d, k in xml.elements(resolved,"cd:constant") do - right[#right+1] = translated(d[k],int) + + local resolved = definitions[e.at.name or ""] + if resolved then + process(resolved) + end + + elseif tag == "keywords" then + + n = n + 1 + local left = tags[n] + local right = { } + + local function processkeyword(e) + right[#right+1] = translatedkeyword(e,interface) + end + + for e in xmlcollected(e,"/*") do + if not e.special then + local tag = e.tg + if tag == "resolve" then + local resolved = definitions[e.at.name or ""] + if resolved then + processkeyword(resolved) + end + elseif tag == "constant" then + processkeyword(e) + else + right[#right+1] = "KEYWORD TODO" + end end end - else - right[#right+1] = translated(d[k],int) - end - end - parameters[#parameters+1] = format(formats.parameter,left,"",concat(right, ", ")) - else - local what = tags[n] - for r, d, k in xml.elements(d[k],"(cd:parameter|cd:inherit)") do - local tag = d[k].tg - local left, right = d[k].at.name or "?", { } - if tag == "inherit" then - local name = d[k].at.name or "?" - local url = format(document.setups.formats.href_as_command[lastmode],name,lastmode,name) - if #parameters > 0 and not find(parameters[#parameters],"<br/>") then - parameters[#parameters+1] = format(formats.parameter,"<br/>","","") + parameters[#parameters+1] = f_keyword(left,concat(right, ", ")) + + elseif tag == "assignments" then + + n = n + 1 + local what = tags[n] + local done = false + + local function processparameter(e,right) + for e in xmlcollected(e,"/*") do + if not e.special then + local tag = e.tg + if tag == "resolve" then + local resolved = definitions[e.at.name or ""] + if resolved then + processparameter(resolved,right) + end + elseif tag == "constant" then + right[#right+1] = translatedparameter(e,interface) + else + right[#right+1] = "PARAMETER TODO" + end + end + end end - parameters[#parameters+1] = format(formats.parameter,what,format(formats.special,translate("inherits",int)),url) - else - for r, d, k in xml.elements(d[k],"(cd:constant|cd:resolve)") do - local tag = d[k].tg - if tag == "resolve" then - local name = d[k].at.name or "" - if name ~= "" then - local resolved = xml.filter(current.root,format("cd:define[@name='%s']",name)) - for r, d, k in xml.elements(resolved,"cd:constant") do - right[#right+1] = translated(d[k],int) + + for e in xmlcollected(e,"/*") do + if not e.special then + local tag = e.tg + local left = translatedconstant(e.at.name,interface) + local right = { } + if tag == "resolve" then + local resolved = definitions[e.at.name or ""] + if resolved then + -- todo + process(resolved) end + elseif tag == "inherit" then + local name = e.at.name or "?" + local url = f_href_as_command_t[lastmode](lastinterface,name,lastmode,name) + parameters[#parameters+1] = f_url(what,translate("cd:inherits",interface),url) + elseif tag == "parameter" then + processparameter(e,right) + parameters[#parameters+1] = f_parameter(what,left,concat(right, ", ")) + else + parameters[#parameters+1] = "PARAMETER TODO" + end + if not done then + done = true + what = "" end - else - right[#right+1] = translated(d[k],int) end end - parameters[#parameters+1] = format(formats.parameter,what,left,concat(right, ", ")) + + what = "" + else + + n = n + 1 + local left = tags[n] + local right = i_setupstrings["cd:"..tag] + + if right then + right = uppercase(right[interface] or right.en or tag) + end + + parameters[#parameters+1] = f_keyword(left,right) + end - what = "" end end - parameters[#parameters+1] = format(formats.parameter,"<br/>","","") + + for e in xmlcollected(command,"/cd:arguments") do + process(e) + end + + else + if texmode then + sequence = formatters["unsupported command '%s%s'"](start or "",name) + else + sequence = formatters["unsupported function '%s%s'"](start or "",name) + end + parameters = { } end - data.parameters = parameters or { } - data.mode = formats.modes[lastmode or "tex"] - return data - else - return nil - end -end --- -- -- -tex = tex or { } + for e in xmlcollected(command,"/cd:instances/cd:constant") do + instances[#instances+1] = f_instance(translatedconstant(e.at.value or "?",interface)) + end + + return { + category = category, + source = source, + mode = f_modes_t[lastmode or "tex"](lastinterface), + view = f_views_t[lastview or "groups"](lastinterface), + sequence = sequence, + parameters = parameters, + instances = instances, + } + end +end -- -- -- @@ -594,8 +575,6 @@ local interfaces = { romanian = 'ro', } -local lastinterface, lastcommand, lastsource, lastmode = "en", "", "", "tex" - local variables = { ['color-background-main-left'] = '#3F3F3F', ['color-background-main-right'] = '#5F5F5F', @@ -604,66 +583,85 @@ local variables = { ['title'] = 'ConTeXt Help Information', } ---~ function lmx.loadedfile(filename) ---~ return io.loaddata(resolvers.findfile(filename)) -- return resolvers.texdatablob(filename) ---~ end +local what = { "environment", "category", "source", "mode", "view" } -local function doit(configuration,filename,hashed) +local function generate(configuration,filename,hashed) - local formats = document.setups.formats - - local start = os.clock() - local detail = hashed.queries or { } + local start = gettime() + local detail = hashed.queries or { } + local variables = setmetatableindex({},variables) if detail then + local lastinterface = detail.interface or defaultinterface or "en" + local lastcommand = detail.command or "" + local lastview = detail.view or "groups" + local lastsource = detail.source or "" + local lastmode = detail.mode or "tex" + + local current = loaded[lastinterface] - lastinterface = detail.interface or lastinterface - lastcommand = detail.command or lastcommand - lastsource = detail.source or lastsource - lastmode = detail.mode or lastmode or "tex" + local title = variables.title .. ": " .. lastinterface + variables.title = title lastcommand = gsub(lastcommand,"%s*^\\*(.+)%s*","%1") - if lastinterface then - report("checking interface: %s",lastinterface) - document.setups.load(format("cont-%s.xml",lastinterface)) + local f_div = f_divs_t[lastinterface] + ----- f_span = f_spans[lastinterface] + + local names = current.names + local groups = current.groups + local refs = { } + local ints = { } + + local function addnames(names) + local target = { } + for k=1,#names do + local namedata = names[k] + local command = namedata[1] + local text = namedata[2] + if command == lastcommand then + target[#target+1] = f_href_in_list_i[lastmode](lastinterface,command,lastmode,text) + else + target[#target+1] = f_href_in_list_t[lastmode](lastinterface,command,lastmode,text) + end + end + return concat(target,"<br/>\n") end - local div = document.setups.div [lastinterface] - local span = document.setups.span[lastinterface] - - local names, refs, ints = document.setups.names(lastinterface), { }, { } - for k=1,#names do - local v = names[k] - refs[k] = format(formats.href_in_list[lastmode],v[1],lastmode,v[2]) + if lastview == "groups" then + local target = { } + for i=1,#groups do + local group = groups[i] + target[#target+1] = f_group(group[1],addnames(group[2])) + end + refs = concat(target,"<br/>\n") + else + refs = addnames(names) end + if lastmode ~= "lua" then - local sorted = table.sortedkeys(interfaces) + local sorted = sortedkeys(interfaces) for k=1,#sorted do local v = sorted[k] - ints[k] = format(formats.interface,interfaces[v],lastmode,v) + ints[k] = f_interface(interfaces[v],lastcommand,lastmode,v) end end - local n = concat(refs,"<br/>") - local i = concat(ints,"<br/><br/>") + local n = refs + local i = concat(ints,"<br/><br/>\n") - if div then - variables.names = format(div,n) - variables.interfaces = format(div,i) + if f_div then + variables.names = f_div(n) + variables.interfaces = f_div(i) else variables.names = n variables.interfaces = i end - -- first we need to add information about mkii/mkiv + -- we only support mkiv - variables.maintitle = "no definition" - variables.maintext = "" - variables.extra = "" + if lastsource and lastsource ~= "" then - if document.setups.showsources and lastsource and lastsource ~= "" then - -- todo: mkii, mkiv, tex (can be different) local name = lastsource local full = resolvers.findfile(name) if full == "" and file.suffix(lastsource) == "tex" then @@ -676,16 +674,21 @@ local function doit(configuration,filename,hashed) end if full == "" then variables.maintitle = lastsource - variables.maintext = format(formats.listing,"no source found") + variables.maintext = f_listing("no source found") else + local data = io.loaddata(full) + data = scite.html(data,file.suffix(full),true) variables.maintitle = name - variables.maintext = format(formats.listing,io.loaddata(full)) + variables.maintext = f_listing(data) end - lastsource = "" + lastsource = "" + variables.extra = "mode: " .. f_modes_t.tex(lastinterface) .. " " .. f_modes_t.lua(lastinterface) + elseif lastcommand and lastcommand ~= "" then - local data = document.setups.collect(lastcommand,lastinterface,lastmode) + + local data = collect(current,lastcommand,lastinterface,lastmode) if data then - local what, extra = { "environment", "category", "source", "mode" }, { } + local extra = { } for k=1,#what do local v = what[k] if data[v] and data[v] ~= "" then @@ -693,12 +696,19 @@ local function doit(configuration,filename,hashed) extra[#extra+1] = v .. ": " .. data[v] end end + local instances = data.instances variables.maintitle = data.sequence - variables.maintext = format(formats.parameters,concat(data.parameters)) + variables.maintext = f_parameters(concat(data.parameters)) .. (#instances > 0 and f_instances(concat(instances,", ")) or "") variables.extra = concat(extra," ") else - variables.maintext = "select command" + variables.maintitle = "no definition" + variables.maintext = "" + variables.extra = "" end + else + variables.maintitle = "no definition" + variables.maintext = "" + variables.extra = "" end else @@ -711,9 +721,9 @@ local function doit(configuration,filename,hashed) local content = lmx.convert('context-help.lmx',false,variables) - report("time spent on page: %0.03f seconds",os.clock()-start) + report("time spent on building page: %0.03f seconds",gettime()-start) return { content = content } end -return doit, true +return generate, true diff --git a/scripts/context/lua/mtx-server.lua b/scripts/context/lua/mtx-server.lua index dba07f1d5..448b20ac5 100644 --- a/scripts/context/lua/mtx-server.lua +++ b/scripts/context/lua/mtx-server.lua @@ -35,7 +35,15 @@ local application = logs.application { helpinfo = helpinfo, } +local tonumber, tostring, loadfile, type = tonumber, tostring, loadfile, type +local find, gsub = string.find, string.gsub +local joinpath, filesuffix, dirname, is_qualified_path = file.join, file.suffix, file.dirname, file.is_qualified_path +local loaddata = io.loaddata +local P, C, patterns, lpegmatch = lpeg.P, lpeg.C, lpeg.patterns, lpeg.match +local formatters = string.formatters +local urlhashed, urlquery = url.hashed, url.query local report = application.report +local gettime = os.gettimeofday or os.clock scripts = scripts or { } scripts.webserver = scripts.webserver or { } @@ -44,7 +52,6 @@ dofile(resolvers.findfile("luat-soc.lua","tex")) local socket = socket or require("socket") ----- http = http or require("socket.http") -- not needed -local format = string.format -- The following two lists are taken from webrick (ruby) and -- extended with a few extra suffixes. @@ -154,10 +161,14 @@ local messages = { [505] = 'HTTP Version Not Supported', } +local f_content_length = formatters["Content-Length: %s\r\n"] +local f_content_type = formatters["Content-Type: %s\r\n"] +local f_error_title = formatters["<head><title>%s %s</title></head><html><h2>%s %s</h2></html>"] + local handlers = { } local function errormessage(client,configuration,n) - local data = format("<head><title>%s %s</title></head><html><h2>%s %s</h2></html>",n,messages[n],n,messages[n]) + local data = f_error_title(n,messages[n],n,messages[n]) report("handling error %s: %s",n,messages[n]) handlers.generic(client,configuration,data,nil,true) end @@ -166,42 +177,43 @@ local validpaths, registered = { }, { } function scripts.webserver.registerpath(name) if not registered[name] then - local cleanname = string.gsub(name,"%.%.","deleted-parent") - report("registering path '%s'",cleanname) + local cleanname = gsub(name,"%.%.","deleted-parent") + report("registering path: %s",cleanname) validpaths[#validpaths+1] = cleanname registered[name] = true end end function handlers.generic(client,configuration,data,suffix,iscontent) + local name = data if not iscontent then - local name = data - report("requested file '%s'",name) - local fullname = file.join(configuration.root,name) - data = io.loaddata(fullname) or "" + report("requested file: %s",name) + local fullname = joinpath(configuration.root,name) + data = loaddata(fullname) or "" if data == "" then for n=1,#validpaths do - local fullname = file.join(validpaths[n],name) - data = io.loaddata(fullname) or "" + local fullname = joinpath(validpaths[n],name) + data = loaddata(fullname) or "" if data ~= "" then - report("sending generic file '%s'",fullname) + report("sending generic file: %s",fullname) break end end else - report("sending generic file '%s'",fullname) + report("sending generic file: %s",fullname) end end if data and data ~= "" then client:send("HTTP/1.1 200 OK\r\n") client:send("Connection: close\r\n") - client:send(format("Content-Length: %s\r\n",#data)) - client:send(format("Content-Type: %s\r\n",(suffix and mimetypes[suffix]) or "text/html")) + client:send(f_content_length(#data)) + client:send(f_content_type(suffix and mimetypes[suffix] or "text/html")) client:send("Cache-Control: no-cache, no-store, must-revalidate, max-age=0\r\n") client:send("\r\n") client:send(data) client:send("\r\n") else + report("unknown file: %s",tostring(name)) errormessage(client,configuration,404) end end @@ -217,9 +229,9 @@ end local loaded = { } function handlers.lua(client,configuration,filename,suffix,iscontent,hashed) -- filename will disappear, and become hashed.filename - local filename = file.join(configuration.scripts,filename) - if not file.is_qualified_path(filename) then - filename = file.join(configuration.root,filename) + local filename = joinpath(configuration.scripts,filename) + if not is_qualified_path(filename) then + filename = joinpath(configuration.root,filename) end -- todo: split url in components, see l-url; rather trivial local result, keep = loaded[filename], false @@ -258,16 +270,19 @@ function handlers.lua(client,configuration,filename,suffix,iscontent,hashed) -- local action = handlers[suffix] or handlers.generic action(client,configuration,result.content,suffix,true) -- content elseif result.filename then - local suffix = file.suffix(result.filename) or "text/html" + local suffix = filesuffix(result.filename) or "text/html" local action = handlers[suffix] or handlers.generic action(client,configuration,result.filename,suffix,false) -- filename else + report("no content of filename in result") errormessage(client,configuration,404) end else + report("no valid result") errormessage(client,configuration,500) end else + report("no result") errormessage(client,configuration,404) end end @@ -276,21 +291,21 @@ handlers.luc = handlers.lua handlers.html = handlers.htm local indices = { "index.htm", "index.html" } -local portnumber = 31415 -- pi suits tex +local portnumber = 8088 -local newline = lpeg.patterns.newline -local spacer = lpeg.patterns.spacer -local whitespace = lpeg.patterns.whitespace -local method = lpeg.P("GET") - + lpeg.P("POST") +local newline = patterns.newline +local spacer = patterns.spacer +local whitespace = patterns.whitespace +local method = P("GET") + + P("POST") local identify = (1-method)^0 - * lpeg.C(method) + * C(method) * spacer^1 - * lpeg.C((1-spacer)^1) + * C((1-spacer)^1) * spacer^1 - * lpeg.P("HTTP/") + * P("HTTP/") * (1-whitespace)^0 - * lpeg.C(lpeg.P(1)^0) + * C(P(1)^0) function scripts.webserver.run(configuration) -- check configuration @@ -314,7 +329,7 @@ function scripts.webserver.run(configuration) if not configuration.index then for i=1,#indices do local name = indices[i] - if lfs.isfile(file.join(configuration.root,name)) then + if lfs.isfile(joinpath(configuration.root,name)) then configuration.index = name -- we will prepend the rootpath later break end @@ -322,7 +337,7 @@ function scripts.webserver.run(configuration) configuration.index = configuration.index or "unknown" end if not configuration.scripts or configuration.scripts == "" then - configuration.scripts = dir.expandname(file.join(configuration.root or ".",configuration.scripts or ".")) + configuration.scripts = dir.expandname(joinpath(configuration.root or ".",configuration.scripts or ".")) end -- so far for checks report("running at port: %s",configuration.port) @@ -333,20 +348,20 @@ function scripts.webserver.run(configuration) local server = assert(socket.bind("*", configuration.port)) local script = configuration.script while true do -- blocking - local start = os.clock() + -- local start = gettime() local client = server:accept() client:settimeout(configuration.timeout or 60) local request, e = client:receive() if e then - errormessage(client,configuration,404) + -- probably a time out + -- errormessage(client,configuration,404) else local from = client:getpeername() report("request from: %s",tostring(from)) report("request data: %s",tostring(request)) - -- local fullurl = string.match(request,"(GET) (.+) HTTP/.*$") or "" -- todo: more clever / post + -- local fullurl = match(request,"(GET) (.+) HTTP/.*$") or "" -- todo: more clever / post -- if fullurl == "" then --- print("!!!!",request) - local method, fullurl, body = lpeg.match(identify,request) + local method, fullurl, body = lpegmatch(identify,request) if method == "" or fullurl == "" then report("no url") errormessage(client,configuration,404) @@ -357,46 +372,50 @@ function scripts.webserver.run(configuration) fullurl = url.unescapeget(fullurl) report("requested url: %s",fullurl) -- fullurl = socket.url.unescape(fullurl) -- happens later - local hashed = url.hashed(fullurl) - local query = url.query(hashed.query) + local hashed = urlhashed(fullurl) + local query = urlquery(hashed.query) local filename = hashed.path -- hm, not query? hashed.body = body if script then filename = script report("forced script: %s",filename) - local suffix = file.suffix(filename) + local suffix = filesuffix(filename) local action = handlers[suffix] or handlers.generic if action then report("performing action: %s",filename) action(client,configuration,filename,suffix,false,hashed) -- filename and no content else + report("invalid action: %s",filename) errormessage(client,configuration,404) end elseif filename then - filename = socket.url.unescape(filename) - report("requested action: %s",filename) - if string.find(filename,"%.%.") then + local rawname = socket.url.unescape(filename) + filename = rawname + report("requested action: %s",filename or "?") + if find(filename,"%.%.") then filename = nil -- invalid path end if filename == nil or filename == "" or filename == "/" then filename = configuration.index report("invalid filename, forcing: %s",filename) end - local suffix = file.suffix(filename) + local suffix = filesuffix(filename) local action = handlers[suffix] or handlers.generic if action then - report("performing action: %s",filename) + report("performing action: %s",filename or "?") action(client,configuration,filename,suffix,false,hashed) -- filename and no content else + report("invalid action: %s",filename or "?") errormessage(client,configuration,404) end else + report("invalid request") errormessage(client,configuration,404) end end end client:close() - report("time spent with client: %0.03f seconds",os.clock()-start) + -- report("time spent with client: %0.03f seconds",gettime()-start) end end @@ -404,8 +423,8 @@ if environment.argument("auto") then local path = resolvers.findfile("mtx-server.lua") or "." scripts.webserver.run { port = environment.argument("port"), - root = environment.argument("root") or file.dirname(path) or ".", - scripts = environment.argument("scripts") or file.dirname(path) or ".", + root = environment.argument("root") or dirname(path) or ".", + scripts = environment.argument("scripts") or dirname(path) or ".", script = environment.argument("script"), } elseif environment.argument("start") then @@ -422,4 +441,4 @@ else application.help() end --- mtxrun --script server --start => http://localhost:31415/mtx-server-ctx-startup.lua +-- mtxrun --script server --start => http://localhost:8088/mtx-server-ctx-startup.lua diff --git a/scripts/context/lua/mtx-unicode.lua b/scripts/context/lua/mtx-unicode.lua new file mode 100644 index 000000000..673febc65 --- /dev/null +++ b/scripts/context/lua/mtx-unicode.lua @@ -0,0 +1,532 @@ +if not modules then modules = { } end modules ['mtx-unicode'] = { + version = 1.002, + comment = "companion to mtxrun.lua", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- This is very old code that I started writing in 2005 but occasionally +-- extended. Don't use it yourself, it's just a sort of reference. The +-- data that we use in ConTeXt is more extensive. +-- +-- In my local tree I keep files in places like this: +-- +-- e:/tex-context/tex/texmf-local/data/unicode/blocks.txt +-- +-- last checked: +-- +-- code freeze tl 2014 / unicode 7 +-- +-- todo: +-- +-- specialcasing ? + +local helpinfo = [[ +<?xml version="1.0"?> +<application> + <metadata> + <entry name="name">mtx-unicode</entry> + <entry name="detail">Checker for char-dat.lua</entry> + <entry name="version">1.02</entry> + </metadata> + <flags> + <category name="basic"> + <subcategory> + <flag name="whatever"><short>do whatever</short></flag> + </subcategory> + </category> + </flags> +</application> +]] + +local application = logs.application { + name = "mtx-unicode", + banner = "Checker for char-def.lua 1.02", + helpinfo = helpinfo, +} + +local gmatch, match, gsub, find, lower, format = string.gmatch, string.match, string.gsub, string.find, string.lower, string.format +local concat = table.concat +local split = string.split +local are_equal = table.are_equal +local tonumber = tonumber +local lpegmatch = lpeg.match +local formatters = string.formatters + +local report = application.report + +scripts = scripts or { } +scripts.unicode = scripts.unicode or { } + +characters = characters or { } +characters.data = characters.data or { } + +fonts = fonts or { } +fonts.encodings = fonts.encodings or { } + +local textfiles = { } +local textdata = { } + +local sparse = false + +local split_space_table = lpeg.tsplitat(" ") +local split_space_two = lpeg.splitat (" ") +local split_range_two = lpeg.splitat ("..") +local split_colon_table = lpeg.tsplitat(lpeg.P(" ")^0 * lpeg.P(";") * lpeg.P(" ")^0) + + +local skipped = { + [0x002C6] = true, -- MODIFIER LETTER CIRCUMFLEX ACCENT + [0x002C7] = true, -- CARON +} + +for i=0x0FE00,0x0FE0F do skipped[i] = true end -- variant selector +for i=0xE0100,0xE01EF do skipped[i] = true end -- variant selector extension + +function scripts.unicode.update() + local unicodedata = texttables.unicodedata + local bidimirroring = texttables.bidimirroring + local linebreak = texttables.linebreak + local eastasianwidth = texttables.eastasianwidth + local standardizedvariants = texttables.standardizedvariants + local arabicshaping = texttables.arabicshaping + local characterdata = characters.data + -- + for unicode, ud in table.sortedpairs(unicodedata) do + if not skipped[unicode] then + local char = rawget(characterdata,unicode) + local description = ud[2] or formatters["UNICODE ENTRY %U"](unicode) + if not find(description,"^<") then + local ld = linebreak[unicode] + local bd = bidimirroring[unicode] + local ed = eastasianwidth[unicode] + local category = lower(ud[3] or "?") + local combining = tonumber(ud[4]) + local direction = lower(ud[5] or "l") -- we could omit 'l' being the default + local linebreak = ld and lower(ld[2] or "xx") + local specials = ud[6] or "" + local cjkwd = ed and lower(ed[2] or "n") + local mirror = bd and tonumber(bd[2],16) + local arabic = nil + if sparse and direction == "l" then + direction = nil + end + if linebreak == "xx" then + linebreak = nil + end + if specials == "" then + specials = nil + else + specials = lpegmatch(split_space_table,specials) -- split(specials," ") + if tonumber(specials[1],16) then + for i=#specials,1,-1 do + specials[i+1] = tonumber(specials[i],16) + end + specials[1] = "char" + else + specials[1] = lower(gsub(specials[1],"[<>]","")) + for i=2,#specials do + specials[i] = tonumber(specials[i],16) + end + end + end + if cjkwd == "n" then + cjkwd = nil + end + local comment + if find(description,"MATHEMATICAL") then + comment = "check math properties" + end + -- there are more than arabic + local as = arabicshaping[unicode] + if as then + arabic = lower(as[3]) + end + -- + if not combining or combining == 0 then + combining = nil + end + if not char then + report("%U : adding entry %a",unicode,description) + char = { + -- adobename = , + category = category, + comment = comment, + cjkwd = cjkwd, + description = description, + direction = direction, + mirror = mirror, + linebreak = linebreak, + unicodeslot = unicode, + specials = specials, + arabic = arabic, + combining = combining, + } + characterdata[unicode] = char + else + if direction then + if char.direction ~= direction then + report("%U : setting direction to %a, %a",unicode,direction,description) + char.direction = direction + end + else + if char.direction then + report("%U : resetting direction from %a, %a",unicode,char.direction,description) + char.direction = nil + end + end + if mirror then + if mirror ~= char.mirror then + report("%U : setting mirror to %a, %a",unicode,mirror,description) + char.mirror = mirror + end + else + if char.mirror then + report("%U : resetting mirror from %a, %a",unicode,char.mirror,description) + char.mirror = nil + end + end + if linebreak then + if linebreak ~= char.linebreak then + report("%U : setting linebreak to %a, %a",unicode,linebreak,description) + char.linebreak = linebreak + end + else + if char.linebreak then + report("%U : resetting linebreak from %a, %a",unicode,char.linebreak,description) + char.linebreak = nil + end + end + if cjkwd then + if cjkwd ~= char.cjkwd then + report("%U : setting cjkwd of to %a, %a",unicode,cjkwd,description) + char.cjkwd = cjkwd + end + else + if char.cjkwd then + report("%U : resetting cjkwd of from %a, %a",unicode,char.cjkwd,description) + char.cjkwd = nil + end + end + if arabic then + if arabic ~= char.arabic then + report("%U : setting arabic to %a, %a",unicode,arabic,description) + char.arabic = arabic + end + else + if char.arabic then + report("%U : resetting arabic from %a, %a",unicode,char.arabic,description) + char.arabic = nil + end + end + if combining then + if combining ~= char.combining then + report("%U : setting combining to %a, %a",unicode,combining,description) + char.combining = combining + end + else + if char.combining then + report("%U : resetting combining from %a, %a",unicode,char.combining,description) + end + end + if specials then + if not char.specials or not are_equal(specials,char.specials) then + local t = { specials[1] } for i=2,#specials do t[i] = formatters["%U"](specials[i]) end + report("%U : setting specials to % + t, %a",unicode,t,description) + char.specials = specials + end + else + local specials = char.specials + if specials then + local t = { } for i=2,#specials do t[i] = formatters["%U"](specials[i]) end + if false then + char.comment = nil + report("%U : resetting specials from % + t, %a",unicode,t,description) + else + local comment = char.comment + if not comment then + char.comment = "check special" + elseif not find(comment,"check special") then + char.comment = comment .. ", check special" + end + report("%U : check specials % + t, %a",unicode,t,description) + end + end + end + end + -- + local visual = char.visual + if not visual and find(description,"MATH") then + if find(description,"BOLD ITALIC") then + visual = "bi" + elseif find(description,"ITALIC") then + visual = "it" + elseif find(description,"BOLD") then + visual = "bf" + end + if visual then + report("%U : setting visual to %a, %a",unicode,visual,description) + char.visual = visual + end + end + -- mathextensible + if category == "sm" or (category == "so" and char.mathclass) then + local mathextensible = char.mathextensible + if mathextensible then + -- already done + elseif find(description,"ABOVE") then + -- skip + elseif find(description,"ARROWHEAD") then + -- skip + elseif find(description,"HALFWIDTH") then + -- skip + elseif find(description,"ANGLE") then + -- skip + elseif find(description,"THROUGH") then + -- skip + elseif find(description,"ARROW") then + -- skip + local u = find(description,"UP") + local d = find(description,"DOWN") + local l = find(description,"LEFT") + local r = find(description,"RIGHT") + if find(description,"ARROWHEAD") then + -- skip + elseif find(description,"HALFWIDTH") then + -- skip + elseif u and d then + if l or r then + mathextensible = 'm' -- mixed + else + mathextensible = 'v' -- vertical + end + elseif u then + if l or r then + mathextensible = 'm' -- mixed + else + mathextensible = "u" -- up + end + elseif d then + if l or r then + mathextensible = 'm' -- mixed + else + mathextensible = "d" -- down + end + elseif l and r then + mathextensible = "h" -- horizontal + elseif r then + mathextensible = "r" -- right + elseif l then + mathextensible = "l" -- left + end + if mathextensible then + report("%U : setting mathextensible to %a, %a",unicode,mathextensible,description) + char.mathextensible = mathextensible + end + end + end + end + end + end + for i=1,#standardizedvariants do + local si = standardizedvariants[i] + local pair, addendum = si[1], string.strip(si[2]) + local first, second = lpegmatch(split_space_two,pair) -- string.splitup(pair," ") + first = tonumber(first,16) + second = tonumber(second,16) + if first then + local d = characterdata[first] + if d then + local v = d.variants + local v = rawget(d,"variants") + if not v then + v = { } + d.variants = v + end + if not v[second] then + report("%U : adding variant %U as %s, %a",first,second,addendum,d.description) + v[second] = addendum + end + end + end + end + for unicode, ud in table.sortedpairs(characterdata) do + if not rawget(ud,"category") and rawget(ud,"variants") then + report("stripping %U (variant, takes from metacharacter)",unicode) + characterdata[unicode] = nil + end + end +end + +local preamble + +local function splitdefinition(str,index) + local l = string.splitlines(str) + local t = { } + if index then + for i=1,#l do + local s = gsub(l[i]," *#.*$","") + if s ~= "" then + local d = lpegmatch(split_colon_table,s) -- split(s,";") + local o = d[1] + local u = tonumber(o,16) + if u then + t[u] = d + else + -- local b, e = match(o,"^([^%.]+)%.%.([^%.]+)$") + local b, e = lpegmatch(split_range_two,o) + if b and e then + b = tonumber(b,16) + e = tonumber(e,16) + for k=b,e do + t[k] = d + end + else + report("problem: %s",s) + end + end + end + end + else + local n = 0 + for i=1,#l do + local s = gsub(l[i]," *#.*$","") + if s ~= "" then + n = n + 1 + t[n] = lpegmatch(split_colon_table,s) -- split(s,";") + end + end + end + return t +end + +function scripts.unicode.load() + local fullname = resolvers.findfile("char-def.lua") + report("using: %s",fullname) + local data = io.loaddata(fullname) + if data then + loadstring(data)() + -- + local fullname = resolvers.findfile("char-ini.lua") + report("using: %s",fullname) + dofile(fullname) + -- + local fullname = resolvers.findfile("char-utf.lua") + report("using: %s",fullname) + dofile(fullname) + -- + local fullname = resolvers.findfile("char-cjk.lua") + report("using: %s",fullname) + dofile(fullname) + -- + preamble = data:gsub("characters%.data%s*=%s*%{.*","") + -- + textfiles = { + unicodedata = resolvers.findfile("unicodedata.txt") or "", + bidimirroring = resolvers.findfile("bidimirroring.txt") or "", + linebreak = resolvers.findfile("linebreak.txt") or "", + eastasianwidth = resolvers.findfile("eastasianwidth.txt") or "", + standardizedvariants = resolvers.findfile("standardizedvariants.txt") or "", + arabicshaping = resolvers.findfile("arabicshaping.txt") or "", + } + -- + textdata = { + unicodedata = textfiles.unicodedata ~= "" and io.loaddata(textfiles.unicodedata) or "", + bidimirroring = textfiles.bidimirroring ~= "" and io.loaddata(textfiles.bidimirroring) or "", + linebreak = textfiles.linebreak ~= "" and io.loaddata(textfiles.linebreak) or "", + eastasianwidth = textfiles.eastasianwidth ~= "" and io.loaddata(textfiles.eastasianwidth) or "", + standardizedvariants = textfiles.standardizedvariants ~= "" and io.loaddata(textfiles.standardizedvariants) or "", + arabicshaping = textfiles.arabicshaping ~= "" and io.loaddata(textfiles.arabicshaping) or "", + } + texttables = { + unicodedata = splitdefinition(textdata.unicodedata,true), + bidimirroring = splitdefinition(textdata.bidimirroring,true), + linebreak = splitdefinition(textdata.linebreak,true), + eastasianwidth = splitdefinition(textdata.eastasianwidth,true), + standardizedvariants = splitdefinition(textdata.standardizedvariants,false), + arabicshaping = splitdefinition(textdata.arabicshaping,true), + } + return true + else + preamble = nil + return false + end +end + +function scripts.unicode.save(filename) + if preamble then + io.savedata(filename,preamble .. table.serialize(characters.data,"characters.data", { hexify = true, noquotes = true } )) + end +end + +function scripts.unicode.extras() -- old code + -- + -- 0000..007F; Basic Latin + -- 0080..00FF; Latin-1 Supplement + -- 0100..017F; Latin Extended-A + -- + local fullname = resolvers.findfile("blocks.txt") or "" + if fullname ~= "" then + local data = io.loaddata(fullname) + local lines = string.splitlines(data) + local map = { } + local blocks = characters.blocks + local result = { } + for i=1,#lines do + local line = gsub(lines[i]," *#.*$","") + if line ~= "" then + local specification = lpegmatch(split_colon_table,line) -- split(s,";") + local range = specification[1] + local description = specification[2] + if range and description then + local start, stop = lpegmatch(split_range_two,range) + if start and stop then + local start = tonumber(start,16) + local stop = tonumber(stop,16) + local name = gsub(lower(description),"[^a-z]+","") + if start and stop then + local b = blocks[name] + if not b then + result[#result+1] = formatters[ [[+ block: ["%s"] = { first = 0x%05X, last = 0x%05X, description = "%S" }]] ](name,start,stop,description) + blocks[name] = { first = start, last = stop, description = description } + elseif b.first ~= start or b.last ~= stop or b.description ~= description then + result[#result+1] = formatters[ [[? block: ["%s"] = { first = 0x%05X, last = 0x%05X, description = "%S" }]] ](name,start,stop,description) + end + end + map[#map+1] = name + end + end + end + end + table.sort(result) + for i=1,#result do + report(result[i]) + end + table.sort(map) + for i=1,#map do + local m = map[i] + if not blocks[m] then + report("obsolete block %a",m) + end + end + end +end + +-- the action + +local filename = environment.files[1] + +if environment.arguments.exporthelp then + application.export(environment.arguments.exporthelp,filename) +else + report("start working on %a, input char-def.lua",lfs.currentdir()) + if scripts.unicode.load() then + scripts.unicode.update() + scripts.unicode.extras() + scripts.unicode.save("char-def-new.lua") + else + report("nothing to do") + end + report("stop working on %a, output char-def-new.lua\n",lfs.currentdir()) +end diff --git a/scripts/context/lua/mtx-update.lua b/scripts/context/lua/mtx-update.lua index daf4f5b16..8a23ba9a6 100644 --- a/scripts/context/lua/mtx-update.lua +++ b/scripts/context/lua/mtx-update.lua @@ -42,6 +42,7 @@ local helpinfo = [[ <flag name="state"><short>update tree using saved state</short></flag> <flag name="cygwin"><short>adapt drive specs to cygwin</short></flag> <flag name="mingw"><short>assume mingw binaries being used</short></flag> + <flag name="silent"><short>less (or no) logging</short></flag> </subcategory> </category> </flags> @@ -84,10 +85,10 @@ scripts.update.texformats = { "plain" } -scripts.update.mpformats = { - -- "metafun", - -- "mpost", -} +-- scripts.update.mpformats = { +-- -- "metafun", +-- -- "mpost", +-- } -- experimental is not functional at the moment @@ -280,6 +281,8 @@ function scripts.update.synchronize() local fonts = states.get("fonts") -- fonts (experimental or special) local goodies = states.get("goodies") -- goodies (like editors) local force = environment.argument("force") + local silent = environment.argument("silent") and "--silent" or "" + local quiet = silent == "" and "" or "--quiet" bin = gsub(bin,"\\","/") @@ -517,9 +520,9 @@ function scripts.update.synchronize() resolvers.load_tree(texroot) -- else we operate in the wrong tree -- update filename database for pdftex/xetex - scripts.update.run(format('mtxrun --tree="%s" --direct --resolve mktexlsr',texroot)) + scripts.update.run(format('mtxrun --tree="%s" %s --direct --resolve mktexlsr %s',texroot,silent,quiet)) -- update filename database for luatex - scripts.update.run(format('mtxrun --tree="%s" --generate',texroot)) + scripts.update.run(format('mtxrun --tree="%s" %s --generate',texroot,silent)) report("update, done") end @@ -538,6 +541,8 @@ function scripts.update.make() report("make, start") local force = environment.argument("force") + local silent = environment.argument("silent") and "--silent" or "" + local quiet = silent == "" and "" or "--quiet" local texroot = scripts.update.fullpath(states.get("paths.root")) local engines = states.get('engines') local goodies = states.get('goodies') @@ -546,43 +551,44 @@ function scripts.update.make() resolvers.load_tree(texroot) - scripts.update.run(format('mtxrun --tree="%s" --direct --resolve mktexlsr',texroot)) - scripts.update.run(format('mtxrun --tree="%s" --generate',texroot)) + scripts.update.run(format('mtxrun --tree="%s" %s --direct --resolve mktexlsr %s',texroot,silent,quiet)) + scripts.update.run(format('mtxrun --tree="%s" %s --generate',texroot,silent)) local askedformats = formats local texformats = table.tohash(scripts.update.texformats) - local mpformats = table.tohash(scripts.update.mpformats) + -- local mpformats = table.tohash(scripts.update.mpformats) for k,v in table.sortedhash(texformats) do if not askedformats[k] then texformats[k] = nil end end - for k,v in table.sortedhash(mpformats) do - if not askedformats[k] then - mpformats[k] = nil - end - end + -- for k,v in table.sortedhash(mpformats) do + -- if not askedformats[k] then + -- mpformats[k] = nil + -- end + -- end local formatlist = concat(table.fromhash(texformats), " ") if formatlist ~= "" then for engine in table.sortedhash(engines) do if engine == "luatex" or engine == "luajittex" then - scripts.update.run(format('mtxrun --tree="%s" --script context --autogenerate --make',texroot)) - scripts.update.run(format('mtxrun --tree="%s" --script context --autogenerate --make --engine=luajittex',texroot)) + scripts.update.run(format('mtxrun --tree="%s" %s --script context --autogenerate --make %s',texroot,silent,silent)) + scripts.update.run(format('mtxrun --tree="%s" %s --script context --autogenerate --make --engine=luajittex %s',texroot,silent,silent)) else - scripts.update.run(format('mtxrun --tree="%s" --script texexec --make --all --%s %s',texroot,engine,formatlist)) + -- scripts.update.run(format('mtxrun --tree="%s" %s --script texexec --make --all %s --%s %s',texroot,silent,silent,engine,formatlist)) + scripts.update.run(format('mtxrun --tree="%s" --resolve %s --script context --resolve --make %s --engine=%s %s',texroot,silent,silent,engine,formatlist)) end end end - local formatlist = concat(table.fromhash(mpformats), " ") - if formatlist ~= "" then - scripts.update.run(format('mtxrun --tree="%s" --script texexec --make --all %s',texroot,formatlist)) - end + -- local formatlist = concat(table.fromhash(mpformats), " ") + -- if formatlist ~= "" then + -- scripts.update.run(format('mtxrun --tree="%s" %s --script texexec --make --all %s %s',texroot,silent,silent,formatlist)) + -- end if not force then report("make, use --force to really make formats") end - scripts.update.run(format('mtxrun --tree="%s" --direct --resolve mktexlsr',texroot)) -- needed for mpost - scripts.update.run(format('mtxrun --tree="%s" --generate',texroot)) + -- scripts.update.run(format('mtxrun --tree="%s" %s --direct --resolve mktexlsr',texroot,silent)) -- needed for mpost + scripts.update.run(format('mtxrun --tree="%s" %s --generate',texroot,silent)) report("make, done") end @@ -649,14 +655,14 @@ if scripts.savestate then for r in gmatch(environment.argument("formats") or "","([^, ]+)") do if valid[r] then states.set("formats." .. r, true) end end - local valid = table.tohash(scripts.update.mpformats) - for r in gmatch(environment.argument("formats") or "","([^, ]+)") do - if valid[r] then states.set("formats." .. r, true) end - end + -- local valid = table.tohash(scripts.update.mpformats) + -- for r in gmatch(environment.argument("formats") or "","([^, ]+)") do + -- if valid[r] then states.set("formats." .. r, true) end + -- end states.set("formats.cont-en", true) states.set("formats.cont-nl", true) - states.set("formats.metafun", true) + -- states.set("formats.metafun", true) for r in gmatch(environment.argument("extras") or "","([^, ]+)") do -- for old times sake if r ~= "all" and not find(r,"^[a-z]%-") then diff --git a/scripts/context/lua/mtxlibs.lua b/scripts/context/lua/mtxlibs.lua index ae9d70108..6eee507ae 100644 --- a/scripts/context/lua/mtxlibs.lua +++ b/scripts/context/lua/mtxlibs.lua @@ -88,6 +88,8 @@ local ownlibs = { "util-str.lua", "util-tab.lua", + "util-fil.lua", + "util-sac.lua", "util-sto.lua", -- "util-lua.lua", -- no need for compiling "util-prs.lua", @@ -120,13 +122,19 @@ local ownlist = { '.', ownpath , ownpath .. "/../sources", -- HH's development path + -- + owntree .. "/../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../texmf/tex/context/base/mkiv", + owntree .. "/../../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../../texmf/tex/context/base/mkiv", + -- owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", - owntree .. "/../../texmf-dist/tex/context/base", owntree .. "/../../texmf/tex/context/base", owntree .. "/../../../texmf-local/tex/context/base", owntree .. "/../../../texmf-context/tex/context/base", - owntree .. "/../../../texmf-dist/tex/context/base", owntree .. "/../../../texmf/tex/context/base", } diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 5c09b3b44..7b711a88d 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3888, stripped down to: 2197 +-- original size: 4734, stripped down to: 2626 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -65,10 +65,14 @@ if not modules then modules={} end modules ['l-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") -_MAJORVERSION=tonumber(major) or 5 -_MINORVERSION=tonumber(minor) or 1 +_MAJORVERSION,_MINORVERSION=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(_MAJORVERSION) or 5 +_MINORVERSION=tonumber(_MINORVERSION) or 1 _LUAVERSION=_MAJORVERSION+_MINORVERSION/10 +if _LUAVERSION<5.2 and jit then + _MINORVERSION=2 + _LUAVERSION=5.2 +end if not lpeg then lpeg=require("lpeg") end @@ -111,21 +115,33 @@ if not package.loaders then end local print,select,tostring=print,select,tostring local inspectors={} -function setinspector(inspector) - inspectors[#inspectors+1]=inspector +function setinspector(kind,inspector) + inspectors[kind]=inspector end function inspect(...) for s=1,select("#",...) do local value=select(s,...) - local done=false - for i=1,#inspectors do - done=inspectors[i](value) - if done then - break + if value==nil then + print("nil") + else + local done=false + local kind=type(value) + local inspector=inspectors[kind] + if inspector then + done=inspector(value) + if done then + break + end + end + for kind,inspector in next,inspectors do + done=inspector(value) + if done then + break + end + end + if not done then + print(tostring(value)) end - end - if not done then - print(tostring(value)) end end end @@ -154,7 +170,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10587, stripped down to: 7815 +-- original size: 10949, stripped down to: 8037 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -444,7 +460,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 36977, stripped down to: 20349 +-- original size: 38185, stripped down to: 20990 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -461,7 +477,7 @@ local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print if setinspector then - setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) + setinspector("lpeg",function(v) if lpegtype(v) then lpegprint(v) return true end end) end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns @@ -481,7 +497,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=P("\r")*(P("\n")+P(true))+P("\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -1248,7 +1264,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 361, stripped down to: 322 +-- original size: 372, stripped down to: 329 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1267,7 +1283,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5694, stripped down to: 2827 +-- original size: 5983, stripped down to: 2959 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1354,9 +1370,10 @@ function string.valid(str,default) return (type(str)=="string" and str~="" and str) or default or nil end string.itself=function(s) return s end -local pattern=Ct(C(1)^0) -function string.totable(str) - return lpegmatch(pattern,str) +local pattern_c=Ct(C(1)^0) +local pattern_b=Ct((C(1)/byte)^0) +function string.totable(str,bytes) + return lpegmatch(bytes and pattern_b or pattern_c,str) end local replacer=lpeg.replacer("@","%%") function string.tformat(fmt,...) @@ -1372,7 +1389,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 35724, stripped down to: 21525 +-- original size: 36997, stripped down to: 22376 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2248,7 +2265,7 @@ function table.print(t,...) end end if setinspector then - setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) + setinspector("table",function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) end function table.sub(t,i,j) return { unpack(t,i,j) } @@ -2348,7 +2365,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8643, stripped down to: 6232 +-- original size: 9001, stripped down to: 6512 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2663,7 +2680,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 4939, stripped down to: 2830 +-- original size: 5146, stripped down to: 2933 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2808,7 +2825,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 1923, stripped down to: 1133 +-- original size: 2010, stripped down to: 1186 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2881,7 +2898,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 15832, stripped down to: 9456 +-- original size: 16390, stripped down to: 9734 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3263,7 +3280,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 20949, stripped down to: 9945 +-- original size: 21648, stripped down to: 10238 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3502,7 +3519,7 @@ local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") function file.join(one,two,three,...) if not two then - return one=="" and one or lpegmatch(stripper,one) + return one=="" and one or lpegmatch(reslasher,one) end if one=="" then return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) @@ -3643,7 +3660,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 1002 +-- original size: 1265, stripped down to: 1038 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3697,7 +3714,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3248, stripped down to: 2266 +-- original size: 3355, stripped down to: 2321 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3785,7 +3802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12531, stripped down to: 5721 +-- original size: 12897, stripped down to: 5882 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4002,7 +4019,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 16765, stripped down to: 11003 +-- original size: 17358, stripped down to: 11378 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4467,7 +4484,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1850, stripped down to: 1568 +-- original size: 1919, stripped down to: 1621 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4539,7 +4556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 37388, stripped down to: 15817 +-- original size: 38699, stripped down to: 16321 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4768,9 +4785,10 @@ if not utf.sub then end end end -function utf.remapper(mapping,option) +function utf.remapper(mapping,option,action) local variant=type(mapping) if variant=="table" then + action=action or mapping if option=="dynamic" then local pattern=false table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) @@ -4779,15 +4797,15 @@ function utf.remapper(mapping,option) return "" else if not pattern then - pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) end return lpegmatch(pattern,str) end end elseif option=="pattern" then - return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return Cs((tabletopattern(mapping)/action+p_utf8char)^0) else - local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + local pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) return function(str) if not str or str=="" then return "" @@ -5157,7 +5175,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 974, stripped down to: 890 +-- original size: 1012, stripped down to: 912 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5197,7 +5215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 34513, stripped down to: 18943 +-- original size: 36053, stripped down to: 19685 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5368,7 +5386,13 @@ function string.autosingle(s,sep) end return ("'"..tostring(s).."'") end -local tracedchars={} +local tracedchars={ [0]= + "[null]","[soh]","[stx]","[etx]","[eot]","[enq]","[ack]","[bel]", + "[bs]","[ht]","[lf]","[vt]","[ff]","[cr]","[so]","[si]", + "[dle]","[dc1]","[dc2]","[dc3]","[dc4]","[nak]","[syn]","[etb]", + "[can]","[em]","[sub]","[esc]","[fs]","[gs]","[rs]","[us]", + "[space]", +} string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) @@ -5885,7 +5909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 25338, stripped down to: 16247 +-- original size: 28680, stripped down to: 18636 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -6314,19 +6338,21 @@ local f_val_str=formatters["%w%q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] +local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") -local serialize=table.serialize -function table.serialize(root,name,specification) +local original_serialize=table.serialize +local function serialize(root,name,specification) if type(specification)=="table" then - return serialize(root,name,specification) + return original_serialize(root,name,specification) end - local t + local t local n=1 + local unknown=false local function simple_table(t) local nt=#t if nt>0 then @@ -6337,6 +6363,7 @@ function table.serialize(root,name,specification) return nil end end + local haszero=t[0] if n==nt then local tt={} for i=1,nt do @@ -6353,6 +6380,23 @@ function table.serialize(root,name,specification) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + tt[i+1]=v + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil @@ -6401,7 +6445,7 @@ function table.serialize(root,name,specification) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if next(v)==nil then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -6413,6 +6457,8 @@ function table.serialize(root,name,specification) end elseif tv=="boolean" then n=n+1 t[n]=f_val_boo(depth,v) + elseif unknown then + n=n+1 t[n]=f_val_str(depth,tostring(v)) end elseif tv=="number" then if tk=="number" then @@ -6421,6 +6467,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_num(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_num(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_num(depth,tostring(k),v) end elseif tv=="string" then if tk=="number" then @@ -6429,6 +6477,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_str(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_str(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),v) end elseif tv=="table" then if next(v)==nil then @@ -6438,6 +6488,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_not(depth,k) + elseif unknown then + n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else local st=simple_table(v) @@ -6449,6 +6501,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_seq(depth,k,st) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_seq(depth,k,st) + elseif unknown then + n=n+1 t[n]=f_key_str_value_seq(depth,tostring(k),st) end end elseif tv=="boolean" then @@ -6458,6 +6512,18 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_boo(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_boo(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_boo(depth,tostring(k),v) + end + else + if tk=="number" then + n=n+1 t[n]=f_key_num_value_str(depth,k,tostring(v)) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_str(depth,k,tostring(v)) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_str(depth,k,tostring(v)) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),tostring(v)) end end end @@ -6490,13 +6556,351 @@ function table.serialize(root,name,specification) root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - do_serialize(root,name,1,0) + local st=simple_table(root) + if st then + return t[1]..f_fin_seq(st) + else + do_serialize(root,name,1,0) + end end end n=n+1 t[n]=f_table_finish() return concat(t,"\n") end +table.serialize=serialize +if setinspector then + setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-fil"] = package.loaded["util-fil"] or true + +-- original size: 3577, stripped down to: 2870 + +if not modules then modules={} end modules ['util-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte=string.byte +local extract=bit32.extract +utilities=utilities or {} +local files={} +utilities.files=files +local zerobased={} +function files.open(filename,zb) + local f=io.open(filename,"rb") + if f then + zerobased[f]=zb or false + end + return f +end +function files.close(f) + zerobased[f]=nil + f:close() +end +function files.size(f) + return f:seek("end") +end +function files.setposition(f,n) + if zerobased[f] then + f:seek("set",n) + else + f:seek("set",n-1) + end +end +function files.getposition(f) + if zerobased[f] then + return f:seek() + else + return f:seek()+1 + end +end +function files.look(f,n,chars) + local p=f:seek() + local s=f:read(n) + f:seek("set",p) + if chars then + return s + else + return byte(s,1,#s) + end +end +function files.skip(f,n) + if n==1 then + f:read(n) + else + f:seek("set",f:seek()+n) + end +end +function files.readbyte(f) + return byte(f:read(1)) +end +function files.readbytes(f,n) + return byte(f:read(n),1,n) +end +function files.readchar(f) + return f:read(1) +end +function files.readstring(f,n) + return f:read(n or 1) +end +function files.readinteger1(f) + local n=byte(f:read(1)) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +files.readcardinal1=files.readbyte +files.readcardinal=files.readcardinal1 +files.readinteger=files.readinteger1 +function files.readcardinal2(f) + local a,b=byte(f:read(2),1,2) + return 0x100*a+b +end +function files.readinteger2(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function files.readcardinal3(f) + local a,b,c=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readcardinal4(f) + local a,b,c,d=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function files.readinteger4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function files.skipshort(f,n) + f:read(2*(n or 1)) +end +function files.skiplong(f,n) + f:read(4*(n or 1)) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sac"] = package.loaded["util-sac"] or true + +-- original size: 4264, stripped down to: 3349 + +if not modules then modules={} end modules ['util-sac']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte,sub=string.byte,string.sub +local extract=bit32.extract +utilities=utilities or {} +local streams={} +utilities.streams=streams +function streams.open(filename,zerobased) + local f=io.loaddata(filename) + return { f,1,#f,zerobased or false } +end +function streams.close() +end +function streams.size(f) + return f and f[3] or 0 +end +function streams.setposition(f,i) + if f[4] then + if i<=0 then + f[2]=1 + else + f[2]=i+1 + end + else + if i<=1 then + f[2]=1 + else + f[2]=i + end + end +end +function streams.getposition(f) + if f[4] then + return f[2]-1 + else + return f[2] + end +end +function streams.look(f,n,chars) + local b=f[2] + local e=b+n-1 + if chars then + return sub(f[1],b,e) + else + return byte(f[1],b,e) + end +end +function streams.skip(f,n) + f[2]=f[2]+n +end +function streams.readbyte(f) + local i=f[2] + f[2]=i+1 + return byte(f[1],i) +end +function streams.readbytes(f,n) + local i=f[2] + local j=i+n + f[2]=j + return byte(f[1],i,j-1) +end +function streams.skipbytes(f,n) + f[2]=f[2]+n +end +function streams.readchar(f) + local i=f[2] + f[2]=i+1 + return sub(f[1],i,i) +end +function streams.readstring(f,n) + local i=f[2] + local j=i+n + f[2]=j + return sub(f[1],i,j-1) +end +function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + local n=byte(f[1],i) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +streams.readcardinal1=streams.readbyte +streams.readcardinal=streams.readcardinal1 +streams.readinteger=streams.readinteger1 +function streams.readcardinal2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + return 0x100*a+b +end +function streams.readinteger2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function streams.readcardinal3(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readcardinal4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function streams.readinteger4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function streams.readfixed4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function streams.skipshort(f,n) + f[2]=f[2]+2*(n or 1) +end +function streams.skiplong(f,n) + f[2]=f[2]+4*(n or 1) +end end -- of closure @@ -6505,7 +6909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4172, stripped down to: 2953 +-- original size: 4100, stripped down to: 2852 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -6583,39 +6987,32 @@ local f_index={ ["table"]=f_table, ["number"]=f_number, } -local t_index={ - ["empty"]={ __index=f_empty }, - ["self"]={ __index=f_self }, - ["table"]={ __index=f_table }, - ["number"]={ __index=f_number }, -} function table.setmetatableindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__index=f_index[f] or f + m.__index=i else - setmetatable(t,t_index[f] or { __index=f }) + setmetatable(t,{ __index=i }) end return t end local f_index={ ["ignore"]=f_ignore, } -local t_index={ - ["ignore"]={ __newindex=f_ignore }, -} function table.setmetatablenewindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__newindex=f_index[f] or f + m.__newindex=i else - setmetatable(t,t_index[f] or { __newindex=f }) + setmetatable(t,{ __newindex=i }) end return t end @@ -6652,7 +7049,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 21780, stripped down to: 15121 +-- original size: 23411, stripped down to: 16177 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6676,6 +7073,8 @@ local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash local sortedkeys=table.sortedkeys local tohash=table.tohash +local hashes={} +utilities.parsers.hashes=hashes local digit=R("09") local space=P(' ') local equal=P("=") @@ -6684,6 +7083,8 @@ local lbrace=P("{") local rbrace=P("}") local lparent=P("(") local rparent=P(")") +local lbracket=P("[") +local rbracket=P("]") local period=S(".") local punctuation=S(".,:;") local spacer=lpegpatterns.spacer @@ -6693,6 +7094,7 @@ local anything=lpegpatterns.anything local endofstring=lpegpatterns.endofstring local nobrace=1-(lbrace+rbrace ) local noparent=1-(lparent+rparent) +local nobracket=1-(lbracket+rbracket) local escape,left,right=P("\\"),P('{'),P('}') lpegpatterns.balanced=P { [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, @@ -6700,6 +7102,7 @@ lpegpatterns.balanced=P { } local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local nestedbrackets=P { lbracket*(nobracket+V(1))^0*rbracket } local spaces=space^0 local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) local content=(1-endofstring)^0 @@ -6808,6 +7211,11 @@ function parsers.settings_to_array(str,strict) return { str } end end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_obey_fences(str) + return lpegmatch(pattern,str) +end local cache_a={} local cache_b={} function parsers.groupedsplitat(symbol,withaction) @@ -6894,9 +7302,15 @@ function parsers.array_to_string(a,separator) end end local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) -function utilities.parsers.settings_to_set(str,t) +function utilities.parsers.settings_to_set(str) return str and lpegmatch(pattern,str) or {} end +hashes.settings_to_set=table.setmetatableindex(function(t,k) + local v=k and lpegmatch(pattern,k) or {} + t[k]=v + return v +end) +getmetatable(hashes.settings_to_set).__mode="kv" function parsers.simple_hash_to_string(h,separator) local t,tn={},0 for k,v in sortedhash(h) do @@ -7173,7 +7587,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2274, stripped down to: 1781 +-- original size: 2350, stripped down to: 1847 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7254,7 +7668,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12482, stripped down to: 8864 +-- original size: 12862, stripped down to: 9104 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7567,7 +7981,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 29359, stripped down to: 20483 +-- original size: 30767, stripped down to: 21355 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7610,6 +8024,9 @@ setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then + if texio.setescape then + texio.setescape(0) + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -7627,6 +8044,8 @@ if tex and (tex.jobname or tex.formatname) then elseif target=="term" then texio_write_nl("term","") io_write(...) + elseif type(target)=="number" then + texio_write_nl(target,...) elseif target~="none" then texio_write_nl("log",target,...) texio_write_nl("term","") @@ -7644,6 +8063,8 @@ if tex and (tex.jobname or tex.formatname) then texio_write("log",...) elseif target=="term" then io_write(...) + elseif type(target)=="number" then + texio_write(target,...) elseif target~="none" then texio_write("log",target,...) io_write(target,...) @@ -7714,7 +8135,7 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end report=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,report_yes(translations[a],formats[b])) @@ -7725,7 +8146,7 @@ if tex and (tex.jobname or tex.formatname) then end end direct=function(a,b,c,...) - if c then + if c~=nil then return direct_yes(translations[a],formatters[formats[b]](c,...)) elseif b then return direct_yes(translations[a],formats[b]) @@ -7736,7 +8157,7 @@ if tex and (tex.jobname or tex.formatname) then end end subreport=function(a,s,b,c,...) - if c then + if c~=nil then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) elseif b then write_nl(target,subreport_yes(translations[a],translations[s],formats[b])) @@ -7747,7 +8168,7 @@ if tex and (tex.jobname or tex.formatname) then end end subdirect=function(a,s,b,c,...) - if c then + if c~=nil then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) elseif b then return subdirect_yes(translations[a],translations[s],formats[b]) @@ -7758,7 +8179,7 @@ if tex and (tex.jobname or tex.formatname) then end end status=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,status_yes(translations[a],formats[b])) @@ -8056,7 +8477,7 @@ function logs.messenger(category,subcategory) end end end -local function setblocked(category,value) +local function setblocked(category,value) if category==true then category,value="*",true elseif category==false then @@ -8071,7 +8492,7 @@ local function setblocked(category,value) end else states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) - for c,_ in next,states do + for c in next,states do local v=data[c] if v then v.state=value @@ -8353,7 +8774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6704, stripped down to: 5343 +-- original size: 6917, stripped down to: 5484 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8474,13 +8895,13 @@ function statistics.show() end end register("lua properties",function() - local list=status.list() - local hashchar=tonumber(list.luatex_hashchars) + local hashchar=tonumber(status.luatex_hashchars) + local hashtype=status.luatex_hashtype local mask=lua.mask or "ascii" return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", jit and "luajit" or "lua", statistics.memused(), - list.luatex_hashtype or "default", + hashtype or "default", hashchar and 2^hashchar or "unknown", mask, mask=="utf" and "τεχ" or "tex") @@ -8534,7 +8955,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5829, stripped down to: 3501 +-- original size: 6039, stripped down to: 3616 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -8681,7 +9102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 4982, stripped down to: 3511 +-- original size: 5142, stripped down to: 3611 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -8811,7 +9232,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3898, stripped down to: 2644 +-- original size: 4030, stripped down to: 2718 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8915,7 +9336,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7757, stripped down to: 6015 +-- original size: 7985, stripped down to: 6153 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9092,7 +9513,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7100, stripped down to: 3978 +-- original size: 7313, stripped down to: 4076 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -9237,7 +9658,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8022, stripped down to: 5038 +-- original size: 8284, stripped down to: 5176 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9424,7 +9845,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6174, stripped down to: 4141 +-- original size: 6358, stripped down to: 4257 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9577,7 +9998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 45848, stripped down to: 27914 +-- original size: 56973, stripped down to: 35872 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -9586,7 +10007,7 @@ if not modules then modules={} end modules ['lxml-tab']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local trace_entities=false trackers .register("xml.entities",function(v) trace_entities=v end) local report_xml=logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} @@ -9594,10 +10015,12 @@ local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub +local sort=table.sort local utfchar=utf.char local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters +do xml.xmlns=xml.xmlns or {} local check=P(false) local parse=check @@ -9614,23 +10037,68 @@ end function xml.resolvens(url) return lpegmatch(parse,lower(url)) or "" end +end local nsremap,resolvens=xml.xmlns,xml.resolvens -local stack={} -local top={} -local dt={} -local at={} -local xmlns={} -local errorstr=nil -local entities={} -local strip=false -local cleanup=false -local utfize=false -local resolve_predefined=false -local unify_predefined=false -local dcache={} -local hcache={} -local acache={} -local mt={} +local stack,level,top,at,xmlnms,errorstr +local entities,parameters +local strip,utfize,resolve,cleanup,resolve_predefined,unify_predefined +local dcache,hcache,acache +local mt,dt,nt +local function preparexmlstate(settings) + if settings then + stack={} + level=0 + top={} + at={} + mt={} + dt={} + nt=0 + xmlns={} + errorstr=nil + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + parameters={} + reported_at_errors={} + dcache={} + hcache={} + acache={} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + else + stack=nil + level=nil + top=nil + at=nil + mt=nil + dt=nil + nt=nil + xmlns=nil + errorstr=nil + strip=nil + utfize=nil + resolve=nil + resolve_predefined=nil + unify_predefined=nil + cleanup=nil + entities=nil + parameters=nil + reported_at_errors=nil + dcache=nil + hcache=nil + acache=nil + end +end local function initialize_mt(root) mt={ __index=root } end @@ -9640,8 +10108,9 @@ end function xml.checkerror(top,toclose) return "" end +local checkns=xml.checkns local function add_attribute(namespace,tag,value) - if cleanup and #value>0 then + if cleanup and value~="" then value=cleanup(value) end if tag=="xmlns" then @@ -9650,21 +10119,30 @@ local function add_attribute(namespace,tag,value) elseif namespace=="" then at[tag]=value elseif namespace=="xmlns" then - xml.checkns(tag,value) + checkns(tag,value) at["xmlns:"..tag]=value else at[namespace..":"..tag]=value end end local function add_empty(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top=stack[#stack] + top=stack[level] dt=top.dt - local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } - dt[#dt+1]=t + nt=#dt+1 + local t={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=top + } + dt[nt]=t setmetatable(t,mt) if at.xmlns then remove(xmlns) @@ -9672,23 +10150,35 @@ local function add_empty(spacing,namespace,tag) at={} end local function add_begin(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + top={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=stack[level] + } setmetatable(top,mt) dt=top.dt - stack[#stack+1]=top + nt=#dt + level=level+1 + stack[level]=top at={} end local function add_end(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end - local toclose=remove(stack) - top=stack[#stack] - if #stack<1 then + local toclose=stack[level] + level=level-1 + top=stack[level] + if level<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") report_xml(errorstr) elseif toclose.tg~=tag then @@ -9696,202 +10186,236 @@ local function add_end(spacing,namespace,tag) report_xml(errorstr) end dt=top.dt - dt[#dt+1]=toclose + nt=#dt+1 + dt[nt]=toclose if toclose.at.xmlns then remove(xmlns) end end -local spaceonly=lpegpatterns.whitespace^0*P(-1) local function add_text(text) - local n=#dt - if cleanup and #text>0 then - if n>0 then - local s=dt[n] + if text=="" then + return + end + if cleanup then + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..cleanup(text) + dt[nt]=s..cleanup(text) else - dt[n+1]=cleanup(text) + nt=nt+1 + dt[nt]=cleanup(text) end else + nt=1 dt[1]=cleanup(text) end else - if n>0 then - local s=dt[n] + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..text + dt[nt]=s..text else - dt[n+1]=text + nt=nt+1 + dt[nt]=text end else + nt=1 dt[1]=text end end end local function add_special(what,spacing,text) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end if strip and (what=="@cm@" or what=="@dt@") then else - dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + nt=nt+1 + dt[nt]={ special=true,ns="",tg=what,dt={ text } } end end local function set_message(txt) errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end -local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute value %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end local function attribute_specification_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute specification %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end -local badentity="&error;" -local badentity="&" -xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, - unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, - unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, -} -local placeholders=xml.placeholders -local function fromhex(s) - local n=tonumber(s,16) - if n then - return utfchar(n) - else - return formatters["h:%s"](s),true - end -end -local function fromdec(s) - local n=tonumber(s) - if n then - return utfchar(n) - else - return formatters["d:%s"](s),true - end -end -local p_rest=(1-P(";"))^0 -local p_many=P(1)^0 -local p_char=lpegpatterns.utf8character -local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) -local predefined_unified={ - [38]="&", - [42]=""", - [47]="'", - [74]="<", - [76]=">", -} -local predefined_simplified={ - [38]="&",amp="&", - [42]='"',quot='"', - [47]="'",apos="'", - [74]="<",lt="<", - [76]=">",gt=">", -} -local nofprivates=0xF0000 -local privates_u={ - [ [[&]] ]="&", - [ [["]] ]=""", - [ [[']] ]="'", - [ [[<]] ]="<", - [ [[>]] ]=">", -} -local privates_p={} -local privates_n={ -} -local escaped=utf.remapper(privates_u,"dynamic") -local unprivatized=utf.remapper(privates_p,"dynamic") -xml.unprivatized=unprivatized -local function unescaped(s) - local p=privates_n[s] - if not p then - nofprivates=nofprivates+1 - p=utfchar(nofprivates) - privates_n[s]=p - s="&"..s..";" - privates_u[p]=s - privates_p[p]=s +local grammar_parsed_text_one +local grammar_parsed_text_two +local handle_hex_entity +local handle_dec_entity +local handle_any_entity_dtd +local handle_any_entity_text +do + local badentity="&" + xml.placeholders={ + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, + unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, + unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, + } + local function fromhex(s) + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return formatters["h:%s"](s),true + end end - return p -end -xml.privatetoken=unescaped -xml.privatecodes=privates_n -local function handle_hex_entity(str) - local h=hcache[str] - if not h then - local n=tonumber(str,16) - h=unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end - elseif utfize then - h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end + local function fromdec(s) + local n=tonumber(s) + if n then + return utfchar(n) else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h="&#x"..str..";" + return formatters["d:%s"](s),true + end + end + local p_rest=(1-P(";"))^0 + local p_many=P(1)^0 + local p_char=lpegpatterns.utf8character + local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) + xml.parsedentitylpeg=parsedentity + local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", + } + local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", + } + local nofprivates=0xF0000 + local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", + } + local privates_p={ + } + local privates_s={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[&]] ]="&U+26;", + [ [[']] ]="&U+27;", + [ [[<]] ]="&U+3C;", + [ [[>]] ]="&U+3E;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } + local privates_n={ + } + local escaped=utf.remapper(privates_u,"dynamic") + local unprivatized=utf.remapper(privates_p,"dynamic") + local unspecialized=utf.remapper(privates_s,"dynamic") + xml.unprivatized=unprivatized + xml.unspecialized=unspecialized + xml.escaped=escaped + local function unescaped(s) + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + privates_s[p]=s end - hcache[str]=h + return p end - return h -end -local function handle_dec_entity(str) - local d=dcache[str] - if not d then - local n=tonumber(str) - d=unify_predefined and predefined_unified[n] - if d then - if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) - end - elseif utfize then - d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) + xml.privatetoken=unescaped + xml.privatecodes=privates_n + xml.specialcodes=privates_s + function xml.addspecialcode(key,value) + privates_s[key]=value or "&"..s..";" + end + handle_hex_entity=function(str) + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - else - if trace_entities then - report_xml("found entity &#%s;",str) + hcache[str]=h + end + return h + end + handle_dec_entity=function(str) + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" end - d="&#"..str..";" + dcache[str]=d end - dcache[str]=d + return d end - return d -end -xml.parsedentitylpeg=parsedentity -local function handle_any_entity(str) - if resolve then - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + handle_any_entity_dtd=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then if trace_entities then report_xml("resolving entity &%s; to predefined %a",str,a) end else if type(resolve)=="function" then - a=resolve(str) or entities[str] + a=resolve(str,entities) or entities[str] else a=entities[str] end @@ -9927,40 +10451,194 @@ local function handle_any_entity(str) end end end - acache[str]=a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; to %a",str,a) - acache[str]=a + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a end - return a - else - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + end + handle_any_entity_text=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then - acache[str]=a - if trace_entities then - report_xml("entity &%s; becomes %a",str,a) - end - elseif str=="" then if trace_entities then - report_xml("invalid entity &%s;",str) + report_xml("resolving entity &%s; to predefined %a",str,a) end - a=badentity - acache[str]=a else - if trace_entities then - report_xml("entity &%s; is made private",str) + if type(resolve)=="function" then + a=resolve(str,entities) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then + if trace_entities then + report_xml("expanding entity &%s; to function call",str) + end + a=a(str) or "" + end + a=lpegmatch(grammar_parsed_text_two,a) or a + if type(a)=="number" then + return "" + else + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + end + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then + if trace_entities then + report_xml("resolving entity &%s; to external %s",str,a) + end + else + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str=="" then + a=badentity + else + a="&"..str..";" + end + end + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a end - a=unescaped(str) - acache[str]=a end + return a + end + end + local p_rest=(1-P(";"))^1 + local spec={ + [0x23]="\\Ux{23}", + [0x24]="\\Ux{24}", + [0x25]="\\Ux{25}", + [0x5C]="\\Ux{5C}", + [0x7B]="\\Ux{7B}", + [0x7C]="\\Ux{7C}", + [0x7D]="\\Ux{7D}", + [0x7E]="\\Ux{7E}", + } + local hash=table.setmetatableindex(spec,function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true end - return a end + local reparsedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + local hash=table.setmetatableindex(function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true + end + end + local unescapedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + xml.reparsedentitylpeg=reparsedentity + xml.unescapedentitylpeg=unescapedentity end +local escaped=xml.escaped +local unescaped=xml.unescaped +local placeholders=xml.placeholders local function handle_end_entity(str) report_xml("error in entity, %a found without ending %a",str,";") return str @@ -9987,14 +10665,18 @@ local name=name_yes+name_nop local utfbom=lpegpatterns.utfbom local spacing=C(space^0) local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 -local hexentitycontent=R("AF","af","09")^0 -local decentitycontent=R("09")^0 +local hexentitycontent=R("AF","af","09")^1 +local decentitycontent=R("09")^1 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) - )+(anyentitycontent/handle_any_entity) + )+(anyentitycontent/handle_any_entity_dtd) +local parsedentity_text=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity_text) local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) +local entity_text=(ampersand/"")*parsedentity_text*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) -local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local text_parsed=(Cs((1-open-ampersand)^1)/add_text+Cs(entity_text)/add_text)^1 local somespace=space^1 local optionalspace=space^0 local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) @@ -10004,7 +10686,7 @@ local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error local attributevalue=value+wrongvalue local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 -local parsedtext=text_parsed/add_text +local parsedtext=text_parsed local unparsedtext=text_unparsed/add_text local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty @@ -10019,21 +10701,52 @@ local endcdata=P("]]")*close local someinstruction=C((1-endinstruction)^0) local somecomment=C((1-endcomment )^0) local somecdata=C((1-endcdata )^0) -local function normalentity(k,v ) entities[k]=v end -local function systementity(k,v,n) entities[k]=v end -local function publicentity(k,v,n) entities[k]=v end +local function weirdentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","weird",k,v) + end + parameters[k]=v +end +local function normalentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","normal",k,v) + end + entities[k]=v +end +local function systementity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","system",k,v) + end + entities[k]=v +end +local function publicentity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","public",k,v) + end + entities[k]=v +end local begindoctype=open*P("!DOCTYPE") local enddoctype=close local beginset=P("[") local endset=P("]") +local wrdtypename=C((1-somespace-P(";"))^1) local doctypename=C((1-somespace-close)^0) local elementdoctype=optionalspace*P("<!ELEMENT")*(1-close)^0*close local basiccomment=begincomment*((1-endcomment)^0)*endcomment +local weirdentitytype=P("%")*(somespace*doctypename*somespace*value)/weirdentity local normalentitytype=(doctypename*somespace*value)/normalentity local publicentitytype=(doctypename*somespace*P("PUBLIC")*somespace*value)/publicentity local systementitytype=(doctypename*somespace*P("SYSTEM")*somespace*value*somespace*P("NDATA")*somespace*doctypename)/systementity -local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype)*optionalspace*close -local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+basiccomment+space)^0*optionalspace*endset +local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype+weirdentitytype)*optionalspace*close +local function weirdresolve(s) + lpegmatch(entitydoctype,parameters[s]) +end +local function normalresolve(s) + lpegmatch(entitydoctype,entities[s]) +end +local entityresolve=P("%")*(wrdtypename/weirdresolve )*P(";")+P("&")*(wrdtypename/normalresolve)*P(";") +entitydoctype=entitydoctype+entityresolve +local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+entityresolve+basiccomment+space)^0*optionalspace*endset local definitiondoctype=doctypename*somespace*doctypeset local publicdoctype=doctypename*somespace*P("PUBLIC")*somespace*value*somespace*value*somespace*doctypeset local systemdoctype=doctypename*somespace*P("SYSTEM")*somespace*value*somespace*doctypeset @@ -10045,11 +10758,15 @@ local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata -local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error -local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 -local grammar_parsed_text=P { "preamble", - preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, +grammar_parsed_text_one=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0, +} +grammar_parsed_text_two=P { "followup", + followup=V("parent")*trailer, parent=beginelement*V("children")^0*endelement, children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } @@ -10059,37 +10776,26 @@ local grammar_unparsed_text=P { "preamble", children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) - settings=settings or {} - strip=settings.strip_cm_and_dt - utfize=settings.utfize_entities - resolve=settings.resolve_entities - resolve_predefined=settings.resolve_predefined_entities - unify_predefined=settings.unify_predefined_entities - cleanup=settings.text_cleanup - entities=settings.entities or {} - if utfize==nil then - settings.utfize_entities=true - utfize=true - end - if resolve_predefined==nil then - settings.resolve_predefined_entities=true - resolve_predefined=true - end - stack,top,at,xmlns,errorstr={},{},{},{},nil - acache,hcache,dcache={},{},{} - reported_attribute_errors={} + settings=settings or {} + preparexmlstate(settings) if settings.parent_root then mt=getmetatable(settings.parent_root) else initialize_mt(top) end - stack[#stack+1]=top + level=level+1 + stack[level]=top top.dt={} dt=top.dt + nt=0 if not data or data=="" then errorstr="empty xml file" elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then + local m=lpegmatch(grammar_parsed_text_one,data) + if m then + m=lpegmatch(grammar_parsed_text_two,data,m) + end + if m then else errorstr="invalid xml file - parsed text" end @@ -10105,8 +10811,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } -setmetatable(result,mt) -setmetatable(result.dt[1],mt) + setmetatable(result,mt) + setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -10148,13 +10854,10 @@ setmetatable(result.dt[1],mt) decimals=dcache, hexadecimals=hcache, names=acache, + intermediates=parameters, } } - strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil - unify_predefined,cleanup,entities=nil,nil,nil - stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil - acache,hcache,dcache=nil,nil,nil - reported_attribute_errors,mt,errorhandler=nil,nil,nil + preparexmlstate() return result end local function xmlconvert(data,settings) @@ -10216,15 +10919,15 @@ function xml.toxml(data) return data end end -local function copy(old,tables) +local function copy(old) if old then - tables=tables or {} local new={} - if not tables[old] then - tables[old]=new - end for k,v in next,old do - new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v + if type(v)=="table" then + new[k]=table.copy(v) + else + new[k]=v + end end local mt=getmetatable(old) if mt then @@ -10257,22 +10960,34 @@ local function verbose_element(e,handlers,escape) local ats=eat and next(eat) and {} if ats then local n=0 - for k,v in next,eat do + for k in next,eat do n=n+1 - ats[n]=f_attribute(k,escaped(v)) + ats[n]=k + end + if n==1 then + local k=ats[1] + ats=f_attribute(k,escaped(eat[k])) + else + sort(ats) + for i=1,n do + local k=ats[i] + ats[i]=f_attribute(k,escaped(eat[k])) + end + ats=concat(ats," ") end end if ern and trace_entities and ern~=ens then ens=ern end + local n=edt and #edt if ens~="" then - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") + handle("<",ens,":",etg," ",ats,">") else handle("<",ens,":",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10283,19 +10998,19 @@ local function verbose_element(e,handlers,escape) handle("</",ens,":",etg,">") else if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") + handle("<",ens,":",etg," ",ats,"/>") else handle("<",ens,":",etg,"/>") end end else - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",etg," ",concat(ats," "),">") + handle("<",etg," ",ats,">") else handle("<",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10306,7 +11021,7 @@ local function verbose_element(e,handlers,escape) handle("</",etg,">") else if ats then - handle("<",etg," ",concat(ats," "),"/>") + handle("<",etg," ",ats,"/>") else handle("<",etg,"/>") end @@ -10323,7 +11038,7 @@ local function verbose_cdata(e,handlers) handlers.handle("<![CDATA[",e.dt[1],"]]>") end local function verbose_doctype(e,handlers) - handlers.handle("<!DOCTYPE ",e.dt[1],">") + handlers.handle("<!DOCTYPE",e.dt[1],">") end local function verbose_root(e,handlers) handlers.serialize(e.dt,handlers) @@ -10366,12 +11081,14 @@ local function serialize(e,handlers,...) end end local function xserialize(e,handlers) - local functions=handlers.functions - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) + if e then + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end end end local handlers={} @@ -10603,7 +11320,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48229, stripped down to: 30684 +-- original size: 53892, stripped down to: 32508 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10618,10 +11335,23 @@ local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string. local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local setmetatableindex=table.setmetatableindex local formatters=string.formatters -local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end -local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end -local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local trace_lpath=false +local trace_lparse=false +local trace_lprofile=false local report_lpath=logs.reporter("xml","lpath") +if trackers then + trackers.register("xml.path",function(v) + trace_lpath=v + end) + trackers.register("xml.parse",function(v) + trace_lparse=v + end) + trackers.register("xml.profile",function(v) + trace_lpath=v + trace_lparse=v + trace_lprofile=v + end) +end local xml=xml local lpathcalls=0 function xml.lpathcalls () return lpathcalls end local lpathcached=0 function xml.lpathcached() return lpathcached end @@ -10980,13 +11710,27 @@ local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") local lp_doequal=P("=")/"==" local lp_or=P("|")/" or " local lp_and=P("&")/" and " -local lp_builtin=P ( - P("text")/"(ll.dt[1] or '')"+ - P("content")/"ll.dt"+ - P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ - P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ - P("ns")/"ll.ns" - )*((spaces*P("(")*spaces*P(")"))/"") +local builtin={ + text="(ll.dt[1] or '')", + content="ll.dt", + name="((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)", + tag="ll.tg", + position="l", + firstindex="1", + firstelement="1", + first="1", + lastindex="(#ll.__p__.dt or 1)", + lastelement="(ll.__p__.en or 1)", + last="#list", + rootposition="order", + order="order", + element="(ll.ei or 1)", + index="(ll.ni or 1)", + match="(ll.mi or 1)", + namespace="ll.ns", + ns="ll.ns", +} +local lp_builtin=lpeg.utfchartabletopattern(builtin)/builtin*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" @@ -11005,7 +11749,7 @@ local rparent=P(")") local noparent=1-(lparent+rparent) local nested=P{lparent*(noparent+V(1))^0*rparent} local value=P(lparent*C((noparent+nested)^0)*rparent) -local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_child=Cc("expr.child(ll,'")*R("az","AZ")*R("az","AZ","--","__")^0*Cc("')") local lp_number=S("+-")*R("09")^1 local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) @@ -11044,6 +11788,7 @@ local template_f_y=[[ local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] +local register_last_match={ kind="axis",axis="last-match" } local register_self={ kind="axis",axis="self" } local register_parent={ kind="axis",axis="parent" } local register_descendant={ kind="axis",axis="descendant" } @@ -11121,7 +11866,7 @@ local pathparser=Ct { "patterns", ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, - axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, @@ -11147,6 +11892,7 @@ local pathparser=Ct { "patterns", preceding=P('preceding::')*Cc(register_preceding ), preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + last_match=P('last-match::')*Cc(register_last_match ), nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -11193,13 +11939,12 @@ local function tagstostring(list) end xml.nodesettostring=nodesettostring local lpath -local lshowoptions={ functions=false } local function lshow(parsed) if type(parsed)=="string" then parsed=lpath(parsed) end report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + table.serialize(parsed,false)) end xml.lshow=lshow local function add_comment(p,str) @@ -11265,140 +12010,168 @@ lpath=function (pattern) end end xml.lpath=lpath -local profiled={} xml.profiled=profiled -local function profiled_apply(list,parsed,nofparsed,order) - local p=profiled[parsed.pattern] - if p then - p.tested=p.tested+1 - else - p={ tested=1,matched=0,finalized=0 } - profiled[parsed.pattern]=p - end - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - p.matched=p.matched+1 - p.finalized=p.finalized+1 - return collected - end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) +do + local profiled={} + xml.profiled=profiled + local lastmatch=nil + local keepmatch=nil + if directives then + directives.register("xml.path.keeplastmatch",function(v) + keepmatch=v + lastmatch=nil + end) + end + apply_axis["last-match"]=function() + return lastmatch or {} + end + local function profiled_apply(list,parsed,nofparsed,order) + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 p.finalized=p.finalized+1 return collected end - return nil + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + p.finalized=p.finalized+1 + return collected + end + return nil + end end - end - if collected then - p.matched=p.matched+1 - end - return collected -end -local function traced_apply(list,parsed,nofparsed,order) - if trace_lparse then - lshow(parsed) - end - report_lpath("collecting: %s",parsed.pattern) - report_lpath("root tags : %s",tagstostring(list)) - report_lpath("order : %s",order or "unset") - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected + if collected then + p.matched=p.matched+1 end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") return collected end - return nil - end - end - return collected -end -local function normal_apply(list,parsed,nofparsed,order) - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - local axis=pi.axis - if axis~="self" then - collected=apply_axis[axis](collected) + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil end - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - return pi.finalizer(collected) end - if not collected or #collected==0 then - local pf=i<nofparsed and parsed[nofparsed].finalizer - if pf then - return pf(collected) + return collected + end + local function normal_apply(list,parsed,nofparsed,order) + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + local axis=pi.axis + if axis~="self" then + collected=apply_axis[axis](collected) + end + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected==0 then + local pf=i<nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) + end + return nil end - return nil end + return collected end - return collected -end -local function applylpath(list,pattern) - if not list then - return - end - local parsed=cache[pattern] - if parsed then - lpathcalls=lpathcalls+1 - lpathcached=lpathcached+1 - elseif type(pattern)=="table" then - lpathcalls=lpathcalls+1 - parsed=pattern - else - parsed=lpath(pattern) or pattern - end - if not parsed then - return + local apply=normal_apply + if trackers then + trackers.register("xml.path,xml.parse,xml.profile",function() + if trace_lprofile then + apply=profiled_apply + elseif trace_lpath then + apply=traced_apply + else + apply=normal_apply + end + end) end - local nofparsed=#parsed - if nofparsed==0 then - return + function xml.applylpath(list,pattern) + if not list then + lastmatch=nil + return + end + local parsed=cache[pattern] + if parsed then + lpathcalls=lpathcalls+1 + lpathcached=lpathcached+1 + elseif type(pattern)=="table" then + lpathcalls=lpathcalls+1 + parsed=pattern + else + parsed=lpath(pattern) or pattern + end + if not parsed then + lastmatch=nil + return + end + local nofparsed=#parsed + if nofparsed==0 then + lastmatch=nil + return + end + local collected=apply({ list },parsed,nofparsed,list.mi) + lastmatch=keepmatch and collected or nil + return collected end - if not trace_lpath then - return normal_apply ({ list },parsed,nofparsed,list.mi) - elseif trace_lprofile then - return profiled_apply({ list },parsed,nofparsed,list.mi) - else - return traced_apply ({ list },parsed,nofparsed,list.mi) + function xml.lastmatch() + return lastmatch end end -xml.applylpath=applylpath +local applylpath=xml.applylpath function xml.filter(root,pattern) return applylpath(root,pattern) end @@ -11676,7 +12449,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3684, stripped down to: 1957 +-- original size: 3787, stripped down to: 2003 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -11745,7 +12518,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 28786, stripped down to: 20578 +-- original size: 30566, stripped down to: 21741 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12079,55 +12852,63 @@ local function include(xmldata,pattern,attribute,recursive,loaddata,level) local ek=collected[c] local name=nil local ekdt=ek.dt - local ekat=ek.at - local ekrt=ek.__p__ - local epdt=ekrt.dt - if not attribute or attribute=="" then - name=(type(ekdt)=="table" and ekdt[1]) or ekdt - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name=ekat[a] - if name then - break + if ekdt then + local ekat=ek.at + local ekrt=ek.__p__ + if ekrt then + local epdt=ekrt.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt end - end - end - local data=nil - if name and name~="" then - data=loaddata(name) or "" - if trace_inclusions then - report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") - end - end - if not data or data=="" then - epdt[ek.ni]="" - elseif ekat["parse"]=="text" then - epdt[ek.ni]=xml.escaped(data) - else - local xi=xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni]="" - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata,level+1) + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then + break + end + end end - local child=xml.body(xi) - child.__p__=ekrt - child.__f__=name - epdt[ek.ni]=child - local inclusions=xmldata.settings.inclusions - if inclusions then - inclusions[#inclusions+1]=name - else - xmldata.settings.inclusions={ name } + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") + end end - if child.er then - local badinclusions=xmldata.settings.badinclusions - if badinclusions then - badinclusions[#badinclusions+1]=name + if not data or data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" else - xmldata.settings.badinclusions={ name } + if recursive then + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local settings=xmldata.settings + local inclusions=settings and settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + elseif settings then + settings.inclusions={ name } + else + settings={ inclusions={ name } } + xmldata.settings=settings + end + if child.er then + local badinclusions=settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + settings.badinclusions={ name } + end + end end end end @@ -12598,7 +13379,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10274, stripped down to: 7538 +-- original size: 10719, stripped down to: 7841 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -12976,7 +13757,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6351, stripped down to: 4919 +-- original size: 6534, stripped down to: 5072 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13146,7 +13927,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11085, stripped down to: 7662 +-- original size: 11444, stripped down to: 7830 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -13402,7 +14183,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 17216, stripped down to: 10657 +-- original size: 18619, stripped down to: 11042 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -13413,6 +14194,7 @@ if not modules then modules={} end modules ['data-exp']={ } local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub local concat,sort=table.concat,table.sort +local sortedkeys=table.sortedkeys local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next @@ -13758,14 +14540,16 @@ local nothing=function() end function resolvers.filtered_from_content(content,pattern) if content and type(pattern)=="string" then local pattern=lower(pattern) - local files=content.files + local files=content.files local remap=content.remap if files and remap then - local n=next(files) + local f=sortedkeys(files) + local n=#f + local i=0 local function iterator() - while n do - local k=n - n=next(files,k) + while i<n do + i=i+1 + local k=f[i] if find(k,pattern) then return files[k],remap and remap[k] or k end @@ -13784,7 +14568,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9216, stripped down to: 6798 +-- original size: 9649, stripped down to: 7131 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -13920,6 +14704,11 @@ local relations=allocate { names={ 'fontconfig','fontconfig file','fontconfig files' }, variable='FONTCONFIG_PATH', }, + pk={ + names={ "pk" }, + variable='PKFONTS', + suffixes={ 'pk' }, + }, }, obsolete={ enc={ @@ -14063,7 +14852,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15618, stripped down to: 11629 +-- original size: 16066, stripped down to: 11938 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -14439,7 +15228,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5347, stripped down to: 4015 +-- original size: 5488, stripped down to: 4101 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -14558,7 +15347,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67003, stripped down to: 46291 +-- original size: 67241, stripped down to: 46427 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15828,10 +16617,18 @@ local function findfiles(filename,filetype,allresults) return result or {},status end function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,true) + end end function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,false)[1] or "" + end end function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") @@ -16106,7 +16903,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 3950, stripped down to: 2935 +-- original size: 4236, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -16170,16 +16967,20 @@ prefixes.pathname=function(str) return cleanpath(dirname((fullname~="" and fullname) or str)) end prefixes.selfautoloc=function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) + local pth=getenv('SELFAUTOLOC') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautoparent=function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) + local pth=getenv('SELFAUTOPARENT') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautodir=function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) + local pth=getenv('SELFAUTODIR') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.home=function(str) - return cleanpath(joinpath(getenv('HOME'),str)) + local pth=getenv('HOME') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.env=prefixes.environment prefixes.rel=prefixes.relative @@ -16224,7 +17025,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 910, stripped down to: 823 +-- original size: 935, stripped down to: 838 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -16254,7 +17055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 475 +-- original size: 548, stripped down to: 483 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -16277,7 +17078,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3310 +-- original size: 3976, stripped down to: 3391 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -16385,7 +17186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5010, stripped down to: 3588 +-- original size: 5148, stripped down to: 3680 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -16504,7 +17305,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3899, stripped down to: 2984 +-- original size: 4000, stripped down to: 3052 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -16595,7 +17396,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8772, stripped down to: 6841 +-- original size: 9036, stripped down to: 7041 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -16832,7 +17633,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8479, stripped down to: 5580 +-- original size: 8712, stripped down to: 5726 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17021,7 +17822,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6569, stripped down to: 5304 +-- original size: 6779, stripped down to: 5444 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17202,7 +18003,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4313, stripped down to: 3227 +-- original size: 4447, stripped down to: 3302 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -17311,7 +18112,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2431, stripped down to: 1996 +-- original size: 2494, stripped down to: 2047 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -17378,7 +18179,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2601, stripped down to: 1627 +-- original size: 2674, stripped down to: 1658 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -17434,7 +18235,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2734, stripped down to: 2354 +-- original size: 2815, stripped down to: 2415 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -17514,7 +18315,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11549, stripped down to: 5905 +-- original size: 11846, stripped down to: 6059 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -17700,7 +18501,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5703, stripped down to: 2507 +-- original size: 5914, stripped down to: 2584 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -17803,7 +18604,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5955, stripped down to: 4926 +-- original size: 6967, stripped down to: 5631 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -17832,7 +18633,7 @@ local function primaryflags() end return concat(flags," ") end -function environment.make_format(name) +function environment.make_format(name,silent) local engine=environment.ownmain or "luatex" local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" @@ -17889,9 +18690,23 @@ function environment.make_format(name) lfs.chdir(olddir) return end - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.execute(command) + local dump=os.platform=="unix" and "\\\\dump" or "\\dump" + if silent then + statistics.starttiming() + local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + local result=os.execute(command) + local runtime=statistics.stoptiming() + if result~=0 then + print(format("%s silent make > fatal error when making format %q",engine,name)) + else + print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) + end + os.remove("temp.log") + else + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + report_format("running command: %s\n",command) + os.execute(command) + end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -17935,10 +18750,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 745793 --- stripped bytes : 269308 +-- original bytes : 797557 +-- stripped bytes : 289197 -- end library merge @@ -17982,6 +18797,8 @@ local ownlibs = { -- order can be made better 'util-str.lua', -- code might move to l-string 'util-tab.lua', + 'util-fil.lua', + 'util-sac.lua', 'util-sto.lua', 'util-prs.lua', 'util-fmt.lua', @@ -18037,13 +18854,21 @@ local ownlibs = { -- order can be made better } --- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/mkiv/data-tmf.lua -- c:/data/develop/context/sources/data-tmf.lua local ownlist = { -- '.', -- ownpath , owntree .. "/../../../../context/sources", -- HH's development path + -- + owntree .. "/../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../texmf/tex/context/base/mkiv", + owntree .. "/../../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../../texmf/tex/context/base/mkiv", + -- owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", owntree .. "/../../texmf/tex/context/base", diff --git a/scripts/context/stubs/install/first-setup.bat b/scripts/context/stubs/install/first-setup.bat index f06ad0e6b..f388b0ac0 100644 --- a/scripts/context/stubs/install/first-setup.bat +++ b/scripts/context/stubs/install/first-setup.bat @@ -38,7 +38,7 @@ REM ~ copy /y bin\x.lua bin\mtx-update.lua REM --mingw --nofiledatabase --engine=luatex -mtxrun --script ./bin/mtx-update.lua --update --force --make --engine=all --context=beta --texroot=%OWNPATH%tex %* +mtxrun --script ./bin/mtx-update.lua --update --force --make --engine=all --context=beta --texroot="%OWNPATH%tex" %* echo. echo. diff --git a/scripts/context/stubs/mswin/contextjit.exe b/scripts/context/stubs/mswin/contextjit.exe Binary files differnew file mode 100644 index 000000000..0e7882cf9 --- /dev/null +++ b/scripts/context/stubs/mswin/contextjit.exe diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 5c09b3b44..7b711a88d 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3888, stripped down to: 2197 +-- original size: 4734, stripped down to: 2626 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -65,10 +65,14 @@ if not modules then modules={} end modules ['l-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") -_MAJORVERSION=tonumber(major) or 5 -_MINORVERSION=tonumber(minor) or 1 +_MAJORVERSION,_MINORVERSION=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(_MAJORVERSION) or 5 +_MINORVERSION=tonumber(_MINORVERSION) or 1 _LUAVERSION=_MAJORVERSION+_MINORVERSION/10 +if _LUAVERSION<5.2 and jit then + _MINORVERSION=2 + _LUAVERSION=5.2 +end if not lpeg then lpeg=require("lpeg") end @@ -111,21 +115,33 @@ if not package.loaders then end local print,select,tostring=print,select,tostring local inspectors={} -function setinspector(inspector) - inspectors[#inspectors+1]=inspector +function setinspector(kind,inspector) + inspectors[kind]=inspector end function inspect(...) for s=1,select("#",...) do local value=select(s,...) - local done=false - for i=1,#inspectors do - done=inspectors[i](value) - if done then - break + if value==nil then + print("nil") + else + local done=false + local kind=type(value) + local inspector=inspectors[kind] + if inspector then + done=inspector(value) + if done then + break + end + end + for kind,inspector in next,inspectors do + done=inspector(value) + if done then + break + end + end + if not done then + print(tostring(value)) end - end - if not done then - print(tostring(value)) end end end @@ -154,7 +170,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10587, stripped down to: 7815 +-- original size: 10949, stripped down to: 8037 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -444,7 +460,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 36977, stripped down to: 20349 +-- original size: 38185, stripped down to: 20990 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -461,7 +477,7 @@ local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print if setinspector then - setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) + setinspector("lpeg",function(v) if lpegtype(v) then lpegprint(v) return true end end) end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns @@ -481,7 +497,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=P("\r")*(P("\n")+P(true))+P("\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -1248,7 +1264,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 361, stripped down to: 322 +-- original size: 372, stripped down to: 329 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1267,7 +1283,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5694, stripped down to: 2827 +-- original size: 5983, stripped down to: 2959 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1354,9 +1370,10 @@ function string.valid(str,default) return (type(str)=="string" and str~="" and str) or default or nil end string.itself=function(s) return s end -local pattern=Ct(C(1)^0) -function string.totable(str) - return lpegmatch(pattern,str) +local pattern_c=Ct(C(1)^0) +local pattern_b=Ct((C(1)/byte)^0) +function string.totable(str,bytes) + return lpegmatch(bytes and pattern_b or pattern_c,str) end local replacer=lpeg.replacer("@","%%") function string.tformat(fmt,...) @@ -1372,7 +1389,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 35724, stripped down to: 21525 +-- original size: 36997, stripped down to: 22376 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2248,7 +2265,7 @@ function table.print(t,...) end end if setinspector then - setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) + setinspector("table",function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) end function table.sub(t,i,j) return { unpack(t,i,j) } @@ -2348,7 +2365,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8643, stripped down to: 6232 +-- original size: 9001, stripped down to: 6512 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2663,7 +2680,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 4939, stripped down to: 2830 +-- original size: 5146, stripped down to: 2933 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2808,7 +2825,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 1923, stripped down to: 1133 +-- original size: 2010, stripped down to: 1186 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2881,7 +2898,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 15832, stripped down to: 9456 +-- original size: 16390, stripped down to: 9734 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3263,7 +3280,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 20949, stripped down to: 9945 +-- original size: 21648, stripped down to: 10238 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3502,7 +3519,7 @@ local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") function file.join(one,two,three,...) if not two then - return one=="" and one or lpegmatch(stripper,one) + return one=="" and one or lpegmatch(reslasher,one) end if one=="" then return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) @@ -3643,7 +3660,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 1002 +-- original size: 1265, stripped down to: 1038 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3697,7 +3714,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3248, stripped down to: 2266 +-- original size: 3355, stripped down to: 2321 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3785,7 +3802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12531, stripped down to: 5721 +-- original size: 12897, stripped down to: 5882 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4002,7 +4019,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 16765, stripped down to: 11003 +-- original size: 17358, stripped down to: 11378 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4467,7 +4484,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1850, stripped down to: 1568 +-- original size: 1919, stripped down to: 1621 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4539,7 +4556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 37388, stripped down to: 15817 +-- original size: 38699, stripped down to: 16321 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4768,9 +4785,10 @@ if not utf.sub then end end end -function utf.remapper(mapping,option) +function utf.remapper(mapping,option,action) local variant=type(mapping) if variant=="table" then + action=action or mapping if option=="dynamic" then local pattern=false table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) @@ -4779,15 +4797,15 @@ function utf.remapper(mapping,option) return "" else if not pattern then - pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) end return lpegmatch(pattern,str) end end elseif option=="pattern" then - return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return Cs((tabletopattern(mapping)/action+p_utf8char)^0) else - local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + local pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) return function(str) if not str or str=="" then return "" @@ -5157,7 +5175,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 974, stripped down to: 890 +-- original size: 1012, stripped down to: 912 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5197,7 +5215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 34513, stripped down to: 18943 +-- original size: 36053, stripped down to: 19685 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5368,7 +5386,13 @@ function string.autosingle(s,sep) end return ("'"..tostring(s).."'") end -local tracedchars={} +local tracedchars={ [0]= + "[null]","[soh]","[stx]","[etx]","[eot]","[enq]","[ack]","[bel]", + "[bs]","[ht]","[lf]","[vt]","[ff]","[cr]","[so]","[si]", + "[dle]","[dc1]","[dc2]","[dc3]","[dc4]","[nak]","[syn]","[etb]", + "[can]","[em]","[sub]","[esc]","[fs]","[gs]","[rs]","[us]", + "[space]", +} string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) @@ -5885,7 +5909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 25338, stripped down to: 16247 +-- original size: 28680, stripped down to: 18636 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -6314,19 +6338,21 @@ local f_val_str=formatters["%w%q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] +local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") -local serialize=table.serialize -function table.serialize(root,name,specification) +local original_serialize=table.serialize +local function serialize(root,name,specification) if type(specification)=="table" then - return serialize(root,name,specification) + return original_serialize(root,name,specification) end - local t + local t local n=1 + local unknown=false local function simple_table(t) local nt=#t if nt>0 then @@ -6337,6 +6363,7 @@ function table.serialize(root,name,specification) return nil end end + local haszero=t[0] if n==nt then local tt={} for i=1,nt do @@ -6353,6 +6380,23 @@ function table.serialize(root,name,specification) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + tt[i+1]=v + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil @@ -6401,7 +6445,7 @@ function table.serialize(root,name,specification) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if next(v)==nil then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -6413,6 +6457,8 @@ function table.serialize(root,name,specification) end elseif tv=="boolean" then n=n+1 t[n]=f_val_boo(depth,v) + elseif unknown then + n=n+1 t[n]=f_val_str(depth,tostring(v)) end elseif tv=="number" then if tk=="number" then @@ -6421,6 +6467,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_num(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_num(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_num(depth,tostring(k),v) end elseif tv=="string" then if tk=="number" then @@ -6429,6 +6477,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_str(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_str(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),v) end elseif tv=="table" then if next(v)==nil then @@ -6438,6 +6488,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_not(depth,k) + elseif unknown then + n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else local st=simple_table(v) @@ -6449,6 +6501,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_seq(depth,k,st) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_seq(depth,k,st) + elseif unknown then + n=n+1 t[n]=f_key_str_value_seq(depth,tostring(k),st) end end elseif tv=="boolean" then @@ -6458,6 +6512,18 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_boo(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_boo(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_boo(depth,tostring(k),v) + end + else + if tk=="number" then + n=n+1 t[n]=f_key_num_value_str(depth,k,tostring(v)) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_str(depth,k,tostring(v)) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_str(depth,k,tostring(v)) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),tostring(v)) end end end @@ -6490,13 +6556,351 @@ function table.serialize(root,name,specification) root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - do_serialize(root,name,1,0) + local st=simple_table(root) + if st then + return t[1]..f_fin_seq(st) + else + do_serialize(root,name,1,0) + end end end n=n+1 t[n]=f_table_finish() return concat(t,"\n") end +table.serialize=serialize +if setinspector then + setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-fil"] = package.loaded["util-fil"] or true + +-- original size: 3577, stripped down to: 2870 + +if not modules then modules={} end modules ['util-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte=string.byte +local extract=bit32.extract +utilities=utilities or {} +local files={} +utilities.files=files +local zerobased={} +function files.open(filename,zb) + local f=io.open(filename,"rb") + if f then + zerobased[f]=zb or false + end + return f +end +function files.close(f) + zerobased[f]=nil + f:close() +end +function files.size(f) + return f:seek("end") +end +function files.setposition(f,n) + if zerobased[f] then + f:seek("set",n) + else + f:seek("set",n-1) + end +end +function files.getposition(f) + if zerobased[f] then + return f:seek() + else + return f:seek()+1 + end +end +function files.look(f,n,chars) + local p=f:seek() + local s=f:read(n) + f:seek("set",p) + if chars then + return s + else + return byte(s,1,#s) + end +end +function files.skip(f,n) + if n==1 then + f:read(n) + else + f:seek("set",f:seek()+n) + end +end +function files.readbyte(f) + return byte(f:read(1)) +end +function files.readbytes(f,n) + return byte(f:read(n),1,n) +end +function files.readchar(f) + return f:read(1) +end +function files.readstring(f,n) + return f:read(n or 1) +end +function files.readinteger1(f) + local n=byte(f:read(1)) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +files.readcardinal1=files.readbyte +files.readcardinal=files.readcardinal1 +files.readinteger=files.readinteger1 +function files.readcardinal2(f) + local a,b=byte(f:read(2),1,2) + return 0x100*a+b +end +function files.readinteger2(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function files.readcardinal3(f) + local a,b,c=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readcardinal4(f) + local a,b,c,d=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function files.readinteger4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function files.skipshort(f,n) + f:read(2*(n or 1)) +end +function files.skiplong(f,n) + f:read(4*(n or 1)) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sac"] = package.loaded["util-sac"] or true + +-- original size: 4264, stripped down to: 3349 + +if not modules then modules={} end modules ['util-sac']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte,sub=string.byte,string.sub +local extract=bit32.extract +utilities=utilities or {} +local streams={} +utilities.streams=streams +function streams.open(filename,zerobased) + local f=io.loaddata(filename) + return { f,1,#f,zerobased or false } +end +function streams.close() +end +function streams.size(f) + return f and f[3] or 0 +end +function streams.setposition(f,i) + if f[4] then + if i<=0 then + f[2]=1 + else + f[2]=i+1 + end + else + if i<=1 then + f[2]=1 + else + f[2]=i + end + end +end +function streams.getposition(f) + if f[4] then + return f[2]-1 + else + return f[2] + end +end +function streams.look(f,n,chars) + local b=f[2] + local e=b+n-1 + if chars then + return sub(f[1],b,e) + else + return byte(f[1],b,e) + end +end +function streams.skip(f,n) + f[2]=f[2]+n +end +function streams.readbyte(f) + local i=f[2] + f[2]=i+1 + return byte(f[1],i) +end +function streams.readbytes(f,n) + local i=f[2] + local j=i+n + f[2]=j + return byte(f[1],i,j-1) +end +function streams.skipbytes(f,n) + f[2]=f[2]+n +end +function streams.readchar(f) + local i=f[2] + f[2]=i+1 + return sub(f[1],i,i) +end +function streams.readstring(f,n) + local i=f[2] + local j=i+n + f[2]=j + return sub(f[1],i,j-1) +end +function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + local n=byte(f[1],i) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +streams.readcardinal1=streams.readbyte +streams.readcardinal=streams.readcardinal1 +streams.readinteger=streams.readinteger1 +function streams.readcardinal2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + return 0x100*a+b +end +function streams.readinteger2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function streams.readcardinal3(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readcardinal4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function streams.readinteger4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function streams.readfixed4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function streams.skipshort(f,n) + f[2]=f[2]+2*(n or 1) +end +function streams.skiplong(f,n) + f[2]=f[2]+4*(n or 1) +end end -- of closure @@ -6505,7 +6909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4172, stripped down to: 2953 +-- original size: 4100, stripped down to: 2852 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -6583,39 +6987,32 @@ local f_index={ ["table"]=f_table, ["number"]=f_number, } -local t_index={ - ["empty"]={ __index=f_empty }, - ["self"]={ __index=f_self }, - ["table"]={ __index=f_table }, - ["number"]={ __index=f_number }, -} function table.setmetatableindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__index=f_index[f] or f + m.__index=i else - setmetatable(t,t_index[f] or { __index=f }) + setmetatable(t,{ __index=i }) end return t end local f_index={ ["ignore"]=f_ignore, } -local t_index={ - ["ignore"]={ __newindex=f_ignore }, -} function table.setmetatablenewindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__newindex=f_index[f] or f + m.__newindex=i else - setmetatable(t,t_index[f] or { __newindex=f }) + setmetatable(t,{ __newindex=i }) end return t end @@ -6652,7 +7049,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 21780, stripped down to: 15121 +-- original size: 23411, stripped down to: 16177 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6676,6 +7073,8 @@ local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash local sortedkeys=table.sortedkeys local tohash=table.tohash +local hashes={} +utilities.parsers.hashes=hashes local digit=R("09") local space=P(' ') local equal=P("=") @@ -6684,6 +7083,8 @@ local lbrace=P("{") local rbrace=P("}") local lparent=P("(") local rparent=P(")") +local lbracket=P("[") +local rbracket=P("]") local period=S(".") local punctuation=S(".,:;") local spacer=lpegpatterns.spacer @@ -6693,6 +7094,7 @@ local anything=lpegpatterns.anything local endofstring=lpegpatterns.endofstring local nobrace=1-(lbrace+rbrace ) local noparent=1-(lparent+rparent) +local nobracket=1-(lbracket+rbracket) local escape,left,right=P("\\"),P('{'),P('}') lpegpatterns.balanced=P { [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, @@ -6700,6 +7102,7 @@ lpegpatterns.balanced=P { } local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local nestedbrackets=P { lbracket*(nobracket+V(1))^0*rbracket } local spaces=space^0 local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) local content=(1-endofstring)^0 @@ -6808,6 +7211,11 @@ function parsers.settings_to_array(str,strict) return { str } end end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_obey_fences(str) + return lpegmatch(pattern,str) +end local cache_a={} local cache_b={} function parsers.groupedsplitat(symbol,withaction) @@ -6894,9 +7302,15 @@ function parsers.array_to_string(a,separator) end end local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) -function utilities.parsers.settings_to_set(str,t) +function utilities.parsers.settings_to_set(str) return str and lpegmatch(pattern,str) or {} end +hashes.settings_to_set=table.setmetatableindex(function(t,k) + local v=k and lpegmatch(pattern,k) or {} + t[k]=v + return v +end) +getmetatable(hashes.settings_to_set).__mode="kv" function parsers.simple_hash_to_string(h,separator) local t,tn={},0 for k,v in sortedhash(h) do @@ -7173,7 +7587,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2274, stripped down to: 1781 +-- original size: 2350, stripped down to: 1847 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7254,7 +7668,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12482, stripped down to: 8864 +-- original size: 12862, stripped down to: 9104 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7567,7 +7981,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 29359, stripped down to: 20483 +-- original size: 30767, stripped down to: 21355 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7610,6 +8024,9 @@ setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then + if texio.setescape then + texio.setescape(0) + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -7627,6 +8044,8 @@ if tex and (tex.jobname or tex.formatname) then elseif target=="term" then texio_write_nl("term","") io_write(...) + elseif type(target)=="number" then + texio_write_nl(target,...) elseif target~="none" then texio_write_nl("log",target,...) texio_write_nl("term","") @@ -7644,6 +8063,8 @@ if tex and (tex.jobname or tex.formatname) then texio_write("log",...) elseif target=="term" then io_write(...) + elseif type(target)=="number" then + texio_write(target,...) elseif target~="none" then texio_write("log",target,...) io_write(target,...) @@ -7714,7 +8135,7 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end report=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,report_yes(translations[a],formats[b])) @@ -7725,7 +8146,7 @@ if tex and (tex.jobname or tex.formatname) then end end direct=function(a,b,c,...) - if c then + if c~=nil then return direct_yes(translations[a],formatters[formats[b]](c,...)) elseif b then return direct_yes(translations[a],formats[b]) @@ -7736,7 +8157,7 @@ if tex and (tex.jobname or tex.formatname) then end end subreport=function(a,s,b,c,...) - if c then + if c~=nil then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) elseif b then write_nl(target,subreport_yes(translations[a],translations[s],formats[b])) @@ -7747,7 +8168,7 @@ if tex and (tex.jobname or tex.formatname) then end end subdirect=function(a,s,b,c,...) - if c then + if c~=nil then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) elseif b then return subdirect_yes(translations[a],translations[s],formats[b]) @@ -7758,7 +8179,7 @@ if tex and (tex.jobname or tex.formatname) then end end status=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,status_yes(translations[a],formats[b])) @@ -8056,7 +8477,7 @@ function logs.messenger(category,subcategory) end end end -local function setblocked(category,value) +local function setblocked(category,value) if category==true then category,value="*",true elseif category==false then @@ -8071,7 +8492,7 @@ local function setblocked(category,value) end else states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) - for c,_ in next,states do + for c in next,states do local v=data[c] if v then v.state=value @@ -8353,7 +8774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6704, stripped down to: 5343 +-- original size: 6917, stripped down to: 5484 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8474,13 +8895,13 @@ function statistics.show() end end register("lua properties",function() - local list=status.list() - local hashchar=tonumber(list.luatex_hashchars) + local hashchar=tonumber(status.luatex_hashchars) + local hashtype=status.luatex_hashtype local mask=lua.mask or "ascii" return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", jit and "luajit" or "lua", statistics.memused(), - list.luatex_hashtype or "default", + hashtype or "default", hashchar and 2^hashchar or "unknown", mask, mask=="utf" and "τεχ" or "tex") @@ -8534,7 +8955,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5829, stripped down to: 3501 +-- original size: 6039, stripped down to: 3616 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -8681,7 +9102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 4982, stripped down to: 3511 +-- original size: 5142, stripped down to: 3611 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -8811,7 +9232,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3898, stripped down to: 2644 +-- original size: 4030, stripped down to: 2718 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8915,7 +9336,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7757, stripped down to: 6015 +-- original size: 7985, stripped down to: 6153 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9092,7 +9513,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7100, stripped down to: 3978 +-- original size: 7313, stripped down to: 4076 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -9237,7 +9658,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8022, stripped down to: 5038 +-- original size: 8284, stripped down to: 5176 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9424,7 +9845,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6174, stripped down to: 4141 +-- original size: 6358, stripped down to: 4257 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9577,7 +9998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 45848, stripped down to: 27914 +-- original size: 56973, stripped down to: 35872 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -9586,7 +10007,7 @@ if not modules then modules={} end modules ['lxml-tab']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local trace_entities=false trackers .register("xml.entities",function(v) trace_entities=v end) local report_xml=logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} @@ -9594,10 +10015,12 @@ local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub +local sort=table.sort local utfchar=utf.char local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters +do xml.xmlns=xml.xmlns or {} local check=P(false) local parse=check @@ -9614,23 +10037,68 @@ end function xml.resolvens(url) return lpegmatch(parse,lower(url)) or "" end +end local nsremap,resolvens=xml.xmlns,xml.resolvens -local stack={} -local top={} -local dt={} -local at={} -local xmlns={} -local errorstr=nil -local entities={} -local strip=false -local cleanup=false -local utfize=false -local resolve_predefined=false -local unify_predefined=false -local dcache={} -local hcache={} -local acache={} -local mt={} +local stack,level,top,at,xmlnms,errorstr +local entities,parameters +local strip,utfize,resolve,cleanup,resolve_predefined,unify_predefined +local dcache,hcache,acache +local mt,dt,nt +local function preparexmlstate(settings) + if settings then + stack={} + level=0 + top={} + at={} + mt={} + dt={} + nt=0 + xmlns={} + errorstr=nil + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + parameters={} + reported_at_errors={} + dcache={} + hcache={} + acache={} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + else + stack=nil + level=nil + top=nil + at=nil + mt=nil + dt=nil + nt=nil + xmlns=nil + errorstr=nil + strip=nil + utfize=nil + resolve=nil + resolve_predefined=nil + unify_predefined=nil + cleanup=nil + entities=nil + parameters=nil + reported_at_errors=nil + dcache=nil + hcache=nil + acache=nil + end +end local function initialize_mt(root) mt={ __index=root } end @@ -9640,8 +10108,9 @@ end function xml.checkerror(top,toclose) return "" end +local checkns=xml.checkns local function add_attribute(namespace,tag,value) - if cleanup and #value>0 then + if cleanup and value~="" then value=cleanup(value) end if tag=="xmlns" then @@ -9650,21 +10119,30 @@ local function add_attribute(namespace,tag,value) elseif namespace=="" then at[tag]=value elseif namespace=="xmlns" then - xml.checkns(tag,value) + checkns(tag,value) at["xmlns:"..tag]=value else at[namespace..":"..tag]=value end end local function add_empty(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top=stack[#stack] + top=stack[level] dt=top.dt - local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } - dt[#dt+1]=t + nt=#dt+1 + local t={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=top + } + dt[nt]=t setmetatable(t,mt) if at.xmlns then remove(xmlns) @@ -9672,23 +10150,35 @@ local function add_empty(spacing,namespace,tag) at={} end local function add_begin(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + top={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=stack[level] + } setmetatable(top,mt) dt=top.dt - stack[#stack+1]=top + nt=#dt + level=level+1 + stack[level]=top at={} end local function add_end(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end - local toclose=remove(stack) - top=stack[#stack] - if #stack<1 then + local toclose=stack[level] + level=level-1 + top=stack[level] + if level<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") report_xml(errorstr) elseif toclose.tg~=tag then @@ -9696,202 +10186,236 @@ local function add_end(spacing,namespace,tag) report_xml(errorstr) end dt=top.dt - dt[#dt+1]=toclose + nt=#dt+1 + dt[nt]=toclose if toclose.at.xmlns then remove(xmlns) end end -local spaceonly=lpegpatterns.whitespace^0*P(-1) local function add_text(text) - local n=#dt - if cleanup and #text>0 then - if n>0 then - local s=dt[n] + if text=="" then + return + end + if cleanup then + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..cleanup(text) + dt[nt]=s..cleanup(text) else - dt[n+1]=cleanup(text) + nt=nt+1 + dt[nt]=cleanup(text) end else + nt=1 dt[1]=cleanup(text) end else - if n>0 then - local s=dt[n] + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..text + dt[nt]=s..text else - dt[n+1]=text + nt=nt+1 + dt[nt]=text end else + nt=1 dt[1]=text end end end local function add_special(what,spacing,text) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end if strip and (what=="@cm@" or what=="@dt@") then else - dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + nt=nt+1 + dt[nt]={ special=true,ns="",tg=what,dt={ text } } end end local function set_message(txt) errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end -local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute value %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end local function attribute_specification_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute specification %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end -local badentity="&error;" -local badentity="&" -xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, - unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, - unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, -} -local placeholders=xml.placeholders -local function fromhex(s) - local n=tonumber(s,16) - if n then - return utfchar(n) - else - return formatters["h:%s"](s),true - end -end -local function fromdec(s) - local n=tonumber(s) - if n then - return utfchar(n) - else - return formatters["d:%s"](s),true - end -end -local p_rest=(1-P(";"))^0 -local p_many=P(1)^0 -local p_char=lpegpatterns.utf8character -local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) -local predefined_unified={ - [38]="&", - [42]=""", - [47]="'", - [74]="<", - [76]=">", -} -local predefined_simplified={ - [38]="&",amp="&", - [42]='"',quot='"', - [47]="'",apos="'", - [74]="<",lt="<", - [76]=">",gt=">", -} -local nofprivates=0xF0000 -local privates_u={ - [ [[&]] ]="&", - [ [["]] ]=""", - [ [[']] ]="'", - [ [[<]] ]="<", - [ [[>]] ]=">", -} -local privates_p={} -local privates_n={ -} -local escaped=utf.remapper(privates_u,"dynamic") -local unprivatized=utf.remapper(privates_p,"dynamic") -xml.unprivatized=unprivatized -local function unescaped(s) - local p=privates_n[s] - if not p then - nofprivates=nofprivates+1 - p=utfchar(nofprivates) - privates_n[s]=p - s="&"..s..";" - privates_u[p]=s - privates_p[p]=s +local grammar_parsed_text_one +local grammar_parsed_text_two +local handle_hex_entity +local handle_dec_entity +local handle_any_entity_dtd +local handle_any_entity_text +do + local badentity="&" + xml.placeholders={ + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, + unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, + unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, + } + local function fromhex(s) + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return formatters["h:%s"](s),true + end end - return p -end -xml.privatetoken=unescaped -xml.privatecodes=privates_n -local function handle_hex_entity(str) - local h=hcache[str] - if not h then - local n=tonumber(str,16) - h=unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end - elseif utfize then - h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end + local function fromdec(s) + local n=tonumber(s) + if n then + return utfchar(n) else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h="&#x"..str..";" + return formatters["d:%s"](s),true + end + end + local p_rest=(1-P(";"))^0 + local p_many=P(1)^0 + local p_char=lpegpatterns.utf8character + local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) + xml.parsedentitylpeg=parsedentity + local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", + } + local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", + } + local nofprivates=0xF0000 + local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", + } + local privates_p={ + } + local privates_s={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[&]] ]="&U+26;", + [ [[']] ]="&U+27;", + [ [[<]] ]="&U+3C;", + [ [[>]] ]="&U+3E;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } + local privates_n={ + } + local escaped=utf.remapper(privates_u,"dynamic") + local unprivatized=utf.remapper(privates_p,"dynamic") + local unspecialized=utf.remapper(privates_s,"dynamic") + xml.unprivatized=unprivatized + xml.unspecialized=unspecialized + xml.escaped=escaped + local function unescaped(s) + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + privates_s[p]=s end - hcache[str]=h + return p end - return h -end -local function handle_dec_entity(str) - local d=dcache[str] - if not d then - local n=tonumber(str) - d=unify_predefined and predefined_unified[n] - if d then - if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) - end - elseif utfize then - d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) + xml.privatetoken=unescaped + xml.privatecodes=privates_n + xml.specialcodes=privates_s + function xml.addspecialcode(key,value) + privates_s[key]=value or "&"..s..";" + end + handle_hex_entity=function(str) + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - else - if trace_entities then - report_xml("found entity &#%s;",str) + hcache[str]=h + end + return h + end + handle_dec_entity=function(str) + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" end - d="&#"..str..";" + dcache[str]=d end - dcache[str]=d + return d end - return d -end -xml.parsedentitylpeg=parsedentity -local function handle_any_entity(str) - if resolve then - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + handle_any_entity_dtd=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then if trace_entities then report_xml("resolving entity &%s; to predefined %a",str,a) end else if type(resolve)=="function" then - a=resolve(str) or entities[str] + a=resolve(str,entities) or entities[str] else a=entities[str] end @@ -9927,40 +10451,194 @@ local function handle_any_entity(str) end end end - acache[str]=a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; to %a",str,a) - acache[str]=a + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a end - return a - else - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + end + handle_any_entity_text=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then - acache[str]=a - if trace_entities then - report_xml("entity &%s; becomes %a",str,a) - end - elseif str=="" then if trace_entities then - report_xml("invalid entity &%s;",str) + report_xml("resolving entity &%s; to predefined %a",str,a) end - a=badentity - acache[str]=a else - if trace_entities then - report_xml("entity &%s; is made private",str) + if type(resolve)=="function" then + a=resolve(str,entities) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then + if trace_entities then + report_xml("expanding entity &%s; to function call",str) + end + a=a(str) or "" + end + a=lpegmatch(grammar_parsed_text_two,a) or a + if type(a)=="number" then + return "" + else + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + end + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then + if trace_entities then + report_xml("resolving entity &%s; to external %s",str,a) + end + else + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str=="" then + a=badentity + else + a="&"..str..";" + end + end + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a end - a=unescaped(str) - acache[str]=a end + return a + end + end + local p_rest=(1-P(";"))^1 + local spec={ + [0x23]="\\Ux{23}", + [0x24]="\\Ux{24}", + [0x25]="\\Ux{25}", + [0x5C]="\\Ux{5C}", + [0x7B]="\\Ux{7B}", + [0x7C]="\\Ux{7C}", + [0x7D]="\\Ux{7D}", + [0x7E]="\\Ux{7E}", + } + local hash=table.setmetatableindex(spec,function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true end - return a end + local reparsedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + local hash=table.setmetatableindex(function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true + end + end + local unescapedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + xml.reparsedentitylpeg=reparsedentity + xml.unescapedentitylpeg=unescapedentity end +local escaped=xml.escaped +local unescaped=xml.unescaped +local placeholders=xml.placeholders local function handle_end_entity(str) report_xml("error in entity, %a found without ending %a",str,";") return str @@ -9987,14 +10665,18 @@ local name=name_yes+name_nop local utfbom=lpegpatterns.utfbom local spacing=C(space^0) local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 -local hexentitycontent=R("AF","af","09")^0 -local decentitycontent=R("09")^0 +local hexentitycontent=R("AF","af","09")^1 +local decentitycontent=R("09")^1 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) - )+(anyentitycontent/handle_any_entity) + )+(anyentitycontent/handle_any_entity_dtd) +local parsedentity_text=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity_text) local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) +local entity_text=(ampersand/"")*parsedentity_text*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) -local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local text_parsed=(Cs((1-open-ampersand)^1)/add_text+Cs(entity_text)/add_text)^1 local somespace=space^1 local optionalspace=space^0 local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) @@ -10004,7 +10686,7 @@ local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error local attributevalue=value+wrongvalue local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 -local parsedtext=text_parsed/add_text +local parsedtext=text_parsed local unparsedtext=text_unparsed/add_text local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty @@ -10019,21 +10701,52 @@ local endcdata=P("]]")*close local someinstruction=C((1-endinstruction)^0) local somecomment=C((1-endcomment )^0) local somecdata=C((1-endcdata )^0) -local function normalentity(k,v ) entities[k]=v end -local function systementity(k,v,n) entities[k]=v end -local function publicentity(k,v,n) entities[k]=v end +local function weirdentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","weird",k,v) + end + parameters[k]=v +end +local function normalentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","normal",k,v) + end + entities[k]=v +end +local function systementity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","system",k,v) + end + entities[k]=v +end +local function publicentity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","public",k,v) + end + entities[k]=v +end local begindoctype=open*P("!DOCTYPE") local enddoctype=close local beginset=P("[") local endset=P("]") +local wrdtypename=C((1-somespace-P(";"))^1) local doctypename=C((1-somespace-close)^0) local elementdoctype=optionalspace*P("<!ELEMENT")*(1-close)^0*close local basiccomment=begincomment*((1-endcomment)^0)*endcomment +local weirdentitytype=P("%")*(somespace*doctypename*somespace*value)/weirdentity local normalentitytype=(doctypename*somespace*value)/normalentity local publicentitytype=(doctypename*somespace*P("PUBLIC")*somespace*value)/publicentity local systementitytype=(doctypename*somespace*P("SYSTEM")*somespace*value*somespace*P("NDATA")*somespace*doctypename)/systementity -local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype)*optionalspace*close -local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+basiccomment+space)^0*optionalspace*endset +local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype+weirdentitytype)*optionalspace*close +local function weirdresolve(s) + lpegmatch(entitydoctype,parameters[s]) +end +local function normalresolve(s) + lpegmatch(entitydoctype,entities[s]) +end +local entityresolve=P("%")*(wrdtypename/weirdresolve )*P(";")+P("&")*(wrdtypename/normalresolve)*P(";") +entitydoctype=entitydoctype+entityresolve +local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+entityresolve+basiccomment+space)^0*optionalspace*endset local definitiondoctype=doctypename*somespace*doctypeset local publicdoctype=doctypename*somespace*P("PUBLIC")*somespace*value*somespace*value*somespace*doctypeset local systemdoctype=doctypename*somespace*P("SYSTEM")*somespace*value*somespace*doctypeset @@ -10045,11 +10758,15 @@ local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata -local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error -local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 -local grammar_parsed_text=P { "preamble", - preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, +grammar_parsed_text_one=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0, +} +grammar_parsed_text_two=P { "followup", + followup=V("parent")*trailer, parent=beginelement*V("children")^0*endelement, children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } @@ -10059,37 +10776,26 @@ local grammar_unparsed_text=P { "preamble", children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) - settings=settings or {} - strip=settings.strip_cm_and_dt - utfize=settings.utfize_entities - resolve=settings.resolve_entities - resolve_predefined=settings.resolve_predefined_entities - unify_predefined=settings.unify_predefined_entities - cleanup=settings.text_cleanup - entities=settings.entities or {} - if utfize==nil then - settings.utfize_entities=true - utfize=true - end - if resolve_predefined==nil then - settings.resolve_predefined_entities=true - resolve_predefined=true - end - stack,top,at,xmlns,errorstr={},{},{},{},nil - acache,hcache,dcache={},{},{} - reported_attribute_errors={} + settings=settings or {} + preparexmlstate(settings) if settings.parent_root then mt=getmetatable(settings.parent_root) else initialize_mt(top) end - stack[#stack+1]=top + level=level+1 + stack[level]=top top.dt={} dt=top.dt + nt=0 if not data or data=="" then errorstr="empty xml file" elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then + local m=lpegmatch(grammar_parsed_text_one,data) + if m then + m=lpegmatch(grammar_parsed_text_two,data,m) + end + if m then else errorstr="invalid xml file - parsed text" end @@ -10105,8 +10811,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } -setmetatable(result,mt) -setmetatable(result.dt[1],mt) + setmetatable(result,mt) + setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -10148,13 +10854,10 @@ setmetatable(result.dt[1],mt) decimals=dcache, hexadecimals=hcache, names=acache, + intermediates=parameters, } } - strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil - unify_predefined,cleanup,entities=nil,nil,nil - stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil - acache,hcache,dcache=nil,nil,nil - reported_attribute_errors,mt,errorhandler=nil,nil,nil + preparexmlstate() return result end local function xmlconvert(data,settings) @@ -10216,15 +10919,15 @@ function xml.toxml(data) return data end end -local function copy(old,tables) +local function copy(old) if old then - tables=tables or {} local new={} - if not tables[old] then - tables[old]=new - end for k,v in next,old do - new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v + if type(v)=="table" then + new[k]=table.copy(v) + else + new[k]=v + end end local mt=getmetatable(old) if mt then @@ -10257,22 +10960,34 @@ local function verbose_element(e,handlers,escape) local ats=eat and next(eat) and {} if ats then local n=0 - for k,v in next,eat do + for k in next,eat do n=n+1 - ats[n]=f_attribute(k,escaped(v)) + ats[n]=k + end + if n==1 then + local k=ats[1] + ats=f_attribute(k,escaped(eat[k])) + else + sort(ats) + for i=1,n do + local k=ats[i] + ats[i]=f_attribute(k,escaped(eat[k])) + end + ats=concat(ats," ") end end if ern and trace_entities and ern~=ens then ens=ern end + local n=edt and #edt if ens~="" then - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") + handle("<",ens,":",etg," ",ats,">") else handle("<",ens,":",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10283,19 +10998,19 @@ local function verbose_element(e,handlers,escape) handle("</",ens,":",etg,">") else if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") + handle("<",ens,":",etg," ",ats,"/>") else handle("<",ens,":",etg,"/>") end end else - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",etg," ",concat(ats," "),">") + handle("<",etg," ",ats,">") else handle("<",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10306,7 +11021,7 @@ local function verbose_element(e,handlers,escape) handle("</",etg,">") else if ats then - handle("<",etg," ",concat(ats," "),"/>") + handle("<",etg," ",ats,"/>") else handle("<",etg,"/>") end @@ -10323,7 +11038,7 @@ local function verbose_cdata(e,handlers) handlers.handle("<![CDATA[",e.dt[1],"]]>") end local function verbose_doctype(e,handlers) - handlers.handle("<!DOCTYPE ",e.dt[1],">") + handlers.handle("<!DOCTYPE",e.dt[1],">") end local function verbose_root(e,handlers) handlers.serialize(e.dt,handlers) @@ -10366,12 +11081,14 @@ local function serialize(e,handlers,...) end end local function xserialize(e,handlers) - local functions=handlers.functions - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) + if e then + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end end end local handlers={} @@ -10603,7 +11320,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48229, stripped down to: 30684 +-- original size: 53892, stripped down to: 32508 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10618,10 +11335,23 @@ local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string. local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local setmetatableindex=table.setmetatableindex local formatters=string.formatters -local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end -local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end -local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local trace_lpath=false +local trace_lparse=false +local trace_lprofile=false local report_lpath=logs.reporter("xml","lpath") +if trackers then + trackers.register("xml.path",function(v) + trace_lpath=v + end) + trackers.register("xml.parse",function(v) + trace_lparse=v + end) + trackers.register("xml.profile",function(v) + trace_lpath=v + trace_lparse=v + trace_lprofile=v + end) +end local xml=xml local lpathcalls=0 function xml.lpathcalls () return lpathcalls end local lpathcached=0 function xml.lpathcached() return lpathcached end @@ -10980,13 +11710,27 @@ local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") local lp_doequal=P("=")/"==" local lp_or=P("|")/" or " local lp_and=P("&")/" and " -local lp_builtin=P ( - P("text")/"(ll.dt[1] or '')"+ - P("content")/"ll.dt"+ - P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ - P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ - P("ns")/"ll.ns" - )*((spaces*P("(")*spaces*P(")"))/"") +local builtin={ + text="(ll.dt[1] or '')", + content="ll.dt", + name="((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)", + tag="ll.tg", + position="l", + firstindex="1", + firstelement="1", + first="1", + lastindex="(#ll.__p__.dt or 1)", + lastelement="(ll.__p__.en or 1)", + last="#list", + rootposition="order", + order="order", + element="(ll.ei or 1)", + index="(ll.ni or 1)", + match="(ll.mi or 1)", + namespace="ll.ns", + ns="ll.ns", +} +local lp_builtin=lpeg.utfchartabletopattern(builtin)/builtin*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" @@ -11005,7 +11749,7 @@ local rparent=P(")") local noparent=1-(lparent+rparent) local nested=P{lparent*(noparent+V(1))^0*rparent} local value=P(lparent*C((noparent+nested)^0)*rparent) -local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_child=Cc("expr.child(ll,'")*R("az","AZ")*R("az","AZ","--","__")^0*Cc("')") local lp_number=S("+-")*R("09")^1 local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) @@ -11044,6 +11788,7 @@ local template_f_y=[[ local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] +local register_last_match={ kind="axis",axis="last-match" } local register_self={ kind="axis",axis="self" } local register_parent={ kind="axis",axis="parent" } local register_descendant={ kind="axis",axis="descendant" } @@ -11121,7 +11866,7 @@ local pathparser=Ct { "patterns", ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, - axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, @@ -11147,6 +11892,7 @@ local pathparser=Ct { "patterns", preceding=P('preceding::')*Cc(register_preceding ), preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + last_match=P('last-match::')*Cc(register_last_match ), nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -11193,13 +11939,12 @@ local function tagstostring(list) end xml.nodesettostring=nodesettostring local lpath -local lshowoptions={ functions=false } local function lshow(parsed) if type(parsed)=="string" then parsed=lpath(parsed) end report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + table.serialize(parsed,false)) end xml.lshow=lshow local function add_comment(p,str) @@ -11265,140 +12010,168 @@ lpath=function (pattern) end end xml.lpath=lpath -local profiled={} xml.profiled=profiled -local function profiled_apply(list,parsed,nofparsed,order) - local p=profiled[parsed.pattern] - if p then - p.tested=p.tested+1 - else - p={ tested=1,matched=0,finalized=0 } - profiled[parsed.pattern]=p - end - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - p.matched=p.matched+1 - p.finalized=p.finalized+1 - return collected - end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) +do + local profiled={} + xml.profiled=profiled + local lastmatch=nil + local keepmatch=nil + if directives then + directives.register("xml.path.keeplastmatch",function(v) + keepmatch=v + lastmatch=nil + end) + end + apply_axis["last-match"]=function() + return lastmatch or {} + end + local function profiled_apply(list,parsed,nofparsed,order) + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 p.finalized=p.finalized+1 return collected end - return nil + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + p.finalized=p.finalized+1 + return collected + end + return nil + end end - end - if collected then - p.matched=p.matched+1 - end - return collected -end -local function traced_apply(list,parsed,nofparsed,order) - if trace_lparse then - lshow(parsed) - end - report_lpath("collecting: %s",parsed.pattern) - report_lpath("root tags : %s",tagstostring(list)) - report_lpath("order : %s",order or "unset") - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected + if collected then + p.matched=p.matched+1 end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") return collected end - return nil - end - end - return collected -end -local function normal_apply(list,parsed,nofparsed,order) - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - local axis=pi.axis - if axis~="self" then - collected=apply_axis[axis](collected) + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil end - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - return pi.finalizer(collected) end - if not collected or #collected==0 then - local pf=i<nofparsed and parsed[nofparsed].finalizer - if pf then - return pf(collected) + return collected + end + local function normal_apply(list,parsed,nofparsed,order) + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + local axis=pi.axis + if axis~="self" then + collected=apply_axis[axis](collected) + end + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected==0 then + local pf=i<nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) + end + return nil end - return nil end + return collected end - return collected -end -local function applylpath(list,pattern) - if not list then - return - end - local parsed=cache[pattern] - if parsed then - lpathcalls=lpathcalls+1 - lpathcached=lpathcached+1 - elseif type(pattern)=="table" then - lpathcalls=lpathcalls+1 - parsed=pattern - else - parsed=lpath(pattern) or pattern - end - if not parsed then - return + local apply=normal_apply + if trackers then + trackers.register("xml.path,xml.parse,xml.profile",function() + if trace_lprofile then + apply=profiled_apply + elseif trace_lpath then + apply=traced_apply + else + apply=normal_apply + end + end) end - local nofparsed=#parsed - if nofparsed==0 then - return + function xml.applylpath(list,pattern) + if not list then + lastmatch=nil + return + end + local parsed=cache[pattern] + if parsed then + lpathcalls=lpathcalls+1 + lpathcached=lpathcached+1 + elseif type(pattern)=="table" then + lpathcalls=lpathcalls+1 + parsed=pattern + else + parsed=lpath(pattern) or pattern + end + if not parsed then + lastmatch=nil + return + end + local nofparsed=#parsed + if nofparsed==0 then + lastmatch=nil + return + end + local collected=apply({ list },parsed,nofparsed,list.mi) + lastmatch=keepmatch and collected or nil + return collected end - if not trace_lpath then - return normal_apply ({ list },parsed,nofparsed,list.mi) - elseif trace_lprofile then - return profiled_apply({ list },parsed,nofparsed,list.mi) - else - return traced_apply ({ list },parsed,nofparsed,list.mi) + function xml.lastmatch() + return lastmatch end end -xml.applylpath=applylpath +local applylpath=xml.applylpath function xml.filter(root,pattern) return applylpath(root,pattern) end @@ -11676,7 +12449,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3684, stripped down to: 1957 +-- original size: 3787, stripped down to: 2003 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -11745,7 +12518,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 28786, stripped down to: 20578 +-- original size: 30566, stripped down to: 21741 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12079,55 +12852,63 @@ local function include(xmldata,pattern,attribute,recursive,loaddata,level) local ek=collected[c] local name=nil local ekdt=ek.dt - local ekat=ek.at - local ekrt=ek.__p__ - local epdt=ekrt.dt - if not attribute or attribute=="" then - name=(type(ekdt)=="table" and ekdt[1]) or ekdt - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name=ekat[a] - if name then - break + if ekdt then + local ekat=ek.at + local ekrt=ek.__p__ + if ekrt then + local epdt=ekrt.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt end - end - end - local data=nil - if name and name~="" then - data=loaddata(name) or "" - if trace_inclusions then - report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") - end - end - if not data or data=="" then - epdt[ek.ni]="" - elseif ekat["parse"]=="text" then - epdt[ek.ni]=xml.escaped(data) - else - local xi=xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni]="" - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata,level+1) + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then + break + end + end end - local child=xml.body(xi) - child.__p__=ekrt - child.__f__=name - epdt[ek.ni]=child - local inclusions=xmldata.settings.inclusions - if inclusions then - inclusions[#inclusions+1]=name - else - xmldata.settings.inclusions={ name } + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") + end end - if child.er then - local badinclusions=xmldata.settings.badinclusions - if badinclusions then - badinclusions[#badinclusions+1]=name + if not data or data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" else - xmldata.settings.badinclusions={ name } + if recursive then + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local settings=xmldata.settings + local inclusions=settings and settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + elseif settings then + settings.inclusions={ name } + else + settings={ inclusions={ name } } + xmldata.settings=settings + end + if child.er then + local badinclusions=settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + settings.badinclusions={ name } + end + end end end end @@ -12598,7 +13379,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10274, stripped down to: 7538 +-- original size: 10719, stripped down to: 7841 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -12976,7 +13757,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6351, stripped down to: 4919 +-- original size: 6534, stripped down to: 5072 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13146,7 +13927,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11085, stripped down to: 7662 +-- original size: 11444, stripped down to: 7830 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -13402,7 +14183,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 17216, stripped down to: 10657 +-- original size: 18619, stripped down to: 11042 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -13413,6 +14194,7 @@ if not modules then modules={} end modules ['data-exp']={ } local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub local concat,sort=table.concat,table.sort +local sortedkeys=table.sortedkeys local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next @@ -13758,14 +14540,16 @@ local nothing=function() end function resolvers.filtered_from_content(content,pattern) if content and type(pattern)=="string" then local pattern=lower(pattern) - local files=content.files + local files=content.files local remap=content.remap if files and remap then - local n=next(files) + local f=sortedkeys(files) + local n=#f + local i=0 local function iterator() - while n do - local k=n - n=next(files,k) + while i<n do + i=i+1 + local k=f[i] if find(k,pattern) then return files[k],remap and remap[k] or k end @@ -13784,7 +14568,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9216, stripped down to: 6798 +-- original size: 9649, stripped down to: 7131 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -13920,6 +14704,11 @@ local relations=allocate { names={ 'fontconfig','fontconfig file','fontconfig files' }, variable='FONTCONFIG_PATH', }, + pk={ + names={ "pk" }, + variable='PKFONTS', + suffixes={ 'pk' }, + }, }, obsolete={ enc={ @@ -14063,7 +14852,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15618, stripped down to: 11629 +-- original size: 16066, stripped down to: 11938 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -14439,7 +15228,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5347, stripped down to: 4015 +-- original size: 5488, stripped down to: 4101 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -14558,7 +15347,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67003, stripped down to: 46291 +-- original size: 67241, stripped down to: 46427 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15828,10 +16617,18 @@ local function findfiles(filename,filetype,allresults) return result or {},status end function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,true) + end end function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,false)[1] or "" + end end function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") @@ -16106,7 +16903,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 3950, stripped down to: 2935 +-- original size: 4236, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -16170,16 +16967,20 @@ prefixes.pathname=function(str) return cleanpath(dirname((fullname~="" and fullname) or str)) end prefixes.selfautoloc=function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) + local pth=getenv('SELFAUTOLOC') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautoparent=function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) + local pth=getenv('SELFAUTOPARENT') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautodir=function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) + local pth=getenv('SELFAUTODIR') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.home=function(str) - return cleanpath(joinpath(getenv('HOME'),str)) + local pth=getenv('HOME') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.env=prefixes.environment prefixes.rel=prefixes.relative @@ -16224,7 +17025,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 910, stripped down to: 823 +-- original size: 935, stripped down to: 838 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -16254,7 +17055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 475 +-- original size: 548, stripped down to: 483 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -16277,7 +17078,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3310 +-- original size: 3976, stripped down to: 3391 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -16385,7 +17186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5010, stripped down to: 3588 +-- original size: 5148, stripped down to: 3680 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -16504,7 +17305,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3899, stripped down to: 2984 +-- original size: 4000, stripped down to: 3052 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -16595,7 +17396,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8772, stripped down to: 6841 +-- original size: 9036, stripped down to: 7041 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -16832,7 +17633,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8479, stripped down to: 5580 +-- original size: 8712, stripped down to: 5726 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17021,7 +17822,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6569, stripped down to: 5304 +-- original size: 6779, stripped down to: 5444 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17202,7 +18003,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4313, stripped down to: 3227 +-- original size: 4447, stripped down to: 3302 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -17311,7 +18112,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2431, stripped down to: 1996 +-- original size: 2494, stripped down to: 2047 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -17378,7 +18179,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2601, stripped down to: 1627 +-- original size: 2674, stripped down to: 1658 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -17434,7 +18235,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2734, stripped down to: 2354 +-- original size: 2815, stripped down to: 2415 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -17514,7 +18315,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11549, stripped down to: 5905 +-- original size: 11846, stripped down to: 6059 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -17700,7 +18501,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5703, stripped down to: 2507 +-- original size: 5914, stripped down to: 2584 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -17803,7 +18604,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5955, stripped down to: 4926 +-- original size: 6967, stripped down to: 5631 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -17832,7 +18633,7 @@ local function primaryflags() end return concat(flags," ") end -function environment.make_format(name) +function environment.make_format(name,silent) local engine=environment.ownmain or "luatex" local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" @@ -17889,9 +18690,23 @@ function environment.make_format(name) lfs.chdir(olddir) return end - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.execute(command) + local dump=os.platform=="unix" and "\\\\dump" or "\\dump" + if silent then + statistics.starttiming() + local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + local result=os.execute(command) + local runtime=statistics.stoptiming() + if result~=0 then + print(format("%s silent make > fatal error when making format %q",engine,name)) + else + print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) + end + os.remove("temp.log") + else + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + report_format("running command: %s\n",command) + os.execute(command) + end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -17935,10 +18750,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 745793 --- stripped bytes : 269308 +-- original bytes : 797557 +-- stripped bytes : 289197 -- end library merge @@ -17982,6 +18797,8 @@ local ownlibs = { -- order can be made better 'util-str.lua', -- code might move to l-string 'util-tab.lua', + 'util-fil.lua', + 'util-sac.lua', 'util-sto.lua', 'util-prs.lua', 'util-fmt.lua', @@ -18037,13 +18854,21 @@ local ownlibs = { -- order can be made better } --- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/mkiv/data-tmf.lua -- c:/data/develop/context/sources/data-tmf.lua local ownlist = { -- '.', -- ownpath , owntree .. "/../../../../context/sources", -- HH's development path + -- + owntree .. "/../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../texmf/tex/context/base/mkiv", + owntree .. "/../../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../../texmf/tex/context/base/mkiv", + -- owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", owntree .. "/../../texmf/tex/context/base", diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 5c09b3b44..7b711a88d 100644 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3888, stripped down to: 2197 +-- original size: 4734, stripped down to: 2626 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -65,10 +65,14 @@ if not modules then modules={} end modules ['l-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") -_MAJORVERSION=tonumber(major) or 5 -_MINORVERSION=tonumber(minor) or 1 +_MAJORVERSION,_MINORVERSION=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(_MAJORVERSION) or 5 +_MINORVERSION=tonumber(_MINORVERSION) or 1 _LUAVERSION=_MAJORVERSION+_MINORVERSION/10 +if _LUAVERSION<5.2 and jit then + _MINORVERSION=2 + _LUAVERSION=5.2 +end if not lpeg then lpeg=require("lpeg") end @@ -111,21 +115,33 @@ if not package.loaders then end local print,select,tostring=print,select,tostring local inspectors={} -function setinspector(inspector) - inspectors[#inspectors+1]=inspector +function setinspector(kind,inspector) + inspectors[kind]=inspector end function inspect(...) for s=1,select("#",...) do local value=select(s,...) - local done=false - for i=1,#inspectors do - done=inspectors[i](value) - if done then - break + if value==nil then + print("nil") + else + local done=false + local kind=type(value) + local inspector=inspectors[kind] + if inspector then + done=inspector(value) + if done then + break + end + end + for kind,inspector in next,inspectors do + done=inspector(value) + if done then + break + end + end + if not done then + print(tostring(value)) end - end - if not done then - print(tostring(value)) end end end @@ -154,7 +170,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10587, stripped down to: 7815 +-- original size: 10949, stripped down to: 8037 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -444,7 +460,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 36977, stripped down to: 20349 +-- original size: 38185, stripped down to: 20990 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -461,7 +477,7 @@ local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print if setinspector then - setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) + setinspector("lpeg",function(v) if lpegtype(v) then lpegprint(v) return true end end) end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns @@ -481,7 +497,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=P("\r")*(P("\n")+P(true))+P("\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -1248,7 +1264,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 361, stripped down to: 322 +-- original size: 372, stripped down to: 329 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1267,7 +1283,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5694, stripped down to: 2827 +-- original size: 5983, stripped down to: 2959 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1354,9 +1370,10 @@ function string.valid(str,default) return (type(str)=="string" and str~="" and str) or default or nil end string.itself=function(s) return s end -local pattern=Ct(C(1)^0) -function string.totable(str) - return lpegmatch(pattern,str) +local pattern_c=Ct(C(1)^0) +local pattern_b=Ct((C(1)/byte)^0) +function string.totable(str,bytes) + return lpegmatch(bytes and pattern_b or pattern_c,str) end local replacer=lpeg.replacer("@","%%") function string.tformat(fmt,...) @@ -1372,7 +1389,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 35724, stripped down to: 21525 +-- original size: 36997, stripped down to: 22376 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2248,7 +2265,7 @@ function table.print(t,...) end end if setinspector then - setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) + setinspector("table",function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) end function table.sub(t,i,j) return { unpack(t,i,j) } @@ -2348,7 +2365,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8643, stripped down to: 6232 +-- original size: 9001, stripped down to: 6512 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2663,7 +2680,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 4939, stripped down to: 2830 +-- original size: 5146, stripped down to: 2933 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2808,7 +2825,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 1923, stripped down to: 1133 +-- original size: 2010, stripped down to: 1186 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2881,7 +2898,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 15832, stripped down to: 9456 +-- original size: 16390, stripped down to: 9734 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3263,7 +3280,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 20949, stripped down to: 9945 +-- original size: 21648, stripped down to: 10238 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3502,7 +3519,7 @@ local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") function file.join(one,two,three,...) if not two then - return one=="" and one or lpegmatch(stripper,one) + return one=="" and one or lpegmatch(reslasher,one) end if one=="" then return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) @@ -3643,7 +3660,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 1002 +-- original size: 1265, stripped down to: 1038 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3697,7 +3714,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3248, stripped down to: 2266 +-- original size: 3355, stripped down to: 2321 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3785,7 +3802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12531, stripped down to: 5721 +-- original size: 12897, stripped down to: 5882 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4002,7 +4019,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 16765, stripped down to: 11003 +-- original size: 17358, stripped down to: 11378 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4467,7 +4484,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1850, stripped down to: 1568 +-- original size: 1919, stripped down to: 1621 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4539,7 +4556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 37388, stripped down to: 15817 +-- original size: 38699, stripped down to: 16321 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4768,9 +4785,10 @@ if not utf.sub then end end end -function utf.remapper(mapping,option) +function utf.remapper(mapping,option,action) local variant=type(mapping) if variant=="table" then + action=action or mapping if option=="dynamic" then local pattern=false table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) @@ -4779,15 +4797,15 @@ function utf.remapper(mapping,option) return "" else if not pattern then - pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) end return lpegmatch(pattern,str) end end elseif option=="pattern" then - return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return Cs((tabletopattern(mapping)/action+p_utf8char)^0) else - local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + local pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) return function(str) if not str or str=="" then return "" @@ -5157,7 +5175,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 974, stripped down to: 890 +-- original size: 1012, stripped down to: 912 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5197,7 +5215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 34513, stripped down to: 18943 +-- original size: 36053, stripped down to: 19685 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5368,7 +5386,13 @@ function string.autosingle(s,sep) end return ("'"..tostring(s).."'") end -local tracedchars={} +local tracedchars={ [0]= + "[null]","[soh]","[stx]","[etx]","[eot]","[enq]","[ack]","[bel]", + "[bs]","[ht]","[lf]","[vt]","[ff]","[cr]","[so]","[si]", + "[dle]","[dc1]","[dc2]","[dc3]","[dc4]","[nak]","[syn]","[etb]", + "[can]","[em]","[sub]","[esc]","[fs]","[gs]","[rs]","[us]", + "[space]", +} string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) @@ -5885,7 +5909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 25338, stripped down to: 16247 +-- original size: 28680, stripped down to: 18636 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -6314,19 +6338,21 @@ local f_val_str=formatters["%w%q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] +local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") -local serialize=table.serialize -function table.serialize(root,name,specification) +local original_serialize=table.serialize +local function serialize(root,name,specification) if type(specification)=="table" then - return serialize(root,name,specification) + return original_serialize(root,name,specification) end - local t + local t local n=1 + local unknown=false local function simple_table(t) local nt=#t if nt>0 then @@ -6337,6 +6363,7 @@ function table.serialize(root,name,specification) return nil end end + local haszero=t[0] if n==nt then local tt={} for i=1,nt do @@ -6353,6 +6380,23 @@ function table.serialize(root,name,specification) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + tt[i+1]=v + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil @@ -6401,7 +6445,7 @@ function table.serialize(root,name,specification) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if next(v)==nil then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -6413,6 +6457,8 @@ function table.serialize(root,name,specification) end elseif tv=="boolean" then n=n+1 t[n]=f_val_boo(depth,v) + elseif unknown then + n=n+1 t[n]=f_val_str(depth,tostring(v)) end elseif tv=="number" then if tk=="number" then @@ -6421,6 +6467,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_num(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_num(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_num(depth,tostring(k),v) end elseif tv=="string" then if tk=="number" then @@ -6429,6 +6477,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_str(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_str(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),v) end elseif tv=="table" then if next(v)==nil then @@ -6438,6 +6488,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_not(depth,k) + elseif unknown then + n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else local st=simple_table(v) @@ -6449,6 +6501,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_seq(depth,k,st) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_seq(depth,k,st) + elseif unknown then + n=n+1 t[n]=f_key_str_value_seq(depth,tostring(k),st) end end elseif tv=="boolean" then @@ -6458,6 +6512,18 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_boo(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_boo(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_boo(depth,tostring(k),v) + end + else + if tk=="number" then + n=n+1 t[n]=f_key_num_value_str(depth,k,tostring(v)) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_str(depth,k,tostring(v)) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_str(depth,k,tostring(v)) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),tostring(v)) end end end @@ -6490,13 +6556,351 @@ function table.serialize(root,name,specification) root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - do_serialize(root,name,1,0) + local st=simple_table(root) + if st then + return t[1]..f_fin_seq(st) + else + do_serialize(root,name,1,0) + end end end n=n+1 t[n]=f_table_finish() return concat(t,"\n") end +table.serialize=serialize +if setinspector then + setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-fil"] = package.loaded["util-fil"] or true + +-- original size: 3577, stripped down to: 2870 + +if not modules then modules={} end modules ['util-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte=string.byte +local extract=bit32.extract +utilities=utilities or {} +local files={} +utilities.files=files +local zerobased={} +function files.open(filename,zb) + local f=io.open(filename,"rb") + if f then + zerobased[f]=zb or false + end + return f +end +function files.close(f) + zerobased[f]=nil + f:close() +end +function files.size(f) + return f:seek("end") +end +function files.setposition(f,n) + if zerobased[f] then + f:seek("set",n) + else + f:seek("set",n-1) + end +end +function files.getposition(f) + if zerobased[f] then + return f:seek() + else + return f:seek()+1 + end +end +function files.look(f,n,chars) + local p=f:seek() + local s=f:read(n) + f:seek("set",p) + if chars then + return s + else + return byte(s,1,#s) + end +end +function files.skip(f,n) + if n==1 then + f:read(n) + else + f:seek("set",f:seek()+n) + end +end +function files.readbyte(f) + return byte(f:read(1)) +end +function files.readbytes(f,n) + return byte(f:read(n),1,n) +end +function files.readchar(f) + return f:read(1) +end +function files.readstring(f,n) + return f:read(n or 1) +end +function files.readinteger1(f) + local n=byte(f:read(1)) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +files.readcardinal1=files.readbyte +files.readcardinal=files.readcardinal1 +files.readinteger=files.readinteger1 +function files.readcardinal2(f) + local a,b=byte(f:read(2),1,2) + return 0x100*a+b +end +function files.readinteger2(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function files.readcardinal3(f) + local a,b,c=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readcardinal4(f) + local a,b,c,d=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function files.readinteger4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function files.skipshort(f,n) + f:read(2*(n or 1)) +end +function files.skiplong(f,n) + f:read(4*(n or 1)) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sac"] = package.loaded["util-sac"] or true + +-- original size: 4264, stripped down to: 3349 + +if not modules then modules={} end modules ['util-sac']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte,sub=string.byte,string.sub +local extract=bit32.extract +utilities=utilities or {} +local streams={} +utilities.streams=streams +function streams.open(filename,zerobased) + local f=io.loaddata(filename) + return { f,1,#f,zerobased or false } +end +function streams.close() +end +function streams.size(f) + return f and f[3] or 0 +end +function streams.setposition(f,i) + if f[4] then + if i<=0 then + f[2]=1 + else + f[2]=i+1 + end + else + if i<=1 then + f[2]=1 + else + f[2]=i + end + end +end +function streams.getposition(f) + if f[4] then + return f[2]-1 + else + return f[2] + end +end +function streams.look(f,n,chars) + local b=f[2] + local e=b+n-1 + if chars then + return sub(f[1],b,e) + else + return byte(f[1],b,e) + end +end +function streams.skip(f,n) + f[2]=f[2]+n +end +function streams.readbyte(f) + local i=f[2] + f[2]=i+1 + return byte(f[1],i) +end +function streams.readbytes(f,n) + local i=f[2] + local j=i+n + f[2]=j + return byte(f[1],i,j-1) +end +function streams.skipbytes(f,n) + f[2]=f[2]+n +end +function streams.readchar(f) + local i=f[2] + f[2]=i+1 + return sub(f[1],i,i) +end +function streams.readstring(f,n) + local i=f[2] + local j=i+n + f[2]=j + return sub(f[1],i,j-1) +end +function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + local n=byte(f[1],i) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +streams.readcardinal1=streams.readbyte +streams.readcardinal=streams.readcardinal1 +streams.readinteger=streams.readinteger1 +function streams.readcardinal2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + return 0x100*a+b +end +function streams.readinteger2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function streams.readcardinal3(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readcardinal4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function streams.readinteger4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function streams.readfixed4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function streams.skipshort(f,n) + f[2]=f[2]+2*(n or 1) +end +function streams.skiplong(f,n) + f[2]=f[2]+4*(n or 1) +end end -- of closure @@ -6505,7 +6909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4172, stripped down to: 2953 +-- original size: 4100, stripped down to: 2852 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -6583,39 +6987,32 @@ local f_index={ ["table"]=f_table, ["number"]=f_number, } -local t_index={ - ["empty"]={ __index=f_empty }, - ["self"]={ __index=f_self }, - ["table"]={ __index=f_table }, - ["number"]={ __index=f_number }, -} function table.setmetatableindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__index=f_index[f] or f + m.__index=i else - setmetatable(t,t_index[f] or { __index=f }) + setmetatable(t,{ __index=i }) end return t end local f_index={ ["ignore"]=f_ignore, } -local t_index={ - ["ignore"]={ __newindex=f_ignore }, -} function table.setmetatablenewindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__newindex=f_index[f] or f + m.__newindex=i else - setmetatable(t,t_index[f] or { __newindex=f }) + setmetatable(t,{ __newindex=i }) end return t end @@ -6652,7 +7049,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 21780, stripped down to: 15121 +-- original size: 23411, stripped down to: 16177 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6676,6 +7073,8 @@ local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash local sortedkeys=table.sortedkeys local tohash=table.tohash +local hashes={} +utilities.parsers.hashes=hashes local digit=R("09") local space=P(' ') local equal=P("=") @@ -6684,6 +7083,8 @@ local lbrace=P("{") local rbrace=P("}") local lparent=P("(") local rparent=P(")") +local lbracket=P("[") +local rbracket=P("]") local period=S(".") local punctuation=S(".,:;") local spacer=lpegpatterns.spacer @@ -6693,6 +7094,7 @@ local anything=lpegpatterns.anything local endofstring=lpegpatterns.endofstring local nobrace=1-(lbrace+rbrace ) local noparent=1-(lparent+rparent) +local nobracket=1-(lbracket+rbracket) local escape,left,right=P("\\"),P('{'),P('}') lpegpatterns.balanced=P { [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, @@ -6700,6 +7102,7 @@ lpegpatterns.balanced=P { } local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local nestedbrackets=P { lbracket*(nobracket+V(1))^0*rbracket } local spaces=space^0 local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) local content=(1-endofstring)^0 @@ -6808,6 +7211,11 @@ function parsers.settings_to_array(str,strict) return { str } end end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_obey_fences(str) + return lpegmatch(pattern,str) +end local cache_a={} local cache_b={} function parsers.groupedsplitat(symbol,withaction) @@ -6894,9 +7302,15 @@ function parsers.array_to_string(a,separator) end end local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) -function utilities.parsers.settings_to_set(str,t) +function utilities.parsers.settings_to_set(str) return str and lpegmatch(pattern,str) or {} end +hashes.settings_to_set=table.setmetatableindex(function(t,k) + local v=k and lpegmatch(pattern,k) or {} + t[k]=v + return v +end) +getmetatable(hashes.settings_to_set).__mode="kv" function parsers.simple_hash_to_string(h,separator) local t,tn={},0 for k,v in sortedhash(h) do @@ -7173,7 +7587,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2274, stripped down to: 1781 +-- original size: 2350, stripped down to: 1847 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7254,7 +7668,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12482, stripped down to: 8864 +-- original size: 12862, stripped down to: 9104 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7567,7 +7981,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 29359, stripped down to: 20483 +-- original size: 30767, stripped down to: 21355 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7610,6 +8024,9 @@ setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then + if texio.setescape then + texio.setescape(0) + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -7627,6 +8044,8 @@ if tex and (tex.jobname or tex.formatname) then elseif target=="term" then texio_write_nl("term","") io_write(...) + elseif type(target)=="number" then + texio_write_nl(target,...) elseif target~="none" then texio_write_nl("log",target,...) texio_write_nl("term","") @@ -7644,6 +8063,8 @@ if tex and (tex.jobname or tex.formatname) then texio_write("log",...) elseif target=="term" then io_write(...) + elseif type(target)=="number" then + texio_write(target,...) elseif target~="none" then texio_write("log",target,...) io_write(target,...) @@ -7714,7 +8135,7 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end report=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,report_yes(translations[a],formats[b])) @@ -7725,7 +8146,7 @@ if tex and (tex.jobname or tex.formatname) then end end direct=function(a,b,c,...) - if c then + if c~=nil then return direct_yes(translations[a],formatters[formats[b]](c,...)) elseif b then return direct_yes(translations[a],formats[b]) @@ -7736,7 +8157,7 @@ if tex and (tex.jobname or tex.formatname) then end end subreport=function(a,s,b,c,...) - if c then + if c~=nil then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) elseif b then write_nl(target,subreport_yes(translations[a],translations[s],formats[b])) @@ -7747,7 +8168,7 @@ if tex and (tex.jobname or tex.formatname) then end end subdirect=function(a,s,b,c,...) - if c then + if c~=nil then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) elseif b then return subdirect_yes(translations[a],translations[s],formats[b]) @@ -7758,7 +8179,7 @@ if tex and (tex.jobname or tex.formatname) then end end status=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,status_yes(translations[a],formats[b])) @@ -8056,7 +8477,7 @@ function logs.messenger(category,subcategory) end end end -local function setblocked(category,value) +local function setblocked(category,value) if category==true then category,value="*",true elseif category==false then @@ -8071,7 +8492,7 @@ local function setblocked(category,value) end else states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) - for c,_ in next,states do + for c in next,states do local v=data[c] if v then v.state=value @@ -8353,7 +8774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6704, stripped down to: 5343 +-- original size: 6917, stripped down to: 5484 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8474,13 +8895,13 @@ function statistics.show() end end register("lua properties",function() - local list=status.list() - local hashchar=tonumber(list.luatex_hashchars) + local hashchar=tonumber(status.luatex_hashchars) + local hashtype=status.luatex_hashtype local mask=lua.mask or "ascii" return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", jit and "luajit" or "lua", statistics.memused(), - list.luatex_hashtype or "default", + hashtype or "default", hashchar and 2^hashchar or "unknown", mask, mask=="utf" and "τεχ" or "tex") @@ -8534,7 +8955,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5829, stripped down to: 3501 +-- original size: 6039, stripped down to: 3616 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -8681,7 +9102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 4982, stripped down to: 3511 +-- original size: 5142, stripped down to: 3611 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -8811,7 +9232,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3898, stripped down to: 2644 +-- original size: 4030, stripped down to: 2718 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8915,7 +9336,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7757, stripped down to: 6015 +-- original size: 7985, stripped down to: 6153 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9092,7 +9513,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7100, stripped down to: 3978 +-- original size: 7313, stripped down to: 4076 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -9237,7 +9658,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8022, stripped down to: 5038 +-- original size: 8284, stripped down to: 5176 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9424,7 +9845,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6174, stripped down to: 4141 +-- original size: 6358, stripped down to: 4257 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9577,7 +9998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 45848, stripped down to: 27914 +-- original size: 56973, stripped down to: 35872 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -9586,7 +10007,7 @@ if not modules then modules={} end modules ['lxml-tab']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local trace_entities=false trackers .register("xml.entities",function(v) trace_entities=v end) local report_xml=logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} @@ -9594,10 +10015,12 @@ local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub +local sort=table.sort local utfchar=utf.char local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters +do xml.xmlns=xml.xmlns or {} local check=P(false) local parse=check @@ -9614,23 +10037,68 @@ end function xml.resolvens(url) return lpegmatch(parse,lower(url)) or "" end +end local nsremap,resolvens=xml.xmlns,xml.resolvens -local stack={} -local top={} -local dt={} -local at={} -local xmlns={} -local errorstr=nil -local entities={} -local strip=false -local cleanup=false -local utfize=false -local resolve_predefined=false -local unify_predefined=false -local dcache={} -local hcache={} -local acache={} -local mt={} +local stack,level,top,at,xmlnms,errorstr +local entities,parameters +local strip,utfize,resolve,cleanup,resolve_predefined,unify_predefined +local dcache,hcache,acache +local mt,dt,nt +local function preparexmlstate(settings) + if settings then + stack={} + level=0 + top={} + at={} + mt={} + dt={} + nt=0 + xmlns={} + errorstr=nil + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + parameters={} + reported_at_errors={} + dcache={} + hcache={} + acache={} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + else + stack=nil + level=nil + top=nil + at=nil + mt=nil + dt=nil + nt=nil + xmlns=nil + errorstr=nil + strip=nil + utfize=nil + resolve=nil + resolve_predefined=nil + unify_predefined=nil + cleanup=nil + entities=nil + parameters=nil + reported_at_errors=nil + dcache=nil + hcache=nil + acache=nil + end +end local function initialize_mt(root) mt={ __index=root } end @@ -9640,8 +10108,9 @@ end function xml.checkerror(top,toclose) return "" end +local checkns=xml.checkns local function add_attribute(namespace,tag,value) - if cleanup and #value>0 then + if cleanup and value~="" then value=cleanup(value) end if tag=="xmlns" then @@ -9650,21 +10119,30 @@ local function add_attribute(namespace,tag,value) elseif namespace=="" then at[tag]=value elseif namespace=="xmlns" then - xml.checkns(tag,value) + checkns(tag,value) at["xmlns:"..tag]=value else at[namespace..":"..tag]=value end end local function add_empty(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top=stack[#stack] + top=stack[level] dt=top.dt - local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } - dt[#dt+1]=t + nt=#dt+1 + local t={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=top + } + dt[nt]=t setmetatable(t,mt) if at.xmlns then remove(xmlns) @@ -9672,23 +10150,35 @@ local function add_empty(spacing,namespace,tag) at={} end local function add_begin(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + top={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=stack[level] + } setmetatable(top,mt) dt=top.dt - stack[#stack+1]=top + nt=#dt + level=level+1 + stack[level]=top at={} end local function add_end(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end - local toclose=remove(stack) - top=stack[#stack] - if #stack<1 then + local toclose=stack[level] + level=level-1 + top=stack[level] + if level<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") report_xml(errorstr) elseif toclose.tg~=tag then @@ -9696,202 +10186,236 @@ local function add_end(spacing,namespace,tag) report_xml(errorstr) end dt=top.dt - dt[#dt+1]=toclose + nt=#dt+1 + dt[nt]=toclose if toclose.at.xmlns then remove(xmlns) end end -local spaceonly=lpegpatterns.whitespace^0*P(-1) local function add_text(text) - local n=#dt - if cleanup and #text>0 then - if n>0 then - local s=dt[n] + if text=="" then + return + end + if cleanup then + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..cleanup(text) + dt[nt]=s..cleanup(text) else - dt[n+1]=cleanup(text) + nt=nt+1 + dt[nt]=cleanup(text) end else + nt=1 dt[1]=cleanup(text) end else - if n>0 then - local s=dt[n] + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..text + dt[nt]=s..text else - dt[n+1]=text + nt=nt+1 + dt[nt]=text end else + nt=1 dt[1]=text end end end local function add_special(what,spacing,text) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end if strip and (what=="@cm@" or what=="@dt@") then else - dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + nt=nt+1 + dt[nt]={ special=true,ns="",tg=what,dt={ text } } end end local function set_message(txt) errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end -local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute value %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end local function attribute_specification_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute specification %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end -local badentity="&error;" -local badentity="&" -xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, - unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, - unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, -} -local placeholders=xml.placeholders -local function fromhex(s) - local n=tonumber(s,16) - if n then - return utfchar(n) - else - return formatters["h:%s"](s),true - end -end -local function fromdec(s) - local n=tonumber(s) - if n then - return utfchar(n) - else - return formatters["d:%s"](s),true - end -end -local p_rest=(1-P(";"))^0 -local p_many=P(1)^0 -local p_char=lpegpatterns.utf8character -local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) -local predefined_unified={ - [38]="&", - [42]=""", - [47]="'", - [74]="<", - [76]=">", -} -local predefined_simplified={ - [38]="&",amp="&", - [42]='"',quot='"', - [47]="'",apos="'", - [74]="<",lt="<", - [76]=">",gt=">", -} -local nofprivates=0xF0000 -local privates_u={ - [ [[&]] ]="&", - [ [["]] ]=""", - [ [[']] ]="'", - [ [[<]] ]="<", - [ [[>]] ]=">", -} -local privates_p={} -local privates_n={ -} -local escaped=utf.remapper(privates_u,"dynamic") -local unprivatized=utf.remapper(privates_p,"dynamic") -xml.unprivatized=unprivatized -local function unescaped(s) - local p=privates_n[s] - if not p then - nofprivates=nofprivates+1 - p=utfchar(nofprivates) - privates_n[s]=p - s="&"..s..";" - privates_u[p]=s - privates_p[p]=s +local grammar_parsed_text_one +local grammar_parsed_text_two +local handle_hex_entity +local handle_dec_entity +local handle_any_entity_dtd +local handle_any_entity_text +do + local badentity="&" + xml.placeholders={ + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, + unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, + unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, + } + local function fromhex(s) + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return formatters["h:%s"](s),true + end end - return p -end -xml.privatetoken=unescaped -xml.privatecodes=privates_n -local function handle_hex_entity(str) - local h=hcache[str] - if not h then - local n=tonumber(str,16) - h=unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end - elseif utfize then - h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end + local function fromdec(s) + local n=tonumber(s) + if n then + return utfchar(n) else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h="&#x"..str..";" + return formatters["d:%s"](s),true + end + end + local p_rest=(1-P(";"))^0 + local p_many=P(1)^0 + local p_char=lpegpatterns.utf8character + local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) + xml.parsedentitylpeg=parsedentity + local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", + } + local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", + } + local nofprivates=0xF0000 + local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", + } + local privates_p={ + } + local privates_s={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[&]] ]="&U+26;", + [ [[']] ]="&U+27;", + [ [[<]] ]="&U+3C;", + [ [[>]] ]="&U+3E;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } + local privates_n={ + } + local escaped=utf.remapper(privates_u,"dynamic") + local unprivatized=utf.remapper(privates_p,"dynamic") + local unspecialized=utf.remapper(privates_s,"dynamic") + xml.unprivatized=unprivatized + xml.unspecialized=unspecialized + xml.escaped=escaped + local function unescaped(s) + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + privates_s[p]=s end - hcache[str]=h + return p end - return h -end -local function handle_dec_entity(str) - local d=dcache[str] - if not d then - local n=tonumber(str) - d=unify_predefined and predefined_unified[n] - if d then - if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) - end - elseif utfize then - d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) + xml.privatetoken=unescaped + xml.privatecodes=privates_n + xml.specialcodes=privates_s + function xml.addspecialcode(key,value) + privates_s[key]=value or "&"..s..";" + end + handle_hex_entity=function(str) + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - else - if trace_entities then - report_xml("found entity &#%s;",str) + hcache[str]=h + end + return h + end + handle_dec_entity=function(str) + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" end - d="&#"..str..";" + dcache[str]=d end - dcache[str]=d + return d end - return d -end -xml.parsedentitylpeg=parsedentity -local function handle_any_entity(str) - if resolve then - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + handle_any_entity_dtd=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then if trace_entities then report_xml("resolving entity &%s; to predefined %a",str,a) end else if type(resolve)=="function" then - a=resolve(str) or entities[str] + a=resolve(str,entities) or entities[str] else a=entities[str] end @@ -9927,40 +10451,194 @@ local function handle_any_entity(str) end end end - acache[str]=a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; to %a",str,a) - acache[str]=a + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a end - return a - else - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + end + handle_any_entity_text=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then - acache[str]=a - if trace_entities then - report_xml("entity &%s; becomes %a",str,a) - end - elseif str=="" then if trace_entities then - report_xml("invalid entity &%s;",str) + report_xml("resolving entity &%s; to predefined %a",str,a) end - a=badentity - acache[str]=a else - if trace_entities then - report_xml("entity &%s; is made private",str) + if type(resolve)=="function" then + a=resolve(str,entities) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then + if trace_entities then + report_xml("expanding entity &%s; to function call",str) + end + a=a(str) or "" + end + a=lpegmatch(grammar_parsed_text_two,a) or a + if type(a)=="number" then + return "" + else + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + end + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then + if trace_entities then + report_xml("resolving entity &%s; to external %s",str,a) + end + else + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str=="" then + a=badentity + else + a="&"..str..";" + end + end + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a end - a=unescaped(str) - acache[str]=a end + return a + end + end + local p_rest=(1-P(";"))^1 + local spec={ + [0x23]="\\Ux{23}", + [0x24]="\\Ux{24}", + [0x25]="\\Ux{25}", + [0x5C]="\\Ux{5C}", + [0x7B]="\\Ux{7B}", + [0x7C]="\\Ux{7C}", + [0x7D]="\\Ux{7D}", + [0x7E]="\\Ux{7E}", + } + local hash=table.setmetatableindex(spec,function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true end - return a end + local reparsedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + local hash=table.setmetatableindex(function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true + end + end + local unescapedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + xml.reparsedentitylpeg=reparsedentity + xml.unescapedentitylpeg=unescapedentity end +local escaped=xml.escaped +local unescaped=xml.unescaped +local placeholders=xml.placeholders local function handle_end_entity(str) report_xml("error in entity, %a found without ending %a",str,";") return str @@ -9987,14 +10665,18 @@ local name=name_yes+name_nop local utfbom=lpegpatterns.utfbom local spacing=C(space^0) local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 -local hexentitycontent=R("AF","af","09")^0 -local decentitycontent=R("09")^0 +local hexentitycontent=R("AF","af","09")^1 +local decentitycontent=R("09")^1 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) - )+(anyentitycontent/handle_any_entity) + )+(anyentitycontent/handle_any_entity_dtd) +local parsedentity_text=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity_text) local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) +local entity_text=(ampersand/"")*parsedentity_text*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) -local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local text_parsed=(Cs((1-open-ampersand)^1)/add_text+Cs(entity_text)/add_text)^1 local somespace=space^1 local optionalspace=space^0 local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) @@ -10004,7 +10686,7 @@ local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error local attributevalue=value+wrongvalue local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 -local parsedtext=text_parsed/add_text +local parsedtext=text_parsed local unparsedtext=text_unparsed/add_text local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty @@ -10019,21 +10701,52 @@ local endcdata=P("]]")*close local someinstruction=C((1-endinstruction)^0) local somecomment=C((1-endcomment )^0) local somecdata=C((1-endcdata )^0) -local function normalentity(k,v ) entities[k]=v end -local function systementity(k,v,n) entities[k]=v end -local function publicentity(k,v,n) entities[k]=v end +local function weirdentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","weird",k,v) + end + parameters[k]=v +end +local function normalentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","normal",k,v) + end + entities[k]=v +end +local function systementity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","system",k,v) + end + entities[k]=v +end +local function publicentity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","public",k,v) + end + entities[k]=v +end local begindoctype=open*P("!DOCTYPE") local enddoctype=close local beginset=P("[") local endset=P("]") +local wrdtypename=C((1-somespace-P(";"))^1) local doctypename=C((1-somespace-close)^0) local elementdoctype=optionalspace*P("<!ELEMENT")*(1-close)^0*close local basiccomment=begincomment*((1-endcomment)^0)*endcomment +local weirdentitytype=P("%")*(somespace*doctypename*somespace*value)/weirdentity local normalentitytype=(doctypename*somespace*value)/normalentity local publicentitytype=(doctypename*somespace*P("PUBLIC")*somespace*value)/publicentity local systementitytype=(doctypename*somespace*P("SYSTEM")*somespace*value*somespace*P("NDATA")*somespace*doctypename)/systementity -local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype)*optionalspace*close -local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+basiccomment+space)^0*optionalspace*endset +local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype+weirdentitytype)*optionalspace*close +local function weirdresolve(s) + lpegmatch(entitydoctype,parameters[s]) +end +local function normalresolve(s) + lpegmatch(entitydoctype,entities[s]) +end +local entityresolve=P("%")*(wrdtypename/weirdresolve )*P(";")+P("&")*(wrdtypename/normalresolve)*P(";") +entitydoctype=entitydoctype+entityresolve +local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+entityresolve+basiccomment+space)^0*optionalspace*endset local definitiondoctype=doctypename*somespace*doctypeset local publicdoctype=doctypename*somespace*P("PUBLIC")*somespace*value*somespace*value*somespace*doctypeset local systemdoctype=doctypename*somespace*P("SYSTEM")*somespace*value*somespace*doctypeset @@ -10045,11 +10758,15 @@ local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata -local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error -local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 -local grammar_parsed_text=P { "preamble", - preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, +grammar_parsed_text_one=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0, +} +grammar_parsed_text_two=P { "followup", + followup=V("parent")*trailer, parent=beginelement*V("children")^0*endelement, children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } @@ -10059,37 +10776,26 @@ local grammar_unparsed_text=P { "preamble", children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) - settings=settings or {} - strip=settings.strip_cm_and_dt - utfize=settings.utfize_entities - resolve=settings.resolve_entities - resolve_predefined=settings.resolve_predefined_entities - unify_predefined=settings.unify_predefined_entities - cleanup=settings.text_cleanup - entities=settings.entities or {} - if utfize==nil then - settings.utfize_entities=true - utfize=true - end - if resolve_predefined==nil then - settings.resolve_predefined_entities=true - resolve_predefined=true - end - stack,top,at,xmlns,errorstr={},{},{},{},nil - acache,hcache,dcache={},{},{} - reported_attribute_errors={} + settings=settings or {} + preparexmlstate(settings) if settings.parent_root then mt=getmetatable(settings.parent_root) else initialize_mt(top) end - stack[#stack+1]=top + level=level+1 + stack[level]=top top.dt={} dt=top.dt + nt=0 if not data or data=="" then errorstr="empty xml file" elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then + local m=lpegmatch(grammar_parsed_text_one,data) + if m then + m=lpegmatch(grammar_parsed_text_two,data,m) + end + if m then else errorstr="invalid xml file - parsed text" end @@ -10105,8 +10811,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } -setmetatable(result,mt) -setmetatable(result.dt[1],mt) + setmetatable(result,mt) + setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -10148,13 +10854,10 @@ setmetatable(result.dt[1],mt) decimals=dcache, hexadecimals=hcache, names=acache, + intermediates=parameters, } } - strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil - unify_predefined,cleanup,entities=nil,nil,nil - stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil - acache,hcache,dcache=nil,nil,nil - reported_attribute_errors,mt,errorhandler=nil,nil,nil + preparexmlstate() return result end local function xmlconvert(data,settings) @@ -10216,15 +10919,15 @@ function xml.toxml(data) return data end end -local function copy(old,tables) +local function copy(old) if old then - tables=tables or {} local new={} - if not tables[old] then - tables[old]=new - end for k,v in next,old do - new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v + if type(v)=="table" then + new[k]=table.copy(v) + else + new[k]=v + end end local mt=getmetatable(old) if mt then @@ -10257,22 +10960,34 @@ local function verbose_element(e,handlers,escape) local ats=eat and next(eat) and {} if ats then local n=0 - for k,v in next,eat do + for k in next,eat do n=n+1 - ats[n]=f_attribute(k,escaped(v)) + ats[n]=k + end + if n==1 then + local k=ats[1] + ats=f_attribute(k,escaped(eat[k])) + else + sort(ats) + for i=1,n do + local k=ats[i] + ats[i]=f_attribute(k,escaped(eat[k])) + end + ats=concat(ats," ") end end if ern and trace_entities and ern~=ens then ens=ern end + local n=edt and #edt if ens~="" then - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") + handle("<",ens,":",etg," ",ats,">") else handle("<",ens,":",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10283,19 +10998,19 @@ local function verbose_element(e,handlers,escape) handle("</",ens,":",etg,">") else if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") + handle("<",ens,":",etg," ",ats,"/>") else handle("<",ens,":",etg,"/>") end end else - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",etg," ",concat(ats," "),">") + handle("<",etg," ",ats,">") else handle("<",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10306,7 +11021,7 @@ local function verbose_element(e,handlers,escape) handle("</",etg,">") else if ats then - handle("<",etg," ",concat(ats," "),"/>") + handle("<",etg," ",ats,"/>") else handle("<",etg,"/>") end @@ -10323,7 +11038,7 @@ local function verbose_cdata(e,handlers) handlers.handle("<![CDATA[",e.dt[1],"]]>") end local function verbose_doctype(e,handlers) - handlers.handle("<!DOCTYPE ",e.dt[1],">") + handlers.handle("<!DOCTYPE",e.dt[1],">") end local function verbose_root(e,handlers) handlers.serialize(e.dt,handlers) @@ -10366,12 +11081,14 @@ local function serialize(e,handlers,...) end end local function xserialize(e,handlers) - local functions=handlers.functions - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) + if e then + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end end end local handlers={} @@ -10603,7 +11320,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48229, stripped down to: 30684 +-- original size: 53892, stripped down to: 32508 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10618,10 +11335,23 @@ local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string. local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local setmetatableindex=table.setmetatableindex local formatters=string.formatters -local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end -local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end -local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local trace_lpath=false +local trace_lparse=false +local trace_lprofile=false local report_lpath=logs.reporter("xml","lpath") +if trackers then + trackers.register("xml.path",function(v) + trace_lpath=v + end) + trackers.register("xml.parse",function(v) + trace_lparse=v + end) + trackers.register("xml.profile",function(v) + trace_lpath=v + trace_lparse=v + trace_lprofile=v + end) +end local xml=xml local lpathcalls=0 function xml.lpathcalls () return lpathcalls end local lpathcached=0 function xml.lpathcached() return lpathcached end @@ -10980,13 +11710,27 @@ local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") local lp_doequal=P("=")/"==" local lp_or=P("|")/" or " local lp_and=P("&")/" and " -local lp_builtin=P ( - P("text")/"(ll.dt[1] or '')"+ - P("content")/"ll.dt"+ - P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ - P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ - P("ns")/"ll.ns" - )*((spaces*P("(")*spaces*P(")"))/"") +local builtin={ + text="(ll.dt[1] or '')", + content="ll.dt", + name="((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)", + tag="ll.tg", + position="l", + firstindex="1", + firstelement="1", + first="1", + lastindex="(#ll.__p__.dt or 1)", + lastelement="(ll.__p__.en or 1)", + last="#list", + rootposition="order", + order="order", + element="(ll.ei or 1)", + index="(ll.ni or 1)", + match="(ll.mi or 1)", + namespace="ll.ns", + ns="ll.ns", +} +local lp_builtin=lpeg.utfchartabletopattern(builtin)/builtin*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" @@ -11005,7 +11749,7 @@ local rparent=P(")") local noparent=1-(lparent+rparent) local nested=P{lparent*(noparent+V(1))^0*rparent} local value=P(lparent*C((noparent+nested)^0)*rparent) -local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_child=Cc("expr.child(ll,'")*R("az","AZ")*R("az","AZ","--","__")^0*Cc("')") local lp_number=S("+-")*R("09")^1 local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) @@ -11044,6 +11788,7 @@ local template_f_y=[[ local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] +local register_last_match={ kind="axis",axis="last-match" } local register_self={ kind="axis",axis="self" } local register_parent={ kind="axis",axis="parent" } local register_descendant={ kind="axis",axis="descendant" } @@ -11121,7 +11866,7 @@ local pathparser=Ct { "patterns", ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, - axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, @@ -11147,6 +11892,7 @@ local pathparser=Ct { "patterns", preceding=P('preceding::')*Cc(register_preceding ), preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + last_match=P('last-match::')*Cc(register_last_match ), nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -11193,13 +11939,12 @@ local function tagstostring(list) end xml.nodesettostring=nodesettostring local lpath -local lshowoptions={ functions=false } local function lshow(parsed) if type(parsed)=="string" then parsed=lpath(parsed) end report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + table.serialize(parsed,false)) end xml.lshow=lshow local function add_comment(p,str) @@ -11265,140 +12010,168 @@ lpath=function (pattern) end end xml.lpath=lpath -local profiled={} xml.profiled=profiled -local function profiled_apply(list,parsed,nofparsed,order) - local p=profiled[parsed.pattern] - if p then - p.tested=p.tested+1 - else - p={ tested=1,matched=0,finalized=0 } - profiled[parsed.pattern]=p - end - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - p.matched=p.matched+1 - p.finalized=p.finalized+1 - return collected - end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) +do + local profiled={} + xml.profiled=profiled + local lastmatch=nil + local keepmatch=nil + if directives then + directives.register("xml.path.keeplastmatch",function(v) + keepmatch=v + lastmatch=nil + end) + end + apply_axis["last-match"]=function() + return lastmatch or {} + end + local function profiled_apply(list,parsed,nofparsed,order) + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 p.finalized=p.finalized+1 return collected end - return nil + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + p.finalized=p.finalized+1 + return collected + end + return nil + end end - end - if collected then - p.matched=p.matched+1 - end - return collected -end -local function traced_apply(list,parsed,nofparsed,order) - if trace_lparse then - lshow(parsed) - end - report_lpath("collecting: %s",parsed.pattern) - report_lpath("root tags : %s",tagstostring(list)) - report_lpath("order : %s",order or "unset") - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected + if collected then + p.matched=p.matched+1 end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") return collected end - return nil - end - end - return collected -end -local function normal_apply(list,parsed,nofparsed,order) - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - local axis=pi.axis - if axis~="self" then - collected=apply_axis[axis](collected) + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil end - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - return pi.finalizer(collected) end - if not collected or #collected==0 then - local pf=i<nofparsed and parsed[nofparsed].finalizer - if pf then - return pf(collected) + return collected + end + local function normal_apply(list,parsed,nofparsed,order) + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + local axis=pi.axis + if axis~="self" then + collected=apply_axis[axis](collected) + end + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected==0 then + local pf=i<nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) + end + return nil end - return nil end + return collected end - return collected -end -local function applylpath(list,pattern) - if not list then - return - end - local parsed=cache[pattern] - if parsed then - lpathcalls=lpathcalls+1 - lpathcached=lpathcached+1 - elseif type(pattern)=="table" then - lpathcalls=lpathcalls+1 - parsed=pattern - else - parsed=lpath(pattern) or pattern - end - if not parsed then - return + local apply=normal_apply + if trackers then + trackers.register("xml.path,xml.parse,xml.profile",function() + if trace_lprofile then + apply=profiled_apply + elseif trace_lpath then + apply=traced_apply + else + apply=normal_apply + end + end) end - local nofparsed=#parsed - if nofparsed==0 then - return + function xml.applylpath(list,pattern) + if not list then + lastmatch=nil + return + end + local parsed=cache[pattern] + if parsed then + lpathcalls=lpathcalls+1 + lpathcached=lpathcached+1 + elseif type(pattern)=="table" then + lpathcalls=lpathcalls+1 + parsed=pattern + else + parsed=lpath(pattern) or pattern + end + if not parsed then + lastmatch=nil + return + end + local nofparsed=#parsed + if nofparsed==0 then + lastmatch=nil + return + end + local collected=apply({ list },parsed,nofparsed,list.mi) + lastmatch=keepmatch and collected or nil + return collected end - if not trace_lpath then - return normal_apply ({ list },parsed,nofparsed,list.mi) - elseif trace_lprofile then - return profiled_apply({ list },parsed,nofparsed,list.mi) - else - return traced_apply ({ list },parsed,nofparsed,list.mi) + function xml.lastmatch() + return lastmatch end end -xml.applylpath=applylpath +local applylpath=xml.applylpath function xml.filter(root,pattern) return applylpath(root,pattern) end @@ -11676,7 +12449,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3684, stripped down to: 1957 +-- original size: 3787, stripped down to: 2003 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -11745,7 +12518,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 28786, stripped down to: 20578 +-- original size: 30566, stripped down to: 21741 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12079,55 +12852,63 @@ local function include(xmldata,pattern,attribute,recursive,loaddata,level) local ek=collected[c] local name=nil local ekdt=ek.dt - local ekat=ek.at - local ekrt=ek.__p__ - local epdt=ekrt.dt - if not attribute or attribute=="" then - name=(type(ekdt)=="table" and ekdt[1]) or ekdt - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name=ekat[a] - if name then - break + if ekdt then + local ekat=ek.at + local ekrt=ek.__p__ + if ekrt then + local epdt=ekrt.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt end - end - end - local data=nil - if name and name~="" then - data=loaddata(name) or "" - if trace_inclusions then - report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") - end - end - if not data or data=="" then - epdt[ek.ni]="" - elseif ekat["parse"]=="text" then - epdt[ek.ni]=xml.escaped(data) - else - local xi=xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni]="" - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata,level+1) + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then + break + end + end end - local child=xml.body(xi) - child.__p__=ekrt - child.__f__=name - epdt[ek.ni]=child - local inclusions=xmldata.settings.inclusions - if inclusions then - inclusions[#inclusions+1]=name - else - xmldata.settings.inclusions={ name } + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") + end end - if child.er then - local badinclusions=xmldata.settings.badinclusions - if badinclusions then - badinclusions[#badinclusions+1]=name + if not data or data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" else - xmldata.settings.badinclusions={ name } + if recursive then + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local settings=xmldata.settings + local inclusions=settings and settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + elseif settings then + settings.inclusions={ name } + else + settings={ inclusions={ name } } + xmldata.settings=settings + end + if child.er then + local badinclusions=settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + settings.badinclusions={ name } + end + end end end end @@ -12598,7 +13379,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10274, stripped down to: 7538 +-- original size: 10719, stripped down to: 7841 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -12976,7 +13757,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6351, stripped down to: 4919 +-- original size: 6534, stripped down to: 5072 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13146,7 +13927,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11085, stripped down to: 7662 +-- original size: 11444, stripped down to: 7830 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -13402,7 +14183,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 17216, stripped down to: 10657 +-- original size: 18619, stripped down to: 11042 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -13413,6 +14194,7 @@ if not modules then modules={} end modules ['data-exp']={ } local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub local concat,sort=table.concat,table.sort +local sortedkeys=table.sortedkeys local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next @@ -13758,14 +14540,16 @@ local nothing=function() end function resolvers.filtered_from_content(content,pattern) if content and type(pattern)=="string" then local pattern=lower(pattern) - local files=content.files + local files=content.files local remap=content.remap if files and remap then - local n=next(files) + local f=sortedkeys(files) + local n=#f + local i=0 local function iterator() - while n do - local k=n - n=next(files,k) + while i<n do + i=i+1 + local k=f[i] if find(k,pattern) then return files[k],remap and remap[k] or k end @@ -13784,7 +14568,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9216, stripped down to: 6798 +-- original size: 9649, stripped down to: 7131 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -13920,6 +14704,11 @@ local relations=allocate { names={ 'fontconfig','fontconfig file','fontconfig files' }, variable='FONTCONFIG_PATH', }, + pk={ + names={ "pk" }, + variable='PKFONTS', + suffixes={ 'pk' }, + }, }, obsolete={ enc={ @@ -14063,7 +14852,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15618, stripped down to: 11629 +-- original size: 16066, stripped down to: 11938 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -14439,7 +15228,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5347, stripped down to: 4015 +-- original size: 5488, stripped down to: 4101 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -14558,7 +15347,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67003, stripped down to: 46291 +-- original size: 67241, stripped down to: 46427 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15828,10 +16617,18 @@ local function findfiles(filename,filetype,allresults) return result or {},status end function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,true) + end end function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,false)[1] or "" + end end function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") @@ -16106,7 +16903,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 3950, stripped down to: 2935 +-- original size: 4236, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -16170,16 +16967,20 @@ prefixes.pathname=function(str) return cleanpath(dirname((fullname~="" and fullname) or str)) end prefixes.selfautoloc=function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) + local pth=getenv('SELFAUTOLOC') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautoparent=function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) + local pth=getenv('SELFAUTOPARENT') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautodir=function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) + local pth=getenv('SELFAUTODIR') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.home=function(str) - return cleanpath(joinpath(getenv('HOME'),str)) + local pth=getenv('HOME') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.env=prefixes.environment prefixes.rel=prefixes.relative @@ -16224,7 +17025,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 910, stripped down to: 823 +-- original size: 935, stripped down to: 838 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -16254,7 +17055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 475 +-- original size: 548, stripped down to: 483 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -16277,7 +17078,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3310 +-- original size: 3976, stripped down to: 3391 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -16385,7 +17186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5010, stripped down to: 3588 +-- original size: 5148, stripped down to: 3680 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -16504,7 +17305,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3899, stripped down to: 2984 +-- original size: 4000, stripped down to: 3052 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -16595,7 +17396,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8772, stripped down to: 6841 +-- original size: 9036, stripped down to: 7041 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -16832,7 +17633,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8479, stripped down to: 5580 +-- original size: 8712, stripped down to: 5726 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17021,7 +17822,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6569, stripped down to: 5304 +-- original size: 6779, stripped down to: 5444 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17202,7 +18003,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4313, stripped down to: 3227 +-- original size: 4447, stripped down to: 3302 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -17311,7 +18112,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2431, stripped down to: 1996 +-- original size: 2494, stripped down to: 2047 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -17378,7 +18179,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2601, stripped down to: 1627 +-- original size: 2674, stripped down to: 1658 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -17434,7 +18235,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2734, stripped down to: 2354 +-- original size: 2815, stripped down to: 2415 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -17514,7 +18315,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11549, stripped down to: 5905 +-- original size: 11846, stripped down to: 6059 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -17700,7 +18501,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5703, stripped down to: 2507 +-- original size: 5914, stripped down to: 2584 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -17803,7 +18604,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5955, stripped down to: 4926 +-- original size: 6967, stripped down to: 5631 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -17832,7 +18633,7 @@ local function primaryflags() end return concat(flags," ") end -function environment.make_format(name) +function environment.make_format(name,silent) local engine=environment.ownmain or "luatex" local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" @@ -17889,9 +18690,23 @@ function environment.make_format(name) lfs.chdir(olddir) return end - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.execute(command) + local dump=os.platform=="unix" and "\\\\dump" or "\\dump" + if silent then + statistics.starttiming() + local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + local result=os.execute(command) + local runtime=statistics.stoptiming() + if result~=0 then + print(format("%s silent make > fatal error when making format %q",engine,name)) + else + print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) + end + os.remove("temp.log") + else + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + report_format("running command: %s\n",command) + os.execute(command) + end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -17935,10 +18750,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 745793 --- stripped bytes : 269308 +-- original bytes : 797557 +-- stripped bytes : 289197 -- end library merge @@ -17982,6 +18797,8 @@ local ownlibs = { -- order can be made better 'util-str.lua', -- code might move to l-string 'util-tab.lua', + 'util-fil.lua', + 'util-sac.lua', 'util-sto.lua', 'util-prs.lua', 'util-fmt.lua', @@ -18037,13 +18854,21 @@ local ownlibs = { -- order can be made better } --- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/mkiv/data-tmf.lua -- c:/data/develop/context/sources/data-tmf.lua local ownlist = { -- '.', -- ownpath , owntree .. "/../../../../context/sources", -- HH's development path + -- + owntree .. "/../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../texmf/tex/context/base/mkiv", + owntree .. "/../../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../../texmf/tex/context/base/mkiv", + -- owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", owntree .. "/../../texmf/tex/context/base", diff --git a/scripts/context/stubs/win64/mtxrun.lua b/scripts/context/stubs/win64/mtxrun.lua index 5c09b3b44..7b711a88d 100644 --- a/scripts/context/stubs/win64/mtxrun.lua +++ b/scripts/context/stubs/win64/mtxrun.lua @@ -56,7 +56,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lua"] = package.loaded["l-lua"] or true --- original size: 3888, stripped down to: 2197 +-- original size: 4734, stripped down to: 2626 if not modules then modules={} end modules ['l-lua']={ version=1.001, @@ -65,10 +65,14 @@ if not modules then modules={} end modules ['l-lua']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local major,minor=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") -_MAJORVERSION=tonumber(major) or 5 -_MINORVERSION=tonumber(minor) or 1 +_MAJORVERSION,_MINORVERSION=string.match(_VERSION,"^[^%d]+(%d+)%.(%d+).*$") +_MAJORVERSION=tonumber(_MAJORVERSION) or 5 +_MINORVERSION=tonumber(_MINORVERSION) or 1 _LUAVERSION=_MAJORVERSION+_MINORVERSION/10 +if _LUAVERSION<5.2 and jit then + _MINORVERSION=2 + _LUAVERSION=5.2 +end if not lpeg then lpeg=require("lpeg") end @@ -111,21 +115,33 @@ if not package.loaders then end local print,select,tostring=print,select,tostring local inspectors={} -function setinspector(inspector) - inspectors[#inspectors+1]=inspector +function setinspector(kind,inspector) + inspectors[kind]=inspector end function inspect(...) for s=1,select("#",...) do local value=select(s,...) - local done=false - for i=1,#inspectors do - done=inspectors[i](value) - if done then - break + if value==nil then + print("nil") + else + local done=false + local kind=type(value) + local inspector=inspectors[kind] + if inspector then + done=inspector(value) + if done then + break + end + end + for kind,inspector in next,inspectors do + done=inspector(value) + if done then + break + end + end + if not done then + print(tostring(value)) end - end - if not done then - print(tostring(value)) end end end @@ -154,7 +170,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-package"] = package.loaded["l-package"] or true --- original size: 10587, stripped down to: 7815 +-- original size: 10949, stripped down to: 8037 if not modules then modules={} end modules ['l-package']={ version=1.001, @@ -444,7 +460,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-lpeg"] = package.loaded["l-lpeg"] or true --- original size: 36977, stripped down to: 20349 +-- original size: 38185, stripped down to: 20990 if not modules then modules={} end modules ['l-lpeg']={ version=1.001, @@ -461,7 +477,7 @@ local floor=math.floor local P,R,S,V,Ct,C,Cs,Cc,Cp,Cmt=lpeg.P,lpeg.R,lpeg.S,lpeg.V,lpeg.Ct,lpeg.C,lpeg.Cs,lpeg.Cc,lpeg.Cp,lpeg.Cmt local lpegtype,lpegmatch,lpegprint=lpeg.type,lpeg.match,lpeg.print if setinspector then - setinspector(function(v) if lpegtype(v) then lpegprint(v) return true end end) + setinspector("lpeg",function(v) if lpegtype(v) then lpegprint(v) return true end end) end lpeg.patterns=lpeg.patterns or {} local patterns=lpeg.patterns @@ -481,7 +497,7 @@ local uppercase=R("AZ") local underscore=P("_") local hexdigit=digit+lowercase+uppercase local cr,lf,crlf=P("\r"),P("\n"),P("\r\n") -local newline=P("\r")*(P("\n")+P(true))+P("\n") +local newline=P("\r")*(P("\n")+P(true))+P("\n") local escaped=P("\\")*anything local squote=P("'") local dquote=P('"') @@ -1248,7 +1264,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-function"] = package.loaded["l-function"] or true --- original size: 361, stripped down to: 322 +-- original size: 372, stripped down to: 329 if not modules then modules={} end modules ['l-functions']={ version=1.001, @@ -1267,7 +1283,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-string"] = package.loaded["l-string"] or true --- original size: 5694, stripped down to: 2827 +-- original size: 5983, stripped down to: 2959 if not modules then modules={} end modules ['l-string']={ version=1.001, @@ -1354,9 +1370,10 @@ function string.valid(str,default) return (type(str)=="string" and str~="" and str) or default or nil end string.itself=function(s) return s end -local pattern=Ct(C(1)^0) -function string.totable(str) - return lpegmatch(pattern,str) +local pattern_c=Ct(C(1)^0) +local pattern_b=Ct((C(1)/byte)^0) +function string.totable(str,bytes) + return lpegmatch(bytes and pattern_b or pattern_c,str) end local replacer=lpeg.replacer("@","%%") function string.tformat(fmt,...) @@ -1372,7 +1389,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-table"] = package.loaded["l-table"] or true --- original size: 35724, stripped down to: 21525 +-- original size: 36997, stripped down to: 22376 if not modules then modules={} end modules ['l-table']={ version=1.001, @@ -2248,7 +2265,7 @@ function table.print(t,...) end end if setinspector then - setinspector(function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) + setinspector("table",function(v) if type(v)=="table" then serialize(print,v,"table") return true end end) end function table.sub(t,i,j) return { unpack(t,i,j) } @@ -2348,7 +2365,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-io"] = package.loaded["l-io"] or true --- original size: 8643, stripped down to: 6232 +-- original size: 9001, stripped down to: 6512 if not modules then modules={} end modules ['l-io']={ version=1.001, @@ -2663,7 +2680,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-number"] = package.loaded["l-number"] or true --- original size: 4939, stripped down to: 2830 +-- original size: 5146, stripped down to: 2933 if not modules then modules={} end modules ['l-number']={ version=1.001, @@ -2808,7 +2825,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-set"] = package.loaded["l-set"] or true --- original size: 1923, stripped down to: 1133 +-- original size: 2010, stripped down to: 1186 if not modules then modules={} end modules ['l-set']={ version=1.001, @@ -2881,7 +2898,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-os"] = package.loaded["l-os"] or true --- original size: 15832, stripped down to: 9456 +-- original size: 16390, stripped down to: 9734 if not modules then modules={} end modules ['l-os']={ version=1.001, @@ -3263,7 +3280,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-file"] = package.loaded["l-file"] or true --- original size: 20949, stripped down to: 9945 +-- original size: 21648, stripped down to: 10238 if not modules then modules={} end modules ['l-file']={ version=1.001, @@ -3502,7 +3519,7 @@ local reslasher=lpeg.replacer(S("\\/"),"/") local deslasher=lpeg.replacer(S("\\/")^1,"/") function file.join(one,two,three,...) if not two then - return one=="" and one or lpegmatch(stripper,one) + return one=="" and one or lpegmatch(reslasher,one) end if one=="" then return lpegmatch(stripper,three and concat({ two,three,... },"/") or two) @@ -3643,7 +3660,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-gzip"] = package.loaded["l-gzip"] or true --- original size: 1211, stripped down to: 1002 +-- original size: 1265, stripped down to: 1038 if not modules then modules={} end modules ['l-gzip']={ version=1.001, @@ -3697,7 +3714,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-md5"] = package.loaded["l-md5"] or true --- original size: 3248, stripped down to: 2266 +-- original size: 3355, stripped down to: 2321 if not modules then modules={} end modules ['l-md5']={ version=1.001, @@ -3785,7 +3802,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-url"] = package.loaded["l-url"] or true --- original size: 12531, stripped down to: 5721 +-- original size: 12897, stripped down to: 5882 if not modules then modules={} end modules ['l-url']={ version=1.001, @@ -4002,7 +4019,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-dir"] = package.loaded["l-dir"] or true --- original size: 16765, stripped down to: 11003 +-- original size: 17358, stripped down to: 11378 if not modules then modules={} end modules ['l-dir']={ version=1.001, @@ -4467,7 +4484,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-boolean"] = package.loaded["l-boolean"] or true --- original size: 1850, stripped down to: 1568 +-- original size: 1919, stripped down to: 1621 if not modules then modules={} end modules ['l-boolean']={ version=1.001, @@ -4539,7 +4556,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-unicode"] = package.loaded["l-unicode"] or true --- original size: 37388, stripped down to: 15817 +-- original size: 38699, stripped down to: 16321 if not modules then modules={} end modules ['l-unicode']={ version=1.001, @@ -4768,9 +4785,10 @@ if not utf.sub then end end end -function utf.remapper(mapping,option) +function utf.remapper(mapping,option,action) local variant=type(mapping) if variant=="table" then + action=action or mapping if option=="dynamic" then local pattern=false table.setmetatablenewindex(mapping,function(t,k,v) rawset(t,k,v) pattern=false end) @@ -4779,15 +4797,15 @@ function utf.remapper(mapping,option) return "" else if not pattern then - pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) end return lpegmatch(pattern,str) end end elseif option=="pattern" then - return Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + return Cs((tabletopattern(mapping)/action+p_utf8char)^0) else - local pattern=Cs((tabletopattern(mapping)/mapping+p_utf8char)^0) + local pattern=Cs((tabletopattern(mapping)/action+p_utf8char)^0) return function(str) if not str or str=="" then return "" @@ -5157,7 +5175,7 @@ do -- create closure to overcome 200 locals limit package.loaded["l-math"] = package.loaded["l-math"] or true --- original size: 974, stripped down to: 890 +-- original size: 1012, stripped down to: 912 if not modules then modules={} end modules ['l-math']={ version=1.001, @@ -5197,7 +5215,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-str"] = package.loaded["util-str"] or true --- original size: 34513, stripped down to: 18943 +-- original size: 36053, stripped down to: 19685 if not modules then modules={} end modules ['util-str']={ version=1.001, @@ -5368,7 +5386,13 @@ function string.autosingle(s,sep) end return ("'"..tostring(s).."'") end -local tracedchars={} +local tracedchars={ [0]= + "[null]","[soh]","[stx]","[etx]","[eot]","[enq]","[ack]","[bel]", + "[bs]","[ht]","[lf]","[vt]","[ff]","[cr]","[so]","[si]", + "[dle]","[dc1]","[dc2]","[dc3]","[dc4]","[nak]","[syn]","[etb]", + "[can]","[em]","[sub]","[esc]","[fs]","[gs]","[rs]","[us]", + "[space]", +} string.tracedchars=tracedchars strings.tracers=tracedchars function string.tracedchar(b) @@ -5885,7 +5909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tab"] = package.loaded["util-tab"] or true --- original size: 25338, stripped down to: 16247 +-- original size: 28680, stripped down to: 18636 if not modules then modules={} end modules ['util-tab']={ version=1.001, @@ -6314,19 +6338,21 @@ local f_val_str=formatters["%w%q,"] local f_val_boo=formatters["%w%l,"] local f_val_not=formatters["%w{},"] local f_val_seq=formatters["%w{ %, t },"] +local f_fin_seq=formatters[" %, t }"] local f_table_return=formatters["return {"] local f_table_name=formatters["%s={"] local f_table_direct=formatters["{"] local f_table_entry=formatters["[%q]={"] local f_table_finish=formatters["}"] local spaces=utilities.strings.newrepeater(" ") -local serialize=table.serialize -function table.serialize(root,name,specification) +local original_serialize=table.serialize +local function serialize(root,name,specification) if type(specification)=="table" then - return serialize(root,name,specification) + return original_serialize(root,name,specification) end - local t + local t local n=1 + local unknown=false local function simple_table(t) local nt=#t if nt>0 then @@ -6337,6 +6363,7 @@ function table.serialize(root,name,specification) return nil end end + local haszero=t[0] if n==nt then local tt={} for i=1,nt do @@ -6353,6 +6380,23 @@ function table.serialize(root,name,specification) end end return tt + elseif haszero and (n==nt+1) then + local tt={} + for i=0,nt do + local v=t[i] + local tv=type(v) + if tv=="number" then + tt[i+1]=v + elseif tv=="string" then + tt[i+1]=format("%q",v) + elseif tv=="boolean" then + tt[i+1]=v and "true" or "false" + else + return nil + end + end + tt[1]="[0] = "..tt[1] + return tt end end return nil @@ -6401,7 +6445,7 @@ function table.serialize(root,name,specification) elseif tv=="string" then n=n+1 t[n]=f_val_str(depth,v) elseif tv=="table" then - if next(v)==nil then + if next(v)==nil then n=n+1 t[n]=f_val_not(depth) else local st=simple_table(v) @@ -6413,6 +6457,8 @@ function table.serialize(root,name,specification) end elseif tv=="boolean" then n=n+1 t[n]=f_val_boo(depth,v) + elseif unknown then + n=n+1 t[n]=f_val_str(depth,tostring(v)) end elseif tv=="number" then if tk=="number" then @@ -6421,6 +6467,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_num(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_num(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_num(depth,tostring(k),v) end elseif tv=="string" then if tk=="number" then @@ -6429,6 +6477,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_str(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_str(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),v) end elseif tv=="table" then if next(v)==nil then @@ -6438,6 +6488,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_not(depth,k) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_not(depth,k) + elseif unknown then + n=n+1 t[n]=f_key_str_value_not(depth,tostring(k)) end else local st=simple_table(v) @@ -6449,6 +6501,8 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_seq(depth,k,st) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_seq(depth,k,st) + elseif unknown then + n=n+1 t[n]=f_key_str_value_seq(depth,tostring(k),st) end end elseif tv=="boolean" then @@ -6458,6 +6512,18 @@ function table.serialize(root,name,specification) n=n+1 t[n]=f_key_str_value_boo(depth,k,v) elseif tk=="boolean" then n=n+1 t[n]=f_key_boo_value_boo(depth,k,v) + elseif unknown then + n=n+1 t[n]=f_key_str_value_boo(depth,tostring(k),v) + end + else + if tk=="number" then + n=n+1 t[n]=f_key_num_value_str(depth,k,tostring(v)) + elseif tk=="string" then + n=n+1 t[n]=f_key_str_value_str(depth,k,tostring(v)) + elseif tk=="boolean" then + n=n+1 t[n]=f_key_boo_value_str(depth,k,tostring(v)) + elseif unknown then + n=n+1 t[n]=f_key_str_value_str(depth,tostring(k),tostring(v)) end end end @@ -6490,13 +6556,351 @@ function table.serialize(root,name,specification) root._w_h_a_t_e_v_e_r_=nil end if next(root)~=nil then - do_serialize(root,name,1,0) + local st=simple_table(root) + if st then + return t[1]..f_fin_seq(st) + else + do_serialize(root,name,1,0) + end end end n=n+1 t[n]=f_table_finish() return concat(t,"\n") end +table.serialize=serialize +if setinspector then + setinspector("table",function(v) if type(v)=="table" then print(serialize(v,"table",{})) return true end end) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-fil"] = package.loaded["util-fil"] or true + +-- original size: 3577, stripped down to: 2870 + +if not modules then modules={} end modules ['util-fil']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte=string.byte +local extract=bit32.extract +utilities=utilities or {} +local files={} +utilities.files=files +local zerobased={} +function files.open(filename,zb) + local f=io.open(filename,"rb") + if f then + zerobased[f]=zb or false + end + return f +end +function files.close(f) + zerobased[f]=nil + f:close() +end +function files.size(f) + return f:seek("end") +end +function files.setposition(f,n) + if zerobased[f] then + f:seek("set",n) + else + f:seek("set",n-1) + end +end +function files.getposition(f) + if zerobased[f] then + return f:seek() + else + return f:seek()+1 + end +end +function files.look(f,n,chars) + local p=f:seek() + local s=f:read(n) + f:seek("set",p) + if chars then + return s + else + return byte(s,1,#s) + end +end +function files.skip(f,n) + if n==1 then + f:read(n) + else + f:seek("set",f:seek()+n) + end +end +function files.readbyte(f) + return byte(f:read(1)) +end +function files.readbytes(f,n) + return byte(f:read(n),1,n) +end +function files.readchar(f) + return f:read(1) +end +function files.readstring(f,n) + return f:read(n or 1) +end +function files.readinteger1(f) + local n=byte(f:read(1)) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +files.readcardinal1=files.readbyte +files.readcardinal=files.readcardinal1 +files.readinteger=files.readinteger1 +function files.readcardinal2(f) + local a,b=byte(f:read(2),1,2) + return 0x100*a+b +end +function files.readinteger2(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function files.readcardinal3(f) + local a,b,c=byte(f:read(3),1,3) + return 0x10000*a+0x100*b+c +end +function files.readcardinal4(f) + local a,b,c,d=byte(f:read(4),1,4) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function files.readinteger4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function files.readfixed4(f) + local a,b,c,d=byte(f:read(4),1,4) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function files.read2dot14(f) + local a,b=byte(f:read(2),1,2) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function files.skipshort(f,n) + f:read(2*(n or 1)) +end +function files.skiplong(f,n) + f:read(4*(n or 1)) +end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +package.loaded["util-sac"] = package.loaded["util-sac"] or true + +-- original size: 4264, stripped down to: 3349 + +if not modules then modules={} end modules ['util-sac']={ + version=1.001, + comment="companion to luat-lib.mkiv", + author="Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright="PRAGMA ADE / ConTeXt Development Team", + license="see context related readme files" +} +local byte,sub=string.byte,string.sub +local extract=bit32.extract +utilities=utilities or {} +local streams={} +utilities.streams=streams +function streams.open(filename,zerobased) + local f=io.loaddata(filename) + return { f,1,#f,zerobased or false } +end +function streams.close() +end +function streams.size(f) + return f and f[3] or 0 +end +function streams.setposition(f,i) + if f[4] then + if i<=0 then + f[2]=1 + else + f[2]=i+1 + end + else + if i<=1 then + f[2]=1 + else + f[2]=i + end + end +end +function streams.getposition(f) + if f[4] then + return f[2]-1 + else + return f[2] + end +end +function streams.look(f,n,chars) + local b=f[2] + local e=b+n-1 + if chars then + return sub(f[1],b,e) + else + return byte(f[1],b,e) + end +end +function streams.skip(f,n) + f[2]=f[2]+n +end +function streams.readbyte(f) + local i=f[2] + f[2]=i+1 + return byte(f[1],i) +end +function streams.readbytes(f,n) + local i=f[2] + local j=i+n + f[2]=j + return byte(f[1],i,j-1) +end +function streams.skipbytes(f,n) + f[2]=f[2]+n +end +function streams.readchar(f) + local i=f[2] + f[2]=i+1 + return sub(f[1],i,i) +end +function streams.readstring(f,n) + local i=f[2] + local j=i+n + f[2]=j + return sub(f[1],i,j-1) +end +function streams.readinteger1(f) + local i=f[2] + f[2]=i+1 + local n=byte(f[1],i) + if n>=0x80 then + return n-0xFF-1 + else + return n + end +end +streams.readcardinal1=streams.readbyte +streams.readcardinal=streams.readcardinal1 +streams.readinteger=streams.readinteger1 +function streams.readcardinal2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + return 0x100*a+b +end +function streams.readinteger2(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1 + else + return n + end +end +function streams.readcardinal3(f) + local i=f[2] + local j=i+2 + f[2]=j+1 + local a,b,c=byte(f[1],i,j) + return 0x10000*a+0x100*b+c +end +function streams.readcardinal4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + return 0x1000000*a+0x10000*b+0x100*c+d +end +function streams.readinteger4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x1000000*a+0x10000*b+0x100*c+d + if n>=0x8000000 then + return n-0xFFFFFFFF-1 + else + return n + end +end +function streams.readfixed4(f) + local i=f[2] + local j=i+3 + f[2]=j+1 + local a,b,c,d=byte(f[1],i,j) + local n=0x100*a+b + if n>=0x8000 then + return n-0xFFFF-1+(0x100*c+d)/0xFFFF + else + return n+(0x100*c+d)/0xFFFF + end +end +function streams.read2dot14(f) + local i=f[2] + local j=i+1 + f[2]=j+1 + local a,b=byte(f[1],i,j) + local n=0x100*a+b + local m=extract(n,0,30) + if n>0x7FFF then + n=extract(n,30,2) + return m/0x4000-4 + else + n=extract(n,30,2) + return n+m/0x4000 + end +end +function streams.skipshort(f,n) + f[2]=f[2]+2*(n or 1) +end +function streams.skiplong(f,n) + f[2]=f[2]+4*(n or 1) +end end -- of closure @@ -6505,7 +6909,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-sto"] = package.loaded["util-sto"] or true --- original size: 4172, stripped down to: 2953 +-- original size: 4100, stripped down to: 2852 if not modules then modules={} end modules ['util-sto']={ version=1.001, @@ -6583,39 +6987,32 @@ local f_index={ ["table"]=f_table, ["number"]=f_number, } -local t_index={ - ["empty"]={ __index=f_empty }, - ["self"]={ __index=f_self }, - ["table"]={ __index=f_table }, - ["number"]={ __index=f_number }, -} function table.setmetatableindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__index=f_index[f] or f + m.__index=i else - setmetatable(t,t_index[f] or { __index=f }) + setmetatable(t,{ __index=i }) end return t end local f_index={ ["ignore"]=f_ignore, } -local t_index={ - ["ignore"]={ __newindex=f_ignore }, -} function table.setmetatablenewindex(t,f) if type(t)~="table" then f,t=t,{} end local m=getmetatable(t) + local i=f_index[f] or f if m then - m.__newindex=f_index[f] or f + m.__newindex=i else - setmetatable(t,t_index[f] or { __newindex=f }) + setmetatable(t,{ __newindex=i }) end return t end @@ -6652,7 +7049,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-prs"] = package.loaded["util-prs"] or true --- original size: 21780, stripped down to: 15121 +-- original size: 23411, stripped down to: 16177 if not modules then modules={} end modules ['util-prs']={ version=1.001, @@ -6676,6 +7073,8 @@ local setmetatableindex=table.setmetatableindex local sortedhash=table.sortedhash local sortedkeys=table.sortedkeys local tohash=table.tohash +local hashes={} +utilities.parsers.hashes=hashes local digit=R("09") local space=P(' ') local equal=P("=") @@ -6684,6 +7083,8 @@ local lbrace=P("{") local rbrace=P("}") local lparent=P("(") local rparent=P(")") +local lbracket=P("[") +local rbracket=P("]") local period=S(".") local punctuation=S(".,:;") local spacer=lpegpatterns.spacer @@ -6693,6 +7094,7 @@ local anything=lpegpatterns.anything local endofstring=lpegpatterns.endofstring local nobrace=1-(lbrace+rbrace ) local noparent=1-(lparent+rparent) +local nobracket=1-(lbracket+rbracket) local escape,left,right=P("\\"),P('{'),P('}') lpegpatterns.balanced=P { [1]=((escape*(left+right))+(1-(left+right))+V(2))^0, @@ -6700,6 +7102,7 @@ lpegpatterns.balanced=P { } local nestedbraces=P { lbrace*(nobrace+V(1))^0*rbrace } local nestedparents=P { lparent*(noparent+V(1))^0*rparent } +local nestedbrackets=P { lbracket*(nobracket+V(1))^0*rbracket } local spaces=space^0 local argument=Cs((lbrace/"")*((nobrace+nestedbraces)^0)*(rbrace/"")) local content=(1-endofstring)^0 @@ -6808,6 +7211,11 @@ function parsers.settings_to_array(str,strict) return { str } end end +local value=P(lbrace*C((nobrace+nestedbraces)^0)*rbrace)+C((nestedbraces+nestedbrackets+nestedparents+(1-comma))^0) +local pattern=spaces*Ct(value*(separator*value)^0) +function parsers.settings_to_array_obey_fences(str) + return lpegmatch(pattern,str) +end local cache_a={} local cache_b={} function parsers.groupedsplitat(symbol,withaction) @@ -6894,9 +7302,15 @@ function parsers.array_to_string(a,separator) end end local pattern=Cf(Ct("")*Cg(C((1-S(", "))^1)*S(", ")^0*Cc(true))^1,rawset) -function utilities.parsers.settings_to_set(str,t) +function utilities.parsers.settings_to_set(str) return str and lpegmatch(pattern,str) or {} end +hashes.settings_to_set=table.setmetatableindex(function(t,k) + local v=k and lpegmatch(pattern,k) or {} + t[k]=v + return v +end) +getmetatable(hashes.settings_to_set).__mode="kv" function parsers.simple_hash_to_string(h,separator) local t,tn={},0 for k,v in sortedhash(h) do @@ -7173,7 +7587,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-fmt"] = package.loaded["util-fmt"] or true --- original size: 2274, stripped down to: 1781 +-- original size: 2350, stripped down to: 1847 if not modules then modules={} end modules ['util-fmt']={ version=1.001, @@ -7254,7 +7668,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-set"] = package.loaded["trac-set"] or true --- original size: 12482, stripped down to: 8864 +-- original size: 12862, stripped down to: 9104 if not modules then modules={} end modules ['trac-set']={ version=1.001, @@ -7567,7 +7981,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-log"] = package.loaded["trac-log"] or true --- original size: 29359, stripped down to: 20483 +-- original size: 30767, stripped down to: 21355 if not modules then modules={} end modules ['trac-log']={ version=1.001, @@ -7610,6 +8024,9 @@ setmetatableindex(logs,function(t,k) t[k]=ignore;return ignore end) local report,subreport,status,settarget,setformats,settranslations local direct,subdirect,writer,pushtarget,poptarget,setlogfile,settimedlog,setprocessor,setformatters,newline if tex and (tex.jobname or tex.formatname) then + if texio.setescape then + texio.setescape(0) + end local function useluawrites() local texio_write_nl=texio.write_nl local texio_write=texio.write @@ -7627,6 +8044,8 @@ if tex and (tex.jobname or tex.formatname) then elseif target=="term" then texio_write_nl("term","") io_write(...) + elseif type(target)=="number" then + texio_write_nl(target,...) elseif target~="none" then texio_write_nl("log",target,...) texio_write_nl("term","") @@ -7644,6 +8063,8 @@ if tex and (tex.jobname or tex.formatname) then texio_write("log",...) elseif target=="term" then io_write(...) + elseif type(target)=="number" then + texio_write(target,...) elseif target~="none" then texio_write("log",target,...) io_write(target,...) @@ -7714,7 +8135,7 @@ if tex and (tex.jobname or tex.formatname) then write_nl(target,"\n") end report=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,report_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,report_yes(translations[a],formats[b])) @@ -7725,7 +8146,7 @@ if tex and (tex.jobname or tex.formatname) then end end direct=function(a,b,c,...) - if c then + if c~=nil then return direct_yes(translations[a],formatters[formats[b]](c,...)) elseif b then return direct_yes(translations[a],formats[b]) @@ -7736,7 +8157,7 @@ if tex and (tex.jobname or tex.formatname) then end end subreport=function(a,s,b,c,...) - if c then + if c~=nil then write_nl(target,subreport_yes(translations[a],translations[s],formatters[formats[b]](c,...))) elseif b then write_nl(target,subreport_yes(translations[a],translations[s],formats[b])) @@ -7747,7 +8168,7 @@ if tex and (tex.jobname or tex.formatname) then end end subdirect=function(a,s,b,c,...) - if c then + if c~=nil then return subdirect_yes(translations[a],translations[s],formatters[formats[b]](c,...)) elseif b then return subdirect_yes(translations[a],translations[s],formats[b]) @@ -7758,7 +8179,7 @@ if tex and (tex.jobname or tex.formatname) then end end status=function(a,b,c,...) - if c then + if c~=nil then write_nl(target,status_yes(translations[a],formatters[formats[b]](c,...))) elseif b then write_nl(target,status_yes(translations[a],formats[b])) @@ -8056,7 +8477,7 @@ function logs.messenger(category,subcategory) end end end -local function setblocked(category,value) +local function setblocked(category,value) if category==true then category,value="*",true elseif category==false then @@ -8071,7 +8492,7 @@ local function setblocked(category,value) end else states=utilities.parsers.settings_to_hash(category,type(states)=="table" and states or nil) - for c,_ in next,states do + for c in next,states do local v=data[c] if v then v.state=value @@ -8353,7 +8774,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-inf"] = package.loaded["trac-inf"] or true --- original size: 6704, stripped down to: 5343 +-- original size: 6917, stripped down to: 5484 if not modules then modules={} end modules ['trac-inf']={ version=1.001, @@ -8474,13 +8895,13 @@ function statistics.show() end end register("lua properties",function() - local list=status.list() - local hashchar=tonumber(list.luatex_hashchars) + local hashchar=tonumber(status.luatex_hashchars) + local hashtype=status.luatex_hashtype local mask=lua.mask or "ascii" return format("engine: %s, used memory: %s, hash type: %s, hash chars: min(%s,40), symbol mask: %s (%s)", jit and "luajit" or "lua", statistics.memused(), - list.luatex_hashtype or "default", + hashtype or "default", hashchar and 2^hashchar or "unknown", mask, mask=="utf" and "τεχ" or "tex") @@ -8534,7 +8955,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-pro"] = package.loaded["trac-pro"] or true --- original size: 5829, stripped down to: 3501 +-- original size: 6039, stripped down to: 3616 if not modules then modules={} end modules ['trac-pro']={ version=1.001, @@ -8681,7 +9102,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lua"] = package.loaded["util-lua"] or true --- original size: 4982, stripped down to: 3511 +-- original size: 5142, stripped down to: 3611 if not modules then modules={} end modules ['util-lua']={ version=1.001, @@ -8811,7 +9232,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-deb"] = package.loaded["util-deb"] or true --- original size: 3898, stripped down to: 2644 +-- original size: 4030, stripped down to: 2718 if not modules then modules={} end modules ['util-deb']={ version=1.001, @@ -8915,7 +9336,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-mrg"] = package.loaded["util-mrg"] or true --- original size: 7757, stripped down to: 6015 +-- original size: 7985, stripped down to: 6153 if not modules then modules={} end modules ['util-mrg']={ version=1.001, @@ -9092,7 +9513,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-tpl"] = package.loaded["util-tpl"] or true --- original size: 7100, stripped down to: 3978 +-- original size: 7313, stripped down to: 4076 if not modules then modules={} end modules ['util-tpl']={ version=1.001, @@ -9237,7 +9658,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-env"] = package.loaded["util-env"] or true --- original size: 8022, stripped down to: 5038 +-- original size: 8284, stripped down to: 5176 if not modules then modules={} end modules ['util-env']={ version=1.001, @@ -9424,7 +9845,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-env"] = package.loaded["luat-env"] or true --- original size: 6174, stripped down to: 4141 +-- original size: 6358, stripped down to: 4257 if not modules then modules={} end modules ['luat-env']={ version=1.001, @@ -9577,7 +9998,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-tab"] = package.loaded["lxml-tab"] or true --- original size: 45848, stripped down to: 27914 +-- original size: 56973, stripped down to: 35872 if not modules then modules={} end modules ['lxml-tab']={ version=1.001, @@ -9586,7 +10007,7 @@ if not modules then modules={} end modules ['lxml-tab']={ copyright="PRAGMA ADE / ConTeXt Development Team", license="see context related readme files" } -local trace_entities=false trackers.register("xml.entities",function(v) trace_entities=v end) +local trace_entities=false trackers .register("xml.entities",function(v) trace_entities=v end) local report_xml=logs and logs.reporter("xml","core") or function(...) print(string.format(...)) end if lpeg.setmaxstack then lpeg.setmaxstack(1000) end xml=xml or {} @@ -9594,10 +10015,12 @@ local xml=xml local concat,remove,insert=table.concat,table.remove,table.insert local type,next,setmetatable,getmetatable,tonumber,rawset=type,next,setmetatable,getmetatable,tonumber,rawset local lower,find,match,gsub=string.lower,string.find,string.match,string.gsub +local sort=table.sort local utfchar=utf.char local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local P,S,R,C,V,C,Cs=lpeg.P,lpeg.S,lpeg.R,lpeg.C,lpeg.V,lpeg.C,lpeg.Cs local formatters=string.formatters +do xml.xmlns=xml.xmlns or {} local check=P(false) local parse=check @@ -9614,23 +10037,68 @@ end function xml.resolvens(url) return lpegmatch(parse,lower(url)) or "" end +end local nsremap,resolvens=xml.xmlns,xml.resolvens -local stack={} -local top={} -local dt={} -local at={} -local xmlns={} -local errorstr=nil -local entities={} -local strip=false -local cleanup=false -local utfize=false -local resolve_predefined=false -local unify_predefined=false -local dcache={} -local hcache={} -local acache={} -local mt={} +local stack,level,top,at,xmlnms,errorstr +local entities,parameters +local strip,utfize,resolve,cleanup,resolve_predefined,unify_predefined +local dcache,hcache,acache +local mt,dt,nt +local function preparexmlstate(settings) + if settings then + stack={} + level=0 + top={} + at={} + mt={} + dt={} + nt=0 + xmlns={} + errorstr=nil + strip=settings.strip_cm_and_dt + utfize=settings.utfize_entities + resolve=settings.resolve_entities + resolve_predefined=settings.resolve_predefined_entities + unify_predefined=settings.unify_predefined_entities + cleanup=settings.text_cleanup + entities=settings.entities or {} + parameters={} + reported_at_errors={} + dcache={} + hcache={} + acache={} + if utfize==nil then + settings.utfize_entities=true + utfize=true + end + if resolve_predefined==nil then + settings.resolve_predefined_entities=true + resolve_predefined=true + end + else + stack=nil + level=nil + top=nil + at=nil + mt=nil + dt=nil + nt=nil + xmlns=nil + errorstr=nil + strip=nil + utfize=nil + resolve=nil + resolve_predefined=nil + unify_predefined=nil + cleanup=nil + entities=nil + parameters=nil + reported_at_errors=nil + dcache=nil + hcache=nil + acache=nil + end +end local function initialize_mt(root) mt={ __index=root } end @@ -9640,8 +10108,9 @@ end function xml.checkerror(top,toclose) return "" end +local checkns=xml.checkns local function add_attribute(namespace,tag,value) - if cleanup and #value>0 then + if cleanup and value~="" then value=cleanup(value) end if tag=="xmlns" then @@ -9650,21 +10119,30 @@ local function add_attribute(namespace,tag,value) elseif namespace=="" then at[tag]=value elseif namespace=="xmlns" then - xml.checkns(tag,value) + checkns(tag,value) at["xmlns:"..tag]=value else at[namespace..":"..tag]=value end end local function add_empty(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top=stack[#stack] + top=stack[level] dt=top.dt - local t={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=top } - dt[#dt+1]=t + nt=#dt+1 + local t={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=top + } + dt[nt]=t setmetatable(t,mt) if at.xmlns then remove(xmlns) @@ -9672,23 +10150,35 @@ local function add_empty(spacing,namespace,tag) at={} end local function add_begin(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end local resolved=namespace=="" and xmlns[#xmlns] or nsremap[namespace] or namespace - top={ ns=namespace or "",rn=resolved,tg=tag,at=at,dt={},__p__=stack[#stack] } + top={ + ns=namespace or "", + rn=resolved, + tg=tag, + at=at, + dt={}, + __p__=stack[level] + } setmetatable(top,mt) dt=top.dt - stack[#stack+1]=top + nt=#dt + level=level+1 + stack[level]=top at={} end local function add_end(spacing,namespace,tag) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end - local toclose=remove(stack) - top=stack[#stack] - if #stack<1 then + local toclose=stack[level] + level=level-1 + top=stack[level] + if level<1 then errorstr=formatters["unable to close %s %s"](tag,xml.checkerror(top,toclose) or "") report_xml(errorstr) elseif toclose.tg~=tag then @@ -9696,202 +10186,236 @@ local function add_end(spacing,namespace,tag) report_xml(errorstr) end dt=top.dt - dt[#dt+1]=toclose + nt=#dt+1 + dt[nt]=toclose if toclose.at.xmlns then remove(xmlns) end end -local spaceonly=lpegpatterns.whitespace^0*P(-1) local function add_text(text) - local n=#dt - if cleanup and #text>0 then - if n>0 then - local s=dt[n] + if text=="" then + return + end + if cleanup then + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..cleanup(text) + dt[nt]=s..cleanup(text) else - dt[n+1]=cleanup(text) + nt=nt+1 + dt[nt]=cleanup(text) end else + nt=1 dt[1]=cleanup(text) end else - if n>0 then - local s=dt[n] + if nt>0 then + local s=dt[nt] if type(s)=="string" then - dt[n]=s..text + dt[nt]=s..text else - dt[n+1]=text + nt=nt+1 + dt[nt]=text end else + nt=1 dt[1]=text end end end local function add_special(what,spacing,text) - if #spacing>0 then - dt[#dt+1]=spacing + if spacing~="" then + nt=nt+1 + dt[nt]=spacing end if strip and (what=="@cm@" or what=="@dt@") then else - dt[#dt+1]={ special=true,ns="",tg=what,dt={ text } } + nt=nt+1 + dt[nt]={ special=true,ns="",tg=what,dt={ text } } end end local function set_message(txt) errorstr="garbage at the end of the file: "..gsub(txt,"([ \n\r\t]*)","") end -local reported_attribute_errors={} local function attribute_value_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute value %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end local function attribute_specification_error(str) - if not reported_attribute_errors[str] then + if not reported_at_errors[str] then report_xml("invalid attribute specification %a",str) - reported_attribute_errors[str]=true + reported_at_errors[str]=true at._error_=str end return str end -local badentity="&error;" -local badentity="&" -xml.placeholders={ - unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, - unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, - unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, -} -local placeholders=xml.placeholders -local function fromhex(s) - local n=tonumber(s,16) - if n then - return utfchar(n) - else - return formatters["h:%s"](s),true - end -end -local function fromdec(s) - local n=tonumber(s) - if n then - return utfchar(n) - else - return formatters["d:%s"](s),true - end -end -local p_rest=(1-P(";"))^0 -local p_many=P(1)^0 -local p_char=lpegpatterns.utf8character -local parsedentity=P("&")*(P("#x")*(p_rest/fromhex)+P("#")*(p_rest/fromdec))*P(";")*P(-1)+(P("#x")*(p_many/fromhex)+P("#")*(p_many/fromdec)) -local predefined_unified={ - [38]="&", - [42]=""", - [47]="'", - [74]="<", - [76]=">", -} -local predefined_simplified={ - [38]="&",amp="&", - [42]='"',quot='"', - [47]="'",apos="'", - [74]="<",lt="<", - [76]=">",gt=">", -} -local nofprivates=0xF0000 -local privates_u={ - [ [[&]] ]="&", - [ [["]] ]=""", - [ [[']] ]="'", - [ [[<]] ]="<", - [ [[>]] ]=">", -} -local privates_p={} -local privates_n={ -} -local escaped=utf.remapper(privates_u,"dynamic") -local unprivatized=utf.remapper(privates_p,"dynamic") -xml.unprivatized=unprivatized -local function unescaped(s) - local p=privates_n[s] - if not p then - nofprivates=nofprivates+1 - p=utfchar(nofprivates) - privates_n[s]=p - s="&"..s..";" - privates_u[p]=s - privates_p[p]=s +local grammar_parsed_text_one +local grammar_parsed_text_two +local handle_hex_entity +local handle_dec_entity +local handle_any_entity_dtd +local handle_any_entity_text +do + local badentity="&" + xml.placeholders={ + unknown_dec_entity=function(str) return str=="" and badentity or formatters["&%s;"](str) end, + unknown_hex_entity=function(str) return formatters["&#x%s;"](str) end, + unknown_any_entity=function(str) return formatters["&#x%s;"](str) end, + } + local function fromhex(s) + local n=tonumber(s,16) + if n then + return utfchar(n) + else + return formatters["h:%s"](s),true + end end - return p -end -xml.privatetoken=unescaped -xml.privatecodes=privates_n -local function handle_hex_entity(str) - local h=hcache[str] - if not h then - local n=tonumber(str,16) - h=unify_predefined and predefined_unified[n] - if h then - if trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end - elseif utfize then - h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" - if not n then - report_xml("utfize, ignoring hex entity &#x%s;",str) - elseif trace_entities then - report_xml("utfize, converting hex entity &#x%s; into %a",str,h) - end + local function fromdec(s) + local n=tonumber(s) + if n then + return utfchar(n) else - if trace_entities then - report_xml("found entity &#x%s;",str) - end - h="&#x"..str..";" + return formatters["d:%s"](s),true + end + end + local p_rest=(1-P(";"))^0 + local p_many=P(1)^0 + local p_char=lpegpatterns.utf8character + local parsedentity=P("&#")*(P("x")*(p_rest/fromhex)+(p_rest/fromdec))*P(";")*P(-1)+P ("#")*(P("x")*(p_many/fromhex)+(p_many/fromdec)) + xml.parsedentitylpeg=parsedentity + local predefined_unified={ + [38]="&", + [42]=""", + [47]="'", + [74]="<", + [76]=">", + } + local predefined_simplified={ + [38]="&",amp="&", + [42]='"',quot='"', + [47]="'",apos="'", + [74]="<",lt="<", + [76]=">",gt=">", + } + local nofprivates=0xF0000 + local privates_u={ + [ [[&]] ]="&", + [ [["]] ]=""", + [ [[']] ]="'", + [ [[<]] ]="<", + [ [[>]] ]=">", + } + local privates_p={ + } + local privates_s={ + [ [["]] ]="&U+22;", + [ [[#]] ]="&U+23;", + [ [[$]] ]="&U+24;", + [ [[%]] ]="&U+25;", + [ [[&]] ]="&U+26;", + [ [[']] ]="&U+27;", + [ [[<]] ]="&U+3C;", + [ [[>]] ]="&U+3E;", + [ [[\]] ]="&U+5C;", + [ [[{]] ]="&U+7B;", + [ [[|]] ]="&U+7C;", + [ [[}]] ]="&U+7D;", + [ [[~]] ]="&U+7E;", + } + local privates_n={ + } + local escaped=utf.remapper(privates_u,"dynamic") + local unprivatized=utf.remapper(privates_p,"dynamic") + local unspecialized=utf.remapper(privates_s,"dynamic") + xml.unprivatized=unprivatized + xml.unspecialized=unspecialized + xml.escaped=escaped + local function unescaped(s) + local p=privates_n[s] + if not p then + nofprivates=nofprivates+1 + p=utfchar(nofprivates) + privates_n[s]=p + s="&"..s..";" + privates_u[p]=s + privates_p[p]=s + privates_s[p]=s end - hcache[str]=h + return p end - return h -end -local function handle_dec_entity(str) - local d=dcache[str] - if not d then - local n=tonumber(str) - d=unify_predefined and predefined_unified[n] - if d then - if trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) - end - elseif utfize then - d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" - if not n then - report_xml("utfize, ignoring dec entity &#%s;",str) - elseif trace_entities then - report_xml("utfize, converting dec entity &#%s; into %a",str,d) + xml.privatetoken=unescaped + xml.privatecodes=privates_n + xml.specialcodes=privates_s + function xml.addspecialcode(key,value) + privates_s[key]=value or "&"..s..";" + end + handle_hex_entity=function(str) + local h=hcache[str] + if not h then + local n=tonumber(str,16) + h=unify_predefined and predefined_unified[n] + if h then + if trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + elseif utfize then + h=(n and utfchar(n)) or xml.unknown_hex_entity(str) or "" + if not n then + report_xml("utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + report_xml("utfize, converting hex entity &#x%s; into %a",str,h) + end + else + if trace_entities then + report_xml("found entity &#x%s;",str) + end + h="&#x"..str..";" end - else - if trace_entities then - report_xml("found entity &#%s;",str) + hcache[str]=h + end + return h + end + handle_dec_entity=function(str) + local d=dcache[str] + if not d then + local n=tonumber(str) + d=unify_predefined and predefined_unified[n] + if d then + if trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + elseif utfize then + d=(n and utfchar(n)) or placeholders.unknown_dec_entity(str) or "" + if not n then + report_xml("utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + report_xml("utfize, converting dec entity &#%s; into %a",str,d) + end + else + if trace_entities then + report_xml("found entity &#%s;",str) + end + d="&#"..str..";" end - d="&#"..str..";" + dcache[str]=d end - dcache[str]=d + return d end - return d -end -xml.parsedentitylpeg=parsedentity -local function handle_any_entity(str) - if resolve then - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + handle_any_entity_dtd=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then if trace_entities then report_xml("resolving entity &%s; to predefined %a",str,a) end else if type(resolve)=="function" then - a=resolve(str) or entities[str] + a=resolve(str,entities) or entities[str] else a=entities[str] end @@ -9927,40 +10451,194 @@ local function handle_any_entity(str) end end end - acache[str]=a - elseif trace_entities then - if not acache[str] then - report_xml("converting entity &%s; to %a",str,a) - acache[str]=a + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a + end end + return a end - return a - else - local a=acache[str] - if not a then - a=resolve_predefined and predefined_simplified[str] + end + handle_any_entity_text=function(str) + if resolve then + local a=resolve_predefined and predefined_simplified[str] if a then - acache[str]=a - if trace_entities then - report_xml("entity &%s; becomes %a",str,a) - end - elseif str=="" then if trace_entities then - report_xml("invalid entity &%s;",str) + report_xml("resolving entity &%s; to predefined %a",str,a) end - a=badentity - acache[str]=a else - if trace_entities then - report_xml("entity &%s; is made private",str) + if type(resolve)=="function" then + a=resolve(str,entities) or entities[str] + else + a=entities[str] + end + if a then + if type(a)=="function" then + if trace_entities then + report_xml("expanding entity &%s; to function call",str) + end + a=a(str) or "" + end + a=lpegmatch(grammar_parsed_text_two,a) or a + if type(a)=="number" then + return "" + else + a=lpegmatch(parsedentity,a) or a + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + end + if trace_entities then + report_xml("resolving entity &%s; to internal %a",str,a) + end + else + local unknown_any_entity=placeholders.unknown_any_entity + if unknown_any_entity then + a=unknown_any_entity(str) or "" + end + if a then + if trace_entities then + report_xml("resolving entity &%s; to external %s",str,a) + end + else + if trace_entities then + report_xml("keeping entity &%s;",str) + end + if str=="" then + a=badentity + else + a="&"..str..";" + end + end + end + end + return a + else + local a=acache[str] + if not a then + a=resolve_predefined and predefined_simplified[str] + if a then + acache[str]=a + if trace_entities then + report_xml("entity &%s; becomes %a",str,a) + end + elseif str=="" then + if trace_entities then + report_xml("invalid entity &%s;",str) + end + a=badentity + acache[str]=a + else + if trace_entities then + report_xml("entity &%s; is made private",str) + end + a=unescaped(str) + acache[str]=a end - a=unescaped(str) - acache[str]=a end + return a + end + end + local p_rest=(1-P(";"))^1 + local spec={ + [0x23]="\\Ux{23}", + [0x24]="\\Ux{24}", + [0x25]="\\Ux{25}", + [0x5C]="\\Ux{5C}", + [0x7B]="\\Ux{7B}", + [0x7C]="\\Ux{7C}", + [0x7D]="\\Ux{7D}", + [0x7E]="\\Ux{7E}", + } + local hash=table.setmetatableindex(spec,function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true end - return a end + local reparsedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + local hash=table.setmetatableindex(function(t,k) + local v=utfchar(k) + t[k]=v + return v + end) + local function fromuni(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["u:%s"](s),true + end + end + local function fromhex(s) + local n=tonumber(s,16) + if n then + return hash[n] + else + return formatters["h:%s"](s),true + end + end + local function fromdec(s) + local n=tonumber(s) + if n then + return hash[n] + else + return formatters["d:%s"](s),true + end + end + local unescapedentity=P("U+")*(p_rest/fromuni)+P("#")*( + P("x")*(p_rest/fromhex)+p_rest/fromdec + ) + xml.reparsedentitylpeg=reparsedentity + xml.unescapedentitylpeg=unescapedentity end +local escaped=xml.escaped +local unescaped=xml.unescaped +local placeholders=xml.placeholders local function handle_end_entity(str) report_xml("error in entity, %a found without ending %a",str,";") return str @@ -9987,14 +10665,18 @@ local name=name_yes+name_nop local utfbom=lpegpatterns.utfbom local spacing=C(space^0) local anyentitycontent=(1-open-semicolon-space-close-ampersand)^0 -local hexentitycontent=R("AF","af","09")^0 -local decentitycontent=R("09")^0 +local hexentitycontent=R("AF","af","09")^1 +local decentitycontent=R("09")^1 local parsedentity=P("#")/""*( P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) - )+(anyentitycontent/handle_any_entity) + )+(anyentitycontent/handle_any_entity_dtd) +local parsedentity_text=P("#")/""*( + P("x")/""*(hexentitycontent/handle_hex_entity)+(decentitycontent/handle_dec_entity) + )+(anyentitycontent/handle_any_entity_text) local entity=(ampersand/"")*parsedentity*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) +local entity_text=(ampersand/"")*parsedentity_text*(semicolon/"")+ampersand*(anyentitycontent/handle_end_entity) local text_unparsed=C((1-open)^1) -local text_parsed=Cs(((1-open-ampersand)^1+entity)^1) +local text_parsed=(Cs((1-open-ampersand)^1)/add_text+Cs(entity_text)/add_text)^1 local somespace=space^1 local optionalspace=space^0 local value=(squote*Cs((entity+(1-squote))^0)*squote)+(dquote*Cs((entity+(1-dquote))^0)*dquote) @@ -10004,7 +10686,7 @@ local wrongvalue=Cs(P(entity+(1-space-endofattributes))^1)/attribute_value_error local attributevalue=value+wrongvalue local attribute=(somespace*name*optionalspace*equal*optionalspace*attributevalue)/add_attribute local attributes=(attribute+somespace^-1*(((1-endofattributes)^1)/attribute_specification_error))^0 -local parsedtext=text_parsed/add_text +local parsedtext=text_parsed local unparsedtext=text_unparsed/add_text local balanced=P { "["*((1-S"[]")+V(1))^0*"]" } local emptyelement=(spacing*open*name*attributes*optionalspace*slash*close)/add_empty @@ -10019,21 +10701,52 @@ local endcdata=P("]]")*close local someinstruction=C((1-endinstruction)^0) local somecomment=C((1-endcomment )^0) local somecdata=C((1-endcdata )^0) -local function normalentity(k,v ) entities[k]=v end -local function systementity(k,v,n) entities[k]=v end -local function publicentity(k,v,n) entities[k]=v end +local function weirdentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","weird",k,v) + end + parameters[k]=v +end +local function normalentity(k,v) + if trace_entities then + report_xml("registering %s entity %a as %a","normal",k,v) + end + entities[k]=v +end +local function systementity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","system",k,v) + end + entities[k]=v +end +local function publicentity(k,v,n) + if trace_entities then + report_xml("registering %s entity %a as %a","public",k,v) + end + entities[k]=v +end local begindoctype=open*P("!DOCTYPE") local enddoctype=close local beginset=P("[") local endset=P("]") +local wrdtypename=C((1-somespace-P(";"))^1) local doctypename=C((1-somespace-close)^0) local elementdoctype=optionalspace*P("<!ELEMENT")*(1-close)^0*close local basiccomment=begincomment*((1-endcomment)^0)*endcomment +local weirdentitytype=P("%")*(somespace*doctypename*somespace*value)/weirdentity local normalentitytype=(doctypename*somespace*value)/normalentity local publicentitytype=(doctypename*somespace*P("PUBLIC")*somespace*value)/publicentity local systementitytype=(doctypename*somespace*P("SYSTEM")*somespace*value*somespace*P("NDATA")*somespace*doctypename)/systementity -local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype)*optionalspace*close -local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+basiccomment+space)^0*optionalspace*endset +local entitydoctype=optionalspace*P("<!ENTITY")*somespace*(systementitytype+publicentitytype+normalentitytype+weirdentitytype)*optionalspace*close +local function weirdresolve(s) + lpegmatch(entitydoctype,parameters[s]) +end +local function normalresolve(s) + lpegmatch(entitydoctype,entities[s]) +end +local entityresolve=P("%")*(wrdtypename/weirdresolve )*P(";")+P("&")*(wrdtypename/normalresolve)*P(";") +entitydoctype=entitydoctype+entityresolve +local doctypeset=beginset*optionalspace*P(elementdoctype+entitydoctype+entityresolve+basiccomment+space)^0*optionalspace*endset local definitiondoctype=doctypename*somespace*doctypeset local publicdoctype=doctypename*somespace*P("PUBLIC")*somespace*value*somespace*value*somespace*doctypeset local systemdoctype=doctypename*somespace*P("SYSTEM")*somespace*value*somespace*doctypeset @@ -10045,11 +10758,15 @@ local cdata=(spacing*begincdata*somecdata*endcdata )/function(...) add_special local doctype=(spacing*begindoctype*somedoctype*enddoctype )/function(...) add_special("@dt@",...) end local crap_parsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata-ampersand local crap_unparsed=1-beginelement-endelement-emptyelement-begininstruction-begincomment-begincdata -local parsedcrap=Cs((crap_parsed^1+entity)^1)/handle_crap_error -local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local parsedcrap=Cs((crap_parsed^1+entity_text)^1)/handle_crap_error +local unparsedcrap=Cs((crap_unparsed )^1)/handle_crap_error local trailer=space^0*(text_unparsed/set_message)^0 -local grammar_parsed_text=P { "preamble", - preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0*V("parent")*trailer, +grammar_parsed_text_one=P { "preamble", + preamble=utfbom^0*instruction^0*(doctype+comment+instruction)^0, +} +grammar_parsed_text_two=P { "followup", + followup=V("parent")*trailer, parent=beginelement*V("children")^0*endelement, children=parsedtext+V("parent")+emptyelement+comment+cdata+instruction+parsedcrap, } @@ -10059,37 +10776,26 @@ local grammar_unparsed_text=P { "preamble", children=unparsedtext+V("parent")+emptyelement+comment+cdata+instruction+unparsedcrap, } local function _xmlconvert_(data,settings) - settings=settings or {} - strip=settings.strip_cm_and_dt - utfize=settings.utfize_entities - resolve=settings.resolve_entities - resolve_predefined=settings.resolve_predefined_entities - unify_predefined=settings.unify_predefined_entities - cleanup=settings.text_cleanup - entities=settings.entities or {} - if utfize==nil then - settings.utfize_entities=true - utfize=true - end - if resolve_predefined==nil then - settings.resolve_predefined_entities=true - resolve_predefined=true - end - stack,top,at,xmlns,errorstr={},{},{},{},nil - acache,hcache,dcache={},{},{} - reported_attribute_errors={} + settings=settings or {} + preparexmlstate(settings) if settings.parent_root then mt=getmetatable(settings.parent_root) else initialize_mt(top) end - stack[#stack+1]=top + level=level+1 + stack[level]=top top.dt={} dt=top.dt + nt=0 if not data or data=="" then errorstr="empty xml file" elseif utfize or resolve then - if lpegmatch(grammar_parsed_text,data) then + local m=lpegmatch(grammar_parsed_text_one,data) + if m then + m=lpegmatch(grammar_parsed_text_two,data,m) + end + if m then else errorstr="invalid xml file - parsed text" end @@ -10105,8 +10811,8 @@ local function _xmlconvert_(data,settings) local result if errorstr and errorstr~="" then result={ dt={ { ns="",tg="error",dt={ errorstr },at={},er=true } } } -setmetatable(result,mt) -setmetatable(result.dt[1],mt) + setmetatable(result,mt) + setmetatable(result.dt[1],mt) setmetatable(stack,mt) local errorhandler=settings.error_handler if errorhandler==false then @@ -10148,13 +10854,10 @@ setmetatable(result.dt[1],mt) decimals=dcache, hexadecimals=hcache, names=acache, + intermediates=parameters, } } - strip,utfize,resolve,resolve_predefined=nil,nil,nil,nil - unify_predefined,cleanup,entities=nil,nil,nil - stack,top,at,xmlns,errorstr=nil,nil,nil,nil,nil - acache,hcache,dcache=nil,nil,nil - reported_attribute_errors,mt,errorhandler=nil,nil,nil + preparexmlstate() return result end local function xmlconvert(data,settings) @@ -10216,15 +10919,15 @@ function xml.toxml(data) return data end end -local function copy(old,tables) +local function copy(old) if old then - tables=tables or {} local new={} - if not tables[old] then - tables[old]=new - end for k,v in next,old do - new[k]=(type(v)=="table" and (tables[v] or copy(v,tables))) or v + if type(v)=="table" then + new[k]=table.copy(v) + else + new[k]=v + end end local mt=getmetatable(old) if mt then @@ -10257,22 +10960,34 @@ local function verbose_element(e,handlers,escape) local ats=eat and next(eat) and {} if ats then local n=0 - for k,v in next,eat do + for k in next,eat do n=n+1 - ats[n]=f_attribute(k,escaped(v)) + ats[n]=k + end + if n==1 then + local k=ats[1] + ats=f_attribute(k,escaped(eat[k])) + else + sort(ats) + for i=1,n do + local k=ats[i] + ats[i]=f_attribute(k,escaped(eat[k])) + end + ats=concat(ats," ") end end if ern and trace_entities and ern~=ens then ens=ern end + local n=edt and #edt if ens~="" then - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",ens,":",etg," ",concat(ats," "),">") + handle("<",ens,":",etg," ",ats,">") else handle("<",ens,":",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10283,19 +10998,19 @@ local function verbose_element(e,handlers,escape) handle("</",ens,":",etg,">") else if ats then - handle("<",ens,":",etg," ",concat(ats," "),"/>") + handle("<",ens,":",etg," ",ats,"/>") else handle("<",ens,":",etg,"/>") end end else - if edt and #edt>0 then + if n and n>0 then if ats then - handle("<",etg," ",concat(ats," "),">") + handle("<",etg," ",ats,">") else handle("<",etg,">") end - for i=1,#edt do + for i=1,n do local e=edt[i] if type(e)=="string" then handle(escaped(e)) @@ -10306,7 +11021,7 @@ local function verbose_element(e,handlers,escape) handle("</",etg,">") else if ats then - handle("<",etg," ",concat(ats," "),"/>") + handle("<",etg," ",ats,"/>") else handle("<",etg,"/>") end @@ -10323,7 +11038,7 @@ local function verbose_cdata(e,handlers) handlers.handle("<![CDATA[",e.dt[1],"]]>") end local function verbose_doctype(e,handlers) - handlers.handle("<!DOCTYPE ",e.dt[1],">") + handlers.handle("<!DOCTYPE",e.dt[1],">") end local function verbose_root(e,handlers) handlers.serialize(e.dt,handlers) @@ -10366,12 +11081,14 @@ local function serialize(e,handlers,...) end end local function xserialize(e,handlers) - local functions=handlers.functions - local etg=e.tg - if etg then - (functions[etg] or functions["@el@"])(e,handlers) - else - functions["@dc@"](e,handlers) + if e then + local functions=handlers.functions + local etg=e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + else + functions["@dc@"](e,handlers) + end end end local handlers={} @@ -10603,7 +11320,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-lpt"] = package.loaded["lxml-lpt"] or true --- original size: 48229, stripped down to: 30684 +-- original size: 53892, stripped down to: 32508 if not modules then modules={} end modules ['lxml-lpt']={ version=1.001, @@ -10618,10 +11335,23 @@ local format,upper,lower,gmatch,gsub,find,rep=string.format,string.upper,string. local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local setmetatableindex=table.setmetatableindex local formatters=string.formatters -local trace_lpath=false if trackers then trackers.register("xml.path",function(v) trace_lpath=v end) end -local trace_lparse=false if trackers then trackers.register("xml.parse",function(v) trace_lparse=v end) end -local trace_lprofile=false if trackers then trackers.register("xml.profile",function(v) trace_lpath=v trace_lparse=v trace_lprofile=v end) end +local trace_lpath=false +local trace_lparse=false +local trace_lprofile=false local report_lpath=logs.reporter("xml","lpath") +if trackers then + trackers.register("xml.path",function(v) + trace_lpath=v + end) + trackers.register("xml.parse",function(v) + trace_lparse=v + end) + trackers.register("xml.profile",function(v) + trace_lpath=v + trace_lparse=v + trace_lprofile=v + end) +end local xml=xml local lpathcalls=0 function xml.lpathcalls () return lpathcalls end local lpathcached=0 function xml.lpathcached() return lpathcached end @@ -10980,13 +11710,27 @@ local lp_noequal=P("!=")/"~="+P("<=")+P(">=")+P("==") local lp_doequal=P("=")/"==" local lp_or=P("|")/" or " local lp_and=P("&")/" and " -local lp_builtin=P ( - P("text")/"(ll.dt[1] or '')"+ - P("content")/"ll.dt"+ - P("name")/"((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)"+P("tag")/"ll.tg"+P("position")/"l"+ - P("firstindex")/"1"+P("lastindex")/"(#ll.__p__.dt or 1)"+P("firstelement")/"1"+P("lastelement")/"(ll.__p__.en or 1)"+P("first")/"1"+P("last")/"#list"+P("rootposition")/"order"+P("order")/"order"+P("element")/"(ll.ei or 1)"+P("index")/"(ll.ni or 1)"+P("match")/"(ll.mi or 1)"+ - P("ns")/"ll.ns" - )*((spaces*P("(")*spaces*P(")"))/"") +local builtin={ + text="(ll.dt[1] or '')", + content="ll.dt", + name="((ll.ns~='' and ll.ns..':'..ll.tg) or ll.tg)", + tag="ll.tg", + position="l", + firstindex="1", + firstelement="1", + first="1", + lastindex="(#ll.__p__.dt or 1)", + lastelement="(ll.__p__.en or 1)", + last="#list", + rootposition="order", + order="order", + element="(ll.ei or 1)", + index="(ll.ni or 1)", + match="(ll.mi or 1)", + namespace="ll.ns", + ns="ll.ns", +} +local lp_builtin=lpeg.utfchartabletopattern(builtin)/builtin*((spaces*P("(")*spaces*P(")"))/"") local lp_attribute=(P("@")+P("attribute::"))/""*Cc("(ll.at and ll.at['")*((R("az","AZ")+S("-_:"))^1)*Cc("'])") local lp_fastpos_p=P("+")^0*R("09")^1*P(-1)/"l==%0" local lp_fastpos_n=P("-")*R("09")^1*P(-1)/"(%0<0 and (#list+%0==l))" @@ -11005,7 +11749,7 @@ local rparent=P(")") local noparent=1-(lparent+rparent) local nested=P{lparent*(noparent+V(1))^0*rparent} local value=P(lparent*C((noparent+nested)^0)*rparent) -local lp_child=Cc("expr.child(ll,'")*R("az","AZ","--","__")^1*Cc("')") +local lp_child=Cc("expr.child(ll,'")*R("az","AZ")*R("az","AZ","--","__")^0*Cc("')") local lp_number=S("+-")*R("09")^1 local lp_string=Cc("'")*R("az","AZ","--","__")^1*Cc("'") local lp_content=(P("'")*(1-P("'"))^0*P("'")+P('"')*(1-P('"'))^0*P('"')) @@ -11044,6 +11788,7 @@ local template_f_y=[[ local template_f_n=[[ return xml.finalizers['%s']['%s'] ]] +local register_last_match={ kind="axis",axis="last-match" } local register_self={ kind="axis",axis="self" } local register_parent={ kind="axis",axis="parent" } local register_descendant={ kind="axis",axis="descendant" } @@ -11121,7 +11866,7 @@ local pathparser=Ct { "patterns", ), protocol=Cg(V("letters"),"protocol")*P("://")+Cg(Cc(nil),"protocol"), step=((V("shortcuts")+P("/")+V("axis"))*spaces*V("nodes")^0+V("error"))*spaces*V("expressions")^0*spaces*V("finalizer")^0, - axis=V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), + axis=V("last_match")+V("descendant")+V("child")+V("parent")+V("self")+V("root")+V("ancestor")+V("descendant_or_self")+V("following_sibling")+V("following")+V("reverse_sibling")+V("preceding_sibling")+V("preceding")+V("ancestor_or_self")+#(1-P(-1))*Cc(register_auto_child), special=special_1+special_2+special_3, initial=(P("/")*spaces*Cc(register_initial_child))^-1, error=(P(1)^1)/register_error, @@ -11147,6 +11892,7 @@ local pathparser=Ct { "patterns", preceding=P('preceding::')*Cc(register_preceding ), preceding_sibling=P('preceding-sibling::')*Cc(register_preceding_sibling ), reverse_sibling=P('reverse-sibling::')*Cc(register_reverse_sibling ), + last_match=P('last-match::')*Cc(register_last_match ), nodes=(V("nodefunction")*spaces*P("(")*V("nodeset")*P(")")+V("nodetest")*V("nodeset"))/register_nodes, expressions=expression/register_expression, letters=R("az")^1, @@ -11193,13 +11939,12 @@ local function tagstostring(list) end xml.nodesettostring=nodesettostring local lpath -local lshowoptions={ functions=false } local function lshow(parsed) if type(parsed)=="string" then parsed=lpath(parsed) end report_lpath("%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern, - table.serialize(parsed,false,lshowoptions)) + table.serialize(parsed,false)) end xml.lshow=lshow local function add_comment(p,str) @@ -11265,140 +12010,168 @@ lpath=function (pattern) end end xml.lpath=lpath -local profiled={} xml.profiled=profiled -local function profiled_apply(list,parsed,nofparsed,order) - local p=profiled[parsed.pattern] - if p then - p.tested=p.tested+1 - else - p={ tested=1,matched=0,finalized=0 } - profiled[parsed.pattern]=p - end - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - p.matched=p.matched+1 - p.finalized=p.finalized+1 - return collected - end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) +do + local profiled={} + xml.profiled=profiled + local lastmatch=nil + local keepmatch=nil + if directives then + directives.register("xml.path.keeplastmatch",function(v) + keepmatch=v + lastmatch=nil + end) + end + apply_axis["last-match"]=function() + return lastmatch or {} + end + local function profiled_apply(list,parsed,nofparsed,order) + local p=profiled[parsed.pattern] + if p then + p.tested=p.tested+1 + else + p={ tested=1,matched=0,finalized=0 } + profiled[parsed.pattern]=p + end + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + p.matched=p.matched+1 p.finalized=p.finalized+1 return collected end - return nil + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + p.finalized=p.finalized+1 + return collected + end + return nil + end end - end - if collected then - p.matched=p.matched+1 - end - return collected -end -local function traced_apply(list,parsed,nofparsed,order) - if trace_lparse then - lshow(parsed) - end - report_lpath("collecting: %s",parsed.pattern) - report_lpath("root tags : %s",tagstostring(list)) - report_lpath("order : %s",order or "unset") - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - collected=apply_axis[pi.axis](collected) - report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) - elseif kind=="finalizer" then - collected=pi.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") - return collected + if collected then + p.matched=p.matched+1 end - if not collected or #collected==0 then - local pn=i<nofparsed and parsed[nofparsed] - if pn and pn.kind=="finalizer" then - collected=pn.finalizer(collected) - report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + local function traced_apply(list,parsed,nofparsed,order) + if trace_lparse then + lshow(parsed) + end + report_lpath("collecting: %s",parsed.pattern) + report_lpath("root tags : %s",tagstostring(list)) + report_lpath("order : %s",order or "unset") + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + collected=apply_axis[pi.axis](collected) + report_lpath("% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + report_lpath("% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + report_lpath("% 10i : ex : %s -> %s",(collected and #collected) or 0,pi.expression,pi.converted) + elseif kind=="finalizer" then + collected=pi.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") return collected end - return nil - end - end - return collected -end -local function normal_apply(list,parsed,nofparsed,order) - local collected=list - for i=1,nofparsed do - local pi=parsed[i] - local kind=pi.kind - if kind=="axis" then - local axis=pi.axis - if axis~="self" then - collected=apply_axis[axis](collected) + if not collected or #collected==0 then + local pn=i<nofparsed and parsed[nofparsed] + if pn and pn.kind=="finalizer" then + collected=pn.finalizer(collected) + report_lpath("% 10i : fi : %s : %s(%s)",(type(collected)=="table" and #collected) or 0,parsed.protocol or xml.defaultprotocol,pn.name,pn.arguments or "") + return collected + end + return nil end - elseif kind=="nodes" then - collected=apply_nodes(collected,pi.nodetest,pi.nodes) - elseif kind=="expression" then - collected=apply_expression(collected,pi.evaluator,order) - elseif kind=="finalizer" then - return pi.finalizer(collected) end - if not collected or #collected==0 then - local pf=i<nofparsed and parsed[nofparsed].finalizer - if pf then - return pf(collected) + return collected + end + local function normal_apply(list,parsed,nofparsed,order) + local collected=list + for i=1,nofparsed do + local pi=parsed[i] + local kind=pi.kind + if kind=="axis" then + local axis=pi.axis + if axis~="self" then + collected=apply_axis[axis](collected) + end + elseif kind=="nodes" then + collected=apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind=="expression" then + collected=apply_expression(collected,pi.evaluator,order) + elseif kind=="finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected==0 then + local pf=i<nofparsed and parsed[nofparsed].finalizer + if pf then + return pf(collected) + end + return nil end - return nil end + return collected end - return collected -end -local function applylpath(list,pattern) - if not list then - return - end - local parsed=cache[pattern] - if parsed then - lpathcalls=lpathcalls+1 - lpathcached=lpathcached+1 - elseif type(pattern)=="table" then - lpathcalls=lpathcalls+1 - parsed=pattern - else - parsed=lpath(pattern) or pattern - end - if not parsed then - return + local apply=normal_apply + if trackers then + trackers.register("xml.path,xml.parse,xml.profile",function() + if trace_lprofile then + apply=profiled_apply + elseif trace_lpath then + apply=traced_apply + else + apply=normal_apply + end + end) end - local nofparsed=#parsed - if nofparsed==0 then - return + function xml.applylpath(list,pattern) + if not list then + lastmatch=nil + return + end + local parsed=cache[pattern] + if parsed then + lpathcalls=lpathcalls+1 + lpathcached=lpathcached+1 + elseif type(pattern)=="table" then + lpathcalls=lpathcalls+1 + parsed=pattern + else + parsed=lpath(pattern) or pattern + end + if not parsed then + lastmatch=nil + return + end + local nofparsed=#parsed + if nofparsed==0 then + lastmatch=nil + return + end + local collected=apply({ list },parsed,nofparsed,list.mi) + lastmatch=keepmatch and collected or nil + return collected end - if not trace_lpath then - return normal_apply ({ list },parsed,nofparsed,list.mi) - elseif trace_lprofile then - return profiled_apply({ list },parsed,nofparsed,list.mi) - else - return traced_apply ({ list },parsed,nofparsed,list.mi) + function xml.lastmatch() + return lastmatch end end -xml.applylpath=applylpath +local applylpath=xml.applylpath function xml.filter(root,pattern) return applylpath(root,pattern) end @@ -11676,7 +12449,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-mis"] = package.loaded["lxml-mis"] or true --- original size: 3684, stripped down to: 1957 +-- original size: 3787, stripped down to: 2003 if not modules then modules={} end modules ['lxml-mis']={ version=1.001, @@ -11745,7 +12518,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-aux"] = package.loaded["lxml-aux"] or true --- original size: 28786, stripped down to: 20578 +-- original size: 30566, stripped down to: 21741 if not modules then modules={} end modules ['lxml-aux']={ version=1.001, @@ -12079,55 +12852,63 @@ local function include(xmldata,pattern,attribute,recursive,loaddata,level) local ek=collected[c] local name=nil local ekdt=ek.dt - local ekat=ek.at - local ekrt=ek.__p__ - local epdt=ekrt.dt - if not attribute or attribute=="" then - name=(type(ekdt)=="table" and ekdt[1]) or ekdt - end - if not name then - for a in gmatch(attribute or "href","([^|]+)") do - name=ekat[a] - if name then - break + if ekdt then + local ekat=ek.at + local ekrt=ek.__p__ + if ekrt then + local epdt=ekrt.dt + if not attribute or attribute=="" then + name=(type(ekdt)=="table" and ekdt[1]) or ekdt end - end - end - local data=nil - if name and name~="" then - data=loaddata(name) or "" - if trace_inclusions then - report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") - end - end - if not data or data=="" then - epdt[ek.ni]="" - elseif ekat["parse"]=="text" then - epdt[ek.ni]=xml.escaped(data) - else - local xi=xmlinheritedconvert(data,xmldata) - if not xi then - epdt[ek.ni]="" - else - if recursive then - include(xi,pattern,attribute,recursive,loaddata,level+1) + if not name then + for a in gmatch(attribute or "href","([^|]+)") do + name=ekat[a] + if name then + break + end + end end - local child=xml.body(xi) - child.__p__=ekrt - child.__f__=name - epdt[ek.ni]=child - local inclusions=xmldata.settings.inclusions - if inclusions then - inclusions[#inclusions+1]=name - else - xmldata.settings.inclusions={ name } + local data=nil + if name and name~="" then + data=loaddata(name) or "" + if trace_inclusions then + report_xml("including %s bytes from %a at level %s by pattern %a and attribute %a (%srecursing)",#data,name,level,pattern,attribute or "",recursive and "" or "not ") + end end - if child.er then - local badinclusions=xmldata.settings.badinclusions - if badinclusions then - badinclusions[#badinclusions+1]=name + if not data or data=="" then + epdt[ek.ni]="" + elseif ekat["parse"]=="text" then + epdt[ek.ni]=xml.escaped(data) + else + local xi=xmlinheritedconvert(data,xmldata) + if not xi then + epdt[ek.ni]="" else - xmldata.settings.badinclusions={ name } + if recursive then + include(xi,pattern,attribute,recursive,loaddata,level+1) + end + local child=xml.body(xi) + child.__p__=ekrt + child.__f__=name + epdt[ek.ni]=child + local settings=xmldata.settings + local inclusions=settings and settings.inclusions + if inclusions then + inclusions[#inclusions+1]=name + elseif settings then + settings.inclusions={ name } + else + settings={ inclusions={ name } } + xmldata.settings=settings + end + if child.er then + local badinclusions=settings.badinclusions + if badinclusions then + badinclusions[#badinclusions+1]=name + else + settings.badinclusions={ name } + end + end end end end @@ -12598,7 +13379,7 @@ do -- create closure to overcome 200 locals limit package.loaded["lxml-xml"] = package.loaded["lxml-xml"] or true --- original size: 10274, stripped down to: 7538 +-- original size: 10719, stripped down to: 7841 if not modules then modules={} end modules ['lxml-xml']={ version=1.001, @@ -12976,7 +13757,7 @@ do -- create closure to overcome 200 locals limit package.loaded["trac-xml"] = package.loaded["trac-xml"] or true --- original size: 6351, stripped down to: 4919 +-- original size: 6534, stripped down to: 5072 if not modules then modules={} end modules ['trac-xml']={ version=1.001, @@ -13146,7 +13927,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-ini"] = package.loaded["data-ini"] or true --- original size: 11085, stripped down to: 7662 +-- original size: 11444, stripped down to: 7830 if not modules then modules={} end modules ['data-ini']={ version=1.001, @@ -13402,7 +14183,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-exp"] = package.loaded["data-exp"] or true --- original size: 17216, stripped down to: 10657 +-- original size: 18619, stripped down to: 11042 if not modules then modules={} end modules ['data-exp']={ version=1.001, @@ -13413,6 +14194,7 @@ if not modules then modules={} end modules ['data-exp']={ } local format,find,gmatch,lower,char,sub=string.format,string.find,string.gmatch,string.lower,string.char,string.sub local concat,sort=table.concat,table.sort +local sortedkeys=table.sortedkeys local lpegmatch,lpegpatterns=lpeg.match,lpeg.patterns local Ct,Cs,Cc,Carg,P,C,S=lpeg.Ct,lpeg.Cs,lpeg.Cc,lpeg.Carg,lpeg.P,lpeg.C,lpeg.S local type,next=type,next @@ -13758,14 +14540,16 @@ local nothing=function() end function resolvers.filtered_from_content(content,pattern) if content and type(pattern)=="string" then local pattern=lower(pattern) - local files=content.files + local files=content.files local remap=content.remap if files and remap then - local n=next(files) + local f=sortedkeys(files) + local n=#f + local i=0 local function iterator() - while n do - local k=n - n=next(files,k) + while i<n do + i=i+1 + local k=f[i] if find(k,pattern) then return files[k],remap and remap[k] or k end @@ -13784,7 +14568,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-env"] = package.loaded["data-env"] or true --- original size: 9216, stripped down to: 6798 +-- original size: 9649, stripped down to: 7131 if not modules then modules={} end modules ['data-env']={ version=1.001, @@ -13920,6 +14704,11 @@ local relations=allocate { names={ 'fontconfig','fontconfig file','fontconfig files' }, variable='FONTCONFIG_PATH', }, + pk={ + names={ "pk" }, + variable='PKFONTS', + suffixes={ 'pk' }, + }, }, obsolete={ enc={ @@ -14063,7 +14852,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmp"] = package.loaded["data-tmp"] or true --- original size: 15618, stripped down to: 11629 +-- original size: 16066, stripped down to: 11938 if not modules then modules={} end modules ['data-tmp']={ version=1.100, @@ -14439,7 +15228,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-met"] = package.loaded["data-met"] or true --- original size: 5347, stripped down to: 4015 +-- original size: 5488, stripped down to: 4101 if not modules then modules={} end modules ['data-met']={ version=1.100, @@ -14558,7 +15347,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-res"] = package.loaded["data-res"] or true --- original size: 67003, stripped down to: 46291 +-- original size: 67241, stripped down to: 46427 if not modules then modules={} end modules ['data-res']={ version=1.001, @@ -15828,10 +16617,18 @@ local function findfiles(filename,filetype,allresults) return result or {},status end function resolvers.findfiles(filename,filetype) - return findfiles(filename,filetype,true) + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,true) + end end function resolvers.findfile(filename,filetype) - return findfiles(filename,filetype,false)[1] or "" + if not filename or filename=="" then + return "" + else + return findfiles(filename,filetype,false)[1] or "" + end end function resolvers.findpath(filename,filetype) return filedirname(findfiles(filename,filetype,false)[1] or "") @@ -16106,7 +16903,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-pre"] = package.loaded["data-pre"] or true --- original size: 3950, stripped down to: 2935 +-- original size: 4236, stripped down to: 3144 if not modules then modules={} end modules ['data-pre']={ version=1.001, @@ -16170,16 +16967,20 @@ prefixes.pathname=function(str) return cleanpath(dirname((fullname~="" and fullname) or str)) end prefixes.selfautoloc=function(str) - return cleanpath(joinpath(getenv('SELFAUTOLOC'),str)) + local pth=getenv('SELFAUTOLOC') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautoparent=function(str) - return cleanpath(joinpath(getenv('SELFAUTOPARENT'),str)) + local pth=getenv('SELFAUTOPARENT') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.selfautodir=function(str) - return cleanpath(joinpath(getenv('SELFAUTODIR'),str)) + local pth=getenv('SELFAUTODIR') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.home=function(str) - return cleanpath(joinpath(getenv('HOME'),str)) + local pth=getenv('HOME') + return cleanpath(str and joinpath(pth,str) or pth) end prefixes.env=prefixes.environment prefixes.rel=prefixes.relative @@ -16224,7 +17025,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-inp"] = package.loaded["data-inp"] or true --- original size: 910, stripped down to: 823 +-- original size: 935, stripped down to: 838 if not modules then modules={} end modules ['data-inp']={ version=1.001, @@ -16254,7 +17055,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-out"] = package.loaded["data-out"] or true --- original size: 530, stripped down to: 475 +-- original size: 548, stripped down to: 483 if not modules then modules={} end modules ['data-out']={ version=1.001, @@ -16277,7 +17078,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-fil"] = package.loaded["data-fil"] or true --- original size: 3863, stripped down to: 3310 +-- original size: 3976, stripped down to: 3391 if not modules then modules={} end modules ['data-fil']={ version=1.001, @@ -16385,7 +17186,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-con"] = package.loaded["data-con"] or true --- original size: 5010, stripped down to: 3588 +-- original size: 5148, stripped down to: 3680 if not modules then modules={} end modules ['data-con']={ version=1.100, @@ -16504,7 +17305,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-use"] = package.loaded["data-use"] or true --- original size: 3899, stripped down to: 2984 +-- original size: 4000, stripped down to: 3052 if not modules then modules={} end modules ['data-use']={ version=1.001, @@ -16595,7 +17396,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-zip"] = package.loaded["data-zip"] or true --- original size: 8772, stripped down to: 6841 +-- original size: 9036, stripped down to: 7041 if not modules then modules={} end modules ['data-zip']={ version=1.001, @@ -16832,7 +17633,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tre"] = package.loaded["data-tre"] or true --- original size: 8479, stripped down to: 5580 +-- original size: 8712, stripped down to: 5726 if not modules then modules={} end modules ['data-tre']={ version=1.001, @@ -17021,7 +17822,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-sch"] = package.loaded["data-sch"] or true --- original size: 6569, stripped down to: 5304 +-- original size: 6779, stripped down to: 5444 if not modules then modules={} end modules ['data-sch']={ version=1.001, @@ -17202,7 +18003,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lua"] = package.loaded["data-lua"] or true --- original size: 4313, stripped down to: 3227 +-- original size: 4447, stripped down to: 3302 if not modules then modules={} end modules ['data-lua']={ version=1.001, @@ -17311,7 +18112,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-aux"] = package.loaded["data-aux"] or true --- original size: 2431, stripped down to: 1996 +-- original size: 2494, stripped down to: 2047 if not modules then modules={} end modules ['data-aux']={ version=1.001, @@ -17378,7 +18179,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-tmf"] = package.loaded["data-tmf"] or true --- original size: 2601, stripped down to: 1627 +-- original size: 2674, stripped down to: 1658 if not modules then modules={} end modules ['data-tmf']={ version=1.001, @@ -17434,7 +18235,7 @@ do -- create closure to overcome 200 locals limit package.loaded["data-lst"] = package.loaded["data-lst"] or true --- original size: 2734, stripped down to: 2354 +-- original size: 2815, stripped down to: 2415 if not modules then modules={} end modules ['data-lst']={ version=1.001, @@ -17514,7 +18315,7 @@ do -- create closure to overcome 200 locals limit package.loaded["util-lib"] = package.loaded["util-lib"] or true --- original size: 11549, stripped down to: 5905 +-- original size: 11846, stripped down to: 6059 if not modules then modules={} end modules ['util-lib']={ version=1.001, @@ -17700,7 +18501,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-sta"] = package.loaded["luat-sta"] or true --- original size: 5703, stripped down to: 2507 +-- original size: 5914, stripped down to: 2584 if not modules then modules={} end modules ['luat-sta']={ version=1.001, @@ -17803,7 +18604,7 @@ do -- create closure to overcome 200 locals limit package.loaded["luat-fmt"] = package.loaded["luat-fmt"] or true --- original size: 5955, stripped down to: 4926 +-- original size: 6967, stripped down to: 5631 if not modules then modules={} end modules ['luat-fmt']={ version=1.001, @@ -17832,7 +18633,7 @@ local function primaryflags() end return concat(flags," ") end -function environment.make_format(name) +function environment.make_format(name,silent) local engine=environment.ownmain or "luatex" local olddir=dir.current() local path=caches.getwritablepath("formats",engine) or "" @@ -17889,9 +18690,23 @@ function environment.make_format(name) lfs.chdir(olddir) return end - local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),os.platform=="unix" and "\\\\" or "\\") - report_format("running command: %s\n",command) - os.execute(command) + local dump=os.platform=="unix" and "\\\\dump" or "\\dump" + if silent then + statistics.starttiming() + local command=format("%s --ini --interaction=batchmode %s --lua=%s %s %s > temp.log",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + local result=os.execute(command) + local runtime=statistics.stoptiming() + if result~=0 then + print(format("%s silent make > fatal error when making format %q",engine,name)) + else + print(format("%s silent make > format %q made in %.3f seconds",engine,name,runtime)) + end + os.remove("temp.log") + else + local command=format("%s --ini %s --lua=%s %s %sdump",engine,primaryflags(),quoted(usedluastub),quoted(fulltexsourcename),dump) + report_format("running command: %s\n",command) + os.execute(command) + end local pattern=file.removesuffix(file.basename(usedluastub)).."-*.mem" local mp=dir.glob(pattern) if mp then @@ -17935,10 +18750,10 @@ end end -- of closure --- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua +-- used libraries : l-lua.lua l-package.lua l-lpeg.lua l-function.lua l-string.lua l-table.lua l-io.lua l-number.lua l-set.lua l-os.lua l-file.lua l-gzip.lua l-md5.lua l-url.lua l-dir.lua l-boolean.lua l-unicode.lua l-math.lua util-str.lua util-tab.lua util-fil.lua util-sac.lua util-sto.lua util-prs.lua util-fmt.lua trac-set.lua trac-log.lua trac-inf.lua trac-pro.lua util-lua.lua util-deb.lua util-mrg.lua util-tpl.lua util-env.lua luat-env.lua lxml-tab.lua lxml-lpt.lua lxml-mis.lua lxml-aux.lua lxml-xml.lua trac-xml.lua data-ini.lua data-exp.lua data-env.lua data-tmp.lua data-met.lua data-res.lua data-pre.lua data-inp.lua data-out.lua data-fil.lua data-con.lua data-use.lua data-zip.lua data-tre.lua data-sch.lua data-lua.lua data-aux.lua data-tmf.lua data-lst.lua util-lib.lua luat-sta.lua luat-fmt.lua -- skipped libraries : - --- original bytes : 745793 --- stripped bytes : 269308 +-- original bytes : 797557 +-- stripped bytes : 289197 -- end library merge @@ -17982,6 +18797,8 @@ local ownlibs = { -- order can be made better 'util-str.lua', -- code might move to l-string 'util-tab.lua', + 'util-fil.lua', + 'util-sac.lua', 'util-sto.lua', 'util-prs.lua', 'util-fmt.lua', @@ -18037,13 +18854,21 @@ local ownlibs = { -- order can be made better } --- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/data-tmf.lua +-- c:/data/develop/tex-context/tex/texmf-win64/bin/../../texmf-context/tex/context/base/mkiv/data-tmf.lua -- c:/data/develop/context/sources/data-tmf.lua local ownlist = { -- '.', -- ownpath , owntree .. "/../../../../context/sources", -- HH's development path + -- + owntree .. "/../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../texmf/tex/context/base/mkiv", + owntree .. "/../../../texmf-local/tex/context/base/mkiv", + owntree .. "/../../../texmf-context/tex/context/base/mkiv", + owntree .. "/../../../texmf/tex/context/base/mkiv", + -- owntree .. "/../../texmf-local/tex/context/base", owntree .. "/../../texmf-context/tex/context/base", owntree .. "/../../texmf/tex/context/base", |