diff options
Diffstat (limited to 'scripts')
| -rw-r--r-- | scripts/context/lua/luatools.lua | 152 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-context.lua | 64 | ||||
| -rw-r--r-- | scripts/context/lua/mtx-update.lua | 3 | ||||
| -rw-r--r-- | scripts/context/lua/mtxrun.lua | 4266 | ||||
| -rw-r--r-- | scripts/context/stubs/mswin/luatools.lua | 152 | ||||
| -rw-r--r-- | scripts/context/stubs/mswin/mtxrun.lua | 4266 | ||||
| -rwxr-xr-x | scripts/context/stubs/unix/luatools | 152 | ||||
| -rwxr-xr-x | scripts/context/stubs/unix/mtxrun | 4266 |
8 files changed, 7647 insertions, 5674 deletions
diff --git a/scripts/context/lua/luatools.lua b/scripts/context/lua/luatools.lua index a8cfbd5b0..2bc943210 100644 --- a/scripts/context/lua/luatools.lua +++ b/scripts/context/lua/luatools.lua @@ -230,6 +230,16 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -279,6 +289,12 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + end -- of closure @@ -387,6 +403,18 @@ function string:split(separator) return c:match(self) end +--~ function lpeg.L(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + lpeg.P(list[l]) +--~ else +--~ p = lpeg.P(list[l]) +--~ end +--~ end +--~ return p +--~ end + end -- of closure @@ -420,6 +448,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key,_ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -1192,21 +1228,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1413,7 +1463,7 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } -local format = string.format +local format, foor, insert = string.format, math.floor, table.insert number = number or { } @@ -1449,7 +1499,18 @@ function number.toset(n) return one:match(tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1914,11 +1975,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return qualified:match(filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return rootbased:match(filename) ~= nil end local slash = lpeg.S("\\/") @@ -3134,6 +3195,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3156,7 +3235,7 @@ debugger = debugger or { } local counters = { } local names = { } local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub -- one @@ -3290,7 +3369,7 @@ local data, done = { }, { } local function set(what,value) if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_array(what) -- inefficient but ok end for i=1,#what do local w = what[i] @@ -3315,6 +3394,19 @@ local function reset() end end +local function enable(what) + set(what,true) +end + +local function disable(what) + if not what or what == "" then + done = { } + reset() + else + set(what,false) + end +end + function trackers.register(what,...) what = lower(what) local w = data[what] @@ -3333,20 +3425,20 @@ function trackers.register(what,...) end function trackers.enable(what) - done = { } - set(what,true) + local e = trackers.enable + trackers.enable, done = enable, { } + enable(string.simpleesc(what)) + trackers.enable, done = e, { } end function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end + local e = trackers.disable + trackers.disable, done = disable, { } + disable(string.simpleesc(what)) + trackers.disable, done = e, { } end -function trackers.reset(what) +function trackers.reset() done = { } reset() end @@ -3423,7 +3515,7 @@ function environment.initialize_arguments(arg) environment.arguments, environment.files, environment.sortedflags = arguments, files, nil for index, argument in pairs(arg) do if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = argument:match("^%-+(.-)=(.-)$") if flag then arguments[flag] = string.unquote(value or "") else diff --git a/scripts/context/lua/mtx-context.lua b/scripts/context/lua/mtx-context.lua index 418387fce..cf26c4e51 100644 --- a/scripts/context/lua/mtx-context.lua +++ b/scripts/context/lua/mtx-context.lua @@ -131,7 +131,6 @@ do if f then f:write("<?xml version='1.0' standalone='yes'?>\n\n") f:write(string.format("<ctx:preplist local='%s'>\n",yn(ctxdata.runlocal))) ---~ for name, value in pairs(ctxdata.prepfiles) do for _, name in ipairs(table.sortedkeys(ctxdata.prepfiles)) do f:write(string.format("\t<ctx:prepfile done='%s'>%s</ctx:prepfile>\n",yn(ctxdata.prepfiles[name]),name)) end @@ -189,8 +188,8 @@ do end end -usedname = resolvers.find_file(ctxdata.ctxname,"tex") -found = usedname ~= "" + usedname = resolvers.find_file(ctxdata.ctxname,"tex") + found = usedname ~= "" if not found and defaultname and defaultname ~= "" and lfs.isfile(defaultname) then usedname, found = defaultname, true @@ -225,36 +224,35 @@ found = usedname ~= "" logs.simple("ctx comment: %s", xml.tostring(message)) end - xml.each(ctxdata.xmldata,"ctx:value[@name='job']", function(ek,e,k) + for r, e, k in xml.elements(ctxdata.xmldata,"ctx:value[@name='job']") do e[k] = ctxdata.variables['job'] or "" - end) + end local commands = { } - xml.each(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/ctx:processor", function(r,d,k) - local ek = d[k] - commands[ek.at and ek.at['name'] or "unknown"] = ek - end) + for e in xml.collected(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/ctx:processor") do + commands[e.at and e.at['name'] or "unknown"] = e + end local suffix = xml.filter(ctxdata.xmldata,"/ctx:job/ctx:preprocess/attribute(suffix)") or ctxdata.suffix local runlocal = xml.filter(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:processors/attribute(local)") runlocal = toboolean(runlocal) - for _, files in ipairs(xml.filters.elements(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:files")) do - for _, pattern in ipairs(xml.filters.elements(files,"ctx:file")) do + for files in xml.collected(ctxdata.xmldata,"/ctx:job/ctx:preprocess/ctx:files") do + for pattern in xml.collected(files,"ctx:file") do preprocessor = pattern.at['processor'] or "" if preprocessor ~= "" then ctxdata.variables['old'] = ctxdata.jobname - xml.each(ctxdata.xmldata,"ctx:value", function(r,d,k) + for r, d, k in xml.elements(ctxdata.xmldata,"ctx:value") do local ek = d[k] local ekat = ek.at['name'] if ekat == 'old' then d[k] = ctxrunner.substitute(ctxdata.variables[ekat] or "") end - end) + end pattern = ctxrunner.justtext(xml.tostring(pattern)) @@ -293,21 +291,21 @@ found = usedname ~= "" if ctxdata.runlocal then newfile = file.basename(newfile) end - xml.each(command,"ctx:old", function(r,d,k) + for r, d, k in xml.elements(command,"ctx:old") do d[k] = ctxrunner.substitute(oldfile) - end) - xml.each(command,"ctx:new", function(r,d,k) + end + for r, d, k in xml.elements(command,"ctx:new") do d[k] = ctxrunner.substitute(newfile) - end) + end ctxdata.variables['old'] = oldfile ctxdata.variables['new'] = newfile - xml.each(command,"ctx:value", function(r,d,k) + for r, d, k in xml.elements(command,"ctx:value") do local ek = d[k] local ekat = ek.at and ek.at['name'] if ekat then d[k] = ctxrunner.substitute(ctxdata.variables[ekat] or "") end - end) + end -- potential optimization: when mtxrun run internal command = xml.text(command) command = ctxrunner.justtext(command) -- command is still xml element here @@ -444,6 +442,12 @@ function scripts.context.multipass.makeoptionfile(jobname,ctxdata,kindofrun,curr if type(environment.argument("track")) == "string" then setvalue ("track" , "\\enabletrackers[%s]") end + if type(environment.argument("trackers")) == "string" then + setvalue ("trackers" , "\\enabletrackers[%s]") + end + if type(environment.argument("directives")) == "string" then + setvalue ("directives", "\\enabledirectives[%s]") + end setfixed ("timing" , "\\usemodule[timing]") setfixed ("batchmode" , "\\batchmode") setfixed ("nonstopmode" , "\\nonstopmode") @@ -1211,8 +1215,15 @@ end -- todo: we need to do a dummy run -function scripts.context.track() - environment.files = { "m-track" } +function scripts.context.trackers() + environment.files = { "m-trackers" } + scripts.context.multipass.nofruns = 1 + scripts.context.run() + -- maybe filter from log +end + +function scripts.context.directives() + environment.files = { "m-directives" } scripts.context.multipass.nofruns = 1 scripts.context.run() -- maybe filter from log @@ -1403,7 +1414,8 @@ expert options: --nostats omit runtime statistics at the end of the run --update update context from website (not to be confused with contextgarden) --profile profile job (use: mtxrun --script profile --analyse) ---track show/set tracker variables +--trackers show/set tracker variables +--directives show/set directive variables --timing generate timing and statistics overview --extra=name process extra (mtx-context-<name> in distribution) --tracefiles show some extra info when locating files (at the tex end) @@ -1462,8 +1474,12 @@ elseif environment.argument("extra") then scripts.context.extra() elseif environment.argument("help") then logs.help(messages.help) -elseif environment.argument("track") and type(environment.argument("track")) == "boolean" then - scripts.context.track() +elseif environment.argument("trackers") and type(environment.argument("trackers")) == "boolean" then + scripts.context.trackers() +elseif environment.argument("directives") and type(environment.argument("directives")) == "boolean" then + scripts.context.directives() +elseif environment.argument("track") and type(environment.argument("track")) == "boolean" then -- for old times sake, will go + scripts.context.trackers() elseif environment.files[1] then -- scripts.context.timed(scripts.context.run) scripts.context.timed(scripts.context.autoctx) diff --git a/scripts/context/lua/mtx-update.lua b/scripts/context/lua/mtx-update.lua index ef05f087d..1d2e0672a 100644 --- a/scripts/context/lua/mtx-update.lua +++ b/scripts/context/lua/mtx-update.lua @@ -69,7 +69,6 @@ scripts.update.base = { { "context/img/", "texmf-context" }, { "misc/setuptex/", "." }, { "misc/web2c", "texmf" }, - { "bin/common/luatex/", "texmf-<platform>" }, { "bin/common/<platform>/", "texmf-<platform>" }, { "bin/context/<platform>/", "texmf-<platform>" }, { "bin/metapost/<platform>/", "texmf-<platform>" }, @@ -87,10 +86,12 @@ scripts.update.engines = { ["xetex"] = { { "base/xetex/", "texmf" }, { "fonts/new/", "texmf" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, -- tools { "bin/xetex/<platform>/", "texmf-<platform>" }, }, ["pdftex"] = { { "fonts/old/", "texmf" }, + { "bin/luatex/<platform>/", "texmf-<platform>" }, -- tools { "bin/pdftex/<platform>/", "texmf-<platform>" }, }, ["all"] = { diff --git a/scripts/context/lua/mtxrun.lua b/scripts/context/lua/mtxrun.lua index 865994073..8bc88c900 100644 --- a/scripts/context/lua/mtxrun.lua +++ b/scripts/context/lua/mtxrun.lua @@ -239,6 +239,16 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -288,6 +298,12 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + end -- of closure @@ -396,6 +412,18 @@ function string:split(separator) return c:match(self) end +--~ function lpeg.L(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + lpeg.P(list[l]) +--~ else +--~ p = lpeg.P(list[l]) +--~ end +--~ end +--~ return p +--~ end + end -- of closure @@ -429,6 +457,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key,_ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -1009,7 +1045,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1038,6 +1074,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1201,21 +1255,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1422,7 +1490,7 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } -local format = string.format +local format, foor, insert = string.format, math.floor, table.insert number = number or { } @@ -1458,7 +1526,18 @@ function number.toset(n) return one:match(tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1923,11 +2002,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return qualified:match(filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return rootbased:match(filename) ~= nil end local slash = lpeg.S("\\/") @@ -2854,6 +2933,406 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in pairs(counters) do + if count > threshold then + local name = getname(func) + if not name:find("for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in pairs(counters) do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_array(what) -- inefficient but ok + end + for i=1,#what do + local w = what[i] + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](value) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(what)) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(what)) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + for k,v in ipairs(setters.list(t)) do + commands.writestatus(t.name,v) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singluar tracker + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +--~ -- old code: +-- +--~ trackers = trackers or { } +--~ local data, done = { }, { } +--~ local function set(what,value) +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end +--~ local function reset() +--~ for d, f in next, data do +--~ for i=1,#f do +--~ f[i](false) +--~ end +--~ end +--~ end +--~ local function enable(what) +--~ set(what,true) +--~ end +--~ local function disable(what) +--~ if not what or what == "" then +--~ done = { } +--~ reset() +--~ else +--~ set(what,false) +--~ end +--~ end +--~ function trackers.register(what,...) +--~ what = lower(what) +--~ local w = data[what] +--~ if not w then +--~ w = { } +--~ data[what] = w +--~ end +--~ for _, fnc in next, { ... } do +--~ local typ = type(fnc) +--~ if typ == "function" then +--~ w[#w+1] = fnc +--~ elseif typ == "string" then +--~ w[#w+1] = function(value) set(fnc,value,nesting) end +--~ end +--~ end +--~ end +--~ function trackers.enable(what) +--~ local e = trackers.enable +--~ trackers.enable, done = enable, { } +--~ enable(string.simpleesc(what)) +--~ trackers.enable, done = e, { } +--~ end +--~ function trackers.disable(what) +--~ local e = trackers.disable +--~ trackers.disable, done = disable, { } +--~ disable(string.simpleesc(what)) +--~ trackers.disable, done = e, { } +--~ end +--~ function trackers.reset() +--~ done = { } +--~ reset() +--~ end +--~ function trackers.list() -- pattern +--~ local list = table.sortedkeys(data) +--~ local user, system = { }, { } +--~ for l=1,#list do +--~ local what = list[l] +--~ if find(what,"^%*") then +--~ system[#system+1] = what +--~ else +--~ user[#user+1] = what +--~ end +--~ end +--~ return user, system +--~ end + end -- of closure @@ -2867,6 +3346,12 @@ if not modules then modules = { } end modules ['lxml-tab'] = { license = "see context related readme files" } +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble + +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -2874,18 +3359,6 @@ parent access; a first version used different trickery but was less optimized to went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> @@ -2896,26 +3369,9 @@ xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable = type, next, setmetatable +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber local format, lower, find = string.format, string.lower, string.find - ---[[ldx-- -<p>This module can be used stand alone but also inside <l n='mkiv'/> in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers.</p> ---ldx]]-- - -local trace_remap = false - -if trackers then - trackers.register("xml.remap", function(v) trace_remap = v end) -end - -function xml.settrace(str,value) - if str == "remap" then - trace_remap = value or false - end -end +local utfchar = unicode.utf8.char --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -3022,25 +3478,25 @@ element.</p> </typing> --ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes - -- not just one big nested table capture (lpeg overflow) local nsremap, resolvens = xml.xmlns, xml.resolvens local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local strip, cleanup, utfize, resolve = false, false, false, false -local mt = { __tostring = xml.text } +local mt = { } -function xml.check_error(top,toclose) - return "" +function initialize_mt(root) -- we will make a xml.new that then sets the mt as field + mt = { __tostring = xml.text, __index = root } end -local strip = false -local cleanup = false +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end -function xml.set_text_cleanup(fnc) - cleanup = fnc +function xml.check_error(top,toclose) + return "" end local function add_attribute(namespace,tag,value) @@ -3058,6 +3514,22 @@ local function add_attribute(namespace,tag,value) end end +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) + end + at = { } +end + local function add_begin(spacing, namespace, tag) if #spacing > 0 then dt[#dt+1] = spacing @@ -3083,28 +3555,12 @@ local function add_end(spacing, namespace, tag) end dt = top.dt dt[#dt+1] = toclose - dt[0] = top + -- dt[0] = top -- nasty circular reference when serializing table if toclose.at.xmlns then remove(xmlns) end end -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - local function add_text(text) if cleanup and #text > 0 then dt[#dt+1] = cleanup(text) @@ -3128,7 +3584,109 @@ local function set_message(txt) errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") end -local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end + +local dcache, hcache, acache = { }, { }, { } + +function xml.unknown_dec_entity_format(str) return format("&%s;", str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&%s;", str) end + +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + if utfize then + local n = tonumber(str,16) + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,c) + end + else + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#" .. str .. ";" + end + hcache[str] = h + end + return h +end +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + if utfize then + local n = tonumber(str) + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,c) + end + else + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&" .. str .. ";" + end + dcache[str] = d + end + return d +end +local function handle_any_entity(str) + if resolve then + local a = entities[str] -- per instance ! + if not a then + a = acache[str] + if not a then + if trace_entities then + logs.report("xml","ignoring entity &%s;",str) + else + -- can be defined in a global mapper and intercepted elsewhere + -- as happens in lxml-tex.lua + end + a = xml.unknown_any_entity_format(str) or "" + acache[str] = a + end + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,r) + acache[str] = a + end + end + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = "&" .. str .. ";" + acache[str] = a + end + return a + end +end + +local P, S, R, C, V, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cs local space = S(' \r\n\t') local open = P('<') @@ -3138,6 +3696,8 @@ local dquote = S('"') local equal = P('=') local slash = P('/') local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') local valid = R('az', 'AZ', '09') + S('_-.') local name_yes = C(valid^1) * colon * C(valid^1) local name_nop = C(P(true)) * C(valid^1) @@ -3147,15 +3707,36 @@ local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture local spacing = C(space^0) -local justtext = C((1-open)^1) + +local entitycontent = (1-open-semicolon)^0 +local entity = ampersand/"" * ( + P("#")/"" * ( + P("x")/"" * (entitycontent/handle_hex_entity) + + (entitycontent/handle_dec_entity) + ) + (entitycontent/handle_any_entity) + ) * (semicolon/"") + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) + local somespace = space^1 local optionalspace = space^0 -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute -local attributes = attribute^0 +local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value + +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error + +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local endofattributes = slash * close + close -- recovery of flacky html +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 -local text = justtext / add_text +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty @@ -3208,42 +3789,72 @@ local doctype = (spacing * begindoctype * somedoctype * enddoct -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special -local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 -local grammar = P { "preamble", +local grammar_parsed_text = P { "preamble", preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- todo: xml.new + properties like entities and strip and such (store in root) +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} -function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, settings.entities or {} + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end stack[#stack+1] = top top.dt = { } dt = top.dt if not data or data == "" then errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" + elseif utfize or resolve then + if grammar_parsed_text:match(data) then + errorstr = "" + else + errorstr = "invalid xml file - parsed text" + end else - errorstr = "" + if grammar_unparsed_text:match(data) then + errorstr = "" + else + errorstr = "invalid xml file - unparsed text" + end end if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } } } setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + local error_handler = settings.error_handler + if error_handler == false then + -- no error message + else + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) + end + end else result = stack[1] end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities, settings = settings } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do @@ -3254,9 +3865,14 @@ function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe us end end end + if errorstr and errorstr ~= "" then + result.error = true + end return result end +xml.convert = xmlconvert + --[[ldx-- <p>Packaging data in an xml like table is done with the following function. Maybe it will go away (when not used).</p> @@ -3289,16 +3905,16 @@ function xml.load(filename) if type(filename) == "string" then local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + local root = xmlconvert(f:read("*all")) f:close() return root else - return xml.convert("") + return xmlconvert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) + return xmlconvert(filename:read("*all")) else - return xml.convert("") + return xmlconvert("") end end @@ -3307,9 +3923,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -3354,217 +3972,305 @@ alternative.</p> -- todo: add <?xml version='1.0' standalone='yes'?> when not present -local fallbackhandle = (tex and tex.sprint) or io.write - -local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) +end + +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) else - -- no need to handle any further + serialize(e,handlers) end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) end + handle("</",ens,":",etg,">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster if ats then - if attributeconverter then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) - end - end + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") end - if ern and trace_remap and ern ~= ens then - ens = ern + end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end + serialize(ei,handlers) end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + handle("<",etg,"/>") end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - handle(e) + serialize(ei,handlers) end - else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + end +end + +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state end end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end -xml.serialize = serialize +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then - found = true - break +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv + end + else + t[k] = v end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) + if settings.name then + handlers[settings.name] = t end end + return t +end + +local nofunction = function() end + +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction end +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + --[[ldx-- -<p>At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.</p> +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> --ldx]]-- -function xml.tostring(root) -- 25% overhead due to collecting +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end return "" end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" +end + +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring +xml.text = xmltext + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -3572,33 +4278,16 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> - -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> ---ldx]]-- - -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() - end -end +xml.string = xmlstring --[[ldx-- <p>A few helpers:</p> --ldx]]-- +function xml.parent(root) + return root.__p__ +end + function xml.body(root) return (root.ri and root.dt[root.ri]) or root end @@ -3611,34 +4300,19 @@ function xml.content(root) -- bugged return (root and root.dt and xml.tostring(root.dt)) or "" end -function xml.isempty(root, pattern) - if pattern == "" or pattern == "*" then - pattern = nil - end - if pattern then - -- todo - return false - else - return not root or not root.dt or #root.dt == 0 or root.dt == "" - end -end - --[[ldx-- <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -3672,96 +4346,403 @@ if not modules then modules = { } end modules ['lxml-pth'] = { license = "see context related readme files" } +-- e.ni is only valid after a filter run + local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, lower, gmatch, gsub, find, rep = string.format, string.lower, string.gmatch, string.gsub, string.find, string.rep +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep --[[ldx-- <p>This module can be used stand alone but also inside <l n='mkiv'/> in which case it hooks into the tracker code. Therefore we provide a few functions that set the tracers. Here we overload a previously defined function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> --ldx]]-- -local trace_lpath = false - -if trackers then - trackers.register("xml.lpath", function(v) trace_lpath = v end) -end +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> -local settrace = xml.settrace -- lxml-tab +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- -function xml.settrace(str,value) - if str == "lpath" then - trace_lpath = value or false - else - settrace(str,value) -- lxml-tab - end -end +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statistics -local lpathcached = 0 -- statistics +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -xml.expressions = xml.expressions or { } +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } local functions = xml.functions local expressions = xml.expressions +local finalizers = xml.finalizers -local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", -} +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } + +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end + end + return fn +end + +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) + +xml.defaultprotocol = "xml" + +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above --- a rather dumb lpeg +-- evaluator (needs to be redone, for the moment copied) -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- todo: apply_axis(list,notable) and collection vs single --- instead of using functions we just parse a few names which saves a call --- later on +local apply_axis = { } -local lp_position = P("position()") / "ps" -local lp_index = P("index()") / "id" -local lp_text = P("text()") / "tx" -local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" -local lp_tag = P("tag()") / "tg" -- (rt.tg or '') -local lp_ns = P("ns()") / "ns" -- (rt.ns or '') -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll + end + end + collected[#collected+1] = rt + end + return collected +end + +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end -local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local dt = list[l].dt + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + end + end + end + return collected +end + +local function collect(list,collected) + local dt = list.dt + if dt then + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + collect(dk,collected) + end + end + end +end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end + +local function collect(list,collected) + local dt = list.dt + if dt then + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + collect(dk,collected) + end + end + end +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] +if ll.special ~= true then -- catch double root + collected[#collected+1] = ll +end + collect(ll,collected) + end + return collected +end + +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end + +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end + +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl + end + end + return collected +end + +apply_axis['attribute'] = function(list) + return { } +end + +apply_axis['following'] = function(list) + return { } +end + +apply_axis['following-sibling'] = function(list) + return { } +end + +apply_axis['namespace'] = function(list) + return { } +end + +apply_axis['preceding'] = function(list) + return { } +end + +apply_axis['preceding-sibling'] = function(list) + return { } +end + +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] + +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } + end + else + local collected = { } + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + collected[#collected+1] = ll + end + elseif ntg ~= ltg then + collected[#collected+1] = ll + end + end + end + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + collected[#collected+1] = ll + end + elseif lns ~= nns then + collected[#collected+1] = ll + end + end + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + collected[#collected+1] = ll + end + elseif not ok then + collected[#collected+1] = ll + end + end + end + end + return collected + end + else + local collected = { } + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + collected[#collected+1] = ll + end + elseif not ok then + collected[#collected+1] = ll + end + end + end + return collected + end +end + +local function apply_expression(list,expression,order) + local collected = { } + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, alleen valid als n=1 + collected[#collected+1] = ll + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 + +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) + +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("first") / "1" + + P("last") / "#list" + + P("position") / "l" + + P("rootposition") / "order" + + P("index") / "ll.ni" + + P("text") / "(ll.dt[1] or '')" + + P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("ll.at['") * R("az","AZ","--","__")^1 * Cc("']") +local lp_fastpos = ((R("09","--","++")^1 * P(-1)) / function(s) return "l==" .. s end) + +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling return t .. "(" end -local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling if expressions[t] then - return "expressions." .. t .. "(" + return "expr." .. t .. "(" else - return "expressions.error(" + return "expr.error(" end end @@ -3771,337 +4752,527 @@ local noparent = 1 - (lparent+rparent) local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} --- if we use a dedicated namespace then we don't need to pass rt and k +local lp_child = Cc("expr.child(e,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content= (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) + +local cleaner -local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" + s = s and s ~= "" and cleaner:match(s) + if s and s ~= "" then + return "expr." .. t .. "(e," .. s ..")" else - return "expressions." .. t .. "(r,k)" + return "expr." .. t .. "(e)" end else - return "expressions.error(" .. t .. ")" + return "expr.error(" .. t .. ")" end end -local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one +local content = + lp_builtin + + lp_attribute + lp_special + lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = lpeg.Cs ( + lp_fastpos + (lpeg.P { lparent * (lpeg.V(1))^0 * rparent + content } )^0 +) + +cleaner = lpeg.Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_string + 1 )^1 ) --- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 +--~ expr -local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,root) return %s end ]] -local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() -end - -local map = { } - -local space = S(' \r\n\t') -local squote = S("'") -local dquote = S('"') -local lparent = P('(') -local rparent = P(')') -local atsign = P('@') -local lbracket = P('[') -local rbracket = P(']') -local exclam = P('!') -local period = P('.') -local eq = P('==') + P('=') -local ne = P('<>') + P('!=') -local star = P('*') -local slash = P('/') -local colon = P(':') -local bar = P('|') -local hat = P('^') -local valid = R('az', 'AZ', '09') + S('_-') -local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* -local name_nop = Cc("*") * C(valid^1) -local name = name_yes + name_nop -local number = C((S('+-')^0 * R('09')^1)) / tonumber -local names = (bar^0 * name)^1 -local morenames = name * (bar^0 * name)^1 -local instructiontag = P('pi::') -local spacing = C(space^0) -local somespace = space^1 -local optionalspace = space^0 -local text = C(valid^0) -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local empty = 1-slash - -local is_eq = lbracket * atsign * name * eq * value * rbracket -local is_ne = lbracket * atsign * name * ne * value * rbracket -local is_attribute = lbracket * atsign * name * rbracket -local is_value = lbracket * value * rbracket -local is_number = lbracket * number * rbracket - -local nobracket = 1-(lbracket+rbracket) -- must be improved -local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - -local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - -local is_one = name -local is_none = exclam * name -local is_one_of = ((lparent * names * rparent) + morenames) -local is_none_of = exclam * ((lparent * names * rparent) + morenames) - -local stay = (period ) -local parent = (period * period ) / function( ) map[#map+1] = { 11 } end -local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end -local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end -local any = (star ) / function( ) map[#map+1] = { 14 } end -local many = (star * star ) / function( ) map[#map+1] = { 15 } end -local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - -local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end -local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end -local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end -local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - -local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end -local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end -local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end -local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - -local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end -local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end -local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end -local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - -local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end -local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end -local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end -local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end -local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end -local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - -local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end -local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - -local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end -local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - -local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end -local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? -local crap = (1-slash)^1 - --- a few ugly goodies: - -local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end -local subroottag = P('^') / function( ) map[#map+1] = { 13 } end -local roottag = P('root::') / function( ) map[#map+1] = { 12 } end -local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end -local childtag = P('child::') -local selftag = P('self::') - --- there will be more and order will be optimized - -local selector = ( - instruction + --- many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty -) +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] -local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, -} +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] -local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false +-- + +local function errorrunner_e(str,cnv) + logs.report("lpath","error in expression: %s => %s",str,cnv) + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false +end + +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } +end + +local function register_expression(expression) + local converted = converter:match(expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) - end - -- print(gsub(table.serialize(map),"[ \n]+"," ")) - return map - end + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } end +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} + +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} + +-- todo: better arg parser + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local function register_error(str) + return { kind = "error", comment = format("unparsed: %s",str) } +end + +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside + + patterns = spaces * V("protocol") * spaces * V("initial") * spaces * V("step") * spaces * + (P("/") * spaces * V("step") * spaces)^0, + + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), + + step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following") + V("following_sibling") + + V("preceding") + V("preceding_sibling") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), + + initial = (P("/") * spaces * Cc(register_initial_child))^-1, + + error = (P(1)^1) / register_error, + + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), + + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, + + s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), + + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, + + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + local cache = { } -function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 - end - if trace or trace_lpath then - xml.lshow(result) - end - return result +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) else - return pattern + return concat(t,"|") end end -function xml.cached_patterns() - return cache +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end end --- we run out of locals (limited to 200) --- --- local fallbackreport = (texio and texio.write) or io.write - -function xml.lshow(pattern,report) --- report = report or fallbackreport - report = report or (texio and texio.write) or io.write - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") +xml.nodesettostring = nodesettostring + +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) + end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly +end + +xml.lshow = lshow + +local function parse_pattern(pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k=1,#lp do - local v = lp[k] - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = parser:match(pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) + else + -- we could have done this with a more complex parsed but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + parsed.comment = "auto-child replaced by auto-descendant-or-self" + parsed[1] = register_auto_descendant_or_self + --~ parsed.comment = "auto-child replaced by auto-descendant" + --~ parsed[1] = register_auto_descendant + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + parsed.comment = "initial-child removed" -- we could also make it a auto-self + remove(parsed,1) end end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end end + return parsed end end -function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } --- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up + +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 else - for i=1,#e do - report(tostring(e[i]) .. "\n") + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + return nil end end + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + logs.report("lpath", "% 10i : ex : %s",(collected and #collected) or 0,pi.expression) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(collected and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + return nil + end + end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> - -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + -- something is wrong + elseif not trace_lpath then + -- normal apply, inline, no self + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + elseif kind == "finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected == 0 then + return nil + end + end + return collected + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed) + else -- trace_lpath + return traced_apply(list,parsed,nofparsed) + end +end -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- +-- internal (parsed) -local functions = xml.functions -local expressions = xml.expressions +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache +end +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 +end -expressions.contains = string.find -expressions.find = string.find -expressions.upper = string.upper -expressions.lower = string.lower -expressions.number = tonumber -expressions.boolean = toboolean +-- external expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end - expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) return false end +expressions.undefined = function(s) + return s == nil +end -functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean + +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) + end + end +end + +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) + end + else + return collected + end end end -functions.name = function(d,k,n) -- ns + tg +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle + +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache +end + +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) + end + else + logs.report("xml","unknown function '%s'",fnc) + end + end +end + +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" +end + +expressions.name = function(e,n) -- ns + tg local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e elseif n < 0 then + local d, k = e.__p__.dt, e.ni for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then @@ -4114,6 +5285,7 @@ functions.name = function(d,k,n) -- ns + tg end end else + local d, k = e.__p__.dt, e.ni for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then @@ -4138,15 +5310,13 @@ functions.name = function(d,k,n) -- ns + tg end end -functions.tag = function(d,k,n) -- only tg +expressions.tag = function(e,n) -- only tg local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail elseif n < 0 then + local d, k = e.__p__.dt, e.ni for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then @@ -4159,6 +5329,7 @@ functions.tag = function(d,k,n) -- only tg end end else + local d, k = e.__p__.dt, e.ni for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then @@ -4174,664 +5345,403 @@ functions.tag = function(d,k,n) -- only tg return (found and found.tg) or "" end -expressions.text = functions.text -expressions.name = functions.name -expressions.tag = functions.tag +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- -local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) +end + +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method +end +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one +end +</typing> +--ldx]]-- + +local wrap, yield = coroutine.wrap, coroutine.yield + +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end - end - end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end + end + return wrap(function() end) +end + +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - break -- else loop - end - end - end - end + return wrap(function() for c=1,#collected do yield(collected[c]) end end) end end - return true + return wrap(function() end) end -xml.traverse = traverse + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-ent'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local type, next = type, next +local texsprint, ctxcatcodes = tex.sprint, tex.ctxcatcodes +local utf = unicode.utf8 +local utfupper = utf.upper --[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> +<p>We do things different now but it's still somewhat experimental</p> --ldx]]-- -local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +xml.entities = xml.entities or { } -- xml.entity_handler == function -xml.filters = { } +-- experimental, this will be done differently -function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk +function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in next, documententities do + allentities[k] = v + end + end end -function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk +function xml.resolved_entity(str) + local e = xml.entities[str] + if e then + local te = type(e) + if te == "function" then + e(str) else - return ekat, rt, dt, dk + texsprint(ctxcatcodes,e) end else - return { }, rt, dt, dk + texsprint(ctxcatcodes,"\\xmle{",str,"}{",utfupper(str),"}") -- we need to use our own upper end end -function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +xml.entities.amp = function() tex.write("&") end +xml.entities.lt = function() tex.write("<") end +xml.entities.gt = function() tex.write(">") end -function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n -end -function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t -end +end -- of closure -function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t -end +do -- create closure to overcome 200 locals limit -function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub = string.format, string.gsub -function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- + + +local function xmlgsub(t,old,new) + local dt = t.dt + if dt then + for k=1,#dt do + local v = dt[k] + if type(v) == "string" then + dt[k] = gsub(v,old,new) + else + xmlgsub(v,old,new) + end end end - return nil, nil, nil, nil end -function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - -- return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" - return (ekat and (ekat[arguments] or (find(arguments,"^[\'\"]") and ekat[sub(arguments,2,-2)]))) or "" -end +xmlgsub = xmlgsub -function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") end - else - return "", rt, dt, dk end end -function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag -end +--~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } +--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end -function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag -end +--~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end +--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end +--~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" ---[[ldx-- -<p>For splitting the filter function from the path specification, we can -use string matching or lpeg matching. Here the difference in speed is -neglectable but the lpeg variant is more robust.</p> ---ldx]]-- +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs --- not faster but hipper ... although ... i can't get rid of the trailing / in the path +-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg +-- +-- 1021:0335:0287:0247 -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" +-- +-- 1559:0257:0288:0190 (last one suggested by roberto) -local slash = P('/') -local name = (R("az","AZ","--","__"))^1 -local path = C(((1-slash)^0 * slash)^1) -local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } -local action = Cc(1) * path * C(name) * argument -local attribute = Cc(2) * path * P('@') * C(name) -local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument +-- escaped = Cs((S("<&>") / xml.escapes + 1)^0) +-- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +local normal = (1 - S("<&>"))^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local escaped = Cs(normal * (special * normal)^0) -local parser = direct + action + attribute +-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) -local filters = xml.filters -local attribute_filter = xml.filters.attributes -local default_filter = xml.filters.default +local normal = (1 - S"&")^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local unescaped = Cs(normal * (special * normal)^0) --- todo: also hash, could be gc'd +-- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) -function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) - else - return default_filter(root,pattern) - end -end +local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) ---~ slightly faster, but first we need a proper test file ---~ ---~ local hash = { } ---~ ---~ function xml.filter(root,pattern) ---~ local h = hash[pattern] ---~ if not h then ---~ local kind, a, b, c = parser:match(pattern) ---~ if kind == 1 then ---~ h = { kind, filters[b] or default_filter, a, b, c } ---~ elseif kind == 2 then ---~ h = { kind, attribute_filter, a, b, c } ---~ else ---~ h = { kind, default_filter, a, b, c } ---~ end ---~ hash[pattern] = h ---~ end ---~ local kind = h[1] ---~ if kind == 1 then ---~ return h[2](root,h[2],h[4]) ---~ elseif kind == 2 then ---~ return h[2](root,h[2],h[3]) ---~ else ---~ return h[2](root,pattern) ---~ end ---~ end +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed ---[[ldx-- -<p>The following functions collect elements and texts.</p> ---ldx]]-- +function xml.escaped (str) return escaped :match(str) end +function xml.unescaped(str) return unescaped:match(str) end +function xml.cleansed (str) return cleansed :match(str) end --- still somewhat bugged -function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr -end +end -- of closure -function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" - else - t[#t+1] = "" +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-aux'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons + +local xmlparseapply, xmlconvert, xmlcopy = xml.parse_apply, xml.convert, xml.copy + +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub + +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) end - else - t[#t+1] = tx or "" end - else - t[#t+1] = "" end - end) - return t + end end -function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg +xml.withelements = withelements + +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end end end - end) - return #t > 0 and {} + end end ---[[ldx-- -<p>Often using an iterators looks nicer in the code than passing handler -functions. The <l n='lua'/> book describes how to use coroutines for that -purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits -code like:</p> +xml.elements_only = xml.collected -<typing> -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) +function xml.each_element(root, pattern, handle, reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end + end + return collected + end end -</typing> -<p>Which will print all the titles in the document. The iterator variant takes -1.5 times the runtime of the function variant which is due to the overhead in -creating the wrapper. So, instead of:</p> +xml.process_elements = xml.each_element -<typing> -function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk +function xml.process_attributes(root, pattern, handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end end - return nil, nil, nil, nil + return collected end -</typing> -<p>We use the function variants in the filters.</p> +--[[ldx-- +<p>The following functions collect elements and texts.</p> --ldx]]-- -local wrap, yield = coroutine.wrap, coroutine.yield +-- are these still needed -> lxml-cmp.lua -function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) end -function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) -end - -function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok -end - -function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) end - end) + end + return collected or { } end -function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end - end) + return t + end end --[[ldx-- <p>We've now arrives at the functions that manipulate the tree.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.inject_element(root, pattern, element, prepend) if root and element then - local matches, collect = { }, nil if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,no_root) end if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d = r.dt + local k = e.ni + if element.ri then + element = element.dt[element.ri].dt else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] + element = element.dt end + local edt if r.ri then - r.dt[r.ri].dt = be + edt = r.dt[r.ri].dt else - d[k].dt = be + edt = d and d[k] and d[k].dt + end + if edt then + local be, af + if prepend then + be, af = xmlcopy(element), edt + else + be, af = edt, xmlcopy(element) + end + for i=1,#af do + be[#be+1] = af[i] + end + if r.ri then + r.dt[r.ri].dt = be + else + d[k].dt = be + end + else + -- r.dt = element.dt -- todo end - else - -- r.dt = element.dt -- todo end end end @@ -4847,32 +5757,31 @@ function xml.insert_element(root, pattern, element, before) -- todo: element als else local matches, collect = { }, nil if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested ---~ elseif element.dt then ---~ for _,v in ipairs(element.dt) do -- i added ---~ insert(d,k,v) ---~ k = k + 1 ---~ end ---~ end - else - local edt = element.dt - if edt then - for i=1,#edt do - insert(d,k,edt[i]) - k = k + 1 + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d = r.dt + local k = e.ni + if not before then + k = k + 1 + end + if element.tg then + insert(d,k,element) -- untested + else + local edt = element.dt + if edt then + for i=1,#edt do + insert(d,k,edt[i]) + k = k + 1 + end end end end @@ -4888,105 +5797,114 @@ xml.inject_element_after = xml.inject_element xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = remove(m[2],m[3]) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + remove(e.__p__.dt,e.ni) + e.ni = nil + end end - return deleted + return collection end function xml.replace_element(root, pattern, element) if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) - end -end - -local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.__p__.dt[e.ni] = element.dt -- maybe not clever enough + end + end end - return data end -function xml.include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata) -- parse="text" (default: xml), encoding="" (todo) -- attribute = attribute or 'href' pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then for a in gmatch(attribute or "href","([^|]+)") do - name = ek.at[a] + name = ekat[a] if name then break end end end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + local xi = xmlconvert(data,settings) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) end - xml.assign(d,k,xi) end end end - xml.each_element(xmldata, pattern, include) end +xml.include = include + function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then - if str == "" then - -- stripped - else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then if str == "" then -- stripped else - t[#t+1] = str + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end end + else +--~ str.ni = i + t[#t+1] = str end - else - t[#t+1] = str end + e.dt = t end - d[k].dt = t end - end) + end end local function rename_space(root, oldspace, newspace) -- fast variant @@ -5011,59 +5929,49 @@ end xml.rename_space = rename_space function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end end + function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end end + function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns + end end - end) -end -function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) + end end -function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true - else - found = true +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn end - return true - end) - return found + end end --[[ldx-- <p>Here are a few synonyms.</p> --ldx]]-- -xml.filters.position = xml.filters.index - -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found - xml.each = xml.each_element xml.process = xml.process_element xml.strip = xml.strip_whitespace @@ -5077,155 +5985,12 @@ xml.before = xml.insert_element_before xml.delete = xml.delete_element xml.replace = xml.replace_element ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- - -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xml.gsub(v,old,new) - end - end - end -end - -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..rep(" ",#s),"\n") - end -end - -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end - ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end - -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) - end - else - return "" - end -end - -function xml.statistics() - return { - lpathcalls = lpathcalls, - lpathcached = lpathcached, - } -end - --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) - ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") - ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) - ---~ xml.settrace("lpath",true) - ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) - ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] - ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) - ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-ent'] = { +if not modules then modules = { } end modules ['lxml-xml'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5233,457 +5998,249 @@ if not modules then modules = { } end modules ['lxml-ent'] = { license = "see context related readme files" } -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, find = string.format, string.gsub, string.find -local utfchar = unicode.utf8.char - ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize -xml.entities = xml.entities or { } -- xml.entity_handler == function - -function xml.entity_handler(e) - return format("[%s]",e) +local function first(collected) + return collected and collected[1] end -local function toutf(s) - return utfchar(tonumber(s,16)) +local function last(collected) + return collected and collected[#collected] end -local function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if find(dk,"&#x.-;") then - d[k] = gsub(dk,"&#x(.-);",toutf) - end - else - utfize(dk) - end - end +local function all(collected) + return collected end -xml.utfize = utfize - -local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if find(e,"^#x") then - return utfchar(tonumber(e:sub(3),16)) - elseif find(e,"^#") then - return utfchar(tonumber(e:sub(2))) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] end + return reversed end end -local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if find(dk,"&.-;") then - d[k] = gsub(dk,"&(.-);",resolve) - end - else - resolve_entities(dk) - end - end - end +local function attribute(collected,name) + local at = collected and collected[1].at + return at and at[name] end -xml.resolve_entities = resolve_entities +local function att(id,name) + local at = id.at + return at and at[name] +end -function xml.utfize_text(str) - if find(str,"&#") then - return (gsub(str,"&#x(.-);",toutf)) - else - return str - end +local function count(collected) + return (collected and #collected) or 0 end -function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if find(str,"&") then - return (gsub(str,"&(.-);",resolve)) - else - return str +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + else + return collected[n] + end end end -function xml.show_text_entities(str) - if find(str,"&") then - return (gsub(str,"&(.-);","[%1]")) - else - return str +local function index(collected) + if collected then + return collected[1].ni end end --- experimental, this will be done differently - -function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in next, documententities do - allentities[k] = v +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them end end end - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub = string.format, string.gsub - ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- - -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end else - xml.gsub(v,old,new) + break -- error end + e = e.__p__ end end + return "" end -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") +local function text(collected) + if collected then + return xmltostring(collected[1]) -- only first as we cannot concat function + else + return "" end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end - ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed - -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end - -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end end - else - return "" + return t end end - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-tra'] = { - version = 1.001, - comment = "companion to trac-tra.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the <anonymous> tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -debugger = debugger or { } - -local counters = { } -local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch - --- one - -local function hook() - local f = getinfo(2,"f").func - local n = getinfo(2,"Sn") --- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end - if f then - local cf = counters[f] - if cf == nil then - counters[f] = 1 - names[f] = n +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] else - counters[f] = cf + 1 + c = collected[#collected-n+1] end + return c and c.tg end end -local function getname(func) - local n = names[func] - if n then - if n.what == "C" then - return n.name or '<anonymous>' + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] else - -- source short_src linedefined what name namewhat nups func - local name = n.name or n.namewhat or n.what - if not name or name == "" then name = "?" end - return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + c = collected[#collected-n+1] end - else - return "unknown" - end -end -function debugger.showstats(printer,threshold) - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - printer("\n") -- ugly but ok - -- table.sort(counters) - for func, count in pairs(counters) do - if count > threshold then - local name = getname(func) - if not name:find("for generator") then - printer(format("%8i %s", count, name)) - total = total + count + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg end end - grandtotal = grandtotal + count - functions = functions + 1 end - printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) end --- two - ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in pairs(counters) do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end - --- rest - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + return t end end -function debugger.enable() - debug.sethook(hook,"c") -end - -function debugger.disable() - debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil -end - -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true - else - function debugger.tracing() return false end ; return false +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end + end end + return true end ---~ debugger.enable() - ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) - ---~ debugger.disable() - ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty -trackers = trackers or { } +-- shortcuts -- we could support xmlfilter(id,pattern,first) -local data, done = { }, { } +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) +end -local function set(what,value) - if type(what) == "string" then - what = aux.settings_to_array(what) - end - for i=1,#what do - local w = what[i] - for d, f in next, data do - if done[d] then - -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](value) - end - end - end - end +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) end -local function reset() - for d, f in next, data do - for i=1,#f do - f[i](false) - end - end +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) end -function trackers.register(what,...) - what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w - end - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end - end - end +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) end -function trackers.enable(what) - done = { } - set(what,true) +function xml.text(id,pattern) + return text(xmlfilter(id,pattern)) end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function xml.raw(id,pattern) + return xmlserialize(xmlfilter(id,pattern)) end -function trackers.reset(what) - done = { } - reset() +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) end +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter + end -- of closure @@ -6135,6 +6692,7 @@ function statistics.timed(action,report) end + end -- of closure do -- create closure to overcome 200 locals limit @@ -9814,11 +10372,13 @@ own.libs = { -- todo: check which ones are really needed 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', + 'trac-tra.lua', 'lxml-tab.lua', - 'lxml-pth.lua', + 'lxml-lpt.lua', 'lxml-ent.lua', 'lxml-mis.lua', - 'trac-tra.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', @@ -9889,7 +10449,7 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) +logs.setprogram('MTXrun',"TDS Runner Tool 1.23",environment.arguments["verbose"] or false) local instance = resolvers.reset() diff --git a/scripts/context/stubs/mswin/luatools.lua b/scripts/context/stubs/mswin/luatools.lua index a8cfbd5b0..2bc943210 100644 --- a/scripts/context/stubs/mswin/luatools.lua +++ b/scripts/context/stubs/mswin/luatools.lua @@ -230,6 +230,16 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -279,6 +289,12 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + end -- of closure @@ -387,6 +403,18 @@ function string:split(separator) return c:match(self) end +--~ function lpeg.L(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + lpeg.P(list[l]) +--~ else +--~ p = lpeg.P(list[l]) +--~ end +--~ end +--~ return p +--~ end + end -- of closure @@ -420,6 +448,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key,_ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -1192,21 +1228,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1413,7 +1463,7 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } -local format = string.format +local format, foor, insert = string.format, math.floor, table.insert number = number or { } @@ -1449,7 +1499,18 @@ function number.toset(n) return one:match(tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1914,11 +1975,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return qualified:match(filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return rootbased:match(filename) ~= nil end local slash = lpeg.S("\\/") @@ -3134,6 +3195,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3156,7 +3235,7 @@ debugger = debugger or { } local counters = { } local names = { } local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub -- one @@ -3290,7 +3369,7 @@ local data, done = { }, { } local function set(what,value) if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_array(what) -- inefficient but ok end for i=1,#what do local w = what[i] @@ -3315,6 +3394,19 @@ local function reset() end end +local function enable(what) + set(what,true) +end + +local function disable(what) + if not what or what == "" then + done = { } + reset() + else + set(what,false) + end +end + function trackers.register(what,...) what = lower(what) local w = data[what] @@ -3333,20 +3425,20 @@ function trackers.register(what,...) end function trackers.enable(what) - done = { } - set(what,true) + local e = trackers.enable + trackers.enable, done = enable, { } + enable(string.simpleesc(what)) + trackers.enable, done = e, { } end function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end + local e = trackers.disable + trackers.disable, done = disable, { } + disable(string.simpleesc(what)) + trackers.disable, done = e, { } end -function trackers.reset(what) +function trackers.reset() done = { } reset() end @@ -3423,7 +3515,7 @@ function environment.initialize_arguments(arg) environment.arguments, environment.files, environment.sortedflags = arguments, files, nil for index, argument in pairs(arg) do if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = argument:match("^%-+(.-)=(.-)$") if flag then arguments[flag] = string.unquote(value or "") else diff --git a/scripts/context/stubs/mswin/mtxrun.lua b/scripts/context/stubs/mswin/mtxrun.lua index 865994073..8bc88c900 100644 --- a/scripts/context/stubs/mswin/mtxrun.lua +++ b/scripts/context/stubs/mswin/mtxrun.lua @@ -239,6 +239,16 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -288,6 +298,12 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + end -- of closure @@ -396,6 +412,18 @@ function string:split(separator) return c:match(self) end +--~ function lpeg.L(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + lpeg.P(list[l]) +--~ else +--~ p = lpeg.P(list[l]) +--~ end +--~ end +--~ return p +--~ end + end -- of closure @@ -429,6 +457,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key,_ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -1009,7 +1045,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1038,6 +1074,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1201,21 +1255,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1422,7 +1490,7 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } -local format = string.format +local format, foor, insert = string.format, math.floor, table.insert number = number or { } @@ -1458,7 +1526,18 @@ function number.toset(n) return one:match(tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1923,11 +2002,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return qualified:match(filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return rootbased:match(filename) ~= nil end local slash = lpeg.S("\\/") @@ -2854,6 +2933,406 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in pairs(counters) do + if count > threshold then + local name = getname(func) + if not name:find("for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in pairs(counters) do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_array(what) -- inefficient but ok + end + for i=1,#what do + local w = what[i] + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](value) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(what)) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(what)) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + for k,v in ipairs(setters.list(t)) do + commands.writestatus(t.name,v) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singluar tracker + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +--~ -- old code: +-- +--~ trackers = trackers or { } +--~ local data, done = { }, { } +--~ local function set(what,value) +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end +--~ local function reset() +--~ for d, f in next, data do +--~ for i=1,#f do +--~ f[i](false) +--~ end +--~ end +--~ end +--~ local function enable(what) +--~ set(what,true) +--~ end +--~ local function disable(what) +--~ if not what or what == "" then +--~ done = { } +--~ reset() +--~ else +--~ set(what,false) +--~ end +--~ end +--~ function trackers.register(what,...) +--~ what = lower(what) +--~ local w = data[what] +--~ if not w then +--~ w = { } +--~ data[what] = w +--~ end +--~ for _, fnc in next, { ... } do +--~ local typ = type(fnc) +--~ if typ == "function" then +--~ w[#w+1] = fnc +--~ elseif typ == "string" then +--~ w[#w+1] = function(value) set(fnc,value,nesting) end +--~ end +--~ end +--~ end +--~ function trackers.enable(what) +--~ local e = trackers.enable +--~ trackers.enable, done = enable, { } +--~ enable(string.simpleesc(what)) +--~ trackers.enable, done = e, { } +--~ end +--~ function trackers.disable(what) +--~ local e = trackers.disable +--~ trackers.disable, done = disable, { } +--~ disable(string.simpleesc(what)) +--~ trackers.disable, done = e, { } +--~ end +--~ function trackers.reset() +--~ done = { } +--~ reset() +--~ end +--~ function trackers.list() -- pattern +--~ local list = table.sortedkeys(data) +--~ local user, system = { }, { } +--~ for l=1,#list do +--~ local what = list[l] +--~ if find(what,"^%*") then +--~ system[#system+1] = what +--~ else +--~ user[#user+1] = what +--~ end +--~ end +--~ return user, system +--~ end + end -- of closure @@ -2867,6 +3346,12 @@ if not modules then modules = { } end modules ['lxml-tab'] = { license = "see context related readme files" } +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble + +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -2874,18 +3359,6 @@ parent access; a first version used different trickery but was less optimized to went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> @@ -2896,26 +3369,9 @@ xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable = type, next, setmetatable +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber local format, lower, find = string.format, string.lower, string.find - ---[[ldx-- -<p>This module can be used stand alone but also inside <l n='mkiv'/> in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers.</p> ---ldx]]-- - -local trace_remap = false - -if trackers then - trackers.register("xml.remap", function(v) trace_remap = v end) -end - -function xml.settrace(str,value) - if str == "remap" then - trace_remap = value or false - end -end +local utfchar = unicode.utf8.char --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -3022,25 +3478,25 @@ element.</p> </typing> --ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes - -- not just one big nested table capture (lpeg overflow) local nsremap, resolvens = xml.xmlns, xml.resolvens local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local strip, cleanup, utfize, resolve = false, false, false, false -local mt = { __tostring = xml.text } +local mt = { } -function xml.check_error(top,toclose) - return "" +function initialize_mt(root) -- we will make a xml.new that then sets the mt as field + mt = { __tostring = xml.text, __index = root } end -local strip = false -local cleanup = false +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end -function xml.set_text_cleanup(fnc) - cleanup = fnc +function xml.check_error(top,toclose) + return "" end local function add_attribute(namespace,tag,value) @@ -3058,6 +3514,22 @@ local function add_attribute(namespace,tag,value) end end +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) + end + at = { } +end + local function add_begin(spacing, namespace, tag) if #spacing > 0 then dt[#dt+1] = spacing @@ -3083,28 +3555,12 @@ local function add_end(spacing, namespace, tag) end dt = top.dt dt[#dt+1] = toclose - dt[0] = top + -- dt[0] = top -- nasty circular reference when serializing table if toclose.at.xmlns then remove(xmlns) end end -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - local function add_text(text) if cleanup and #text > 0 then dt[#dt+1] = cleanup(text) @@ -3128,7 +3584,109 @@ local function set_message(txt) errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") end -local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end + +local dcache, hcache, acache = { }, { }, { } + +function xml.unknown_dec_entity_format(str) return format("&%s;", str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&%s;", str) end + +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + if utfize then + local n = tonumber(str,16) + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,c) + end + else + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#" .. str .. ";" + end + hcache[str] = h + end + return h +end +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + if utfize then + local n = tonumber(str) + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,c) + end + else + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&" .. str .. ";" + end + dcache[str] = d + end + return d +end +local function handle_any_entity(str) + if resolve then + local a = entities[str] -- per instance ! + if not a then + a = acache[str] + if not a then + if trace_entities then + logs.report("xml","ignoring entity &%s;",str) + else + -- can be defined in a global mapper and intercepted elsewhere + -- as happens in lxml-tex.lua + end + a = xml.unknown_any_entity_format(str) or "" + acache[str] = a + end + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,r) + acache[str] = a + end + end + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = "&" .. str .. ";" + acache[str] = a + end + return a + end +end + +local P, S, R, C, V, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cs local space = S(' \r\n\t') local open = P('<') @@ -3138,6 +3696,8 @@ local dquote = S('"') local equal = P('=') local slash = P('/') local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') local valid = R('az', 'AZ', '09') + S('_-.') local name_yes = C(valid^1) * colon * C(valid^1) local name_nop = C(P(true)) * C(valid^1) @@ -3147,15 +3707,36 @@ local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture local spacing = C(space^0) -local justtext = C((1-open)^1) + +local entitycontent = (1-open-semicolon)^0 +local entity = ampersand/"" * ( + P("#")/"" * ( + P("x")/"" * (entitycontent/handle_hex_entity) + + (entitycontent/handle_dec_entity) + ) + (entitycontent/handle_any_entity) + ) * (semicolon/"") + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) + local somespace = space^1 local optionalspace = space^0 -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute -local attributes = attribute^0 +local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value + +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error + +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local endofattributes = slash * close + close -- recovery of flacky html +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 -local text = justtext / add_text +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty @@ -3208,42 +3789,72 @@ local doctype = (spacing * begindoctype * somedoctype * enddoct -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special -local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 -local grammar = P { "preamble", +local grammar_parsed_text = P { "preamble", preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- todo: xml.new + properties like entities and strip and such (store in root) +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} -function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, settings.entities or {} + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end stack[#stack+1] = top top.dt = { } dt = top.dt if not data or data == "" then errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" + elseif utfize or resolve then + if grammar_parsed_text:match(data) then + errorstr = "" + else + errorstr = "invalid xml file - parsed text" + end else - errorstr = "" + if grammar_unparsed_text:match(data) then + errorstr = "" + else + errorstr = "invalid xml file - unparsed text" + end end if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } } } setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + local error_handler = settings.error_handler + if error_handler == false then + -- no error message + else + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) + end + end else result = stack[1] end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities, settings = settings } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do @@ -3254,9 +3865,14 @@ function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe us end end end + if errorstr and errorstr ~= "" then + result.error = true + end return result end +xml.convert = xmlconvert + --[[ldx-- <p>Packaging data in an xml like table is done with the following function. Maybe it will go away (when not used).</p> @@ -3289,16 +3905,16 @@ function xml.load(filename) if type(filename) == "string" then local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + local root = xmlconvert(f:read("*all")) f:close() return root else - return xml.convert("") + return xmlconvert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) + return xmlconvert(filename:read("*all")) else - return xml.convert("") + return xmlconvert("") end end @@ -3307,9 +3923,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -3354,217 +3972,305 @@ alternative.</p> -- todo: add <?xml version='1.0' standalone='yes'?> when not present -local fallbackhandle = (tex and tex.sprint) or io.write - -local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) +end + +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) else - -- no need to handle any further + serialize(e,handlers) end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) end + handle("</",ens,":",etg,">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster if ats then - if attributeconverter then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) - end - end + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") end - if ern and trace_remap and ern ~= ens then - ens = ern + end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end + serialize(ei,handlers) end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + handle("<",etg,"/>") end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - handle(e) + serialize(ei,handlers) end - else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + end +end + +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state end end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end -xml.serialize = serialize +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then - found = true - break +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv + end + else + t[k] = v end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) + if settings.name then + handlers[settings.name] = t end end + return t +end + +local nofunction = function() end + +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction end +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + --[[ldx-- -<p>At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.</p> +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> --ldx]]-- -function xml.tostring(root) -- 25% overhead due to collecting +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end return "" end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" +end + +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring +xml.text = xmltext + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -3572,33 +4278,16 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> - -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> ---ldx]]-- - -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() - end -end +xml.string = xmlstring --[[ldx-- <p>A few helpers:</p> --ldx]]-- +function xml.parent(root) + return root.__p__ +end + function xml.body(root) return (root.ri and root.dt[root.ri]) or root end @@ -3611,34 +4300,19 @@ function xml.content(root) -- bugged return (root and root.dt and xml.tostring(root.dt)) or "" end -function xml.isempty(root, pattern) - if pattern == "" or pattern == "*" then - pattern = nil - end - if pattern then - -- todo - return false - else - return not root or not root.dt or #root.dt == 0 or root.dt == "" - end -end - --[[ldx-- <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -3672,96 +4346,403 @@ if not modules then modules = { } end modules ['lxml-pth'] = { license = "see context related readme files" } +-- e.ni is only valid after a filter run + local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, lower, gmatch, gsub, find, rep = string.format, string.lower, string.gmatch, string.gsub, string.find, string.rep +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep --[[ldx-- <p>This module can be used stand alone but also inside <l n='mkiv'/> in which case it hooks into the tracker code. Therefore we provide a few functions that set the tracers. Here we overload a previously defined function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> --ldx]]-- -local trace_lpath = false - -if trackers then - trackers.register("xml.lpath", function(v) trace_lpath = v end) -end +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> -local settrace = xml.settrace -- lxml-tab +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- -function xml.settrace(str,value) - if str == "lpath" then - trace_lpath = value or false - else - settrace(str,value) -- lxml-tab - end -end +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statistics -local lpathcached = 0 -- statistics +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -xml.expressions = xml.expressions or { } +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } local functions = xml.functions local expressions = xml.expressions +local finalizers = xml.finalizers -local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", -} +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } + +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end + end + return fn +end + +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) + +xml.defaultprotocol = "xml" + +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above --- a rather dumb lpeg +-- evaluator (needs to be redone, for the moment copied) -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- todo: apply_axis(list,notable) and collection vs single --- instead of using functions we just parse a few names which saves a call --- later on +local apply_axis = { } -local lp_position = P("position()") / "ps" -local lp_index = P("index()") / "id" -local lp_text = P("text()") / "tx" -local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" -local lp_tag = P("tag()") / "tg" -- (rt.tg or '') -local lp_ns = P("ns()") / "ns" -- (rt.ns or '') -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll + end + end + collected[#collected+1] = rt + end + return collected +end + +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end -local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local dt = list[l].dt + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + end + end + end + return collected +end + +local function collect(list,collected) + local dt = list.dt + if dt then + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + collect(dk,collected) + end + end + end +end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end + +local function collect(list,collected) + local dt = list.dt + if dt then + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + collect(dk,collected) + end + end + end +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] +if ll.special ~= true then -- catch double root + collected[#collected+1] = ll +end + collect(ll,collected) + end + return collected +end + +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end + +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end + +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl + end + end + return collected +end + +apply_axis['attribute'] = function(list) + return { } +end + +apply_axis['following'] = function(list) + return { } +end + +apply_axis['following-sibling'] = function(list) + return { } +end + +apply_axis['namespace'] = function(list) + return { } +end + +apply_axis['preceding'] = function(list) + return { } +end + +apply_axis['preceding-sibling'] = function(list) + return { } +end + +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] + +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } + end + else + local collected = { } + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + collected[#collected+1] = ll + end + elseif ntg ~= ltg then + collected[#collected+1] = ll + end + end + end + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + collected[#collected+1] = ll + end + elseif lns ~= nns then + collected[#collected+1] = ll + end + end + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + collected[#collected+1] = ll + end + elseif not ok then + collected[#collected+1] = ll + end + end + end + end + return collected + end + else + local collected = { } + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + collected[#collected+1] = ll + end + elseif not ok then + collected[#collected+1] = ll + end + end + end + return collected + end +end + +local function apply_expression(list,expression,order) + local collected = { } + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, alleen valid als n=1 + collected[#collected+1] = ll + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 + +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) + +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("first") / "1" + + P("last") / "#list" + + P("position") / "l" + + P("rootposition") / "order" + + P("index") / "ll.ni" + + P("text") / "(ll.dt[1] or '')" + + P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("ll.at['") * R("az","AZ","--","__")^1 * Cc("']") +local lp_fastpos = ((R("09","--","++")^1 * P(-1)) / function(s) return "l==" .. s end) + +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling return t .. "(" end -local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling if expressions[t] then - return "expressions." .. t .. "(" + return "expr." .. t .. "(" else - return "expressions.error(" + return "expr.error(" end end @@ -3771,337 +4752,527 @@ local noparent = 1 - (lparent+rparent) local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} --- if we use a dedicated namespace then we don't need to pass rt and k +local lp_child = Cc("expr.child(e,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content= (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) + +local cleaner -local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" + s = s and s ~= "" and cleaner:match(s) + if s and s ~= "" then + return "expr." .. t .. "(e," .. s ..")" else - return "expressions." .. t .. "(r,k)" + return "expr." .. t .. "(e)" end else - return "expressions.error(" .. t .. ")" + return "expr.error(" .. t .. ")" end end -local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one +local content = + lp_builtin + + lp_attribute + lp_special + lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = lpeg.Cs ( + lp_fastpos + (lpeg.P { lparent * (lpeg.V(1))^0 * rparent + content } )^0 +) + +cleaner = lpeg.Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_string + 1 )^1 ) --- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 +--~ expr -local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,root) return %s end ]] -local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() -end - -local map = { } - -local space = S(' \r\n\t') -local squote = S("'") -local dquote = S('"') -local lparent = P('(') -local rparent = P(')') -local atsign = P('@') -local lbracket = P('[') -local rbracket = P(']') -local exclam = P('!') -local period = P('.') -local eq = P('==') + P('=') -local ne = P('<>') + P('!=') -local star = P('*') -local slash = P('/') -local colon = P(':') -local bar = P('|') -local hat = P('^') -local valid = R('az', 'AZ', '09') + S('_-') -local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* -local name_nop = Cc("*") * C(valid^1) -local name = name_yes + name_nop -local number = C((S('+-')^0 * R('09')^1)) / tonumber -local names = (bar^0 * name)^1 -local morenames = name * (bar^0 * name)^1 -local instructiontag = P('pi::') -local spacing = C(space^0) -local somespace = space^1 -local optionalspace = space^0 -local text = C(valid^0) -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local empty = 1-slash - -local is_eq = lbracket * atsign * name * eq * value * rbracket -local is_ne = lbracket * atsign * name * ne * value * rbracket -local is_attribute = lbracket * atsign * name * rbracket -local is_value = lbracket * value * rbracket -local is_number = lbracket * number * rbracket - -local nobracket = 1-(lbracket+rbracket) -- must be improved -local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - -local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - -local is_one = name -local is_none = exclam * name -local is_one_of = ((lparent * names * rparent) + morenames) -local is_none_of = exclam * ((lparent * names * rparent) + morenames) - -local stay = (period ) -local parent = (period * period ) / function( ) map[#map+1] = { 11 } end -local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end -local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end -local any = (star ) / function( ) map[#map+1] = { 14 } end -local many = (star * star ) / function( ) map[#map+1] = { 15 } end -local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - -local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end -local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end -local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end -local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - -local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end -local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end -local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end -local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - -local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end -local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end -local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end -local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - -local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end -local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end -local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end -local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end -local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end -local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - -local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end -local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - -local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end -local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - -local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end -local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? -local crap = (1-slash)^1 - --- a few ugly goodies: - -local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end -local subroottag = P('^') / function( ) map[#map+1] = { 13 } end -local roottag = P('root::') / function( ) map[#map+1] = { 12 } end -local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end -local childtag = P('child::') -local selftag = P('self::') - --- there will be more and order will be optimized - -local selector = ( - instruction + --- many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty -) +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] -local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, -} +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] -local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false +-- + +local function errorrunner_e(str,cnv) + logs.report("lpath","error in expression: %s => %s",str,cnv) + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false +end + +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } +end + +local function register_expression(expression) + local converted = converter:match(expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) - end - -- print(gsub(table.serialize(map),"[ \n]+"," ")) - return map - end + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } end +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} + +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} + +-- todo: better arg parser + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local function register_error(str) + return { kind = "error", comment = format("unparsed: %s",str) } +end + +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside + + patterns = spaces * V("protocol") * spaces * V("initial") * spaces * V("step") * spaces * + (P("/") * spaces * V("step") * spaces)^0, + + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), + + step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following") + V("following_sibling") + + V("preceding") + V("preceding_sibling") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), + + initial = (P("/") * spaces * Cc(register_initial_child))^-1, + + error = (P(1)^1) / register_error, + + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), + + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, + + s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), + + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, + + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + local cache = { } -function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 - end - if trace or trace_lpath then - xml.lshow(result) - end - return result +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) else - return pattern + return concat(t,"|") end end -function xml.cached_patterns() - return cache +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end end --- we run out of locals (limited to 200) --- --- local fallbackreport = (texio and texio.write) or io.write - -function xml.lshow(pattern,report) --- report = report or fallbackreport - report = report or (texio and texio.write) or io.write - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") +xml.nodesettostring = nodesettostring + +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) + end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly +end + +xml.lshow = lshow + +local function parse_pattern(pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k=1,#lp do - local v = lp[k] - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = parser:match(pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) + else + -- we could have done this with a more complex parsed but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + parsed.comment = "auto-child replaced by auto-descendant-or-self" + parsed[1] = register_auto_descendant_or_self + --~ parsed.comment = "auto-child replaced by auto-descendant" + --~ parsed[1] = register_auto_descendant + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + parsed.comment = "initial-child removed" -- we could also make it a auto-self + remove(parsed,1) end end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end end + return parsed end end -function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } --- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up + +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 else - for i=1,#e do - report(tostring(e[i]) .. "\n") + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + return nil end end + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + logs.report("lpath", "% 10i : ex : %s",(collected and #collected) or 0,pi.expression) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(collected and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + return nil + end + end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> - -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + -- something is wrong + elseif not trace_lpath then + -- normal apply, inline, no self + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + elseif kind == "finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected == 0 then + return nil + end + end + return collected + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed) + else -- trace_lpath + return traced_apply(list,parsed,nofparsed) + end +end -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- +-- internal (parsed) -local functions = xml.functions -local expressions = xml.expressions +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache +end +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 +end -expressions.contains = string.find -expressions.find = string.find -expressions.upper = string.upper -expressions.lower = string.lower -expressions.number = tonumber -expressions.boolean = toboolean +-- external expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end - expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) return false end +expressions.undefined = function(s) + return s == nil +end -functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean + +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) + end + end +end + +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) + end + else + return collected + end end end -functions.name = function(d,k,n) -- ns + tg +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle + +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache +end + +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) + end + else + logs.report("xml","unknown function '%s'",fnc) + end + end +end + +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" +end + +expressions.name = function(e,n) -- ns + tg local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e elseif n < 0 then + local d, k = e.__p__.dt, e.ni for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then @@ -4114,6 +5285,7 @@ functions.name = function(d,k,n) -- ns + tg end end else + local d, k = e.__p__.dt, e.ni for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then @@ -4138,15 +5310,13 @@ functions.name = function(d,k,n) -- ns + tg end end -functions.tag = function(d,k,n) -- only tg +expressions.tag = function(e,n) -- only tg local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail elseif n < 0 then + local d, k = e.__p__.dt, e.ni for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then @@ -4159,6 +5329,7 @@ functions.tag = function(d,k,n) -- only tg end end else + local d, k = e.__p__.dt, e.ni for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then @@ -4174,664 +5345,403 @@ functions.tag = function(d,k,n) -- only tg return (found and found.tg) or "" end -expressions.text = functions.text -expressions.name = functions.name -expressions.tag = functions.tag +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- -local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) +end + +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method +end +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one +end +</typing> +--ldx]]-- + +local wrap, yield = coroutine.wrap, coroutine.yield + +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end - end - end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end + end + return wrap(function() end) +end + +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - break -- else loop - end - end - end - end + return wrap(function() for c=1,#collected do yield(collected[c]) end end) end end - return true + return wrap(function() end) end -xml.traverse = traverse + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-ent'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local type, next = type, next +local texsprint, ctxcatcodes = tex.sprint, tex.ctxcatcodes +local utf = unicode.utf8 +local utfupper = utf.upper --[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> +<p>We do things different now but it's still somewhat experimental</p> --ldx]]-- -local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +xml.entities = xml.entities or { } -- xml.entity_handler == function -xml.filters = { } +-- experimental, this will be done differently -function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk +function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in next, documententities do + allentities[k] = v + end + end end -function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk +function xml.resolved_entity(str) + local e = xml.entities[str] + if e then + local te = type(e) + if te == "function" then + e(str) else - return ekat, rt, dt, dk + texsprint(ctxcatcodes,e) end else - return { }, rt, dt, dk + texsprint(ctxcatcodes,"\\xmle{",str,"}{",utfupper(str),"}") -- we need to use our own upper end end -function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +xml.entities.amp = function() tex.write("&") end +xml.entities.lt = function() tex.write("<") end +xml.entities.gt = function() tex.write(">") end -function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n -end -function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t -end +end -- of closure -function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t -end +do -- create closure to overcome 200 locals limit -function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub = string.format, string.gsub -function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- + + +local function xmlgsub(t,old,new) + local dt = t.dt + if dt then + for k=1,#dt do + local v = dt[k] + if type(v) == "string" then + dt[k] = gsub(v,old,new) + else + xmlgsub(v,old,new) + end end end - return nil, nil, nil, nil end -function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - -- return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" - return (ekat and (ekat[arguments] or (find(arguments,"^[\'\"]") and ekat[sub(arguments,2,-2)]))) or "" -end +xmlgsub = xmlgsub -function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") end - else - return "", rt, dt, dk end end -function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag -end +--~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } +--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end -function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag -end +--~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end +--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end +--~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" ---[[ldx-- -<p>For splitting the filter function from the path specification, we can -use string matching or lpeg matching. Here the difference in speed is -neglectable but the lpeg variant is more robust.</p> ---ldx]]-- +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs --- not faster but hipper ... although ... i can't get rid of the trailing / in the path +-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg +-- +-- 1021:0335:0287:0247 -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" +-- +-- 1559:0257:0288:0190 (last one suggested by roberto) -local slash = P('/') -local name = (R("az","AZ","--","__"))^1 -local path = C(((1-slash)^0 * slash)^1) -local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } -local action = Cc(1) * path * C(name) * argument -local attribute = Cc(2) * path * P('@') * C(name) -local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument +-- escaped = Cs((S("<&>") / xml.escapes + 1)^0) +-- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +local normal = (1 - S("<&>"))^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local escaped = Cs(normal * (special * normal)^0) -local parser = direct + action + attribute +-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) -local filters = xml.filters -local attribute_filter = xml.filters.attributes -local default_filter = xml.filters.default +local normal = (1 - S"&")^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local unescaped = Cs(normal * (special * normal)^0) --- todo: also hash, could be gc'd +-- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) -function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) - else - return default_filter(root,pattern) - end -end +local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) ---~ slightly faster, but first we need a proper test file ---~ ---~ local hash = { } ---~ ---~ function xml.filter(root,pattern) ---~ local h = hash[pattern] ---~ if not h then ---~ local kind, a, b, c = parser:match(pattern) ---~ if kind == 1 then ---~ h = { kind, filters[b] or default_filter, a, b, c } ---~ elseif kind == 2 then ---~ h = { kind, attribute_filter, a, b, c } ---~ else ---~ h = { kind, default_filter, a, b, c } ---~ end ---~ hash[pattern] = h ---~ end ---~ local kind = h[1] ---~ if kind == 1 then ---~ return h[2](root,h[2],h[4]) ---~ elseif kind == 2 then ---~ return h[2](root,h[2],h[3]) ---~ else ---~ return h[2](root,pattern) ---~ end ---~ end +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed ---[[ldx-- -<p>The following functions collect elements and texts.</p> ---ldx]]-- +function xml.escaped (str) return escaped :match(str) end +function xml.unescaped(str) return unescaped:match(str) end +function xml.cleansed (str) return cleansed :match(str) end --- still somewhat bugged -function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr -end +end -- of closure -function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" - else - t[#t+1] = "" +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-aux'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons + +local xmlparseapply, xmlconvert, xmlcopy = xml.parse_apply, xml.convert, xml.copy + +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub + +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) end - else - t[#t+1] = tx or "" end - else - t[#t+1] = "" end - end) - return t + end end -function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg +xml.withelements = withelements + +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end end end - end) - return #t > 0 and {} + end end ---[[ldx-- -<p>Often using an iterators looks nicer in the code than passing handler -functions. The <l n='lua'/> book describes how to use coroutines for that -purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits -code like:</p> +xml.elements_only = xml.collected -<typing> -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) +function xml.each_element(root, pattern, handle, reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end + end + return collected + end end -</typing> -<p>Which will print all the titles in the document. The iterator variant takes -1.5 times the runtime of the function variant which is due to the overhead in -creating the wrapper. So, instead of:</p> +xml.process_elements = xml.each_element -<typing> -function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk +function xml.process_attributes(root, pattern, handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end end - return nil, nil, nil, nil + return collected end -</typing> -<p>We use the function variants in the filters.</p> +--[[ldx-- +<p>The following functions collect elements and texts.</p> --ldx]]-- -local wrap, yield = coroutine.wrap, coroutine.yield +-- are these still needed -> lxml-cmp.lua -function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) end -function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) -end - -function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok -end - -function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) end - end) + end + return collected or { } end -function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end - end) + return t + end end --[[ldx-- <p>We've now arrives at the functions that manipulate the tree.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.inject_element(root, pattern, element, prepend) if root and element then - local matches, collect = { }, nil if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,no_root) end if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d = r.dt + local k = e.ni + if element.ri then + element = element.dt[element.ri].dt else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] + element = element.dt end + local edt if r.ri then - r.dt[r.ri].dt = be + edt = r.dt[r.ri].dt else - d[k].dt = be + edt = d and d[k] and d[k].dt + end + if edt then + local be, af + if prepend then + be, af = xmlcopy(element), edt + else + be, af = edt, xmlcopy(element) + end + for i=1,#af do + be[#be+1] = af[i] + end + if r.ri then + r.dt[r.ri].dt = be + else + d[k].dt = be + end + else + -- r.dt = element.dt -- todo end - else - -- r.dt = element.dt -- todo end end end @@ -4847,32 +5757,31 @@ function xml.insert_element(root, pattern, element, before) -- todo: element als else local matches, collect = { }, nil if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested ---~ elseif element.dt then ---~ for _,v in ipairs(element.dt) do -- i added ---~ insert(d,k,v) ---~ k = k + 1 ---~ end ---~ end - else - local edt = element.dt - if edt then - for i=1,#edt do - insert(d,k,edt[i]) - k = k + 1 + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d = r.dt + local k = e.ni + if not before then + k = k + 1 + end + if element.tg then + insert(d,k,element) -- untested + else + local edt = element.dt + if edt then + for i=1,#edt do + insert(d,k,edt[i]) + k = k + 1 + end end end end @@ -4888,105 +5797,114 @@ xml.inject_element_after = xml.inject_element xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = remove(m[2],m[3]) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + remove(e.__p__.dt,e.ni) + e.ni = nil + end end - return deleted + return collection end function xml.replace_element(root, pattern, element) if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) - end -end - -local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.__p__.dt[e.ni] = element.dt -- maybe not clever enough + end + end end - return data end -function xml.include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata) -- parse="text" (default: xml), encoding="" (todo) -- attribute = attribute or 'href' pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then for a in gmatch(attribute or "href","([^|]+)") do - name = ek.at[a] + name = ekat[a] if name then break end end end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + local xi = xmlconvert(data,settings) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) end - xml.assign(d,k,xi) end end end - xml.each_element(xmldata, pattern, include) end +xml.include = include + function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then - if str == "" then - -- stripped - else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then if str == "" then -- stripped else - t[#t+1] = str + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end end + else +--~ str.ni = i + t[#t+1] = str end - else - t[#t+1] = str end + e.dt = t end - d[k].dt = t end - end) + end end local function rename_space(root, oldspace, newspace) -- fast variant @@ -5011,59 +5929,49 @@ end xml.rename_space = rename_space function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end end + function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end end + function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns + end end - end) -end -function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) + end end -function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true - else - found = true +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn end - return true - end) - return found + end end --[[ldx-- <p>Here are a few synonyms.</p> --ldx]]-- -xml.filters.position = xml.filters.index - -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found - xml.each = xml.each_element xml.process = xml.process_element xml.strip = xml.strip_whitespace @@ -5077,155 +5985,12 @@ xml.before = xml.insert_element_before xml.delete = xml.delete_element xml.replace = xml.replace_element ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- - -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xml.gsub(v,old,new) - end - end - end -end - -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..rep(" ",#s),"\n") - end -end - -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end - ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end - -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) - end - else - return "" - end -end - -function xml.statistics() - return { - lpathcalls = lpathcalls, - lpathcached = lpathcached, - } -end - --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) - ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") - ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) - ---~ xml.settrace("lpath",true) - ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) - ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] - ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) - ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-ent'] = { +if not modules then modules = { } end modules ['lxml-xml'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5233,457 +5998,249 @@ if not modules then modules = { } end modules ['lxml-ent'] = { license = "see context related readme files" } -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, find = string.format, string.gsub, string.find -local utfchar = unicode.utf8.char - ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize -xml.entities = xml.entities or { } -- xml.entity_handler == function - -function xml.entity_handler(e) - return format("[%s]",e) +local function first(collected) + return collected and collected[1] end -local function toutf(s) - return utfchar(tonumber(s,16)) +local function last(collected) + return collected and collected[#collected] end -local function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if find(dk,"&#x.-;") then - d[k] = gsub(dk,"&#x(.-);",toutf) - end - else - utfize(dk) - end - end +local function all(collected) + return collected end -xml.utfize = utfize - -local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if find(e,"^#x") then - return utfchar(tonumber(e:sub(3),16)) - elseif find(e,"^#") then - return utfchar(tonumber(e:sub(2))) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] end + return reversed end end -local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if find(dk,"&.-;") then - d[k] = gsub(dk,"&(.-);",resolve) - end - else - resolve_entities(dk) - end - end - end +local function attribute(collected,name) + local at = collected and collected[1].at + return at and at[name] end -xml.resolve_entities = resolve_entities +local function att(id,name) + local at = id.at + return at and at[name] +end -function xml.utfize_text(str) - if find(str,"&#") then - return (gsub(str,"&#x(.-);",toutf)) - else - return str - end +local function count(collected) + return (collected and #collected) or 0 end -function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if find(str,"&") then - return (gsub(str,"&(.-);",resolve)) - else - return str +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + else + return collected[n] + end end end -function xml.show_text_entities(str) - if find(str,"&") then - return (gsub(str,"&(.-);","[%1]")) - else - return str +local function index(collected) + if collected then + return collected[1].ni end end --- experimental, this will be done differently - -function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in next, documententities do - allentities[k] = v +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them end end end - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub = string.format, string.gsub - ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- - -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end else - xml.gsub(v,old,new) + break -- error end + e = e.__p__ end end + return "" end -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") +local function text(collected) + if collected then + return xmltostring(collected[1]) -- only first as we cannot concat function + else + return "" end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end - ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed - -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end - -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end end - else - return "" + return t end end - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-tra'] = { - version = 1.001, - comment = "companion to trac-tra.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the <anonymous> tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -debugger = debugger or { } - -local counters = { } -local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch - --- one - -local function hook() - local f = getinfo(2,"f").func - local n = getinfo(2,"Sn") --- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end - if f then - local cf = counters[f] - if cf == nil then - counters[f] = 1 - names[f] = n +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] else - counters[f] = cf + 1 + c = collected[#collected-n+1] end + return c and c.tg end end -local function getname(func) - local n = names[func] - if n then - if n.what == "C" then - return n.name or '<anonymous>' + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] else - -- source short_src linedefined what name namewhat nups func - local name = n.name or n.namewhat or n.what - if not name or name == "" then name = "?" end - return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + c = collected[#collected-n+1] end - else - return "unknown" - end -end -function debugger.showstats(printer,threshold) - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - printer("\n") -- ugly but ok - -- table.sort(counters) - for func, count in pairs(counters) do - if count > threshold then - local name = getname(func) - if not name:find("for generator") then - printer(format("%8i %s", count, name)) - total = total + count + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg end end - grandtotal = grandtotal + count - functions = functions + 1 end - printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) end --- two - ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in pairs(counters) do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end - --- rest - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + return t end end -function debugger.enable() - debug.sethook(hook,"c") -end - -function debugger.disable() - debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil -end - -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true - else - function debugger.tracing() return false end ; return false +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end + end end + return true end ---~ debugger.enable() - ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) - ---~ debugger.disable() - ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty -trackers = trackers or { } +-- shortcuts -- we could support xmlfilter(id,pattern,first) -local data, done = { }, { } +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) +end -local function set(what,value) - if type(what) == "string" then - what = aux.settings_to_array(what) - end - for i=1,#what do - local w = what[i] - for d, f in next, data do - if done[d] then - -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](value) - end - end - end - end +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) end -local function reset() - for d, f in next, data do - for i=1,#f do - f[i](false) - end - end +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) end -function trackers.register(what,...) - what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w - end - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end - end - end +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) end -function trackers.enable(what) - done = { } - set(what,true) +function xml.text(id,pattern) + return text(xmlfilter(id,pattern)) end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function xml.raw(id,pattern) + return xmlserialize(xmlfilter(id,pattern)) end -function trackers.reset(what) - done = { } - reset() +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) end +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter + end -- of closure @@ -6135,6 +6692,7 @@ function statistics.timed(action,report) end + end -- of closure do -- create closure to overcome 200 locals limit @@ -9814,11 +10372,13 @@ own.libs = { -- todo: check which ones are really needed 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', + 'trac-tra.lua', 'lxml-tab.lua', - 'lxml-pth.lua', + 'lxml-lpt.lua', 'lxml-ent.lua', 'lxml-mis.lua', - 'trac-tra.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', @@ -9889,7 +10449,7 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) +logs.setprogram('MTXrun',"TDS Runner Tool 1.23",environment.arguments["verbose"] or false) local instance = resolvers.reset() diff --git a/scripts/context/stubs/unix/luatools b/scripts/context/stubs/unix/luatools index a8cfbd5b0..2bc943210 100755 --- a/scripts/context/stubs/unix/luatools +++ b/scripts/context/stubs/unix/luatools @@ -230,6 +230,16 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -279,6 +289,12 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + end -- of closure @@ -387,6 +403,18 @@ function string:split(separator) return c:match(self) end +--~ function lpeg.L(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + lpeg.P(list[l]) +--~ else +--~ p = lpeg.P(list[l]) +--~ end +--~ end +--~ return p +--~ end + end -- of closure @@ -420,6 +448,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key,_ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -1192,21 +1228,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1413,7 +1463,7 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } -local format = string.format +local format, foor, insert = string.format, math.floor, table.insert number = number or { } @@ -1449,7 +1499,18 @@ function number.toset(n) return one:match(tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1914,11 +1975,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return qualified:match(filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return rootbased:match(filename) ~= nil end local slash = lpeg.S("\\/") @@ -3134,6 +3195,24 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + end -- of closure @@ -3156,7 +3235,7 @@ debugger = debugger or { } local counters = { } local names = { } local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub -- one @@ -3290,7 +3369,7 @@ local data, done = { }, { } local function set(what,value) if type(what) == "string" then - what = aux.settings_to_array(what) + what = aux.settings_to_array(what) -- inefficient but ok end for i=1,#what do local w = what[i] @@ -3315,6 +3394,19 @@ local function reset() end end +local function enable(what) + set(what,true) +end + +local function disable(what) + if not what or what == "" then + done = { } + reset() + else + set(what,false) + end +end + function trackers.register(what,...) what = lower(what) local w = data[what] @@ -3333,20 +3425,20 @@ function trackers.register(what,...) end function trackers.enable(what) - done = { } - set(what,true) + local e = trackers.enable + trackers.enable, done = enable, { } + enable(string.simpleesc(what)) + trackers.enable, done = e, { } end function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end + local e = trackers.disable + trackers.disable, done = disable, { } + disable(string.simpleesc(what)) + trackers.disable, done = e, { } end -function trackers.reset(what) +function trackers.reset() done = { } reset() end @@ -3423,7 +3515,7 @@ function environment.initialize_arguments(arg) environment.arguments, environment.files, environment.sortedflags = arguments, files, nil for index, argument in pairs(arg) do if index > 0 then - local flag, value = argument:match("^%-+(.+)=(.-)$") + local flag, value = argument:match("^%-+(.-)=(.-)$") if flag then arguments[flag] = string.unquote(value or "") else diff --git a/scripts/context/stubs/unix/mtxrun b/scripts/context/stubs/unix/mtxrun index 865994073..8bc88c900 100755 --- a/scripts/context/stubs/unix/mtxrun +++ b/scripts/context/stubs/unix/mtxrun @@ -239,6 +239,16 @@ function string:pattesc() return (gsub(self,".",patterns_escapes)) end +local simple_escapes = { + ["-"] = "%-", + ["."] = "%.", + ["*"] = ".*", +} + +function string:simpleesc() + return (gsub(self,".",simple_escapes)) +end + function string:tohash() local t = { } for s in gmatch(self,"([^, ]+)") do -- lpeg @@ -288,6 +298,12 @@ function string:compactlong() -- strips newlines and leading spaces return self end +function string:striplong() -- strips newlines and leading spaces + self = gsub(self,"^%s*","") + self = gsub(self,"[\n\r]+ *","\n") + return self +end + end -- of closure @@ -396,6 +412,18 @@ function string:split(separator) return c:match(self) end +--~ function lpeg.L(list,pp) +--~ local p = pp +--~ for l=1,#list do +--~ if p then +--~ p = p + lpeg.P(list[l]) +--~ else +--~ p = lpeg.P(list[l]) +--~ end +--~ end +--~ return p +--~ end + end -- of closure @@ -429,6 +457,14 @@ function table.strip(tab) return lst end +function table.keys(t) + local k = { } + for key,_ in next, t do + k[#k+1] = key + end + return k +end + local function compare(a,b) return (tostring(a) < tostring(b)) end @@ -1009,7 +1045,7 @@ function table.tofile(filename,root,name,reduce,noquotes,hexify) end end -local function flatten(t,f,complete) +local function flatten(t,f,complete) -- is this used? meybe a variant with next, ... for i=1,#t do local v = t[i] if type(v) == "table" then @@ -1038,6 +1074,24 @@ end table.flatten_one_level = table.unnest +-- a better one: + +local function flattened(t,f) + if not f then + f = { } + end + for k, v in next, t do + if type(v) == "table" then + flattened(v,f) + else + f[k] = v + end + end + return f +end + +table.flattened = flattened + -- the next three may disappear function table.remove_value(t,value) -- todo: n @@ -1201,21 +1255,35 @@ function table.reverse(t) return tt end ---~ function table.keys(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return k ---~ end +function table.insert_before_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i,extra) + return + end + end + insert(t,1,extra) +end ---~ function table.keys_as_string(t) ---~ local k = { } ---~ for k,_ in next, t do ---~ k[#k+1] = k ---~ end ---~ return concat(k,"") ---~ end +function table.insert_after_value(t,value,extra) + for i=1,#t do + if t[i] == extra then + remove(t,i) + end + end + for i=1,#t do + if t[i] == value then + insert(t,i+1,extra) + return + end + end + insert(t,#t+1,extra) +end end -- of closure @@ -1422,7 +1490,7 @@ if not modules then modules = { } end modules ['l-number'] = { license = "see context related readme files" } -local format = string.format +local format, foor, insert = string.format, math.floor, table.insert number = number or { } @@ -1458,7 +1526,18 @@ function number.toset(n) return one:match(tostring(n)) end - +function number.bits(n,zero) + local t, i = { }, (zero and 0) or 1 + while n > 0 do + local m = n % 2 + if m > 0 then + insert(t,1,i) + end + n = floor(n/2) + i = i + 1 + end + return t +end end -- of closure @@ -1923,11 +2002,11 @@ local rootbased = lpeg.P("/") + letter*lpeg.P(":") -- ./name ../name /name c: :// name/name function file.is_qualified_path(filename) - return qualified:match(filename) + return qualified:match(filename) ~= nil end function file.is_rootbased_path(filename) - return rootbased:match(filename) + return rootbased:match(filename) ~= nil end local slash = lpeg.S("\\/") @@ -2854,6 +2933,406 @@ function aux.accesstable(target) return t end +-- as we use this a lot ... + +--~ function aux.cachefunction(action,weak) +--~ local cache = { } +--~ if weak then +--~ setmetatable(cache, { __mode = "kv" } ) +--~ end +--~ local function reminder(str) +--~ local found = cache[str] +--~ if not found then +--~ found = action(str) +--~ cache[str] = found +--~ end +--~ return found +--~ end +--~ return reminder, cache +--~ end + + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['trac-tra'] = { + version = 1.001, + comment = "companion to trac-tra.mkiv", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- the <anonymous> tag is kind of generic and used for functions that are not +-- bound to a variable, like node.new, node.copy etc (contrary to for instance +-- node.has_attribute which is bound to a has_attribute local variable in mkiv) + +local getinfo = debug.getinfo +local type, next = type, next +local concat = table.concat +local format, find, lower, gmatch, gsub = string.format, string.find, string.lower, string.gmatch, string.gsub + +debugger = debugger or { } + +local counters = { } +local names = { } + +-- one + +local function hook() + local f = getinfo(2,"f").func + local n = getinfo(2,"Sn") +-- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end + if f then + local cf = counters[f] + if cf == nil then + counters[f] = 1 + names[f] = n + else + counters[f] = cf + 1 + end + end +end +local function getname(func) + local n = names[func] + if n then + if n.what == "C" then + return n.name or '<anonymous>' + else + -- source short_src linedefined what name namewhat nups func + local name = n.name or n.namewhat or n.what + if not name or name == "" then name = "?" end + return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + end + else + return "unknown" + end +end +function debugger.showstats(printer,threshold) + printer = printer or texio.write or print + threshold = threshold or 0 + local total, grandtotal, functions = 0, 0, 0 + printer("\n") -- ugly but ok + -- table.sort(counters) + for func, count in pairs(counters) do + if count > threshold then + local name = getname(func) + if not name:find("for generator") then + printer(format("%8i %s", count, name)) + total = total + count + end + end + grandtotal = grandtotal + count + functions = functions + 1 + end + printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +end + +-- two + +--~ local function hook() +--~ local n = getinfo(2) +--~ if n.what=="C" and not n.name then +--~ local f = tostring(debug.traceback()) +--~ local cf = counters[f] +--~ if cf == nil then +--~ counters[f] = 1 +--~ names[f] = n +--~ else +--~ counters[f] = cf + 1 +--~ end +--~ end +--~ end +--~ function debugger.showstats(printer,threshold) +--~ printer = printer or texio.write or print +--~ threshold = threshold or 0 +--~ local total, grandtotal, functions = 0, 0, 0 +--~ printer("\n") -- ugly but ok +--~ -- table.sort(counters) +--~ for func, count in pairs(counters) do +--~ if count > threshold then +--~ printer(format("%8i %s", count, func)) +--~ total = total + count +--~ end +--~ grandtotal = grandtotal + count +--~ functions = functions + 1 +--~ end +--~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) +--~ end + +-- rest + +function debugger.savestats(filename,threshold) + local f = io.open(filename,'w') + if f then + debugger.showstats(function(str) f:write(str) end,threshold) + f:close() + end +end + +function debugger.enable() + debug.sethook(hook,"c") +end + +function debugger.disable() + debug.sethook() +--~ counters[debug.getinfo(2,"f").func] = nil +end + +function debugger.tracing() + local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 + if n > 0 then + function debugger.tracing() return true end ; return true + else + function debugger.tracing() return false end ; return false + end +end + +--~ debugger.enable() + +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) +--~ print(math.sin(1*.5)) + +--~ debugger.disable() + +--~ print("") +--~ debugger.showstats() +--~ print("") +--~ debugger.showstats(print,3) + +setters = setters or { } +setters.data = setters.data or { } + +local function set(t,what,value) + local data, done = t.data, t.done + if type(what) == "string" then + what = aux.settings_to_array(what) -- inefficient but ok + end + for i=1,#what do + local w = what[i] + for d, f in next, data do + if done[d] then + -- prevent recursion due to wildcards + elseif find(d,w) then + done[d] = true + for i=1,#f do + f[i](value) + end + end + end + end +end + +local function reset(t) + for d, f in next, t.data do + for i=1,#f do + f[i](false) + end + end +end + +local function enable(t,what) + set(t,what,true) +end + +local function disable(t,what) + local data = t.data + if not what or what == "" then + t.done = { } + reset(t) + else + set(t,what,false) + end +end + +function setters.register(t,what,...) + local data = t.data + what = lower(what) + local w = data[what] + if not w then + w = { } + data[what] = w + end + for _, fnc in next, { ... } do + local typ = type(fnc) + if typ == "function" then + w[#w+1] = fnc + elseif typ == "string" then + w[#w+1] = function(value) set(t,fnc,value,nesting) end + end + end +end + +function setters.enable(t,what) + local e = t.enable + t.enable, t.done = enable, { } + enable(t,string.simpleesc(what)) + t.enable, t.done = e, { } +end + +function setters.disable(t,what) + local e = t.disable + t.disable, t.done = disable, { } + disable(t,string.simpleesc(what)) + t.disable, t.done = e, { } +end + +function setters.reset(t) + t.done = { } + reset(t) +end + +function setters.list(t) -- pattern + local list = table.sortedkeys(t.data) + local user, system = { }, { } + for l=1,#list do + local what = list[l] + if find(what,"^%*") then + system[#system+1] = what + else + user[#user+1] = what + end + end + return user, system +end + +function setters.show(t) + commands.writestatus("","") + for k,v in ipairs(setters.list(t)) do + commands.writestatus(t.name,v) + end + commands.writestatus("","") +end + +-- we could have used a bit of oo and the trackers:enable syntax but +-- there is already a lot of code around using the singluar tracker + +function setters.new(name) + local t + t = { + data = { }, + name = name, + enable = function(...) setters.enable (t,...) end, + disable = function(...) setters.disable (t,...) end, + register = function(...) setters.register(t,...) end, + list = function(...) setters.list (t,...) end, + show = function(...) setters.show (t,...) end, + } + setters.data[name] = t + return t +end + +trackers = setters.new("trackers") +directives = setters.new("directives") + +-- nice trick: we overload two of the directives related functions with variants that +-- do tracing (itself using a tracker) .. proof of concept + +local trace_directives = false local trace_directives = false trackers.register("system.directives", function(v) trace_directives = v end) + +local e = directives.enable +local d = directives.disable + +function directives.enable(...) + commands.writestatus("directives","enabling: %s",concat({...}," ")) + e(...) +end + +function directives.disable(...) + commands.writestatus("directives","disabling: %s",concat({...}," ")) + d(...) +end + +--~ -- old code: +-- +--~ trackers = trackers or { } +--~ local data, done = { }, { } +--~ local function set(what,value) +--~ if type(what) == "string" then +--~ what = aux.settings_to_array(what) -- inefficient but ok +--~ end +--~ for i=1,#what do +--~ local w = what[i] +--~ for d, f in next, data do +--~ if done[d] then +--~ -- prevent recursion due to wildcards +--~ elseif find(d,w) then +--~ done[d] = true +--~ for i=1,#f do +--~ f[i](value) +--~ end +--~ end +--~ end +--~ end +--~ end +--~ local function reset() +--~ for d, f in next, data do +--~ for i=1,#f do +--~ f[i](false) +--~ end +--~ end +--~ end +--~ local function enable(what) +--~ set(what,true) +--~ end +--~ local function disable(what) +--~ if not what or what == "" then +--~ done = { } +--~ reset() +--~ else +--~ set(what,false) +--~ end +--~ end +--~ function trackers.register(what,...) +--~ what = lower(what) +--~ local w = data[what] +--~ if not w then +--~ w = { } +--~ data[what] = w +--~ end +--~ for _, fnc in next, { ... } do +--~ local typ = type(fnc) +--~ if typ == "function" then +--~ w[#w+1] = fnc +--~ elseif typ == "string" then +--~ w[#w+1] = function(value) set(fnc,value,nesting) end +--~ end +--~ end +--~ end +--~ function trackers.enable(what) +--~ local e = trackers.enable +--~ trackers.enable, done = enable, { } +--~ enable(string.simpleesc(what)) +--~ trackers.enable, done = e, { } +--~ end +--~ function trackers.disable(what) +--~ local e = trackers.disable +--~ trackers.disable, done = disable, { } +--~ disable(string.simpleesc(what)) +--~ trackers.disable, done = e, { } +--~ end +--~ function trackers.reset() +--~ done = { } +--~ reset() +--~ end +--~ function trackers.list() -- pattern +--~ local list = table.sortedkeys(data) +--~ local user, system = { }, { } +--~ for l=1,#list do +--~ local what = list[l] +--~ if find(what,"^%*") then +--~ system[#system+1] = what +--~ else +--~ user[#user+1] = what +--~ end +--~ end +--~ return user, system +--~ end + end -- of closure @@ -2867,6 +3346,12 @@ if not modules then modules = { } end modules ['lxml-tab'] = { license = "see context related readme files" } +-- this module needs a cleanup: check latest lpeg, passing args, (sub)grammar, etc etc +-- stripping spaces from e.g. cont-en.xml saves .2 sec runtime so it's not worth the +-- trouble + +local trace_entities = false trackers.register("xml.entities", function(v) trace_entities = v end) + --[[ldx-- <p>The parser used here is inspired by the variant discussed in the lua book, but handles comment and processing instructions, has a different structure, provides @@ -2874,18 +3359,6 @@ parent access; a first version used different trickery but was less optimized to went this route. First we had a find based parser, now we have an <l n='lpeg'/> based one. The find based parser can be found in l-xml-edu.lua along with other older code.</p> -<p>Expecially the lpath code is experimental, we will support some of xpath, but -only things that make sense for us; as compensation it is possible to hook in your -own functions. Apart from preprocessing content for <l n='context'/> we also need -this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> -files.</p> - -<typing> -a/b/c /*/c -a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) -a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) -</typing> - <p>Beware, the interface may change. For instance at, ns, tg, dt may get more verbose names. Once the code is stable we will also remove some tracing and optimize the code.</p> @@ -2896,26 +3369,9 @@ xml = xml or { } --~ local xml = xml local concat, remove, insert = table.concat, table.remove, table.insert -local type, next, setmetatable = type, next, setmetatable +local type, next, setmetatable, getmetatable, tonumber = type, next, setmetatable, getmetatable, tonumber local format, lower, find = string.format, string.lower, string.find - ---[[ldx-- -<p>This module can be used stand alone but also inside <l n='mkiv'/> in -which case it hooks into the tracker code. Therefore we provide a few -functions that set the tracers.</p> ---ldx]]-- - -local trace_remap = false - -if trackers then - trackers.register("xml.remap", function(v) trace_remap = v end) -end - -function xml.settrace(str,value) - if str == "remap" then - trace_remap = value or false - end -end +local utfchar = unicode.utf8.char --[[ldx-- <p>First a hack to enable namespace resolving. A namespace is characterized by @@ -3022,25 +3478,25 @@ element.</p> </typing> --ldx]]-- -xml.strip_cm_and_dt = false -- an extra global flag, in case we have many includes - -- not just one big nested table capture (lpeg overflow) local nsremap, resolvens = xml.xmlns, xml.resolvens local stack, top, dt, at, xmlns, errorstr, entities = {}, {}, {}, {}, {}, nil, {} +local strip, cleanup, utfize, resolve = false, false, false, false -local mt = { __tostring = xml.text } +local mt = { } -function xml.check_error(top,toclose) - return "" +function initialize_mt(root) -- we will make a xml.new that then sets the mt as field + mt = { __tostring = xml.text, __index = root } end -local strip = false -local cleanup = false +function xml.setproperty(root,k,v) + getmetatable(root).__index[k] = v +end -function xml.set_text_cleanup(fnc) - cleanup = fnc +function xml.check_error(top,toclose) + return "" end local function add_attribute(namespace,tag,value) @@ -3058,6 +3514,22 @@ local function add_attribute(namespace,tag,value) end end +local function add_empty(spacing, namespace, tag) + if #spacing > 0 then + dt[#dt+1] = spacing + end + local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace + top = stack[#stack] + dt = top.dt + local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } + dt[#dt+1] = t + setmetatable(t, mt) + if at.xmlns then + remove(xmlns) + end + at = { } +end + local function add_begin(spacing, namespace, tag) if #spacing > 0 then dt[#dt+1] = spacing @@ -3083,28 +3555,12 @@ local function add_end(spacing, namespace, tag) end dt = top.dt dt[#dt+1] = toclose - dt[0] = top + -- dt[0] = top -- nasty circular reference when serializing table if toclose.at.xmlns then remove(xmlns) end end -local function add_empty(spacing, namespace, tag) - if #spacing > 0 then - dt[#dt+1] = spacing - end - local resolved = (namespace == "" and xmlns[#xmlns]) or nsremap[namespace] or namespace - top = stack[#stack] - dt = top.dt - local t = { ns=namespace or "", rn=resolved, tg=tag, at=at, dt={}, __p__ = top } - dt[#dt+1] = t - setmetatable(t, mt) - if at.xmlns then - remove(xmlns) - end - at = { } -end - local function add_text(text) if cleanup and #text > 0 then dt[#dt+1] = cleanup(text) @@ -3128,7 +3584,109 @@ local function set_message(txt) errorstr = "garbage at the end of the file: " .. gsub(txt,"([ \n\r\t]*)","") end -local P, S, R, C, V = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V +local reported_attribute_errors = { } + +local function attribute_value_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute value: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end +local function attribute_specification_error(str) + if not reported_attribute_errors[str] then + logs.report("xml","invalid attribute specification: %q",str) + reported_attribute_errors[str] = true + at._error_ = str + end + return str +end + +local dcache, hcache, acache = { }, { }, { } + +function xml.unknown_dec_entity_format(str) return format("&%s;", str) end +function xml.unknown_hex_entity_format(str) return format("&#x%s;",str) end +function xml.unknown_any_entity_format(str) return format("&%s;", str) end + +local function handle_hex_entity(str) + local h = hcache[str] + if not h then + if utfize then + local n = tonumber(str,16) + h = (n and utfchar(n)) or xml.unknown_hex_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring hex entity &#x%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting hex entity &#x%s; into %s",str,c) + end + else + if trace_entities then + logs.report("xml","found entity &#x%s;",str) + end + h = "&#" .. str .. ";" + end + hcache[str] = h + end + return h +end +local function handle_dec_entity(str) + local d = dcache[str] + if not d then + if utfize then + local n = tonumber(str) + d = (n and utfchar(n)) or xml.unknown_dec_entity_format(str) or "" + if not n then + logs.report("xml","utfize, ignoring dec entity &#%s;",str) + elseif trace_entities then + logs.report("xml","utfize, converting dec entity &#%s; into %s",str,c) + end + else + if trace_entities then + logs.report("xml","found entity &#%s;",str) + end + d = "&" .. str .. ";" + end + dcache[str] = d + end + return d +end +local function handle_any_entity(str) + if resolve then + local a = entities[str] -- per instance ! + if not a then + a = acache[str] + if not a then + if trace_entities then + logs.report("xml","ignoring entity &%s;",str) + else + -- can be defined in a global mapper and intercepted elsewhere + -- as happens in lxml-tex.lua + end + a = xml.unknown_any_entity_format(str) or "" + acache[str] = a + end + elseif trace_entities then + if not acache[str] then + logs.report("xml","converting entity &%s; into %s",str,r) + acache[str] = a + end + end + return a + else + local a = acache[str] + if not a then + if trace_entities then + logs.report("xml","found entity &%s;",str) + end + a = "&" .. str .. ";" + acache[str] = a + end + return a + end +end + +local P, S, R, C, V, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cs local space = S(' \r\n\t') local open = P('<') @@ -3138,6 +3696,8 @@ local dquote = S('"') local equal = P('=') local slash = P('/') local colon = P(':') +local semicolon = P(';') +local ampersand = P('&') local valid = R('az', 'AZ', '09') + S('_-.') local name_yes = C(valid^1) * colon * C(valid^1) local name_nop = C(P(true)) * C(valid^1) @@ -3147,15 +3707,36 @@ local utfbom = P('\000\000\254\255') + P('\255\254\000\000') + P('\255\254') + P('\254\255') + P('\239\187\191') -- no capture local spacing = C(space^0) -local justtext = C((1-open)^1) + +local entitycontent = (1-open-semicolon)^0 +local entity = ampersand/"" * ( + P("#")/"" * ( + P("x")/"" * (entitycontent/handle_hex_entity) + + (entitycontent/handle_dec_entity) + ) + (entitycontent/handle_any_entity) + ) * (semicolon/"") + +local text_unparsed = C((1-open)^1) +local text_parsed = Cs(((1-open-ampersand)^1 + entity)^1) + local somespace = space^1 local optionalspace = space^0 -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local attribute = (somespace * name * optionalspace * equal * optionalspace * value) / add_attribute -local attributes = attribute^0 +local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -- ampersand and < also invalid in value + +local whatever = space * name * optionalspace * equal +local wrongvalue = C(P(1-whatever-close)^1 + P(1-close)^1) / attribute_value_error + +local attributevalue = value + wrongvalue + +local attribute = (somespace * name * optionalspace * equal * optionalspace * attributevalue) / add_attribute +----- attributes = (attribute)^0 + +local endofattributes = slash * close + close -- recovery of flacky html +local attributes = (attribute + somespace^-1 * (((1-endofattributes)^1)/attribute_specification_error))^0 -local text = justtext / add_text +local parsedtext = text_parsed / add_text +local unparsedtext = text_unparsed / add_text local balanced = P { "[" * ((1 - S"[]") + V(1))^0 * "]" } -- taken from lpeg manual, () example local emptyelement = (spacing * open * name * attributes * optionalspace * slash * close) / add_empty @@ -3208,42 +3789,72 @@ local doctype = (spacing * begindoctype * somedoctype * enddoct -- local cdata = (lpeg.Cc("@cd@") * spacing * begincdata * somecdata * endcdata ) / add_special -- local doctype = (lpeg.Cc("@dt@") * spacing * begindoctype * somedoctype * enddoctype ) / add_special -local trailer = space^0 * (justtext/set_message)^0 +local trailer = space^0 * (text_unparsed/set_message)^0 -- comment + emptyelement + text + cdata + instruction + V("parent"), -- 6.5 seconds on 40 MB database file -- text + comment + emptyelement + cdata + instruction + V("parent"), -- 5.8 -- text + V("parent") + emptyelement + comment + cdata + instruction, -- 5.5 -local grammar = P { "preamble", +local grammar_parsed_text = P { "preamble", preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, parent = beginelement * V("children")^0 * endelement, - children = text + V("parent") + emptyelement + comment + cdata + instruction, + children = parsedtext + V("parent") + emptyelement + comment + cdata + instruction, } --- todo: xml.new + properties like entities and strip and such (store in root) +local grammar_unparsed_text = P { "preamble", + preamble = utfbom^0 * instruction^0 * (doctype + comment + instruction)^0 * V("parent") * trailer, + parent = beginelement * V("children")^0 * endelement, + children = unparsedtext + V("parent") + emptyelement + comment + cdata + instruction, +} -function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe use table met k/v (given_entities may disapear) - strip = strip_cm_and_dt or xml.strip_cm_and_dt - stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, given_entities or {} +local function xmlconvert(data, settings) + settings = settings or { } -- no_root strip_cm_and_dt given_entities parent_root error_handler + strip = settings.strip_cm_and_dt + utfize = settings.utfize_entities + resolve = settings.resolve_entities + cleanup = settings.text_cleanup + stack, top, at, xmlns, errorstr, result, entities = {}, {}, {}, {}, nil, nil, settings.entities or {} + reported_attribute_errors = { } + if settings.parent_root then + mt = getmetatable(settings.parent_root) + else + initialize_mt(top) + end stack[#stack+1] = top top.dt = { } dt = top.dt if not data or data == "" then errorstr = "empty xml file" - elseif not grammar:match(data) then - errorstr = "invalid xml file" + elseif utfize or resolve then + if grammar_parsed_text:match(data) then + errorstr = "" + else + errorstr = "invalid xml file - parsed text" + end else - errorstr = "" + if grammar_unparsed_text:match(data) then + errorstr = "" + else + errorstr = "invalid xml file - unparsed text" + end end if errorstr and errorstr ~= "" then - result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } }, error = true } + result = { dt = { { ns = "", tg = "error", dt = { errorstr }, at={}, er = true } } } setmetatable(stack, mt) - if xml.error_handler then xml.error_handler("load",errorstr) end + local error_handler = settings.error_handler + if error_handler == false then + -- no error message + else + error_handler = error_handler or xml.error_handler + if error_handler then + xml.error_handler("load",errorstr) + end + end else result = stack[1] end - if not no_root then - result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities } + if not settings.no_root then + result = { special = true, ns = "", tg = '@rt@', dt = result.dt, at={}, entities = entities, settings = settings } setmetatable(result, mt) local rdt = result.dt for k=1,#rdt do @@ -3254,9 +3865,14 @@ function xml.convert(data, no_root, strip_cm_and_dt, given_entities) -- maybe us end end end + if errorstr and errorstr ~= "" then + result.error = true + end return result end +xml.convert = xmlconvert + --[[ldx-- <p>Packaging data in an xml like table is done with the following function. Maybe it will go away (when not used).</p> @@ -3289,16 +3905,16 @@ function xml.load(filename) if type(filename) == "string" then local f = io.open(filename,'r') if f then - local root = xml.convert(f:read("*all")) + local root = xmlconvert(f:read("*all")) f:close() return root else - return xml.convert("") + return xmlconvert("") end elseif filename then -- filehandle - return xml.convert(filename:read("*all")) + return xmlconvert(filename:read("*all")) else - return xml.convert("") + return xmlconvert("") end end @@ -3307,9 +3923,11 @@ end valid trees, which is what the next function does.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.toxml(data) if type(data) == "string" then - local root = { xml.convert(data,true) } + local root = { xmlconvert(data,no_root) } return (#root > 1 and root) or root[1] else return data @@ -3354,217 +3972,305 @@ alternative.</p> -- todo: add <?xml version='1.0' standalone='yes'?> when not present -local fallbackhandle = (tex and tex.sprint) or io.write - -local function serialize(e, handle, textconverter, attributeconverter, specialconverter, nocommands) - if not e then - return - elseif not nocommands then - local ec = e.command - if ec ~= nil then -- we can have all kind of types - if e.special then - local etg, edt = e.tg, e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) - return - else - -- no need to handle any further - end - end - end - local xc = xml.command - if xc then - xc(e,ec) - return +function xml.checkbom(root) -- can be made faster + if root.ri then + local dt, found = root.dt, false + for k=1,#dt do + local v = dt[k] + if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then + found = true + break end end + if not found then + insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) + insert(dt, 2, "\n" ) + end end - handle = handle or fallbackhandle - local etg = e.tg - if etg then - if e.special then - local edt = e.dt - local spc = specialconverter and specialconverter[etg] - if spc then - local result = spc(edt[1]) - if result then - handle(result) +end + +--[[ldx-- +<p>At the cost of some 25% runtime overhead you can first convert the tree to a string +and then handle the lot.</p> +--ldx]]-- + +-- new experimental reorganized serialize + +local function verbose_element(e,handlers) + local handle = handlers.handle + local serialize = handlers.serialize + local ens, etg, eat, edt, ern = e.ns, e.tg, e.at, e.dt, e.rn + local ats = eat and next(eat) and { } + if ats then + for k,v in next, eat do + ats[#ats+1] = format('%s=%q',k,v) + end + end + if ern and trace_remap and ern ~= ens then + ens = ern + end + if ens ~= "" then + if edt and #edt > 0 then + if ats then + handle("<",ens,":",etg," ",concat(ats," "),">") + else + handle("<",ens,":",etg,">") + end + for i=1,#edt do + local e = edt[i] + if type(e) == "string" then + handle(e) else - -- no need to handle any further + serialize(e,handlers) end - elseif etg == "@pi@" then - -- handle(format("<?%s?>",edt[1])) - handle("<?" .. edt[1] .. "?>") - elseif etg == "@cm@" then - -- handle(format("<!--%s-->",edt[1])) - handle("<!--" .. edt[1] .. "-->") - elseif etg == "@cd@" then - -- handle(format("<![CDATA[%s]]>",edt[1])) - handle("<![CDATA[" .. edt[1] .. "]]>") - elseif etg == "@dt@" then - -- handle(format("<!DOCTYPE %s>",edt[1])) - handle("<!DOCTYPE " .. edt[1] .. ">") - elseif etg == "@rt@" then - serialize(edt,handle,textconverter,attributeconverter,specialconverter,nocommands) end + handle("</",ens,":",etg,">") else - local ens, eat, edt, ern = e.ns, e.at, e.dt, e.rn - local ats = eat and next(eat) and { } -- type test maybe faster if ats then - if attributeconverter then - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,attributeconverter(v)) - end - else - for k,v in next, eat do - ats[#ats+1] = format('%s=%q',k,v) - end - end + handle("<",ens,":",etg," ",concat(ats," "),"/>") + else + handle("<",ens,":",etg,"/>") end - if ern and trace_remap and ern ~= ens then - ens = ern + end + else + if edt and #edt > 0 then + if ats then + handle("<",etg," ",concat(ats," "),">") + else + handle("<",etg,">") end - if ens ~= "" then - if edt and #edt > 0 then - if ats then - -- handle(format("<%s:%s %s>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s:%s>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. ">") - end - for i=1,#edt do - local e = edt[i] - if type(e) == "string" then - if textconverter then - handle(textconverter(e)) - else - handle(e) - end - else - serialize(e,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s:%s>",ens,etg)) - handle("</" .. ens .. ":" .. etg .. ">") + for i=1,#edt do + local ei = edt[i] + if type(ei) == "string" then + handle(ei) else - if ats then - -- handle(format("<%s:%s %s/>",ens,etg,concat(ats," "))) - handle("<" .. ens .. ":" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s:%s/>",ens,etg)) - handle("<" .. ens .. ":" .. etg .. "/>") - end + serialize(ei,handlers) end + end + handle("</",etg,">") + else + if ats then + handle("<",etg," ",concat(ats," "),"/>") else - if edt and #edt > 0 then - if ats then - -- handle(format("<%s %s>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. ">") - else - -- handle(format("<%s>",etg)) - handle("<" .. etg .. ">") - end - for i=1,#edt do - local ei = edt[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end - end - -- handle(format("</%s>",etg)) - handle("</" .. etg .. ">") - else - if ats then - -- handle(format("<%s %s/>",etg,concat(ats," "))) - handle("<" .. etg .. " " .. concat(ats," ") .. "/>") - else - -- handle(format("<%s/>",etg)) - handle("<" .. etg .. "/>") - end - end + handle("<",etg,"/>") end end - elseif type(e) == "string" then - if textconverter then - handle(textconverter(e)) + end +end + +local function verbose_pi(e,handlers) + handlers.handle("<?",e.dt[1],"?>") +end + +local function verbose_comment(e,handlers) + handlers.handle("<!--",e.dt[1],"-->") +end + +local function verbose_cdata(e,handlers) + handlers.handle("<![CDATA[", e.dt[1],"]]>") +end + +local function verbose_doctype(e,handlers) + handlers.handle("<!DOCTYPE ",e.dt[1],">") +end + +local function verbose_root(e,handlers) + handlers.serialize(e.dt,handlers) +end + +local function verbose_text(e,handlers) + handlers.handle(e) +end + +local function verbose_document(e,handlers) + local serialize = handlers.serialize + local functions = handlers.functions + for i=1,#e do + local ei = e[i] + if type(ei) == "string" then + functions["@tx@"](ei,handlers) else - handle(e) + serialize(ei,handlers) end - else - for i=1,#e do - local ei = e[i] - if type(ei) == "string" then - if textconverter then - handle(textconverter(ei)) - else - handle(ei) - end - else - serialize(ei,handle,textconverter,attributeconverter,specialconverter,nocommands) - end + end +end + +local function serialize(e,handlers,...) + local initialize = handlers.initialize + local finalize = handlers.finalize + local functions = handlers.functions + if initialize then + local state = initialize(...) + if not state == true then + return state end end + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end + if finalize then + return finalize() + end end -xml.serialize = serialize +local function xserialize(e,handlers) + local functions = handlers.functions + local etg = e.tg + if etg then + (functions[etg] or functions["@el@"])(e,handlers) + -- elseif type(e) == "string" then + -- functions["@tx@"](e,handlers) + else + functions["@dc@"](e,handlers) + end +end -function xml.checkbom(root) -- can be made faster - if root.ri then - local dt, found = root.dt, false - for k=1,#dt do - local v = dt[k] - if type(v) == "table" and v.special and v.tg == "@pi" and find(v.dt,"xml.*version=") then - found = true - break +local handlers = { } + +local function newhandlers(settings) + local t = table.copy(handlers.verbose or { }) -- merge + if settings then + for k,v in next, settings do + if type(v) == "table" then + tk = t[k] if not tk then tk = { } t[k] = tk end + for kk,vv in next, v do + tk[kk] = vv + end + else + t[k] = v end end - if not found then - insert(dt, 1, { special=true, ns="", tg="@pi@", dt = { "xml version='1.0' standalone='yes'"} } ) - insert(dt, 2, "\n" ) + if settings.name then + handlers[settings.name] = t end end + return t +end + +local nofunction = function() end + +function xml.sethandlersfunction(handler,name,fnc) + handler.functions[name] = fnc or nofunction end +function xml.gethandlersfunction(handler,name) + return handler.functions[name] +end + +function xml.gethandlers(name) + return handlers[name] +end + +newhandlers { + name = "verbose", + initialize = false, -- faster than nil and mt lookup + finalize = false, -- faster than nil and mt lookup + serialize = xserialize, + handle = print, + functions = { + ["@dc@"] = verbose_document, + ["@dt@"] = verbose_doctype, + ["@rt@"] = verbose_root, + ["@el@"] = verbose_element, + ["@pi@"] = verbose_pi, + ["@cm@"] = verbose_comment, + ["@cd@"] = verbose_cdata, + ["@tx@"] = verbose_text, + } +} + --[[ldx-- -<p>At the cost of some 25% runtime overhead you can first convert the tree to a string -and then handle the lot.</p> +<p>How you deal with saving data depends on your preferences. For a 40 MB database +file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> + +<lines> +1.3 : load data from file to string +6.1 : convert string into tree +5.3 : saving in file using xmlsave +6.8 : converting to string using xml.tostring +3.6 : saving converted string in file +</lines> + +<p>Beware, these were timing with the old routine but measurements will not be that +much different I guess.</p> --ldx]]-- -function xml.tostring(root) -- 25% overhead due to collecting +-- maybe this will move to lxml-xml + +local result + +local xmlfilehandler = newhandlers { + name = "file", + initialize = function(name) result = io.open(name,"wb") return result end, + finalize = function() result:close() return true end, + handle = function(...) result:write(...) end, +} + +-- no checking on writeability here but not faster either +-- +-- local xmlfilehandler = newhandlers { +-- initialize = function(name) io.output(name,"wb") return true end, +-- finalize = function() io.close() return true end, +-- handle = io.write, +-- } + + +function xml.save(root,name) + serialize(root,xmlfilehandler,name) +end + +local result + +local xmlstringhandler = newhandlers { + name = "string", + initialize = function() result = { } return result end, + finalize = function() return concat(result) end, + handle = function(...) result[#result+1] = concat { ... } end +} + +local function xmltostring(root) -- 25% overhead due to collecting if root then if type(root) == 'string' then return root - elseif next(root) then -- next is faster than type (and >0 test) - local result = { } - serialize(root,function(s) result[#result+1] = s end) -- brrr, slow (direct printing is faster) - return concat(result,"") + else -- if next(root) then -- next is faster than type (and >0 test) + return serialize(root,xmlstringhandler) or "" end end return "" end +local function xmltext(root) -- inline + return (root and xmltostring(root)) or "" +end + +function initialize_mt(root) + mt = { __tostring = xmltext, __index = root } +end + +xml.defaulthandlers = handlers +xml.newhandlers = newhandlers +xml.serialize = serialize +xml.tostring = xmltostring +xml.text = xmltext + --[[ldx-- <p>The next function operated on the content only and needs a handle function that accepts a string.</p> --ldx]]-- -function xml.string(e,handle) +local function xmlstring(e,handle) if not handle or (e.special and e.tg ~= "@rt@") then -- nothing elseif e.tg then local edt = e.dt if edt then for i=1,#edt do - xml.string(edt[i],handle) + xmlstring(edt[i],handle) end end else @@ -3572,33 +4278,16 @@ function xml.string(e,handle) end end ---[[ldx-- -<p>How you deal with saving data depends on your preferences. For a 40 MB database -file the timing on a 2.3 Core Duo are as follows (time in seconds):</p> - -<lines> -1.3 : load data from file to string -6.1 : convert string into tree -5.3 : saving in file using xmlsave -6.8 : converting to string using xml.tostring -3.6 : saving converted string in file -</lines> - -<p>The save function is given below.</p> ---ldx]]-- - -function xml.save(root,name) - local f = io.open(name,"w") - if f then - xml.serialize(root,function(s) f:write(s) end) - f:close() - end -end +xml.string = xmlstring --[[ldx-- <p>A few helpers:</p> --ldx]]-- +function xml.parent(root) + return root.__p__ +end + function xml.body(root) return (root.ri and root.dt[root.ri]) or root end @@ -3611,34 +4300,19 @@ function xml.content(root) -- bugged return (root and root.dt and xml.tostring(root.dt)) or "" end -function xml.isempty(root, pattern) - if pattern == "" or pattern == "*" then - pattern = nil - end - if pattern then - -- todo - return false - else - return not root or not root.dt or #root.dt == 0 or root.dt == "" - end -end - --[[ldx-- <p>The next helper erases an element but keeps the table as it is, and since empty strings are not serialized (effectively) it does not harm. Copying the table would take more time. Usage:</p> - -<typing> -dt[k] = xml.empty() or xml.empty(dt,k) -</typing> --ldx]]-- -function xml.empty(dt,k) - if dt and k then - dt[k] = "" - return dt[k] - else - return "" +function xml.erase(dt,k) + if dt then + if k then + dt[k] = "" + else for k=1,#dt do + dt[1] = { "" } + end end end end @@ -3672,96 +4346,403 @@ if not modules then modules = { } end modules ['lxml-pth'] = { license = "see context related readme files" } +-- e.ni is only valid after a filter run + local concat, remove, insert = table.concat, table.remove, table.insert local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, lower, gmatch, gsub, find, rep = string.format, string.lower, string.gmatch, string.gsub, string.find, string.rep +local format, upper, lower, gmatch, gsub, find, rep = string.format, string.upper, string.lower, string.gmatch, string.gsub, string.find, string.rep --[[ldx-- <p>This module can be used stand alone but also inside <l n='mkiv'/> in which case it hooks into the tracker code. Therefore we provide a few functions that set the tracers. Here we overload a previously defined function.</p> +<p>If I can get in the mood I will make a variant that is XSLT compliant +but I wonder if it makes sense.</P> --ldx]]-- -local trace_lpath = false - -if trackers then - trackers.register("xml.lpath", function(v) trace_lpath = v end) -end +--[[ldx-- +<p>Expecially the lpath code is experimental, we will support some of xpath, but +only things that make sense for us; as compensation it is possible to hook in your +own functions. Apart from preprocessing content for <l n='context'/> we also need +this module for process management, like handling <l n='ctx'/> and <l n='rlx'/> +files.</p> -local settrace = xml.settrace -- lxml-tab +<typing> +a/b/c /*/c +a/b/c/first() a/b/c/last() a/b/c/index(n) a/b/c/index(-n) +a/b/c/text() a/b/c/text(1) a/b/c/text(-1) a/b/c/text(n) +</typing> +--ldx]]-- -function xml.settrace(str,value) - if str == "lpath" then - trace_lpath = value or false - else - settrace(str,value) -- lxml-tab - end -end +local trace_lpath = false if trackers then trackers.register("xml.path", function(v) trace_lpath = v end) end +local trace_lparse = false if trackers then trackers.register("xml.parse", function(v) trace_lparse = v end) end +local trace_lprofile = false if trackers then trackers.register("xml.profile", function(v) trace_lpath = v trace_lparse = v trace_lprofile = v end) end --[[ldx-- -<p>We've now arrived at an intersting part: accessing the tree using a subset +<p>We've now arrived at an interesting part: accessing the tree using a subset of <l n='xpath'/> and since we're not compatible we call it <l n='lpath'/>. We will explain more about its usage in other documents.</p> --ldx]]-- -local lpathcalls = 0 -- statistics -local lpathcached = 0 -- statistics +local lpathcalls = 0 function xml.lpathcalls () return lpathcalls end +local lpathcached = 0 function xml.lpathcached() return lpathcached end -xml.functions = xml.functions or { } -xml.expressions = xml.expressions or { } +xml.functions = xml.functions or { } -- internal +xml.expressions = xml.expressions or { } -- in expressions +xml.finalizers = xml.finalizers or { } -- fast do-with ... (with return value other than collection) +xml.specialhandler = xml.specialhandler or { } local functions = xml.functions local expressions = xml.expressions +local finalizers = xml.finalizers -local actions = { - [10] = "stay", - [11] = "parent", - [12] = "subtree root", - [13] = "document root", - [14] = "any", - [15] = "many", - [16] = "initial", - [20] = "match", - [21] = "match one of", - [22] = "match and attribute eq", - [23] = "match and attribute ne", - [24] = "match one of and attribute eq", - [25] = "match one of and attribute ne", - [27] = "has attribute", - [28] = "has value", - [29] = "fast match", - [30] = "select", - [31] = "expression", - [40] = "processing instruction", -} +finalizers.xml = finalizers.xml or { } +finalizers.tex = finalizers.tex or { } + +local function fallback (t, name) + local fn = finalizers[name] + if fn then + t[name] = fn + else + logs.report("xml","unknown sub finalizer '%s'",tostring(name)) + fn = function() end + end + return fn +end + +setmetatable(finalizers.xml, { __index = fallback }) +setmetatable(finalizers.tex, { __index = fallback }) + +xml.defaultprotocol = "xml" + +-- as xsl does not follow xpath completely here we will also +-- be more liberal especially with regards to the use of | and +-- the rootpath: +-- +-- test : all 'test' under current +-- /test : 'test' relative to current +-- a|b|c : set of names +-- (a|b|c) : idem +-- ! : not +-- +-- after all, we're not doing transformations but filtering. in +-- addition we provide filter functions (last bit) +-- +-- todo: optimizer +-- +-- .. : parent +-- * : all kids +-- / : anchor here +-- // : /**/ +-- ** : all in between +-- +-- so far we had (more practical as we don't transform) +-- +-- {/test} : kids 'test' under current node +-- {test} : any kid with tag 'test' +-- {//test} : same as above --- a rather dumb lpeg +-- evaluator (needs to be redone, for the moment copied) -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- todo: apply_axis(list,notable) and collection vs single --- instead of using functions we just parse a few names which saves a call --- later on +local apply_axis = { } -local lp_position = P("position()") / "ps" -local lp_index = P("index()") / "id" -local lp_text = P("text()") / "tx" -local lp_name = P("name()") / "(ns~='' and ns..':'..tg)" -- "((rt.ns~='' and rt.ns..':'..rt.tg) or '')" -local lp_tag = P("tag()") / "tg" -- (rt.tg or '') -local lp_ns = P("ns()") / "ns" -- (rt.ns or '') -local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") -local lp_doequal = P("=") / "==" -local lp_attribute = P("@") / "" * Cc("(at['") * R("az","AZ","--","__")^1 * Cc("'] or '')") +apply_axis['root'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + local rt = ll + while ll do + ll = ll.__p__ + if ll then + rt = ll + end + end + collected[#collected+1] = rt + end + return collected +end + +apply_axis['self'] = function(list) +--~ local collected = { } +--~ for l=1,#list do +--~ collected[#collected+1] = list[l] +--~ end +--~ return collected + return list +end -local lp_lua_function = C(R("az","AZ","--","__")^1 * (P(".") * R("az","AZ","--","__")^1)^1) * P("(") / function(t) -- todo: better . handling +apply_axis['child'] = function(list) + local collected = { } + for l=1,#list do + local dt = list[l].dt + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + end + end + end + return collected +end + +local function collect(list,collected) + local dt = list.dt + if dt then + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + collect(dk,collected) + end + end + end +end +apply_axis['descendant'] = function(list) + local collected = { } + for l=1,#list do + collect(list[l],collected) + end + return collected +end + +local function collect(list,collected) + local dt = list.dt + if dt then + for k=1,#dt do + local dk = dt[k] + if dk.tg then + collected[#collected+1] = dk + dk.ni = k -- refresh + collect(dk,collected) + end + end + end +end +apply_axis['descendant-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] +if ll.special ~= true then -- catch double root + collected[#collected+1] = ll +end + collect(ll,collected) + end + return collected +end + +apply_axis['ancestor'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end + +apply_axis['ancestor-or-self'] = function(list) + local collected = { } + for l=1,#list do + local ll = list[l] + collected[#collected+1] = ll + while ll do + ll = ll.__p__ + if ll then + collected[#collected+1] = ll + end + end + end + return collected +end + +apply_axis['parent'] = function(list) + local collected = { } + for l=1,#list do + local pl = list[l].__p__ + if pl then + collected[#collected+1] = pl + end + end + return collected +end + +apply_axis['attribute'] = function(list) + return { } +end + +apply_axis['following'] = function(list) + return { } +end + +apply_axis['following-sibling'] = function(list) + return { } +end + +apply_axis['namespace'] = function(list) + return { } +end + +apply_axis['preceding'] = function(list) + return { } +end + +apply_axis['preceding-sibling'] = function(list) + return { } +end + +apply_axis['auto-descendant-or-self'] = apply_axis['descendant-or-self'] +apply_axis['auto-descendant'] = apply_axis['descendant'] +apply_axis['auto-child'] = apply_axis['child'] +apply_axis['auto-self'] = apply_axis['self'] +apply_axis['initial-child'] = apply_axis['child'] + +local function apply_nodes(list,directive,nodes) + -- todo: nodes[1] etc ... negated node name in set ... when needed + -- ... currently ignored + local maxn = #nodes + if maxn == 3 then --optimized loop + local nns, ntg = nodes[2], nodes[3] + if not nns and not ntg then -- wildcard + if directive then + return list + else + return { } + end + else + local collected = { } + if not nns then -- only check tag + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + if directive then + if ntg == ltg then + collected[#collected+1] = ll + end + elseif ntg ~= ltg then + collected[#collected+1] = ll + end + end + end + elseif not ntg then -- only check namespace + for l=1,#list do + local ll = list[l] + local lns = ll.rn or ll.ns + if lns then + if directive then + if lns == nns then + collected[#collected+1] = ll + end + elseif lns ~= nns then + collected[#collected+1] = ll + end + end + end + else -- check both + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = ltg == ntg and lns == nns + if directive then + if ok then + collected[#collected+1] = ll + end + elseif not ok then + collected[#collected+1] = ll + end + end + end + end + return collected + end + else + local collected = { } + for l=1,#list do + local ll = list[l] + local ltg = ll.tg + if ltg then + local lns = ll.rn or ll.ns + local ok = false + for n=1,maxn,3 do + local nns, ntg = nodes[n+1], nodes[n+2] + ok = (not ntg or ltg == ntg) and (not nns or lns == nns) + if ok then + break + end + end + if directive then + if ok then + collected[#collected+1] = ll + end + elseif not ok then + collected[#collected+1] = ll + end + end + end + return collected + end +end + +local function apply_expression(list,expression,order) + local collected = { } + for l=1,#list do + local ll = list[l] + if expression(list,ll,l,order) then -- nasty, alleen valid als n=1 + collected[#collected+1] = ll + end + end + return collected +end + +local P, V, C, Cs, Cc, Ct, R, S, Cg, Cb = lpeg.P, lpeg.V, lpeg.C, lpeg.Cs, lpeg.Cc, lpeg.Ct, lpeg.R, lpeg.S, lpeg.Cg, lpeg.Cb + +local spaces = S(" \n\r\t\f")^0 + +local lp_space = S(" \n\r\t\f") +local lp_any = P(1) + +local lp_noequal = P("!=") / "~=" + P("<=") + P(">=") + P("==") +local lp_doequal = P("=") / "==" +local lp_or = P("|") / " or " +local lp_and = P("&") / " and " + +local lp_builtin = P ( + P("first") / "1" + + P("last") / "#list" + + P("position") / "l" + + P("rootposition") / "order" + + P("index") / "ll.ni" + + P("text") / "(ll.dt[1] or '')" + + P("name") / "(ll.ns~='' and ll.ns..':'..ll.tg)" + + P("tag") / "ll.tg" + + P("ns") / "ll.ns" + ) * ((spaces * P("(") * spaces * P(")"))/"") + +local lp_attribute = (P("@") + P("attribute::")) / "" * Cc("ll.at['") * R("az","AZ","--","__")^1 * Cc("']") +local lp_fastpos = ((R("09","--","++")^1 * P(-1)) / function(s) return "l==" .. s end) + +local lp_reserved = C("and") + C("or") + C("not") + C("div") + C("mod") + C("true") + C("false") + +local lp_lua_function = C(R("az","AZ","__")^1 * (P(".") * R("az","AZ","__")^1)^1) * ("(") / function(t) -- todo: better . handling return t .. "(" end -local lp_function = C(R("az","AZ","--","__")^1) * P("(") / function(t) -- todo: better . handling +local lp_function = C(R("az","AZ","__")^1) * P("(") / function(t) -- todo: better . handling if expressions[t] then - return "expressions." .. t .. "(" + return "expr." .. t .. "(" else - return "expressions.error(" + return "expr.error(" end end @@ -3771,337 +4752,527 @@ local noparent = 1 - (lparent+rparent) local nested = lpeg.P{lparent * (noparent + lpeg.V(1))^0 * rparent} local value = lpeg.P(lparent * lpeg.C((noparent + nested)^0) * rparent) -- lpeg.P{"("*C(((1-S("()"))+V(1))^0)*")"} --- if we use a dedicated namespace then we don't need to pass rt and k +local lp_child = Cc("expr.child(e,'") * R("az","AZ","--","__")^1 * Cc("')") +local lp_string = Cc("'") * R("az","AZ","--","__")^1 * Cc("'") +local lp_content= (P("'") * (1-P("'"))^0 * P("'") + P('"') * (1-P('"'))^0 * P('"')) + +local cleaner -local lp_special = (C(P("name")+P("text")+P("tag"))) * value / function(t,s) +local lp_special = (C(P("name")+P("text")+P("tag")+P("count")+P("child"))) * value / function(t,s) if expressions[t] then - if s then - return "expressions." .. t .. "(r,k," .. s ..")" + s = s and s ~= "" and cleaner:match(s) + if s and s ~= "" then + return "expr." .. t .. "(e," .. s ..")" else - return "expressions." .. t .. "(r,k)" + return "expr." .. t .. "(e)" end else - return "expressions.error(" .. t .. ")" + return "expr.error(" .. t .. ")" end end -local converter = lpeg.Cs ( ( - lp_position + - lp_index + - lp_text + lp_name + -- fast one +local content = + lp_builtin + + lp_attribute + lp_special + lp_noequal + lp_doequal + - lp_attribute + - lp_lua_function + - lp_function + + lp_or + lp_and + + lp_reserved + + lp_lua_function + lp_function + + lp_content + -- too fragile + lp_child + + lp_any + +local converter = lpeg.Cs ( + lp_fastpos + (lpeg.P { lparent * (lpeg.V(1))^0 * rparent + content } )^0 +) + +cleaner = lpeg.Cs ( ( +--~ lp_fastpos + + lp_reserved + + lp_string + 1 )^1 ) --- expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1 +--~ expr -local template = [[ - return function(expressions,r,d,k,e,dt,ns,tg,id,ps) - local at, tx = e.at or { }, dt[1] or "" +local template_e = [[ + local expr = xml.expressions + return function(list,ll,l,root) return %s end ]] -local function make_expression(str) - str = converter:match(str) - return str, loadstring(format(template,str))() -end - -local map = { } - -local space = S(' \r\n\t') -local squote = S("'") -local dquote = S('"') -local lparent = P('(') -local rparent = P(')') -local atsign = P('@') -local lbracket = P('[') -local rbracket = P(']') -local exclam = P('!') -local period = P('.') -local eq = P('==') + P('=') -local ne = P('<>') + P('!=') -local star = P('*') -local slash = P('/') -local colon = P(':') -local bar = P('|') -local hat = P('^') -local valid = R('az', 'AZ', '09') + S('_-') -local name_yes = C(valid^1 + star) * colon * C(valid^1 + star) -- permits ns:* *:tg *:* -local name_nop = Cc("*") * C(valid^1) -local name = name_yes + name_nop -local number = C((S('+-')^0 * R('09')^1)) / tonumber -local names = (bar^0 * name)^1 -local morenames = name * (bar^0 * name)^1 -local instructiontag = P('pi::') -local spacing = C(space^0) -local somespace = space^1 -local optionalspace = space^0 -local text = C(valid^0) -local value = (squote * C((1 - squote)^0) * squote) + (dquote * C((1 - dquote)^0) * dquote) -local empty = 1-slash - -local is_eq = lbracket * atsign * name * eq * value * rbracket -local is_ne = lbracket * atsign * name * ne * value * rbracket -local is_attribute = lbracket * atsign * name * rbracket -local is_value = lbracket * value * rbracket -local is_number = lbracket * number * rbracket - -local nobracket = 1-(lbracket+rbracket) -- must be improved -local is_expression = lbracket * C(((C(nobracket^1))/make_expression)) * rbracket - -local is_expression = lbracket * (C(nobracket^1))/make_expression * rbracket - -local is_one = name -local is_none = exclam * name -local is_one_of = ((lparent * names * rparent) + morenames) -local is_none_of = exclam * ((lparent * names * rparent) + morenames) - -local stay = (period ) -local parent = (period * period ) / function( ) map[#map+1] = { 11 } end -local subtreeroot = (slash + hat ) / function( ) map[#map+1] = { 12 } end -local documentroot = (hat * hat ) / function( ) map[#map+1] = { 13 } end -local any = (star ) / function( ) map[#map+1] = { 14 } end -local many = (star * star ) / function( ) map[#map+1] = { 15 } end -local initial = (hat * hat * hat ) / function( ) map[#map+1] = { 16 } end - -local match = (is_one ) / function(...) map[#map+1] = { 20, true , ... } end -local match_one_of = (is_one_of ) / function(...) map[#map+1] = { 21, true , ... } end -local dont_match = (is_none ) / function(...) map[#map+1] = { 20, false, ... } end -local dont_match_one_of = (is_none_of ) / function(...) map[#map+1] = { 21, false, ... } end - -local match_and_eq = (is_one * is_eq ) / function(...) map[#map+1] = { 22, true , ... } end -local match_and_ne = (is_one * is_ne ) / function(...) map[#map+1] = { 23, true , ... } end -local dont_match_and_eq = (is_none * is_eq ) / function(...) map[#map+1] = { 22, false, ... } end -local dont_match_and_ne = (is_none * is_ne ) / function(...) map[#map+1] = { 23, false, ... } end - -local match_one_of_and_eq = (is_one_of * is_eq ) / function(...) map[#map+1] = { 24, true , ... } end -local match_one_of_and_ne = (is_one_of * is_ne ) / function(...) map[#map+1] = { 25, true , ... } end -local dont_match_one_of_and_eq = (is_none_of * is_eq ) / function(...) map[#map+1] = { 24, false, ... } end -local dont_match_one_of_and_ne = (is_none_of * is_ne ) / function(...) map[#map+1] = { 25, false, ... } end - -local has_attribute = (is_one * is_attribute) / function(...) map[#map+1] = { 27, true , ... } end -local has_value = (is_one * is_value ) / function(...) map[#map+1] = { 28, true , ... } end -local dont_has_attribute = (is_none * is_attribute) / function(...) map[#map+1] = { 27, false, ... } end -local dont_has_value = (is_none * is_value ) / function(...) map[#map+1] = { 28, false, ... } end -local position = (is_one * is_number ) / function(...) map[#map+1] = { 30, true, ... } end -local dont_position = (is_none * is_number ) / function(...) map[#map+1] = { 30, false, ... } end - -local expression = (is_one * is_expression)/ function(...) map[#map+1] = { 31, true, ... } end -local dont_expression = (is_none * is_expression)/ function(...) map[#map+1] = { 31, false, ... } end - -local self_expression = ( is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, true, "*", "*", ... } end -local dont_self_expression = (exclam * is_expression) / function(...) if #map == 0 then map[#map+1] = { 11 } end - map[#map+1] = { 31, false, "*", "*", ... } end - -local instruction = (instructiontag * text ) / function(...) map[#map+1] = { 40, ... } end -local nothing = (empty ) / function( ) map[#map+1] = { 15 } end -- 15 ? -local crap = (1-slash)^1 - --- a few ugly goodies: - -local docroottag = P('^^') / function( ) map[#map+1] = { 12 } end -local subroottag = P('^') / function( ) map[#map+1] = { 13 } end -local roottag = P('root::') / function( ) map[#map+1] = { 12 } end -local parenttag = P('parent::') / function( ) map[#map+1] = { 11 } end -local childtag = P('child::') -local selftag = P('self::') - --- there will be more and order will be optimized - -local selector = ( - instruction + --- many + any + -- brrr, not here ! - parent + stay + - dont_position + position + - dont_match_one_of_and_eq + dont_match_one_of_and_ne + - match_one_of_and_eq + match_one_of_and_ne + - dont_match_and_eq + dont_match_and_ne + - match_and_eq + match_and_ne + - dont_expression + expression + - dont_self_expression + self_expression + - has_attribute + has_value + - dont_match_one_of + match_one_of + - dont_match + match + - many + any + - crap + empty -) +local template_f_y = [[ + local finalizer = xml.finalizers['%s']['%s'] + return function(collection) + return finalizer(collection,%s) + end +]] -local grammar = P { "startup", - startup = (initial + documentroot + subtreeroot + roottag + docroottag + subroottag)^0 * V("followup"), - followup = ((slash + parenttag + childtag + selftag)^0 * selector)^1, -} +local template_f_n = [[ + return xml.finalizers['%s']['%s'] +]] -local function compose(str) - if not str or str == "" then - -- wildcard - return true - elseif str == '/' then - -- root - return false +-- + +local function errorrunner_e(str,cnv) + logs.report("lpath","error in expression: %s => %s",str,cnv) + return false +end +local function errorrunner_f(str,arg) + logs.report("lpath","error in finalizer: %s(%s)",str,arg or "") + return false +end + +local function register_nodes(nodetest,nodes) + return { kind = "nodes", nodetest = nodetest, nodes = nodes } +end + +local function register_expression(expression) + local converted = converter:match(expression) + local runner = loadstring(format(template_e,converted)) + runner = (runner and runner()) or function() errorrunner_e(expression,converted) end + return { kind = "expression", expression = expression, converted = converted, evaluator = runner } +end + +local function register_finalizer(protocol,name,arguments) + local runner + if arguments and arguments ~= "" then + runner = loadstring(format(template_f_y,protocol or xml.defaultprotocol,name,arguments)) else - map = { } - grammar:match(str) - if #map == 0 then - return true - else - local m = map[1][1] - if #map == 1 then - if m == 14 or m == 15 then - -- wildcard - return true - elseif m == 12 then - -- root - return false - end - elseif #map == 2 and m == 12 and map[2][1] == 20 then - -- return { { 29, map[2][2], map[2][3], map[2][4], map[2][5] } } - map[2][1] = 29 - return { map[2] } - end - if m ~= 11 and m ~= 12 and m ~= 13 and m ~= 14 and m ~= 15 and m ~= 16 then - insert(map, 1, { 16 }) - end - -- print(gsub(table.serialize(map),"[ \n]+"," ")) - return map - end + runner = loadstring(format(template_f_n,protocol or xml.defaultprotocol,name)) end + runner = (runner and runner()) or function() errorrunner_f(name,arguments) end + return { kind = "finalizer", name = name, arguments = arguments, finalizer = runner } end +local expression = P { "ex", + ex = "[" * C((V("sq") + V("dq") + (1 - S("[]")) + V("ex"))^0) * "]", + sq = "'" * (1 - S("'"))^0 * "'", + dq = '"' * (1 - S('"'))^0 * '"', +} + +local arguments = P { "ar", + ar = "(" * Cs((V("sq") + V("dq") + V("nq") + P(1-P(")")))^0) * ")", + nq = ((1 - S("),'\""))^1) / function(s) return format("%q",s) end, + sq = P("'") * (1 - P("'"))^0 * P("'"), + dq = P('"') * (1 - P('"'))^0 * P('"'), +} + +-- todo: better arg parser + +local register_self = { kind = "axis", axis = "self" } -- , apply = apply_axis["self"] } +local register_parent = { kind = "axis", axis = "parent" } -- , apply = apply_axis["parent"] } +local register_descendant = { kind = "axis", axis = "descendant" } -- , apply = apply_axis["descendant"] } +local register_child = { kind = "axis", axis = "child" } -- , apply = apply_axis["child"] } +local register_descendant_or_self = { kind = "axis", axis = "descendant-or-self" } -- , apply = apply_axis["descendant-or-self"] } +local register_root = { kind = "axis", axis = "root" } -- , apply = apply_axis["root"] } +local register_ancestor = { kind = "axis", axis = "ancestor" } -- , apply = apply_axis["ancestor"] } +local register_ancestor_or_self = { kind = "axis", axis = "ancestor-or-self" } -- , apply = apply_axis["ancestor-or-self"] } +local register_attribute = { kind = "axis", axis = "attribute" } -- , apply = apply_axis["attribute"] } +local register_namespace = { kind = "axis", axis = "namespace" } -- , apply = apply_axis["namespace"] } +local register_following = { kind = "axis", axis = "following" } -- , apply = apply_axis["following"] } +local register_following_sibling = { kind = "axis", axis = "following-sibling" } -- , apply = apply_axis["following-sibling"] } +local register_preceding = { kind = "axis", axis = "preceding" } -- , apply = apply_axis["preceding"] } +local register_preceding_sibling = { kind = "axis", axis = "preceding-sibling" } -- , apply = apply_axis["preceding-sibling"] } + +local register_auto_descendant_or_self = { kind = "axis", axis = "auto-descendant-or-self" } -- , apply = apply_axis["auto-descendant-or-self"] } +local register_auto_descendant = { kind = "axis", axis = "auto-descendant" } -- , apply = apply_axis["auto-descendant"] } +local register_auto_self = { kind = "axis", axis = "auto-self" } -- , apply = apply_axis["auto-self"] } +local register_auto_child = { kind = "axis", axis = "auto-child" } -- , apply = apply_axis["auto-child"] } + +local register_initial_child = { kind = "axis", axis = "initial-child" } -- , apply = apply_axis["initial-child"] } + +local register_all_nodes = { kind = "nodes", nodetest = true, nodes = { true, false, false } } + +local function register_error(str) + return { kind = "error", comment = format("unparsed: %s",str) } +end + +local parser = Ct { "patterns", -- can be made a bit faster by moving pattern outside + + patterns = spaces * V("protocol") * spaces * V("initial") * spaces * V("step") * spaces * + (P("/") * spaces * V("step") * spaces)^0, + + protocol = Cg(V("letters"),"protocol") * P("://") + Cg(Cc(nil),"protocol"), + + step = (V("shortcuts") + V("axis") * spaces * V("nodes")^0 + V("error")) * spaces * V("expressions")^0 * spaces * V("finalizer")^0, + + axis = V("descendant") + V("child") + V("parent") + V("self") + V("root") + V("ancestor") + + V("descendant_or_self") + V("following") + V("following_sibling") + + V("preceding") + V("preceding_sibling") + V("ancestor_or_self") + + #(1-P(-1)) * Cc(register_auto_child), + + initial = (P("/") * spaces * Cc(register_initial_child))^-1, + + error = (P(1)^1) / register_error, + + shortcuts_a = V("s_descendant_or_self") + V("s_descendant") + V("s_child") + V("s_parent") + V("s_self") + V("s_root") + V("s_ancestor"), + + shortcuts = V("shortcuts_a") * (spaces * "/" * spaces * V("shortcuts_a"))^0, + + s_descendant_or_self = P("/") * Cc(register_descendant_or_self), + s_descendant = P("**") * Cc(register_descendant), + s_child = P("*") * Cc(register_child ), + s_parent = P("..") * Cc(register_parent ), + s_self = P("." ) * Cc(register_self ), + s_root = P("^^") * Cc(register_root ), + s_ancestor = P("^") * Cc(register_ancestor ), + + descendant = P("descendant::") * Cc(register_descendant ), + child = P("child::") * Cc(register_child ), + parent = P("parent::") * Cc(register_parent ), + self = P("self::") * Cc(register_self ), + root = P('root::') * Cc(register_root ), + ancestor = P('ancestor::') * Cc(register_ancestor ), + descendant_or_self = P('descendant-or-self::') * Cc(register_descendant_or_self ), + ancestor_or_self = P('ancestor-or-self::') * Cc(register_ancestor_or_self ), + -- attribute = P('attribute::') * Cc(register_attribute ), + -- namespace = P('namespace::') * Cc(register_namespace ), + following = P('following::') * Cc(register_following ), + following_sibling = P('following-sibling::') * Cc(register_following_sibling ), + preceding = P('preceding::') * Cc(register_preceding ), + preceding_sibling = P('preceding-sibling::') * Cc(register_preceding_sibling ), + + nodes = (V("nodefunction") * spaces * P("(") * V("nodeset") * P(")") + V("nodetest") * V("nodeset")) / register_nodes, + + expressions = expression / register_expression, + + letters = R("az")^1, + name = (1-lpeg.S("/[]()|:*!"))^1, + negate = P("!") * Cc(false), + + nodefunction = V("negate") + P("not") * Cc(false) + Cc(true), + nodetest = V("negate") + Cc(true), + nodename = (V("negate") + Cc(true)) * spaces * ((V("wildnodename") * P(":") * V("wildnodename")) + (Cc(false) * V("wildnodename"))), + wildnodename = (C(V("name")) + P("*") * Cc(false)) * #(1-P("(")), + nodeset = spaces * Ct(V("nodename") * (spaces * P("|") * spaces * V("nodename"))^0) * spaces, + + finalizer = (Cb("protocol") * P("/")^-1 * C(V("name")) * arguments * P(-1)) / register_finalizer, + +} + local cache = { } -function xml.lpath(pattern,trace) - lpathcalls = lpathcalls + 1 - if type(pattern) == "string" then - local result = cache[pattern] - if result == nil then -- can be false which is valid -) - result = compose(pattern) - cache[pattern] = result - lpathcached = lpathcached + 1 - end - if trace or trace_lpath then - xml.lshow(result) - end - return result +local function nodesettostring(set,nodetest) + local t = { } + for i=1,#set,3 do + local directive, ns, tg = set[i], set[i+1], set[i+2] + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + tg = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + t[#t+1] = (directive and tg) or format("not(%s)",tg) + end + if nodetest == false then + return format("not(%s)",concat(t,"|")) else - return pattern + return concat(t,"|") end end -function xml.cached_patterns() - return cache +local function tagstostring(list) + if #list == 0 then + return "no elements" + else + local t = { } + for i=1, #list do + local li = list[i] + local ns, tg = li.ns, li.tg + if not ns or ns == "" then ns = "*" end + if not tg or tg == "" then tg = "*" end + t[#t+1] = (tg == "@rt@" and "[root]") or format("%s:%s",ns,tg) + end + return concat(t," ") + end end --- we run out of locals (limited to 200) --- --- local fallbackreport = (texio and texio.write) or io.write - -function xml.lshow(pattern,report) --- report = report or fallbackreport - report = report or (texio and texio.write) or io.write - local lp = xml.lpath(pattern) - if lp == false then - report(" -: root\n") - elseif lp == true then - report(" -: wildcard\n") +xml.nodesettostring = nodesettostring + +local function lshow(parsed) + if type(parsed) == "string" then + parsed = parse_pattern(parsed) + end + local s = table.serialize_functions -- ugly + table.serialize_functions = false -- ugly + logs.report("lpath","%s://%s => %s",parsed.protocol or xml.defaultprotocol,parsed.pattern,table.serialize(parsed,false)) + table.serialize_functions = s -- ugly +end + +xml.lshow = lshow + +local function parse_pattern(pattern) -- the gain of caching is rather minimal + lpathcalls = lpathcalls + 1 + if type(pattern) == "table" then + return pattern else - if type(pattern) == "string" then - report(format("pattern: %s\n",pattern)) - end - for k=1,#lp do - local v = lp[k] - if #v > 1 then - local t = { } - for i=2,#v do - local vv = v[i] - if type(vv) == "string" then - t[#t+1] = (vv ~= "" and vv) or "#" - elseif type(vv) == "boolean" then - t[#t+1] = (vv and "==") or "<>" + local parsed = cache[pattern] + if parsed then + lpathcached = lpathcached + 1 + else + parsed = parser:match(pattern) + if parsed then + parsed.pattern = pattern + local np = #parsed + if np == 0 then + parsed = { pattern = pattern, register_self, state = "parsing error" } + logs.report("lpath","parsing error in '%s'",pattern) + lshow(parsed) + else + -- we could have done this with a more complex parsed but this + -- is cleaner + local pi = parsed[1] + if pi.axis == "auto-child" then + parsed.comment = "auto-child replaced by auto-descendant-or-self" + parsed[1] = register_auto_descendant_or_self + --~ parsed.comment = "auto-child replaced by auto-descendant" + --~ parsed[1] = register_auto_descendant + elseif pi.axis == "initial-child" and np > 1 and parsed[2].axis then + parsed.comment = "initial-child removed" -- we could also make it a auto-self + remove(parsed,1) end end - report(format("%2i: %s %s -> %s\n", k,v[1],actions[v[1]],concat(t," "))) else - report(format("%2i: %s %s\n", k,v[1],actions[v[1]])) + parsed = { pattern = pattern } + end + cache[pattern] = parsed + if trace_lparse and not trace_lprofile then + lshow(parsed) end end + return parsed end end -function xml.xshow(e,...) -- also handy when report is given, use () to isolate first e - local t = { ... } --- local report = (type(t[#t]) == "function" and t[#t]) or fallbackreport - local report = (type(t[#t]) == "function" and t[#t]) or (texio and texio.write) or io.write - if e == nil then - report("<!-- no element -->\n") - elseif type(e) ~= "table" then - report(tostring(e)) - elseif e.tg then - report(tostring(e) .. "\n") +-- we can move all calls inline and then merge the trace back +-- technically we can combine axis and the next nodes which is +-- what we did before but this a bit cleaner (but slower too) +-- but interesting is that it's not that much faster when we +-- go inline +-- +-- beware: we need to return a collection even when we filter +-- else the (simple) cache gets messed up + +-- caching found lookups saves not that much (max .1 sec on a 8 sec run) +-- and it also messes up finalizers + +local profiled = { } xml.profiled = profiled + +local function profiled_apply(list,parsed,nofparsed) + local p = profiled[parsed.pattern] + if p then + p.tested = p.tested + 1 else - for i=1,#e do - report(tostring(e[i]) .. "\n") + p = { tested = 1, matched = 0, finalized = 0 } + profiled[parsed.pattern] = p + end + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + p.matched = p.matched + 1 + p.finalized = p.finalized + 1 + return collected + end + if not collected or #collected == 0 then + return nil end end + if collected then + p.matched = p.matched + 1 + end + return collected +end + +local function traced_apply(list,parsed,nofparsed) + if trace_lparse then + lshow(parsed) + end + logs.report("lpath", "collecting : %s",parsed.pattern) + logs.report("lpath", " root tags : %s",tagstostring(list)) + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + collected = apply_axis[pi.axis](collected) + logs.report("lpath", "% 10i : ax : %s",(collected and #collected) or 0,pi.axis) + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + logs.report("lpath", "% 10i : ns : %s",(collected and #collected) or 0,nodesettostring(pi.nodes,pi.nodetest)) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + logs.report("lpath", "% 10i : ex : %s",(collected and #collected) or 0,pi.expression) + elseif kind == "finalizer" then + collected = pi.finalizer(collected) + logs.report("lpath", "% 10i : fi : %s : %s(%s)",(collected and #collected) or 0,parsed.protocol or xml.defaultprotocol,pi.name,pi.arguments or "") + return collected + end + if not collected or #collected == 0 then + return nil + end + end + return collected end ---[[ldx-- -<p>An <l n='lpath'/> is converted to a table with instructions for traversing the -tree. Hoever, simple cases are signaled by booleans. Because we don't know in -advance what we want to do with the found element the handle gets three arguments:</p> - -<lines> -<t>r</t> : the root element of the data table -<t>d</t> : the data table of the result -<t>t</t> : the index in the data table of the result -</lines> +local function parse_apply(list,pattern) + -- we avoid an extra call + local parsed = cache[pattern] + if parsed then + lpathcalls = lpathcalls + 1 + lpathcached = lpathcached + 1 + elseif type(pattern) == "table" then + lpathcalls = lpathcalls + 1 + parsed = pattern + else + parsed = parse_pattern(pattern) or pattern + end + if not parsed then + return + end + local nofparsed = #parsed + if nofparsed == 0 then + -- something is wrong + elseif not trace_lpath then + -- normal apply, inline, no self + local collected = list + for i=1,nofparsed do + local pi = parsed[i] + local kind = pi.kind + if kind == "axis" then + local axis = pi.axis + if axis ~= "self" then + collected = apply_axis[axis](collected) + end + elseif kind == "nodes" then + collected = apply_nodes(collected,pi.nodetest,pi.nodes) + elseif kind == "expression" then + collected = apply_expression(collected,pi.evaluator,i) + elseif kind == "finalizer" then + return pi.finalizer(collected) + end + if not collected or #collected == 0 then + return nil + end + end + return collected + elseif trace_lprofile then + return profiled_apply(list,parsed,nofparsed) + else -- trace_lpath + return traced_apply(list,parsed,nofparsed) + end +end -<p> Access to the root and data table makes it possible to construct insert and delete -functions.</p> ---ldx]]-- +-- internal (parsed) -local functions = xml.functions -local expressions = xml.expressions +expressions.child = function(e,pattern) + return parse_apply({ e },pattern) -- todo: cache +end +expressions.count = function(e,pattern) + local collected = parse_apply({ e },pattern) -- todo: cache + return (collected and #collected) or 0 +end -expressions.contains = string.find -expressions.find = string.find -expressions.upper = string.upper -expressions.lower = string.lower -expressions.number = tonumber -expressions.boolean = toboolean +-- external expressions.oneof = function(s,...) -- slow local t = {...} for i=1,#t do if s == t[i] then return true end end return false end - expressions.error = function(str) - xml.error_handler("unknown function in lpath expression",str or "?") + xml.error_handler("unknown function in lpath expression",tostring(str or "?")) return false end +expressions.undefined = function(s) + return s == nil +end -functions.text = function(root,k,n) -- unchecked, maybe one deeper - local t = type(t) - if t == "string" then - return t - else -- todo n - local rdt = root.dt - return (rdt and rdt[k]) or root[k] or "" +expressions.contains = find +expressions.find = find +expressions.upper = upper +expressions.lower = lower +expressions.number = tonumber +expressions.boolean = toboolean + +-- user interface + +local function traverse(root,pattern,handle) + logs.report("xml","use 'xml.selection' instead for '%s'",pattern) + local collected = parse_apply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + handle(r,r.dt,e.ni) + end + end +end + +local function selection(root,pattern,handle) + local collected = parse_apply({ root },pattern) + if collected then + if handle then + for c=1,#collected do + handle(collected[c]) + end + else + return collected + end end end -functions.name = function(d,k,n) -- ns + tg +xml.parse_parser = parser +xml.parse_pattern = parse_pattern +xml.parse_apply = parse_apply +xml.traverse = traverse -- old method, r, d, k +xml.selection = selection -- new method, simple handle + +local lpath = parse_pattern + +xml.lpath = lpath + +function xml.cached_patterns() + return cache +end + +-- generic function finalizer (independant namespace) + +local function dofunction(collected,fnc) + if collected then + local f = functions[fnc] + if f then + for c=1,#collected do + f(collected[c]) + end + else + logs.report("xml","unknown function '%s'",fnc) + end + end +end + +xml.finalizers.xml["function"] = dofunction +xml.finalizers.tex["function"] = dofunction + +-- functions + +expressions.text = function(e,n) + local rdt = e.__p__.dt + return (rdt and rdt[n]) or "" +end + +expressions.name = function(e,n) -- ns + tg local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk + n = tonumber(n) or 0 + if n == 0 then + found = type(e) == "table" and e elseif n < 0 then + local d, k = e.__p__.dt, e.ni for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then @@ -4114,6 +5285,7 @@ functions.name = function(d,k,n) -- ns + tg end end else + local d, k = e.__p__.dt, e.ni for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then @@ -4138,15 +5310,13 @@ functions.name = function(d,k,n) -- ns + tg end end -functions.tag = function(d,k,n) -- only tg +expressions.tag = function(e,n) -- only tg local found = false - n = n or 0 - if not k then - -- not found - elseif n == 0 then - local dk = d[k] - found = dk and (type(dk) == "table") and dk + n = tonumber(n) or 0 + if n == 0 then + found = (type(e) == "table") and e -- seems to fail elseif n < 0 then + local d, k = e.__p__.dt, e.ni for i=k-1,1,-1 do local di = d[i] if type(di) == "table" then @@ -4159,6 +5329,7 @@ functions.tag = function(d,k,n) -- only tg end end else + local d, k = e.__p__.dt, e.ni for i=k+1,#d,1 do local di = d[i] if type(di) == "table" then @@ -4174,664 +5345,403 @@ functions.tag = function(d,k,n) -- only tg return (found and found.tg) or "" end -expressions.text = functions.text -expressions.name = functions.name -expressions.tag = functions.tag +--[[ldx-- +<p>This is the main filter function. It returns whatever is asked for.</p> +--ldx]]-- -local function traverse(root,pattern,handle,reverse,index,parent,wildcard) -- multiple only for tags, not for namespaces - if not root then -- error - return false - elseif pattern == false then -- root - handle(root,root.dt,root.ri) - return false - elseif pattern == true then -- wildcard - local rootdt = root.dt - if rootdt then - local start, stop, step = 1, #rootdt, 1 - if reverse then - start, stop, step = stop, start, -1 - end - for k=start,stop,step do - if handle(root,rootdt,root.ri or k) then return false end - if not traverse(rootdt[k],true,handle,reverse) then return false end - end +function xml.filter(root,pattern) -- no longer funny attribute handling here + return parse_apply({ root },pattern) +end + +--[[ldx-- +<p>Often using an iterators looks nicer in the code than passing handler +functions. The <l n='lua'/> book describes how to use coroutines for that +purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits +code like:</p> + +<typing> +for r, d, k in xml.elements(xml.load('text.xml'),"title") do + print(d[k]) -- old method +end +for e in xml.collected(xml.load('text.xml'),"title") do + print(e) -- new one +end +</typing> +--ldx]]-- + +local wrap, yield = coroutine.wrap, coroutine.yield + +function xml.elements(root,pattern,reverse) -- r, d, k + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) + else + return wrap(function() for c=1,#collected do + local e = collected[c] local r = e.__p__ yield(r,r.dt,e.ni) + end end) end - return false - elseif root.dt then - index = index or 1 - local action = pattern[index] - local command = action[1] - if command == 29 then -- fast case /oeps - local rootdt = root.dt - for k=1,#rootdt do - local e = rootdt[k] - local tg = e.tg - if e.tg then - local ns = e.rn or e.ns - local ns_a, tg_a = action[3], action[4] - local matched = (ns_a == "*" or ns == ns_a) and (tg_a == "*" or tg == tg_a) - if not action[2] then matched = not matched end - if matched then - if handle(root,rootdt,k) then return false end - end - end - end - elseif command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end + end + return wrap(function() end) +end + +function xml.collected(root,pattern,reverse) -- e + local collected = parse_apply({ root },pattern) + if collected then + if reverse then + return wrap(function() for c=#collected,1,-1 do yield(collected[c]) end end) else - if (command == 16 or command == 12) and index == 1 then -- initial - -- wildcard = true - wildcard = command == 16 -- ok? - index = index + 1 - action = pattern[index] - command = action and action[1] or 0 -- something is wrong - end - if command == 11 then -- parent - local ep = root.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - else - local rootdt = root.dt - local start, stop, step, n, dn = 1, #rootdt, 1, 0, 1 - if command == 30 then - if action[5] < 0 then - start, stop, step = stop, start, -1 - dn = -1 - end - elseif reverse and index == #pattern then - start, stop, step = stop, start, -1 - end - local idx = 0 - local hsh = { } -- this will slooow down the lot - for k=start,stop,step do -- we used to have functions for all but a case is faster - local e = rootdt[k] - local ns, tg = e.rn or e.ns, e.tg - if tg then - -- we can optimize this for simple searches, but it probably does not pay off - hsh[tg] = (hsh[tg] or 0) + 1 - idx = idx + 1 - if command == 30 then - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - n = n + dn - if n == action[5] then - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - break - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - else - local matched, multiple = false, false - if command == 20 then -- match - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - elseif command == 21 then -- match one of - multiple = true - for i=3,#action,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - elseif command == 22 then -- eq - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - matched = matched and e.at[action[6]] == action[7] - elseif command == 23 then -- ne - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = mached and e.at[action[6]] ~= action[7] - elseif command == 24 then -- one of eq - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] == action[#action] - elseif command == 25 then -- one of ne - multiple = true - for i=3,#action-2,2 do - local ns_a, tg_a = action[i], action[i+1] - if (ns_a == "*" or ns == ns_a) and (tg == "*" or tg == tg_a) then - matched = true - break - end - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[#action-1]] ~= action[#action] - elseif command == 27 then -- has attribute - local ns_a, tg_a = action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and e.at[action[5]] - elseif command == 28 then -- has value - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - matched = matched and edt and edt[1] == action[5] - elseif command == 31 then - local edt, ns_a, tg_a = e.dt, action[3], action[4] - if tg == tg_a then - matched = ns_a == "*" or ns == ns_a - elseif tg_a == '*' then - matched, multiple = ns_a == "*" or ns == ns_a, true - else - matched = false - end - if not action[2] then matched = not matched end - if matched then - matched = action[6](expressions,root,rootdt,k,e,edt,ns,tg,idx,hsh[tg] or 1) - end - end - if matched then -- combine tg test and at test - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - if wildcard then - if multiple then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - else - -- maybe or multiple; anyhow, check on (section|title) vs just section and title in example in lxml - if not traverse(e,pattern,handle,reverse,index,root) then return false end - end - end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 14 then -- any - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root) then return false end - end - elseif command == 15 then -- many - if index == #pattern then - if handle(root,rootdt,root.ri or k) then return false end - else - if not traverse(e,pattern,handle,reverse,index+1,root,true) then return false end - end - -- not here : 11 - elseif command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,root,index+1) then return false end - elseif handle(root,rootdt,k) then - return false - end - elseif command == 40 and e.special and tg == "@pi@" then -- pi - local pi = action[2] - if pi ~= "" then - local pt = e.dt[1] - if pt and pt:find(pi) then - if handle(root,rootdt,k) then - return false - end - end - elseif handle(root,rootdt,k) then - return false - end - elseif wildcard then - if not traverse(e,pattern,handle,reverse,index,root,true) then return false end - end - end - else - -- not here : 11 - if command == 11 then -- parent - local ep = e.__p__ or parent - if index < #pattern then - if not traverse(ep,pattern,handle,reverse,index+1,root) then return false end - elseif handle(root,rootdt,k) then - return false - end - break -- else loop - end - end - end - end + return wrap(function() for c=1,#collected do yield(collected[c]) end end) end end - return true + return wrap(function() end) end -xml.traverse = traverse + +end -- of closure + +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-ent'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +local type, next = type, next +local texsprint, ctxcatcodes = tex.sprint, tex.ctxcatcodes +local utf = unicode.utf8 +local utfupper = utf.upper --[[ldx-- -<p>Next come all kind of locators and manipulators. The most generic function here -is <t>xml.filter(root,pattern)</t>. All registers functions in the filters namespace -can be path of a search path, as in:</p> +<p>We provide (at least here) two entity handlers. The more extensive +resolver consults a hash first, tries to convert to <l n='utf'/> next, +and finaly calls a handler when defines. When this all fails, the +original entity is returned.</p> -<typing> -local r, d, k = xml.filter(root,"/a/b/c/position(4)" -</typing> +<p>We do things different now but it's still somewhat experimental</p> --ldx]]-- -local traverse, lpath, convert = xml.traverse, xml.lpath, xml.convert +xml.entities = xml.entities or { } -- xml.entity_handler == function -xml.filters = { } +-- experimental, this will be done differently -function xml.filters.default(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk +function xml.merge_entities(root) + local documententities = root.entities + local allentities = xml.entities + if documententities then + for k, v in next, documententities do + allentities[k] = v + end + end end -function xml.filters.attributes(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - if ekat then - if arguments then - return ekat[arguments] or "", rt, dt, dk +function xml.resolved_entity(str) + local e = xml.entities[str] + if e then + local te = type(e) + if te == "function" then + e(str) else - return ekat, rt, dt, dk + texsprint(ctxcatcodes,e) end else - return { }, rt, dt, dk + texsprint(ctxcatcodes,"\\xmle{",str,"}{",utfupper(str),"}") -- we need to use our own upper end end -function xml.filters.reverse(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +xml.entities.amp = function() tex.write("&") end +xml.entities.lt = function() tex.write("<") end +xml.entities.gt = function() tex.write(">") end -function xml.filters.count(root,pattern,everything) - local n = 0 - traverse(root, lpath(pattern), function(r,d,t) - if everything or type(d[t]) == "table" then - n = n + 1 - end - end) - return n -end -function xml.filters.elements(root, pattern) -- == all - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e then - t[#t+1] = e - end - end) - return t -end +end -- of closure -function xml.filters.texts(root, pattern) - local t = { } - traverse(root, lpath(pattern), function(r,d,k) - local e = d[k] - if e and e.dt then - t[#t+1] = e.dt - end - end) - return t -end +do -- create closure to overcome 200 locals limit -function xml.filters.first(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end) - return dt and dt[dk], rt, dt, dk -end +if not modules then modules = { } end modules ['lxml-mis'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} -function xml.filters.last(root,pattern) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt,dt,dk = r,d,k return true end, 'reverse') - return dt and dt[dk], rt, dt, dk -end +local concat = table.concat +local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring +local format, gsub = string.format, string.gsub -function xml.filters.index(root,pattern,arguments) - local rt, dt, dk, reverse, i = nil, nil, nil, false, tonumber(arguments or '1') or 1 - if i and i ~= 0 then - if i < 0 then - reverse, i = true, -i - end - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk, i = r, d, k, i-1 return i == 0 end, reverse) - if i == 0 then - return dt and dt[dk], rt, dt, dk +--[[ldx-- +<p>The following helper functions best belong to the <t>lmxl-ini</t> +module. Some are here because we need then in the <t>mk</t> +document and other manuals, others came up when playing with +this module. Since this module is also used in <l n='mtxrun'/> we've +put them here instead of loading mode modules there then needed.</p> +--ldx]]-- + + +local function xmlgsub(t,old,new) + local dt = t.dt + if dt then + for k=1,#dt do + local v = dt[k] + if type(v) == "string" then + dt[k] = gsub(v,old,new) + else + xmlgsub(v,old,new) + end end end - return nil, nil, nil, nil end -function xml.filters.attribute(root,pattern,arguments) - local rt, dt, dk - traverse(root, lpath(pattern), function(r,d,k) rt, dt, dk = r, d, k return true end) - local ekat = (dt and dt[dk] and dt[dk].at) or (rt and rt.at) - -- return (ekat and (ekat[arguments] or ekat[gsub(arguments,"^([\"\'])(.*)%1$","%2")])) or "" - return (ekat and (ekat[arguments] or (find(arguments,"^[\'\"]") and ekat[sub(arguments,2,-2)]))) or "" -end +xmlgsub = xmlgsub -function xml.filters.text(root,pattern,arguments) -- ?? why index, tostring slow - local dtk, rt, dt, dk = xml.filters.index(root,pattern,arguments) - if dtk then -- n - local dtkdt = dtk.dt - if not dtkdt then - return "", rt, dt, dk - elseif #dtkdt == 1 and type(dtkdt[1]) == "string" then - return dtkdt[1], rt, dt, dk - else - return xml.tostring(dtkdt), rt, dt, dk +function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual + if d and k then + local dkm = d[k-1] + if dkm and type(dkm) == "string" then + local s = match(dkm,"\n(%s+)") + xmlgsub(dk,"\n"..rep(" ",#s),"\n") end - else - return "", rt, dt, dk end end -function xml.filters.tag(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.tag(d,k,n and tonumber(n)) - return true - end) - return tag -end +--~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } +--~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end -function xml.filters.name(root,pattern,n) - local tag = "" - traverse(root, lpath(pattern), function(r,d,k) - tag = xml.functions.name(d,k,n and tonumber(n)) - return true - end) - return tag -end +--~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end +--~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end +--~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" ---[[ldx-- -<p>For splitting the filter function from the path specification, we can -use string matching or lpeg matching. Here the difference in speed is -neglectable but the lpeg variant is more robust.</p> ---ldx]]-- +local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs --- not faster but hipper ... although ... i can't get rid of the trailing / in the path +-- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg +-- +-- 1021:0335:0287:0247 -local P, S, R, C, V, Cc = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc +-- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" +-- +-- 1559:0257:0288:0190 (last one suggested by roberto) -local slash = P('/') -local name = (R("az","AZ","--","__"))^1 -local path = C(((1-slash)^0 * slash)^1) -local argument = P { "(" * C(((1 - S("()")) + V(1))^0) * ")" } -local action = Cc(1) * path * C(name) * argument -local attribute = Cc(2) * path * P('@') * C(name) -local direct = Cc(3) * Cc("../*") * slash^0 * C(name) * argument +-- escaped = Cs((S("<&>") / xml.escapes + 1)^0) +-- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) +local normal = (1 - S("<&>"))^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local escaped = Cs(normal * (special * normal)^0) -local parser = direct + action + attribute +-- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) -local filters = xml.filters -local attribute_filter = xml.filters.attributes -local default_filter = xml.filters.default +local normal = (1 - S"&")^0 +local special = P("<")/"<" + P(">")/">" + P("&")/"&" +local unescaped = Cs(normal * (special * normal)^0) --- todo: also hash, could be gc'd +-- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) -function xml.filter(root,pattern) - local kind, a, b, c = parser:match(pattern) - if kind == 1 or kind == 3 then - return (filters[b] or default_filter)(root,a,c) - elseif kind == 2 then - return attribute_filter(root,a,b) - else - return default_filter(root,pattern) - end -end +local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) ---~ slightly faster, but first we need a proper test file ---~ ---~ local hash = { } ---~ ---~ function xml.filter(root,pattern) ---~ local h = hash[pattern] ---~ if not h then ---~ local kind, a, b, c = parser:match(pattern) ---~ if kind == 1 then ---~ h = { kind, filters[b] or default_filter, a, b, c } ---~ elseif kind == 2 then ---~ h = { kind, attribute_filter, a, b, c } ---~ else ---~ h = { kind, default_filter, a, b, c } ---~ end ---~ hash[pattern] = h ---~ end ---~ local kind = h[1] ---~ if kind == 1 then ---~ return h[2](root,h[2],h[4]) ---~ elseif kind == 2 then ---~ return h[2](root,h[2],h[3]) ---~ else ---~ return h[2](root,pattern) ---~ end ---~ end +xml.escaped_pattern = escaped +xml.unescaped_pattern = unescaped +xml.cleansed_pattern = cleansed ---[[ldx-- -<p>The following functions collect elements and texts.</p> ---ldx]]-- +function xml.escaped (str) return escaped :match(str) end +function xml.unescaped(str) return unescaped:match(str) end +function xml.cleansed (str) return cleansed :match(str) end --- still somewhat bugged -function xml.collect_elements(root, pattern, ignorespaces) - local rr, dd = { }, { } - traverse(root, lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk then - if ignorespaces and type(dk) == "string" and dk:find("[^%S]") then - -- ignore - else - local n = #rr+1 - rr[n], dd[n] = r, dk - end - end - end) - return dd, rr -end +end -- of closure -function xml.collect_texts(root, pattern, flatten) - local t = { } -- no r collector - traverse(root, lpath(pattern), function(r,d,k) - if d then - local ek = d[k] - local tx = ek and ek.dt - if flatten then - if tx then - t[#t+1] = xml.tostring(tx) or "" - else - t[#t+1] = "" +do -- create closure to overcome 200 locals limit + +if not modules then modules = { } end modules ['lxml-aux'] = { + version = 1.001, + comment = "this module is the basis for the lxml-* ones", + author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", + copyright = "PRAGMA ADE / ConTeXt Development Team", + license = "see context related readme files" +} + +-- not all functions here make sense anymore vbut we keep them for +-- compatibility reasons + +local xmlparseapply, xmlconvert, xmlcopy = xml.parse_apply, xml.convert, xml.copy + +local type = type +local insert, remove = table.insert, table.remove +local gmatch, gsub = string.gmatch, string.gsub + +local function withelements(e,handle,depth) + if e and handle then + local edt = e.dt + if edt then + depth = depth or 0 + for i=1,#edt do + local e = edt[i] + if type(e) == "table" then + handle(e,depth) + withelements(e,handle,depth+1) end - else - t[#t+1] = tx or "" end - else - t[#t+1] = "" end - end) - return t + end end -function xml.collect_tags(root, pattern, nonamespace) - local t = { } - xml.traverse(root, xml.lpath(pattern), function(r,d,k) - local dk = d and d[k] - if dk and type(dk) == "table" then - local ns, tg = e.ns, e.tg - if nonamespace then - t[#t+1] = tg -- if needed we can return an extra table - elseif ns == "" then - t[#t+1] = tg - else - t[#t+1] = ns .. ":" .. tg +xml.withelements = withelements + +function xml.withelement(e,n,handle) -- slow + if e and n ~= 0 and handle then + local edt = e.dt + if edt then + if n > 0 then + for i=1,#edt do + local ei = edt[i] + if type(ei) == "table" then + if n == 1 then + handle(ei) + return + else + n = n - 1 + end + end + end + elseif n < 0 then + for i=#edt,1,-1 do + local ei = edt[i] + if type(ei) == "table" then + if n == -1 then + handle(ei) + return + else + n = n + 1 + end + end + end end end - end) - return #t > 0 and {} + end end ---[[ldx-- -<p>Often using an iterators looks nicer in the code than passing handler -functions. The <l n='lua'/> book describes how to use coroutines for that -purpose (<url href='http://www.lua.org/pil/9.3.html'/>). This permits -code like:</p> +xml.elements_only = xml.collected -<typing> -for r, d, k in xml.elements(xml.load('text.xml'),"title") do - print(d[k]) +function xml.each_element(root, pattern, handle, reverse) + local collected = xmlparseapply({ root },pattern) + if collected then + if reverse then + for c=#collected,1,-1 do + handle(collected[c]) + end + else + for c=1,#collected do + handle(collected[c]) + end + end + return collected + end end -</typing> -<p>Which will print all the titles in the document. The iterator variant takes -1.5 times the runtime of the function variant which is due to the overhead in -creating the wrapper. So, instead of:</p> +xml.process_elements = xml.each_element -<typing> -function xml.filters.first(root,pattern) - for rt,dt,dk in xml.elements(root,pattern) - return dt and dt[dk], rt, dt, dk +function xml.process_attributes(root, pattern, handle) + local collected = xmlparseapply({ root },pattern) + if collected and handle then + for c=1,#collected do + handle(collected[c].at) + end end - return nil, nil, nil, nil + return collected end -</typing> -<p>We use the function variants in the filters.</p> +--[[ldx-- +<p>The following functions collect elements and texts.</p> --ldx]]-- -local wrap, yield = coroutine.wrap, coroutine.yield +-- are these still needed -> lxml-cmp.lua -function xml.elements(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), yield, reverse) end) +function xml.collect_elements(root, pattern) + return xmlparseapply({ root },pattern) end -function xml.elements_only(root,pattern,reverse) - return wrap(function() traverse(root, lpath(pattern), function(r,d,k) yield(d[k]) end, reverse) end) -end - -function xml.each_element(root, pattern, handle, reverse) - local ok - traverse(root, lpath(pattern), function(r,d,k) ok = true handle(r,d,k) end, reverse) - return ok -end - -function xml.process_elements(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then - for i=1,#dkdt do - local v = dkdt[i] - if v.tg then handle(v) end - end +function xml.collect_texts(root, pattern, flatten) -- todo: variant with handle + local collected = xmlparseapply({ root },pattern) + if collected and flatten then + local xmltostring = xml.tostring + for c=1,#collected do + collected[c] = xmltostring(collected[c].dt) end - end) + end + return collected or { } end -function xml.process_attributes(root, pattern, handle) - traverse(root, lpath(pattern), function(r,d,k) - local ek = d[k] - local a = ek.at or { } - handle(a) - if next(a) then -- next is faster than type (and >0 test) - ek.at = a - else - ek.at = nil +function xml.collect_tags(root, pattern, nonamespace) + local collected = xmlparseapply({ root },pattern) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace then + t[#t+1] = tg + elseif ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end end - end) + return t + end end --[[ldx-- <p>We've now arrives at the functions that manipulate the tree.</p> --ldx]]-- +local no_root = { no_root = true } + function xml.inject_element(root, pattern, element, prepend) if root and element then - local matches, collect = { }, nil if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,no_root) end if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=1,#matches do - local m = matches[i] - local r, d, k, element, edt = m[1], m[2], m[3], m[4], nil - if element.ri then - element = element.dt[element.ri].dt - else - element = element.dt - end - if r.ri then - edt = r.dt[r.ri].dt - else - edt = d and d[k] and d[k].dt - end - if edt then - local be, af - if prepend then - be, af = xml.copy(element), edt + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d = r.dt + local k = e.ni + if element.ri then + element = element.dt[element.ri].dt else - be, af = edt, xml.copy(element) - end - for i=1,#af do - be[#be+1] = af[i] + element = element.dt end + local edt if r.ri then - r.dt[r.ri].dt = be + edt = r.dt[r.ri].dt else - d[k].dt = be + edt = d and d[k] and d[k].dt + end + if edt then + local be, af + if prepend then + be, af = xmlcopy(element), edt + else + be, af = edt, xmlcopy(element) + end + for i=1,#af do + be[#be+1] = af[i] + end + if r.ri then + r.dt[r.ri].dt = be + else + d[k].dt = be + end + else + -- r.dt = element.dt -- todo end - else - -- r.dt = element.dt -- todo end end end @@ -4847,32 +5757,31 @@ function xml.insert_element(root, pattern, element, before) -- todo: element als else local matches, collect = { }, nil if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then - collect = function(r,d,k) matches[#matches+1] = { r, d, k, element } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - local r, d, k, element = m[1], m[2], m[3], m[4] - if not before then k = k + 1 end - if element.tg then - insert(d,k,element) -- untested ---~ elseif element.dt then ---~ for _,v in ipairs(element.dt) do -- i added ---~ insert(d,k,v) ---~ k = k + 1 ---~ end ---~ end - else - local edt = element.dt - if edt then - for i=1,#edt do - insert(d,k,edt[i]) - k = k + 1 + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + local r = e.__p__ + local d = r.dt + local k = e.ni + if not before then + k = k + 1 + end + if element.tg then + insert(d,k,element) -- untested + else + local edt = element.dt + if edt then + for i=1,#edt do + insert(d,k,edt[i]) + k = k + 1 + end end end end @@ -4888,105 +5797,114 @@ xml.inject_element_after = xml.inject_element xml.inject_element_before = function(r,p,e) xml.inject_element(r,p,e,true) end function xml.delete_element(root, pattern) - local matches, deleted = { }, { } - local collect = function(r,d,k) matches[#matches+1] = { r, d, k } end - traverse(root, lpath(pattern), collect) - for i=#matches,1,-1 do - local m = matches[i] - deleted[#deleted+1] = remove(m[2],m[3]) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + remove(e.__p__.dt,e.ni) + e.ni = nil + end end - return deleted + return collection end function xml.replace_element(root, pattern, element) if type(element) == "string" then - element = convert(element,true) + element = xmlconvert(element,true) end if element and element.ri then element = element.dt[element.ri] end if element then - traverse(root, lpath(pattern), function(rm, d, k) - d[k] = element.dt -- maybe not clever enough - end) - end -end - -local function load_data(name) -- == io.loaddata - local f, data = io.open(name), "" - if f then - data = f:read("*all",'b') -- 'b' ? - f:close() + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.__p__.dt[e.ni] = element.dt -- maybe not clever enough + end + end end - return data end -function xml.include(xmldata,pattern,attribute,recursive,loaddata) +local function include(xmldata,pattern,attribute,recursive,loaddata) -- parse="text" (default: xml), encoding="" (todo) -- attribute = attribute or 'href' pattern = pattern or 'include' - loaddata = loaddata or load_data - local function include(r,d,k) - local ek, name = d[k], nil - if not attribute or attribute == "" then + loaddata = loaddata or io.loaddata + local collected = xmlparseapply({ xmldata },pattern) + if collected then + for c=1,#collected do + local ek = collected[c] + local name = nil local ekdt = ek.dt - name = (type(ekdt) == "table" and ekdt[1]) or ekdt - end - if not name then - if ek.at then + local ekat = ek.at + local epdt = ek.__p__.dt + if not attribute or attribute == "" then + name = (type(ekdt) == "table" and ekdt[1]) or ekdt -- ckeck, probably always tab or str + end + if not name then for a in gmatch(attribute or "href","([^|]+)") do - name = ek.at[a] + name = ekat[a] if name then break end end end - end - local data = (name and name ~= "" and loaddata(name)) or "" - if data == "" then - xml.empty(d,k) - elseif ek.at["parse"] == "text" then -- for the moment hard coded - d[k] = xml.escaped(data) - else - local xi = xml.convert(data) - if not xi then - xml.empty(d,k) + local data = (name and name ~= "" and loaddata(name)) or "" + if data == "" then + epdt[ek.ni] = "" -- xml.empty(d,k) + elseif ekat["parse"] == "text" then + -- for the moment hard coded + epdt[ek.ni] = xml.escaped(data) -- d[k] = xml.escaped(data) else - if recursive then - xml.include(xi,pattern,attribute,recursive,loaddata) + local settings = xmldata.settings + settings.parent_root = xmldata -- to be tested + local xi = xmlconvert(data,settings) + if not xi then + epdt[ek.ni] = "" -- xml.empty(d,k) + else + if recursive then + include(xi,pattern,attribute,recursive,loaddata) + end + epdt[ek.ni] = xml.body(xi) -- xml.assign(d,k,xi) end - xml.assign(d,k,xi) end end end - xml.each_element(xmldata, pattern, include) end +xml.include = include + function xml.strip_whitespace(root, pattern, nolines) -- strips all leading and trailing space ! - traverse(root, lpath(pattern), function(r,d,k) - local dkdt = d[k].dt - if dkdt then -- can be optimized - local t = { } - for i=1,#dkdt do - local str = dkdt[i] - if type(str) == "string" then - if str == "" then - -- stripped - else - if nolines then - str = gsub(str,"[ \n\r\t]+"," ") - end + local collected = xmlparseapply({ root },pattern) + if collected then + for i=1,#collected do + local e = collected[i] + local edt = e.dt + if edt then + local t = { } + for i=1,#edt do + local str = edt[i] + if type(str) == "string" then if str == "" then -- stripped else - t[#t+1] = str + if nolines then + str = gsub(str,"[ \n\r\t]+"," ") + end + if str == "" then + -- stripped + else + t[#t+1] = str + end end + else +--~ str.ni = i + t[#t+1] = str end - else - t[#t+1] = str end + e.dt = t end - d[k].dt = t end - end) + end end local function rename_space(root, oldspace, newspace) -- fast variant @@ -5011,59 +5929,49 @@ end xml.rename_space = rename_space function xml.remap_tag(root, pattern, newtg) - traverse(root, lpath(pattern), function(r,d,k) - d[k].tg = newtg - end) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].tg = newtg + end + end end + function xml.remap_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - d[k].ns = newns - end) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + collected[c].ns = newns + end + end end + function xml.check_namespace(root, pattern, newns) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - if (not dk.rn or dk.rn == "") and dk.ns == "" then - dk.rn = newns + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + if (not e.rn or e.rn == "") and e.ns == "" then + e.rn = newns + end end - end) -end -function xml.remap_name(root, pattern, newtg, newns, newrn) - traverse(root, lpath(pattern), function(r,d,k) - local dk = d[k] - dk.tg = newtg - dk.ns = newns - dk.rn = newrn - end) + end end -function xml.filters.found(root,pattern,check_content) - local found = false - traverse(root, lpath(pattern), function(r,d,k) - if check_content then - local dk = d and d[k] - found = dk and dk.dt and next(dk.dt) and true - else - found = true +function xml.remap_name(root, pattern, newtg, newns, newrn) + local collected = xmlparseapply({ root },pattern) + if collected then + for c=1,#collected do + local e = collected[c] + e.tg, e.ns, e.rn = newtg, newns, newrn end - return true - end) - return found + end end --[[ldx-- <p>Here are a few synonyms.</p> --ldx]]-- -xml.filters.position = xml.filters.index - -xml.count = xml.filters.count -xml.index = xml.filters.index -xml.position = xml.filters.index -xml.first = xml.filters.first -xml.last = xml.filters.last -xml.found = xml.filters.found - xml.each = xml.each_element xml.process = xml.process_element xml.strip = xml.strip_whitespace @@ -5077,155 +5985,12 @@ xml.before = xml.insert_element_before xml.delete = xml.delete_element xml.replace = xml.replace_element ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- - -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) - else - xml.gsub(v,old,new) - end - end - end -end - -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..rep(" ",#s),"\n") - end -end - -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end - ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end - -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) - end - else - return "" - end -end - -function xml.statistics() - return { - lpathcalls = lpathcalls, - lpathcached = lpathcached, - } -end - --- xml.set_text_cleanup(xml.show_text_entities) --- xml.set_text_cleanup(xml.resolve_text_entities) - ---~ xml.lshow("/../../../a/(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!(b|c)[@d='e']/f") ---~ xml.lshow("/../../../a/!b[@d!='e']/f") - ---~ x = xml.convert([[ ---~ <a> ---~ <b n='01'>01</b> ---~ <b n='02'>02</b> ---~ <b n='03'>03</b> ---~ <b n='04'>OK</b> ---~ <b n='05'>05</b> ---~ <b n='06'>06</b> ---~ <b n='07'>ALSO OK</b> ---~ </a> ---~ ]]) - ---~ xml.settrace("lpath",true) - ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == 'ok']")) ---~ xml.xshow(xml.first(x,"b[position() > 2 and position() < 5 and text() == upper('ok')]")) ---~ xml.xshow(xml.first(x,"b[@n=='03' or @n=='08']")) ---~ xml.xshow(xml.all (x,"b[number(@n)>2 and number(@n)<6]")) ---~ xml.xshow(xml.first(x,"b[find(text(),'ALSO')]")) - ---~ str = [[ ---~ <?xml version="1.0" encoding="utf-8"?> ---~ <story line='mojca'> ---~ <windows>my secret</mouse> ---~ </story> ---~ ]] - ---~ x = xml.convert([[ ---~ <a><b n='01'>01</b><b n='02'>02</b><x>xx</x><b n='03'>03</b><b n='04'>OK</b></a> ---~ ]]) ---~ xml.xshow(xml.first(x,"b[tag(2) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-1) == 'x']")) ---~ xml.xshow(xml.first(x,"b[tag(-2) == 'x']")) - ---~ print(xml.filter(x,"b/tag(2)")) ---~ print(xml.filter(x,"b/tag(1)")) - end -- of closure do -- create closure to overcome 200 locals limit -if not modules then modules = { } end modules ['lxml-ent'] = { +if not modules then modules = { } end modules ['lxml-xml'] = { version = 1.001, comment = "this module is the basis for the lxml-* ones", author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", @@ -5233,457 +5998,249 @@ if not modules then modules = { } end modules ['lxml-ent'] = { license = "see context related readme files" } -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub, find = string.format, string.gsub, string.find -local utfchar = unicode.utf8.char - ---[[ldx-- -<p>We provide (at least here) two entity handlers. The more extensive -resolver consults a hash first, tries to convert to <l n='utf'/> next, -and finaly calls a handler when defines. When this all fails, the -original entity is returned.</p> ---ldx]]-- +local finalizers = xml.finalizers.xml +local xmlfilter = xml.filter -- we could inline this one for speed +local xmltostring = xml.tostring +local xmlserialize = xml.serialize -xml.entities = xml.entities or { } -- xml.entity_handler == function - -function xml.entity_handler(e) - return format("[%s]",e) +local function first(collected) + return collected and collected[1] end -local function toutf(s) - return utfchar(tonumber(s,16)) +local function last(collected) + return collected and collected[#collected] end -local function utfize(root) - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - -- test prevents copying if no match - if find(dk,"&#x.-;") then - d[k] = gsub(dk,"&#x(.-);",toutf) - end - else - utfize(dk) - end - end +local function all(collected) + return collected end -xml.utfize = utfize - -local function resolve(e) -- hex encoded always first, just to avoid mkii fallbacks - if find(e,"^#x") then - return utfchar(tonumber(e:sub(3),16)) - elseif find(e,"^#") then - return utfchar(tonumber(e:sub(2))) - else - local ee = xml.entities[e] -- we cannot shortcut this one (is reloaded) - if ee then - return ee - else - local h = xml.entity_handler - return (h and h(e)) or "&" .. e .. ";" +local function reverse(collected) + if collected then + local reversed = { } + for c=#collected,1,-1 do + reversed[#reversed+1] = collected[c] end + return reversed end end -local function resolve_entities(root) - if not root.special or root.tg == "@rt@" then - local d = root.dt - for k=1,#d do - local dk = d[k] - if type(dk) == "string" then - if find(dk,"&.-;") then - d[k] = gsub(dk,"&(.-);",resolve) - end - else - resolve_entities(dk) - end - end - end +local function attribute(collected,name) + local at = collected and collected[1].at + return at and at[name] end -xml.resolve_entities = resolve_entities +local function att(id,name) + local at = id.at + return at and at[name] +end -function xml.utfize_text(str) - if find(str,"&#") then - return (gsub(str,"&#x(.-);",toutf)) - else - return str - end +local function count(collected) + return (collected and #collected) or 0 end -function xml.resolve_text_entities(str) -- maybe an lpeg. maybe resolve inline - if find(str,"&") then - return (gsub(str,"&(.-);",resolve)) - else - return str +local function position(collected,n) + if collected then + n = tonumber(n) or 0 + if n < 0 then + return collected[#collected + n + 1] + else + return collected[n] + end end end -function xml.show_text_entities(str) - if find(str,"&") then - return (gsub(str,"&(.-);","[%1]")) - else - return str +local function index(collected) + if collected then + return collected[1].ni end end --- experimental, this will be done differently - -function xml.merge_entities(root) - local documententities = root.entities - local allentities = xml.entities - if documententities then - for k, v in next, documententities do - allentities[k] = v +local function attributes(collected,arguments) + if collected then + local at = collected[1].at + if arguments then + return at[arguments] + elseif next(at) then + return at -- all of them end end end - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['lxml-mis'] = { - version = 1.001, - comment = "this module is the basis for the lxml-* ones", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - -local concat = table.concat -local type, next, tonumber, tostring, setmetatable, loadstring = type, next, tonumber, tostring, setmetatable, loadstring -local format, gsub = string.format, string.gsub - ---[[ldx-- -<p>The following helper functions best belong to the <t>lmxl-ini</t> -module. Some are here because we need then in the <t>mk</t> -document and other manuals, others came up when playing with -this module. Since this module is also used in <l n='mtxrun'/> we've -put them here instead of loading mode modules there then needed.</p> ---ldx]]-- - -function xml.gsub(t,old,new) - local dt = t.dt - if dt then - for k=1,#dt do - local v = dt[k] - if type(v) == "string" then - dt[k] = gsub(v,old,new) +local function chainattribute(collected,arguments) -- todo: optional levels + if collected then + local e = collected[1] + while e do + local at = e.at + if at then + local a = at[arguments] + if a then + return a + end else - xml.gsub(v,old,new) + break -- error end + e = e.__p__ end end + return "" end -function xml.strip_leading_spaces(dk,d,k) -- cosmetic, for manual - if d and k and d[k-1] and type(d[k-1]) == "string" then - local s = d[k-1]:match("\n(%s+)") - xml.gsub(dk,"\n"..string.rep(" ",#s),"\n") +local function text(collected) + if collected then + return xmltostring(collected[1]) -- only first as we cannot concat function + else + return "" end end -function xml.serialize_path(root,lpath,handle) - local dk, r, d, k = xml.first(root,lpath) - dk = xml.copy(dk) - xml.strip_leading_spaces(dk,d,k) - xml.serialize(dk,handle) -end - ---~ xml.escapes = { ['&'] = '&', ['<'] = '<', ['>'] = '>', ['"'] = '"' } ---~ xml.unescapes = { } for k,v in pairs(xml.escapes) do xml.unescapes[v] = k end - ---~ function xml.escaped (str) return (gsub(str,"(.)" , xml.escapes )) end ---~ function xml.unescaped(str) return (gsub(str,"(&.-;)", xml.unescapes)) end ---~ function xml.cleansed (str) return (gsub(str,"<.->" , '' )) end -- "%b<>" - -local P, S, R, C, V, Cc, Cs = lpeg.P, lpeg.S, lpeg.R, lpeg.C, lpeg.V, lpeg.Cc, lpeg.Cs - --- 100 * 2500 * "oeps< oeps> oeps&" : gsub:lpeg|lpeg|lpeg --- --- 1021:0335:0287:0247 - --- 10 * 1000 * "oeps< oeps> oeps& asfjhalskfjh alskfjh alskfjh alskfjh ;al J;LSFDJ" --- --- 1559:0257:0288:0190 (last one suggested by roberto) - --- escaped = Cs((S("<&>") / xml.escapes + 1)^0) --- escaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) -local normal = (1 - S("<&>"))^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local escaped = Cs(normal * (special * normal)^0) - --- 100 * 1000 * "oeps< oeps> oeps&" : gsub:lpeg == 0153:0280:0151:0080 (last one by roberto) - --- unescaped = Cs((S("<")/"<" + S(">")/">" + S("&")/"&" + 1)^0) --- unescaped = Cs((((P("&")/"") * (P("lt")/"<" + P("gt")/">" + P("amp")/"&") * (P(";")/"")) + 1)^0) -local normal = (1 - S"&")^0 -local special = P("<")/"<" + P(">")/">" + P("&")/"&" -local unescaped = Cs(normal * (special * normal)^0) - --- 100 * 5000 * "oeps <oeps bla='oeps' foo='bar'> oeps </oeps> oeps " : gsub:lpeg == 623:501 msec (short tags, less difference) - -local cleansed = Cs(((P("<") * (1-P(">"))^0 * P(">"))/"" + 1)^0) - -xml.escaped_pattern = escaped -xml.unescaped_pattern = unescaped -xml.cleansed_pattern = cleansed - -function xml.escaped (str) return escaped :match(str) end -function xml.unescaped(str) return unescaped:match(str) end -function xml.cleansed (str) return cleansed :match(str) end - -function xml.join(t,separator,lastseparator) - if #t > 0 then - local result = { } - for k,v in pairs(t) do - result[k] = xml.tostring(v) - end - if lastseparator then - return concat(result,separator or "",1,#result-1) .. (lastseparator or "") .. result[#result] - else - return concat(result,separator) +local function texts(collected) + if collected then + local t = { } + for c=1,#collected do + local e = collection[c] + if e and e.dt then + t[#t+1] = e.dt + end end - else - return "" + return t end end - -end -- of closure - -do -- create closure to overcome 200 locals limit - -if not modules then modules = { } end modules ['trac-tra'] = { - version = 1.001, - comment = "companion to trac-tra.mkiv", - author = "Hans Hagen, PRAGMA-ADE, Hasselt NL", - copyright = "PRAGMA ADE / ConTeXt Development Team", - license = "see context related readme files" -} - --- the <anonymous> tag is kind of generic and used for functions that are not --- bound to a variable, like node.new, node.copy etc (contrary to for instance --- node.has_attribute which is bound to a has_attribute local variable in mkiv) - -debugger = debugger or { } - -local counters = { } -local names = { } -local getinfo = debug.getinfo -local format, find, lower, gmatch = string.format, string.find, string.lower, string.gmatch - --- one - -local function hook() - local f = getinfo(2,"f").func - local n = getinfo(2,"Sn") --- if n.what == "C" and n.name then print (n.namewhat .. ': ' .. n.name) end - if f then - local cf = counters[f] - if cf == nil then - counters[f] = 1 - names[f] = n +local function tag(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] else - counters[f] = cf + 1 + c = collected[#collected-n+1] end + return c and c.tg end end -local function getname(func) - local n = names[func] - if n then - if n.what == "C" then - return n.name or '<anonymous>' + +local function name(collected,n) + if collected then + local c + if n == 0 or not n then + c = collected[1] + elseif n > 1 then + c = collected[n] else - -- source short_src linedefined what name namewhat nups func - local name = n.name or n.namewhat or n.what - if not name or name == "" then name = "?" end - return format("%s : %s : %s", n.short_src or "unknown source", n.linedefined or "--", name) + c = collected[#collected-n+1] end - else - return "unknown" - end -end -function debugger.showstats(printer,threshold) - printer = printer or texio.write or print - threshold = threshold or 0 - local total, grandtotal, functions = 0, 0, 0 - printer("\n") -- ugly but ok - -- table.sort(counters) - for func, count in pairs(counters) do - if count > threshold then - local name = getname(func) - if not name:find("for generator") then - printer(format("%8i %s", count, name)) - total = total + count + if c then + if c.ns == "" then + return c.tg + else + return c.ns .. ":" .. c.tg end end - grandtotal = grandtotal + count - functions = functions + 1 end - printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) end --- two - ---~ local function hook() ---~ local n = getinfo(2) ---~ if n.what=="C" and not n.name then ---~ local f = tostring(debug.traceback()) ---~ local cf = counters[f] ---~ if cf == nil then ---~ counters[f] = 1 ---~ names[f] = n ---~ else ---~ counters[f] = cf + 1 ---~ end ---~ end ---~ end ---~ function debugger.showstats(printer,threshold) ---~ printer = printer or texio.write or print ---~ threshold = threshold or 0 ---~ local total, grandtotal, functions = 0, 0, 0 ---~ printer("\n") -- ugly but ok ---~ -- table.sort(counters) ---~ for func, count in pairs(counters) do ---~ if count > threshold then ---~ printer(format("%8i %s", count, func)) ---~ total = total + count ---~ end ---~ grandtotal = grandtotal + count ---~ functions = functions + 1 ---~ end ---~ printer(format("functions: %s, total: %s, grand total: %s, threshold: %s\n", functions, total, grandtotal, threshold)) ---~ end - --- rest - -function debugger.savestats(filename,threshold) - local f = io.open(filename,'w') - if f then - debugger.showstats(function(str) f:write(str) end,threshold) - f:close() +local function tags(collected,nonamespace) + if collected then + local t = { } + for c=1,#collected do + local e = collected[c] + local ns, tg = e.ns, e.tg + if nonamespace or ns == "" then + t[#t+1] = tg + else + t[#t+1] = ns .. ":" .. tg + end + end + return t end end -function debugger.enable() - debug.sethook(hook,"c") -end - -function debugger.disable() - debug.sethook() ---~ counters[debug.getinfo(2,"f").func] = nil -end - -function debugger.tracing() - local n = tonumber(os.env['MTX.TRACE.CALLS']) or tonumber(os.env['MTX_TRACE_CALLS']) or 0 - if n > 0 then - function debugger.tracing() return true end ; return true - else - function debugger.tracing() return false end ; return false +local function empty(collected) + if collected then + for c=1,#collected do + local e = collected[c] + if e then + local edt = e.dt + if edt then + local n = #edt + if n == 1 then + local edk = edt[1] + local typ = type(edk) + if typ == "table" then + return false + elseif edk ~= "" then -- maybe an extra tester for spacing only + return false + end + elseif n > 1 then + return false + end + end + end + end end + return true end ---~ debugger.enable() - ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) ---~ print(math.sin(1*.5)) - ---~ debugger.disable() - ---~ print("") ---~ debugger.showstats() ---~ print("") ---~ debugger.showstats(print,3) +finalizers.first = first +finalizers.last = last +finalizers.all = all +finalizers.reverse = reverse +finalizers.elements = all +finalizers.default = all +finalizers.attribute = attribute +finalizers.att = att +finalizers.count = count +finalizers.position = position +finalizers.index = index +finalizers.attributes = attributes +finalizers.chainattribute = chainattribute +finalizers.text = text +finalizers.texts = texts +finalizers.tag = tag +finalizers.name = name +finalizers.tags = tags +finalizers.empty = empty -trackers = trackers or { } +-- shortcuts -- we could support xmlfilter(id,pattern,first) -local data, done = { }, { } +function xml.first(id,pattern) + return first(xmlfilter(id,pattern)) +end -local function set(what,value) - if type(what) == "string" then - what = aux.settings_to_array(what) - end - for i=1,#what do - local w = what[i] - for d, f in next, data do - if done[d] then - -- prevent recursion due to wildcards - elseif find(d,w) then - done[d] = true - for i=1,#f do - f[i](value) - end - end - end - end +function xml.last(id,pattern) + return last(xmlfilter(id,pattern)) end -local function reset() - for d, f in next, data do - for i=1,#f do - f[i](false) - end - end +function xml.count(id,pattern) + return count(xmlfilter(id,pattern)) end -function trackers.register(what,...) - what = lower(what) - local w = data[what] - if not w then - w = { } - data[what] = w - end - for _, fnc in next, { ... } do - local typ = type(fnc) - if typ == "function" then - w[#w+1] = fnc - elseif typ == "string" then - w[#w+1] = function(value) set(fnc,value,nesting) end - end - end +function xml.attribute(id,pattern,a,default) + return attribute(xmlfilter(id,pattern),a,default) end -function trackers.enable(what) - done = { } - set(what,true) +function xml.text(id,pattern) + return text(xmlfilter(id,pattern)) end -function trackers.disable(what) - done = { } - if not what or what == "" then - trackers.reset(what) - else - set(what,false) - end +function xml.raw(id,pattern) + return xmlserialize(xmlfilter(id,pattern)) end -function trackers.reset(what) - done = { } - reset() +function xml.position(id,pattern,n) + return position(xmlfilter(id,pattern),n) end -function trackers.list() -- pattern - local list = table.sortedkeys(data) - local user, system = { }, { } - for l=1,#list do - local what = list[l] - if find(what,"^%*") then - system[#system+1] = what - else - user[#user+1] = what - end - end - return user, system +function xml.empty(id,pattern) + return empty(xmlfilter(id,pattern)) end +xml.all = xml.filter +xml.index = xml.position +xml.found = xml.filter + end -- of closure @@ -6135,6 +6692,7 @@ function statistics.timed(action,report) end + end -- of closure do -- create closure to overcome 200 locals limit @@ -9814,11 +10372,13 @@ own.libs = { -- todo: check which ones are really needed 'l-utils.lua', 'l-aux.lua', -- 'l-xml.lua', + 'trac-tra.lua', 'lxml-tab.lua', - 'lxml-pth.lua', + 'lxml-lpt.lua', 'lxml-ent.lua', 'lxml-mis.lua', - 'trac-tra.lua', + 'lxml-aux.lua', + 'lxml-xml.lua', 'luat-env.lua', 'trac-inf.lua', 'trac-log.lua', @@ -9889,7 +10449,7 @@ if not resolvers then os.exit() end -logs.setprogram('MTXrun',"TDS Runner Tool 1.22",environment.arguments["verbose"] or false) +logs.setprogram('MTXrun',"TDS Runner Tool 1.23",environment.arguments["verbose"] or false) local instance = resolvers.reset() |